Update hivegrid-cloud reference to latest main
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e98d38e716
commit
7b21524c86
3781 files changed
+373484
-225607
No files matched your search
@@ -3,6 +3,20 @@ name: Deploy
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
# Skip the staging deploy for test-/docs-only merges (e.g. coverage waves, doc updates).
|
||||
# paths-ignore skips the run only when EVERY changed file matches a pattern; any app or
|
||||
# config source change still deploys. Manual deploys via workflow_dispatch are unaffected.
|
||||
paths-ignore:
|
||||
- "**/*.test.ts"
|
||||
- "**/*.test.tsx"
|
||||
- "**/*.test.mjs"
|
||||
- "**/*.test.js"
|
||||
- "**/*.spec.ts"
|
||||
- "docs/**"
|
||||
- ".planning/**"
|
||||
- "**/*.md"
|
||||
- ".gitea/workflows/**"
|
||||
- ".github/workflows/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
action:
|
||||
@@ -18,10 +32,10 @@ on:
|
||||
default: "false"
|
||||
|
||||
env:
|
||||
TOOLCTRL_GITEA_HOST_IP: ${{ vars.TOOLCTRL_GITEA_HOST_IP }}
|
||||
HIVEGRID_GITEA_HOST_IP: ${{ vars.HIVEGRID_GITEA_HOST_IP }}
|
||||
|
||||
concurrency:
|
||||
group: boilerplate-deploy
|
||||
group: hivegrid-deploy
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
@@ -32,12 +46,12 @@ jobs:
|
||||
steps:
|
||||
- name: Resolve internal Gitea host
|
||||
run: |
|
||||
if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then
|
||||
if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then
|
||||
echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
|
||||
if [ -n "${HIVEGRID_GITEA_HOST_IP:-}" ]; then
|
||||
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
|
||||
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
|
||||
exit 1
|
||||
fi
|
||||
printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts
|
||||
printf '%s git.tool-ctrl.de\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
|
||||
fi
|
||||
|
||||
- name: Checkout code
|
||||
@@ -120,12 +134,12 @@ jobs:
|
||||
|
||||
- name: Setup hosts override and Docker CLI
|
||||
run: |
|
||||
if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then
|
||||
if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then
|
||||
echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
|
||||
if [ -n "${HIVEGRID_GITEA_HOST_IP:-}" ]; then
|
||||
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
|
||||
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
|
||||
exit 1
|
||||
fi
|
||||
printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts
|
||||
printf '%s git.tool-ctrl.de\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
|
||||
fi
|
||||
|
||||
MISSING_TOOLS=""
|
||||
@@ -191,7 +205,7 @@ jobs:
|
||||
git cat-file -e "$IMAGE_TAG^{commit}"
|
||||
git merge-base --is-ancestor "$IMAGE_TAG" origin/main
|
||||
|
||||
for TARGET in web api worker; do
|
||||
for TARGET in web api worker mcp; do
|
||||
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
||||
SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}')"
|
||||
STAGING_DIGEST="$(docker buildx imagetools inspect "$IMAGE:staging" --format '{{json .Manifest.Digest}}')"
|
||||
@@ -215,7 +229,7 @@ jobs:
|
||||
cd "$GITHUB_WORKSPACE"
|
||||
BUILD_PIDS=()
|
||||
BUILD_TARGETS=()
|
||||
for TARGET in web api worker; do
|
||||
for TARGET in web api worker mcp; do
|
||||
(
|
||||
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
||||
CACHE_REF="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/build-cache:$TARGET"
|
||||
@@ -229,22 +243,28 @@ jobs:
|
||||
--build-arg VITE_SENTRY_ENVIRONMENT="$VITE_SENTRY_ENVIRONMENT"
|
||||
--build-arg VITE_SENTRY_RELEASE="$SHA"
|
||||
--build-arg VITE_OTEL_EXPORTER_OTLP_ENDPOINT="$VITE_OTEL_EXPORTER_OTLP_ENDPOINT"
|
||||
--build-arg VITE_OTEL_SERVICE_NAME="boilerplate-web"
|
||||
--build-arg VITE_OTEL_SERVICE_NAME="hivegrid-web"
|
||||
)
|
||||
fi
|
||||
# `ignore-error=true` on the cache export: writing the build cache
|
||||
# to the registry is an optimization, not part of producing/pushing
|
||||
# the image. The Gitea registry rate-limits (HTTP 429) build-cache
|
||||
# blob writes under bursts of back-to-back deploys (made worse by
|
||||
# web/api/worker exporting to the same cache in parallel); without
|
||||
# this flag a 429 on a cache layer fails the whole build even though
|
||||
# the image itself built and pushed fine.
|
||||
docker buildx build --push --target "$TARGET" --file Dockerfile \
|
||||
--cache-from "type=registry,ref=$CACHE_REF" \
|
||||
--cache-to "type=registry,ref=$CACHE_REF,mode=max,ignore-error=true" \
|
||||
--build-arg APP_REVISION="$SHA" \
|
||||
"${WEB_BUILD_ARGS[@]}" \
|
||||
--tag "$IMAGE:$IMAGE_TAG" .
|
||||
# The registry cache is read-only in CI. Exporting build-cache blobs
|
||||
# to Gitea is optional and has repeatedly failed staging deploys with
|
||||
# registry-side 499/context deadline errors after the image itself
|
||||
# built successfully.
|
||||
for ATTEMPT in 1 2 3; do
|
||||
if docker buildx build --push --target "$TARGET" --provenance=false --file Dockerfile \
|
||||
--cache-from "type=registry,ref=$CACHE_REF" \
|
||||
--build-arg APP_REVISION="$SHA" \
|
||||
"${WEB_BUILD_ARGS[@]}" \
|
||||
--tag "$IMAGE:$IMAGE_TAG" .; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$ATTEMPT" -lt 3 ]; then
|
||||
echo "Docker build/push failed for $TARGET on attempt $ATTEMPT; retrying"
|
||||
sleep $((ATTEMPT * 20))
|
||||
fi
|
||||
done
|
||||
echo "Docker build/push failed for $TARGET after 3 attempts"
|
||||
exit 1
|
||||
) &
|
||||
BUILD_PIDS+=("$!")
|
||||
BUILD_TARGETS+=("$TARGET")
|
||||
@@ -275,7 +295,7 @@ jobs:
|
||||
echo "Cosign signing skipped: COSIGN_PRIVATE_KEY or COSIGN_PASSWORD not configured"
|
||||
exit 0
|
||||
fi
|
||||
for TARGET in web api worker; do
|
||||
for TARGET in web api worker mcp; do
|
||||
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
||||
cosign sign --yes --key env://COSIGN_PRIVATE_KEY "$IMAGE:$IMAGE_TAG"
|
||||
done
|
||||
@@ -300,16 +320,19 @@ jobs:
|
||||
|
||||
node scripts/coolify-ensure-worker-groups.mjs staging
|
||||
|
||||
- name: Configure Coolify staging migration command
|
||||
# Migrations and the guarded seed run inside the web container entrypoint
|
||||
# (start.sh, gated by MIGRATE_ON_BOOT/SEED_ON_BOOT) because Coolify's
|
||||
# pre/post_deployment_command never fired for docker-image applications.
|
||||
# This step upserts the boot flags on the web resource and clears the
|
||||
# dead hook slots so they are not mistaken for an active mechanism.
|
||||
- name: Configure Coolify staging migration boot flags
|
||||
env:
|
||||
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
||||
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
|
||||
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
||||
MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts
|
||||
SEED_COMMAND: cd /myapp && SEED_DEMO_DATA=true ALLOW_STAGING_DEMO_SEED=true NODE_ENV=production node packages/db/dist/production-seed.server.js
|
||||
run: |
|
||||
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
||||
echo "Coolify migration command configuration skipped for promote"
|
||||
echo "Coolify migration boot flag configuration skipped for promote"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -323,6 +346,7 @@ jobs:
|
||||
|
||||
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
|
||||
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
|
||||
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
|
||||
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
|
||||
APPS_JSON="$(mktemp)"
|
||||
trap 'rm -f "${APPS_JSON:-}"' EXIT
|
||||
@@ -331,6 +355,7 @@ jobs:
|
||||
"$SELECTED_COOLIFY_API_URL/applications" > "$APPS_JSON"
|
||||
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
||||
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
||||
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.MCP_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
||||
select_worker_resource() {
|
||||
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
|
||||
}
|
||||
@@ -340,19 +365,20 @@ jobs:
|
||||
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
|
||||
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
|
||||
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
|
||||
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
|
||||
echo "Coolify staging web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
|
||||
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$MCP_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
|
||||
echo "Coolify staging web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
|
||||
exit 1
|
||||
fi
|
||||
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
if ! printf ",%s," "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
|
||||
echo "Coolify staging deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
|
||||
echo "Coolify staging deploy resources must include the web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
|
||||
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
|
||||
{
|
||||
echo "STAGING_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
|
||||
echo "STAGING_MCP_RESOURCE_UUID=$MCP_RESOURCE_UUID"
|
||||
echo "STAGING_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
|
||||
echo "STAGING_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
|
||||
echo "STAGING_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
|
||||
@@ -362,14 +388,19 @@ jobs:
|
||||
echo "STAGING_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
||||
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
||||
curl --fail --silent --show-error --request PATCH \
|
||||
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data "$MIGRATION_PAYLOAD" \
|
||||
"$SELECTED_COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null
|
||||
node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \
|
||||
MIGRATE_ON_BOOT=true \
|
||||
SEED_ON_BOOT=true \
|
||||
SEED_DEMO_DATA=true \
|
||||
ALLOW_STAGING_DEMO_SEED=true
|
||||
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
||||
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
curl --fail --silent --show-error --request PATCH \
|
||||
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
|
||||
--header "Content-Type: application/json" \
|
||||
@@ -384,26 +415,11 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for TARGET in web api worker; do
|
||||
for TARGET in web api worker mcp; do
|
||||
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
||||
docker buildx imagetools create --tag "$IMAGE:staging" "$IMAGE:$IMAGE_TAG"
|
||||
done
|
||||
|
||||
- name: Pre-pull staging images on Coolify host
|
||||
env:
|
||||
COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }}
|
||||
COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }}
|
||||
COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }}
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_PAT: ${{ secrets.REGISTRY_PAT }}
|
||||
run: |
|
||||
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
||||
echo "Staging image pre-pull skipped for promote"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
timeout 240s scripts/coolify-prepull-images.sh || echo "Staging image pre-pull skipped after timeout; Coolify will pull during deploy"
|
||||
|
||||
- name: Trigger Coolify staging deploy
|
||||
env:
|
||||
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
||||
@@ -457,7 +473,7 @@ jobs:
|
||||
EXPECTED_IMAGE_ALIAS="${4:-staging}"
|
||||
EXPECTED_SERVICE="${5:-}"
|
||||
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
if [ -z "$HEALTH_URL" ]; then
|
||||
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
|
||||
echo "$LABEL is an HTTP resource but has no health URL"
|
||||
@@ -555,9 +571,6 @@ jobs:
|
||||
env:
|
||||
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
||||
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
||||
COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }}
|
||||
COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }}
|
||||
COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }}
|
||||
run: |
|
||||
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
||||
echo "Staging dependent health wait skipped for promote"
|
||||
@@ -580,35 +593,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# SSH helper for worker health verification (Coolify has no exec REST API);
|
||||
# used by verify_live_revision to run the health probe via docker exec.
|
||||
if [ -z "$COOLIFY_STAGING_SSH_HOST" ] || [ -z "$COOLIFY_STAGING_SSH_KEY" ]; then
|
||||
echo "Coolify staging SSH configuration is incomplete (worker health verification requires SSH + docker exec)"
|
||||
exit 1
|
||||
fi
|
||||
REMOTE_SSH_KEY_FILE="$(mktemp)"
|
||||
REMOTE_SSH_KNOWN_HOSTS="$(mktemp)"
|
||||
trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT
|
||||
# Materialise the key via an intermediate var so the secret is written to
|
||||
# a file (never stdout/logs); avoids leaking it in CI output.
|
||||
REMOTE_SSH_KEY_DATA="$COOLIFY_STAGING_SSH_KEY"
|
||||
printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE"
|
||||
unset REMOTE_SSH_KEY_DATA
|
||||
chmod 600 "$REMOTE_SSH_KEY_FILE"
|
||||
REMOTE_SSH_USER="${COOLIFY_STAGING_SSH_USER:-root}"
|
||||
REMOTE_SSH_HOST="$COOLIFY_STAGING_SSH_HOST"
|
||||
if command -v ssh-keyscan >/dev/null 2>&1; then
|
||||
ssh-keyscan -H "$REMOTE_SSH_HOST" > "$REMOTE_SSH_KNOWN_HOSTS" 2>/dev/null || true
|
||||
fi
|
||||
if [ -s "$REMOTE_SSH_KNOWN_HOSTS" ]; then
|
||||
REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=yes"
|
||||
else
|
||||
REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=accept-new"
|
||||
fi
|
||||
remote_ssh() {
|
||||
ssh -i "$REMOTE_SSH_KEY_FILE" -o BatchMode=yes -o ConnectTimeout=10 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" $REMOTE_SSH_HOSTKEY_OPT "${REMOTE_SSH_USER}@${REMOTE_SSH_HOST}" "$@"
|
||||
}
|
||||
|
||||
DEPLOYMENT_INDEX=0
|
||||
DEPLOYMENT_WAIT_PIDS=""
|
||||
for DEPLOYMENT_UUID in $(printf "%s" "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
|
||||
@@ -636,7 +620,7 @@ jobs:
|
||||
RESOURCE_KIND="${6:-auto}"
|
||||
EXPECTED_WORKER_GROUP="${7:-}"
|
||||
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
|
||||
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
||||
@@ -644,74 +628,7 @@ jobs:
|
||||
echo "$LABEL worker image tag does not match expected alias"
|
||||
exit 1
|
||||
fi
|
||||
# Coolify 4.x exposes no command-execution REST API, so verify worker
|
||||
# health by running the one-shot health probe inside the live container
|
||||
# over SSH (docker exec) on the Coolify host. Fail-closed: any SSH /
|
||||
# container-resolution / exec failure blocks the deploy.
|
||||
WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')"
|
||||
if [ -z "$WORKER_CONTAINER" ]; then
|
||||
echo "$LABEL worker container not found on Coolify host"
|
||||
exit 1
|
||||
fi
|
||||
# health.js exits non-zero when the worker reports a non-ready
|
||||
# status (e.g. "misconfigured"), yet it still prints the diagnostic
|
||||
# JSON (status + errors) to stdout. Capture stdout regardless of the
|
||||
# exit code (|| true) so the real reason surfaces in the checks below
|
||||
# instead of a generic "command unavailable". Empty output means the
|
||||
# docker exec itself could not run (container gone, SSH failure, ...).
|
||||
WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ -z "$WORKER_HEALTH_OUTPUT" ]; then
|
||||
echo "$LABEL worker docker exec health command unavailable (no output from health probe)"
|
||||
exit 1
|
||||
fi
|
||||
WORKER_HEALTH_JSON="$(mktemp)"
|
||||
printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON"
|
||||
if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then
|
||||
rm -f "$WORKER_HEALTH_JSON"
|
||||
echo "$LABEL worker execute output is not JSON"
|
||||
exit 1
|
||||
fi
|
||||
WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")"
|
||||
rm -f "$WORKER_HEALTH_JSON"
|
||||
if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then
|
||||
echo "$LABEL worker health revision does not match image tag yet"
|
||||
return 1
|
||||
fi
|
||||
if [ "$WORKER_ENABLED" != "true" ]; then
|
||||
echo "$LABEL worker health reports workerEnabled=false"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$WORKER_STATUS" != "ready" ]; then
|
||||
echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then
|
||||
echo "$LABEL worker group does not match expected queue group"
|
||||
exit 1
|
||||
fi
|
||||
EXPECTED_WORKER_QUEUES=""
|
||||
case "$EXPECTED_WORKER_GROUP" in
|
||||
general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;;
|
||||
mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;;
|
||||
import)
|
||||
if [ -n "$WORKER_QUEUES" ]; then
|
||||
EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports"
|
||||
fi
|
||||
;;
|
||||
webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;;
|
||||
plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;;
|
||||
esac
|
||||
for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do
|
||||
if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then
|
||||
echo "$LABEL worker queues do not match expected queue group"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "$LABEL worker live revision matches image tag"
|
||||
echo "$LABEL worker Coolify status is healthy and image tag matches expected alias"
|
||||
return
|
||||
fi
|
||||
if [ -z "$HEALTH_URL" ]; then
|
||||
@@ -761,6 +678,10 @@ jobs:
|
||||
RESOURCE_KIND="api"
|
||||
EXPECTED_SERVICE="api"
|
||||
EXPECTED_WORKER_GROUP=""
|
||||
if [ "$RESOURCE_UUID" = "$STAGING_MCP_RESOURCE_UUID" ]; then
|
||||
RESOURCE_KIND="mcp"
|
||||
EXPECTED_SERVICE="mcp"
|
||||
fi
|
||||
if printf ",%s," "$STAGING_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
|
||||
RESOURCE_KIND="worker"
|
||||
EXPECTED_SERVICE=""
|
||||
@@ -829,16 +750,17 @@ jobs:
|
||||
|
||||
node scripts/coolify-ensure-worker-groups.mjs production
|
||||
|
||||
- name: Configure Coolify production migration command
|
||||
# Same entrypoint-based migration mechanism as staging: boot flags on the
|
||||
# web resource, dead Coolify hook slots cleared. Production runs the
|
||||
# ungated seed (no demo flags) inside the new container after migrate.
|
||||
- name: Configure Coolify production migration boot flags
|
||||
env:
|
||||
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
||||
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
|
||||
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
||||
MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts
|
||||
SEED_COMMAND: cd /myapp && NODE_ENV=production node packages/db/dist/production-seed.server.js
|
||||
run: |
|
||||
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
||||
echo "Coolify production migration command configuration skipped for staging"
|
||||
echo "Coolify production migration boot flag configuration skipped for staging"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -850,6 +772,7 @@ jobs:
|
||||
|
||||
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
|
||||
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
|
||||
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
|
||||
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
|
||||
APPS_JSON="$(mktemp)"
|
||||
trap 'rm -f "${APPS_JSON:-}"' EXIT
|
||||
@@ -858,6 +781,7 @@ jobs:
|
||||
"$COOLIFY_API_URL/applications" > "$APPS_JSON"
|
||||
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
||||
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
||||
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.MCP_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
||||
select_worker_resource() {
|
||||
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
|
||||
}
|
||||
@@ -867,19 +791,20 @@ jobs:
|
||||
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
|
||||
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
|
||||
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
|
||||
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
|
||||
echo "Coolify production web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
|
||||
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$MCP_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
|
||||
echo "Coolify production web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
|
||||
exit 1
|
||||
fi
|
||||
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
if ! printf ",%s," "$RESOLVED_COOLIFY_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
|
||||
echo "Coolify production deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
|
||||
echo "Coolify production deploy resources must include the web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
|
||||
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
|
||||
{
|
||||
echo "PRODUCTION_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
|
||||
echo "PRODUCTION_MCP_RESOURCE_UUID=$MCP_RESOURCE_UUID"
|
||||
echo "PRODUCTION_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
|
||||
echo "PRODUCTION_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
|
||||
echo "PRODUCTION_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
|
||||
@@ -889,14 +814,17 @@ jobs:
|
||||
echo "PRODUCTION_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
||||
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
||||
curl --fail --silent --show-error --request PATCH \
|
||||
--header "Authorization: Bearer $COOLIFY_TOKEN" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data "$MIGRATION_PAYLOAD" \
|
||||
"$COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null
|
||||
node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \
|
||||
MIGRATE_ON_BOOT=true \
|
||||
SEED_ON_BOOT=true
|
||||
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
||||
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
||||
curl --fail --silent --show-error --request PATCH \
|
||||
--header "Authorization: Bearer $COOLIFY_TOKEN" \
|
||||
--header "Content-Type: application/json" \
|
||||
@@ -911,7 +839,7 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for TARGET in web api worker; do
|
||||
for TARGET in web api worker mcp; do
|
||||
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
||||
docker buildx imagetools create --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"
|
||||
done
|
||||
@@ -968,7 +896,7 @@ jobs:
|
||||
EXPECTED_IMAGE_ALIAS="${4:-main}"
|
||||
EXPECTED_SERVICE="${5:-}"
|
||||
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
if [ -z "$HEALTH_URL" ]; then
|
||||
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
|
||||
echo "$LABEL is an HTTP resource but has no health URL"
|
||||
@@ -1118,12 +1046,6 @@ jobs:
|
||||
env:
|
||||
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
||||
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
||||
# AD-24: Prod ist Multi-Server. Worker laufen auf der worker-VM (privates Netz), nur ueber
|
||||
# die control-Bastion erreichbar -> Worker-Health-Verifikation (docker exec) per SSH-ProxyJump.
|
||||
COOLIFY_PROD_SSH_KEY: ${{ secrets.COOLIFY_PROD_SSH_KEY }}
|
||||
COOLIFY_PROD_SSH_KNOWN_HOSTS: ${{ secrets.COOLIFY_PROD_SSH_KNOWN_HOSTS }}
|
||||
COOLIFY_PROD_SSH_HOST: ${{ vars.COOLIFY_PROD_SSH_HOST }}
|
||||
COOLIFY_PROD_WORKER_PRIVATE_IP: ${{ vars.COOLIFY_PROD_WORKER_PRIVATE_IP }}
|
||||
run: |
|
||||
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
||||
echo "Production dependent health wait skipped for staging"
|
||||
@@ -1164,32 +1086,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# SSH helper for worker health verification (Coolify has no exec REST API). AD-24: Prod ist
|
||||
# Multi-Server -> Worker laufen auf der worker-VM (privates Netz), erreichbar nur ueber die
|
||||
# control-Bastion. ProxyJump + gepinnte Host-Keys (kein StrictHostKeyChecking=no). Secrets
|
||||
# werden via Zwischen-Variable in Dateien geschrieben (nie stdout/Logs).
|
||||
if [ -z "$COOLIFY_PROD_SSH_HOST" ] || [ -z "$COOLIFY_PROD_SSH_KEY" ] || [ -z "$COOLIFY_PROD_SSH_KNOWN_HOSTS" ] || [ -z "$COOLIFY_PROD_WORKER_PRIVATE_IP" ]; then
|
||||
echo "Coolify production SSH configuration is incomplete (worker health verification requires SSH + docker exec)"
|
||||
exit 1
|
||||
fi
|
||||
REMOTE_SSH_KEY_FILE="$(mktemp)"
|
||||
REMOTE_SSH_KNOWN_HOSTS="$(mktemp)"
|
||||
trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT
|
||||
REMOTE_SSH_KEY_DATA="$COOLIFY_PROD_SSH_KEY"
|
||||
printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE"
|
||||
unset REMOTE_SSH_KEY_DATA
|
||||
chmod 600 "$REMOTE_SSH_KEY_FILE"
|
||||
REMOTE_SSH_KNOWN_HOSTS_DATA="$COOLIFY_PROD_SSH_KNOWN_HOSTS"
|
||||
printf '%s\n' "$REMOTE_SSH_KNOWN_HOSTS_DATA" > "$REMOTE_SSH_KNOWN_HOSTS"
|
||||
unset REMOTE_SSH_KNOWN_HOSTS_DATA
|
||||
# ProxyCommand (nicht ProxyJump): ProxyJump reicht den -i-Key nicht zuverlaessig an den
|
||||
# Jump-Hop weiter (MaxAuthTries -> Permission denied). ProxyCommand gibt der Bastion den Key
|
||||
# explizit (IdentitiesOnly), Host-Keys fuer beide Hops gepinnt.
|
||||
REMOTE_SSH_PROXY="ssh -i $REMOTE_SSH_KEY_FILE -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile=$REMOTE_SSH_KNOWN_HOSTS -o StrictHostKeyChecking=yes -W %h:%p root@${COOLIFY_PROD_SSH_HOST}"
|
||||
remote_ssh() {
|
||||
ssh -i "$REMOTE_SSH_KEY_FILE" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" -o StrictHostKeyChecking=yes -o ProxyCommand="$REMOTE_SSH_PROXY" "root@${COOLIFY_PROD_WORKER_PRIVATE_IP}" "$@"
|
||||
}
|
||||
|
||||
verify_live_revision() {
|
||||
APP_JSON="$1"
|
||||
LABEL="$2"
|
||||
@@ -1199,7 +1095,7 @@ jobs:
|
||||
RESOURCE_KIND="${6:-auto}"
|
||||
EXPECTED_WORKER_GROUP="${7:-}"
|
||||
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
||||
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
|
||||
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
||||
@@ -1207,74 +1103,7 @@ jobs:
|
||||
echo "$LABEL worker image tag does not match expected alias"
|
||||
exit 1
|
||||
fi
|
||||
# Coolify 4.x exposes no command-execution REST API, so verify worker
|
||||
# health by running the one-shot health probe inside the live container
|
||||
# over SSH (docker exec) on the Coolify host. Fail-closed: any SSH /
|
||||
# container-resolution / exec failure blocks the deploy.
|
||||
WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')"
|
||||
if [ -z "$WORKER_CONTAINER" ]; then
|
||||
echo "$LABEL worker container not found on Coolify host"
|
||||
exit 1
|
||||
fi
|
||||
# health.js exits non-zero when the worker reports a non-ready
|
||||
# status (e.g. "misconfigured"), yet it still prints the diagnostic
|
||||
# JSON (status + errors) to stdout. Capture stdout regardless of the
|
||||
# exit code (|| true) so the real reason surfaces in the checks below
|
||||
# instead of a generic "command unavailable". Empty output means the
|
||||
# docker exec itself could not run (container gone, SSH failure, ...).
|
||||
WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)"
|
||||
if [ -z "$WORKER_HEALTH_OUTPUT" ]; then
|
||||
echo "$LABEL worker docker exec health command unavailable (no output from health probe)"
|
||||
exit 1
|
||||
fi
|
||||
WORKER_HEALTH_JSON="$(mktemp)"
|
||||
printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON"
|
||||
if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then
|
||||
rm -f "$WORKER_HEALTH_JSON"
|
||||
echo "$LABEL worker execute output is not JSON"
|
||||
exit 1
|
||||
fi
|
||||
WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")"
|
||||
WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")"
|
||||
rm -f "$WORKER_HEALTH_JSON"
|
||||
if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then
|
||||
echo "$LABEL worker health revision does not match image tag yet"
|
||||
return 1
|
||||
fi
|
||||
if [ "$WORKER_ENABLED" != "true" ]; then
|
||||
echo "$LABEL worker health reports workerEnabled=false"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$WORKER_STATUS" != "ready" ]; then
|
||||
echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then
|
||||
echo "$LABEL worker group does not match expected queue group"
|
||||
exit 1
|
||||
fi
|
||||
EXPECTED_WORKER_QUEUES=""
|
||||
case "$EXPECTED_WORKER_GROUP" in
|
||||
general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;;
|
||||
mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;;
|
||||
import)
|
||||
if [ -n "$WORKER_QUEUES" ]; then
|
||||
EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports"
|
||||
fi
|
||||
;;
|
||||
webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;;
|
||||
plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;;
|
||||
esac
|
||||
for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do
|
||||
if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then
|
||||
echo "$LABEL worker queues do not match expected queue group"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "$LABEL worker live revision matches image tag"
|
||||
echo "$LABEL worker Coolify status is healthy and image tag matches expected alias"
|
||||
return
|
||||
fi
|
||||
if [ -z "$HEALTH_URL" ]; then
|
||||
@@ -1324,6 +1153,10 @@ jobs:
|
||||
RESOURCE_KIND="api"
|
||||
EXPECTED_SERVICE="api"
|
||||
EXPECTED_WORKER_GROUP=""
|
||||
if [ "$RESOURCE_UUID" = "$PRODUCTION_MCP_RESOURCE_UUID" ]; then
|
||||
RESOURCE_KIND="mcp"
|
||||
EXPECTED_SERVICE="mcp"
|
||||
fi
|
||||
if printf ",%s," "$PRODUCTION_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
|
||||
RESOURCE_KIND="worker"
|
||||
EXPECTED_SERVICE=""
|
||||
@@ -1390,7 +1223,7 @@ jobs:
|
||||
SENTRY_ENVIRONMENT=staging
|
||||
RELEASE_SHA="${SHA}"
|
||||
fi
|
||||
for PROJECT in boilerplate-api boilerplate-worker boilerplate-web; do
|
||||
for PROJECT in api worker web; do
|
||||
bash scripts/ci/sentry-release.sh \
|
||||
--project "$PROJECT" \
|
||||
--environment "$SENTRY_ENVIRONMENT" \
|
||||
|
||||
Reference in new issue
Block a user