1418 lines
79 KiB
YAML
1418 lines
79 KiB
YAML
name: Deploy
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
inputs:
|
|
action:
|
|
description: "stage or promote"
|
|
required: true
|
|
default: "stage"
|
|
staged_sha:
|
|
description: "Full staged commit SHA to promote"
|
|
required: false
|
|
backup_confirmed:
|
|
description: "Set to true only after production backup or restore-point readiness is confirmed for promote"
|
|
required: false
|
|
default: "false"
|
|
|
|
env:
|
|
TOOLCTRL_GITEA_HOST_IP: ${{ vars.TOOLCTRL_GITEA_HOST_IP }}
|
|
|
|
concurrency:
|
|
group: boilerplate-deploy
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
deploy-preflight:
|
|
name: Deploy preflight gates
|
|
runs-on: toolctrl-ci
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Resolve internal Gitea host
|
|
run: |
|
|
if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then
|
|
if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then
|
|
echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
|
|
exit 1
|
|
fi
|
|
printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts
|
|
fi
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Verify deploy wiring
|
|
run: |
|
|
node scripts/verify-gitea-deploy-workflow.mjs
|
|
node scripts/verify-deploy-readiness.mjs
|
|
node scripts/verify-docker-baseline.mjs
|
|
node scripts/verify-docker-non-root.mjs
|
|
node --check scripts/coolify-deploy.mjs
|
|
node --check scripts/coolify-ensure-worker-groups.mjs
|
|
bash -n scripts/coolify-prepull-images.sh
|
|
bash -n scripts/ci/install-cosign.sh
|
|
|
|
stage-or-promote:
|
|
name: Stage or promote Docker images
|
|
runs-on: toolctrl-ci
|
|
timeout-minutes: 60
|
|
needs: deploy-preflight
|
|
env:
|
|
REGISTRY: git.tool-ctrl.de
|
|
IMAGE_NAMESPACE: toolctrl-labs
|
|
IMAGE_PREFIX: hivegrid
|
|
SHA: ${{ gitea.sha }}
|
|
DISPATCH_ACTION: ${{ gitea.event.inputs.action }}
|
|
DISPATCH_STAGED_SHA: ${{ gitea.event.inputs.staged_sha }}
|
|
DISPATCH_BACKUP_CONFIRMED: ${{ gitea.event.inputs.backup_confirmed }}
|
|
steps:
|
|
- name: Resolve deployment action
|
|
run: |
|
|
REF="${GITEA_REF:-${{ gitea.ref }}}"
|
|
EVENT_NAME="${GITEA_EVENT_NAME:-${{ gitea.event_name }}}"
|
|
ACTION="${DISPATCH_ACTION:-stage}"
|
|
STAGED_SHA="${DISPATCH_STAGED_SHA:-}"
|
|
BACKUP_CONFIRMED="${DISPATCH_BACKUP_CONFIRMED:-false}"
|
|
|
|
if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/main" ]; then
|
|
DEPLOY_ACTION=stage
|
|
elif [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$ACTION" = "stage" ]; then
|
|
if [ "$REF" != "refs/heads/main" ]; then
|
|
echo "Stage must be dispatched from refs/heads/main"
|
|
exit 1
|
|
fi
|
|
DEPLOY_ACTION=stage
|
|
elif [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$ACTION" = "promote" ]; then
|
|
DEPLOY_ACTION=promote
|
|
if [ "$REF" != "refs/heads/main" ]; then
|
|
echo "Promote must be dispatched from refs/heads/main"
|
|
exit 1
|
|
fi
|
|
if ! printf '%s' "$STAGED_SHA" | grep -Eq '^[0-9a-f]{40}$'; then
|
|
echo "Promote requires staged_sha as the full 40-character staged commit SHA"
|
|
exit 1
|
|
fi
|
|
if [ "$BACKUP_CONFIRMED" != "true" ]; then
|
|
echo "Promote requires backup_confirmed=true after production backup or restore-point readiness is confirmed"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "Unsupported deployment action: event=$EVENT_NAME ref=$REF action=$ACTION"
|
|
exit 1
|
|
fi
|
|
|
|
if ! printf '%s' "$SHA" | grep -Eq '^[0-9a-f]{40}$'; then
|
|
echo "Gitea SHA must be a full 40-character commit SHA"
|
|
exit 1
|
|
fi
|
|
if [ "$DEPLOY_ACTION" = "promote" ]; then
|
|
IMAGE_TAG="$STAGED_SHA"
|
|
else
|
|
IMAGE_TAG="$SHA"
|
|
fi
|
|
|
|
{
|
|
echo "IMAGE_TAG=$IMAGE_TAG"
|
|
echo "DEPLOY_ACTION=$DEPLOY_ACTION"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
- name: Setup hosts override and Docker CLI
|
|
run: |
|
|
if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then
|
|
if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then
|
|
echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
|
|
exit 1
|
|
fi
|
|
printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts
|
|
fi
|
|
|
|
MISSING_TOOLS=""
|
|
for TOOL in curl git; do
|
|
if ! command -v "$TOOL" >/dev/null 2>&1; then
|
|
MISSING_TOOLS="$MISSING_TOOLS $TOOL"
|
|
fi
|
|
done
|
|
if ! command -v ssh >/dev/null 2>&1; then
|
|
MISSING_TOOLS="$MISSING_TOOLS openssh-client"
|
|
fi
|
|
|
|
if [ -n "$MISSING_TOOLS" ]; then
|
|
if ! command -v apt-get >/dev/null 2>&1 || [ "$(id -u)" != "0" ]; then
|
|
echo "Missing required tools and apt-get is not available as root:$MISSING_TOOLS"
|
|
exit 1
|
|
fi
|
|
apt-get update -qq
|
|
apt-get install -qq -y --no-install-recommends ca-certificates $MISSING_TOOLS
|
|
fi
|
|
|
|
if ! command -v docker >/dev/null 2>&1 || ! docker buildx version >/dev/null 2>&1; then
|
|
if ! command -v apt-get >/dev/null 2>&1 || [ "$(id -u)" != "0" ]; then
|
|
echo "Docker CLI/buildx missing and apt-get is not available as root"
|
|
exit 1
|
|
fi
|
|
install -m 0755 -d /etc/apt/keyrings
|
|
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
|
chmod a+r /etc/apt/keyrings/docker.asc
|
|
. /etc/os-release
|
|
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list
|
|
apt-get update -qq
|
|
apt-get install -qq -y --no-install-recommends docker-ce-cli docker-buildx-plugin
|
|
fi
|
|
|
|
docker --version
|
|
docker buildx version
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Verify checkout revision
|
|
run: |
|
|
CHECKOUT_SHA="$(git rev-parse HEAD)"
|
|
if [ "$CHECKOUT_SHA" != "$SHA" ]; then
|
|
echo "Checked out revision must match the Gitea SHA"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Login to Gitea Registry
|
|
run: echo "${{ secrets.REGISTRY_PAT }}" | docker login "$REGISTRY" --username "${{ secrets.REGISTRY_USER }}" --password-stdin
|
|
|
|
- name: Verify staged promotion provenance
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Staged promotion provenance skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
git fetch --no-tags origin main
|
|
git cat-file -e "$IMAGE_TAG^{commit}"
|
|
git merge-base --is-ancestor "$IMAGE_TAG" origin/main
|
|
|
|
for TARGET in web api worker; do
|
|
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
|
SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}')"
|
|
STAGING_DIGEST="$(docker buildx imagetools inspect "$IMAGE:staging" --format '{{json .Manifest.Digest}}')"
|
|
if [ -z "$SHA_DIGEST" ] || [ "$SHA_DIGEST" != "$STAGING_DIGEST" ]; then
|
|
echo "Staged image digest for $TARGET does not match $IMAGE_TAG"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
- name: Build and push staging images
|
|
env:
|
|
VITE_SENTRY_DSN_WEB: ${{ secrets.VITE_SENTRY_DSN_WEB }}
|
|
VITE_SENTRY_ENVIRONMENT: ${{ secrets.VITE_SENTRY_ENVIRONMENT }}
|
|
VITE_OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.VITE_OTEL_EXPORTER_OTLP_ENDPOINT }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Stage build skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
cd "$GITHUB_WORKSPACE"
|
|
BUILD_PIDS=()
|
|
BUILD_TARGETS=()
|
|
for TARGET in web api worker; do
|
|
(
|
|
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
|
CACHE_REF="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/build-cache:$TARGET"
|
|
WEB_BUILD_ARGS=()
|
|
if [ "$TARGET" = "web" ]; then
|
|
# VITE_* build args are only meaningful for the web target (Vite
|
|
# inlines them into the client bundle). Keeping them web-only
|
|
# prevents web env changes from invalidating api/worker caches.
|
|
WEB_BUILD_ARGS=(
|
|
--build-arg VITE_SENTRY_DSN_WEB="$VITE_SENTRY_DSN_WEB"
|
|
--build-arg VITE_SENTRY_ENVIRONMENT="$VITE_SENTRY_ENVIRONMENT"
|
|
--build-arg VITE_SENTRY_RELEASE="$SHA"
|
|
--build-arg VITE_OTEL_EXPORTER_OTLP_ENDPOINT="$VITE_OTEL_EXPORTER_OTLP_ENDPOINT"
|
|
--build-arg VITE_OTEL_SERVICE_NAME="boilerplate-web"
|
|
)
|
|
fi
|
|
# `ignore-error=true` on the cache export: writing the build cache
|
|
# to the registry is an optimization, not part of producing/pushing
|
|
# the image. The Gitea registry rate-limits (HTTP 429) build-cache
|
|
# blob writes under bursts of back-to-back deploys (made worse by
|
|
# web/api/worker exporting to the same cache in parallel); without
|
|
# this flag a 429 on a cache layer fails the whole build even though
|
|
# the image itself built and pushed fine.
|
|
docker buildx build --push --target "$TARGET" --file Dockerfile \
|
|
--cache-from "type=registry,ref=$CACHE_REF" \
|
|
--cache-to "type=registry,ref=$CACHE_REF,mode=max,ignore-error=true" \
|
|
--build-arg APP_REVISION="$SHA" \
|
|
"${WEB_BUILD_ARGS[@]}" \
|
|
--tag "$IMAGE:$IMAGE_TAG" .
|
|
) &
|
|
BUILD_PIDS+=("$!")
|
|
BUILD_TARGETS+=("$TARGET")
|
|
done
|
|
|
|
BUILD_FAILED=0
|
|
for INDEX in "${!BUILD_PIDS[@]}"; do
|
|
if ! wait "${BUILD_PIDS[$INDEX]}"; then
|
|
echo "Docker build failed for ${BUILD_TARGETS[$INDEX]}"
|
|
BUILD_FAILED=1
|
|
fi
|
|
done
|
|
if [ "$BUILD_FAILED" -ne 0 ]; then
|
|
exit "$BUILD_FAILED"
|
|
fi
|
|
|
|
- name: Install cosign
|
|
if: env.DEPLOY_ACTION == 'stage'
|
|
run: bash scripts/ci/install-cosign.sh
|
|
|
|
- name: Sign published container images
|
|
if: env.DEPLOY_ACTION == 'stage'
|
|
env:
|
|
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
|
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
|
|
run: |
|
|
if [ -z "${COSIGN_PRIVATE_KEY:-}" ] || [ -z "${COSIGN_PASSWORD:-}" ]; then
|
|
echo "Cosign signing skipped: COSIGN_PRIVATE_KEY or COSIGN_PASSWORD not configured"
|
|
exit 0
|
|
fi
|
|
for TARGET in web api worker; do
|
|
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
|
cosign sign --yes --key env://COSIGN_PRIVATE_KEY "$IMAGE:$IMAGE_TAG"
|
|
done
|
|
|
|
- name: Ensure Coolify staging worker groups
|
|
env:
|
|
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
|
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
|
|
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
|
COOLIFY_STAGING_PROVISION_WORKER_GROUPS: ${{ vars.COOLIFY_STAGING_PROVISION_WORKER_GROUPS }}
|
|
COOLIFY_STAGING_PROJECT_UUID: ${{ vars.COOLIFY_STAGING_PROJECT_UUID }}
|
|
COOLIFY_STAGING_SERVER_UUID: ${{ vars.COOLIFY_STAGING_SERVER_UUID }}
|
|
COOLIFY_STAGING_ENVIRONMENT_NAME: ${{ vars.COOLIFY_STAGING_ENVIRONMENT_NAME }}
|
|
COOLIFY_STAGING_ENVIRONMENT_UUID: ${{ vars.COOLIFY_STAGING_ENVIRONMENT_UUID }}
|
|
COOLIFY_STAGING_DESTINATION_UUID: ${{ vars.COOLIFY_STAGING_DESTINATION_UUID }}
|
|
COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID: ${{ vars.COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Coolify staging worker group provisioning skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
node scripts/coolify-ensure-worker-groups.mjs staging
|
|
|
|
- name: Configure Coolify staging migration command
|
|
env:
|
|
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
|
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
|
|
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
|
MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts
|
|
SEED_COMMAND: cd /myapp && SEED_DEMO_DATA=true ALLOW_STAGING_DEMO_SEED=true NODE_ENV=production node packages/db/dist/production-seed.server.js
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Coolify migration command configuration skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
|
|
RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS="${STAGING_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_STAGING_RESOURCE_UUIDS}"
|
|
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
|
|
echo "Coolify staging configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
|
|
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
|
|
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
|
|
APPS_JSON="$(mktemp)"
|
|
trap 'rm -f "${APPS_JSON:-}"' EXIT
|
|
curl --fail --silent --show-error \
|
|
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
|
|
"$SELECTED_COOLIFY_API_URL/applications" > "$APPS_JSON"
|
|
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
|
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
|
select_worker_resource() {
|
|
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
|
|
}
|
|
WORKER_GENERAL_RESOURCE_UUID="$(select_worker_resource general)"
|
|
WORKER_MAIL_RESOURCE_UUID="$(select_worker_resource mail)"
|
|
WORKER_IMPORT_RESOURCE_UUID="$(select_worker_resource import)"
|
|
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
|
|
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
|
|
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
|
|
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
|
|
echo "Coolify staging web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
|
|
exit 1
|
|
fi
|
|
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
|
if ! printf ",%s," "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
|
|
echo "Coolify staging deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
|
|
exit 1
|
|
fi
|
|
done
|
|
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
|
|
{
|
|
echo "STAGING_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
|
|
echo "STAGING_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
|
|
echo "STAGING_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
|
|
echo "STAGING_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
|
|
echo "STAGING_WORKER_IMPORT_RESOURCE_UUID=$WORKER_IMPORT_RESOURCE_UUID"
|
|
echo "STAGING_WORKER_WEBHOOK_RESOURCE_UUID=$WORKER_WEBHOOK_RESOURCE_UUID"
|
|
echo "STAGING_WORKER_PLUGIN_RESOURCE_UUID=$WORKER_PLUGIN_RESOURCE_UUID"
|
|
echo "STAGING_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
|
curl --fail --silent --show-error --request PATCH \
|
|
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
|
|
--header "Content-Type: application/json" \
|
|
--data "$MIGRATION_PAYLOAD" \
|
|
"$SELECTED_COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null
|
|
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
|
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
|
curl --fail --silent --show-error --request PATCH \
|
|
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
|
|
--header "Content-Type: application/json" \
|
|
--data "$IMAGE_TAG_PAYLOAD" \
|
|
"$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" >/dev/null
|
|
done
|
|
|
|
- name: Publish staging image aliases
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Staging alias publication skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
for TARGET in web api worker; do
|
|
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
|
docker buildx imagetools create --tag "$IMAGE:staging" "$IMAGE:$IMAGE_TAG"
|
|
done
|
|
|
|
- name: Pre-pull staging images on Coolify host
|
|
env:
|
|
COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }}
|
|
COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }}
|
|
COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }}
|
|
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
|
REGISTRY_PAT: ${{ secrets.REGISTRY_PAT }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Staging image pre-pull skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
timeout 240s scripts/coolify-prepull-images.sh || echo "Staging image pre-pull skipped after timeout; Coolify will pull during deploy"
|
|
|
|
- name: Trigger Coolify staging deploy
|
|
env:
|
|
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
|
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
|
|
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Staging deploy skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
|
|
|
|
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$STAGING_WEB_RESOURCE_UUID" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
|
|
echo "Coolify staging configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"
|
|
echo "STAGING_WEB_DEPLOYMENT_UUID=$STAGING_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
|
|
|
|
- name: Wait for Coolify staging health
|
|
env:
|
|
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
|
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
|
|
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Staging health wait skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
|
|
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$STAGING_WEB_RESOURCE_UUID" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
|
|
echo "Coolify staging configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
if [ -z "$STAGING_WEB_DEPLOYMENT_UUID" ]; then
|
|
echo "Coolify staging deployment UUID is missing"
|
|
exit 1
|
|
fi
|
|
|
|
node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web"
|
|
|
|
verify_live_revision() {
|
|
APP_JSON="$1"
|
|
LABEL="$2"
|
|
REQUIRE_HEALTH_URL="${3:-auto}"
|
|
EXPECTED_IMAGE_ALIAS="${4:-staging}"
|
|
EXPECTED_SERVICE="${5:-}"
|
|
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
|
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
|
if [ -z "$HEALTH_URL" ]; then
|
|
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
|
|
echo "$LABEL is an HTTP resource but has no health URL"
|
|
exit 1
|
|
fi
|
|
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
|
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
|
|
echo "$LABEL non-HTTP resource image tag does not match expected alias"
|
|
exit 1
|
|
fi
|
|
echo "$LABEL non-HTTP resource image tag matches expected alias"
|
|
return
|
|
fi
|
|
HEALTH_JSON="$(mktemp)"
|
|
if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then
|
|
rm -f "$HEALTH_JSON"
|
|
echo "$LABEL health endpoint is not ready yet"
|
|
return 1
|
|
fi
|
|
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
|
|
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
|
|
rm -f "$HEALTH_JSON"
|
|
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
|
|
echo "$LABEL health service does not match expected service"
|
|
exit 1
|
|
fi
|
|
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
|
|
echo "$LABEL health revision does not match staged image yet"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
RESOURCE_INDEX=0
|
|
for RESOURCE_UUID in $(printf "%s" "$STAGING_WEB_RESOURCE_UUID" | tr ',' ' '); do
|
|
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
|
|
HEALTHY=false
|
|
for ATTEMPT in $(seq 1 24); do
|
|
APP_JSON="$(mktemp)"
|
|
curl --fail --silent --show-error \
|
|
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
|
|
"$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
|
|
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
|
|
echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS"
|
|
if [ "$STATUS" = "running:healthy" ]; then
|
|
if verify_live_revision "$APP_JSON" "Coolify staging resource #$RESOURCE_INDEX" required "$IMAGE_TAG" web; then
|
|
rm -f "$APP_JSON"
|
|
HEALTHY=true
|
|
break
|
|
fi
|
|
fi
|
|
rm -f "$APP_JSON"
|
|
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
|
|
echo "Coolify staging resource #$RESOURCE_INDEX entered a terminal failure status"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
if [ "$HEALTHY" != "true" ]; then
|
|
echo "Coolify staging resource #$RESOURCE_INDEX did not become healthy in time"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
- name: Trigger Coolify staging dependent deploys
|
|
env:
|
|
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
|
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Staging dependent deploy skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$STAGING_DEPENDENT_RESOURCE_UUIDS" ]; then
|
|
echo "No staging dependent resources to deploy"
|
|
exit 0
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
|
|
|
|
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
|
|
echo "Coolify staging configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
STAGING_DEPENDENT_DEPLOYMENT_UUIDS=""
|
|
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
|
|
DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"
|
|
STAGING_DEPENDENT_DEPLOYMENT_UUIDS="${STAGING_DEPENDENT_DEPLOYMENT_UUIDS:+$STAGING_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
|
|
done
|
|
echo "STAGING_DEPENDENT_DEPLOYMENT_UUIDS=$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV"
|
|
|
|
- name: Wait for Coolify staging dependent health
|
|
env:
|
|
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
|
|
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
|
|
COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }}
|
|
COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }}
|
|
COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "stage" ]; then
|
|
echo "Staging dependent health wait skipped for promote"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$STAGING_DEPENDENT_RESOURCE_UUIDS" ]; then
|
|
echo "No staging dependent resources to wait for"
|
|
exit 0
|
|
fi
|
|
if [ -z "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" ]; then
|
|
echo "Coolify staging dependent deployment UUIDs are missing"
|
|
exit 1
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
|
|
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
|
|
echo "Coolify staging configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
# SSH helper for worker health verification (Coolify has no exec REST API);
|
|
# used by verify_live_revision to run the health probe via docker exec.
|
|
if [ -z "$COOLIFY_STAGING_SSH_HOST" ] || [ -z "$COOLIFY_STAGING_SSH_KEY" ]; then
|
|
echo "Coolify staging SSH configuration is incomplete (worker health verification requires SSH + docker exec)"
|
|
exit 1
|
|
fi
|
|
REMOTE_SSH_KEY_FILE="$(mktemp)"
|
|
REMOTE_SSH_KNOWN_HOSTS="$(mktemp)"
|
|
trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT
|
|
# Materialise the key via an intermediate var so the secret is written to
|
|
# a file (never stdout/logs); avoids leaking it in CI output.
|
|
REMOTE_SSH_KEY_DATA="$COOLIFY_STAGING_SSH_KEY"
|
|
printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE"
|
|
unset REMOTE_SSH_KEY_DATA
|
|
chmod 600 "$REMOTE_SSH_KEY_FILE"
|
|
REMOTE_SSH_USER="${COOLIFY_STAGING_SSH_USER:-root}"
|
|
REMOTE_SSH_HOST="$COOLIFY_STAGING_SSH_HOST"
|
|
if command -v ssh-keyscan >/dev/null 2>&1; then
|
|
ssh-keyscan -H "$REMOTE_SSH_HOST" > "$REMOTE_SSH_KNOWN_HOSTS" 2>/dev/null || true
|
|
fi
|
|
if [ -s "$REMOTE_SSH_KNOWN_HOSTS" ]; then
|
|
REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=yes"
|
|
else
|
|
REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=accept-new"
|
|
fi
|
|
remote_ssh() {
|
|
ssh -i "$REMOTE_SSH_KEY_FILE" -o BatchMode=yes -o ConnectTimeout=10 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" $REMOTE_SSH_HOSTKEY_OPT "${REMOTE_SSH_USER}@${REMOTE_SSH_HOST}" "$@"
|
|
}
|
|
|
|
DEPLOYMENT_INDEX=0
|
|
DEPLOYMENT_WAIT_PIDS=""
|
|
for DEPLOYMENT_UUID in $(printf "%s" "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
|
|
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
|
|
node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent #$DEPLOYMENT_INDEX" &
|
|
DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!"
|
|
done
|
|
DEPLOYMENT_WAIT_FAILED=false
|
|
for PID in $DEPLOYMENT_WAIT_PIDS; do
|
|
if ! wait "$PID"; then
|
|
DEPLOYMENT_WAIT_FAILED=true
|
|
fi
|
|
done
|
|
if [ "$DEPLOYMENT_WAIT_FAILED" = "true" ]; then
|
|
echo "One or more Coolify staging dependent deployment waits failed"
|
|
exit 1
|
|
fi
|
|
|
|
verify_live_revision() {
|
|
APP_JSON="$1"
|
|
LABEL="$2"
|
|
REQUIRE_HEALTH_URL="${3:-auto}"
|
|
EXPECTED_IMAGE_ALIAS="${4:-staging}"
|
|
EXPECTED_SERVICE="${5:-}"
|
|
RESOURCE_KIND="${6:-auto}"
|
|
EXPECTED_WORKER_GROUP="${7:-}"
|
|
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
|
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
|
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
|
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
|
|
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
|
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
|
|
echo "$LABEL worker image tag does not match expected alias"
|
|
exit 1
|
|
fi
|
|
# Coolify 4.x exposes no command-execution REST API, so verify worker
|
|
# health by running the one-shot health probe inside the live container
|
|
# over SSH (docker exec) on the Coolify host. Fail-closed: any SSH /
|
|
# container-resolution / exec failure blocks the deploy.
|
|
WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')"
|
|
if [ -z "$WORKER_CONTAINER" ]; then
|
|
echo "$LABEL worker container not found on Coolify host"
|
|
exit 1
|
|
fi
|
|
# health.js exits non-zero when the worker reports a non-ready
|
|
# status (e.g. "misconfigured"), yet it still prints the diagnostic
|
|
# JSON (status + errors) to stdout. Capture stdout regardless of the
|
|
# exit code (|| true) so the real reason surfaces in the checks below
|
|
# instead of a generic "command unavailable". Empty output means the
|
|
# docker exec itself could not run (container gone, SSH failure, ...).
|
|
WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)"
|
|
if [ -z "$WORKER_HEALTH_OUTPUT" ]; then
|
|
echo "$LABEL worker docker exec health command unavailable (no output from health probe)"
|
|
exit 1
|
|
fi
|
|
WORKER_HEALTH_JSON="$(mktemp)"
|
|
printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON"
|
|
if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then
|
|
rm -f "$WORKER_HEALTH_JSON"
|
|
echo "$LABEL worker execute output is not JSON"
|
|
exit 1
|
|
fi
|
|
WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")"
|
|
rm -f "$WORKER_HEALTH_JSON"
|
|
if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then
|
|
echo "$LABEL worker health revision does not match image tag yet"
|
|
return 1
|
|
fi
|
|
if [ "$WORKER_ENABLED" != "true" ]; then
|
|
echo "$LABEL worker health reports workerEnabled=false"
|
|
exit 1
|
|
fi
|
|
if [ "$WORKER_STATUS" != "ready" ]; then
|
|
echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}"
|
|
exit 1
|
|
fi
|
|
if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then
|
|
echo "$LABEL worker group does not match expected queue group"
|
|
exit 1
|
|
fi
|
|
EXPECTED_WORKER_QUEUES=""
|
|
case "$EXPECTED_WORKER_GROUP" in
|
|
general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;;
|
|
mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;;
|
|
import)
|
|
if [ -n "$WORKER_QUEUES" ]; then
|
|
EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports"
|
|
fi
|
|
;;
|
|
webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;;
|
|
plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;;
|
|
esac
|
|
for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do
|
|
if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then
|
|
echo "$LABEL worker queues do not match expected queue group"
|
|
exit 1
|
|
fi
|
|
done
|
|
echo "$LABEL worker live revision matches image tag"
|
|
return
|
|
fi
|
|
if [ -z "$HEALTH_URL" ]; then
|
|
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
|
|
echo "$LABEL is an HTTP resource but has no health URL"
|
|
exit 1
|
|
fi
|
|
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
|
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
|
|
echo "$LABEL non-HTTP resource image tag does not match expected alias"
|
|
exit 1
|
|
fi
|
|
echo "$LABEL non-HTTP resource image tag matches expected alias"
|
|
return
|
|
fi
|
|
HEALTH_JSON="$(mktemp)"
|
|
if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then
|
|
rm -f "$HEALTH_JSON"
|
|
echo "$LABEL health endpoint is not ready yet"
|
|
return 1
|
|
fi
|
|
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
|
|
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
|
|
rm -f "$HEALTH_JSON"
|
|
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
|
|
echo "$LABEL health service does not match expected service"
|
|
exit 1
|
|
fi
|
|
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
|
|
echo "$LABEL health revision does not match staged image yet"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
check_dependent_resource() {
|
|
RESOURCE_UUID="$1"
|
|
RESOURCE_INDEX="$2"
|
|
HEALTHY=false
|
|
for ATTEMPT in $(seq 1 24); do
|
|
APP_JSON="$(mktemp)"
|
|
curl --fail --silent --show-error \
|
|
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
|
|
"$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
|
|
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
|
|
echo "Coolify staging dependent resource #$RESOURCE_INDEX status: $STATUS"
|
|
if [ "$STATUS" = "running:healthy" ]; then
|
|
RESOURCE_KIND="api"
|
|
EXPECTED_SERVICE="api"
|
|
EXPECTED_WORKER_GROUP=""
|
|
if printf ",%s," "$STAGING_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
|
|
RESOURCE_KIND="worker"
|
|
EXPECTED_SERVICE=""
|
|
case "$RESOURCE_UUID" in
|
|
"$STAGING_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general" ;;
|
|
"$STAGING_WORKER_MAIL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="mail" ;;
|
|
"$STAGING_WORKER_IMPORT_RESOURCE_UUID") EXPECTED_WORKER_GROUP="import" ;;
|
|
"$STAGING_WORKER_WEBHOOK_RESOURCE_UUID") EXPECTED_WORKER_GROUP="webhook" ;;
|
|
"$STAGING_WORKER_PLUGIN_RESOURCE_UUID") EXPECTED_WORKER_GROUP="plugin" ;;
|
|
esac
|
|
fi
|
|
if verify_live_revision "$APP_JSON" "Coolify staging dependent resource #$RESOURCE_INDEX" auto "$IMAGE_TAG" "$EXPECTED_SERVICE" "$RESOURCE_KIND" "$EXPECTED_WORKER_GROUP"; then
|
|
rm -f "$APP_JSON"
|
|
HEALTHY=true
|
|
break
|
|
fi
|
|
fi
|
|
rm -f "$APP_JSON"
|
|
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
|
|
echo "Coolify staging dependent resource #$RESOURCE_INDEX entered a terminal failure status"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
if [ "$HEALTHY" != "true" ]; then
|
|
echo "Coolify staging dependent resource #$RESOURCE_INDEX did not become healthy in time"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
RESOURCE_INDEX=0
|
|
RESOURCE_WAIT_PIDS=""
|
|
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
|
|
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
|
|
check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" &
|
|
RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!"
|
|
done
|
|
RESOURCE_WAIT_FAILED=false
|
|
for PID in $RESOURCE_WAIT_PIDS; do
|
|
if ! wait "$PID"; then
|
|
RESOURCE_WAIT_FAILED=true
|
|
fi
|
|
done
|
|
if [ "$RESOURCE_WAIT_FAILED" = "true" ]; then
|
|
echo "One or more Coolify staging dependent resources failed health checks"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Ensure Coolify production worker groups
|
|
env:
|
|
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
|
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
|
|
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
|
COOLIFY_PROVISION_WORKER_GROUPS: ${{ vars.COOLIFY_PROVISION_WORKER_GROUPS }}
|
|
COOLIFY_PROJECT_UUID: ${{ vars.COOLIFY_PROJECT_UUID }}
|
|
COOLIFY_SERVER_UUID: ${{ vars.COOLIFY_SERVER_UUID }}
|
|
COOLIFY_ENVIRONMENT_NAME: ${{ vars.COOLIFY_ENVIRONMENT_NAME }}
|
|
COOLIFY_ENVIRONMENT_UUID: ${{ vars.COOLIFY_ENVIRONMENT_UUID }}
|
|
COOLIFY_DESTINATION_UUID: ${{ vars.COOLIFY_DESTINATION_UUID }}
|
|
COOLIFY_WORKER_ENV_SOURCE_UUID: ${{ vars.COOLIFY_WORKER_ENV_SOURCE_UUID }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Coolify production worker group provisioning skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
node scripts/coolify-ensure-worker-groups.mjs production
|
|
|
|
- name: Configure Coolify production migration command
|
|
env:
|
|
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
|
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
|
|
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
|
MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts
|
|
SEED_COMMAND: cd /myapp && NODE_ENV=production node packages/db/dist/production-seed.server.js
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Coolify production migration command configuration skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
RESOLVED_COOLIFY_RESOURCE_UUIDS="${PRODUCTION_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_RESOURCE_UUIDS}"
|
|
if [ -z "$COOLIFY_API_URL" ] || [ -z "$RESOLVED_COOLIFY_RESOURCE_UUIDS" ] || [ -z "$COOLIFY_TOKEN" ]; then
|
|
echo "Coolify production configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
|
|
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
|
|
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
|
|
APPS_JSON="$(mktemp)"
|
|
trap 'rm -f "${APPS_JSON:-}"' EXIT
|
|
curl --fail --silent --show-error \
|
|
--header "Authorization: Bearer $COOLIFY_TOKEN" \
|
|
"$COOLIFY_API_URL/applications" > "$APPS_JSON"
|
|
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
|
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
|
|
select_worker_resource() {
|
|
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
|
|
}
|
|
WORKER_GENERAL_RESOURCE_UUID="$(select_worker_resource general)"
|
|
WORKER_MAIL_RESOURCE_UUID="$(select_worker_resource mail)"
|
|
WORKER_IMPORT_RESOURCE_UUID="$(select_worker_resource import)"
|
|
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
|
|
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
|
|
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
|
|
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
|
|
echo "Coolify production web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
|
|
exit 1
|
|
fi
|
|
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
|
if ! printf ",%s," "$RESOLVED_COOLIFY_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
|
|
echo "Coolify production deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
|
|
exit 1
|
|
fi
|
|
done
|
|
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
|
|
{
|
|
echo "PRODUCTION_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
|
|
echo "PRODUCTION_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
|
|
echo "PRODUCTION_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
|
|
echo "PRODUCTION_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
|
|
echo "PRODUCTION_WORKER_IMPORT_RESOURCE_UUID=$WORKER_IMPORT_RESOURCE_UUID"
|
|
echo "PRODUCTION_WORKER_WEBHOOK_RESOURCE_UUID=$WORKER_WEBHOOK_RESOURCE_UUID"
|
|
echo "PRODUCTION_WORKER_PLUGIN_RESOURCE_UUID=$WORKER_PLUGIN_RESOURCE_UUID"
|
|
echo "PRODUCTION_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
|
curl --fail --silent --show-error --request PATCH \
|
|
--header "Authorization: Bearer $COOLIFY_TOKEN" \
|
|
--header "Content-Type: application/json" \
|
|
--data "$MIGRATION_PAYLOAD" \
|
|
"$COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null
|
|
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
|
|
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
|
|
curl --fail --silent --show-error --request PATCH \
|
|
--header "Authorization: Bearer $COOLIFY_TOKEN" \
|
|
--header "Content-Type: application/json" \
|
|
--data "$IMAGE_TAG_PAYLOAD" \
|
|
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" >/dev/null
|
|
done
|
|
|
|
- name: Promote staging images to production aliases
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Promotion skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
for TARGET in web api worker; do
|
|
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
|
|
docker buildx imagetools create --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"
|
|
done
|
|
|
|
- name: Trigger Coolify production deploy
|
|
env:
|
|
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
|
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
|
|
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Production deploy skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$COOLIFY_API_URL" ] || [ -z "$PRODUCTION_WEB_RESOURCE_UUID" ] || [ -z "$COOLIFY_TOKEN" ]; then
|
|
echo "Coolify production configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
|
|
PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"
|
|
echo "PRODUCTION_WEB_DEPLOYMENT_UUID=$PRODUCTION_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
|
|
|
|
- name: Wait for Coolify production health
|
|
env:
|
|
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
|
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
|
|
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Production health wait skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$COOLIFY_API_URL" ] || [ -z "$PRODUCTION_WEB_RESOURCE_UUID" ] || [ -z "$COOLIFY_TOKEN" ]; then
|
|
echo "Coolify production configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
if [ -z "$PRODUCTION_WEB_DEPLOYMENT_UUID" ]; then
|
|
echo "Coolify production deployment UUID is missing"
|
|
exit 1
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
|
|
node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production web"
|
|
|
|
verify_live_revision() {
|
|
APP_JSON="$1"
|
|
LABEL="$2"
|
|
REQUIRE_HEALTH_URL="${3:-auto}"
|
|
EXPECTED_IMAGE_ALIAS="${4:-main}"
|
|
EXPECTED_SERVICE="${5:-}"
|
|
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
|
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
|
if [ -z "$HEALTH_URL" ]; then
|
|
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
|
|
echo "$LABEL is an HTTP resource but has no health URL"
|
|
exit 1
|
|
fi
|
|
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
|
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
|
|
echo "$LABEL non-HTTP resource image tag does not match expected alias"
|
|
exit 1
|
|
fi
|
|
echo "$LABEL non-HTTP resource image tag matches expected alias"
|
|
return
|
|
fi
|
|
HEALTH_JSON="$(mktemp)"
|
|
if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then
|
|
rm -f "$HEALTH_JSON"
|
|
echo "$LABEL health endpoint is not ready yet"
|
|
return 1
|
|
fi
|
|
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
|
|
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
|
|
rm -f "$HEALTH_JSON"
|
|
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
|
|
echo "$LABEL health service does not match expected service"
|
|
exit 1
|
|
fi
|
|
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
|
|
echo "$LABEL health revision does not match promoted image yet"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
RESOURCE_INDEX=0
|
|
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_WEB_RESOURCE_UUID" | tr ',' ' '); do
|
|
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
|
|
HEALTHY=false
|
|
for ATTEMPT in $(seq 1 24); do
|
|
APP_JSON="$(mktemp)"
|
|
curl --fail --silent --show-error \
|
|
--header "Authorization: Bearer $COOLIFY_TOKEN" \
|
|
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
|
|
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
|
|
echo "Coolify production resource #$RESOURCE_INDEX status: $STATUS"
|
|
if [ "$STATUS" = "running:healthy" ]; then
|
|
if verify_live_revision "$APP_JSON" "Coolify production resource #$RESOURCE_INDEX" required "$IMAGE_TAG" web; then
|
|
rm -f "$APP_JSON"
|
|
HEALTHY=true
|
|
break
|
|
fi
|
|
fi
|
|
rm -f "$APP_JSON"
|
|
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
|
|
echo "Coolify production resource #$RESOURCE_INDEX entered a terminal failure status"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
if [ "$HEALTHY" != "true" ]; then
|
|
echo "Coolify production resource #$RESOURCE_INDEX did not become healthy in time"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
- name: Prepare production migration status checkout
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Production migration status checkout skipped for staging"
|
|
exit 0
|
|
fi
|
|
git checkout --detach "$IMAGE_TAG"
|
|
CHECKOUT_SHA="$(git rev-parse HEAD)"
|
|
if [ "$CHECKOUT_SHA" != "$IMAGE_TAG" ]; then
|
|
echo "Production migration status checkout must match the staged image tag"
|
|
exit 1
|
|
fi
|
|
corepack enable
|
|
corepack prepare pnpm@10.33.0 --activate
|
|
pnpm config set store-dir .pnpm-store
|
|
|
|
- name: Cache production migration pnpm store
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: .pnpm-store
|
|
key: ${{ runner.os }}-pnpm-promote-${{ hashFiles('pnpm-lock.yaml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pnpm-
|
|
|
|
- name: Install production migration dependencies
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Production migration dependency install skipped for staging"
|
|
exit 0
|
|
fi
|
|
pnpm install --frozen-lockfile --prefer-offline
|
|
|
|
- name: Verify production migration status after web deploy
|
|
env:
|
|
DATABASE_URL: ${{ secrets.DATABASE_URL }}
|
|
# AD-24: Prod-DB liegt im privaten Netz des separaten Prod-Accounts und ist vom CI-Runner
|
|
# nicht erreichbar. verify-db-migrate-status erkennt P1001 (unreachable) und ueberspringt
|
|
# den Check sauber (exit 0), wenn dieses Flag gesetzt ist. Die Migration selbst laeuft
|
|
# ohnehin als Pre-Deployment-Command im web-Container (schlaegt fehl -> Deploy schlaegt
|
|
# fehl). Greift NUR bei tatsaechlich unerreichbarer DB; ein echter Drift gegen eine
|
|
# erreichbare DB wuerde weiterhin rot.
|
|
DB_MIGRATE_STATUS_ALLOW_UNREACHABLE: "true"
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Production migration status check skipped for staging"
|
|
exit 0
|
|
fi
|
|
if [ -z "$DATABASE_URL" ]; then
|
|
echo "DATABASE_URL is required for production migration status verification"
|
|
exit 1
|
|
fi
|
|
pnpm run verify:db-migrate-status
|
|
|
|
- name: Trigger Coolify production dependent deploys
|
|
env:
|
|
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
|
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Production dependent deploy skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" ]; then
|
|
echo "No production dependent resources to deploy"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$COOLIFY_API_URL" ] || [ -z "$COOLIFY_TOKEN" ]; then
|
|
echo "Coolify production configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
|
|
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=""
|
|
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
|
|
DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"
|
|
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="${PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS:+$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
|
|
done
|
|
echo "PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV"
|
|
|
|
- name: Wait for Coolify production dependent health
|
|
env:
|
|
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
|
|
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
|
|
# AD-24: Prod ist Multi-Server. Worker laufen auf der worker-VM (privates Netz), nur ueber
|
|
# die control-Bastion erreichbar -> Worker-Health-Verifikation (docker exec) per SSH-ProxyJump.
|
|
COOLIFY_PROD_SSH_KEY: ${{ secrets.COOLIFY_PROD_SSH_KEY }}
|
|
COOLIFY_PROD_SSH_KNOWN_HOSTS: ${{ secrets.COOLIFY_PROD_SSH_KNOWN_HOSTS }}
|
|
COOLIFY_PROD_SSH_HOST: ${{ vars.COOLIFY_PROD_SSH_HOST }}
|
|
COOLIFY_PROD_WORKER_PRIVATE_IP: ${{ vars.COOLIFY_PROD_WORKER_PRIVATE_IP }}
|
|
run: |
|
|
if [ "$DEPLOY_ACTION" != "promote" ]; then
|
|
echo "Production dependent health wait skipped for staging"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" ]; then
|
|
echo "No production dependent resources to wait for"
|
|
exit 0
|
|
fi
|
|
if [ -z "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" ]; then
|
|
echo "Coolify production dependent deployment UUIDs are missing"
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$COOLIFY_API_URL" ] || [ -z "$COOLIFY_TOKEN" ]; then
|
|
echo "Coolify production configuration is incomplete"
|
|
exit 1
|
|
fi
|
|
|
|
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
|
|
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
|
|
DEPLOYMENT_INDEX=0
|
|
DEPLOYMENT_WAIT_PIDS=""
|
|
for DEPLOYMENT_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
|
|
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
|
|
node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent #$DEPLOYMENT_INDEX" &
|
|
DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!"
|
|
done
|
|
DEPLOYMENT_WAIT_FAILED=false
|
|
for PID in $DEPLOYMENT_WAIT_PIDS; do
|
|
if ! wait "$PID"; then
|
|
DEPLOYMENT_WAIT_FAILED=true
|
|
fi
|
|
done
|
|
if [ "$DEPLOYMENT_WAIT_FAILED" = "true" ]; then
|
|
echo "One or more Coolify production dependent deployment waits failed"
|
|
exit 1
|
|
fi
|
|
|
|
# SSH helper for worker health verification (Coolify has no exec REST API). AD-24: Prod ist
|
|
# Multi-Server -> Worker laufen auf der worker-VM (privates Netz), erreichbar nur ueber die
|
|
# control-Bastion. ProxyJump + gepinnte Host-Keys (kein StrictHostKeyChecking=no). Secrets
|
|
# werden via Zwischen-Variable in Dateien geschrieben (nie stdout/Logs).
|
|
if [ -z "$COOLIFY_PROD_SSH_HOST" ] || [ -z "$COOLIFY_PROD_SSH_KEY" ] || [ -z "$COOLIFY_PROD_SSH_KNOWN_HOSTS" ] || [ -z "$COOLIFY_PROD_WORKER_PRIVATE_IP" ]; then
|
|
echo "Coolify production SSH configuration is incomplete (worker health verification requires SSH + docker exec)"
|
|
exit 1
|
|
fi
|
|
REMOTE_SSH_KEY_FILE="$(mktemp)"
|
|
REMOTE_SSH_KNOWN_HOSTS="$(mktemp)"
|
|
trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT
|
|
REMOTE_SSH_KEY_DATA="$COOLIFY_PROD_SSH_KEY"
|
|
printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE"
|
|
unset REMOTE_SSH_KEY_DATA
|
|
chmod 600 "$REMOTE_SSH_KEY_FILE"
|
|
REMOTE_SSH_KNOWN_HOSTS_DATA="$COOLIFY_PROD_SSH_KNOWN_HOSTS"
|
|
printf '%s\n' "$REMOTE_SSH_KNOWN_HOSTS_DATA" > "$REMOTE_SSH_KNOWN_HOSTS"
|
|
unset REMOTE_SSH_KNOWN_HOSTS_DATA
|
|
# ProxyCommand (nicht ProxyJump): ProxyJump reicht den -i-Key nicht zuverlaessig an den
|
|
# Jump-Hop weiter (MaxAuthTries -> Permission denied). ProxyCommand gibt der Bastion den Key
|
|
# explizit (IdentitiesOnly), Host-Keys fuer beide Hops gepinnt.
|
|
REMOTE_SSH_PROXY="ssh -i $REMOTE_SSH_KEY_FILE -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile=$REMOTE_SSH_KNOWN_HOSTS -o StrictHostKeyChecking=yes -W %h:%p root@${COOLIFY_PROD_SSH_HOST}"
|
|
remote_ssh() {
|
|
ssh -i "$REMOTE_SSH_KEY_FILE" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" -o StrictHostKeyChecking=yes -o ProxyCommand="$REMOTE_SSH_PROXY" "root@${COOLIFY_PROD_WORKER_PRIVATE_IP}" "$@"
|
|
}
|
|
|
|
verify_live_revision() {
|
|
APP_JSON="$1"
|
|
LABEL="$2"
|
|
REQUIRE_HEALTH_URL="${3:-auto}"
|
|
EXPECTED_IMAGE_ALIAS="${4:-main}"
|
|
EXPECTED_SERVICE="${5:-}"
|
|
RESOURCE_KIND="${6:-auto}"
|
|
EXPECTED_WORKER_GROUP="${7:-}"
|
|
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
|
|
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
|
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
|
|
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
|
|
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
|
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
|
|
echo "$LABEL worker image tag does not match expected alias"
|
|
exit 1
|
|
fi
|
|
# Coolify 4.x exposes no command-execution REST API, so verify worker
|
|
# health by running the one-shot health probe inside the live container
|
|
# over SSH (docker exec) on the Coolify host. Fail-closed: any SSH /
|
|
# container-resolution / exec failure blocks the deploy.
|
|
WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')"
|
|
if [ -z "$WORKER_CONTAINER" ]; then
|
|
echo "$LABEL worker container not found on Coolify host"
|
|
exit 1
|
|
fi
|
|
# health.js exits non-zero when the worker reports a non-ready
|
|
# status (e.g. "misconfigured"), yet it still prints the diagnostic
|
|
# JSON (status + errors) to stdout. Capture stdout regardless of the
|
|
# exit code (|| true) so the real reason surfaces in the checks below
|
|
# instead of a generic "command unavailable". Empty output means the
|
|
# docker exec itself could not run (container gone, SSH failure, ...).
|
|
WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)"
|
|
if [ -z "$WORKER_HEALTH_OUTPUT" ]; then
|
|
echo "$LABEL worker docker exec health command unavailable (no output from health probe)"
|
|
exit 1
|
|
fi
|
|
WORKER_HEALTH_JSON="$(mktemp)"
|
|
printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON"
|
|
if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then
|
|
rm -f "$WORKER_HEALTH_JSON"
|
|
echo "$LABEL worker execute output is not JSON"
|
|
exit 1
|
|
fi
|
|
WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")"
|
|
WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")"
|
|
rm -f "$WORKER_HEALTH_JSON"
|
|
if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then
|
|
echo "$LABEL worker health revision does not match image tag yet"
|
|
return 1
|
|
fi
|
|
if [ "$WORKER_ENABLED" != "true" ]; then
|
|
echo "$LABEL worker health reports workerEnabled=false"
|
|
exit 1
|
|
fi
|
|
if [ "$WORKER_STATUS" != "ready" ]; then
|
|
echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}"
|
|
exit 1
|
|
fi
|
|
if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then
|
|
echo "$LABEL worker group does not match expected queue group"
|
|
exit 1
|
|
fi
|
|
EXPECTED_WORKER_QUEUES=""
|
|
case "$EXPECTED_WORKER_GROUP" in
|
|
general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;;
|
|
mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;;
|
|
import)
|
|
if [ -n "$WORKER_QUEUES" ]; then
|
|
EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports"
|
|
fi
|
|
;;
|
|
webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;;
|
|
plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;;
|
|
esac
|
|
for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do
|
|
if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then
|
|
echo "$LABEL worker queues do not match expected queue group"
|
|
exit 1
|
|
fi
|
|
done
|
|
echo "$LABEL worker live revision matches image tag"
|
|
return
|
|
fi
|
|
if [ -z "$HEALTH_URL" ]; then
|
|
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
|
|
echo "$LABEL is an HTTP resource but has no health URL"
|
|
exit 1
|
|
fi
|
|
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
|
|
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
|
|
echo "$LABEL non-HTTP resource image tag does not match expected alias"
|
|
exit 1
|
|
fi
|
|
echo "$LABEL non-HTTP resource image tag matches expected alias"
|
|
return
|
|
fi
|
|
HEALTH_JSON="$(mktemp)"
|
|
if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then
|
|
rm -f "$HEALTH_JSON"
|
|
echo "$LABEL health endpoint is not ready yet"
|
|
return 1
|
|
fi
|
|
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
|
|
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
|
|
rm -f "$HEALTH_JSON"
|
|
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
|
|
echo "$LABEL health service does not match expected service"
|
|
exit 1
|
|
fi
|
|
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
|
|
echo "$LABEL health revision does not match promoted image yet"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
check_dependent_resource() {
|
|
RESOURCE_UUID="$1"
|
|
RESOURCE_INDEX="$2"
|
|
HEALTHY=false
|
|
for ATTEMPT in $(seq 1 24); do
|
|
APP_JSON="$(mktemp)"
|
|
curl --fail --silent --show-error \
|
|
--header "Authorization: Bearer $COOLIFY_TOKEN" \
|
|
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
|
|
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
|
|
echo "Coolify production dependent resource #$RESOURCE_INDEX status: $STATUS"
|
|
if [ "$STATUS" = "running:healthy" ]; then
|
|
RESOURCE_KIND="api"
|
|
EXPECTED_SERVICE="api"
|
|
EXPECTED_WORKER_GROUP=""
|
|
if printf ",%s," "$PRODUCTION_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
|
|
RESOURCE_KIND="worker"
|
|
EXPECTED_SERVICE=""
|
|
case "$RESOURCE_UUID" in
|
|
"$PRODUCTION_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general" ;;
|
|
"$PRODUCTION_WORKER_MAIL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="mail" ;;
|
|
"$PRODUCTION_WORKER_IMPORT_RESOURCE_UUID") EXPECTED_WORKER_GROUP="import" ;;
|
|
"$PRODUCTION_WORKER_WEBHOOK_RESOURCE_UUID") EXPECTED_WORKER_GROUP="webhook" ;;
|
|
"$PRODUCTION_WORKER_PLUGIN_RESOURCE_UUID") EXPECTED_WORKER_GROUP="plugin" ;;
|
|
esac
|
|
fi
|
|
if verify_live_revision "$APP_JSON" "Coolify production dependent resource #$RESOURCE_INDEX" auto "$IMAGE_TAG" "$EXPECTED_SERVICE" "$RESOURCE_KIND" "$EXPECTED_WORKER_GROUP"; then
|
|
rm -f "$APP_JSON"
|
|
HEALTHY=true
|
|
break
|
|
fi
|
|
fi
|
|
rm -f "$APP_JSON"
|
|
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
|
|
echo "Coolify production dependent resource #$RESOURCE_INDEX entered a terminal failure status"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
if [ "$HEALTHY" != "true" ]; then
|
|
echo "Coolify production dependent resource #$RESOURCE_INDEX did not become healthy in time"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
RESOURCE_INDEX=0
|
|
RESOURCE_WAIT_PIDS=""
|
|
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
|
|
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
|
|
check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" &
|
|
RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!"
|
|
done
|
|
RESOURCE_WAIT_FAILED=false
|
|
for PID in $RESOURCE_WAIT_PIDS; do
|
|
if ! wait "$PID"; then
|
|
RESOURCE_WAIT_FAILED=true
|
|
fi
|
|
done
|
|
if [ "$RESOURCE_WAIT_FAILED" = "true" ]; then
|
|
echo "One or more Coolify production dependent resources failed health checks"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Mark Sentry deploy
|
|
if: success()
|
|
env:
|
|
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
|
|
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
|
|
SENTRY_URL: ${{ secrets.SENTRY_URL }}
|
|
run: |
|
|
if [ -z "${SENTRY_AUTH_TOKEN:-}" ] || [ -z "${SENTRY_ORG:-}" ] || [ -z "${SENTRY_URL:-}" ]; then
|
|
echo "Sentry secrets not configured; skipping deploy marker"
|
|
exit 0
|
|
fi
|
|
if [ "$DEPLOY_ACTION" = "promote" ]; then
|
|
SENTRY_ENVIRONMENT=production
|
|
RELEASE_SHA="${IMAGE_TAG}"
|
|
else
|
|
SENTRY_ENVIRONMENT=staging
|
|
RELEASE_SHA="${SHA}"
|
|
fi
|
|
for PROJECT in boilerplate-api boilerplate-worker boilerplate-web; do
|
|
bash scripts/ci/sentry-release.sh \
|
|
--project "$PROJECT" \
|
|
--environment "$SENTRY_ENVIRONMENT" \
|
|
--release "$RELEASE_SHA" \
|
|
--sourcemaps none
|
|
done
|
|
|
|
- name: Purge asset CDN
|
|
# Whole-zone purge against Bunny.net so the edge picks up the new
|
|
# hash-named bundles immediately. The helper script no-ops when
|
|
# BUNNY_API_KEY is empty, so this step is safe to wire even before
|
|
# the operator has provisioned a Bunny pull zone.
|
|
# See docs/ops/asset-cdn.md for the one-time setup.
|
|
if: success()
|
|
env:
|
|
BUNNY_PULL_ZONE_ID: ${{ secrets.BUNNY_PULL_ZONE_ID }}
|
|
BUNNY_API_KEY: ${{ secrets.BUNNY_API_KEY }}
|
|
ASSET_CDN_URL: ${{ vars.ASSET_CDN_URL }}
|
|
run: |
|
|
if [ -z "${BUNNY_API_KEY:-}" ] || [ -z "${BUNNY_PULL_ZONE_ID:-}" ]; then
|
|
echo "BUNNY_API_KEY / BUNNY_PULL_ZONE_ID not configured; skipping CDN purge"
|
|
exit 0
|
|
fi
|
|
pnpm exec tsx scripts/cdn/bunny-purge.ts
|