import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { afterEach, describe, it } from "vitest"; import { formatGiteaDeployWorkflowResult, verifyGiteaDeployWorkflow } from "./verify-gitea-deploy-workflow.mjs"; const LEGACY_WORKFLOW_FIXTURE = `name: Stage main / Promote staging to production on: push: branches: [main] paths-ignore: - "scripts/ci/docker-smoke-suite.sh" - "scripts/docker-api-runtime-smoke.sh" - "scripts/docker-hocuspocus-runtime-smoke.sh" - "scripts/docker-mcp-runtime-smoke.sh" - "scripts/docker-portal-domain-controller-runtime-smoke.sh" - "scripts/docker-web-runtime-smoke.sh" - "scripts/docker-worker-runtime-smoke.sh" - "scripts/lib/docker-rls-role-fixture.sh" - "scripts/lib/docker-runtime-smoke.sh" - "scripts/lib/run-command-with-timeout.mjs" - "scripts/verify-gitea-ci-workflow.mjs" - "scripts/verify-portal-domain-controller-deploy.mjs" # A manual run is intentionally production-only and input-free. Starting it # is the operator's audited confirmation that backup/restore-point readiness # was checked; the workflow discovers the release that is live on staging. workflow_dispatch: env: HIVEGRID_GITEA_HOST_IP: \${{ vars.HIVEGRID_GITEA_HOST_IP }} concurrency: group: hivegrid-deploy cancel-in-progress: false jobs: stage-or-promote: runs-on: toolctrl-deploy env: REGISTRY: git.tool-ctrl.de IMAGE_NAMESPACE: toolctrl-labs IMAGE_PREFIX: hivegrid DEPLOY_BUILDER_NAME: hivegrid-deploy-v0312-gc1 DEPLOY_BUILDKIT_IMAGE: moby/buildkit:v0.31.2@sha256:2f5adac4ecd194d9f8c10b7b5d7bceb5186853db1b26e5abd3a657af0b7e26ec SHA: \${{ gitea.sha }} steps: - name: Resolve deployment action run: | REF="\${GITEA_REF:-\${{ gitea.ref }}}" EVENT_NAME="\${GITEA_EVENT_NAME:-\${{ gitea.event_name }}}" if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/main" ]; then DEPLOY_ACTION=stage elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then DEPLOY_ACTION=promote if [ "$REF" != "refs/heads/main" ]; then echo "Promote must be dispatched from refs/heads/main" exit 1 fi else echo "Unsupported deployment action: event=$EVENT_NAME ref=$REF" exit 1 fi if ! printf '%s' "$SHA" | grep -Eq '^[0-9a-f]{40}$'; then echo "Gitea SHA must be a full 40-character commit SHA" exit 1 fi if [ "$DEPLOY_ACTION" = "stage" ]; then IMAGE_TAG="$SHA" echo "IMAGE_TAG=$IMAGE_TAG" >> "$GITHUB_ENV" fi echo "DEPLOY_ACTION=$DEPLOY_ACTION" >> "$GITHUB_ENV" - name: Setup hosts override and Docker CLI run: | if [ -n "\${HIVEGRID_GITEA_HOST_IP:-}" ]; then if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token" exit 1 fi printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts fi - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - name: Quiesce orphaned deploy BuildKit run: | BUILDER_CONTAINER="buildx_buildkit_\${DEPLOY_BUILDER_NAME}0" if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1 && [ "$(docker inspect --format '{{.State.Running}}' "$BUILDER_CONTAINER")" = "true" ]; then docker stop "$BUILDER_CONTAINER" fi - name: Verify deploy wiring run: | node scripts/verify-gitea-deploy-workflow.mjs node scripts/verify-deploy-readiness.mjs node scripts/verify-docker-baseline.mjs node scripts/verify-docker-non-root.mjs node scripts/verify-portal-domain-controller-deploy.mjs node --check scripts/coolify-deploy.mjs node --check scripts/coolify-ensure-worker-groups.mjs node --check scripts/ci/verify-deploy-buildkit-config.mjs bash -n scripts/coolify-prepull-images.sh bash -n scripts/ci/install-cosign.sh - name: Verify checkout revision run: | CHECKOUT_SHA="$(git rev-parse HEAD)" if [ "$CHECKOUT_SHA" != "$SHA" ]; then echo "Checked out revision must match the Gitea SHA" exit 1 fi - name: Login to Gitea Registry run: echo "\${{ secrets.REGISTRY_PAT }}" | docker login "$REGISTRY" --username "\${{ secrets.REGISTRY_USER }}" --password-stdin - name: Verify staged promotion provenance run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Staged promotion provenance skipped for staging" exit 0 fi git fetch --no-tags origin main git cat-file -e "$IMAGE_TAG^{commit}" git merge-base --is-ancestor "$IMAGE_TAG" origin/main for TARGET in web api worker mcp; do IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}')" STAGING_DIGEST="$(docker buildx imagetools inspect "$IMAGE:staging" --format '{{json .Manifest.Digest}}')" if [ -z "$SHA_DIGEST" ] || [ "$SHA_DIGEST" != "$STAGING_DIGEST" ]; then echo "Staged image digest for $TARGET does not match $IMAGE_TAG" exit 1 fi done - name: Start isolated deploy BuildKit if: env.DEPLOY_ACTION == 'stage' run: | BUILDER_CONTAINER="buildx_buildkit_\${DEPLOY_BUILDER_NAME}0" if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1; then ACTUAL_BUILDKIT_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$BUILDER_CONTAINER")" if [ "$ACTUAL_BUILDKIT_IMAGE" != "$DEPLOY_BUILDKIT_IMAGE" ]; then exit 1 fi ACTUAL_RESTART_POLICY="$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$BUILDER_CONTAINER")" if [ "$ACTUAL_RESTART_POLICY" != "no" ]; then exit 1 fi fi if docker buildx inspect "$DEPLOY_BUILDER_NAME" >/dev/null 2>&1; then docker buildx inspect --bootstrap "$DEPLOY_BUILDER_NAME" else docker buildx create \ --name "$DEPLOY_BUILDER_NAME" \ --driver docker-container \ --driver-opt "image=$DEPLOY_BUILDKIT_IMAGE" \ --driver-opt "restart-policy=no" \ --buildkitd-config scripts/ci/deploy-buildkitd.toml \ --bootstrap fi ACTUAL_BUILDKIT_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$BUILDER_CONTAINER")" ACTUAL_RESTART_POLICY="$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$BUILDER_CONTAINER")" if [ "$ACTUAL_BUILDKIT_IMAGE" != "$DEPLOY_BUILDKIT_IMAGE" ] || [ "$ACTUAL_RESTART_POLICY" != "no" ]; then exit 1 fi docker exec "$BUILDER_CONTAINER" cat /etc/buildkit/buildkitd.toml \ | node scripts/ci/verify-deploy-buildkit-config.mjs scripts/ci/deploy-buildkitd.toml if [ -z "\${HIVEGRID_GITEA_HOST_IP:-}" ]; then echo "HIVEGRID_GITEA_HOST_IP is required for the isolated deploy builder" exit 1 fi if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token" exit 1 fi docker exec \\ --env HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" \\ --interactive \\ "$BUILDER_CONTAINER" \\ sh -eu <<'HIVEGRID_BUILDKIT_HOST_OVERRIDE' HOSTS_TMP="$(mktemp)" trap 'rm -f "$HOSTS_TMP"' EXIT HUP INT TERM awk -v host="git.tool-ctrl.de" ' { rewritten = "" for (field = 1; field <= NF; field += 1) { if (field > 1 && $field == host) { continue } rewritten = rewritten == "" ? $field : rewritten " " $field } if (rewritten != "") { print rewritten } } ' /etc/hosts > "$HOSTS_TMP" cat "$HOSTS_TMP" > /etc/hosts printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts BUILDKIT_HOST_COUNT="$(awk -v host="git.tool-ctrl.de" '{ for (field = 2; field <= NF; field += 1) if ($field == host) count += 1 } END { print count + 0 }' /etc/hosts)" BUILDKIT_EXACT_HOST_COUNT="$(awk -v host="git.tool-ctrl.de" -v ip="$HIVEGRID_GITEA_HOST_IP" '$1 == ip { for (field = 2; field <= NF; field += 1) if ($field == host) count += 1 } END { print count + 0 }' /etc/hosts)" if [ "$BUILDKIT_HOST_COUNT" -ne 1 ] || [ "$BUILDKIT_EXACT_HOST_COUNT" -ne 1 ]; then echo "BuildKit registry host override did not converge to exactly one validated mapping" exit 1 fi HIVEGRID_BUILDKIT_HOST_OVERRIDE - name: Build and push staging images env: VITE_HOCUSPOCUS_URL: \${{ vars.VITE_HOCUSPOCUS_URL }} run: | BUILD_PIDS=() BUILD_TARGETS=() for TARGET in web api worker mcp; do ( IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" WEB_BUILD_ARGS=() if [ "$TARGET" = "web" ]; then WEB_BUILD_ARGS=(--build-arg VITE_SENTRY_DSN_WEB="$VITE_SENTRY_DSN_WEB" --build-arg VITE_SENTRY_RELEASE="$SHA" --build-arg VITE_HOCUSPOCUS_URL="$VITE_HOCUSPOCUS_URL") fi docker buildx build --builder "$DEPLOY_BUILDER_NAME" --push --target "$TARGET" --file Dockerfile --cache-from "type=registry,ref=$IMAGE:staging" --cache-to "type=inline" --build-arg APP_REVISION="$SHA" "\${WEB_BUILD_ARGS[@]}" --tag "$IMAGE:$IMAGE_TAG" . ) & BUILD_PIDS+=("$!") BUILD_TARGETS+=("$TARGET") done BUILD_FAILED=0 for INDEX in "\${!BUILD_PIDS[@]}"; do if ! wait "\${BUILD_PIDS[$INDEX]}"; then echo "Docker build failed for \${BUILD_TARGETS[$INDEX]}" BUILD_FAILED=1 fi done if [ "$BUILD_FAILED" -ne 0 ]; then exit "$BUILD_FAILED" fi for TARGET in web api worker mcp; do IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" docker buildx imagetools create --prefer-index=false --tag "$IMAGE:staging" "$IMAGE:$IMAGE_TAG" done - name: Stop isolated deploy BuildKit if: always() run: | if [ "\${DEPLOY_ACTION:-}" != "stage" ]; then exit 0 fi BUILDER_CONTAINER="buildx_buildkit_\${DEPLOY_BUILDER_NAME}0" docker inspect "$BUILDER_CONTAINER" >/dev/null docker stop "$BUILDER_CONTAINER" - name: Install cosign if: env.DEPLOY_ACTION == 'stage' run: bash scripts/ci/install-cosign.sh - name: Validate Coolify staging runtime preflight env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_STAGING_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" - name: Ensure Coolify staging worker groups run: node scripts/coolify-ensure-worker-groups.mjs staging - name: Configure Coolify staging migration boot flags env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} run: | WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const apps = [{ uuid: "web", docker_registry_image_name: process.env.WEB_IMAGE, docker_registry_image_tag: process.env.IMAGE_TAG }]; const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");')" API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" node -e 'process.stdout.write("api")')" MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e 'process.stdout.write("mcp")')" WORKER_GENERAL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=general WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-general")')" WORKER_MAIL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=mail WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-mail")')" WORKER_IMPORT_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=import WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-import")')" WORKER_WEBHOOK_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=webhook WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-webhook")')" WORKER_PLUGIN_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=plugin WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-plugin")')" WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID" for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do if ! printf ",%s," "$COOLIFY_STAGING_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then echo "Coolify staging deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications" exit 1 fi done node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID" node scripts/coolify-deploy.mjs assert-no-app-revision-override "$API_RESOURCE_UUID" DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS" echo "STAGING_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID" echo "STAGING_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS" echo "STAGING_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS" MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\ --header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\ --header "Content-Type: application/json" \\ --data "$MIGRATION_PAYLOAD" \\ "$COOLIFY_STAGING_API_URL/applications/$WEB_RESOURCE_UUID" node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \\ MIGRATE_ON_BOOT=true \\ SEED_ON_BOOT=true \\ SEED_DEMO_DATA=true \\ ALLOW_STAGING_DEMO_SEED=true \\ SEED_DEMO_PRO_MANAGED=true IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')" for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\ --header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\ --header "Content-Type: application/json" \\ --data "$IMAGE_TAG_PAYLOAD" \\ "$COOLIFY_STAGING_API_URL/applications/$RESOURCE_UUID" done - name: Trigger Coolify staging deploy env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")" echo "STAGING_WEB_DEPLOYMENT_UUID=$STAGING_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV" - name: Wait for Coolify staging health env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web" RESOURCE_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$STAGING_WEB_RESOURCE_UUID" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) APP_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\ --header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\ "$COOLIFY_STAGING_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS" HEALTH_URL="https://staging-web.example.invalid" HEALTH_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON" HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" EXPECTED_SERVICE="\${5:-web}" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then exit 1 fi test "$STATUS" = "running:healthy" done - name: Trigger Coolify staging dependent deploys env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-deploy.mjs configure-mcp-healthcheck "$STAGING_MCP_RESOURCE_UUID" "$IMAGE_TAG" STAGING_DEPENDENT_DEPLOYMENT_UUIDS="" STAGING_DEPENDENT_TRIGGER_FAILED=false DEPLOYMENT_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1)) if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then STAGING_DEPENDENT_DEPLOYMENT_UUIDS="\${STAGING_DEPENDENT_DEPLOYMENT_UUIDS:+$STAGING_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID" if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent deploy #$DEPLOYMENT_INDEX"; then STAGING_DEPENDENT_TRIGGER_FAILED=true fi else STAGING_DEPENDENT_TRIGGER_FAILED=true fi done echo "STAGING_DEPENDENT_DEPLOYMENT_UUIDS=$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV" echo "STAGING_DEPENDENT_TRIGGER_FAILED=$STAGING_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV" - name: Wait for Coolify staging dependent health env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" DEPLOYMENT_WAIT_PIDS="" for DEPLOYMENT_UUID in $(printf "%s" "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent" & DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!" done for PID in $DEPLOYMENT_WAIT_PIDS; do wait "$PID" done RESOURCE_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) APP_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\ --header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\ "$COOLIFY_STAGING_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify staging dependent resource #$RESOURCE_INDEX status: $STATUS" HEALTH_URL="https://staging-api.example.invalid" HTTP_RESOURCE="$(node -e 'process.stdout.write("true")')" WORKER_RESOURCE="$(node -e 'process.stdout.write("false")')" if printf ",%s," "$STAGING_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then WORKER_RESOURCE="$(node -e 'process.stdout.write("true")')" fi EXPECTED_WORKER_GROUP="\${7:-general}" if [ "$WORKER_RESOURCE" = "true" ]; then CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("staging")')" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL worker image tag does not match expected alias" exit 1 fi if [ -z "$EXPECTED_WORKER_GROUP" ]; then exit 1 fi EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP" CONFIGURED_HEALTH_COMMAND="$EXPECTED_HEALTH_COMMAND" CONFIGURED_HEALTH_TYPE="cmd" HEALTH_CHECK_ENABLED="true" CUSTOM_HEALTHCHECK_FOUND="false" if [ "$HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_HEALTH_COMMAND" != "$EXPECTED_HEALTH_COMMAND" ] || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then echo "$LABEL worker healthcheck is not bound to the deployed revision" exit 1 fi echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy" return fi EXPECTED_MCP_HEALTH_COMMAND="node apps/mcp/dist/deploy-health.js $IMAGE_TAG" CONFIGURED_MCP_HEALTH_COMMAND="$EXPECTED_MCP_HEALTH_COMMAND" CONFIGURED_MCP_HEALTH_TYPE="cmd" MCP_HEALTH_CHECK_ENABLED="true" MCP_CUSTOM_HEALTHCHECK_FOUND="false" if [ "$MCP_HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_MCP_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_MCP_HEALTH_COMMAND" != "$EXPECTED_MCP_HEALTH_COMMAND" ] || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then exit 1 fi if [ -z "$HEALTH_URL" ] && [ "$HTTP_RESOURCE" = "true" ]; then echo "$LABEL is an HTTP resource but has no health URL" exit 1 fi EXPECTED_IMAGE_ALIAS="\${4:-staging}" CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("staging")')" if [ -z "$HEALTH_URL" ] && [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then exit 1 fi HEALTH_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON" HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" EXPECTED_SERVICE="\${5:-api}" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then exit 1 fi test "$STATUS" = "running:healthy" done check_dependent_resource() { RESOURCE_UUID="$1" RESOURCE_INDEX="$2" test -n "$RESOURCE_UUID" } RESOURCE_WAIT_PIDS="" for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" & RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!" done for PID in $RESOURCE_WAIT_PIDS; do wait "$PID" done if [ "\${STAGING_DEPENDENT_TRIGGER_FAILED:-false}" = "true" ]; then echo "One or more Coolify staging dependent deploy triggers failed" exit 1 fi - name: Promote staging images to production aliases run: | for TARGET in web api worker mcp; do IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG" done - name: Validate Coolify production runtime preflight env: COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }} COOLIFY_RESOURCE_UUIDS: \${{ vars.COOLIFY_RESOURCE_UUIDS }} COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" - name: Ensure Coolify production worker groups run: node scripts/coolify-ensure-worker-groups.mjs production - name: Configure Coolify production migration boot flags env: COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }} COOLIFY_RESOURCE_UUIDS: \${{ vars.COOLIFY_RESOURCE_UUIDS }} COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }} run: | WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$COOLIFY_RESOURCE_UUIDS" node -e 'const apps = [{ uuid: "web", docker_registry_image_name: process.env.WEB_IMAGE, docker_registry_image_tag: process.env.IMAGE_TAG }]; const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");')" API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" node -e 'process.stdout.write("api")')" MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e 'process.stdout.write("mcp")')" WORKER_GENERAL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=general WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-general")')" WORKER_MAIL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=mail WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-mail")')" WORKER_IMPORT_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=import WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-import")')" WORKER_WEBHOOK_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=webhook WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-webhook")')" WORKER_PLUGIN_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=plugin WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-plugin")')" WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID" for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do if ! printf ",%s," "$COOLIFY_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then echo "Coolify production deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications" exit 1 fi done node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID" node scripts/coolify-deploy.mjs assert-no-app-revision-override "$API_RESOURCE_UUID" DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS" echo "PRODUCTION_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID" echo "PRODUCTION_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS" echo "PRODUCTION_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS" MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\ --header "Authorization: Bearer $COOLIFY_TOKEN" \\ --header "Content-Type: application/json" \\ --data "$MIGRATION_PAYLOAD" \\ "$COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \\ MIGRATE_ON_BOOT=true \\ SEED_ON_BOOT=true IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')" for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\ --header "Authorization: Bearer $COOLIFY_TOKEN" \\ --header "Content-Type: application/json" \\ --data "$IMAGE_TAG_PAYLOAD" \\ "$COOLIFY_API_URL/applications/$RESOURCE_UUID" done - name: Trigger Coolify production deploy env: COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }} COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")" echo "PRODUCTION_WEB_DEPLOYMENT_UUID=$PRODUCTION_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV" - name: Wait for Coolify production health env: COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }} COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production web" RESOURCE_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_WEB_RESOURCE_UUID" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) APP_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\ --header "Authorization: Bearer $COOLIFY_TOKEN" \\ "$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify production resource #$RESOURCE_INDEX status: $STATUS" HEALTH_URL="https://production-web.example.invalid" HEALTH_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON" HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" EXPECTED_SERVICE="\${5:-web}" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then exit 1 fi test "$STATUS" = "running:healthy" done - name: Verify production migration status after web deploy if: env.DEPLOY_ACTION == 'promote' env: DATABASE_URL: \${{ secrets.DATABASE_URL }} run: | if [ -z "$DATABASE_URL" ]; then echo "DATABASE_URL is required for production migration status verification" exit 1 fi pnpm run verify:db-migrate-status - name: Trigger Coolify production dependent deploys env: COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }} COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs configure-mcp-healthcheck "$PRODUCTION_MCP_RESOURCE_UUID" "$IMAGE_TAG" PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="" PRODUCTION_DEPENDENT_TRIGGER_FAILED=false DEPLOYMENT_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1)) if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="\${PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS:+$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID" if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent deploy #$DEPLOYMENT_INDEX"; then PRODUCTION_DEPENDENT_TRIGGER_FAILED=true fi else PRODUCTION_DEPENDENT_TRIGGER_FAILED=true fi done echo "PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV" echo "PRODUCTION_DEPENDENT_TRIGGER_FAILED=$PRODUCTION_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV" - name: Wait for Coolify production dependent health env: COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }} COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" DEPLOYMENT_WAIT_PIDS="" for DEPLOYMENT_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent" & DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!" done for PID in $DEPLOYMENT_WAIT_PIDS; do wait "$PID" done RESOURCE_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) APP_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\ --header "Authorization: Bearer $COOLIFY_TOKEN" \\ "$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify production dependent resource #$RESOURCE_INDEX status: $STATUS" HEALTH_URL="https://production-api.example.invalid" HTTP_RESOURCE="$(node -e 'process.stdout.write("true")')" WORKER_RESOURCE="$(node -e 'process.stdout.write("false")')" if printf ",%s," "$PRODUCTION_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then WORKER_RESOURCE="$(node -e 'process.stdout.write("true")')" fi EXPECTED_WORKER_GROUP="\${7:-general}" if [ "$WORKER_RESOURCE" = "true" ]; then CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("main")')" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL worker image tag does not match expected alias" exit 1 fi if [ -z "$EXPECTED_WORKER_GROUP" ]; then exit 1 fi EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP" CONFIGURED_HEALTH_COMMAND="$EXPECTED_HEALTH_COMMAND" CONFIGURED_HEALTH_TYPE="cmd" HEALTH_CHECK_ENABLED="true" CUSTOM_HEALTHCHECK_FOUND="false" if [ "$HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_HEALTH_COMMAND" != "$EXPECTED_HEALTH_COMMAND" ] || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then echo "$LABEL worker healthcheck is not bound to the deployed revision" exit 1 fi echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy" return fi EXPECTED_MCP_HEALTH_COMMAND="node apps/mcp/dist/deploy-health.js $IMAGE_TAG" CONFIGURED_MCP_HEALTH_COMMAND="$EXPECTED_MCP_HEALTH_COMMAND" CONFIGURED_MCP_HEALTH_TYPE="cmd" MCP_HEALTH_CHECK_ENABLED="true" MCP_CUSTOM_HEALTHCHECK_FOUND="false" if [ "$MCP_HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_MCP_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_MCP_HEALTH_COMMAND" != "$EXPECTED_MCP_HEALTH_COMMAND" ] || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then exit 1 fi if [ -z "$HEALTH_URL" ] && [ "$HTTP_RESOURCE" = "true" ]; then echo "$LABEL is an HTTP resource but has no health URL" exit 1 fi EXPECTED_IMAGE_ALIAS="\${4:-main}" CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("main")')" if [ -z "$HEALTH_URL" ] && [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then exit 1 fi HEALTH_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON" HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" EXPECTED_SERVICE="\${5:-api}" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then exit 1 fi test "$STATUS" = "running:healthy" done check_dependent_resource() { RESOURCE_UUID="$1" RESOURCE_INDEX="$2" test -n "$RESOURCE_UUID" } RESOURCE_WAIT_PIDS="" for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" & RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!" done for PID in $RESOURCE_WAIT_PIDS; do wait "$PID" done if [ "\${PRODUCTION_DEPENDENT_TRIGGER_FAILED:-false}" = "true" ]; then echo "One or more Coolify production dependent deploy triggers failed" exit 1 fi - name: Mark Sentry deploy if: success() env: SENTRY_AUTH_TOKEN: \${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_ORG: \${{ secrets.SENTRY_ORG }} SENTRY_URL: \${{ secrets.SENTRY_URL }} run: | if [ -z "\${SENTRY_AUTH_TOKEN:-}" ] || [ -z "\${SENTRY_ORG:-}" ] || [ -z "\${SENTRY_URL:-}" ]; then echo "Sentry secrets not configured; skipping deploy marker" exit 0 fi if [ "$DEPLOY_ACTION" = "promote" ]; then SENTRY_ENVIRONMENT=production RELEASE_SHA="\${IMAGE_TAG}" else SENTRY_ENVIRONMENT=staging RELEASE_SHA="\${SHA}" fi for PROJECT in api worker web; do bash scripts/ci/sentry-release.sh \ --project "$PROJECT" \ --environment "$SENTRY_ENVIRONMENT" \ --release "$RELEASE_SHA" \ --sourcemaps none done `; const fixtures = []; const STAGING_ONE_SHOT_FIXTURE = ` - name: Run staging one-shot database migration env: MIGRATOR_DATABASE_URL: \${{ secrets.STAGING_MIGRATOR_DATABASE_URL }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then exit 0; fi if [ -z "\${MIGRATOR_DATABASE_URL:-}" ]; then exit 1; fi DIRECT_URL="$MIGRATOR_DATABASE_URL" \\ docker run --rm --pull=always \\ -e DIRECT_URL \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web:$IMAGE_TAG" migrate `; const PRODUCTION_ONE_SHOT_FIXTURE = ` - name: Run production one-shot database migration env: MIGRATOR_DATABASE_URL: \${{ secrets.MIGRATOR_DATABASE_URL }} run: | if [ "$DEPLOY_ACTION" != "promote" ]; then exit 0; fi if [ -z "\${MIGRATOR_DATABASE_URL:-}" ]; then exit 1; fi DIRECT_URL="$MIGRATOR_DATABASE_URL" \\ docker run --rm --pull=always \\ -e DIRECT_URL \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web:$IMAGE_TAG" migrate `; const STAGING_CONTROLLER_DRAIN_FIXTURE = ` - name: Stop and drain Coolify staging portal-domain-controller env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION="$(node scripts/coolify-portal-domain-controller.mjs snapshot-release \\ staging \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\ "$COOLIFY_STAGING_RESOURCE_UUIDS" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker")" echo "STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION=$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION" >> "$GITHUB_ENV" echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true" >> "$GITHUB_ENV" node scripts/coolify-portal-domain-controller.mjs stop-and-drain \\ staging \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\ "$COOLIFY_STAGING_RESOURCE_UUIDS" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" `; const STAGING_CONTROLLER_ACCEPTANCE_AND_RECOVERY_FIXTURE = ` - name: Trigger and verify Coolify staging portal-domain-controller env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-portal-domain-controller.mjs set-release \\ staging \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\ "$COOLIFY_STAGING_RESOURCE_UUIDS" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\ "$IMAGE_TAG" STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")" node scripts/coolify-deploy.mjs wait "$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID" "Coolify staging portal-domain-controller" node scripts/coolify-portal-domain-controller.mjs assert-release \\ staging \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\ "$COOLIFY_STAGING_RESOURCE_UUIDS" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\ "$IMAGE_TAG" \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID" echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true" >> "$GITHUB_ENV" - name: Restore previous Coolify staging portal-domain-controller release on rollout failure if: always() env: COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then exit 0; fi if [ "\${STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED:-false}" != "true" ]; then exit 0; fi if [ "\${STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED:-false}" = "true" ]; then exit 0; fi SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-portal-domain-controller.mjs set-release \\ staging \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\ "$COOLIFY_STAGING_RESOURCE_UUIDS" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION" RECOVERY_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")" node scripts/coolify-deploy.mjs wait "$RECOVERY_DEPLOYMENT_UUID" "Coolify staging portal-domain-controller recovery" node scripts/coolify-portal-domain-controller.mjs assert-release \\ staging \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\ "$COOLIFY_STAGING_RESOURCE_UUIDS" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\ "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\ "$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION" \\ "$RECOVERY_DEPLOYMENT_UUID" `; const EXACT_REVISION_GATES_FIXTURE = ` - name: Resolve live staging release run: | if [ "$DEPLOY_ACTION" != "promote" ]; then exit 0; fi STAGING_HEALTH_JSON="$(mktemp)" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\ https://staging-app.hivegrid.de/healthz > "$STAGING_HEALTH_JSON" STAGED_SHA="$(node -e 'const payload = { revision: "a".repeat(40) }; process.stdout.write(payload.revision || "");')" STAGING_SERVICE="$(node -e 'process.stdout.write("hivegrid-web")')" STAGING_SURFACE="$(node -e 'process.stdout.write("core")')" if [ "$STAGING_SERVICE" != "hivegrid-web" ] || [ "$STAGING_SURFACE" != "core" ]; then exit 1; fi if ! printf '%s' "$STAGED_SHA" | grep -Eq '^[0-9a-f]{40}$'; then exit 1; fi git fetch --no-tags origin main git cat-file -e "$STAGED_SHA^{commit}" git merge-base --is-ancestor "$STAGED_SHA" origin/main echo "IMAGE_TAG=$STAGED_SHA" >> "$GITHUB_ENV" - name: Verify checkout revision run: | git fetch --no-tags origin main git cat-file -e "$IMAGE_TAG^{commit}" git merge-base --is-ancestor "$IMAGE_TAG" origin/main git checkout --detach "$IMAGE_TAG" CHECKOUT_SHA="$(git rev-parse HEAD)" if [ "$CHECKOUT_SHA" != "$IMAGE_TAG" ]; then exit 1; fi if [ "$DEPLOY_ACTION" = "stage" ] && [ "$IMAGE_TAG" != "$SHA" ]; then exit 1; fi - name: Install exact-revision dependencies run: pnpm install --frozen-lockfile --prefer-offline - name: Verify exact-revision tenant policy contract run: pnpm run verify:tenant-policy-contract - name: Verify exact-revision RLS release readiness run: pnpm run verify:rls-release-readiness - name: Verify exact-revision live RLS isolation run: pnpm run verify:rls-live env: HIVEGRID_RLS_REQUIRE_MIGRATION_CHAIN_PARITY: "1" `; const RUNTIME_DB_ASSERTIONS_FIXTURE = ` node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web node scripts/coolify-deploy.mjs assert-runtime-db-env "$ADMIN_RESOURCE_UUID" hivegrid_app admin node scripts/coolify-deploy.mjs assert-runtime-db-env "$PORTAL_RESOURCE_UUID" hivegrid_portal portal node scripts/coolify-deploy.mjs assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app service node scripts/coolify-deploy.mjs assert-runtime-db-env "$MCP_RESOURCE_UUID" hivegrid_app service node scripts/coolify-deploy.mjs assert-runtime-db-env "$HOCUSPOCUS_RESOURCE_UUID" hivegrid_app service for WORKER_RESOURCE_UUID in "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do node scripts/coolify-deploy.mjs assert-runtime-db-env "$WORKER_RESOURCE_UUID" hivegrid_worker service done node scripts/coolify-deploy.mjs assert-web-push-env "$WEB_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')" if ! printf '%s' "$WEB_IMAGE_DIGEST" | grep -Eq '^sha256:[0-9a-f]{64}$'; then exit 1; fi node scripts/coolify-deploy.mjs set-web-surface-release \\ "$WEB_RESOURCE_UUID" \\ "$ADMIN_RESOURCE_UUID" \\ "$PORTAL_RESOURCE_UUID" \\ "$WEB_IMAGE" \\ "$IMAGE_TAG" \\ "$WEB_IMAGE_DIGEST" echo "STAGING_ADMIN_RESOURCE_UUID=$ADMIN_RESOURCE_UUID" echo "STAGING_PORTAL_RESOURCE_UUID=$PORTAL_RESOURCE_UUID" echo "STAGING_WEB_IMAGE_DIGEST=$WEB_IMAGE_DIGEST" echo "PRODUCTION_ADMIN_RESOURCE_UUID=$ADMIN_RESOURCE_UUID" echo "PRODUCTION_PORTAL_RESOURCE_UUID=$PORTAL_RESOURCE_UUID" echo "PRODUCTION_WEB_IMAGE_DIGEST=$WEB_IMAGE_DIGEST"`; const VALID_WORKFLOW = LEGACY_WORKFLOW_FIXTURE.replace( / - name: Verify checkout revision[\s\S]*? - name: Login to Gitea Registry/, () => `${EXACT_REVISION_GATES_FIXTURE} - name: Login to Gitea Registry` ) .replaceAll("for TARGET in web api worker mcp; do", "for TARGET in web api worker mcp hocuspocus portal-domain-controller; do") .replace( ' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n WEB_BUILD_ARGS=()', ' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n DOCKERFILE="Dockerfile"\n if [ "$TARGET" = "hocuspocus" ]; then\n DOCKERFILE="apps/hocuspocus/Dockerfile"\n fi\n WEB_BUILD_ARGS=()' ) .replace("--file Dockerfile", '--file "$DOCKERFILE"') .replaceAll( '"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"', '"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"' ) .replaceAll( ' MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"', ' MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"\n HOCUSPOCUS_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"' ) .replaceAll( ' MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e \'process.stdout.write("mcp")\')"', ' MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e \'process.stdout.write("mcp")\')"\n HOCUSPOCUS_RESOURCE_UUID="$(HOCUSPOCUS_IMAGE="$HOCUSPOCUS_IMAGE" node -e \'process.stdout.write("hocuspocus")\')"' ) .replaceAll('"$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID"', '"$MCP_RESOURCE_UUID" "$HOCUSPOCUS_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID"') .replaceAll( 'DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"', 'DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"' ) .replaceAll( ' DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"', () => `${RUNTIME_DB_ASSERTIONS_FIXTURE}\n DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"` ) .replace( 'EXPECTED_IMAGE_ALIAS="\${4:-staging}"', 'EXPECTED_IMAGE_ALIAS="\${4:-staging}"\n HOCUSPOCUS_RESOURCE_UUID="$STAGING_HOCUSPOCUS_RESOURCE_UUID"\n RESOURCE_KIND="hocuspocus"\n echo "$LABEL Hocuspocus Docker health and image revision are healthy"\n RESOURCE_KIND="\${6:-auto}"' ) .replace( 'EXPECTED_IMAGE_ALIAS="\${4:-main}"', 'EXPECTED_IMAGE_ALIAS="\${4:-main}"\n HOCUSPOCUS_RESOURCE_UUID="$PRODUCTION_HOCUSPOCUS_RESOURCE_UUID"\n RESOURCE_KIND="hocuspocus"\n echo "$LABEL Hocuspocus Docker health and image revision are healthy"\n RESOURCE_KIND="\${6:-auto}"' ) .replace(" - name: Ensure Coolify staging worker groups\n", `${STAGING_ONE_SHOT_FIXTURE} - name: Ensure Coolify staging worker groups\n`) .replace("Configure Coolify staging migration boot flags", "Configure Coolify staging runtime resources") .replace( / node scripts\/coolify-deploy\.mjs set-envs "\$WEB_RESOURCE_UUID" \\\n MIGRATE_ON_BOOT=true \\\n SEED_ON_BOOT=true \\\n SEED_DEMO_DATA=true \\\n ALLOW_STAGING_DEMO_SEED=true \\\n SEED_DEMO_PRO_MANAGED=true/, "" ) .replace(" - name: Ensure Coolify production worker groups\n", `${PRODUCTION_ONE_SHOT_FIXTURE} - name: Ensure Coolify production worker groups\n`) .replace("Configure Coolify production migration boot flags", "Configure Coolify production runtime resources") .replace( / node scripts\/coolify-deploy\.mjs set-envs "\$WEB_RESOURCE_UUID" \\\n MIGRATE_ON_BOOT=true \\\n SEED_ON_BOOT=true/, "" ) .replace("DATABASE_URL: \${{ secrets.DATABASE_URL }}", "DATABASE_URL: \${{ secrets.MIGRATOR_DATABASE_URL }}") .replace( ' SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_STAGING_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"', ' SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"\n SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"\n node scripts/coolify-deploy.mjs assert-runtime-preflight \\\n "$COOLIFY_STAGING_RESOURCE_UUIDS" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"' ) .replace( ' SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"', ' SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"\n SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"\n node scripts/coolify-deploy.mjs assert-runtime-preflight \\\n "$COOLIFY_RESOURCE_UUIDS" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"' ) .replace(/^ WEB_RESOURCE_UUID=.*$/gm, (line) => { const allowlist = line.includes("COOLIFY_STAGING_RESOURCE_UUIDS") ? "$COOLIFY_STAGING_RESOURCE_UUIDS" : "$COOLIFY_RESOURCE_UUIDS"; return ` WEB_SURFACE_RESOURCE_UUIDS="$(node scripts/coolify-deploy.mjs resolve-web-surfaces "${allowlist}" "$WEB_IMAGE")" WEB_RESOURCE_UUID="\${WEB_SURFACE_RESOURCE_UUIDS%%,*}" REMAINING_WEB_SURFACE_RESOURCE_UUIDS="\${WEB_SURFACE_RESOURCE_UUIDS#*,}" ADMIN_RESOURCE_UUID="\${REMAINING_WEB_SURFACE_RESOURCE_UUIDS%%,*}" PORTAL_RESOURCE_UUID="\${REMAINING_WEB_SURFACE_RESOURCE_UUIDS#*,}"`; }) .replaceAll( '"$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID"', '"$WEB_RESOURCE_UUID" "$ADMIN_RESOURCE_UUID" "$PORTAL_RESOURCE_UUID" "$API_RESOURCE_UUID"' ) .replaceAll( ' node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"', ' node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"\n node scripts/coolify-deploy.mjs assert-no-app-revision-override "$ADMIN_RESOURCE_UUID"\n node scripts/coolify-deploy.mjs assert-no-app-revision-override "$PORTAL_RESOURCE_UUID"' ) .replace( ' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"', ' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging hivegrid-web"\n STAGING_ADMIN_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_ADMIN_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$STAGING_ADMIN_DEPLOYMENT_UUID" "Coolify staging hivegrid-admin"\n STAGING_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$STAGING_PORTAL_DEPLOYMENT_UUID" "Coolify staging hivegrid-portal"' ) .replace( ' PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"', ' PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production hivegrid-web"\n PRODUCTION_ADMIN_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_ADMIN_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$PRODUCTION_ADMIN_DEPLOYMENT_UUID" "Coolify production hivegrid-admin"\n PRODUCTION_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$PRODUCTION_PORTAL_DEPLOYMENT_UUID" "Coolify production hivegrid-portal"' ) .replace( ' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web"', ' if [ -z "\${STAGING_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_ADMIN_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_PORTAL_DEPLOYMENT_UUID:-}" ]; then echo "web surface deployment UUIDs are missing"; exit 1; fi' ) .replace( ' node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production web"', ' if [ -z "\${PRODUCTION_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "\${PRODUCTION_ADMIN_DEPLOYMENT_UUID:-}" ] || [ -z "\${PRODUCTION_PORTAL_DEPLOYMENT_UUID:-}" ]; then echo "web surface deployment UUIDs are missing"; exit 1; fi' ) .replace( " for RESOURCE_UUID in $(printf \"%s\" \"$STAGING_WEB_RESOURCE_UUID\" | tr ',' ' '); do", ' ROLLED_OUT_WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format \'{{json .Manifest.Digest}}\' | tr -d \'"\')"\n if [ "$ROLLED_OUT_WEB_IMAGE_DIGEST" != "$STAGING_WEB_IMAGE_DIGEST" ]; then exit 1; fi\n node scripts/coolify-deploy.mjs assert-web-surface-release \\\n "$STAGING_WEB_RESOURCE_UUID" \\\n "$STAGING_ADMIN_RESOURCE_UUID" \\\n "$STAGING_PORTAL_RESOURCE_UUID" \\\n "$WEB_IMAGE" \\\n "$IMAGE_TAG" \\\n "$ROLLED_OUT_WEB_IMAGE_DIGEST" \\\n "$STAGING_WEB_DEPLOYMENT_UUID" \\\n "$STAGING_ADMIN_DEPLOYMENT_UUID" \\\n "$STAGING_PORTAL_DEPLOYMENT_UUID"\n for SURFACE in core admin portal; do\n EXPECTED_SERVICE="hivegrid-web"\n EXPECTED_SERVICE="hivegrid-admin"\n EXPECTED_SERVICE="hivegrid-portal"\n RESOURCE_UUID="$STAGING_WEB_RESOURCE_UUID"\n verify_live_revision "$APP_JSON" "Coolify staging resource" required "$IMAGE_TAG" "$EXPECTED_SERVICE" "$SURFACE"' ) .replace( " for RESOURCE_UUID in $(printf \"%s\" \"$PRODUCTION_WEB_RESOURCE_UUID\" | tr ',' ' '); do", ' ROLLED_OUT_WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format \'{{json .Manifest.Digest}}\' | tr -d \'"\')"\n if [ "$ROLLED_OUT_WEB_IMAGE_DIGEST" != "$PRODUCTION_WEB_IMAGE_DIGEST" ]; then exit 1; fi\n node scripts/coolify-deploy.mjs assert-web-surface-release \\\n "$PRODUCTION_WEB_RESOURCE_UUID" \\\n "$PRODUCTION_ADMIN_RESOURCE_UUID" \\\n "$PRODUCTION_PORTAL_RESOURCE_UUID" \\\n "$WEB_IMAGE" \\\n "$IMAGE_TAG" \\\n "$ROLLED_OUT_WEB_IMAGE_DIGEST" \\\n "$PRODUCTION_WEB_DEPLOYMENT_UUID" \\\n "$PRODUCTION_ADMIN_DEPLOYMENT_UUID" \\\n "$PRODUCTION_PORTAL_DEPLOYMENT_UUID"\n for SURFACE in core admin portal; do\n EXPECTED_SERVICE="hivegrid-web"\n EXPECTED_SERVICE="hivegrid-admin"\n EXPECTED_SERVICE="hivegrid-portal"\n RESOURCE_UUID="$PRODUCTION_WEB_RESOURCE_UUID"\n verify_live_revision "$APP_JSON" "Coolify production resource" required "$IMAGE_TAG" "$EXPECTED_SERVICE" "$SURFACE"' ) .replaceAll( ' HEALTH_SERVICE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");\' "$HEALTH_JSON")"', ' HEALTH_SERVICE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");\' "$HEALTH_JSON")"\n EXPECTED_SURFACE="${6:-}"\n HEALTH_SURFACE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");\' "$HEALTH_JSON")"\n if [ -n "$EXPECTED_SURFACE" ] && [ "$HEALTH_SURFACE" != "$EXPECTED_SURFACE" ]; then exit 1; fi\n READINESS_JSON="$(mktemp)"\n if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/readyz" > "$READINESS_JSON"; then exit 1; fi\n READINESS_STATUS="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");\' "$READINESS_JSON")"\n READINESS_SERVICE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");\' "$READINESS_JSON")"\n READINESS_SURFACE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");\' "$READINESS_JSON")"\n READINESS_REVISION="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");\' "$READINESS_JSON")"\n if [ "$READINESS_STATUS" != "ready" ]; then exit 1; fi\n if [ "$READINESS_REVISION" != "$IMAGE_TAG" ]; then exit 1; fi\n if [ -n "$EXPECTED_SERVICE" ] && [ "$READINESS_SERVICE" != "$EXPECTED_SERVICE" ]; then exit 1; fi\n if [ -n "$EXPECTED_SURFACE" ] && [ "$READINESS_SURFACE" != "$EXPECTED_SURFACE" ]; then exit 1; fi\n EXPECTED_WEB_DIGEST_TAG="sha256-${STAGING_WEB_IMAGE_DIGEST#sha256:}"\n EXPECTED_WEB_DIGEST_TAG="sha256-${PRODUCTION_WEB_IMAGE_DIGEST#sha256:}"\n CONFIGURED_IMAGE_TAG="$EXPECTED_WEB_DIGEST_TAG"\n if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_WEB_DIGEST_TAG" ]; then exit 1; fi' ) .replace(" - name: Trigger Coolify staging deploy\n", `${STAGING_CONTROLLER_DRAIN_FIXTURE} - name: Trigger Coolify staging deploy\n`) .replace(" - name: Promote staging images to production aliases\n", `${STAGING_CONTROLLER_ACCEPTANCE_AND_RECOVERY_FIXTURE} - name: Promote staging images to production aliases\n`); const VALID_COSIGN_INSTALLER = `#!/usr/bin/env bash set -euo pipefail COSIGN_VERSION="\${COSIGN_VERSION:-v3.0.6}" COSIGN_SHA256="\${COSIGN_SHA256:-c956e5dfcac53d52bcf058360d579472f0c1d2d9b69f55209e256fe7783f4c74}" curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --location "https://github.com/sigstore/cosign/releases/download/\${COSIGN_VERSION}/cosign-linux-amd64" --output /tmp/cosign printf '%s %s\\n' "$COSIGN_SHA256" /tmp/cosign | sha256sum --check --status install -m 0755 /tmp/cosign /usr/local/bin/cosign `; const VALID_COOLIFY_DEPLOY_HELPER = `const apiTimeoutMs = 20000; const delayMs = Number(process.env.COOLIFY_DEPLOYMENT_POLL_MS || "2000"); const attempts = Number(process.env.COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS || "225"); const COOLIFY_CUID_PATTERN = /^[a-z][a-z0-9]{23}$/; const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/; const OCI_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; const WORKER_GROUP_NAMES = Object.freeze(["general", "mail", "import", "webhook", "plugin"]); const CORE_PUBLIC_HOST_ENV_KEYS = Object.freeze(["APP_BASE_DOMAIN", "WEB_APP_HOST", "ADMIN_APP_HOST", "API_HOST"]); const WEB_SURFACE_IDENTITIES = { core: "hivegrid-web", admin: "hivegrid-admin", portal: "hivegrid-portal" }; async function coolify() { try { return await fetch("https://example.invalid", { signal: AbortSignal.timeout(apiTimeoutMs) }); } catch { throw new Error("Coolify API request failed or timed out"); } } async function readApplicationEnvs(resourceUuid) { return coolify("/applications/" + encodeURIComponent(resourceUuid) + "/envs"); } function envKey(entry) { return entry?.key || ""; } function envValue(entry) { return entry?.value ?? entry?.real_value ?? ""; } function envEntries(payload) { if (Array.isArray(payload)) return payload; if (Array.isArray(payload?.data)) return payload.data; throw new Error("Coolify environment response was malformed"); } function normalizeCoolifyBoolean(value) { return value === true ? "true" : value === false ? "false" : ""; } function runtimeEnvEntries(payload) { const runtime = []; const keys = new Set(); for (const entry of envEntries(payload)) { if (normalizeCoolifyBoolean(entry?.is_preview) === "true") continue; if (keys.has(entry.key)) throw new Error("Coolify runtime environment response contained duplicate non-preview keys"); keys.add(entry.key); runtime.push(entry); } return runtime; } function applicationEntries(payload) { if (Array.isArray(payload)) return payload; if (Array.isArray(payload?.data)) return payload.data; throw new Error("Coolify applications response was malformed"); } function selectWebSurfacePair(apps, allowed) { const allowedWebImageApps = apps.filter((app) => allowed.has(app.uuid)); const coreMatches = allowedWebImageApps.filter((app) => app.name === WEB_SURFACE_IDENTITIES.core); const adminMatches = allowedWebImageApps.filter((app) => app.name === WEB_SURFACE_IDENTITIES.admin); const portalMatches = allowedWebImageApps.filter((app) => app.name === WEB_SURFACE_IDENTITIES.portal); if (allowedWebImageApps.length !== 3 || coreMatches.length !== 1 || adminMatches.length !== 1 || portalMatches.length !== 1) throw new Error("invalid trio"); const core = coreMatches[0]; const admin = adminMatches[0]; const portal = portalMatches[0]; if (new Set([core.uuid, admin.uuid, portal.uuid]).size !== 3) throw new Error("duplicate trio"); return { core, admin, portal }; } function selectUniqueAllowedImageApplication(apps, allowed, expectedImage, label) { const matches = apps.filter((app) => allowed.has(app.uuid) && app.docker_registry_image_name === expectedImage); if (matches.length !== 1) throw new Error(label + " runtime missing"); return matches[0]; } function assertRuntimeDatabaseEntries(entries, expectedRole, surface) { const values = new Map(entries.map((entry) => [envKey(entry), envValue(entry)])); if (values.has("APP_REVISION")) throw new Error("runtime APP_REVISION overrides must be removed before migration"); if (surface === "web" && values.has("PORTAL_SESSION_SECRET")) throw new Error("hivegrid-web runtime must not contain PORTAL_SESSION_SECRET"); if (!values.get("DATABASE_POOL_MAX")) throw new Error("explicit positive DATABASE_POOL_MAX"); if (!values.get("DATABASE_URL")?.includes(":6432/")) throw new Error("PgBouncer port 6432"); return values; } function assertWebPushEntries() {} async function assertRuntimePreflight(allowedCsv, webImage, apiImage, mcpImage, hocuspocusImage, workerImage) { const allowed = new Set(allowedCsv.split(",").filter(Boolean)); const apps = applicationEntries(await coolify("/applications")); const webPair = selectWebSurfacePair(apps, allowed, webImage); const api = selectUniqueAllowedImageApplication(apps, allowed, apiImage, "API"); const mcp = selectUniqueAllowedImageApplication(apps, allowed, mcpImage, "MCP"); const hocuspocus = selectUniqueAllowedImageApplication(apps, allowed, hocuspocusImage, "Hocuspocus"); const workers = WORKER_GROUP_NAMES.map((group) => ({ group, key: "WORKER_QUEUE_GROUP", image: workerImage })); const runtimeResources = [webPair.core, webPair.admin, webPair.portal, api, mcp, hocuspocus, ...workers].map((app) => ({ app, entries: [], role: "hivegrid_app", surface: "service", })); if (new Set(runtimeResources.map(({ app }) => app.uuid)).size !== runtimeResources.length) throw new Error("ten distinct resources"); for (const resource of runtimeResources) assertRuntimeDatabaseEntries(resource.entries, resource.role, resource.surface); if (false) throw new Error("same normalized PORTAL_APP_HOST"); if (false) throw new Error("hivegrid-portal and worker-mail must set APP_PROTOCOL=https"); CORE_PUBLIC_HOST_ENV_KEYS.map((key) => key); if (false) throw new Error("PORTAL_APP_HOST must be distinct from core APP_BASE_DOMAIN, WEB_APP_HOST, ADMIN_APP_HOST, and API_HOST"); const coreRuntime = runtimeResources[0]; const webhookWorker = workers[3]; assertWebPushEntries(coreRuntime?.entries || [], webhookWorker?.entries || []); } async function restoreWebSurfaceReleases() {} function coolifyDigestTag(digest) { return digest.replace(":", "-"); } function webSurfaceReleasePayload(digest) { return { pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: coolifyDigestTag(digest) }; } async function setWebSurfaceRelease(coreResourceUuid, adminResourceUuid, portalResourceUuid, expectedImage, revision, digest) { if (!FULL_GIT_SHA_PATTERN.test(revision)) throw new Error("invalid revision"); if (!OCI_DIGEST_PATTERN.test(digest)) throw new Error("invalid digest"); const mutations = [{ uuid: coreResourceUuid }, { uuid: adminResourceUuid }, { uuid: portalResourceUuid }]; const patched = []; for (const resource of mutations) patched.push(resource); const updated = []; const digestTag = coolifyDigestTag(digest); if (updated.some((app) => normalizeCoolifyValue(app?.docker_registry_image_tag) !== digestTag)) throw new Error("drift"); await restoreWebSurfaceReleases(patched); } function assertCompletedWebSurfaceDeployment(payload) { return payload.deployment_uuid && payload.application_name && payload.application_id; } async function assertWebSurfaceRelease() { const applications = []; const deployments = []; const index = 0; const digestTag = "sha256-digest"; assertWebSurfacePairByUuid(applications[0], applications[1], applications[2]); if (normalizeCoolifyValue(applications[0]?.docker_registry_image_tag) !== digestTag) throw new Error("digest drift"); if (normalizeCoolifyValue(applications[0]?.status) !== "running:healthy") throw new Error("unhealthy"); assertCompletedWebSurfaceDeployment(deployments[index], "deployment", applications[index], "hivegrid-web"); } function isPortalForbiddenEnvKey(key) { return key === "SESSION_SECRET" || /AUTHENTIK_|OAUTH|BILLING|LAGO|MOLLIE|STRIPE|COOLIFY|SSH|TRAEFIK|INGRESS|CLOUDFLARE|PORTAL_SERVER_|DOMAIN_CONTROLLER/.test(key); } function isAllowedCaptchaInstanceUrl(value) { return value.startsWith("https://"); } function isValidRedisConnectionUrl(value) { return value.startsWith("redis://") || value.startsWith("rediss://"); } function assertPortalAuthDependencies(values) { const provider = values.get("CAPTCHA_PROVIDER"); if (provider === "cap") { isAllowedCaptchaInstanceUrl(values.get("CAP_INSTANCE_URL")); values.get("CAP_SITEKEY_LOGIN"); values.get("CAP_SITE_SECRET_LOGIN") || values.get("CAP_SITE_SECRET"); } else { isAllowedCaptchaInstanceUrl(values.get("MCAPTCHA_INSTANCE_URL")); values.get("MCAPTCHA_SITEKEY_LOGIN"); values.get("MCAPTCHA_ACCOUNT_SECRET"); } isValidRedisConnectionUrl(values.get("REDIS_URL")); isValidRedisConnectionUrl(values.get("BULLMQ_REDIS_URL")); } async function assertPortalRuntime(values) { if (values.get("WEB_RUNTIME_SURFACE") !== "portal") throw new Error("surface"); if (!values.get("PORTAL_SESSION_SECRET")) throw new Error("session"); assertPortalAuthDependencies(values); return isPortalForbiddenEnvKey("SESSION_SECRET"); } async function validateRequiredImages(allowedCsv, expectedImages) { const allowed = new Set(allowedCsv.split(",").filter(Boolean)); const apps = applicationEntries(await coolify("/applications")); const requiredApps = []; for (const expectedImage of expectedImages) { const matches = apps.filter((app) => allowed.has(app?.uuid) && app?.docker_registry_image_name === expectedImage); if (matches.length !== 1) throw new Error("invalid inventory"); requiredApps.push(matches[0]); } const requiredEnvs = await Promise.all( requiredApps.map((app) => readApplicationEnvs(app.uuid).then(runtimeEnvEntries)) ); if (requiredEnvs.some((envs) => envs.some((entry) => envKey(entry) === "APP_REVISION"))) throw new Error("revision override"); } async function assertNoAppRevisionOverride(resourceUuid) { const envs = runtimeEnvEntries(await coolify("/applications/" + encodeURIComponent(resourceUuid) + "/envs")); if (envs.some((entry) => envKey(entry) === "APP_REVISION")) throw new Error("Coolify APP_REVISION override must be removed before deployment"); } async function setEnvs(resourceUuid) { const existingPayload = await coolify("/applications/" + encodeURIComponent(resourceUuid) + "/envs"); const existing = runtimeEnvEntries(existingPayload); return existing; } function requireCoolifyCuid(value) { if (!COOLIFY_CUID_PATTERN.test(value)) throw new Error("invalid Coolify CUID"); } function deploymentUuid(payload, resourceUuid = "") { if (!Array.isArray(payload.deployments)) return ""; const queuedDeployment = resourceUuid ? payload.deployments.find((deployment) => deployment?.resource_uuid === resourceUuid) : payload.deployments[0]; const value = queuedDeployment?.deployment_uuid || ""; return COOLIFY_CUID_PATTERN.test(value) ? value : ""; } async function trigger(resourceUuid) { requireCoolifyCuid(resourceUuid, "Coolify resource UUID"); const payload = await coolify(\`/deploy?uuid=\${encodeURIComponent(resourceUuid)}&force=true\`, { method: "POST" }); return deploymentUuid(payload, resourceUuid); } function desiredMcpHealthConfig(expectedRevision) { return { docker_registry_image_tag: expectedRevision, health_check_enabled: true, health_check_type: "cmd", health_check_command: \`node apps/mcp/dist/deploy-health.js \${expectedRevision}\`, }; } function mcpHealthConfigDrift() { return []; } function expectedMcpImageName() { return "registry.example.test/toolctrl-labs/hivegrid/mcp"; } function expectedWorkerImageName() { return "registry.example.test/toolctrl-labs/hivegrid/worker"; } function isExpectedMcpImage(app) { const actual = app?.docker_registry_image_name || ""; const expected = expectedMcpImageName(); return Boolean(expected) && actual === expected; } async function configureMcpHealthcheck(resourceUuid, expectedRevision) { const envPayload = await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}/envs\`); const envs = runtimeEnvEntries(envPayload); if (envs.some((entry) => envKey(entry) === "APP_REVISION" && envValue(entry).trim())) throw new Error("APP_REVISION override"); const current = await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}\`); if (!isExpectedMcpImage(current)) throw new Error("wrong MCP image"); if (current.custom_healthcheck_found !== false) throw new Error("image healthcheck must be absent"); await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}\`, { method: "PATCH", body: JSON.stringify(desiredMcpHealthConfig(expectedRevision)), }); const updated = await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}\`); if (mcpHealthConfigDrift(updated, expectedRevision).length > 0) throw new Error("healthcheck drift"); } function legacyRollbackResourceKind(app) { const image = app?.docker_registry_image_name || ""; const expectedWorkerImage = expectedWorkerImageName(); const expectedMcpImage = expectedMcpImageName(); if (image === expectedWorkerImage) return "worker"; if (image === expectedMcpImage) return "mcp"; return ""; } function legacyRollbackHealthConfigDrift() { return []; } async function configureLegacyRollbackHealthcheck(resourceUuid, expectedRevision, expectedWorkerGroup = "") { const resourcePath = \`/applications/\${encodeURIComponent(resourceUuid)}\`; const envPayload = await coolify(\`\${resourcePath}/envs\`); const envs = runtimeEnvEntries(envPayload); if (envs.some((entry) => envKey(entry) === "APP_REVISION")) throw new Error("APP_REVISION override"); const current = await coolify(resourcePath); const resourceKind = legacyRollbackResourceKind(current); if (resourceKind === "worker" && !expectedWorkerGroup) throw new Error("worker group required"); await coolify(resourcePath, { method: "PATCH", body: JSON.stringify({ docker_registry_image_tag: expectedRevision, health_check_enabled: false, health_check_command: "" }), }); const updated = await coolify(resourcePath); if (legacyRollbackHealthConfigDrift(updated, expectedRevision).length > 0) throw new Error("legacy rollback drift"); } async function waitForDeployment(uuid) { requireCoolifyCuid(uuid, "Coolify deployment UUID"); } `; function writeFile(rootDir, relativePath, content) { const absolutePath = path.join(rootDir, relativePath); fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); fs.writeFileSync(absolutePath, content); } function createFixture(workflowContent = VALID_WORKFLOW, helperContent = VALID_COOLIFY_DEPLOY_HELPER) { const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "gitea-deploy-workflow-")); fixtures.push(rootDir); if (workflowContent !== null) { writeFile(rootDir, ".gitea/workflows/deploy.yml", workflowContent); writeFile( rootDir, "scripts/ci/deploy-buildkitd.toml", '[history]\n maxAge = 86400\n maxEntries = 20\n\n[worker.oci]\n gc = true\n reservedSpace = "2GB"\n maxUsedSpace = "30GB"\n minFreeSpace = "100GB"\n max-parallelism = 12\n' ); writeFile( rootDir, "scripts/ci/verify-deploy-buildkit-config.mjs", fs.readFileSync(path.join(process.cwd(), "scripts/ci/verify-deploy-buildkit-config.mjs"), "utf8") ); writeFile(rootDir, "scripts/ci/install-cosign.sh", VALID_COSIGN_INSTALLER); writeFile(rootDir, "scripts/coolify-deploy.mjs", helperContent); writeFile( rootDir, "start.sh", "node /myapp/scripts/verify-migrator-role.mjs\nnode /myapp/scripts/adopt-prisma-baseline.mjs --adopt\nnode /myapp/node_modules/prisma/build/index.js migrate deploy --config prisma.config.ts\nnode /myapp/scripts/adopt-prisma-baseline.mjs --verify-current-schema\nnode /myapp/packages/db/dist/deploy-permission-catalog-sync.server.js\n" ); writeFile( rootDir, "scripts/verify-migrator-role.mjs", "if (session.sessionRole !== REQUIRED_MIGRATOR_ROLE) throw new Error();\nif (role.isSuperuser !== false) throw new Error();\nif (role.bypassesRls !== false) throw new Error();\nconst grantedRole = 'hivegrid_rls_definer';\n" ); writeFile(rootDir, "scripts/adopt-prisma-baseline.mjs", "export async function adoptPrismaBaseline() {}\n"); writeFile( rootDir, "Dockerfile", "COPY --from=build /myapp/scripts/verify-migrator-role.mjs /myapp/scripts/verify-migrator-role.mjs\nCOPY --from=build /myapp/scripts/adopt-prisma-baseline.mjs /myapp/scripts/adopt-prisma-baseline.mjs\n" ); } return rootDir; } afterEach(() => { while (fixtures.length > 0) { fs.rmSync(fixtures.pop(), { recursive: true, force: true }); } }); describe("verify-gitea-deploy-workflow", () => { it("passes for the stage-and-promote Gitea Registry workflow", () => { const result = verifyGiteaDeployWorkflow(createFixture()); const output = formatGiteaDeployWorkflowResult(result); assert.deepEqual(result.failures, []); assert.ok(result.passes.some((pass) => /runner is "toolctrl-deploy"/.test(pass))); assert.ok(result.passes.some((pass) => /pinned docker-container BuildKit/.test(pass))); assert.ok(result.passes.some((pass) => /history, cache growth, and BuildKit parallelism are bounded/.test(pass))); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: deploy has no second runner acquisition for a separate preflight job")); assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy job "stage-or-promote" retains preflight and rollout on one runner')); assert.ok( result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-gitea-deploy-workflow.mjs"') ); assert.ok( result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-deploy-readiness.mjs"') ); assert.ok( result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-docker-baseline.mjs"') ); assert.ok( result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-docker-non-root.mjs"') ); assert.ok( result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node --check scripts/coolify-deploy.mjs"') ); assert.ok( result.passes.some( (pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node --check scripts/coolify-ensure-worker-groups.mjs"' ) ); assert.ok( result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "bash -n scripts/coolify-prepull-images.sh"') ); assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "bash -n scripts/ci/install-cosign.sh"')); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: cosign install uses scripts/ci/install-cosign.sh")); assert.ok(result.passes.some((pass) => pass === "scripts/ci/install-cosign.sh: cosign download is verified by SHA256")); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: retained deploy job does not expose production database credentials")); assert.ok( result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: deploy preflight stays lightweight and leaves full CI gates to the CI workflow") ); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: registry login uses REGISTRY_USER secret")); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: registry login uses REGISTRY_PAT secret")); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: main-branch push trigger is defined for staging deployment")); assert.ok( result.passes.some( (pass) => pass === ".gitea/workflows/deploy.yml: workflow_dispatch is an input-free production promotion while main pushes stage" ) ); assert.ok( result.passes.some( (pass) => pass === ".gitea/workflows/deploy.yml: promote reuses the exact-revision checkout and install for the post-web migration status gate without a second cache or install" ) ); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: deployment concurrency serializes staging and production promotion")); assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: builds Docker target "web"')); assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: pushes Docker target "worker"')); assert.ok( result.passes.some( (pass) => pass === ".gitea/workflows/deploy.yml: image paths match existing Coolify hivegrid/{web|api|worker|mcp|hocuspocus} resources" ) ); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: full SHA, staging, and promoted main tag generation is present")); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: staging aliases are not pushed from individual target builds")); assert.ok( result.passes.some( (pass) => pass === ".gitea/workflows/deploy.yml: one-click promote resolves the full live staging SHA with audited backup-readiness intent" ) ); assert.ok( result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: promote verifies staged commit ancestry and staging image digests before retagging") ); assert.ok(result.passes.some((pass) => /staging migration is a fail-closed DIRECT_URL-only one-shot container/.test(pass))); assert.ok(result.passes.some((pass) => /production migration is a fail-closed DIRECT_URL-only one-shot container/.test(pass))); assert.ok( result.passes.some((pass) => /Coolify staging runtime config pairs web surfaces, clears dead hooks, and enforces role-specific RLS credentials/.test(pass) ) ); assert.ok(result.passes.some((pass) => /leaves the operator-only SEED_DEMO_USER_PASSWORD untouched/.test(pass))); assert.ok(result.passes.some((pass) => /Docker image publication passes APP_REVISION build arg/.test(pass))); assert.ok(result.passes.some((pass) => /do not query nonexistent legacy build-cache refs/.test(pass))); assert.ok(result.passes.some((pass) => /embed portable inline cache in the immutable SHA image/.test(pass))); assert.ok(result.passes.some((pass) => /do not use a separate failure-prone registry cache export/.test(pass))); assert.ok(result.passes.some((pass) => /staging Docker image builds run in parallel and collect per-target failures/.test(pass))); assert.ok(result.passes.some((pass) => /Vite build args are scoped to the web Docker target/.test(pass))); assert.ok(result.passes.some((pass) => /public Hocuspocus WebSocket URL is passed from a Gitea variable/.test(pass))); assert.ok( result.passes.some((pass) => /Coolify production runtime config pairs web surfaces, clears dead hooks, and enforces role-specific RLS credentials/.test(pass) ) ); assert.ok(result.passes.some((pass) => /production Sentry deploy marker uses promoted IMAGE_TAG/.test(pass))); assert.ok(result.passes.some((pass) => /deployment waits poll every 2 seconds while preserving the 450-second failure budget/.test(pass))); assert.ok(result.passes.some((pass) => /Sentry deploy marker covers staging SHA and all runtime projects/.test(pass))); assert.ok(result.passes.some((pass) => /Coolify staging image-tag patch covers API, MCP, Hocuspocus, and every worker group/.test(pass))); assert.ok(result.passes.some((pass) => /Coolify production image-tag patch covers API, MCP, Hocuspocus, and every worker group/.test(pass))); assert.ok(result.passes.some((pass) => /staging read-only eleven-resource runtime preflight runs before migration/.test(pass))); assert.ok(result.passes.some((pass) => /production read-only eleven-resource runtime preflight runs before migration/.test(pass))); assert.ok(result.passes.some((pass) => /staging one-shot migration runs before Coolify deploy/.test(pass))); assert.ok(result.passes.some((pass) => /production one-shot migration runs before Coolify mutation and deploy/.test(pass))); assert.ok(result.passes.some((pass) => /Coolify staging health wait polls application status/.test(pass))); assert.ok( result.passes.some( (pass) => pass === ".gitea/workflows/deploy.yml: promote preserves the verified full-SHA manifest digest in production aliases without rebuilding" ) ); assert.match(output, /^PASS Gitea deploy workflow verification/); assert.match(output, /Summary: \d+ passed, 0 observed, 0 failed/); }); it("fails when the exact-revision live RLS gate permits a feature-branch overlay", () => { const workflow = VALID_WORKFLOW.replace(' env:\n HIVEGRID_RLS_REQUIRE_MIGRATION_CHAIN_PARITY: "1"\n', ""); const result = verifyGiteaDeployWorkflow(createFixture(workflow)); assert.match(result.failures.join("\n"), /exact-revision live RLS gate must set HIVEGRID_RLS_REQUIRE_MIGRATION_CHAIN_PARITY: "1"/); }); it("fails when a web surface is selected through the first image match", () => { const imageFirstWorkflow = VALID_WORKFLOW.replace( ' WEB_SURFACE_RESOURCE_UUIDS="$(node scripts/coolify-deploy.mjs resolve-web-surfaces "$COOLIFY_STAGING_RESOURCE_UUIDS" "$WEB_IMAGE")"\n WEB_RESOURCE_UUID="${WEB_SURFACE_RESOURCE_UUIDS%%,*}"\n REMAINING_WEB_SURFACE_RESOURCE_UUIDS="${WEB_SURFACE_RESOURCE_UUIDS#*,}"\n ADMIN_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS%%,*}"\n PORTAL_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS#*,}"', ' WEB_RESOURCE_UUID="$(node -e \'const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE); process.stdout.write(app.uuid)\')"\n ADMIN_RESOURCE_UUID="admin"\n PORTAL_RESOURCE_UUID="portal"' ); const result = verifyGiteaDeployWorkflow(createFixture(imageFirstWorkflow)); assert.match(result.failures.join("\n"), /web resources must not be selected through the first matching image/); }); it("fails when portal is omitted from a paired deploy trigger", () => { const partialTrigger = VALID_WORKFLOW.replace( ' PRODUCTION_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_RESOURCE_UUID")"\n', "" ); const result = verifyGiteaDeployWorkflow(createFixture(partialTrigger)); assert.match( result.failures.join("\n"), /production must run trigger\/wait for core < admin < portal without failure aggregation/, ); }); it("fails when a core deployment trigger or wait is backgrounded before the portal trigger", () => { const backgroundedTrigger = VALID_WORKFLOW.replace( ' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"\n', ' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")" &\n', ); const backgroundedWait = VALID_WORKFLOW.replace( ' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging hivegrid-web"\n', ' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging hivegrid-web" &\n', ); assert.match( verifyGiteaDeployWorkflow(createFixture(backgroundedTrigger)).failures.join("\n"), /staging must run trigger\/wait for core < admin < portal without failure aggregation/, ); assert.match( verifyGiteaDeployWorkflow(createFixture(backgroundedWait)).failures.join("\n"), /staging must run trigger\/wait for core < admin < portal without failure aggregation/, ); }); it("fails when live health polling reabsorbs a terminal web deployment wait", () => { const terminalWaitInHealth = VALID_WORKFLOW.replace( ' if [ -z "\${STAGING_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_ADMIN_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_PORTAL_DEPLOYMENT_UUID:-}" ]; then echo "web surface deployment UUIDs are missing"; exit 1; fi', ' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web retry"', ); const result = verifyGiteaDeployWorkflow(createFixture(terminalWaitInHealth)); assert.match( result.failures.join("\n"), /staging live polling must not absorb deployment wait failures and must verify all three named digest-pinned surfaces/, ); }); it("fails when a reviewed deploy-ignore script is removed or replaced by a broad wildcard", () => { const missingExactPath = VALID_WORKFLOW.replace(' - "scripts/docker-worker-runtime-smoke.sh"\n', ""); const broadWildcard = VALID_WORKFLOW.replace( ' - "scripts/docker-worker-runtime-smoke.sh"\n', ' - "scripts/**"\n', ); assert.match( verifyGiteaDeployWorkflow(createFixture(missingExactPath)).failures.join("\n"), /deploy paths-ignore must contain only the reviewed exact CI\/Docker-smoke script paths.*missing: scripts\/docker-worker-runtime-smoke\.sh/, ); assert.match( verifyGiteaDeployWorkflow(createFixture(broadWildcard)).failures.join("\n"), /deploy paths-ignore must not use a broad scripts\/\*\* or scripts\/\* pattern/, ); }); it("fails when controller recovery is not always-run or restores IMAGE_TAG instead of the snapshot", () => { const conditionalRecovery = VALID_WORKFLOW.replace( " - name: Restore previous Coolify staging portal-domain-controller release on rollout failure\n if: always()\n", ' - name: Restore previous Coolify staging portal-domain-controller release on rollout failure\n if: env.DEPLOY_ACTION == "stage"\n', ); const wrongRecoveryRelease = VALID_WORKFLOW.replace( ' "$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"\n RECOVERY_DEPLOYMENT_UUID=', ' "$IMAGE_TAG"\n RECOVERY_DEPLOYMENT_UUID=', ); assert.match( verifyGiteaDeployWorkflow(createFixture(conditionalRecovery)).failures.join("\n"), /staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/, ); assert.match( verifyGiteaDeployWorkflow(createFixture(wrongRecoveryRelease)).failures.join("\n"), /staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/, ); }); it("fails when controller recovery ordering is weakened around drain, acceptance, or restore", () => { const armLine = ' echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true" >> "$GITHUB_ENV"\n'; const armAfterDrain = VALID_WORKFLOW.replace(armLine, "").replace( ' "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n - name: Trigger Coolify staging deploy', ` "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n${armLine} - name: Trigger Coolify staging deploy`, ); const acceptedLine = ' echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true" >> "$GITHUB_ENV"\n'; const acceptedBeforeProof = VALID_WORKFLOW.replace(acceptedLine, "").replace( " node scripts/coolify-portal-domain-controller.mjs assert-release \\\n", `${acceptedLine} node scripts/coolify-portal-domain-controller.mjs assert-release \\\n`, ); const recoveryStepIndex = VALID_WORKFLOW.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure"); assert.notEqual(recoveryStepIndex, -1); const missingRecoverySetRelease = VALID_WORKFLOW.slice(0, recoveryStepIndex) + VALID_WORKFLOW.slice(recoveryStepIndex).replace( "coolify-portal-domain-controller.mjs set-release", "coolify-portal-domain-controller.mjs removed-set-release", ); for (const weakenedWorkflow of [armAfterDrain, acceptedBeforeProof, missingRecoverySetRelease]) { assert.match( verifyGiteaDeployWorkflow(createFixture(weakenedWorkflow)).failures.join("\n"), /staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/, ); } }); it("fails when the controller recovery arm flag is absent", () => { const missingArm = VALID_WORKFLOW.replace( ' echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true" >> "$GITHUB_ENV"\n', "", ); const result = verifyGiteaDeployWorkflow(createFixture(missingArm)); assert.match( result.failures.join("\n"), /staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/, ); }); it("fails when the always-run controller recovery masks trigger failures", () => { const recoveryStepIndex = VALID_WORKFLOW.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure"); assert.notEqual(recoveryStepIndex, -1); const maskedRecovery = VALID_WORKFLOW.slice(0, recoveryStepIndex) + VALID_WORKFLOW.slice(recoveryStepIndex).replace( 'trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")', 'trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" || true)', ); assert.notEqual(maskedRecovery, VALID_WORKFLOW); const result = verifyGiteaDeployWorkflow(createFixture(maskedRecovery)); assert.match(result.failures.join("\n"), /Coolify deploy trigger must not use shell success masking/); }); it("fails when the pair mutation no longer enforces one registry digest", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace("if (!OCI_DIGEST_PATTERN.test(digest))", "if (false)"); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /web surface release mutation must patch, reread, and compensate all three resources as one SHA\/digest trio/); }); it("fails when rollout acceptance stops re-resolving and proving the web digest", () => { const unsafeWorkflow = VALID_WORKFLOW.replaceAll("ROLLED_OUT_WEB_IMAGE_DIGEST", "UNVERIFIED_WEB_IMAGE_DIGEST"); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must re-resolve the full-SHA digest and bind all three completed web deployments to it/); }); it("fails when post-rollout proof no longer binds the exact completed deployments", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( ' assertCompletedWebSurfaceDeployment(deployments[index], "deployment", applications[index], "hivegrid-web");\n', "" ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /post-rollout proof must bind all three healthy named resources and exact completed deployments to one OCI digest/); }); it("fails when the portal runtime verifier allows the core session secret", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace('key === "SESSION_SECRET" || ', ""); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /portal runtime verifier must enforce its dedicated surface\/session and credential denylist/); }); it("fails when the portal runtime verifier skips its CAPTCHA and Redis dependencies", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(" assertPortalAuthDependencies(values);\n", ""); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /portal runtime verifier must fail closed on incomplete login CAPTCHA or Redis\/BullMQ configuration/); }); it("fails when the Coolify helper substitutes application start for force deploy", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( "/deploy?uuid=${encodeURIComponent(resourceUuid)}&force=true", "/applications/${encodeURIComponent(resourceUuid)}/start?force=true" ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /trigger must POST \/deploy\?uuid=&force=true/); }); it("fails when the Coolify helper accepts the first unrelated deployment", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( /const queuedDeployment = resourceUuid\s*\? payload\.deployments\.find\(\(deployment\) => deployment\?\.resource_uuid === resourceUuid\)\s*:\s*payload\.deployments\[0\];/, "const queuedDeployment = payload.deployments[0];" ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /must fail closed unless deployments contains the requested resource UUID/); }); it("fails when the Coolify helper emits unvalidated deployment identifiers", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace('return COOLIFY_CUID_PATTERN.test(value) ? value : "";', "return value;"); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /must validate 24-character Coolify CUIDs/); }); it("fails when malformed Coolify environment payloads pass open", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(' throw new Error("Coolify environment response was malformed");', " return [];"); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /malformed Coolify environment payloads must fail closed/); }); it("fails when runtime env mutation paths trust preview entries", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( "const existing = runtimeEnvEntries(existingPayload);", "const existing = envEntries(existingPayload);" ).replace("const envs = runtimeEnvEntries(envPayload);", "const envs = envEntries(envPayload);"); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /set-envs and legacy rollback must canonicalize non-preview runtime env entries/); }); it("fails when the eleven-resource runtime preflight is absent before worker reconciliation", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( / - name: Validate Coolify staging runtime preflight[\s\S]*?(?= - name: Ensure Coolify staging worker groups)/, "" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/); }); it("fails when the staging read-only eleven-resource preflight runs after migration", () => { const preflightThenMigration = VALID_WORKFLOW.match( / - name: Validate Coolify staging runtime preflight[\s\S]*?(?= - name: Ensure Coolify staging worker groups)/ ); assert.ok(preflightThenMigration); const preflight = preflightThenMigration[0].match( / - name: Validate Coolify staging runtime preflight[\s\S]*?(?= - name: Run staging one-shot database migration)/ )?.[0]; const migration = preflightThenMigration[0].match(/ - name: Run staging one-shot database migration[\s\S]*$/)?.[0]; assert.ok(preflight); assert.ok(migration); const unsafeWorkflow = VALID_WORKFLOW.replace(preflightThenMigration[0], `${migration}${preflight}`); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/); }); it("fails when the staging pre-migration eleven-resource preflight mutates Coolify", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( ' "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n', ' "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n node scripts/coolify-deploy.mjs set-envs "worker-mail-resource" APP_PROTOCOL=https\n' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/); }); it("fails when staging Coolify mutation moves before the one-shot migration", () => { const runtimeStep = VALID_WORKFLOW.match( / - name: Configure Coolify staging runtime resources[\s\S]*?(?= - name: Trigger Coolify staging deploy)/ )?.[0]; assert.ok(runtimeStep); const withoutRuntime = VALID_WORKFLOW.replace(runtimeStep, ""); const unsafeWorkflow = withoutRuntime.replace( " - name: Run staging one-shot database migration\n", `${runtimeStep} - name: Run staging one-shot database migration\n` ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/); }); it("fails when core inventory does not reject APP_REVISION before mutations", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( / const requiredEnvs = await Promise\.all\([\s\S]*?throw new Error\("revision override"\);\n/, "" ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /core deploy inventory must fail closed.*APP_REVISION override/); }); it("fails when web or API mutable revision overrides are not rejected", () => { const unsafeWorkflow = VALID_WORKFLOW.replace(' node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"\n', ""); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging web and API must reject APP_REVISION Coolify overrides/); }); it("fails when a direct curl call has no connection or total timeout", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( "curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH", "curl --fail --silent --show-error --request PATCH" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /direct curl calls must set --connect-timeout and --max-time/); }); it("fails when MCP revision-bound command health configuration is removed", () => { const unsafeWorkflow = VALID_WORKFLOW.replaceAll( /\s*node scripts\/coolify-deploy\.mjs configure-mcp-healthcheck "\$(?:STAGING|PRODUCTION)_MCP_RESOURCE_UUID" "\$IMAGE_TAG"/g, "" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging MCP must configure revision-bound command health before deploy/); assert.match(result.failures.join("\n"), /production MCP must configure revision-bound command health before deploy/); }); it("fails when worker command health is no longer bound to IMAGE_TAG", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( 'EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP"', 'EXPECTED_HEALTH_COMMAND="node apps/worker/dist/health.js"' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging dependent health wait must bind the worker command healthcheck to IMAGE_TAG and EXPECTED_WORKER_GROUP/); }); it("fails when image-bundled healthchecks can override revision-bound command health", () => { const unsafeWorkflow = VALID_WORKFLOW.replaceAll(' || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]', "").replaceAll( ' || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]', "" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /dependent health wait must bind the worker command healthcheck/); assert.match(result.failures.join("\n"), /dependent health wait must bind MCP command health/); }); it("fails when partial dependent trigger failures are not persisted for post-trigger diagnostics", () => { const unsafeWorkflow = VALID_WORKFLOW.replace(' echo "STAGING_DEPENDENT_TRIGGER_FAILED=$STAGING_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV"\n', ""); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must persist queued deployment UUIDs, record trigger failures, wait for diagnostics, and then fail the rollout/); }); it("fails when MCP reconciliation ignores a built-in image healthcheck", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( ' if (current.custom_healthcheck_found !== false) throw new Error("image healthcheck must be absent");\n', "" ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /must atomically configure and verify MCP revision-bound command health/); }); it("fails when MCP reconciliation does not verify the target image", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(' if (!isExpectedMcpImage(current)) throw new Error("wrong MCP image");\n', ""); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /must atomically configure and verify MCP revision-bound command health/); }); it("fails when MCP target identity accepts any image with an mcp suffix", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace("return Boolean(expected) && actual === expected;", "return /(?:^|\\/)mcp$/.test(actual);"); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /must atomically configure and verify MCP revision-bound command health/); }); it("fails when legacy rollback target kind trusts a display name", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( /function legacyRollbackResourceKind\(app\) \{[\s\S]*?\n\}/, 'function legacyRollbackResourceKind(app) {\n if (/worker/.test(app?.name || "")) return "worker";\n return "";\n}' ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /legacy rollback must reject revision overrides and non-runtime targets/); }); it("fails when legacy rollback accepts another product by image suffix", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace( 'if (image === expectedWorkerImage) return "worker";', 'if (/(?:^|\\/)worker$/.test(image)) return "worker";' ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /legacy rollback must reject revision overrides and non-runtime targets/); }); it("fails when legacy rollback leaves command health enabled", () => { const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace("health_check_enabled: false", "health_check_enabled: true"); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper)); assert.match(result.failures.join("\n"), /legacy rollback must reject revision overrides and non-runtime targets/); }); it("fails when the workflow file is missing", () => { const result = verifyGiteaDeployWorkflow(createFixture(null)); assert.match(result.failures.join("\n"), /\.gitea\/workflows\/deploy\.yml: missing required Gitea deploy workflow file/); }); it("fails when faster polling shortens the deployment failure budget", () => { const shortenedBudgetHelper = VALID_COOLIFY_DEPLOY_HELPER.replace('WAIT_ATTEMPTS || "225"', 'WAIT_ATTEMPTS || "90"'); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, shortenedBudgetHelper)); assert.match(result.failures.join("\n"), /2000 ms x 225 attempts to preserve the 450-second failure budget/); }); it("fails when a hosted runner is used", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" runs-on: toolctrl-deploy\n", " runs-on: ubuntu-latest\n"))); assert.match(result.failures.join("\n"), /\.gitea\/workflows\/deploy\.yml:\d+: runs-on must be "toolctrl-deploy" but found "ubuntu-latest"/); }); it("fails when a separate preflight job reacquires a runner", () => { const splitWorkflow = VALID_WORKFLOW.replace( "jobs:\n", "jobs:\n deploy-preflight:\n runs-on: toolctrl-deploy\n steps:\n - name: Split preflight\n run: node --version\n\n" ); const result = verifyGiteaDeployWorkflow(createFixture(splitWorkflow)); assert.match(result.failures.join("\n"), /preflight and rollout must remain in one retained deploy job/); }); it("fails when the deploy preflight omits a required gate", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node scripts/verify-docker-baseline.mjs\n", ""))); assert.match(result.failures.join("\n"), /deploy preflight missing command "node scripts\/verify-docker-baseline\.mjs"/); }); it("fails when the deploy preflight omits deploy readiness consistency checks", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node scripts/verify-deploy-readiness.mjs\n", ""))); assert.match(result.failures.join("\n"), /deploy preflight missing command "node scripts\/verify-deploy-readiness\.mjs"/); }); it("fails when the deploy preflight omits the Coolify helper syntax check", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node --check scripts/coolify-deploy.mjs\n", ""))); assert.match(result.failures.join("\n"), /deploy preflight missing command "node --check scripts\/coolify-deploy\.mjs"/); }); it("fails when the deploy preflight omits the Coolify worker provisioner syntax check", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node --check scripts/coolify-ensure-worker-groups.mjs\n", ""))); assert.match(result.failures.join("\n"), /deploy preflight missing command "node --check scripts\/coolify-ensure-worker-groups\.mjs"/); }); it("fails when the deploy preflight omits the pre-pull script syntax check", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" bash -n scripts/coolify-prepull-images.sh\n", ""))); assert.match(result.failures.join("\n"), /deploy preflight missing command "bash -n scripts\/coolify-prepull-images\.sh"/); }); it("fails when the deploy preflight omits the cosign installer syntax check", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" bash -n scripts/ci/install-cosign.sh\n", ""))); assert.match(result.failures.join("\n"), /deploy preflight missing command "bash -n scripts\/ci\/install-cosign\.sh"/); }); it("fails when cosign is installed from a mutable latest URL", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( "run: bash scripts/ci/install-cosign.sh", "run: curl -sSL https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64 -o /usr/local/bin/cosign" ) ) ); assert.match(result.failures.join("\n"), /cosign installer must not use mutable releases\/latest download URLs/); assert.match(result.failures.join("\n"), /cosign install must use scripts\/ci\/install-cosign\.sh/); }); it("fails when the pinned cosign installer omits checksum verification", () => { const rootDir = createFixture(); writeFile(rootDir, "scripts/ci/install-cosign.sh", VALID_COSIGN_INSTALLER.replace(" | sha256sum --check --status", "")); const result = verifyGiteaDeployWorkflow(rootDir); assert.match(result.failures.join("\n"), /scripts\/ci\/install-cosign\.sh: must verify the cosign download with a pinned SHA256/); }); it("fails when deploy preflight reintroduces full CI checks", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( " bash -n scripts/coolify-prepull-images.sh\n", " bash -n scripts/coolify-prepull-images.sh\n pnpm run verify:i18n\n pnpm run typecheck\n pnpm run lint\n pnpm run build\n pnpm test\n" ) ) ); assert.match( result.failures.join("\n"), /deploy preflight must stay lightweight and not run heavy CI commands: .*pnpm run verify:i18n.*pnpm run typecheck.*pnpm run lint.*pnpm run test.*pnpm run build/ ); }); it("fails when deploy preflight uses shorthand heavy pnpm commands", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( " bash -n scripts/coolify-prepull-images.sh\n", " bash -n scripts/coolify-prepull-images.sh\n pnpm typecheck\n pnpm lint\n pnpm build\n" ) ) ); assert.match( result.failures.join("\n"), /deploy preflight must stay lightweight and not run heavy CI commands: .*pnpm run typecheck.*pnpm run lint.*pnpm run build/ ); }); it("fails when deploy preflight exposes the production database secret at job scope", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace(" runs-on: toolctrl-deploy\n", " runs-on: toolctrl-deploy\n env:\n DATABASE_URL: ${{ secrets.MIGRATOR_DATABASE_URL }}\n") ) ); assert.match(result.failures.join("\n"), /retained deploy job must not expose secrets\.DATABASE_URL at job scope/); }); it("fails when a secret-backed token environment variable is printed", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( ' SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"\n', ' SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"\n echo "$COOLIFY_TOKEN"\n' ) ) ); assert.match(result.failures.join("\n"), /workflow must not print or query-string secret-backed environment variables/); }); it("passes when deploy preflight has no database URL", () => { const result = verifyGiteaDeployWorkflow(createFixture()); assert.deepEqual(result.failures, []); assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: retained deploy job does not expose production database credentials")); }); it("fails when a Coolify selected environment variable is assigned twice in one run block", () => { const duplicateSelectedAssignment = VALID_WORKFLOW.replace( ' SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"\n STAGING_WEB_DEPLOYMENT_UUID=', ' SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"\n SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"\n STAGING_WEB_DEPLOYMENT_UUID=' ); const result = verifyGiteaDeployWorkflow(createFixture(duplicateSelectedAssignment)); assert.match(result.failures.join("\n"), /SELECTED_COOLIFY_API_URL is assigned 2 times in one run block/); }); it("fails when promote migration status reintroduces a second install", () => { const duplicatePromoteInstall = VALID_WORKFLOW.replace( " - name: Verify production migration status after web deploy\n", " - name: Install production migration dependencies\n run: pnpm install --frozen-lockfile --prefer-offline\n - name: Verify production migration status after web deploy\n" ); const result = verifyGiteaDeployWorkflow(createFixture(duplicatePromoteInstall)); assert.match( result.failures.join("\n"), /promote must reuse the initial IMAGE_TAG checkout and frozen install, skip redundant migration checkout\/cache\/install steps/ ); }); it("fails when a staging branch trigger is used", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" branches: [main]", " branches: [main, staging]"))); assert.match(result.failures.join("\n"), /staging branch trigger is forbidden; staging must deploy from main artifacts/); }); it("fails when workflow_dispatch does not promote", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace("DEPLOY_ACTION=promote", "DEPLOY_ACTION=stage"))); assert.match(result.failures.join("\n"), /workflow_dispatch must be an input-free production promotion while main pushes stage/); }); it("fails when workflow_dispatch promotion is not restricted to main", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( ` if [ "$REF" != "refs/heads/main" ]; then echo "Promote must be dispatched from refs/heads/main" exit 1 fi `, "" ) ) ); assert.match(result.failures.join("\n"), /input-free production promotion must be restricted to refs\/heads\/main/); }); it("fails when workflow_dispatch reintroduces manual inputs", () => { const workflowWithManualInput = VALID_WORKFLOW.replace(" workflow_dispatch:\n", " workflow_dispatch:\n inputs:\n staged_sha:\n required: true\n"); const result = verifyGiteaDeployWorkflow(createFixture(workflowWithManualInput)); assert.match(result.failures.join("\n"), /workflow_dispatch must be an input-free production promotion while main pushes stage/); }); it("fails when promote uses the workflow revision instead of the live staging SHA", () => { const unsafePromoteWorkflow = VALID_WORKFLOW.replace('echo "IMAGE_TAG=$STAGED_SHA" >> "$GITHUB_ENV"', 'echo "IMAGE_TAG=$SHA" >> "$GITHUB_ENV"'); const result = verifyGiteaDeployWorkflow(createFixture(unsafePromoteWorkflow)); assert.match(result.failures.join("\n"), /one-click promote must resolve and validate the full live staging SHA before exact-revision checkout/); }); it("fails when promote does not verify staged commit ancestry", () => { const unsafePromoteWorkflow = VALID_WORKFLOW.replaceAll(' git merge-base --is-ancestor "$IMAGE_TAG" origin/main\n', ""); const result = verifyGiteaDeployWorkflow(createFixture(unsafePromoteWorkflow)); assert.match(result.failures.join("\n"), /promote must verify staged commit ancestry and staging image digests before retagging/); }); it("fails when promote does not inspect source images before retagging", () => { const unsafePromoteWorkflow = VALID_WORKFLOW.replace( ' SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format \'{{json .Manifest.Digest}}\')"\n', "" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafePromoteWorkflow)); assert.match(result.failures.join("\n"), /promote must verify staged commit ancestry and staging image digests before retagging/); }); it("fails when SHA tag generation is missing", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(/ SHA: \$\{\{ gitea\.sha \}\}\n/g, ""))); assert.match(result.failures.join("\n"), /missing full SHA image tag generation/); }); it("fails when checkout revision is not verified before image publication", () => { const withoutCheckoutGuard = VALID_WORKFLOW.replace( / - name: Verify checkout revision[\s\S]*? - name: Login to Gitea Registry/s, " - name: Login to Gitea Registry" ); const result = verifyGiteaDeployWorkflow(createFixture(withoutCheckoutGuard)); assert.match( result.failures.join("\n"), /must check out IMAGE_TAG and verify git rev-parse HEAD matches the exact staged or promoted SHA before publication/ ); }); it("fails when the immutable image tag is shortened", () => { const shortenedWorkflow = VALID_WORKFLOW.replace('IMAGE_TAG="$SHA"', 'IMAGE_TAG="${SHA:0:7}"'); const result = verifyGiteaDeployWorkflow(createFixture(shortenedWorkflow)); assert.match(result.failures.join("\n"), /missing full SHA image tag generation/); }); it("fails when legacy generic registry variables are used", () => { const legacyWorkflow = VALID_WORKFLOW.replace(" IMAGE_NAMESPACE: toolctrl-labs\n", " CONTAINER_REGISTRY_NAMESPACE: toolctrl-labs\n"); const result = verifyGiteaDeployWorkflow(createFixture(legacyWorkflow)); assert.match(result.failures.join("\n"), /legacy generic registry variables are forbidden/); }); it("fails when latest is generated", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace('--tag "$IMAGE:staging"', '--tag "$IMAGE:latest"'))); assert.match(result.failures.join("\n"), /mutable latest tag is forbidden/); }); it("fails when staging aliases are pushed during individual target builds", () => { const unsafeWorkflow = VALID_WORKFLOW.replace('--tag "$IMAGE:$IMAGE_TAG" .', '--tag "$IMAGE:$IMAGE_TAG" --tag "$IMAGE:staging" .'); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging aliases must be retagged only after all staging target builds succeed/); }); it("fails when a forbidden E2E command is present", () => { const result = verifyGiteaDeployWorkflow(createFixture(`${VALID_WORKFLOW} - name: E2E\n run: pnpm exec playwright test\n`)); assert.match(result.failures.join("\n"), /forbidden E2E\/browser command found: run: pnpm exec playwright test/); }); it("fails when registry login is softened at step level", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( " - name: Login to Gitea Registry\n run:", " - name: Login to Gitea Registry\n continue-on-error: true\n run:" ) ) ); assert.match(result.failures.join("\n"), /required registry login step must not use continue-on-error or always\(\)/); }); it("fails when image publication disables errexit", () => { const result = verifyGiteaDeployWorkflow( createFixture(VALID_WORKFLOW.replace(" run: |\n BUILD_PIDS=()\n", " run: |\n set +e\n BUILD_PIDS=()\n")), ); assert.match(result.failures.join("\n"), /required Docker image publication step must not use shell success masking/); }); it("fails when staging Docker builds query an unpublished legacy cache ref", () => { const withLegacyBuildCache = VALID_WORKFLOW.replace( ' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n', ' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n CACHE_REF="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/build-cache:$TARGET"\n' ).replace( ' --cache-from "type=registry,ref=$IMAGE:staging"', ' --cache-from "type=registry,ref=$CACHE_REF" --cache-from "type=registry,ref=$IMAGE:staging"' ); const result = verifyGiteaDeployWorkflow(createFixture(withLegacyBuildCache)); assert.match(result.failures.join("\n"), /must not query legacy build-cache refs that the workflow never publishes/); }); it("fails when staging Docker builds omit the portable inline cache", () => { const withoutInlineCache = VALID_WORKFLOW.replace(' --cache-from "type=registry,ref=$IMAGE:staging" --cache-to "type=inline"', ""); const result = verifyGiteaDeployWorkflow(createFixture(withoutInlineCache)); assert.match(result.failures.join("\n"), /must import \$IMAGE:staging and export type=inline cache/); }); it("fails when staging Docker builds use a separate registry cache export", () => { const withRegistryCacheExport = VALID_WORKFLOW.replace( ' --cache-to "type=inline"', ' --cache-to "type=inline" --cache-to "type=registry,ref=$IMAGE:build-cache,mode=max"' ); const result = verifyGiteaDeployWorkflow(createFixture(withRegistryCacheExport)); assert.match(result.failures.join("\n"), /must not use a separate type=registry cache-to export/); }); it("fails when deploy builds fall back to the embedded Docker daemon builder", () => { const unsafeWorkflow = VALID_WORKFLOW.replace(' --builder "$DEPLOY_BUILDER_NAME"', ""); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /staging image builds must pass --builder/); }); it("fails when the isolated deploy BuildKit image is not digest pinned", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( "moby/buildkit:v0.31.2@sha256:2f5adac4ecd194d9f8c10b7b5d7bceb5186853db1b26e5abd3a657af0b7e26ec", "moby/buildkit:latest" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /deploy must pin moby\/buildkit:v0\.31\.2@sha256/); }); it("fails when the bounded deploy builder cache is made run-scoped", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( "DEPLOY_BUILDER_NAME: hivegrid-deploy-v0312-gc1", "DEPLOY_BUILDER_NAME: hivegrid-deploy-${{ gitea.run_id }}" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /use persistent builder hivegrid-deploy-v0312-gc1/); }); it("fails when fresh-client reuse skips host-container image validation", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( ' if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1; then\n ACTUAL_BUILDKIT_IMAGE=', ' if docker buildx inspect "$DEPLOY_BUILDER_NAME" >/dev/null 2>&1; then\n ACTUAL_BUILDKIT_IMAGE=' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /reuse or create the isolated pinned BuildKit with image-drift validation/); }); it("fails when deploy BuildKit can restart outside a workflow", () => { const unsafeWorkflow = VALID_WORKFLOW.replace('--driver-opt "restart-policy=no"', '--driver-opt "restart-policy=unless-stopped"'); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /reuse or create the isolated pinned BuildKit/); }); it("fails when a later deploy does not quiesce an orphaned host builder", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( / - name: Quiesce orphaned deploy BuildKit[\s\S]*? - name: Verify deploy wiring/, " - name: Verify deploy wiring" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /every deploy must quiesce the host builder before gates/); }); it("fails when the running BuildKit config is not verified", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( "node scripts/ci/verify-deploy-buildkit-config.mjs scripts/ci/deploy-buildkitd.toml", "node scripts/ci/unverified-buildkit-config.mjs scripts/ci/deploy-buildkitd.toml" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /reuse or create the isolated pinned BuildKit with image-drift validation/); }); it("fails when the running BuildKit config verifier is missing", () => { const rootDir = createFixture(); fs.rmSync(path.join(rootDir, "scripts/ci/verify-deploy-buildkit-config.mjs")); const result = verifyGiteaDeployWorkflow(rootDir); assert.match(result.failures.join("\n"), /missing running builder config verifier/); }); it("fails when isolated BuildKit does not require the registry host override", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( 'if [ -z "${HIVEGRID_GITEA_HOST_IP:-}" ]; then', 'if [ -n "${HIVEGRID_GITEA_HOST_IP:-}" ]; then' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit appends without replacing a stale registry host mapping", () => { const unsafeWorkflow = VALID_WORKFLOW.replace('cat "$HOSTS_TMP" > /etc/hosts', 'cat "$HOSTS_TMP" > /tmp/unapplied-hosts'); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit accepts a stale mapping beside the exact registry host mapping", () => { const unsafeWorkflow = VALID_WORKFLOW.replace('"$BUILDKIT_HOST_COUNT" -ne 1', '"$BUILDKIT_HOST_COUNT" -lt 1'); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit no longer removes stale registry host tokens", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( "if (field > 1 && $field == host) {\n continue", "if (false) {\n continue" ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit no longer resets the rewritten hosts line per record", () => { const unsafeWorkflow = VALID_WORKFLOW.replace(' rewritten = ""\n for (field', " for (field"); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit moves the rewritten hosts reset outside the record block", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( ' {\n rewritten = ""', ' BEGIN { rewritten = "" }\n {' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit moves the entire hosts rewrite into an AWK BEGIN block", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( ' awk -v host="git.tool-ctrl.de" \'\n {', ' awk -v host="git.tool-ctrl.de" \'\n BEGIN {' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit hard-codes a passing registry host count", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( `END { print count + 0 }' /etc/hosts)"`, `END { print 1 }' /etc/hosts)"` ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit counts a different hostname", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( 'BUILDKIT_HOST_COUNT="$(awk -v host="git.tool-ctrl.de"', 'BUILDKIT_HOST_COUNT="$(awk -v host="localhost"' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit drops the validated-IP predicate", () => { const unsafeWorkflow = VALID_WORKFLOW.replace("'$1 == ip { for (field", "'1 == 1 { for (field"); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated BuildKit validates the IP only after writing its hosts file", () => { const validationBlock = ` if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token" exit 1 fi `; const validationIndex = VALID_WORKFLOW.lastIndexOf(validationBlock); assert.notEqual(validationIndex, -1); const withoutValidation = VALID_WORKFLOW.slice(0, validationIndex) + VALID_WORKFLOW.slice(validationIndex + validationBlock.length); const builderAppend = ` printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts `; const appendIndex = withoutValidation.lastIndexOf(builderAppend); assert.notEqual(appendIndex, -1); const insertIndex = appendIndex + builderAppend.length; const unsafeWorkflow = withoutValidation.slice(0, insertIndex) + validationBlock + withoutValidation.slice(insertIndex); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when the missing BuildKit host override guard is pass-open", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( 'echo "HIVEGRID_GITEA_HOST_IP is required for the isolated deploy builder"\n exit 1\n fi\n if ! HIVEGRID_GITEA_HOST_IP=', 'echo "HIVEGRID_GITEA_HOST_IP is required for the isolated deploy builder"\n :\n fi\n if ! HIVEGRID_GITEA_HOST_IP=' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when the invalid BuildKit host override guard is pass-open", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( 'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n exit 1\n fi\n docker exec', 'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n :\n fi\n docker exec' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when the BuildKit host count mismatch guard is pass-open", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( 'echo "BuildKit registry host override did not converge to exactly one validated mapping"\n exit 1\n fi', 'echo "BuildKit registry host override did not converge to exactly one validated mapping"\n :\n fi' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/); }); it("fails when isolated deploy BuildKit shutdown is not always-run", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( ' - name: Stop isolated deploy BuildKit\n if: always()\n', ' - name: Stop isolated deploy BuildKit\n if: success()\n' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must be stopped by an always\(\) stage cleanup without deleting its cache/); }); it("fails when isolated deploy BuildKit shutdown deletes the persistent cache", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( ' docker inspect "$BUILDER_CONTAINER" >/dev/null\n docker stop "$BUILDER_CONTAINER"\n', ' docker inspect "$BUILDER_CONTAINER" >/dev/null\n docker buildx rm --force "$DEPLOY_BUILDER_NAME"\n' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /must be stopped by an always\(\) stage cleanup without deleting its cache/); }); it("fails when isolated deploy BuildKit can exceed its shared-host disk bound", () => { const rootDir = createFixture(); writeFile(rootDir, "scripts/ci/deploy-buildkitd.toml", '[worker.oci]\n gc = true\n maxUsedSpace = "300GB"\n'); const result = verifyGiteaDeployWorkflow(rootDir); assert.match(result.failures.join("\n"), /must enforce history 86400s\/20 entries and OCI GC at 2GB reserved\/30GB max\/100GB free/); }); it("fails when isolated deploy BuildKit history retention is unbounded", () => { const rootDir = createFixture(); writeFile( rootDir, "scripts/ci/deploy-buildkitd.toml", '[history]\n maxAge = 86400\n maxEntries = 2000\n\n[worker.oci]\n gc = true\n reservedSpace = "2GB"\n maxUsedSpace = "30GB"\n minFreeSpace = "100GB"\n max-parallelism = 12\n' ); const result = verifyGiteaDeployWorkflow(rootDir); assert.match(result.failures.join("\n"), /must enforce history 86400s\/20 entries/); }); it("fails when staging Docker builds are not tracked as parallel jobs", () => { const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" BUILD_PIDS=()\n", ""))); assert.match( result.failures.join("\n"), /staging Docker image builds must run web\/api\/worker in parallel and collect per-target failures before tagging/ ); }); it("fails when Vite build args are not scoped to the web Docker target", () => { const withoutWebScopedViteArgs = VALID_WORKFLOW.replace( / WEB_BUILD_ARGS=\(\)[\s\S]*? docker buildx build --builder "\$DEPLOY_BUILDER_NAME" --push --target "\$TARGET"/, ' docker buildx build --builder "$DEPLOY_BUILDER_NAME" --push --target "$TARGET"' ); const result = verifyGiteaDeployWorkflow(createFixture(withoutWebScopedViteArgs)); assert.match(result.failures.join("\n"), /Vite build args must be scoped to the web Docker target/); }); it("fails when the public Hocuspocus URL is not passed into the web image build", () => { const withoutHocuspocusBuildArg = VALID_WORKFLOW.replace(" VITE_HOCUSPOCUS_URL: \${{ vars.VITE_HOCUSPOCUS_URL }}\n", "").replace( ' --build-arg VITE_HOCUSPOCUS_URL="$VITE_HOCUSPOCUS_URL"', "" ); const result = verifyGiteaDeployWorkflow(createFixture(withoutHocuspocusBuildArg)); assert.match(result.failures.join("\n"), /web image build must map vars\.VITE_HOCUSPOCUS_URL and pass --build-arg VITE_HOCUSPOCUS_URL/); }); it("fails when the staging one-shot migration step is missing", () => { const withoutMigration = VALID_WORKFLOW.replace( / - name: Run staging one-shot database migration[\s\S]*? - name: Ensure Coolify staging worker groups/s, " - name: Ensure Coolify staging worker groups" ); const result = verifyGiteaDeployWorkflow(createFixture(withoutMigration)); assert.match(result.failures.join("\n"), /missing staging one-shot database migration step/); }); it("fails when runtime metadata is not patched through the application API", () => { const withoutRuntimePatch = VALID_WORKFLOW.replaceAll("--request PATCH", "--request GET"); const result = verifyGiteaDeployWorkflow(createFixture(withoutRuntimePatch)); assert.match(result.failures.join("\n"), /Coolify staging runtime config must enforce app\/worker RLS credentials/); }); it("fails when the Coolify hook slots are not cleared", () => { const withActiveHooks = VALID_COOLIFY_DEPLOY_HELPER.replace( /pre_deployment_command: "", post_deployment_command: ""/, 'pre_deployment_command: "cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts", post_deployment_command: ""' ); const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, withActiveHooks)); assert.match(result.failures.join("\n"), /web surface release mutation must patch, reread, and compensate all three resources/); }); it("fails when the staging one-shot migration omits DIRECT_URL", () => { const withoutDirectUrl = VALID_WORKFLOW.replace(" -e DIRECT_URL \\\n", ""); const result = verifyGiteaDeployWorkflow(createFixture(withoutDirectUrl)); assert.match(result.failures.join("\n"), /staging migration must use only its step-scoped migrator secret/); }); it("fails when a seed flag is forwarded to the staging migrator", () => { const withSeedFlag = VALID_WORKFLOW.replace(" -e DIRECT_URL \\\n", " -e DIRECT_URL \\\n -e SEED_ONCE=true \\\n"); const result = verifyGiteaDeployWorkflow(createFixture(withSeedFlag)); assert.match(result.failures.join("\n"), /staging migration must use only its step-scoped migrator secret/); }); it("fails when runtime resources disable RLS", () => { const withRlsBypass = VALID_WORKFLOW.replaceAll( ' node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web', ' RLS_RUNTIME_MODE=off node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web' ); const result = verifyGiteaDeployWorkflow(createFixture(withRlsBypass)); assert.match(result.failures.join("\n"), /must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app/); }); it("fails when deploy attempts to forward a demo seed credential", () => { const withManagedPassword = VALID_WORKFLOW.replace("migrate\n", "migrate\n SEED_DEMO_USER_PASSWORD=ci-managed\n"); const result = verifyGiteaDeployWorkflow(createFixture(withManagedPassword)); assert.match(result.failures.join("\n"), /must not create or overwrite the operator-only SEED_DEMO_USER_PASSWORD/); }); it("fails when the staging one-shot migration runs after Coolify deploy", () => { const migrationMatch = VALID_WORKFLOW.match( / - name: Run staging one-shot database migration[\s\S]*?(?= - name: Ensure Coolify staging worker groups)/ ); assert.ok(migrationMatch); const migrationStep = migrationMatch[0]; const withoutMigration = VALID_WORKFLOW.replace(migrationStep, ""); const afterCoolify = withoutMigration.replace( " - name: Promote staging images to production aliases\n", `${migrationStep} - name: Promote staging images to production aliases\n` ); const result = verifyGiteaDeployWorkflow(createFixture(afterCoolify)); assert.match(result.failures.join("\n"), /staging one-shot migration must run before Coolify deploy/); }); it("fails when the Coolify staging health wait step is missing", () => { const withoutWait = VALID_WORKFLOW.replace( / - name: Wait for Coolify staging health[\s\S]*? - name: Promote staging images to production aliases/s, " - name: Promote staging images to production aliases" ); const result = verifyGiteaDeployWorkflow(createFixture(withoutWait)); assert.match(result.failures.join("\n"), /missing Coolify staging health wait step/); }); it("fails when staging web surfaces skip the live readiness endpoint", () => { const missingReadiness = VALID_WORKFLOW.replace(/(- name: Wait for Coolify staging health[\s\S]*?)"\$HEALTH_URL\/readyz"/, '$1"$HEALTH_URL/healthz"'); const result = verifyGiteaDeployWorkflow(createFixture(missingReadiness)); assert.match(result.failures.join("\n"), /Coolify staging web health wait must require \/readyz status=ready with matching service, surface, and revision/); }); it("fails when production web surfaces skip the live readiness endpoint", () => { const missingReadiness = VALID_WORKFLOW.replace(/(- name: Wait for Coolify production health[\s\S]*?)"\$HEALTH_URL\/readyz"/, '$1"$HEALTH_URL/healthz"'); const result = verifyGiteaDeployWorkflow(createFixture(missingReadiness)); assert.match( result.failures.join("\n"), /Coolify production web health wait must require \/readyz status=ready with matching service, surface, and revision/ ); }); it("fails when staging readiness trusts the expected revision instead of parsing the live response", () => { const spoofedRevision = VALID_WORKFLOW.replace( /(- name: Wait for Coolify staging health[\s\S]*?)READINESS_REVISION="\$\(node -e[^\n]+/, '$1READINESS_REVISION="$IMAGE_TAG"' ); const result = verifyGiteaDeployWorkflow(createFixture(spoofedRevision)); assert.match(result.failures.join("\n"), /Coolify staging web health wait must require \/readyz status=ready with matching service, surface, and revision/); }); it("fails when a web readiness proof prints a database role name", () => { const unredactedReadiness = VALID_WORKFLOW.replace( ' echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS"', ' echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS"\n echo "hivegrid_portal"' ); const result = verifyGiteaDeployWorkflow(createFixture(unredactedReadiness)); assert.match(result.failures.join("\n"), /Coolify staging readiness proof must not print database role names or readiness payloads/); }); it("fails when staging health verification pass-opens a missing health URL", () => { const passOpen = VALID_WORKFLOW.replace( ' HEALTH_URL="https://staging-web.example.invalid"', ' if [ -z "$HEALTH_URL" ]; then\n echo "$LABEL has no health URL; live revision probe skipped"\n return\n fi\n HEALTH_URL="https://staging-web.example.invalid"' ); const result = verifyGiteaDeployWorkflow(createFixture(passOpen)); assert.match(result.failures.join("\n"), /Coolify staging health wait must not pass-open missing health URLs for HTTP resources/); }); it("fails when production health verification pass-opens a missing health URL", () => { const passOpen = VALID_WORKFLOW.replace( ' HEALTH_URL="https://production-web.example.invalid"', ' if [ -z "$HEALTH_URL" ]; then\n echo "$LABEL has no health URL; live revision probe skipped"\n return\n fi\n HEALTH_URL="https://production-web.example.invalid"' ); const result = verifyGiteaDeployWorkflow(createFixture(passOpen)); assert.match(result.failures.join("\n"), /Coolify production health wait must not pass-open missing health URLs for HTTP resources/); }); it("fails when staging health revision is not parsed from the live health response", () => { const staleRevision = VALID_WORKFLOW.replace( /HEALTH_REVISION="\$\(node -e 'const fs = require\("node:fs"\); const payload = JSON\.parse\(fs\.readFileSync\(process\.argv\[1\], "utf8"\)\); process\.stdout\.write\(payload\.revision \|\| ""\);' "\$HEALTH_JSON"\)"/, 'HEALTH_REVISION="$IMAGE_TAG"' ); const result = verifyGiteaDeployWorkflow(createFixture(staleRevision)); assert.match(result.failures.join("\n"), /Coolify staging health wait must verify \/healthz revision against IMAGE_TAG/); }); it("fails when production health revision is not parsed from the live health response", () => { const staleRevision = VALID_WORKFLOW.replace( /HEALTH_URL="https:\/\/production-web\.example\.invalid"[\s\S]*?if \[ "\$HEALTH_REVISION" != "\$IMAGE_TAG" \]; then\n exit 1\n fi/, 'HEALTH_URL="https://production-web.example.invalid"\n curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$APP_JSON"\n HEALTH_REVISION="$IMAGE_TAG"' ); const result = verifyGiteaDeployWorkflow(createFixture(staleRevision)); assert.match(result.failures.join("\n"), /Coolify production health wait must verify \/healthz revision against IMAGE_TAG/); }); it("fails when staging dependent health revision is not parsed from the live health response", () => { const staleRevision = VALID_WORKFLOW.replace( /HEALTH_URL="https:\/\/staging-api\.example\.invalid"[\s\S]*?if \[ "\$HEALTH_REVISION" != "\$IMAGE_TAG" \]; then\n exit 1\n fi/, 'HEALTH_URL="https://staging-api.example.invalid"\n curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$APP_JSON"\n HEALTH_REVISION="$IMAGE_TAG"' ); const result = verifyGiteaDeployWorkflow(createFixture(staleRevision)); assert.match(result.failures.join("\n"), /Coolify staging dependent health wait must verify \/healthz revision against IMAGE_TAG/); }); it("fails when production dependent health revision is not parsed from the live health response", () => { const staleRevision = VALID_WORKFLOW.replace( /HEALTH_URL="https:\/\/production-api\.example\.invalid"[\s\S]*?if \[ "\$HEALTH_REVISION" != "\$IMAGE_TAG" \]; then\n exit 1\n fi/, 'HEALTH_URL="https://production-api.example.invalid"\n curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$APP_JSON"\n HEALTH_REVISION="$IMAGE_TAG"' ); const result = verifyGiteaDeployWorkflow(createFixture(staleRevision)); assert.match(result.failures.join("\n"), /Coolify production dependent health wait must verify \/healthz revision against IMAGE_TAG/); }); it("fails when staging web health does not verify service identity", () => { const missingServiceCheck = VALID_WORKFLOW.replace(' EXPECTED_SERVICE="hivegrid-portal"\n', ""); const result = verifyGiteaDeployWorkflow(createFixture(missingServiceCheck)); assert.match(result.failures.join("\n"), /Coolify staging web health must verify hivegrid-web\/core, hivegrid-admin\/admin, and hivegrid-portal\/portal/); }); it("fails when staging dependent health does not verify API service identity", () => { const missingServiceCheck = VALID_WORKFLOW.replace(' EXPECTED_SERVICE="${5:-api}"\n', ""); const result = verifyGiteaDeployWorkflow(createFixture(missingServiceCheck)); assert.match(result.failures.join("\n"), /Coolify staging dependent health wait must verify service=api/); }); it("fails when staging dependent non-HTTP resources compare image tags to the full SHA", () => { const staleAliasCheck = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-staging}"', 'EXPECTED_IMAGE_ALIAS="$IMAGE_TAG"'); const result = verifyGiteaDeployWorkflow(createFixture(staleAliasCheck)); assert.match( result.failures.join("\n"), /Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/ ); }); it("fails when staging dependent non-HTTP resources expect the production alias", () => { const staleAlias = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-staging}"', 'EXPECTED_IMAGE_ALIAS="\${4:-main}"'); const result = verifyGiteaDeployWorkflow(createFixture(staleAlias)); assert.match( result.failures.join("\n"), /Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/ ); }); it("fails when production dependent non-HTTP resources compare image tags to the full SHA", () => { const staleAliasCheck = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-main}"', 'EXPECTED_IMAGE_ALIAS="$IMAGE_TAG"'); const result = verifyGiteaDeployWorkflow(createFixture(staleAliasCheck)); assert.match( result.failures.join("\n"), /Coolify production dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/ ); }); it("fails when production dependent non-HTTP resources expect the staging alias", () => { const staleAlias = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-main}"', 'EXPECTED_IMAGE_ALIAS="\${4:-staging}"'); const result = verifyGiteaDeployWorkflow(createFixture(staleAlias)); assert.match( result.failures.join("\n"), /Coolify production dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/ ); }); it("fails when dependent health wait can HTTP-probe worker resources", () => { const workerProbeRegression = VALID_WORKFLOW.replace( / WORKER_RESOURCE="\$\(node -e 'process\.stdout\.write\("false"\)'\)"[\s\S]*? if \[ -z "\$HEALTH_URL" \] && \[ "\$HTTP_RESOURCE" = "true" \]; then/, ' if [ -z "$HEALTH_URL" ] && [ "$HTTP_RESOURCE" = "true" ]; then' ); const result = verifyGiteaDeployWorkflow(createFixture(workerProbeRegression)); assert.match( result.failures.join("\n"), /Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/ ); }); it("fails when dependent health wait omits the revision-bound worker acknowledgement", () => { const missingWorkerAcknowledgement = VALID_WORKFLOW.replace( ' echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy"\n', "" ); const result = verifyGiteaDeployWorkflow(createFixture(missingWorkerAcknowledgement)); assert.match( result.failures.join("\n"), /Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/ ); }); it("fails when dependent deployment waits do not aggregate background failures", () => { const missingWaitAggregation = VALID_WORKFLOW.replace( ' for PID in $DEPLOYMENT_WAIT_PIDS; do\n wait "$PID"\n done', ' echo "deployment waits started"' ); const result = verifyGiteaDeployWorkflow(createFixture(missingWaitAggregation)); assert.match(result.failures.join("\n"), /Coolify staging dependent deployment waits must fan out with PID aggregation/); }); it("fails when staging dependent deployments are enqueued without a serialized deployment wait", () => { const burstQueuedDeployments = VALID_WORKFLOW.replace( ` if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent deploy #$DEPLOYMENT_INDEX"; then STAGING_DEPENDENT_TRIGGER_FAILED=true fi `, "" ); const result = verifyGiteaDeployWorkflow(createFixture(burstQueuedDeployments)); assert.match(result.failures.join("\n"), /Coolify staging must wait for each dependent deployment before enqueueing the next resource/); }); it("fails when production dependent deployments are enqueued without a serialized deployment wait", () => { const burstQueuedDeployments = VALID_WORKFLOW.replace( ` if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent deploy #$DEPLOYMENT_INDEX"; then PRODUCTION_DEPENDENT_TRIGGER_FAILED=true fi `, "" ); const result = verifyGiteaDeployWorkflow(createFixture(burstQueuedDeployments)); assert.match(result.failures.join("\n"), /Coolify production must wait for each dependent deployment before enqueueing the next resource/); }); it("fails when dependent resource health checks do not aggregate background failures", () => { const missingResourceAggregation = VALID_WORKFLOW.replace( ' for PID in $RESOURCE_WAIT_PIDS; do\n wait "$PID"\n done', ' echo "resource health checks started"' ); const result = verifyGiteaDeployWorkflow(createFixture(missingResourceAggregation)); assert.match( result.failures.join("\n"), /Coolify staging dependent health wait must fan out STAGING_DEPENDENT_RESOURCE_UUIDS one UUID per worker and aggregate failures/ ); }); it("fails when a Coolify trigger is pass-opened with shell control operators", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( 'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"', 'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")" || true' ) ) ); assert.match(result.failures.join("\n"), /Coolify deploy trigger must not use shell success masking/); }); it("fails when a later Coolify trigger is pass-opened", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( 'PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"', 'PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")" || true' ) ) ); assert.match(result.failures.join("\n"), /Coolify deploy trigger must not use shell success masking/); }); it("fails when a Coolify trigger helper is removed", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( 'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"', 'STAGING_WEB_DEPLOYMENT_UUID="$STAGING_WEB_RESOURCE_UUID"' ) ) ); assert.match(result.failures.join("\n"), /Coolify staging deploy must trigger only STAGING_WEB_RESOURCE_UUID first/); }); it("fails when a Coolify trigger prints the raw response instead of capturing the deployment UUID", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( 'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"', 'node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID"' ) ) ); assert.match( result.failures.join("\n"), /staging must run trigger\/wait for core < admin < portal without failure aggregation/, ); }); it("fails when staging Coolify configuration falls back to production credentials", () => { const result = verifyGiteaDeployWorkflow( createFixture(`${VALID_WORKFLOW} - name: Unsafe staging fallback run: | SELECTED_COOLIFY_API_URL="\${COOLIFY_STAGING_API_URL:-$COOLIFY_API_URL}" SELECTED_COOLIFY_TOKEN="\${COOLIFY_STAGING_TOKEN:-$COOLIFY_TOKEN}" `) ); assert.match(result.failures.join("\n"), /staging Coolify configuration must not fall back to production Coolify URL or token/); }); it("fails when a staging Coolify step directly references production variables", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( "COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}", "COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}\n COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}" ).replace('"$COOLIFY_STAGING_API_URL/deploy?uuid=$STAGING_WEB_RESOURCE_UUID"', '"$COOLIFY_API_URL/deploy?uuid=$STAGING_WEB_RESOURCE_UUID"') ) ); assert.match(result.failures.join("\n"), /staging Coolify steps must not reference production Coolify URL or token variables/); }); it("fails when a production Coolify step directly references staging variables", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( "COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}", "COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}\n COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}" ).replace('"$COOLIFY_API_URL/deploy?uuid=$PRODUCTION_WEB_RESOURCE_UUID"', '"$COOLIFY_STAGING_API_URL/deploy?uuid=$PRODUCTION_WEB_RESOURCE_UUID"') ) ); assert.match(result.failures.join("\n"), /production Coolify steps must not reference staging Coolify URL or token variables/); }); it("fails when staging dependent Coolify resources are deployed as a comma-valued UUID", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( ` DEPLOYMENT_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1)) if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then STAGING_DEPENDENT_DEPLOYMENT_UUIDS="\${STAGING_DEPENDENT_DEPLOYMENT_UUIDS:+$STAGING_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID" if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent deploy #$DEPLOYMENT_INDEX"; then STAGING_DEPENDENT_TRIGGER_FAILED=true fi else STAGING_DEPENDENT_TRIGGER_FAILED=true fi done`, ` DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_DEPENDENT_RESOURCE_UUIDS")" STAGING_DEPENDENT_DEPLOYMENT_UUIDS="$DEPLOYMENT_UUID"` ) ) ); assert.match(result.failures.join("\n"), /Coolify staging dependent resources must deploy one UUID per request/); }); it("fails when production dependent Coolify resources are deployed as a comma-valued UUID", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( ` DEPLOYMENT_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1)) if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="\${PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS:+$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID" if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent deploy #$DEPLOYMENT_INDEX"; then PRODUCTION_DEPENDENT_TRIGGER_FAILED=true fi else PRODUCTION_DEPENDENT_TRIGGER_FAILED=true fi done`, ` DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS")" PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="$DEPLOYMENT_UUID"` ) ) ); assert.match(result.failures.join("\n"), /Coolify production dependent resources must deploy one UUID per request/); }); it("fails when production image promotion ends with an unconditional true", () => { const result = verifyGiteaDeployWorkflow( createFixture( VALID_WORKFLOW.replace( ' docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"\n done\n', ' docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"\n done\n true\n' ) ) ); assert.match(result.failures.join("\n"), /required production image promotion step must not use shell success masking/); }); it("fails when a Coolify trigger is missing", () => { const withoutCoolify = VALID_WORKFLOW.replace( / - name: Trigger Coolify staging deploy[\s\S]*? - name: Promote staging images/s, " - name: Promote staging images" ); const result = verifyGiteaDeployWorkflow(createFixture(withoutCoolify)); assert.match( result.failures.join("\n"), /Coolify trigger must use production and staging Coolify configuration slots|missing Coolify staging health wait step/ ); }); it("fails when api and worker Coolify resource coverage is missing", () => { const webOnlyAssertions = ' node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web'; const webOnlyWorkflow = VALID_WORKFLOW.replaceAll(RUNTIME_DB_ASSERTIONS_FIXTURE, webOnlyAssertions); const result = verifyGiteaDeployWorkflow(createFixture(webOnlyWorkflow)); assert.match( result.failures.join("\n"), /staging must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app, portal as hivegrid_portal, and every worker as hivegrid_worker/ ); assert.match( result.failures.join("\n"), /production must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app, portal as hivegrid_portal, and every worker as hivegrid_worker/ ); }); it("fails when the shared Web Push runtime preflight is missing", () => { const withoutWebPushPreflight = VALID_WORKFLOW.replaceAll( ' node scripts/coolify-deploy.mjs assert-web-push-env "$WEB_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID"\n', "" ); const result = verifyGiteaDeployWorkflow(createFixture(withoutWebPushPreflight)); assert.match(result.failures.join("\n"), /staging must verify matching web and webhook-worker VAPID configuration/); assert.match(result.failures.join("\n"), /production must verify matching web and webhook-worker VAPID configuration/); }); it("fails when the Authentik JIT-capable web surface is assigned to another runtime resource", () => { const wrongSurfaces = VALID_WORKFLOW.replaceAll( 'assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web', 'assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app service' ).replaceAll('assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app service', 'assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app web'); const result = verifyGiteaDeployWorkflow(createFixture(wrongSurfaces)); assert.match( result.failures.join("\n"), /must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app, portal as hivegrid_portal, and every worker as hivegrid_worker/ ); }); it("fails when the mail worker Coolify resource is missing", () => { const withoutMailWorker = VALID_WORKFLOW.replace(/WORKER_MAIL_RESOURCE_UUID/g, "WORKER_REMOVED_RESOURCE_UUID") .replace( /\$WORKER_GENERAL_RESOURCE_UUID,\$WORKER_REMOVED_RESOURCE_UUID,\$WORKER_IMPORT_RESOURCE_UUID,\$WORKER_WEBHOOK_RESOURCE_UUID,\$WORKER_PLUGIN_RESOURCE_UUID/g, "$WORKER_GENERAL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID" ) .replace(/"\$WORKER_GENERAL_RESOURCE_UUID" "\$WORKER_REMOVED_RESOURCE_UUID"/g, '"$WORKER_GENERAL_RESOURCE_UUID"'); const result = verifyGiteaDeployWorkflow(createFixture(withoutMailWorker)); assert.match(result.failures.join("\n"), /Coolify staging image-tag patch must cover API, MCP, Hocuspocus, and every worker group/); assert.match(result.failures.join("\n"), /Coolify production image-tag patch must cover API, MCP, Hocuspocus, and every worker group/); }); it("fails when a literal Coolify URL is present", () => { const literalUrlWorkflow = `${VALID_WORKFLOW} - name: Unsafe literal Coolify URL run: curl https://coolify.invalid/api/v1/applications `; const result = verifyGiteaDeployWorkflow(createFixture(literalUrlWorkflow)); assert.match(result.failures.join("\n"), /literal Coolify\/webhook URL is forbidden/); }); it("fails when promote rebuilds instead of retagging staging images", () => { const rebuildWorkflow = VALID_WORKFLOW.replace( 'docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"', 'docker buildx build --push --target "$TARGET" --tag "$IMAGE:main" .' ); const result = verifyGiteaDeployWorkflow(createFixture(rebuildWorkflow)); assert.match(result.failures.join("\n"), /promote must preserve the verified full-SHA manifest digest in production aliases without rebuilding/); }); it("fails when the production Sentry deploy marker uses the dispatcher SHA instead of the promoted image tag", () => { const staleSentryWorkflow = VALID_WORKFLOW.replace('RELEASE_SHA="${IMAGE_TAG}"', 'RELEASE_SHA="${PROMOTE_SHA:-${SHA}}"'); const result = verifyGiteaDeployWorkflow(createFixture(staleSentryWorkflow)); assert.match(result.failures.join("\n"), /production Sentry deploy marker must release IMAGE_TAG for promote/); }); it("fails when a literal secret-shaped value is present", () => { const result = verifyGiteaDeployWorkflow( createFixture(`${VALID_WORKFLOW} - name: Bad token\n env:\n TOKEN: ghp_1234567890abcdef1234567890abcdef\n`) ); assert.match(result.failures.join("\n"), /\.gitea\/workflows\/deploy\.yml:\d+: literal secret-shaped value is forbidden/); }); it("fails when the internal host override uses the reserved GITEA_ prefix", () => { const result = verifyGiteaDeployWorkflow(createFixture(`env:\n GITEA_HOST_IP: \${{ vars.GITEA_HOST_IP }}\n${VALID_WORKFLOW}`)); assert.match(result.failures.join("\n"), /GITEA_HOST_IP is forbidden because Gitea reserves the GITEA_ variable prefix/); }); it("fails when the runner host override loses its own token validation", () => { const validation = 'if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e \'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)\'; then'; const validationIndex = VALID_WORKFLOW.indexOf(validation); assert.notEqual(validationIndex, -1); const unsafeWorkflow = VALID_WORKFLOW.slice(0, validationIndex) + "if ! true; then" + VALID_WORKFLOW.slice(validationIndex + validation.length); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/); }); it("fails when the runner host validation guard is pass-open", () => { const unsafeWorkflow = VALID_WORKFLOW.replace( 'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n exit 1\n fi\n printf', 'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n :\n fi\n printf' ); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/); }); it("fails when the runner validates its host override only after writing it", () => { const validationBlock = ` if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token" exit 1 fi `; const validationIndex = VALID_WORKFLOW.indexOf(validationBlock); assert.notEqual(validationIndex, -1); const withoutValidation = VALID_WORKFLOW.slice(0, validationIndex) + VALID_WORKFLOW.slice(validationIndex + validationBlock.length); const runnerAppend = ` printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts `; const appendIndex = withoutValidation.indexOf(runnerAppend); assert.notEqual(appendIndex, -1); const insertIndex = appendIndex + runnerAppend.length; const unsafeWorkflow = withoutValidation.slice(0, insertIndex) + validationBlock + withoutValidation.slice(insertIndex); const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow)); assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/); }); it("fails when the internal host override is written without token validation", () => { const result = verifyGiteaDeployWorkflow( createFixture( `${VALID_WORKFLOW}\n host-smoke:\n runs-on: toolctrl-deploy\n steps:\n - name: Resolve internal host\n run: |\n if [ -n \"\${HIVEGRID_GITEA_HOST_IP:-}\" ]; then\n printf '%s git.tool-ctrl.de\\n' \"$HIVEGRID_GITEA_HOST_IP\" >> /etc/hosts\n fi\n` ) ); assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/); }); it("CLI exits non-zero and prints path-specific output for an invalid workflow", () => { const rootDir = createFixture(null); const scriptPath = path.resolve("scripts/verify-gitea-deploy-workflow.mjs"); assert.throws( () => execFileSync(process.execPath, [scriptPath], { cwd: rootDir, encoding: "utf8", stdio: "pipe" }), (error) => { assert.equal(error.status, 1); assert.match(error.stderr, /^FAIL Gitea deploy workflow verification/); assert.match(error.stderr, /FAIL \.gitea\/workflows\/deploy\.yml: missing required Gitea deploy workflow file/); return true; } ); }); });