toolctrl-labs/HiveGrid-Wireframes
9
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
58df5360aab2953ca8070f736eec2a135b855ea2
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/scripts/verify-gitea-deploy-workflow.test.mjs
T

3097 lines
180 KiB
JavaScript
Raw Blame History

import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { afterEach, describe, it } from "vitest";
import { formatGiteaDeployWorkflowResult, verifyGiteaDeployWorkflow } from "./verify-gitea-deploy-workflow.mjs";
const LEGACY_WORKFLOW_FIXTURE = `name: Stage main / Promote staging to production
on:
push:
branches: [main]
paths-ignore:
- "scripts/ci/docker-smoke-suite.sh"
- "scripts/docker-api-runtime-smoke.sh"
- "scripts/docker-hocuspocus-runtime-smoke.sh"
- "scripts/docker-mcp-runtime-smoke.sh"
- "scripts/docker-portal-domain-controller-runtime-smoke.sh"
- "scripts/docker-web-runtime-smoke.sh"
- "scripts/docker-worker-runtime-smoke.sh"
- "scripts/lib/docker-rls-role-fixture.sh"
- "scripts/lib/docker-runtime-smoke.sh"
- "scripts/lib/run-command-with-timeout.mjs"
- "scripts/verify-gitea-ci-workflow.mjs"
- "scripts/verify-portal-domain-controller-deploy.mjs"
# A manual run is intentionally production-only and input-free. Starting it
# is the operator's audited confirmation that backup/restore-point readiness
# was checked; the workflow discovers the release that is live on staging.
workflow_dispatch:
env:
HIVEGRID_GITEA_HOST_IP: \${{ vars.HIVEGRID_GITEA_HOST_IP }}
concurrency:
group: hivegrid-deploy
cancel-in-progress: false
jobs:
stage-or-promote:
runs-on: toolctrl-deploy
env:
REGISTRY: git.tool-ctrl.de
IMAGE_NAMESPACE: toolctrl-labs
IMAGE_PREFIX: hivegrid
DEPLOY_BUILDER_NAME: hivegrid-deploy-v0312-gc1
DEPLOY_BUILDKIT_IMAGE: moby/buildkit:v0.31.2@sha256:2f5adac4ecd194d9f8c10b7b5d7bceb5186853db1b26e5abd3a657af0b7e26ec
SHA: \${{ gitea.sha }}
steps:
- name: Resolve deployment action
run: |
REF="\${GITEA_REF:-\${{ gitea.ref }}}"
EVENT_NAME="\${GITEA_EVENT_NAME:-\${{ gitea.event_name }}}"
if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/main" ]; then
DEPLOY_ACTION=stage
elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then
DEPLOY_ACTION=promote
if [ "$REF" != "refs/heads/main" ]; then
echo "Promote must be dispatched from refs/heads/main"
exit 1
fi
else
echo "Unsupported deployment action: event=$EVENT_NAME ref=$REF"
exit 1
fi
if ! printf '%s' "$SHA" | grep -Eq '^[0-9a-f]{40}$'; then
echo "Gitea SHA must be a full 40-character commit SHA"
exit 1
fi
if [ "$DEPLOY_ACTION" = "stage" ]; then
IMAGE_TAG="$SHA"
echo "IMAGE_TAG=$IMAGE_TAG" >> "$GITHUB_ENV"
fi
echo "DEPLOY_ACTION=$DEPLOY_ACTION" >> "$GITHUB_ENV"
- name: Setup hosts override and Docker CLI
run: |
if [ -n "\${HIVEGRID_GITEA_HOST_IP:-}" ]; then
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
fi
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Quiesce orphaned deploy BuildKit
run: |
BUILDER_CONTAINER="buildx_buildkit_\${DEPLOY_BUILDER_NAME}0"
if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1 && [ "$(docker inspect --format '{{.State.Running}}' "$BUILDER_CONTAINER")" = "true" ]; then
docker stop "$BUILDER_CONTAINER"
fi
- name: Verify deploy wiring
run: |
node scripts/verify-gitea-deploy-workflow.mjs
node scripts/verify-deploy-readiness.mjs
node scripts/verify-docker-baseline.mjs
node scripts/verify-docker-non-root.mjs
node scripts/verify-portal-domain-controller-deploy.mjs
node --check scripts/coolify-deploy.mjs
node --check scripts/coolify-ensure-worker-groups.mjs
node --check scripts/ci/verify-deploy-buildkit-config.mjs
bash -n scripts/coolify-prepull-images.sh
bash -n scripts/ci/install-cosign.sh
- name: Verify checkout revision
run: |
CHECKOUT_SHA="$(git rev-parse HEAD)"
if [ "$CHECKOUT_SHA" != "$SHA" ]; then
echo "Checked out revision must match the Gitea SHA"
exit 1
fi
- name: Login to Gitea Registry
run: echo "\${{ secrets.REGISTRY_PAT }}" | docker login "$REGISTRY" --username "\${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Verify staged promotion provenance
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Staged promotion provenance skipped for staging"
exit 0
fi
git fetch --no-tags origin main
git cat-file -e "$IMAGE_TAG^{commit}"
git merge-base --is-ancestor "$IMAGE_TAG" origin/main
for TARGET in web api worker mcp; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}')"
STAGING_DIGEST="$(docker buildx imagetools inspect "$IMAGE:staging" --format '{{json .Manifest.Digest}}')"
if [ -z "$SHA_DIGEST" ] || [ "$SHA_DIGEST" != "$STAGING_DIGEST" ]; then
echo "Staged image digest for $TARGET does not match $IMAGE_TAG"
exit 1
fi
done
- name: Start isolated deploy BuildKit
if: env.DEPLOY_ACTION == 'stage'
run: |
BUILDER_CONTAINER="buildx_buildkit_\${DEPLOY_BUILDER_NAME}0"
if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1; then
ACTUAL_BUILDKIT_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$BUILDER_CONTAINER")"
if [ "$ACTUAL_BUILDKIT_IMAGE" != "$DEPLOY_BUILDKIT_IMAGE" ]; then
exit 1
fi
ACTUAL_RESTART_POLICY="$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$BUILDER_CONTAINER")"
if [ "$ACTUAL_RESTART_POLICY" != "no" ]; then
exit 1
fi
fi
if docker buildx inspect "$DEPLOY_BUILDER_NAME" >/dev/null 2>&1; then
docker buildx inspect --bootstrap "$DEPLOY_BUILDER_NAME"
else
docker buildx create \
--name "$DEPLOY_BUILDER_NAME" \
--driver docker-container \
--driver-opt "image=$DEPLOY_BUILDKIT_IMAGE" \
--driver-opt "restart-policy=no" \
--buildkitd-config scripts/ci/deploy-buildkitd.toml \
--bootstrap
fi
ACTUAL_BUILDKIT_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$BUILDER_CONTAINER")"
ACTUAL_RESTART_POLICY="$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$BUILDER_CONTAINER")"
if [ "$ACTUAL_BUILDKIT_IMAGE" != "$DEPLOY_BUILDKIT_IMAGE" ] || [ "$ACTUAL_RESTART_POLICY" != "no" ]; then
exit 1
fi
docker exec "$BUILDER_CONTAINER" cat /etc/buildkit/buildkitd.toml \
| node scripts/ci/verify-deploy-buildkit-config.mjs scripts/ci/deploy-buildkitd.toml
if [ -z "\${HIVEGRID_GITEA_HOST_IP:-}" ]; then
echo "HIVEGRID_GITEA_HOST_IP is required for the isolated deploy builder"
exit 1
fi
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
docker exec \\
--env HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" \\
--interactive \\
"$BUILDER_CONTAINER" \\
sh -eu <<'HIVEGRID_BUILDKIT_HOST_OVERRIDE'
HOSTS_TMP="$(mktemp)"
trap 'rm -f "$HOSTS_TMP"' EXIT HUP INT TERM
awk -v host="git.tool-ctrl.de" '
{
rewritten = ""
for (field = 1; field <= NF; field += 1) {
if (field > 1 && $field == host) {
continue
}
rewritten = rewritten == "" ? $field : rewritten " " $field
}
if (rewritten != "") {
print rewritten
}
}
' /etc/hosts > "$HOSTS_TMP"
cat "$HOSTS_TMP" > /etc/hosts
printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
BUILDKIT_HOST_COUNT="$(awk -v host="git.tool-ctrl.de" '{ for (field = 2; field <= NF; field += 1) if ($field == host) count += 1 } END { print count + 0 }' /etc/hosts)"
BUILDKIT_EXACT_HOST_COUNT="$(awk -v host="git.tool-ctrl.de" -v ip="$HIVEGRID_GITEA_HOST_IP" '$1 == ip { for (field = 2; field <= NF; field += 1) if ($field == host) count += 1 } END { print count + 0 }' /etc/hosts)"
if [ "$BUILDKIT_HOST_COUNT" -ne 1 ] || [ "$BUILDKIT_EXACT_HOST_COUNT" -ne 1 ]; then
echo "BuildKit registry host override did not converge to exactly one validated mapping"
exit 1
fi
HIVEGRID_BUILDKIT_HOST_OVERRIDE
- name: Build and push staging images
env:
VITE_HOCUSPOCUS_URL: \${{ vars.VITE_HOCUSPOCUS_URL }}
run: |
BUILD_PIDS=()
BUILD_TARGETS=()
for TARGET in web api worker mcp; do
(
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
WEB_BUILD_ARGS=()
if [ "$TARGET" = "web" ]; then
WEB_BUILD_ARGS=(--build-arg VITE_SENTRY_DSN_WEB="$VITE_SENTRY_DSN_WEB" --build-arg VITE_SENTRY_RELEASE="$SHA" --build-arg VITE_HOCUSPOCUS_URL="$VITE_HOCUSPOCUS_URL")
fi
docker buildx build --builder "$DEPLOY_BUILDER_NAME" --push --target "$TARGET" --file Dockerfile --cache-from "type=registry,ref=$IMAGE:staging" --cache-to "type=inline" --build-arg APP_REVISION="$SHA" "\${WEB_BUILD_ARGS[@]}" --tag "$IMAGE:$IMAGE_TAG" .
) &
BUILD_PIDS+=("$!")
BUILD_TARGETS+=("$TARGET")
done
BUILD_FAILED=0
for INDEX in "\${!BUILD_PIDS[@]}"; do
if ! wait "\${BUILD_PIDS[$INDEX]}"; then
echo "Docker build failed for \${BUILD_TARGETS[$INDEX]}"
BUILD_FAILED=1
fi
done
if [ "$BUILD_FAILED" -ne 0 ]; then
exit "$BUILD_FAILED"
fi
for TARGET in web api worker mcp; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
docker buildx imagetools create --prefer-index=false --tag "$IMAGE:staging" "$IMAGE:$IMAGE_TAG"
done
- name: Stop isolated deploy BuildKit
if: always()
run: |
if [ "\${DEPLOY_ACTION:-}" != "stage" ]; then
exit 0
fi
BUILDER_CONTAINER="buildx_buildkit_\${DEPLOY_BUILDER_NAME}0"
docker inspect "$BUILDER_CONTAINER" >/dev/null
docker stop "$BUILDER_CONTAINER"
- name: Install cosign
if: env.DEPLOY_ACTION == 'stage'
run: bash scripts/ci/install-cosign.sh
- name: Validate Coolify staging runtime preflight
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_STAGING_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
- name: Ensure Coolify staging worker groups
run: node scripts/coolify-ensure-worker-groups.mjs staging
- name: Configure Coolify staging migration boot flags
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
run: |
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const apps = [{ uuid: "web", docker_registry_image_name: process.env.WEB_IMAGE, docker_registry_image_tag: process.env.IMAGE_TAG }]; const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");')"
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" node -e 'process.stdout.write("api")')"
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e 'process.stdout.write("mcp")')"
WORKER_GENERAL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=general WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-general")')"
WORKER_MAIL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=mail WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-mail")')"
WORKER_IMPORT_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=import WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-import")')"
WORKER_WEBHOOK_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=webhook WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-webhook")')"
WORKER_PLUGIN_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=plugin WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-plugin")')"
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
if ! printf ",%s," "$COOLIFY_STAGING_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
echo "Coolify staging deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
exit 1
fi
done
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$API_RESOURCE_UUID"
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
echo "STAGING_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
echo "STAGING_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
echo "STAGING_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\
--header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\
--header "Content-Type: application/json" \\
--data "$MIGRATION_PAYLOAD" \\
"$COOLIFY_STAGING_API_URL/applications/$WEB_RESOURCE_UUID"
node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \\
MIGRATE_ON_BOOT=true \\
SEED_ON_BOOT=true \\
SEED_DEMO_DATA=true \\
ALLOW_STAGING_DEMO_SEED=true \\
SEED_DEMO_PRO_MANAGED=true
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\
--header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\
--header "Content-Type: application/json" \\
--data "$IMAGE_TAG_PAYLOAD" \\
"$COOLIFY_STAGING_API_URL/applications/$RESOURCE_UUID"
done
- name: Trigger Coolify staging deploy
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"
echo "STAGING_WEB_DEPLOYMENT_UUID=$STAGING_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
- name: Wait for Coolify staging health
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web"
RESOURCE_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$STAGING_WEB_RESOURCE_UUID" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\
--header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\
"$COOLIFY_STAGING_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS"
HEALTH_URL="https://staging-web.example.invalid"
HEALTH_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON"
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
EXPECTED_SERVICE="\${5:-web}"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
exit 1
fi
test "$STATUS" = "running:healthy"
done
- name: Trigger Coolify staging dependent deploys
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
node scripts/coolify-deploy.mjs configure-mcp-healthcheck "$STAGING_MCP_RESOURCE_UUID" "$IMAGE_TAG"
STAGING_DEPENDENT_DEPLOYMENT_UUIDS=""
STAGING_DEPENDENT_TRIGGER_FAILED=false
DEPLOYMENT_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then
STAGING_DEPENDENT_DEPLOYMENT_UUIDS="\${STAGING_DEPENDENT_DEPLOYMENT_UUIDS:+$STAGING_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent deploy #$DEPLOYMENT_INDEX"; then
STAGING_DEPENDENT_TRIGGER_FAILED=true
fi
else
STAGING_DEPENDENT_TRIGGER_FAILED=true
fi
done
echo "STAGING_DEPENDENT_DEPLOYMENT_UUIDS=$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV"
echo "STAGING_DEPENDENT_TRIGGER_FAILED=$STAGING_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV"
- name: Wait for Coolify staging dependent health
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
DEPLOYMENT_WAIT_PIDS=""
for DEPLOYMENT_UUID in $(printf "%s" "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent" &
DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!"
done
for PID in $DEPLOYMENT_WAIT_PIDS; do
wait "$PID"
done
RESOURCE_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\
--header "Authorization: Bearer $COOLIFY_STAGING_TOKEN" \\
"$COOLIFY_STAGING_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
echo "Coolify staging dependent resource #$RESOURCE_INDEX status: $STATUS"
HEALTH_URL="https://staging-api.example.invalid"
HTTP_RESOURCE="$(node -e 'process.stdout.write("true")')"
WORKER_RESOURCE="$(node -e 'process.stdout.write("false")')"
if printf ",%s," "$STAGING_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
WORKER_RESOURCE="$(node -e 'process.stdout.write("true")')"
fi
EXPECTED_WORKER_GROUP="\${7:-general}"
if [ "$WORKER_RESOURCE" = "true" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("staging")')"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL worker image tag does not match expected alias"
exit 1
fi
if [ -z "$EXPECTED_WORKER_GROUP" ]; then
exit 1
fi
EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP"
CONFIGURED_HEALTH_COMMAND="$EXPECTED_HEALTH_COMMAND"
CONFIGURED_HEALTH_TYPE="cmd"
HEALTH_CHECK_ENABLED="true"
CUSTOM_HEALTHCHECK_FOUND="false"
if [ "$HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_HEALTH_COMMAND" != "$EXPECTED_HEALTH_COMMAND" ] || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
echo "$LABEL worker healthcheck is not bound to the deployed revision"
exit 1
fi
echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy"
return
fi
EXPECTED_MCP_HEALTH_COMMAND="node apps/mcp/dist/deploy-health.js $IMAGE_TAG"
CONFIGURED_MCP_HEALTH_COMMAND="$EXPECTED_MCP_HEALTH_COMMAND"
CONFIGURED_MCP_HEALTH_TYPE="cmd"
MCP_HEALTH_CHECK_ENABLED="true"
MCP_CUSTOM_HEALTHCHECK_FOUND="false"
if [ "$MCP_HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_MCP_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_MCP_HEALTH_COMMAND" != "$EXPECTED_MCP_HEALTH_COMMAND" ] || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
exit 1
fi
if [ -z "$HEALTH_URL" ] && [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
exit 1
fi
EXPECTED_IMAGE_ALIAS="\${4:-staging}"
CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("staging")')"
if [ -z "$HEALTH_URL" ] && [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
exit 1
fi
HEALTH_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON"
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
EXPECTED_SERVICE="\${5:-api}"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
exit 1
fi
test "$STATUS" = "running:healthy"
done
check_dependent_resource() {
RESOURCE_UUID="$1"
RESOURCE_INDEX="$2"
test -n "$RESOURCE_UUID"
}
RESOURCE_WAIT_PIDS=""
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" &
RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!"
done
for PID in $RESOURCE_WAIT_PIDS; do
wait "$PID"
done
if [ "\${STAGING_DEPENDENT_TRIGGER_FAILED:-false}" = "true" ]; then
echo "One or more Coolify staging dependent deploy triggers failed"
exit 1
fi
- name: Promote staging images to production aliases
run: |
for TARGET in web api worker mcp; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"
done
- name: Validate Coolify production runtime preflight
env:
COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: \${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
- name: Ensure Coolify production worker groups
run: node scripts/coolify-ensure-worker-groups.mjs production
- name: Configure Coolify production migration boot flags
env:
COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: \${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }}
run: |
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$COOLIFY_RESOURCE_UUIDS" node -e 'const apps = [{ uuid: "web", docker_registry_image_name: process.env.WEB_IMAGE, docker_registry_image_tag: process.env.IMAGE_TAG }]; const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");')"
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" node -e 'process.stdout.write("api")')"
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e 'process.stdout.write("mcp")')"
WORKER_GENERAL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=general WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-general")')"
WORKER_MAIL_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=mail WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-mail")')"
WORKER_IMPORT_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=import WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-import")')"
WORKER_WEBHOOK_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=webhook WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-webhook")')"
WORKER_PLUGIN_RESOURCE_UUID="$(WORKER_QUEUE_GROUP=plugin WORKER_IMAGE="$WORKER_IMAGE" node -e 'process.stdout.write("worker-plugin")')"
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
if ! printf ",%s," "$COOLIFY_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
echo "Coolify production deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
exit 1
fi
done
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$API_RESOURCE_UUID"
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
echo "PRODUCTION_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
echo "PRODUCTION_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
echo "PRODUCTION_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\
--header "Authorization: Bearer $COOLIFY_TOKEN" \\
--header "Content-Type: application/json" \\
--data "$MIGRATION_PAYLOAD" \\
"$COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID"
node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \\
MIGRATE_ON_BOOT=true \\
SEED_ON_BOOT=true
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \\
--header "Authorization: Bearer $COOLIFY_TOKEN" \\
--header "Content-Type: application/json" \\
--data "$IMAGE_TAG_PAYLOAD" \\
"$COOLIFY_API_URL/applications/$RESOURCE_UUID"
done
- name: Trigger Coolify production deploy
env:
COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }}
COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"
echo "PRODUCTION_WEB_DEPLOYMENT_UUID=$PRODUCTION_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
- name: Wait for Coolify production health
env:
COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }}
COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production web"
RESOURCE_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_WEB_RESOURCE_UUID" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\
--header "Authorization: Bearer $COOLIFY_TOKEN" \\
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
echo "Coolify production resource #$RESOURCE_INDEX status: $STATUS"
HEALTH_URL="https://production-web.example.invalid"
HEALTH_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON"
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
EXPECTED_SERVICE="\${5:-web}"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
exit 1
fi
test "$STATUS" = "running:healthy"
done
- name: Verify production migration status after web deploy
if: env.DEPLOY_ACTION == 'promote'
env:
DATABASE_URL: \${{ secrets.DATABASE_URL }}
run: |
if [ -z "$DATABASE_URL" ]; then
echo "DATABASE_URL is required for production migration status verification"
exit 1
fi
pnpm run verify:db-migrate-status
- name: Trigger Coolify production dependent deploys
env:
COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }}
COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
node scripts/coolify-deploy.mjs configure-mcp-healthcheck "$PRODUCTION_MCP_RESOURCE_UUID" "$IMAGE_TAG"
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=""
PRODUCTION_DEPENDENT_TRIGGER_FAILED=false
DEPLOYMENT_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="\${PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS:+$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent deploy #$DEPLOYMENT_INDEX"; then
PRODUCTION_DEPENDENT_TRIGGER_FAILED=true
fi
else
PRODUCTION_DEPENDENT_TRIGGER_FAILED=true
fi
done
echo "PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV"
echo "PRODUCTION_DEPENDENT_TRIGGER_FAILED=$PRODUCTION_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV"
- name: Wait for Coolify production dependent health
env:
COOLIFY_API_URL: \${{ vars.COOLIFY_API_URL }}
COOLIFY_TOKEN: \${{ secrets.COOLIFY_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
DEPLOYMENT_WAIT_PIDS=""
for DEPLOYMENT_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent" &
DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!"
done
for PID in $DEPLOYMENT_WAIT_PIDS; do
wait "$PID"
done
RESOURCE_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\
--header "Authorization: Bearer $COOLIFY_TOKEN" \\
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const app = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
echo "Coolify production dependent resource #$RESOURCE_INDEX status: $STATUS"
HEALTH_URL="https://production-api.example.invalid"
HTTP_RESOURCE="$(node -e 'process.stdout.write("true")')"
WORKER_RESOURCE="$(node -e 'process.stdout.write("false")')"
if printf ",%s," "$PRODUCTION_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
WORKER_RESOURCE="$(node -e 'process.stdout.write("true")')"
fi
EXPECTED_WORKER_GROUP="\${7:-general}"
if [ "$WORKER_RESOURCE" = "true" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("main")')"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL worker image tag does not match expected alias"
exit 1
fi
if [ -z "$EXPECTED_WORKER_GROUP" ]; then
exit 1
fi
EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP"
CONFIGURED_HEALTH_COMMAND="$EXPECTED_HEALTH_COMMAND"
CONFIGURED_HEALTH_TYPE="cmd"
HEALTH_CHECK_ENABLED="true"
CUSTOM_HEALTHCHECK_FOUND="false"
if [ "$HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_HEALTH_COMMAND" != "$EXPECTED_HEALTH_COMMAND" ] || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
echo "$LABEL worker healthcheck is not bound to the deployed revision"
exit 1
fi
echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy"
return
fi
EXPECTED_MCP_HEALTH_COMMAND="node apps/mcp/dist/deploy-health.js $IMAGE_TAG"
CONFIGURED_MCP_HEALTH_COMMAND="$EXPECTED_MCP_HEALTH_COMMAND"
CONFIGURED_MCP_HEALTH_TYPE="cmd"
MCP_HEALTH_CHECK_ENABLED="true"
MCP_CUSTOM_HEALTHCHECK_FOUND="false"
if [ "$MCP_HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_MCP_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_MCP_HEALTH_COMMAND" != "$EXPECTED_MCP_HEALTH_COMMAND" ] || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
exit 1
fi
if [ -z "$HEALTH_URL" ] && [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
exit 1
fi
EXPECTED_IMAGE_ALIAS="\${4:-main}"
CONFIGURED_IMAGE_TAG="$(node -e 'process.stdout.write("main")')"
if [ -z "$HEALTH_URL" ] && [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
exit 1
fi
HEALTH_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON"
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
EXPECTED_SERVICE="\${5:-api}"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
exit 1
fi
test "$STATUS" = "running:healthy"
done
check_dependent_resource() {
RESOURCE_UUID="$1"
RESOURCE_INDEX="$2"
test -n "$RESOURCE_UUID"
}
RESOURCE_WAIT_PIDS=""
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" &
RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!"
done
for PID in $RESOURCE_WAIT_PIDS; do
wait "$PID"
done
if [ "\${PRODUCTION_DEPENDENT_TRIGGER_FAILED:-false}" = "true" ]; then
echo "One or more Coolify production dependent deploy triggers failed"
exit 1
fi
- name: Mark Sentry deploy
if: success()
env:
SENTRY_AUTH_TOKEN: \${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: \${{ secrets.SENTRY_ORG }}
SENTRY_URL: \${{ secrets.SENTRY_URL }}
run: |
if [ -z "\${SENTRY_AUTH_TOKEN:-}" ] || [ -z "\${SENTRY_ORG:-}" ] || [ -z "\${SENTRY_URL:-}" ]; then
echo "Sentry secrets not configured; skipping deploy marker"
exit 0
fi
if [ "$DEPLOY_ACTION" = "promote" ]; then
SENTRY_ENVIRONMENT=production
RELEASE_SHA="\${IMAGE_TAG}"
else
SENTRY_ENVIRONMENT=staging
RELEASE_SHA="\${SHA}"
fi
for PROJECT in api worker web; do
bash scripts/ci/sentry-release.sh \
--project "$PROJECT" \
--environment "$SENTRY_ENVIRONMENT" \
--release "$RELEASE_SHA" \
--sourcemaps none
done
`;
const fixtures = [];
const STAGING_ONE_SHOT_FIXTURE = ` - name: Run staging one-shot database migration
env:
MIGRATOR_DATABASE_URL: \${{ secrets.STAGING_MIGRATOR_DATABASE_URL }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then exit 0; fi
if [ -z "\${MIGRATOR_DATABASE_URL:-}" ]; then exit 1; fi
DIRECT_URL="$MIGRATOR_DATABASE_URL" \\
docker run --rm --pull=always \\
-e DIRECT_URL \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web:$IMAGE_TAG" migrate
`;
const PRODUCTION_ONE_SHOT_FIXTURE = ` - name: Run production one-shot database migration
env:
MIGRATOR_DATABASE_URL: \${{ secrets.MIGRATOR_DATABASE_URL }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then exit 0; fi
if [ -z "\${MIGRATOR_DATABASE_URL:-}" ]; then exit 1; fi
DIRECT_URL="$MIGRATOR_DATABASE_URL" \\
docker run --rm --pull=always \\
-e DIRECT_URL \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web:$IMAGE_TAG" migrate
`;
const STAGING_CONTROLLER_DRAIN_FIXTURE = ` - name: Stop and drain Coolify staging portal-domain-controller
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION="$(node scripts/coolify-portal-domain-controller.mjs snapshot-release \\
staging \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\
"$COOLIFY_STAGING_RESOURCE_UUIDS" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker")"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION=$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION" >> "$GITHUB_ENV"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true" >> "$GITHUB_ENV"
node scripts/coolify-portal-domain-controller.mjs stop-and-drain \\
staging \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\
"$COOLIFY_STAGING_RESOURCE_UUIDS" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
`;
const STAGING_CONTROLLER_ACCEPTANCE_AND_RECOVERY_FIXTURE = ` - name: Trigger and verify Coolify staging portal-domain-controller
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
node scripts/coolify-portal-domain-controller.mjs set-release \\
staging \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\
"$COOLIFY_STAGING_RESOURCE_UUIDS" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\
"$IMAGE_TAG"
STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")"
node scripts/coolify-deploy.mjs wait "$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID" "Coolify staging portal-domain-controller"
node scripts/coolify-portal-domain-controller.mjs assert-release \\
staging \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\
"$COOLIFY_STAGING_RESOURCE_UUIDS" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\
"$IMAGE_TAG" \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true" >> "$GITHUB_ENV"
- name: Restore previous Coolify staging portal-domain-controller release on rollout failure
if: always()
env:
COOLIFY_STAGING_API_URL: \${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: \${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: \${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then exit 0; fi
if [ "\${STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED:-false}" != "true" ]; then exit 0; fi
if [ "\${STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED:-false}" = "true" ]; then exit 0; fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
node scripts/coolify-portal-domain-controller.mjs set-release \\
staging \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\
"$COOLIFY_STAGING_RESOURCE_UUIDS" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"
RECOVERY_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")"
node scripts/coolify-deploy.mjs wait "$RECOVERY_DEPLOYMENT_UUID" "Coolify staging portal-domain-controller recovery"
node scripts/coolify-portal-domain-controller.mjs assert-release \\
staging \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \\
"$COOLIFY_STAGING_RESOURCE_UUIDS" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \\
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \\
"$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION" \\
"$RECOVERY_DEPLOYMENT_UUID"
`;
const EXACT_REVISION_GATES_FIXTURE = ` - name: Resolve live staging release
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then exit 0; fi
STAGING_HEALTH_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \\
https://staging-app.hivegrid.de/healthz > "$STAGING_HEALTH_JSON"
STAGED_SHA="$(node -e 'const payload = { revision: "a".repeat(40) }; process.stdout.write(payload.revision || "");')"
STAGING_SERVICE="$(node -e 'process.stdout.write("hivegrid-web")')"
STAGING_SURFACE="$(node -e 'process.stdout.write("core")')"
if [ "$STAGING_SERVICE" != "hivegrid-web" ] || [ "$STAGING_SURFACE" != "core" ]; then exit 1; fi
if ! printf '%s' "$STAGED_SHA" | grep -Eq '^[0-9a-f]{40}$'; then exit 1; fi
git fetch --no-tags origin main
git cat-file -e "$STAGED_SHA^{commit}"
git merge-base --is-ancestor "$STAGED_SHA" origin/main
echo "IMAGE_TAG=$STAGED_SHA" >> "$GITHUB_ENV"
- name: Verify checkout revision
run: |
git fetch --no-tags origin main
git cat-file -e "$IMAGE_TAG^{commit}"
git merge-base --is-ancestor "$IMAGE_TAG" origin/main
git checkout --detach "$IMAGE_TAG"
CHECKOUT_SHA="$(git rev-parse HEAD)"
if [ "$CHECKOUT_SHA" != "$IMAGE_TAG" ]; then exit 1; fi
if [ "$DEPLOY_ACTION" = "stage" ] && [ "$IMAGE_TAG" != "$SHA" ]; then exit 1; fi
- name: Install exact-revision dependencies
run: pnpm install --frozen-lockfile --prefer-offline
- name: Verify exact-revision tenant policy contract
run: pnpm run verify:tenant-policy-contract
- name: Verify exact-revision RLS release readiness
run: pnpm run verify:rls-release-readiness
- name: Verify exact-revision live RLS isolation
run: pnpm run verify:rls-live
env:
HIVEGRID_RLS_REQUIRE_MIGRATION_CHAIN_PARITY: "1"
`;
const RUNTIME_DB_ASSERTIONS_FIXTURE = ` node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web
node scripts/coolify-deploy.mjs assert-runtime-db-env "$ADMIN_RESOURCE_UUID" hivegrid_app admin
node scripts/coolify-deploy.mjs assert-runtime-db-env "$PORTAL_RESOURCE_UUID" hivegrid_portal portal
node scripts/coolify-deploy.mjs assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app service
node scripts/coolify-deploy.mjs assert-runtime-db-env "$MCP_RESOURCE_UUID" hivegrid_app service
node scripts/coolify-deploy.mjs assert-runtime-db-env "$HOCUSPOCUS_RESOURCE_UUID" hivegrid_app service
for WORKER_RESOURCE_UUID in "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
node scripts/coolify-deploy.mjs assert-runtime-db-env "$WORKER_RESOURCE_UUID" hivegrid_worker service
done
node scripts/coolify-deploy.mjs assert-web-push-env "$WEB_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID"
WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
if ! printf '%s' "$WEB_IMAGE_DIGEST" | grep -Eq '^sha256:[0-9a-f]{64}$'; then exit 1; fi
node scripts/coolify-deploy.mjs set-web-surface-release \\
"$WEB_RESOURCE_UUID" \\
"$ADMIN_RESOURCE_UUID" \\
"$PORTAL_RESOURCE_UUID" \\
"$WEB_IMAGE" \\
"$IMAGE_TAG" \\
"$WEB_IMAGE_DIGEST"
echo "STAGING_ADMIN_RESOURCE_UUID=$ADMIN_RESOURCE_UUID"
echo "STAGING_PORTAL_RESOURCE_UUID=$PORTAL_RESOURCE_UUID"
echo "STAGING_WEB_IMAGE_DIGEST=$WEB_IMAGE_DIGEST"
echo "PRODUCTION_ADMIN_RESOURCE_UUID=$ADMIN_RESOURCE_UUID"
echo "PRODUCTION_PORTAL_RESOURCE_UUID=$PORTAL_RESOURCE_UUID"
echo "PRODUCTION_WEB_IMAGE_DIGEST=$WEB_IMAGE_DIGEST"`;
const VALID_WORKFLOW = LEGACY_WORKFLOW_FIXTURE.replace(
/ - name: Verify checkout revision[\s\S]*? - name: Login to Gitea Registry/,
() => `${EXACT_REVISION_GATES_FIXTURE} - name: Login to Gitea Registry`
)
.replaceAll("for TARGET in web api worker mcp; do", "for TARGET in web api worker mcp hocuspocus portal-domain-controller; do")
.replace(
' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n WEB_BUILD_ARGS=()',
' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n DOCKERFILE="Dockerfile"\n if [ "$TARGET" = "hocuspocus" ]; then\n DOCKERFILE="apps/hocuspocus/Dockerfile"\n fi\n WEB_BUILD_ARGS=()'
)
.replace("--file Dockerfile", '--file "$DOCKERFILE"')
.replaceAll(
'"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"',
'"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"'
)
.replaceAll(
' MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"',
' MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"\n HOCUSPOCUS_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"'
)
.replaceAll(
' MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e \'process.stdout.write("mcp")\')"',
' MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" node -e \'process.stdout.write("mcp")\')"\n HOCUSPOCUS_RESOURCE_UUID="$(HOCUSPOCUS_IMAGE="$HOCUSPOCUS_IMAGE" node -e \'process.stdout.write("hocuspocus")\')"'
)
.replaceAll('"$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID"', '"$MCP_RESOURCE_UUID" "$HOCUSPOCUS_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID"')
.replaceAll(
'DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"',
'DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"'
)
.replaceAll(
' DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"',
() =>
`${RUNTIME_DB_ASSERTIONS_FIXTURE}\n DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"`
)
.replace(
'EXPECTED_IMAGE_ALIAS="\${4:-staging}"',
'EXPECTED_IMAGE_ALIAS="\${4:-staging}"\n HOCUSPOCUS_RESOURCE_UUID="$STAGING_HOCUSPOCUS_RESOURCE_UUID"\n RESOURCE_KIND="hocuspocus"\n echo "$LABEL Hocuspocus Docker health and image revision are healthy"\n RESOURCE_KIND="\${6:-auto}"'
)
.replace(
'EXPECTED_IMAGE_ALIAS="\${4:-main}"',
'EXPECTED_IMAGE_ALIAS="\${4:-main}"\n HOCUSPOCUS_RESOURCE_UUID="$PRODUCTION_HOCUSPOCUS_RESOURCE_UUID"\n RESOURCE_KIND="hocuspocus"\n echo "$LABEL Hocuspocus Docker health and image revision are healthy"\n RESOURCE_KIND="\${6:-auto}"'
)
.replace(" - name: Ensure Coolify staging worker groups\n", `${STAGING_ONE_SHOT_FIXTURE} - name: Ensure Coolify staging worker groups\n`)
.replace("Configure Coolify staging migration boot flags", "Configure Coolify staging runtime resources")
.replace(
/ node scripts\/coolify-deploy\.mjs set-envs "\$WEB_RESOURCE_UUID" \\\n MIGRATE_ON_BOOT=true \\\n SEED_ON_BOOT=true \\\n SEED_DEMO_DATA=true \\\n ALLOW_STAGING_DEMO_SEED=true \\\n SEED_DEMO_PRO_MANAGED=true/,
""
)
.replace(" - name: Ensure Coolify production worker groups\n", `${PRODUCTION_ONE_SHOT_FIXTURE} - name: Ensure Coolify production worker groups\n`)
.replace("Configure Coolify production migration boot flags", "Configure Coolify production runtime resources")
.replace(
/ node scripts\/coolify-deploy\.mjs set-envs "\$WEB_RESOURCE_UUID" \\\n MIGRATE_ON_BOOT=true \\\n SEED_ON_BOOT=true/,
""
)
.replace("DATABASE_URL: \${{ secrets.DATABASE_URL }}", "DATABASE_URL: \${{ secrets.MIGRATOR_DATABASE_URL }}")
.replace(
' SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_STAGING_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"',
' SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"\n SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"\n node scripts/coolify-deploy.mjs assert-runtime-preflight \\\n "$COOLIFY_STAGING_RESOURCE_UUIDS" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"'
)
.replace(
' SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs validate-required-images "$COOLIFY_RESOURCE_UUIDS" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"',
' SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"\n SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"\n node scripts/coolify-deploy.mjs assert-runtime-preflight \\\n "$COOLIFY_RESOURCE_UUIDS" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus" \\\n "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"'
)
.replace(/^ WEB_RESOURCE_UUID=.*$/gm, (line) => {
const allowlist = line.includes("COOLIFY_STAGING_RESOURCE_UUIDS") ? "$COOLIFY_STAGING_RESOURCE_UUIDS" : "$COOLIFY_RESOURCE_UUIDS";
return ` WEB_SURFACE_RESOURCE_UUIDS="$(node scripts/coolify-deploy.mjs resolve-web-surfaces "${allowlist}" "$WEB_IMAGE")"
WEB_RESOURCE_UUID="\${WEB_SURFACE_RESOURCE_UUIDS%%,*}"
REMAINING_WEB_SURFACE_RESOURCE_UUIDS="\${WEB_SURFACE_RESOURCE_UUIDS#*,}"
ADMIN_RESOURCE_UUID="\${REMAINING_WEB_SURFACE_RESOURCE_UUIDS%%,*}"
PORTAL_RESOURCE_UUID="\${REMAINING_WEB_SURFACE_RESOURCE_UUIDS#*,}"`;
})
.replaceAll(
'"$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID"',
'"$WEB_RESOURCE_UUID" "$ADMIN_RESOURCE_UUID" "$PORTAL_RESOURCE_UUID" "$API_RESOURCE_UUID"'
)
.replaceAll(
' node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"',
' node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"\n node scripts/coolify-deploy.mjs assert-no-app-revision-override "$ADMIN_RESOURCE_UUID"\n node scripts/coolify-deploy.mjs assert-no-app-revision-override "$PORTAL_RESOURCE_UUID"'
)
.replace(
' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"',
' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging hivegrid-web"\n STAGING_ADMIN_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_ADMIN_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$STAGING_ADMIN_DEPLOYMENT_UUID" "Coolify staging hivegrid-admin"\n STAGING_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$STAGING_PORTAL_DEPLOYMENT_UUID" "Coolify staging hivegrid-portal"'
)
.replace(
' PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"',
' PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production hivegrid-web"\n PRODUCTION_ADMIN_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_ADMIN_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$PRODUCTION_ADMIN_DEPLOYMENT_UUID" "Coolify production hivegrid-admin"\n PRODUCTION_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_RESOURCE_UUID")"\n node scripts/coolify-deploy.mjs wait "$PRODUCTION_PORTAL_DEPLOYMENT_UUID" "Coolify production hivegrid-portal"'
)
.replace(
' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web"',
' if [ -z "\${STAGING_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_ADMIN_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_PORTAL_DEPLOYMENT_UUID:-}" ]; then echo "web surface deployment UUIDs are missing"; exit 1; fi'
)
.replace(
' node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production web"',
' if [ -z "\${PRODUCTION_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "\${PRODUCTION_ADMIN_DEPLOYMENT_UUID:-}" ] || [ -z "\${PRODUCTION_PORTAL_DEPLOYMENT_UUID:-}" ]; then echo "web surface deployment UUIDs are missing"; exit 1; fi'
)
.replace(
" for RESOURCE_UUID in $(printf \"%s\" \"$STAGING_WEB_RESOURCE_UUID\" | tr ',' ' '); do",
' ROLLED_OUT_WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format \'{{json .Manifest.Digest}}\' | tr -d \'"\')"\n if [ "$ROLLED_OUT_WEB_IMAGE_DIGEST" != "$STAGING_WEB_IMAGE_DIGEST" ]; then exit 1; fi\n node scripts/coolify-deploy.mjs assert-web-surface-release \\\n "$STAGING_WEB_RESOURCE_UUID" \\\n "$STAGING_ADMIN_RESOURCE_UUID" \\\n "$STAGING_PORTAL_RESOURCE_UUID" \\\n "$WEB_IMAGE" \\\n "$IMAGE_TAG" \\\n "$ROLLED_OUT_WEB_IMAGE_DIGEST" \\\n "$STAGING_WEB_DEPLOYMENT_UUID" \\\n "$STAGING_ADMIN_DEPLOYMENT_UUID" \\\n "$STAGING_PORTAL_DEPLOYMENT_UUID"\n for SURFACE in core admin portal; do\n EXPECTED_SERVICE="hivegrid-web"\n EXPECTED_SERVICE="hivegrid-admin"\n EXPECTED_SERVICE="hivegrid-portal"\n RESOURCE_UUID="$STAGING_WEB_RESOURCE_UUID"\n verify_live_revision "$APP_JSON" "Coolify staging resource" required "$IMAGE_TAG" "$EXPECTED_SERVICE" "$SURFACE"'
)
.replace(
" for RESOURCE_UUID in $(printf \"%s\" \"$PRODUCTION_WEB_RESOURCE_UUID\" | tr ',' ' '); do",
' ROLLED_OUT_WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format \'{{json .Manifest.Digest}}\' | tr -d \'"\')"\n if [ "$ROLLED_OUT_WEB_IMAGE_DIGEST" != "$PRODUCTION_WEB_IMAGE_DIGEST" ]; then exit 1; fi\n node scripts/coolify-deploy.mjs assert-web-surface-release \\\n "$PRODUCTION_WEB_RESOURCE_UUID" \\\n "$PRODUCTION_ADMIN_RESOURCE_UUID" \\\n "$PRODUCTION_PORTAL_RESOURCE_UUID" \\\n "$WEB_IMAGE" \\\n "$IMAGE_TAG" \\\n "$ROLLED_OUT_WEB_IMAGE_DIGEST" \\\n "$PRODUCTION_WEB_DEPLOYMENT_UUID" \\\n "$PRODUCTION_ADMIN_DEPLOYMENT_UUID" \\\n "$PRODUCTION_PORTAL_DEPLOYMENT_UUID"\n for SURFACE in core admin portal; do\n EXPECTED_SERVICE="hivegrid-web"\n EXPECTED_SERVICE="hivegrid-admin"\n EXPECTED_SERVICE="hivegrid-portal"\n RESOURCE_UUID="$PRODUCTION_WEB_RESOURCE_UUID"\n verify_live_revision "$APP_JSON" "Coolify production resource" required "$IMAGE_TAG" "$EXPECTED_SERVICE" "$SURFACE"'
)
.replaceAll(
' HEALTH_SERVICE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");\' "$HEALTH_JSON")"',
' HEALTH_SERVICE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");\' "$HEALTH_JSON")"\n EXPECTED_SURFACE="${6:-}"\n HEALTH_SURFACE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");\' "$HEALTH_JSON")"\n if [ -n "$EXPECTED_SURFACE" ] && [ "$HEALTH_SURFACE" != "$EXPECTED_SURFACE" ]; then exit 1; fi\n READINESS_JSON="$(mktemp)"\n if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/readyz" > "$READINESS_JSON"; then exit 1; fi\n READINESS_STATUS="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");\' "$READINESS_JSON")"\n READINESS_SERVICE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");\' "$READINESS_JSON")"\n READINESS_SURFACE="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");\' "$READINESS_JSON")"\n READINESS_REVISION="$(node -e \'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");\' "$READINESS_JSON")"\n if [ "$READINESS_STATUS" != "ready" ]; then exit 1; fi\n if [ "$READINESS_REVISION" != "$IMAGE_TAG" ]; then exit 1; fi\n if [ -n "$EXPECTED_SERVICE" ] && [ "$READINESS_SERVICE" != "$EXPECTED_SERVICE" ]; then exit 1; fi\n if [ -n "$EXPECTED_SURFACE" ] && [ "$READINESS_SURFACE" != "$EXPECTED_SURFACE" ]; then exit 1; fi\n EXPECTED_WEB_DIGEST_TAG="sha256-${STAGING_WEB_IMAGE_DIGEST#sha256:}"\n EXPECTED_WEB_DIGEST_TAG="sha256-${PRODUCTION_WEB_IMAGE_DIGEST#sha256:}"\n CONFIGURED_IMAGE_TAG="$EXPECTED_WEB_DIGEST_TAG"\n if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_WEB_DIGEST_TAG" ]; then exit 1; fi'
)
.replace(" - name: Trigger Coolify staging deploy\n", `${STAGING_CONTROLLER_DRAIN_FIXTURE} - name: Trigger Coolify staging deploy\n`)
.replace(" - name: Promote staging images to production aliases\n", `${STAGING_CONTROLLER_ACCEPTANCE_AND_RECOVERY_FIXTURE} - name: Promote staging images to production aliases\n`);
const VALID_COSIGN_INSTALLER = `#!/usr/bin/env bash
set -euo pipefail
COSIGN_VERSION="\${COSIGN_VERSION:-v3.0.6}"
COSIGN_SHA256="\${COSIGN_SHA256:-c956e5dfcac53d52bcf058360d579472f0c1d2d9b69f55209e256fe7783f4c74}"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --location "https://github.com/sigstore/cosign/releases/download/\${COSIGN_VERSION}/cosign-linux-amd64" --output /tmp/cosign
printf '%s %s\\n' "$COSIGN_SHA256" /tmp/cosign | sha256sum --check --status
install -m 0755 /tmp/cosign /usr/local/bin/cosign
`;
const VALID_COOLIFY_DEPLOY_HELPER = `const apiTimeoutMs = 20000;
const delayMs = Number(process.env.COOLIFY_DEPLOYMENT_POLL_MS || "2000");
const attempts = Number(process.env.COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS || "225");
const COOLIFY_CUID_PATTERN = /^[a-z][a-z0-9]{23}$/;
const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/;
const OCI_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/;
const WORKER_GROUP_NAMES = Object.freeze(["general", "mail", "import", "webhook", "plugin"]);
const CORE_PUBLIC_HOST_ENV_KEYS = Object.freeze(["APP_BASE_DOMAIN", "WEB_APP_HOST", "ADMIN_APP_HOST", "API_HOST"]);
const WEB_SURFACE_IDENTITIES = { core: "hivegrid-web", admin: "hivegrid-admin", portal: "hivegrid-portal" };
async function coolify() {
try { return await fetch("https://example.invalid", { signal: AbortSignal.timeout(apiTimeoutMs) }); }
catch { throw new Error("Coolify API request failed or timed out"); }
}
async function readApplicationEnvs(resourceUuid) {
return coolify("/applications/" + encodeURIComponent(resourceUuid) + "/envs");
}
function envKey(entry) { return entry?.key || ""; }
function envValue(entry) { return entry?.value ?? entry?.real_value ?? ""; }
function envEntries(payload) {
if (Array.isArray(payload)) return payload;
if (Array.isArray(payload?.data)) return payload.data;
throw new Error("Coolify environment response was malformed");
}
function normalizeCoolifyBoolean(value) { return value === true ? "true" : value === false ? "false" : ""; }
function runtimeEnvEntries(payload) {
const runtime = [];
const keys = new Set();
for (const entry of envEntries(payload)) {
if (normalizeCoolifyBoolean(entry?.is_preview) === "true") continue;
if (keys.has(entry.key)) throw new Error("Coolify runtime environment response contained duplicate non-preview keys");
keys.add(entry.key);
runtime.push(entry);
}
return runtime;
}
function applicationEntries(payload) {
if (Array.isArray(payload)) return payload;
if (Array.isArray(payload?.data)) return payload.data;
throw new Error("Coolify applications response was malformed");
}
function selectWebSurfacePair(apps, allowed) {
const allowedWebImageApps = apps.filter((app) => allowed.has(app.uuid));
const coreMatches = allowedWebImageApps.filter((app) => app.name === WEB_SURFACE_IDENTITIES.core);
const adminMatches = allowedWebImageApps.filter((app) => app.name === WEB_SURFACE_IDENTITIES.admin);
const portalMatches = allowedWebImageApps.filter((app) => app.name === WEB_SURFACE_IDENTITIES.portal);
if (allowedWebImageApps.length !== 3 || coreMatches.length !== 1 || adminMatches.length !== 1 || portalMatches.length !== 1) throw new Error("invalid trio");
const core = coreMatches[0];
const admin = adminMatches[0];
const portal = portalMatches[0];
if (new Set([core.uuid, admin.uuid, portal.uuid]).size !== 3) throw new Error("duplicate trio");
return { core, admin, portal };
}
function selectUniqueAllowedImageApplication(apps, allowed, expectedImage, label) {
const matches = apps.filter((app) => allowed.has(app.uuid) && app.docker_registry_image_name === expectedImage);
if (matches.length !== 1) throw new Error(label + " runtime missing");
return matches[0];
}
function assertRuntimeDatabaseEntries(entries, expectedRole, surface) {
const values = new Map(entries.map((entry) => [envKey(entry), envValue(entry)]));
if (values.has("APP_REVISION")) throw new Error("runtime APP_REVISION overrides must be removed before migration");
if (surface === "web" && values.has("PORTAL_SESSION_SECRET")) throw new Error("hivegrid-web runtime must not contain PORTAL_SESSION_SECRET");
if (!values.get("DATABASE_POOL_MAX")) throw new Error("explicit positive DATABASE_POOL_MAX");
if (!values.get("DATABASE_URL")?.includes(":6432/")) throw new Error("PgBouncer port 6432");
return values;
}
function assertWebPushEntries() {}
async function assertRuntimePreflight(allowedCsv, webImage, apiImage, mcpImage, hocuspocusImage, workerImage) {
const allowed = new Set(allowedCsv.split(",").filter(Boolean));
const apps = applicationEntries(await coolify("/applications"));
const webPair = selectWebSurfacePair(apps, allowed, webImage);
const api = selectUniqueAllowedImageApplication(apps, allowed, apiImage, "API");
const mcp = selectUniqueAllowedImageApplication(apps, allowed, mcpImage, "MCP");
const hocuspocus = selectUniqueAllowedImageApplication(apps, allowed, hocuspocusImage, "Hocuspocus");
const workers = WORKER_GROUP_NAMES.map((group) => ({ group, key: "WORKER_QUEUE_GROUP", image: workerImage }));
const runtimeResources = [webPair.core, webPair.admin, webPair.portal, api, mcp, hocuspocus, ...workers].map((app) => ({
app,
entries: [],
role: "hivegrid_app",
surface: "service",
}));
if (new Set(runtimeResources.map(({ app }) => app.uuid)).size !== runtimeResources.length) throw new Error("ten distinct resources");
for (const resource of runtimeResources) assertRuntimeDatabaseEntries(resource.entries, resource.role, resource.surface);
if (false) throw new Error("same normalized PORTAL_APP_HOST");
if (false) throw new Error("hivegrid-portal and worker-mail must set APP_PROTOCOL=https");
CORE_PUBLIC_HOST_ENV_KEYS.map((key) => key);
if (false) throw new Error("PORTAL_APP_HOST must be distinct from core APP_BASE_DOMAIN, WEB_APP_HOST, ADMIN_APP_HOST, and API_HOST");
const coreRuntime = runtimeResources[0];
const webhookWorker = workers[3];
assertWebPushEntries(coreRuntime?.entries || [], webhookWorker?.entries || []);
}
async function restoreWebSurfaceReleases() {}
function coolifyDigestTag(digest) {
return digest.replace(":", "-");
}
function webSurfaceReleasePayload(digest) {
return { pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: coolifyDigestTag(digest) };
}
async function setWebSurfaceRelease(coreResourceUuid, adminResourceUuid, portalResourceUuid, expectedImage, revision, digest) {
if (!FULL_GIT_SHA_PATTERN.test(revision)) throw new Error("invalid revision");
if (!OCI_DIGEST_PATTERN.test(digest)) throw new Error("invalid digest");
const mutations = [{ uuid: coreResourceUuid }, { uuid: adminResourceUuid }, { uuid: portalResourceUuid }];
const patched = [];
for (const resource of mutations) patched.push(resource);
const updated = [];
const digestTag = coolifyDigestTag(digest);
if (updated.some((app) => normalizeCoolifyValue(app?.docker_registry_image_tag) !== digestTag)) throw new Error("drift");
await restoreWebSurfaceReleases(patched);
}
function assertCompletedWebSurfaceDeployment(payload) {
return payload.deployment_uuid && payload.application_name && payload.application_id;
}
async function assertWebSurfaceRelease() {
const applications = [];
const deployments = [];
const index = 0;
const digestTag = "sha256-digest";
assertWebSurfacePairByUuid(applications[0], applications[1], applications[2]);
if (normalizeCoolifyValue(applications[0]?.docker_registry_image_tag) !== digestTag) throw new Error("digest drift");
if (normalizeCoolifyValue(applications[0]?.status) !== "running:healthy") throw new Error("unhealthy");
assertCompletedWebSurfaceDeployment(deployments[index], "deployment", applications[index], "hivegrid-web");
}
function isPortalForbiddenEnvKey(key) {
return key === "SESSION_SECRET" || /AUTHENTIK_|OAUTH|BILLING|LAGO|MOLLIE|STRIPE|COOLIFY|SSH|TRAEFIK|INGRESS|CLOUDFLARE|PORTAL_SERVER_|DOMAIN_CONTROLLER/.test(key);
}
function isAllowedCaptchaInstanceUrl(value) { return value.startsWith("https://"); }
function isValidRedisConnectionUrl(value) { return value.startsWith("redis://") || value.startsWith("rediss://"); }
function assertPortalAuthDependencies(values) {
const provider = values.get("CAPTCHA_PROVIDER");
if (provider === "cap") {
isAllowedCaptchaInstanceUrl(values.get("CAP_INSTANCE_URL"));
values.get("CAP_SITEKEY_LOGIN");
values.get("CAP_SITE_SECRET_LOGIN") || values.get("CAP_SITE_SECRET");
} else {
isAllowedCaptchaInstanceUrl(values.get("MCAPTCHA_INSTANCE_URL"));
values.get("MCAPTCHA_SITEKEY_LOGIN");
values.get("MCAPTCHA_ACCOUNT_SECRET");
}
isValidRedisConnectionUrl(values.get("REDIS_URL"));
isValidRedisConnectionUrl(values.get("BULLMQ_REDIS_URL"));
}
async function assertPortalRuntime(values) {
if (values.get("WEB_RUNTIME_SURFACE") !== "portal") throw new Error("surface");
if (!values.get("PORTAL_SESSION_SECRET")) throw new Error("session");
assertPortalAuthDependencies(values);
return isPortalForbiddenEnvKey("SESSION_SECRET");
}
async function validateRequiredImages(allowedCsv, expectedImages) {
const allowed = new Set(allowedCsv.split(",").filter(Boolean));
const apps = applicationEntries(await coolify("/applications"));
const requiredApps = [];
for (const expectedImage of expectedImages) {
const matches = apps.filter((app) => allowed.has(app?.uuid) && app?.docker_registry_image_name === expectedImage);
if (matches.length !== 1) throw new Error("invalid inventory");
requiredApps.push(matches[0]);
}
const requiredEnvs = await Promise.all(
requiredApps.map((app) => readApplicationEnvs(app.uuid).then(runtimeEnvEntries))
);
if (requiredEnvs.some((envs) => envs.some((entry) => envKey(entry) === "APP_REVISION"))) throw new Error("revision override");
}
async function assertNoAppRevisionOverride(resourceUuid) {
const envs = runtimeEnvEntries(await coolify("/applications/" + encodeURIComponent(resourceUuid) + "/envs"));
if (envs.some((entry) => envKey(entry) === "APP_REVISION")) throw new Error("Coolify APP_REVISION override must be removed before deployment");
}
async function setEnvs(resourceUuid) {
const existingPayload = await coolify("/applications/" + encodeURIComponent(resourceUuid) + "/envs");
const existing = runtimeEnvEntries(existingPayload);
return existing;
}
function requireCoolifyCuid(value) {
if (!COOLIFY_CUID_PATTERN.test(value)) throw new Error("invalid Coolify CUID");
}
function deploymentUuid(payload, resourceUuid = "") {
if (!Array.isArray(payload.deployments)) return "";
const queuedDeployment = resourceUuid
? payload.deployments.find((deployment) => deployment?.resource_uuid === resourceUuid)
: payload.deployments[0];
const value = queuedDeployment?.deployment_uuid || "";
return COOLIFY_CUID_PATTERN.test(value) ? value : "";
}
async function trigger(resourceUuid) {
requireCoolifyCuid(resourceUuid, "Coolify resource UUID");
const payload = await coolify(\`/deploy?uuid=\${encodeURIComponent(resourceUuid)}&force=true\`, { method: "POST" });
return deploymentUuid(payload, resourceUuid);
}
function desiredMcpHealthConfig(expectedRevision) {
return {
docker_registry_image_tag: expectedRevision,
health_check_enabled: true,
health_check_type: "cmd",
health_check_command: \`node apps/mcp/dist/deploy-health.js \${expectedRevision}\`,
};
}
function mcpHealthConfigDrift() { return []; }
function expectedMcpImageName() { return "registry.example.test/toolctrl-labs/hivegrid/mcp"; }
function expectedWorkerImageName() { return "registry.example.test/toolctrl-labs/hivegrid/worker"; }
function isExpectedMcpImage(app) {
const actual = app?.docker_registry_image_name || "";
const expected = expectedMcpImageName();
return Boolean(expected) && actual === expected;
}
async function configureMcpHealthcheck(resourceUuid, expectedRevision) {
const envPayload = await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}/envs\`);
const envs = runtimeEnvEntries(envPayload);
if (envs.some((entry) => envKey(entry) === "APP_REVISION" && envValue(entry).trim())) throw new Error("APP_REVISION override");
const current = await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}\`);
if (!isExpectedMcpImage(current)) throw new Error("wrong MCP image");
if (current.custom_healthcheck_found !== false) throw new Error("image healthcheck must be absent");
await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}\`, {
method: "PATCH",
body: JSON.stringify(desiredMcpHealthConfig(expectedRevision)),
});
const updated = await coolify(\`/applications/\${encodeURIComponent(resourceUuid)}\`);
if (mcpHealthConfigDrift(updated, expectedRevision).length > 0) throw new Error("healthcheck drift");
}
function legacyRollbackResourceKind(app) {
const image = app?.docker_registry_image_name || "";
const expectedWorkerImage = expectedWorkerImageName();
const expectedMcpImage = expectedMcpImageName();
if (image === expectedWorkerImage) return "worker";
if (image === expectedMcpImage) return "mcp";
return "";
}
function legacyRollbackHealthConfigDrift() { return []; }
async function configureLegacyRollbackHealthcheck(resourceUuid, expectedRevision, expectedWorkerGroup = "") {
const resourcePath = \`/applications/\${encodeURIComponent(resourceUuid)}\`;
const envPayload = await coolify(\`\${resourcePath}/envs\`);
const envs = runtimeEnvEntries(envPayload);
if (envs.some((entry) => envKey(entry) === "APP_REVISION")) throw new Error("APP_REVISION override");
const current = await coolify(resourcePath);
const resourceKind = legacyRollbackResourceKind(current);
if (resourceKind === "worker" && !expectedWorkerGroup) throw new Error("worker group required");
await coolify(resourcePath, {
method: "PATCH",
body: JSON.stringify({ docker_registry_image_tag: expectedRevision, health_check_enabled: false, health_check_command: "" }),
});
const updated = await coolify(resourcePath);
if (legacyRollbackHealthConfigDrift(updated, expectedRevision).length > 0) throw new Error("legacy rollback drift");
}
async function waitForDeployment(uuid) {
requireCoolifyCuid(uuid, "Coolify deployment UUID");
}
`;
function writeFile(rootDir, relativePath, content) {
const absolutePath = path.join(rootDir, relativePath);
fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
fs.writeFileSync(absolutePath, content);
}
function createFixture(workflowContent = VALID_WORKFLOW, helperContent = VALID_COOLIFY_DEPLOY_HELPER) {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "gitea-deploy-workflow-"));
fixtures.push(rootDir);
if (workflowContent !== null) {
writeFile(rootDir, ".gitea/workflows/deploy.yml", workflowContent);
writeFile(
rootDir,
"scripts/ci/deploy-buildkitd.toml",
'[history]\n maxAge = 86400\n maxEntries = 20\n\n[worker.oci]\n gc = true\n reservedSpace = "2GB"\n maxUsedSpace = "30GB"\n minFreeSpace = "100GB"\n max-parallelism = 12\n'
);
writeFile(
rootDir,
"scripts/ci/verify-deploy-buildkit-config.mjs",
fs.readFileSync(path.join(process.cwd(), "scripts/ci/verify-deploy-buildkit-config.mjs"), "utf8")
);
writeFile(rootDir, "scripts/ci/install-cosign.sh", VALID_COSIGN_INSTALLER);
writeFile(rootDir, "scripts/coolify-deploy.mjs", helperContent);
writeFile(
rootDir,
"start.sh",
"node /myapp/scripts/verify-migrator-role.mjs\nnode /myapp/scripts/adopt-prisma-baseline.mjs --adopt\nnode /myapp/node_modules/prisma/build/index.js migrate deploy --config prisma.config.ts\nnode /myapp/scripts/adopt-prisma-baseline.mjs --verify-current-schema\nnode /myapp/packages/db/dist/deploy-permission-catalog-sync.server.js\n"
);
writeFile(
rootDir,
"scripts/verify-migrator-role.mjs",
"if (session.sessionRole !== REQUIRED_MIGRATOR_ROLE) throw new Error();\nif (role.isSuperuser !== false) throw new Error();\nif (role.bypassesRls !== false) throw new Error();\nconst grantedRole = 'hivegrid_rls_definer';\n"
);
writeFile(rootDir, "scripts/adopt-prisma-baseline.mjs", "export async function adoptPrismaBaseline() {}\n");
writeFile(
rootDir,
"Dockerfile",
"COPY --from=build /myapp/scripts/verify-migrator-role.mjs /myapp/scripts/verify-migrator-role.mjs\nCOPY --from=build /myapp/scripts/adopt-prisma-baseline.mjs /myapp/scripts/adopt-prisma-baseline.mjs\n"
);
}
return rootDir;
}
afterEach(() => {
while (fixtures.length > 0) {
fs.rmSync(fixtures.pop(), { recursive: true, force: true });
}
});
describe("verify-gitea-deploy-workflow", () => {
it("passes for the stage-and-promote Gitea Registry workflow", () => {
const result = verifyGiteaDeployWorkflow(createFixture());
const output = formatGiteaDeployWorkflowResult(result);
assert.deepEqual(result.failures, []);
assert.ok(result.passes.some((pass) => /runner is "toolctrl-deploy"/.test(pass)));
assert.ok(result.passes.some((pass) => /pinned docker-container BuildKit/.test(pass)));
assert.ok(result.passes.some((pass) => /history, cache growth, and BuildKit parallelism are bounded/.test(pass)));
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: deploy has no second runner acquisition for a separate preflight job"));
assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy job "stage-or-promote" retains preflight and rollout on one runner'));
assert.ok(
result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-gitea-deploy-workflow.mjs"')
);
assert.ok(
result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-deploy-readiness.mjs"')
);
assert.ok(
result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-docker-baseline.mjs"')
);
assert.ok(
result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node scripts/verify-docker-non-root.mjs"')
);
assert.ok(
result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node --check scripts/coolify-deploy.mjs"')
);
assert.ok(
result.passes.some(
(pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "node --check scripts/coolify-ensure-worker-groups.mjs"'
)
);
assert.ok(
result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "bash -n scripts/coolify-prepull-images.sh"')
);
assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: deploy preflight includes command "bash -n scripts/ci/install-cosign.sh"'));
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: cosign install uses scripts/ci/install-cosign.sh"));
assert.ok(result.passes.some((pass) => pass === "scripts/ci/install-cosign.sh: cosign download is verified by SHA256"));
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: retained deploy job does not expose production database credentials"));
assert.ok(
result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: deploy preflight stays lightweight and leaves full CI gates to the CI workflow")
);
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: registry login uses REGISTRY_USER secret"));
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: registry login uses REGISTRY_PAT secret"));
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: main-branch push trigger is defined for staging deployment"));
assert.ok(
result.passes.some(
(pass) => pass === ".gitea/workflows/deploy.yml: workflow_dispatch is an input-free production promotion while main pushes stage"
)
);
assert.ok(
result.passes.some(
(pass) =>
pass ===
".gitea/workflows/deploy.yml: promote reuses the exact-revision checkout and install for the post-web migration status gate without a second cache or install"
)
);
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: deployment concurrency serializes staging and production promotion"));
assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: builds Docker target "web"'));
assert.ok(result.passes.some((pass) => pass === '.gitea/workflows/deploy.yml: pushes Docker target "worker"'));
assert.ok(
result.passes.some(
(pass) => pass === ".gitea/workflows/deploy.yml: image paths match existing Coolify hivegrid/{web|api|worker|mcp|hocuspocus} resources"
)
);
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: full SHA, staging, and promoted main tag generation is present"));
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: staging aliases are not pushed from individual target builds"));
assert.ok(
result.passes.some(
(pass) => pass === ".gitea/workflows/deploy.yml: one-click promote resolves the full live staging SHA with audited backup-readiness intent"
)
);
assert.ok(
result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: promote verifies staged commit ancestry and staging image digests before retagging")
);
assert.ok(result.passes.some((pass) => /staging migration is a fail-closed DIRECT_URL-only one-shot container/.test(pass)));
assert.ok(result.passes.some((pass) => /production migration is a fail-closed DIRECT_URL-only one-shot container/.test(pass)));
assert.ok(
result.passes.some((pass) =>
/Coolify staging runtime config pairs web surfaces, clears dead hooks, and enforces role-specific RLS credentials/.test(pass)
)
);
assert.ok(result.passes.some((pass) => /leaves the operator-only SEED_DEMO_USER_PASSWORD untouched/.test(pass)));
assert.ok(result.passes.some((pass) => /Docker image publication passes APP_REVISION build arg/.test(pass)));
assert.ok(result.passes.some((pass) => /do not query nonexistent legacy build-cache refs/.test(pass)));
assert.ok(result.passes.some((pass) => /embed portable inline cache in the immutable SHA image/.test(pass)));
assert.ok(result.passes.some((pass) => /do not use a separate failure-prone registry cache export/.test(pass)));
assert.ok(result.passes.some((pass) => /staging Docker image builds run in parallel and collect per-target failures/.test(pass)));
assert.ok(result.passes.some((pass) => /Vite build args are scoped to the web Docker target/.test(pass)));
assert.ok(result.passes.some((pass) => /public Hocuspocus WebSocket URL is passed from a Gitea variable/.test(pass)));
assert.ok(
result.passes.some((pass) =>
/Coolify production runtime config pairs web surfaces, clears dead hooks, and enforces role-specific RLS credentials/.test(pass)
)
);
assert.ok(result.passes.some((pass) => /production Sentry deploy marker uses promoted IMAGE_TAG/.test(pass)));
assert.ok(result.passes.some((pass) => /deployment waits poll every 2 seconds while preserving the 450-second failure budget/.test(pass)));
assert.ok(result.passes.some((pass) => /Sentry deploy marker covers staging SHA and all runtime projects/.test(pass)));
assert.ok(result.passes.some((pass) => /Coolify staging image-tag patch covers API, MCP, Hocuspocus, and every worker group/.test(pass)));
assert.ok(result.passes.some((pass) => /Coolify production image-tag patch covers API, MCP, Hocuspocus, and every worker group/.test(pass)));
assert.ok(result.passes.some((pass) => /staging read-only eleven-resource runtime preflight runs before migration/.test(pass)));
assert.ok(result.passes.some((pass) => /production read-only eleven-resource runtime preflight runs before migration/.test(pass)));
assert.ok(result.passes.some((pass) => /staging one-shot migration runs before Coolify deploy/.test(pass)));
assert.ok(result.passes.some((pass) => /production one-shot migration runs before Coolify mutation and deploy/.test(pass)));
assert.ok(result.passes.some((pass) => /Coolify staging health wait polls application status/.test(pass)));
assert.ok(
result.passes.some(
(pass) =>
pass ===
".gitea/workflows/deploy.yml: promote preserves the verified full-SHA manifest digest in production aliases without rebuilding"
)
);
assert.match(output, /^PASS Gitea deploy workflow verification/);
assert.match(output, /Summary: \d+ passed, 0 observed, 0 failed/);
});
it("fails when the exact-revision live RLS gate permits a feature-branch overlay", () => {
const workflow = VALID_WORKFLOW.replace(' env:\n HIVEGRID_RLS_REQUIRE_MIGRATION_CHAIN_PARITY: "1"\n', "");
const result = verifyGiteaDeployWorkflow(createFixture(workflow));
assert.match(result.failures.join("\n"), /exact-revision live RLS gate must set HIVEGRID_RLS_REQUIRE_MIGRATION_CHAIN_PARITY: "1"/);
});
it("fails when a web surface is selected through the first image match", () => {
const imageFirstWorkflow = VALID_WORKFLOW.replace(
' WEB_SURFACE_RESOURCE_UUIDS="$(node scripts/coolify-deploy.mjs resolve-web-surfaces "$COOLIFY_STAGING_RESOURCE_UUIDS" "$WEB_IMAGE")"\n WEB_RESOURCE_UUID="${WEB_SURFACE_RESOURCE_UUIDS%%,*}"\n REMAINING_WEB_SURFACE_RESOURCE_UUIDS="${WEB_SURFACE_RESOURCE_UUIDS#*,}"\n ADMIN_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS%%,*}"\n PORTAL_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS#*,}"',
' WEB_RESOURCE_UUID="$(node -e \'const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE); process.stdout.write(app.uuid)\')"\n ADMIN_RESOURCE_UUID="admin"\n PORTAL_RESOURCE_UUID="portal"'
);
const result = verifyGiteaDeployWorkflow(createFixture(imageFirstWorkflow));
assert.match(result.failures.join("\n"), /web resources must not be selected through the first matching image/);
});
it("fails when portal is omitted from a paired deploy trigger", () => {
const partialTrigger = VALID_WORKFLOW.replace(
' PRODUCTION_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_RESOURCE_UUID")"\n',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(partialTrigger));
assert.match(
result.failures.join("\n"),
/production must run trigger\/wait for core < admin < portal without failure aggregation/,
);
});
it("fails when a core deployment trigger or wait is backgrounded before the portal trigger", () => {
const backgroundedTrigger = VALID_WORKFLOW.replace(
' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"\n',
' STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")" &\n',
);
const backgroundedWait = VALID_WORKFLOW.replace(
' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging hivegrid-web"\n',
' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging hivegrid-web" &\n',
);
assert.match(
verifyGiteaDeployWorkflow(createFixture(backgroundedTrigger)).failures.join("\n"),
/staging must run trigger\/wait for core < admin < portal without failure aggregation/,
);
assert.match(
verifyGiteaDeployWorkflow(createFixture(backgroundedWait)).failures.join("\n"),
/staging must run trigger\/wait for core < admin < portal without failure aggregation/,
);
});
it("fails when live health polling reabsorbs a terminal web deployment wait", () => {
const terminalWaitInHealth = VALID_WORKFLOW.replace(
' if [ -z "\${STAGING_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_ADMIN_DEPLOYMENT_UUID:-}" ] || [ -z "\${STAGING_PORTAL_DEPLOYMENT_UUID:-}" ]; then echo "web surface deployment UUIDs are missing"; exit 1; fi',
' node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web retry"',
);
const result = verifyGiteaDeployWorkflow(createFixture(terminalWaitInHealth));
assert.match(
result.failures.join("\n"),
/staging live polling must not absorb deployment wait failures and must verify all three named digest-pinned surfaces/,
);
});
it("fails when a reviewed deploy-ignore script is removed or replaced by a broad wildcard", () => {
const missingExactPath = VALID_WORKFLOW.replace(' - "scripts/docker-worker-runtime-smoke.sh"\n', "");
const broadWildcard = VALID_WORKFLOW.replace(
' - "scripts/docker-worker-runtime-smoke.sh"\n',
' - "scripts/**"\n',
);
assert.match(
verifyGiteaDeployWorkflow(createFixture(missingExactPath)).failures.join("\n"),
/deploy paths-ignore must contain only the reviewed exact CI\/Docker-smoke script paths.*missing: scripts\/docker-worker-runtime-smoke\.sh/,
);
assert.match(
verifyGiteaDeployWorkflow(createFixture(broadWildcard)).failures.join("\n"),
/deploy paths-ignore must not use a broad scripts\/\*\* or scripts\/\* pattern/,
);
});
it("fails when controller recovery is not always-run or restores IMAGE_TAG instead of the snapshot", () => {
const conditionalRecovery = VALID_WORKFLOW.replace(
" - name: Restore previous Coolify staging portal-domain-controller release on rollout failure\n if: always()\n",
' - name: Restore previous Coolify staging portal-domain-controller release on rollout failure\n if: env.DEPLOY_ACTION == "stage"\n',
);
const wrongRecoveryRelease = VALID_WORKFLOW.replace(
' "$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"\n RECOVERY_DEPLOYMENT_UUID=',
' "$IMAGE_TAG"\n RECOVERY_DEPLOYMENT_UUID=',
);
assert.match(
verifyGiteaDeployWorkflow(createFixture(conditionalRecovery)).failures.join("\n"),
/staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/,
);
assert.match(
verifyGiteaDeployWorkflow(createFixture(wrongRecoveryRelease)).failures.join("\n"),
/staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/,
);
});
it("fails when controller recovery ordering is weakened around drain, acceptance, or restore", () => {
const armLine = ' echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true" >> "$GITHUB_ENV"\n';
const armAfterDrain = VALID_WORKFLOW.replace(armLine, "").replace(
' "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n - name: Trigger Coolify staging deploy',
` "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n${armLine} - name: Trigger Coolify staging deploy`,
);
const acceptedLine = ' echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true" >> "$GITHUB_ENV"\n';
const acceptedBeforeProof = VALID_WORKFLOW.replace(acceptedLine, "").replace(
" node scripts/coolify-portal-domain-controller.mjs assert-release \\\n",
`${acceptedLine} node scripts/coolify-portal-domain-controller.mjs assert-release \\\n`,
);
const recoveryStepIndex = VALID_WORKFLOW.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure");
assert.notEqual(recoveryStepIndex, -1);
const missingRecoverySetRelease =
VALID_WORKFLOW.slice(0, recoveryStepIndex) +
VALID_WORKFLOW.slice(recoveryStepIndex).replace(
"coolify-portal-domain-controller.mjs set-release",
"coolify-portal-domain-controller.mjs removed-set-release",
);
for (const weakenedWorkflow of [armAfterDrain, acceptedBeforeProof, missingRecoverySetRelease]) {
assert.match(
verifyGiteaDeployWorkflow(createFixture(weakenedWorkflow)).failures.join("\n"),
/staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/,
);
}
});
it("fails when the controller recovery arm flag is absent", () => {
const missingArm = VALID_WORKFLOW.replace(
' echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true" >> "$GITHUB_ENV"\n',
"",
);
const result = verifyGiteaDeployWorkflow(createFixture(missingArm));
assert.match(
result.failures.join("\n"),
/staging must arm recovery before drain and restore the previous release unless the new controller release is accepted/,
);
});
it("fails when the always-run controller recovery masks trigger failures", () => {
const recoveryStepIndex = VALID_WORKFLOW.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure");
assert.notEqual(recoveryStepIndex, -1);
const maskedRecovery =
VALID_WORKFLOW.slice(0, recoveryStepIndex) +
VALID_WORKFLOW.slice(recoveryStepIndex).replace(
'trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")',
'trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" || true)',
);
assert.notEqual(maskedRecovery, VALID_WORKFLOW);
const result = verifyGiteaDeployWorkflow(createFixture(maskedRecovery));
assert.match(result.failures.join("\n"), /Coolify deploy trigger must not use shell success masking/);
});
it("fails when the pair mutation no longer enforces one registry digest", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace("if (!OCI_DIGEST_PATTERN.test(digest))", "if (false)");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /web surface release mutation must patch, reread, and compensate all three resources as one SHA\/digest trio/);
});
it("fails when rollout acceptance stops re-resolving and proving the web digest", () => {
const unsafeWorkflow = VALID_WORKFLOW.replaceAll("ROLLED_OUT_WEB_IMAGE_DIGEST", "UNVERIFIED_WEB_IMAGE_DIGEST");
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must re-resolve the full-SHA digest and bind all three completed web deployments to it/);
});
it("fails when post-rollout proof no longer binds the exact completed deployments", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
' assertCompletedWebSurfaceDeployment(deployments[index], "deployment", applications[index], "hivegrid-web");\n',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /post-rollout proof must bind all three healthy named resources and exact completed deployments to one OCI digest/);
});
it("fails when the portal runtime verifier allows the core session secret", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace('key === "SESSION_SECRET" || ', "");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /portal runtime verifier must enforce its dedicated surface\/session and credential denylist/);
});
it("fails when the portal runtime verifier skips its CAPTCHA and Redis dependencies", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(" assertPortalAuthDependencies(values);\n", "");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /portal runtime verifier must fail closed on incomplete login CAPTCHA or Redis\/BullMQ configuration/);
});
it("fails when the Coolify helper substitutes application start for force deploy", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
"/deploy?uuid=${encodeURIComponent(resourceUuid)}&force=true",
"/applications/${encodeURIComponent(resourceUuid)}/start?force=true"
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /trigger must POST \/deploy\?uuid=<encoded>&force=true/);
});
it("fails when the Coolify helper accepts the first unrelated deployment", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
/const queuedDeployment = resourceUuid\s*\? payload\.deployments\.find\(\(deployment\) => deployment\?\.resource_uuid === resourceUuid\)\s*:\s*payload\.deployments\[0\];/,
"const queuedDeployment = payload.deployments[0];"
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /must fail closed unless deployments contains the requested resource UUID/);
});
it("fails when the Coolify helper emits unvalidated deployment identifiers", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace('return COOLIFY_CUID_PATTERN.test(value) ? value : "";', "return value;");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /must validate 24-character Coolify CUIDs/);
});
it("fails when malformed Coolify environment payloads pass open", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(' throw new Error("Coolify environment response was malformed");', " return [];");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /malformed Coolify environment payloads must fail closed/);
});
it("fails when runtime env mutation paths trust preview entries", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
"const existing = runtimeEnvEntries(existingPayload);",
"const existing = envEntries(existingPayload);"
).replace("const envs = runtimeEnvEntries(envPayload);", "const envs = envEntries(envPayload);");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /set-envs and legacy rollback must canonicalize non-preview runtime env entries/);
});
it("fails when the eleven-resource runtime preflight is absent before worker reconciliation", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
/ - name: Validate Coolify staging runtime preflight[\s\S]*?(?= - name: Ensure Coolify staging worker groups)/,
""
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/);
});
it("fails when the staging read-only eleven-resource preflight runs after migration", () => {
const preflightThenMigration = VALID_WORKFLOW.match(
/ - name: Validate Coolify staging runtime preflight[\s\S]*?(?= - name: Ensure Coolify staging worker groups)/
);
assert.ok(preflightThenMigration);
const preflight = preflightThenMigration[0].match(
/ - name: Validate Coolify staging runtime preflight[\s\S]*?(?= - name: Run staging one-shot database migration)/
)?.[0];
const migration = preflightThenMigration[0].match(/ - name: Run staging one-shot database migration[\s\S]*$/)?.[0];
assert.ok(preflight);
assert.ok(migration);
const unsafeWorkflow = VALID_WORKFLOW.replace(preflightThenMigration[0], `${migration}${preflight}`);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/);
});
it("fails when the staging pre-migration eleven-resource preflight mutates Coolify", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
' "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n',
' "$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"\n node scripts/coolify-deploy.mjs set-envs "worker-mail-resource" APP_PROTOCOL=https\n'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/);
});
it("fails when staging Coolify mutation moves before the one-shot migration", () => {
const runtimeStep = VALID_WORKFLOW.match(
/ - name: Configure Coolify staging runtime resources[\s\S]*?(?= - name: Trigger Coolify staging deploy)/
)?.[0];
assert.ok(runtimeStep);
const withoutRuntime = VALID_WORKFLOW.replace(runtimeStep, "");
const unsafeWorkflow = withoutRuntime.replace(
" - name: Run staging one-shot database migration\n",
`${runtimeStep} - name: Run staging one-shot database migration\n`
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging must enforce read-only eleven-resource runtime preflight < one-shot migration/);
});
it("fails when core inventory does not reject APP_REVISION before mutations", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
/ const requiredEnvs = await Promise\.all\([\s\S]*?throw new Error\("revision override"\);\n/,
""
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /core deploy inventory must fail closed.*APP_REVISION override/);
});
it("fails when web or API mutable revision overrides are not rejected", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(' node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"\n', "");
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging web and API must reject APP_REVISION Coolify overrides/);
});
it("fails when a direct curl call has no connection or total timeout", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
"curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH",
"curl --fail --silent --show-error --request PATCH"
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /direct curl calls must set --connect-timeout and --max-time/);
});
it("fails when MCP revision-bound command health configuration is removed", () => {
const unsafeWorkflow = VALID_WORKFLOW.replaceAll(
/\s*node scripts\/coolify-deploy\.mjs configure-mcp-healthcheck "\$(?:STAGING|PRODUCTION)_MCP_RESOURCE_UUID" "\$IMAGE_TAG"/g,
""
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging MCP must configure revision-bound command health before deploy/);
assert.match(result.failures.join("\n"), /production MCP must configure revision-bound command health before deploy/);
});
it("fails when worker command health is no longer bound to IMAGE_TAG", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
'EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP"',
'EXPECTED_HEALTH_COMMAND="node apps/worker/dist/health.js"'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging dependent health wait must bind the worker command healthcheck to IMAGE_TAG and EXPECTED_WORKER_GROUP/);
});
it("fails when image-bundled healthchecks can override revision-bound command health", () => {
const unsafeWorkflow = VALID_WORKFLOW.replaceAll(' || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]', "").replaceAll(
' || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /dependent health wait must bind the worker command healthcheck/);
assert.match(result.failures.join("\n"), /dependent health wait must bind MCP command health/);
});
it("fails when partial dependent trigger failures are not persisted for post-trigger diagnostics", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(' echo "STAGING_DEPENDENT_TRIGGER_FAILED=$STAGING_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV"\n', "");
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must persist queued deployment UUIDs, record trigger failures, wait for diagnostics, and then fail the rollout/);
});
it("fails when MCP reconciliation ignores a built-in image healthcheck", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
' if (current.custom_healthcheck_found !== false) throw new Error("image healthcheck must be absent");\n',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /must atomically configure and verify MCP revision-bound command health/);
});
it("fails when MCP reconciliation does not verify the target image", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(' if (!isExpectedMcpImage(current)) throw new Error("wrong MCP image");\n', "");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /must atomically configure and verify MCP revision-bound command health/);
});
it("fails when MCP target identity accepts any image with an mcp suffix", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace("return Boolean(expected) && actual === expected;", "return /(?:^|\\/)mcp$/.test(actual);");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /must atomically configure and verify MCP revision-bound command health/);
});
it("fails when legacy rollback target kind trusts a display name", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
/function legacyRollbackResourceKind\(app\) \{[\s\S]*?\n\}/,
'function legacyRollbackResourceKind(app) {\n if (/worker/.test(app?.name || "")) return "worker";\n return "";\n}'
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /legacy rollback must reject revision overrides and non-runtime targets/);
});
it("fails when legacy rollback accepts another product by image suffix", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace(
'if (image === expectedWorkerImage) return "worker";',
'if (/(?:^|\\/)worker$/.test(image)) return "worker";'
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /legacy rollback must reject revision overrides and non-runtime targets/);
});
it("fails when legacy rollback leaves command health enabled", () => {
const unsafeHelper = VALID_COOLIFY_DEPLOY_HELPER.replace("health_check_enabled: false", "health_check_enabled: true");
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, unsafeHelper));
assert.match(result.failures.join("\n"), /legacy rollback must reject revision overrides and non-runtime targets/);
});
it("fails when the workflow file is missing", () => {
const result = verifyGiteaDeployWorkflow(createFixture(null));
assert.match(result.failures.join("\n"), /\.gitea\/workflows\/deploy\.yml: missing required Gitea deploy workflow file/);
});
it("fails when faster polling shortens the deployment failure budget", () => {
const shortenedBudgetHelper = VALID_COOLIFY_DEPLOY_HELPER.replace('WAIT_ATTEMPTS || "225"', 'WAIT_ATTEMPTS || "90"');
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, shortenedBudgetHelper));
assert.match(result.failures.join("\n"), /2000 ms x 225 attempts to preserve the 450-second failure budget/);
});
it("fails when a hosted runner is used", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" runs-on: toolctrl-deploy\n", " runs-on: ubuntu-latest\n")));
assert.match(result.failures.join("\n"), /\.gitea\/workflows\/deploy\.yml:\d+: runs-on must be "toolctrl-deploy" but found "ubuntu-latest"/);
});
it("fails when a separate preflight job reacquires a runner", () => {
const splitWorkflow = VALID_WORKFLOW.replace(
"jobs:\n",
"jobs:\n deploy-preflight:\n runs-on: toolctrl-deploy\n steps:\n - name: Split preflight\n run: node --version\n\n"
);
const result = verifyGiteaDeployWorkflow(createFixture(splitWorkflow));
assert.match(result.failures.join("\n"), /preflight and rollout must remain in one retained deploy job/);
});
it("fails when the deploy preflight omits a required gate", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node scripts/verify-docker-baseline.mjs\n", "")));
assert.match(result.failures.join("\n"), /deploy preflight missing command "node scripts\/verify-docker-baseline\.mjs"/);
});
it("fails when the deploy preflight omits deploy readiness consistency checks", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node scripts/verify-deploy-readiness.mjs\n", "")));
assert.match(result.failures.join("\n"), /deploy preflight missing command "node scripts\/verify-deploy-readiness\.mjs"/);
});
it("fails when the deploy preflight omits the Coolify helper syntax check", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node --check scripts/coolify-deploy.mjs\n", "")));
assert.match(result.failures.join("\n"), /deploy preflight missing command "node --check scripts\/coolify-deploy\.mjs"/);
});
it("fails when the deploy preflight omits the Coolify worker provisioner syntax check", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" node --check scripts/coolify-ensure-worker-groups.mjs\n", "")));
assert.match(result.failures.join("\n"), /deploy preflight missing command "node --check scripts\/coolify-ensure-worker-groups\.mjs"/);
});
it("fails when the deploy preflight omits the pre-pull script syntax check", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" bash -n scripts/coolify-prepull-images.sh\n", "")));
assert.match(result.failures.join("\n"), /deploy preflight missing command "bash -n scripts\/coolify-prepull-images\.sh"/);
});
it("fails when the deploy preflight omits the cosign installer syntax check", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" bash -n scripts/ci/install-cosign.sh\n", "")));
assert.match(result.failures.join("\n"), /deploy preflight missing command "bash -n scripts\/ci\/install-cosign\.sh"/);
});
it("fails when cosign is installed from a mutable latest URL", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
"run: bash scripts/ci/install-cosign.sh",
"run: curl -sSL https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64 -o /usr/local/bin/cosign"
)
)
);
assert.match(result.failures.join("\n"), /cosign installer must not use mutable releases\/latest download URLs/);
assert.match(result.failures.join("\n"), /cosign install must use scripts\/ci\/install-cosign\.sh/);
});
it("fails when the pinned cosign installer omits checksum verification", () => {
const rootDir = createFixture();
writeFile(rootDir, "scripts/ci/install-cosign.sh", VALID_COSIGN_INSTALLER.replace(" | sha256sum --check --status", ""));
const result = verifyGiteaDeployWorkflow(rootDir);
assert.match(result.failures.join("\n"), /scripts\/ci\/install-cosign\.sh: must verify the cosign download with a pinned SHA256/);
});
it("fails when deploy preflight reintroduces full CI checks", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
" bash -n scripts/coolify-prepull-images.sh\n",
" bash -n scripts/coolify-prepull-images.sh\n pnpm run verify:i18n\n pnpm run typecheck\n pnpm run lint\n pnpm run build\n pnpm test\n"
)
)
);
assert.match(
result.failures.join("\n"),
/deploy preflight must stay lightweight and not run heavy CI commands: .*pnpm run verify:i18n.*pnpm run typecheck.*pnpm run lint.*pnpm run test.*pnpm run build/
);
});
it("fails when deploy preflight uses shorthand heavy pnpm commands", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
" bash -n scripts/coolify-prepull-images.sh\n",
" bash -n scripts/coolify-prepull-images.sh\n pnpm typecheck\n pnpm lint\n pnpm build\n"
)
)
);
assert.match(
result.failures.join("\n"),
/deploy preflight must stay lightweight and not run heavy CI commands: .*pnpm run typecheck.*pnpm run lint.*pnpm run build/
);
});
it("fails when deploy preflight exposes the production database secret at job scope", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(" runs-on: toolctrl-deploy\n", " runs-on: toolctrl-deploy\n env:\n DATABASE_URL: ${{ secrets.MIGRATOR_DATABASE_URL }}\n")
)
);
assert.match(result.failures.join("\n"), /retained deploy job must not expose secrets\.DATABASE_URL at job scope/);
});
it("fails when a secret-backed token environment variable is printed", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
' SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"\n',
' SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"\n echo "$COOLIFY_TOKEN"\n'
)
)
);
assert.match(result.failures.join("\n"), /workflow must not print or query-string secret-backed environment variables/);
});
it("passes when deploy preflight has no database URL", () => {
const result = verifyGiteaDeployWorkflow(createFixture());
assert.deepEqual(result.failures, []);
assert.ok(result.passes.some((pass) => pass === ".gitea/workflows/deploy.yml: retained deploy job does not expose production database credentials"));
});
it("fails when a Coolify selected environment variable is assigned twice in one run block", () => {
const duplicateSelectedAssignment = VALID_WORKFLOW.replace(
' SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"\n STAGING_WEB_DEPLOYMENT_UUID=',
' SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"\n SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"\n STAGING_WEB_DEPLOYMENT_UUID='
);
const result = verifyGiteaDeployWorkflow(createFixture(duplicateSelectedAssignment));
assert.match(result.failures.join("\n"), /SELECTED_COOLIFY_API_URL is assigned 2 times in one run block/);
});
it("fails when promote migration status reintroduces a second install", () => {
const duplicatePromoteInstall = VALID_WORKFLOW.replace(
" - name: Verify production migration status after web deploy\n",
" - name: Install production migration dependencies\n run: pnpm install --frozen-lockfile --prefer-offline\n - name: Verify production migration status after web deploy\n"
);
const result = verifyGiteaDeployWorkflow(createFixture(duplicatePromoteInstall));
assert.match(
result.failures.join("\n"),
/promote must reuse the initial IMAGE_TAG checkout and frozen install, skip redundant migration checkout\/cache\/install steps/
);
});
it("fails when a staging branch trigger is used", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" branches: [main]", " branches: [main, staging]")));
assert.match(result.failures.join("\n"), /staging branch trigger is forbidden; staging must deploy from main artifacts/);
});
it("fails when workflow_dispatch does not promote", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace("DEPLOY_ACTION=promote", "DEPLOY_ACTION=stage")));
assert.match(result.failures.join("\n"), /workflow_dispatch must be an input-free production promotion while main pushes stage/);
});
it("fails when workflow_dispatch promotion is not restricted to main", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
` if [ "$REF" != "refs/heads/main" ]; then
echo "Promote must be dispatched from refs/heads/main"
exit 1
fi
`,
""
)
)
);
assert.match(result.failures.join("\n"), /input-free production promotion must be restricted to refs\/heads\/main/);
});
it("fails when workflow_dispatch reintroduces manual inputs", () => {
const workflowWithManualInput = VALID_WORKFLOW.replace(" workflow_dispatch:\n", " workflow_dispatch:\n inputs:\n staged_sha:\n required: true\n");
const result = verifyGiteaDeployWorkflow(createFixture(workflowWithManualInput));
assert.match(result.failures.join("\n"), /workflow_dispatch must be an input-free production promotion while main pushes stage/);
});
it("fails when promote uses the workflow revision instead of the live staging SHA", () => {
const unsafePromoteWorkflow = VALID_WORKFLOW.replace('echo "IMAGE_TAG=$STAGED_SHA" >> "$GITHUB_ENV"', 'echo "IMAGE_TAG=$SHA" >> "$GITHUB_ENV"');
const result = verifyGiteaDeployWorkflow(createFixture(unsafePromoteWorkflow));
assert.match(result.failures.join("\n"), /one-click promote must resolve and validate the full live staging SHA before exact-revision checkout/);
});
it("fails when promote does not verify staged commit ancestry", () => {
const unsafePromoteWorkflow = VALID_WORKFLOW.replaceAll(' git merge-base --is-ancestor "$IMAGE_TAG" origin/main\n', "");
const result = verifyGiteaDeployWorkflow(createFixture(unsafePromoteWorkflow));
assert.match(result.failures.join("\n"), /promote must verify staged commit ancestry and staging image digests before retagging/);
});
it("fails when promote does not inspect source images before retagging", () => {
const unsafePromoteWorkflow = VALID_WORKFLOW.replace(
' SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format \'{{json .Manifest.Digest}}\')"\n',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafePromoteWorkflow));
assert.match(result.failures.join("\n"), /promote must verify staged commit ancestry and staging image digests before retagging/);
});
it("fails when SHA tag generation is missing", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(/ SHA: \$\{\{ gitea\.sha \}\}\n/g, "")));
assert.match(result.failures.join("\n"), /missing full SHA image tag generation/);
});
it("fails when checkout revision is not verified before image publication", () => {
const withoutCheckoutGuard = VALID_WORKFLOW.replace(
/ - name: Verify checkout revision[\s\S]*? - name: Login to Gitea Registry/s,
" - name: Login to Gitea Registry"
);
const result = verifyGiteaDeployWorkflow(createFixture(withoutCheckoutGuard));
assert.match(
result.failures.join("\n"),
/must check out IMAGE_TAG and verify git rev-parse HEAD matches the exact staged or promoted SHA before publication/
);
});
it("fails when the immutable image tag is shortened", () => {
const shortenedWorkflow = VALID_WORKFLOW.replace('IMAGE_TAG="$SHA"', 'IMAGE_TAG="${SHA:0:7}"');
const result = verifyGiteaDeployWorkflow(createFixture(shortenedWorkflow));
assert.match(result.failures.join("\n"), /missing full SHA image tag generation/);
});
it("fails when legacy generic registry variables are used", () => {
const legacyWorkflow = VALID_WORKFLOW.replace(" IMAGE_NAMESPACE: toolctrl-labs\n", " CONTAINER_REGISTRY_NAMESPACE: toolctrl-labs\n");
const result = verifyGiteaDeployWorkflow(createFixture(legacyWorkflow));
assert.match(result.failures.join("\n"), /legacy generic registry variables are forbidden/);
});
it("fails when latest is generated", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace('--tag "$IMAGE:staging"', '--tag "$IMAGE:latest"')));
assert.match(result.failures.join("\n"), /mutable latest tag is forbidden/);
});
it("fails when staging aliases are pushed during individual target builds", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace('--tag "$IMAGE:$IMAGE_TAG" .', '--tag "$IMAGE:$IMAGE_TAG" --tag "$IMAGE:staging" .');
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging aliases must be retagged only after all staging target builds succeed/);
});
it("fails when a forbidden E2E command is present", () => {
const result = verifyGiteaDeployWorkflow(createFixture(`${VALID_WORKFLOW} - name: E2E\n run: pnpm exec playwright test\n`));
assert.match(result.failures.join("\n"), /forbidden E2E\/browser command found: run: pnpm exec playwright test/);
});
it("fails when registry login is softened at step level", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
" - name: Login to Gitea Registry\n run:",
" - name: Login to Gitea Registry\n continue-on-error: true\n run:"
)
)
);
assert.match(result.failures.join("\n"), /required registry login step must not use continue-on-error or always\(\)/);
});
it("fails when image publication disables errexit", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(VALID_WORKFLOW.replace(" run: |\n BUILD_PIDS=()\n", " run: |\n set +e\n BUILD_PIDS=()\n")),
);
assert.match(result.failures.join("\n"), /required Docker image publication step must not use shell success masking/);
});
it("fails when staging Docker builds query an unpublished legacy cache ref", () => {
const withLegacyBuildCache = VALID_WORKFLOW.replace(
' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n',
' IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"\n CACHE_REF="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/build-cache:$TARGET"\n'
).replace(
' --cache-from "type=registry,ref=$IMAGE:staging"',
' --cache-from "type=registry,ref=$CACHE_REF" --cache-from "type=registry,ref=$IMAGE:staging"'
);
const result = verifyGiteaDeployWorkflow(createFixture(withLegacyBuildCache));
assert.match(result.failures.join("\n"), /must not query legacy build-cache refs that the workflow never publishes/);
});
it("fails when staging Docker builds omit the portable inline cache", () => {
const withoutInlineCache = VALID_WORKFLOW.replace(' --cache-from "type=registry,ref=$IMAGE:staging" --cache-to "type=inline"', "");
const result = verifyGiteaDeployWorkflow(createFixture(withoutInlineCache));
assert.match(result.failures.join("\n"), /must import \$IMAGE:staging and export type=inline cache/);
});
it("fails when staging Docker builds use a separate registry cache export", () => {
const withRegistryCacheExport = VALID_WORKFLOW.replace(
' --cache-to "type=inline"',
' --cache-to "type=inline" --cache-to "type=registry,ref=$IMAGE:build-cache,mode=max"'
);
const result = verifyGiteaDeployWorkflow(createFixture(withRegistryCacheExport));
assert.match(result.failures.join("\n"), /must not use a separate type=registry cache-to export/);
});
it("fails when deploy builds fall back to the embedded Docker daemon builder", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(' --builder "$DEPLOY_BUILDER_NAME"', "");
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /staging image builds must pass --builder/);
});
it("fails when the isolated deploy BuildKit image is not digest pinned", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
"moby/buildkit:v0.31.2@sha256:2f5adac4ecd194d9f8c10b7b5d7bceb5186853db1b26e5abd3a657af0b7e26ec",
"moby/buildkit:latest"
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /deploy must pin moby\/buildkit:v0\.31\.2@sha256/);
});
it("fails when the bounded deploy builder cache is made run-scoped", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
"DEPLOY_BUILDER_NAME: hivegrid-deploy-v0312-gc1",
"DEPLOY_BUILDER_NAME: hivegrid-deploy-${{ gitea.run_id }}"
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /use persistent builder hivegrid-deploy-v0312-gc1/);
});
it("fails when fresh-client reuse skips host-container image validation", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
' if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1; then\n ACTUAL_BUILDKIT_IMAGE=',
' if docker buildx inspect "$DEPLOY_BUILDER_NAME" >/dev/null 2>&1; then\n ACTUAL_BUILDKIT_IMAGE='
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /reuse or create the isolated pinned BuildKit with image-drift validation/);
});
it("fails when deploy BuildKit can restart outside a workflow", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace('--driver-opt "restart-policy=no"', '--driver-opt "restart-policy=unless-stopped"');
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /reuse or create the isolated pinned BuildKit/);
});
it("fails when a later deploy does not quiesce an orphaned host builder", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
/ - name: Quiesce orphaned deploy BuildKit[\s\S]*? - name: Verify deploy wiring/,
" - name: Verify deploy wiring"
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /every deploy must quiesce the host builder before gates/);
});
it("fails when the running BuildKit config is not verified", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
"node scripts/ci/verify-deploy-buildkit-config.mjs scripts/ci/deploy-buildkitd.toml",
"node scripts/ci/unverified-buildkit-config.mjs scripts/ci/deploy-buildkitd.toml"
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /reuse or create the isolated pinned BuildKit with image-drift validation/);
});
it("fails when the running BuildKit config verifier is missing", () => {
const rootDir = createFixture();
fs.rmSync(path.join(rootDir, "scripts/ci/verify-deploy-buildkit-config.mjs"));
const result = verifyGiteaDeployWorkflow(rootDir);
assert.match(result.failures.join("\n"), /missing running builder config verifier/);
});
it("fails when isolated BuildKit does not require the registry host override", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
'if [ -z "${HIVEGRID_GITEA_HOST_IP:-}" ]; then',
'if [ -n "${HIVEGRID_GITEA_HOST_IP:-}" ]; then'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit appends without replacing a stale registry host mapping", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace('cat "$HOSTS_TMP" > /etc/hosts', 'cat "$HOSTS_TMP" > /tmp/unapplied-hosts');
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit accepts a stale mapping beside the exact registry host mapping", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace('"$BUILDKIT_HOST_COUNT" -ne 1', '"$BUILDKIT_HOST_COUNT" -lt 1');
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit no longer removes stale registry host tokens", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
"if (field > 1 && $field == host) {\n continue",
"if (false) {\n continue"
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit no longer resets the rewritten hosts line per record", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(' rewritten = ""\n for (field', " for (field");
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit moves the rewritten hosts reset outside the record block", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
' {\n rewritten = ""',
' BEGIN { rewritten = "" }\n {'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit moves the entire hosts rewrite into an AWK BEGIN block", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
' awk -v host="git.tool-ctrl.de" \'\n {',
' awk -v host="git.tool-ctrl.de" \'\n BEGIN {'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit hard-codes a passing registry host count", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
`END { print count + 0 }' /etc/hosts)"`,
`END { print 1 }' /etc/hosts)"`
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit counts a different hostname", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
'BUILDKIT_HOST_COUNT="$(awk -v host="git.tool-ctrl.de"',
'BUILDKIT_HOST_COUNT="$(awk -v host="localhost"'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit drops the validated-IP predicate", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace("'$1 == ip { for (field", "'1 == 1 { for (field");
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated BuildKit validates the IP only after writing its hosts file", () => {
const validationBlock = ` if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
`;
const validationIndex = VALID_WORKFLOW.lastIndexOf(validationBlock);
assert.notEqual(validationIndex, -1);
const withoutValidation =
VALID_WORKFLOW.slice(0, validationIndex) + VALID_WORKFLOW.slice(validationIndex + validationBlock.length);
const builderAppend = ` printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
`;
const appendIndex = withoutValidation.lastIndexOf(builderAppend);
assert.notEqual(appendIndex, -1);
const insertIndex = appendIndex + builderAppend.length;
const unsafeWorkflow =
withoutValidation.slice(0, insertIndex) + validationBlock + withoutValidation.slice(insertIndex);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when the missing BuildKit host override guard is pass-open", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
'echo "HIVEGRID_GITEA_HOST_IP is required for the isolated deploy builder"\n exit 1\n fi\n if ! HIVEGRID_GITEA_HOST_IP=',
'echo "HIVEGRID_GITEA_HOST_IP is required for the isolated deploy builder"\n :\n fi\n if ! HIVEGRID_GITEA_HOST_IP='
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when the invalid BuildKit host override guard is pass-open", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n exit 1\n fi\n docker exec',
'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n :\n fi\n docker exec'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when the BuildKit host count mismatch guard is pass-open", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
'echo "BuildKit registry host override did not converge to exactly one validated mapping"\n exit 1\n fi',
'echo "BuildKit registry host override did not converge to exactly one validated mapping"\n :\n fi'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must require, validate, replace, and verify exactly one git\.tool-ctrl\.de mapping/);
});
it("fails when isolated deploy BuildKit shutdown is not always-run", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
' - name: Stop isolated deploy BuildKit\n if: always()\n',
' - name: Stop isolated deploy BuildKit\n if: success()\n'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must be stopped by an always\(\) stage cleanup without deleting its cache/);
});
it("fails when isolated deploy BuildKit shutdown deletes the persistent cache", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
' docker inspect "$BUILDER_CONTAINER" >/dev/null\n docker stop "$BUILDER_CONTAINER"\n',
' docker inspect "$BUILDER_CONTAINER" >/dev/null\n docker buildx rm --force "$DEPLOY_BUILDER_NAME"\n'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /must be stopped by an always\(\) stage cleanup without deleting its cache/);
});
it("fails when isolated deploy BuildKit can exceed its shared-host disk bound", () => {
const rootDir = createFixture();
writeFile(rootDir, "scripts/ci/deploy-buildkitd.toml", '[worker.oci]\n gc = true\n maxUsedSpace = "300GB"\n');
const result = verifyGiteaDeployWorkflow(rootDir);
assert.match(result.failures.join("\n"), /must enforce history 86400s\/20 entries and OCI GC at 2GB reserved\/30GB max\/100GB free/);
});
it("fails when isolated deploy BuildKit history retention is unbounded", () => {
const rootDir = createFixture();
writeFile(
rootDir,
"scripts/ci/deploy-buildkitd.toml",
'[history]\n maxAge = 86400\n maxEntries = 2000\n\n[worker.oci]\n gc = true\n reservedSpace = "2GB"\n maxUsedSpace = "30GB"\n minFreeSpace = "100GB"\n max-parallelism = 12\n'
);
const result = verifyGiteaDeployWorkflow(rootDir);
assert.match(result.failures.join("\n"), /must enforce history 86400s\/20 entries/);
});
it("fails when staging Docker builds are not tracked as parallel jobs", () => {
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW.replace(" BUILD_PIDS=()\n", "")));
assert.match(
result.failures.join("\n"),
/staging Docker image builds must run web\/api\/worker in parallel and collect per-target failures before tagging/
);
});
it("fails when Vite build args are not scoped to the web Docker target", () => {
const withoutWebScopedViteArgs = VALID_WORKFLOW.replace(
/ WEB_BUILD_ARGS=\(\)[\s\S]*? docker buildx build --builder "\$DEPLOY_BUILDER_NAME" --push --target "\$TARGET"/,
' docker buildx build --builder "$DEPLOY_BUILDER_NAME" --push --target "$TARGET"'
);
const result = verifyGiteaDeployWorkflow(createFixture(withoutWebScopedViteArgs));
assert.match(result.failures.join("\n"), /Vite build args must be scoped to the web Docker target/);
});
it("fails when the public Hocuspocus URL is not passed into the web image build", () => {
const withoutHocuspocusBuildArg = VALID_WORKFLOW.replace(" VITE_HOCUSPOCUS_URL: \${{ vars.VITE_HOCUSPOCUS_URL }}\n", "").replace(
' --build-arg VITE_HOCUSPOCUS_URL="$VITE_HOCUSPOCUS_URL"',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(withoutHocuspocusBuildArg));
assert.match(result.failures.join("\n"), /web image build must map vars\.VITE_HOCUSPOCUS_URL and pass --build-arg VITE_HOCUSPOCUS_URL/);
});
it("fails when the staging one-shot migration step is missing", () => {
const withoutMigration = VALID_WORKFLOW.replace(
/ - name: Run staging one-shot database migration[\s\S]*? - name: Ensure Coolify staging worker groups/s,
" - name: Ensure Coolify staging worker groups"
);
const result = verifyGiteaDeployWorkflow(createFixture(withoutMigration));
assert.match(result.failures.join("\n"), /missing staging one-shot database migration step/);
});
it("fails when runtime metadata is not patched through the application API", () => {
const withoutRuntimePatch = VALID_WORKFLOW.replaceAll("--request PATCH", "--request GET");
const result = verifyGiteaDeployWorkflow(createFixture(withoutRuntimePatch));
assert.match(result.failures.join("\n"), /Coolify staging runtime config must enforce app\/worker RLS credentials/);
});
it("fails when the Coolify hook slots are not cleared", () => {
const withActiveHooks = VALID_COOLIFY_DEPLOY_HELPER.replace(
/pre_deployment_command: "", post_deployment_command: ""/,
'pre_deployment_command: "cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts", post_deployment_command: ""'
);
const result = verifyGiteaDeployWorkflow(createFixture(VALID_WORKFLOW, withActiveHooks));
assert.match(result.failures.join("\n"), /web surface release mutation must patch, reread, and compensate all three resources/);
});
it("fails when the staging one-shot migration omits DIRECT_URL", () => {
const withoutDirectUrl = VALID_WORKFLOW.replace(" -e DIRECT_URL \\\n", "");
const result = verifyGiteaDeployWorkflow(createFixture(withoutDirectUrl));
assert.match(result.failures.join("\n"), /staging migration must use only its step-scoped migrator secret/);
});
it("fails when a seed flag is forwarded to the staging migrator", () => {
const withSeedFlag = VALID_WORKFLOW.replace(" -e DIRECT_URL \\\n", " -e DIRECT_URL \\\n -e SEED_ONCE=true \\\n");
const result = verifyGiteaDeployWorkflow(createFixture(withSeedFlag));
assert.match(result.failures.join("\n"), /staging migration must use only its step-scoped migrator secret/);
});
it("fails when runtime resources disable RLS", () => {
const withRlsBypass = VALID_WORKFLOW.replaceAll(
' node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web',
' RLS_RUNTIME_MODE=off node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web'
);
const result = verifyGiteaDeployWorkflow(createFixture(withRlsBypass));
assert.match(result.failures.join("\n"), /must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app/);
});
it("fails when deploy attempts to forward a demo seed credential", () => {
const withManagedPassword = VALID_WORKFLOW.replace("migrate\n", "migrate\n SEED_DEMO_USER_PASSWORD=ci-managed\n");
const result = verifyGiteaDeployWorkflow(createFixture(withManagedPassword));
assert.match(result.failures.join("\n"), /must not create or overwrite the operator-only SEED_DEMO_USER_PASSWORD/);
});
it("fails when the staging one-shot migration runs after Coolify deploy", () => {
const migrationMatch = VALID_WORKFLOW.match(
/ - name: Run staging one-shot database migration[\s\S]*?(?= - name: Ensure Coolify staging worker groups)/
);
assert.ok(migrationMatch);
const migrationStep = migrationMatch[0];
const withoutMigration = VALID_WORKFLOW.replace(migrationStep, "");
const afterCoolify = withoutMigration.replace(
" - name: Promote staging images to production aliases\n",
`${migrationStep} - name: Promote staging images to production aliases\n`
);
const result = verifyGiteaDeployWorkflow(createFixture(afterCoolify));
assert.match(result.failures.join("\n"), /staging one-shot migration must run before Coolify deploy/);
});
it("fails when the Coolify staging health wait step is missing", () => {
const withoutWait = VALID_WORKFLOW.replace(
/ - name: Wait for Coolify staging health[\s\S]*? - name: Promote staging images to production aliases/s,
" - name: Promote staging images to production aliases"
);
const result = verifyGiteaDeployWorkflow(createFixture(withoutWait));
assert.match(result.failures.join("\n"), /missing Coolify staging health wait step/);
});
it("fails when staging web surfaces skip the live readiness endpoint", () => {
const missingReadiness = VALID_WORKFLOW.replace(/(- name: Wait for Coolify staging health[\s\S]*?)"\$HEALTH_URL\/readyz"/, '$1"$HEALTH_URL/healthz"');
const result = verifyGiteaDeployWorkflow(createFixture(missingReadiness));
assert.match(result.failures.join("\n"), /Coolify staging web health wait must require \/readyz status=ready with matching service, surface, and revision/);
});
it("fails when production web surfaces skip the live readiness endpoint", () => {
const missingReadiness = VALID_WORKFLOW.replace(/(- name: Wait for Coolify production health[\s\S]*?)"\$HEALTH_URL\/readyz"/, '$1"$HEALTH_URL/healthz"');
const result = verifyGiteaDeployWorkflow(createFixture(missingReadiness));
assert.match(
result.failures.join("\n"),
/Coolify production web health wait must require \/readyz status=ready with matching service, surface, and revision/
);
});
it("fails when staging readiness trusts the expected revision instead of parsing the live response", () => {
const spoofedRevision = VALID_WORKFLOW.replace(
/(- name: Wait for Coolify staging health[\s\S]*?)READINESS_REVISION="\$\(node -e[^\n]+/,
'$1READINESS_REVISION="$IMAGE_TAG"'
);
const result = verifyGiteaDeployWorkflow(createFixture(spoofedRevision));
assert.match(result.failures.join("\n"), /Coolify staging web health wait must require \/readyz status=ready with matching service, surface, and revision/);
});
it("fails when a web readiness proof prints a database role name", () => {
const unredactedReadiness = VALID_WORKFLOW.replace(
' echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS"',
' echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS"\n echo "hivegrid_portal"'
);
const result = verifyGiteaDeployWorkflow(createFixture(unredactedReadiness));
assert.match(result.failures.join("\n"), /Coolify staging readiness proof must not print database role names or readiness payloads/);
});
it("fails when staging health verification pass-opens a missing health URL", () => {
const passOpen = VALID_WORKFLOW.replace(
' HEALTH_URL="https://staging-web.example.invalid"',
' if [ -z "$HEALTH_URL" ]; then\n echo "$LABEL has no health URL; live revision probe skipped"\n return\n fi\n HEALTH_URL="https://staging-web.example.invalid"'
);
const result = verifyGiteaDeployWorkflow(createFixture(passOpen));
assert.match(result.failures.join("\n"), /Coolify staging health wait must not pass-open missing health URLs for HTTP resources/);
});
it("fails when production health verification pass-opens a missing health URL", () => {
const passOpen = VALID_WORKFLOW.replace(
' HEALTH_URL="https://production-web.example.invalid"',
' if [ -z "$HEALTH_URL" ]; then\n echo "$LABEL has no health URL; live revision probe skipped"\n return\n fi\n HEALTH_URL="https://production-web.example.invalid"'
);
const result = verifyGiteaDeployWorkflow(createFixture(passOpen));
assert.match(result.failures.join("\n"), /Coolify production health wait must not pass-open missing health URLs for HTTP resources/);
});
it("fails when staging health revision is not parsed from the live health response", () => {
const staleRevision = VALID_WORKFLOW.replace(
/HEALTH_REVISION="\$\(node -e 'const fs = require\("node:fs"\); const payload = JSON\.parse\(fs\.readFileSync\(process\.argv\[1\], "utf8"\)\); process\.stdout\.write\(payload\.revision \|\| ""\);' "\$HEALTH_JSON"\)"/,
'HEALTH_REVISION="$IMAGE_TAG"'
);
const result = verifyGiteaDeployWorkflow(createFixture(staleRevision));
assert.match(result.failures.join("\n"), /Coolify staging health wait must verify \/healthz revision against IMAGE_TAG/);
});
it("fails when production health revision is not parsed from the live health response", () => {
const staleRevision = VALID_WORKFLOW.replace(
/HEALTH_URL="https:\/\/production-web\.example\.invalid"[\s\S]*?if \[ "\$HEALTH_REVISION" != "\$IMAGE_TAG" \]; then\n exit 1\n fi/,
'HEALTH_URL="https://production-web.example.invalid"\n curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$APP_JSON"\n HEALTH_REVISION="$IMAGE_TAG"'
);
const result = verifyGiteaDeployWorkflow(createFixture(staleRevision));
assert.match(result.failures.join("\n"), /Coolify production health wait must verify \/healthz revision against IMAGE_TAG/);
});
it("fails when staging dependent health revision is not parsed from the live health response", () => {
const staleRevision = VALID_WORKFLOW.replace(
/HEALTH_URL="https:\/\/staging-api\.example\.invalid"[\s\S]*?if \[ "\$HEALTH_REVISION" != "\$IMAGE_TAG" \]; then\n exit 1\n fi/,
'HEALTH_URL="https://staging-api.example.invalid"\n curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$APP_JSON"\n HEALTH_REVISION="$IMAGE_TAG"'
);
const result = verifyGiteaDeployWorkflow(createFixture(staleRevision));
assert.match(result.failures.join("\n"), /Coolify staging dependent health wait must verify \/healthz revision against IMAGE_TAG/);
});
it("fails when production dependent health revision is not parsed from the live health response", () => {
const staleRevision = VALID_WORKFLOW.replace(
/HEALTH_URL="https:\/\/production-api\.example\.invalid"[\s\S]*?if \[ "\$HEALTH_REVISION" != "\$IMAGE_TAG" \]; then\n exit 1\n fi/,
'HEALTH_URL="https://production-api.example.invalid"\n curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$APP_JSON"\n HEALTH_REVISION="$IMAGE_TAG"'
);
const result = verifyGiteaDeployWorkflow(createFixture(staleRevision));
assert.match(result.failures.join("\n"), /Coolify production dependent health wait must verify \/healthz revision against IMAGE_TAG/);
});
it("fails when staging web health does not verify service identity", () => {
const missingServiceCheck = VALID_WORKFLOW.replace(' EXPECTED_SERVICE="hivegrid-portal"\n', "");
const result = verifyGiteaDeployWorkflow(createFixture(missingServiceCheck));
assert.match(result.failures.join("\n"), /Coolify staging web health must verify hivegrid-web\/core, hivegrid-admin\/admin, and hivegrid-portal\/portal/);
});
it("fails when staging dependent health does not verify API service identity", () => {
const missingServiceCheck = VALID_WORKFLOW.replace(' EXPECTED_SERVICE="${5:-api}"\n', "");
const result = verifyGiteaDeployWorkflow(createFixture(missingServiceCheck));
assert.match(result.failures.join("\n"), /Coolify staging dependent health wait must verify service=api/);
});
it("fails when staging dependent non-HTTP resources compare image tags to the full SHA", () => {
const staleAliasCheck = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-staging}"', 'EXPECTED_IMAGE_ALIAS="$IMAGE_TAG"');
const result = verifyGiteaDeployWorkflow(createFixture(staleAliasCheck));
assert.match(
result.failures.join("\n"),
/Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/
);
});
it("fails when staging dependent non-HTTP resources expect the production alias", () => {
const staleAlias = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-staging}"', 'EXPECTED_IMAGE_ALIAS="\${4:-main}"');
const result = verifyGiteaDeployWorkflow(createFixture(staleAlias));
assert.match(
result.failures.join("\n"),
/Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/
);
});
it("fails when production dependent non-HTTP resources compare image tags to the full SHA", () => {
const staleAliasCheck = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-main}"', 'EXPECTED_IMAGE_ALIAS="$IMAGE_TAG"');
const result = verifyGiteaDeployWorkflow(createFixture(staleAliasCheck));
assert.match(
result.failures.join("\n"),
/Coolify production dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/
);
});
it("fails when production dependent non-HTTP resources expect the staging alias", () => {
const staleAlias = VALID_WORKFLOW.replace('EXPECTED_IMAGE_ALIAS="\${4:-main}"', 'EXPECTED_IMAGE_ALIAS="\${4:-staging}"');
const result = verifyGiteaDeployWorkflow(createFixture(staleAlias));
assert.match(
result.failures.join("\n"),
/Coolify production dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/
);
});
it("fails when dependent health wait can HTTP-probe worker resources", () => {
const workerProbeRegression = VALID_WORKFLOW.replace(
/ WORKER_RESOURCE="\$\(node -e 'process\.stdout\.write\("false"\)'\)"[\s\S]*? if \[ -z "\$HEALTH_URL" \] && \[ "\$HTTP_RESOURCE" = "true" \]; then/,
' if [ -z "$HEALTH_URL" ] && [ "$HTTP_RESOURCE" = "true" ]; then'
);
const result = verifyGiteaDeployWorkflow(createFixture(workerProbeRegression));
assert.match(
result.failures.join("\n"),
/Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/
);
});
it("fails when dependent health wait omits the revision-bound worker acknowledgement", () => {
const missingWorkerAcknowledgement = VALID_WORKFLOW.replace(
' echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy"\n',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(missingWorkerAcknowledgement));
assert.match(
result.failures.join("\n"),
/Coolify staging dependent health wait must fail HTTP resources without health URLs and verify Hocuspocus\/worker\/non-HTTP revisions explicitly/
);
});
it("fails when dependent deployment waits do not aggregate background failures", () => {
const missingWaitAggregation = VALID_WORKFLOW.replace(
' for PID in $DEPLOYMENT_WAIT_PIDS; do\n wait "$PID"\n done',
' echo "deployment waits started"'
);
const result = verifyGiteaDeployWorkflow(createFixture(missingWaitAggregation));
assert.match(result.failures.join("\n"), /Coolify staging dependent deployment waits must fan out with PID aggregation/);
});
it("fails when staging dependent deployments are enqueued without a serialized deployment wait", () => {
const burstQueuedDeployments = VALID_WORKFLOW.replace(
` if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent deploy #$DEPLOYMENT_INDEX"; then
STAGING_DEPENDENT_TRIGGER_FAILED=true
fi
`,
""
);
const result = verifyGiteaDeployWorkflow(createFixture(burstQueuedDeployments));
assert.match(result.failures.join("\n"), /Coolify staging must wait for each dependent deployment before enqueueing the next resource/);
});
it("fails when production dependent deployments are enqueued without a serialized deployment wait", () => {
const burstQueuedDeployments = VALID_WORKFLOW.replace(
` if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent deploy #$DEPLOYMENT_INDEX"; then
PRODUCTION_DEPENDENT_TRIGGER_FAILED=true
fi
`,
""
);
const result = verifyGiteaDeployWorkflow(createFixture(burstQueuedDeployments));
assert.match(result.failures.join("\n"), /Coolify production must wait for each dependent deployment before enqueueing the next resource/);
});
it("fails when dependent resource health checks do not aggregate background failures", () => {
const missingResourceAggregation = VALID_WORKFLOW.replace(
' for PID in $RESOURCE_WAIT_PIDS; do\n wait "$PID"\n done',
' echo "resource health checks started"'
);
const result = verifyGiteaDeployWorkflow(createFixture(missingResourceAggregation));
assert.match(
result.failures.join("\n"),
/Coolify staging dependent health wait must fan out STAGING_DEPENDENT_RESOURCE_UUIDS one UUID per worker and aggregate failures/
);
});
it("fails when a Coolify trigger is pass-opened with shell control operators", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"',
'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")" || true'
)
)
);
assert.match(result.failures.join("\n"), /Coolify deploy trigger must not use shell success masking/);
});
it("fails when a later Coolify trigger is pass-opened", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
'PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"',
'PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")" || true'
)
)
);
assert.match(result.failures.join("\n"), /Coolify deploy trigger must not use shell success masking/);
});
it("fails when a Coolify trigger helper is removed", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"',
'STAGING_WEB_DEPLOYMENT_UUID="$STAGING_WEB_RESOURCE_UUID"'
)
)
);
assert.match(result.failures.join("\n"), /Coolify staging deploy must trigger only STAGING_WEB_RESOURCE_UUID first/);
});
it("fails when a Coolify trigger prints the raw response instead of capturing the deployment UUID", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
'STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"',
'node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID"'
)
)
);
assert.match(
result.failures.join("\n"),
/staging must run trigger\/wait for core < admin < portal without failure aggregation/,
);
});
it("fails when staging Coolify configuration falls back to production credentials", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(`${VALID_WORKFLOW}
- name: Unsafe staging fallback
run: |
SELECTED_COOLIFY_API_URL="\${COOLIFY_STAGING_API_URL:-$COOLIFY_API_URL}"
SELECTED_COOLIFY_TOKEN="\${COOLIFY_STAGING_TOKEN:-$COOLIFY_TOKEN}"
`)
);
assert.match(result.failures.join("\n"), /staging Coolify configuration must not fall back to production Coolify URL or token/);
});
it("fails when a staging Coolify step directly references production variables", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
"COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}",
"COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}\n COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}"
).replace('"$COOLIFY_STAGING_API_URL/deploy?uuid=$STAGING_WEB_RESOURCE_UUID"', '"$COOLIFY_API_URL/deploy?uuid=$STAGING_WEB_RESOURCE_UUID"')
)
);
assert.match(result.failures.join("\n"), /staging Coolify steps must not reference production Coolify URL or token variables/);
});
it("fails when a production Coolify step directly references staging variables", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
"COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}",
"COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}\n COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}"
).replace('"$COOLIFY_API_URL/deploy?uuid=$PRODUCTION_WEB_RESOURCE_UUID"', '"$COOLIFY_STAGING_API_URL/deploy?uuid=$PRODUCTION_WEB_RESOURCE_UUID"')
)
);
assert.match(result.failures.join("\n"), /production Coolify steps must not reference staging Coolify URL or token variables/);
});
it("fails when staging dependent Coolify resources are deployed as a comma-valued UUID", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
` DEPLOYMENT_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then
STAGING_DEPENDENT_DEPLOYMENT_UUIDS="\${STAGING_DEPENDENT_DEPLOYMENT_UUIDS:+$STAGING_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent deploy #$DEPLOYMENT_INDEX"; then
STAGING_DEPENDENT_TRIGGER_FAILED=true
fi
else
STAGING_DEPENDENT_TRIGGER_FAILED=true
fi
done`,
` DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_DEPENDENT_RESOURCE_UUIDS")"
STAGING_DEPENDENT_DEPLOYMENT_UUIDS="$DEPLOYMENT_UUID"`
)
)
);
assert.match(result.failures.join("\n"), /Coolify staging dependent resources must deploy one UUID per request/);
});
it("fails when production dependent Coolify resources are deployed as a comma-valued UUID", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
` DEPLOYMENT_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="\${PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS:+$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent deploy #$DEPLOYMENT_INDEX"; then
PRODUCTION_DEPENDENT_TRIGGER_FAILED=true
fi
else
PRODUCTION_DEPENDENT_TRIGGER_FAILED=true
fi
done`,
` DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS")"
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="$DEPLOYMENT_UUID"`
)
)
);
assert.match(result.failures.join("\n"), /Coolify production dependent resources must deploy one UUID per request/);
});
it("fails when production image promotion ends with an unconditional true", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
VALID_WORKFLOW.replace(
' docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"\n done\n',
' docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"\n done\n true\n'
)
)
);
assert.match(result.failures.join("\n"), /required production image promotion step must not use shell success masking/);
});
it("fails when a Coolify trigger is missing", () => {
const withoutCoolify = VALID_WORKFLOW.replace(
/ - name: Trigger Coolify staging deploy[\s\S]*? - name: Promote staging images/s,
" - name: Promote staging images"
);
const result = verifyGiteaDeployWorkflow(createFixture(withoutCoolify));
assert.match(
result.failures.join("\n"),
/Coolify trigger must use production and staging Coolify configuration slots|missing Coolify staging health wait step/
);
});
it("fails when api and worker Coolify resource coverage is missing", () => {
const webOnlyAssertions = ' node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web';
const webOnlyWorkflow = VALID_WORKFLOW.replaceAll(RUNTIME_DB_ASSERTIONS_FIXTURE, webOnlyAssertions);
const result = verifyGiteaDeployWorkflow(createFixture(webOnlyWorkflow));
assert.match(
result.failures.join("\n"),
/staging must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app, portal as hivegrid_portal, and every worker as hivegrid_worker/
);
assert.match(
result.failures.join("\n"),
/production must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app, portal as hivegrid_portal, and every worker as hivegrid_worker/
);
});
it("fails when the shared Web Push runtime preflight is missing", () => {
const withoutWebPushPreflight = VALID_WORKFLOW.replaceAll(
' node scripts/coolify-deploy.mjs assert-web-push-env "$WEB_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID"\n',
""
);
const result = verifyGiteaDeployWorkflow(createFixture(withoutWebPushPreflight));
assert.match(result.failures.join("\n"), /staging must verify matching web and webhook-worker VAPID configuration/);
assert.match(result.failures.join("\n"), /production must verify matching web and webhook-worker VAPID configuration/);
});
it("fails when the Authentik JIT-capable web surface is assigned to another runtime resource", () => {
const wrongSurfaces = VALID_WORKFLOW.replaceAll(
'assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web',
'assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app service'
).replaceAll('assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app service', 'assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app web');
const result = verifyGiteaDeployWorkflow(createFixture(wrongSurfaces));
assert.match(
result.failures.join("\n"),
/must verify core web\/admin\/API\/MCP\/Hocuspocus as hivegrid_app, portal as hivegrid_portal, and every worker as hivegrid_worker/
);
});
it("fails when the mail worker Coolify resource is missing", () => {
const withoutMailWorker = VALID_WORKFLOW.replace(/WORKER_MAIL_RESOURCE_UUID/g, "WORKER_REMOVED_RESOURCE_UUID")
.replace(
/\$WORKER_GENERAL_RESOURCE_UUID,\$WORKER_REMOVED_RESOURCE_UUID,\$WORKER_IMPORT_RESOURCE_UUID,\$WORKER_WEBHOOK_RESOURCE_UUID,\$WORKER_PLUGIN_RESOURCE_UUID/g,
"$WORKER_GENERAL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
)
.replace(/"\$WORKER_GENERAL_RESOURCE_UUID" "\$WORKER_REMOVED_RESOURCE_UUID"/g, '"$WORKER_GENERAL_RESOURCE_UUID"');
const result = verifyGiteaDeployWorkflow(createFixture(withoutMailWorker));
assert.match(result.failures.join("\n"), /Coolify staging image-tag patch must cover API, MCP, Hocuspocus, and every worker group/);
assert.match(result.failures.join("\n"), /Coolify production image-tag patch must cover API, MCP, Hocuspocus, and every worker group/);
});
it("fails when a literal Coolify URL is present", () => {
const literalUrlWorkflow = `${VALID_WORKFLOW}
- name: Unsafe literal Coolify URL
run: curl https://coolify.invalid/api/v1/applications
`;
const result = verifyGiteaDeployWorkflow(createFixture(literalUrlWorkflow));
assert.match(result.failures.join("\n"), /literal Coolify\/webhook URL is forbidden/);
});
it("fails when promote rebuilds instead of retagging staging images", () => {
const rebuildWorkflow = VALID_WORKFLOW.replace(
'docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"',
'docker buildx build --push --target "$TARGET" --tag "$IMAGE:main" .'
);
const result = verifyGiteaDeployWorkflow(createFixture(rebuildWorkflow));
assert.match(result.failures.join("\n"), /promote must preserve the verified full-SHA manifest digest in production aliases without rebuilding/);
});
it("fails when the production Sentry deploy marker uses the dispatcher SHA instead of the promoted image tag", () => {
const staleSentryWorkflow = VALID_WORKFLOW.replace('RELEASE_SHA="${IMAGE_TAG}"', 'RELEASE_SHA="${PROMOTE_SHA:-${SHA}}"');
const result = verifyGiteaDeployWorkflow(createFixture(staleSentryWorkflow));
assert.match(result.failures.join("\n"), /production Sentry deploy marker must release IMAGE_TAG for promote/);
});
it("fails when a literal secret-shaped value is present", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(`${VALID_WORKFLOW} - name: Bad token\n env:\n TOKEN: ghp_1234567890abcdef1234567890abcdef\n`)
);
assert.match(result.failures.join("\n"), /\.gitea\/workflows\/deploy\.yml:\d+: literal secret-shaped value is forbidden/);
});
it("fails when the internal host override uses the reserved GITEA_ prefix", () => {
const result = verifyGiteaDeployWorkflow(createFixture(`env:\n GITEA_HOST_IP: \${{ vars.GITEA_HOST_IP }}\n${VALID_WORKFLOW}`));
assert.match(result.failures.join("\n"), /GITEA_HOST_IP is forbidden because Gitea reserves the GITEA_ variable prefix/);
});
it("fails when the runner host override loses its own token validation", () => {
const validation =
'if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e \'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)\'; then';
const validationIndex = VALID_WORKFLOW.indexOf(validation);
assert.notEqual(validationIndex, -1);
const unsafeWorkflow =
VALID_WORKFLOW.slice(0, validationIndex) +
"if ! true; then" +
VALID_WORKFLOW.slice(validationIndex + validation.length);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/);
});
it("fails when the runner host validation guard is pass-open", () => {
const unsafeWorkflow = VALID_WORKFLOW.replace(
'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n exit 1\n fi\n printf',
'echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"\n :\n fi\n printf'
);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/);
});
it("fails when the runner validates its host override only after writing it", () => {
const validationBlock = ` if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
`;
const validationIndex = VALID_WORKFLOW.indexOf(validationBlock);
assert.notEqual(validationIndex, -1);
const withoutValidation =
VALID_WORKFLOW.slice(0, validationIndex) + VALID_WORKFLOW.slice(validationIndex + validationBlock.length);
const runnerAppend = ` printf '%s git.tool-ctrl.de\\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
`;
const appendIndex = withoutValidation.indexOf(runnerAppend);
assert.notEqual(appendIndex, -1);
const insertIndex = appendIndex + runnerAppend.length;
const unsafeWorkflow =
withoutValidation.slice(0, insertIndex) + validationBlock + withoutValidation.slice(insertIndex);
const result = verifyGiteaDeployWorkflow(createFixture(unsafeWorkflow));
assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/);
});
it("fails when the internal host override is written without token validation", () => {
const result = verifyGiteaDeployWorkflow(
createFixture(
`${VALID_WORKFLOW}\n host-smoke:\n runs-on: toolctrl-deploy\n steps:\n - name: Resolve internal host\n run: |\n if [ -n \"\${HIVEGRID_GITEA_HOST_IP:-}\" ]; then\n printf '%s git.tool-ctrl.de\\n' \"$HIVEGRID_GITEA_HOST_IP\" >> /etc/hosts\n fi\n`
)
);
assert.match(result.failures.join("\n"), /HIVEGRID_GITEA_HOST_IP must be validated before writing to \/etc\/hosts/);
});
it("CLI exits non-zero and prints path-specific output for an invalid workflow", () => {
const rootDir = createFixture(null);
const scriptPath = path.resolve("scripts/verify-gitea-deploy-workflow.mjs");
assert.throws(
() => execFileSync(process.execPath, [scriptPath], { cwd: rootDir, encoding: "utf8", stdio: "pipe" }),
(error) => {
assert.equal(error.status, 1);
assert.match(error.stderr, /^FAIL Gitea deploy workflow verification/);
assert.match(error.stderr, /FAIL \.gitea\/workflows\/deploy\.yml: missing required Gitea deploy workflow file/);
return true;
}
);
});
});
Reference in new issue View Git Blame Copy Permalink