#!/usr/bin/env node import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const REQUIRED_RUNTIME_DEPENDENCIES = ["bullmq", "ioredis", "pino", "prom-client"]; const FORBIDDEN_RUNTIME_DEPENDENCIES = ["@prisma/client", "pg", "prisma", "ssh2", "dockerode", "@aws-sdk/client-s3"]; const DEPLOY_TARGET_LOOP_PATTERN = /for TARGET in ([^;\n]+); do/gu; export function verifyPortalDomainControllerDeploy(rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")) { const passes = []; const failures = []; const check = (condition, success, failure) => { (condition ? passes : failures).push(condition ? success : failure); }; const read = (relativePath) => fs.readFileSync(path.join(rootDir, relativePath), "utf8"); const parse = (relativePath) => JSON.parse(read(relativePath)); let dockerfile; let deployWorkflow; let ciWorkflow; let dockerSmokeSuite; let productionClosureHelper; let coolifyHelper; let runtimeSmoke; let appManifest; let runtimeManifest; let controllerBuild; let deployHealthSource; try { dockerfile = read("Dockerfile"); deployWorkflow = read(".gitea/workflows/deploy.yml"); ciWorkflow = read(".gitea/workflows/ci.yml"); dockerSmokeSuite = read("scripts/ci/docker-smoke-suite.sh"); productionClosureHelper = read("scripts/ci/deploy-production-closures.sh"); coolifyHelper = read("scripts/coolify-portal-domain-controller.mjs"); runtimeSmoke = read("scripts/docker-portal-domain-controller-runtime-smoke.sh"); appManifest = parse("apps/portal-domain-controller/package.json"); runtimeManifest = parse("runtime/portal-domain-controller/package.json"); controllerBuild = read("apps/portal-domain-controller/scripts/build.mjs"); deployHealthSource = read("apps/portal-domain-controller/src/deploy-health.ts"); } catch (error) { return { passes, failures: [`controller deploy input missing or malformed: ${error instanceof Error ? error.message : String(error)}`] }; } check( appManifest.name === "@hivegrid/portal-domain-controller" && appManifest.private === true, "controller uses a private @hivegrid workspace package", "apps/portal-domain-controller must be the private @hivegrid/portal-domain-controller package" ); const runtimeDependencies = Object.keys(runtimeManifest.dependencies ?? {}).sort(); check( REQUIRED_RUNTIME_DEPENDENCIES.every((name) => runtimeDependencies.includes(name)), "runtime manifest declares the complete external dependency closure", `runtime/portal-domain-controller is missing required dependencies: ${REQUIRED_RUNTIME_DEPENDENCIES.filter((name) => !runtimeDependencies.includes(name)).join(", ")}` ); check( FORBIDDEN_RUNTIME_DEPENDENCIES.every((name) => !runtimeDependencies.includes(name)), "runtime manifest excludes database, SSH, Docker and storage clients", `controller runtime contains forbidden dependencies: ${FORBIDDEN_RUNTIME_DEPENDENCIES.filter((name) => runtimeDependencies.includes(name)).join(", ")}` ); const runtimeBody = dockerTargetBody(dockerfile, "portal-domain-controller"); const productionBundleBody = dockerTargetBody(dockerfile, "production-deps-bundle"); const controllerProductionDepsBody = dockerTargetBody(dockerfile, "portal-domain-controller-production-deps"); const productionBundleCommands = dockerRunCommands(productionBundleBody); const controllerProductionDepsCommands = dockerRunCommands(controllerProductionDepsBody); const expectedBundleCommand = "RUN --mount=type=cache,id=hivegrid-pnpm-store-v2,target=/pnpm/store,sharing=locked bash scripts/ci/deploy-production-closures.sh"; const expectedControllerDeploy = "pnpm --offline --config.inject-workspace-packages=true --filter @hivegrid/runtime-portal-domain-controller deploy --prod --no-optional /prod/portal-domain-controller"; const controllerDeployBody = normalizeCommand(shellFunctionBody(productionClosureHelper, "deploy_portal_controller")); const helperTasks = shellArrayItems(productionClosureHelper, "tasks"); const helperTargets = shellArrayItems(productionClosureHelper, "targets"); const runWaveBody = normalizeCommand(shellFunctionBody(productionClosureHelper, "run_wave")); const normalizedProductionClosureHelper = normalizeCommand(productionClosureHelper); const expectedControllerSbom = "RUN SBOM_TARGET=portal-domain-controller SBOM_ROOT_PATH=/prod/portal-domain-controller SBOM_NODE_MODULES_PATH=/prod/portal-domain-controller/node_modules SBOM_OUTPUT_PATH=/prod/portal-domain-controller/sbom-pnpm.json node /myapp/scripts/generate-sbom.mjs"; check(Boolean(runtimeBody), "Dockerfile defines the controller image", "Dockerfile is missing FROM base AS portal-domain-controller"); check( /^FROM\s+build\s+AS\s+production-deps-bundle\s*$/imu.test(dockerfile) && productionBundleCommands.length === 1 && productionBundleCommands[0] === expectedBundleCommand, "Docker build delegates the common locked warm store directly to the closure helper", "Dockerfile must build production-deps-bundle from build with only one locked hivegrid-pnpm-store-v2 invocation of scripts/ci/deploy-production-closures.sh" ); check( /^FROM\s+production-deps-bundle\s+AS\s+portal-domain-controller-production-deps\s*$/imu.test(dockerfile) && controllerDeployBody === expectedControllerDeploy && !/prune_runtime_metadata|find\s+[^\n]*node_modules[^\n]*(?:\*\.map|\*\.d\.ts)/u.test(productionClosureHelper), "controller production closure inherits the common bundle without metadata work under the store lock", "controller production dependency stage must inherit production-deps-bundle and the helper must deploy the exact filtered closure without locked metadata find/delete work" ); check( helperTasks.filter((task) => task === "deploy_portal_controller").length === 1 && helperTargets.filter((target) => target === "portal-domain-controller").length === 1 && /local -a pids=\(\)/u.test(runWaveBody) && /"\$task"\s*&/u.test(runWaveBody) && /pids\+=\("\$!"\)/u.test(runWaveBody) && /if wait "\$\{pids\[\$index\]\}"; then/u.test(runWaveBody) && /if \[\[ "\$status" -eq 0 \]\]; then status="\$child_status"/u.test(runWaveBody) && /return "\$status"/u.test(runWaveBody) && !hasFailOpenBypass(productionClosureHelper), "controller closure participates in every fully reaped concurrent wave and propagates failure", "controller production closure helper must include its task and target once, collect its PID in run_wave, await every PID and return the first failure" ); const initialWaveIndex = normalizedProductionClosureHelper.indexOf("run_wave offline-first || initial_status=$?"); const cleanupIndex = normalizedProductionClosureHelper.indexOf('for target in "${targets[@]}"; do rm -rf -- "/prod/${target}" done'); const fetchIndex = normalizedProductionClosureHelper.indexOf("if pnpm fetch --frozen-lockfile; then"); const retryWaveIndex = normalizedProductionClosureHelper.indexOf("run_wave after-fetch || retry_status=$?"); check( initialWaveIndex >= 0 && cleanupIndex > initialWaveIndex && fetchIndex > cleanupIndex && retryWaveIndex > fetchIndex && (normalizedProductionClosureHelper.match(/\brun_wave\s+(?:offline-first|after-fetch)\b/gu) ?? []).length === 2 && (normalizedProductionClosureHelper.match(/\bpnpm\s+fetch\s+--frozen-lockfile\b/gu) ?? []).length === 1 && (normalizedProductionClosureHelper.match(/\brm\s+-rf\b/gu) ?? []).length === 1 && /if \[\[ "\$initial_status" -eq 0 \]\]; then[\s\S]*exit 0[\s\S]*fi/u.test(normalizedProductionClosureHelper) && /else fetch_status=\$\?[\s\S]*exit "\$fetch_status"[\s\S]*fi/u.test(normalizedProductionClosureHelper) && /if \[\[ "\$retry_status" -ne 0 \]\]; then[\s\S]*exit "\$retry_status"[\s\S]*fi/u.test(normalizedProductionClosureHelper) && !/sentinel|sha256sum|lockfile[_-]?hash/iu.test(productionClosureHelper) && !/\brm\s+-rf\s+(?:--\s+)?["']?\/prod["']?(?:\s|;|$)/u.test(productionClosureHelper) && !/\brm\s+-rf\s+(?:--\s+)?["']?\/prod\/\*/u.test(productionClosureHelper) && !hasFailOpenBypass(productionClosureHelper), "controller closure uses the one-fetch exact-target fallback and one complete retry without a sentinel", "controller production closure helper must try offline first, clean only all five exact /prod targets, fetch once, retry one full wave and propagate fetch or retry failure without a sentinel" ); check( controllerProductionDepsCommands.length === 1 && controllerProductionDepsCommands[0] === expectedControllerSbom, "controller dependency stage generates only its target-specific installed-tree SBOM", "controller production dependency stage must contain only its target-specific installed-tree SBOM" ); check( runtimeBody?.includes( "COPY --exclude=**/*.map --exclude=**/*.d.ts --from=portal-domain-controller-production-deps /prod/portal-domain-controller/node_modules /myapp/node_modules" ), "controller final dependency COPY excludes maps and declarations", "controller final dependency COPY must use both --exclude=**/*.map and --exclude=**/*.d.ts" ); check( Boolean(runtimeBody) && /^USER\s+node\s*$/imu.test(runtimeBody) && /^CMD\s+\["node",\s*"apps\/portal-domain-controller\/dist\/index\.js"\]\s*$/imu.test(runtimeBody), "controller image runs the compiled app as the non-root node user", "controller image must run its compiled entry point as USER node" ); check( Boolean(runtimeBody) && /PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR="\/data\/coolify\/proxy\/dynamic\/hivegrid-portal-domains"/u.test(runtimeBody) && /PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE="\/data\/coolify\/proxy\/dynamic\/\.hivegrid-portal-domains\.reload"/u.test(runtimeBody) && !/ENV\s+PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE=/u.test(runtimeBody), "controller image fixes only the narrow Traefik paths and leaves routing environment-specific", "controller image must fix the dedicated config/sentinel paths without baking an environment-specific backend service" ); check( Boolean(runtimeBody) && /HEALTHCHECK\s+--interval=30s\b[^\n]*--start-interval=2s\b[^\n]*--retries=3\b[\s\S]*CMD node apps\/portal-domain-controller\/dist\/deploy-health\.js "\$APP_REVISION"/u.test(runtimeBody) && /src\/deploy-health\.ts/u.test(controllerBuild) && /\/healthz/u.test(deployHealthSource) && /\/readyz/u.test(deployHealthSource) && /health\.revision !== expectedRevision/u.test(deployHealthSource) && /\/myapp\/\.image-revision/u.test(deployHealthSource) && /hivegrid-portal-domain-controller/u.test(deployHealthSource) && /org\.opencontainers\.image\.sbom="\/myapp\/sbom-pnpm\.json"/u.test(runtimeBody), "controller image carries a fast-start compiled healthz, readyz, immutable revision and SBOM contract", "controller image must use a 2s start interval with the 30s/3 steady budget and run its compiled healthz, readyz and baked APP_REVISION probe with the target-specific SBOM" ); check( Boolean(runtimeBody) && !/(?:DATABASE_URL|DIRECT_URL|COOLIFY_TOKEN|CLOUDFLARE_API|DOCKER_HOST|SSH_PRIVATE_KEY)\s*=/u.test(runtimeBody), "controller image declares no privileged runtime credentials", "controller final image must not declare database, Coolify, Cloudflare, Docker or SSH credentials" ); const sourceRoot = path.join(rootDir, "apps/portal-domain-controller/src"); const productionSources = listFiles(sourceRoot).filter((file) => file.endsWith(".ts") && !file.endsWith(".test.ts")); const productionSourceEntries = productionSources.map((file) => ({ file, source: fs.readFileSync(file, "utf8") })); const productionSource = productionSourceEntries.map(({ source }) => source).join("\n"); const claimLockSource = productionSourceEntries.find(({ file }) => path.basename(file) === "claim-lock.ts")?.source ?? ""; const indexSource = productionSourceEntries.find(({ file }) => path.basename(file) === "index.ts")?.source ?? ""; const processExecutionOutsideClaimLock = productionSourceEntries.some( ({ file, source }) => path.basename(file) !== "claim-lock.ts" && /node:child_process/u.test(source) ); const reviewedFlockBoundary = /^import \{ spawn \} from "node:child_process";$/mu.test(claimLockSource) && /^const FLOCK_BINARY = "\/usr\/bin\/flock";$/mu.test(claimLockSource) && /spawn\(binary, \["--exclusive", "--wait", timeoutSeconds, "3"\], \{\s*stdio: \["ignore", "ignore", "ignore", fd\],\s*\}\)/u.test(claimLockSource) && !/\b(?:exec|execFile|fork|spawnSync)\s*\(/u.test(claimLockSource) && !/\bshell\s*:/u.test(claimLockSource); check( !/(?:from\s+["'](?:@hivegrid\/db|@prisma\/client|pg|ssh2|dockerode))/u.test(productionSource) && !processExecutionOutsideClaimLock && reviewedFlockBoundary, "controller source has no database, SSH or Docker client boundary and process execution is limited to reviewed /usr/bin/flock fencing", "controller source imports a forbidden database, SSH, Docker or non-flock process-execution capability" ); check( Boolean(runtimeBody) && /apt-get install -y --no-install-recommends util-linux/u.test(runtimeBody), "controller image installs the kernel flock binary explicitly", "controller final image must install util-linux for the reviewed /usr/bin/flock fence" ); check( /createHmac/u.test(productionSource) && /timingSafeEqual/u.test(productionSource) && /contentHash/u.test(productionSource), "controller queue protocol is content-hashed and HMAC authenticated", "controller contracts must bind a content hash to a timing-safe HMAC signature" ); check( /O_NOFOLLOW/u.test(productionSource) && /await rename\(/u.test(productionSource) && /await handle\.sync\(\)/u.test(productionSource), "Traefik writes reject symlinks and use fsync plus atomic rename", "Traefik writer must use O_NOFOLLOW, fsync and same-directory atomic rename" ); check( /normalizePortalDomainBackendService/u.test(productionSource) && /PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE/u.test(productionSource) && /service:\s*\$\{JSON\.stringify\(backendService\)\}/u.test(productionSource) && /Host\(\\`\$\{hostname\}\\`\)/u.test(productionSource), "Traefik renderer uses an exact Host matcher and provider-qualified environment routing", "Traefik renderer must use exact Host() and a validated provider-qualified backend service" ); const persistedConfigReconcileIndex = indexSource.indexOf("await writer.reconcilePersistedDomainConfigs()"); const queueStartupIndex = indexSource.indexOf("await createDomainQueueRuntime("); check( /service: "noop@internal"/u.test(productionSource) && /"418": 404/u.test(productionSource) && /tls: \{\}/u.test(productionSource) && /Math\.max\(1, Buffer\.byteLength\(exactHostRule, "utf8"\) - 1\)/u.test(productionSource) && /let expected = tombstone/u.test(productionSource) && /candidate\.state\.action !== "deprovision"/u.test(productionSource) && /if \(await fileMatches\(configPath, active\)\) continue/u.test(productionSource) && /if \(await fileMatches\(configPath, legacyActive\)\) expected = active/u.test(productionSource) && /renderTraefikDomainConfigForKey/u.test(productionSource) && /if \(persisted\.length > 0\) await this\.#triggerReload\(guard\)/u.test(productionSource) && persistedConfigReconcileIndex >= 0 && queueStartupIndex > persistedConfigReconcileIndex, "controller renders rollback-compatible exact-host backend-free 404 tombstones and safely reconciles every persisted route before queue startup", "controller must render the reviewed noop 418-to-404 TLS tombstone below legacy exact-rule priority, trust only known active config, reconcile deprovisioned states, and replay one reload before consuming commands" ); const shutdownQueueCloseIndex = indexSource.lastIndexOf("await queue.close()"); const shutdownClaimLockCloseIndex = indexSource.lastIndexOf("await claimLock.close()"); const shutdownHttpCloseIndex = indexSource.lastIndexOf("await http.close()"); check( shutdownQueueCloseIndex >= 0 && shutdownClaimLockCloseIndex > shutdownQueueCloseIndex && shutdownHttpCloseIndex > shutdownClaimLockCloseIndex, "controller shutdown removes mesh health only after queue and claim-lock drainage", "controller shutdown must close queue, then claim lock, then the mesh health server" ); check( /PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME/u.test(productionSource) && /#openSafeReloadTrigger/u.test(productionSource) && /handle\.truncate\(0\)/u.test(productionSource) && /handle\.sync\(\)/u.test(productionSource), "Traefik mutations touch the narrow watched-root sentinel through a safe file handle", "Traefik mutations must safely truncate, write and fsync the dedicated reload sentinel" ); check( coolifyHelper.includes("/data/coolify/proxy/dynamic/hivegrid-portal-domains") && coolifyHelper.includes("/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload") && coolifyHelper.includes("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE") && coolifyHelper.includes("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT") && coolifyHelper.includes("PORTAL_CUSTOM_DOMAIN_CNAME_TARGET") && coolifyHelper.includes('staging: "staging-domains.portal.hivegrid.de"') && coolifyHelper.includes('production: "domains.portal.hivegrid.de"') && coolifyHelper.includes('staging: "10.42.0.3:19120:8080"') && coolifyHelper.includes('production: "10.42.0.8:19120:8080"') && coolifyHelper.includes("ports_mappings") && coolifyHelper.includes("PORTAL_DOMAIN_ORCHESTRATION_REQUIRED") && coolifyHelper.includes("CONTROLLER_ALLOWED_ENV_KEYS") && coolifyHelper.includes("file_storages") && coolifyHelper.includes("persistent_storages") && coolifyHelper.includes("/storages") && coolifyHelper.includes("timingSafeEqual") && coolifyHelper.includes("running:healthy") && coolifyHelper.includes("custom_healthcheck_found") && coolifyHelper.includes("apps/portal-domain-controller/dist/deploy-health.js") && coolifyHelper.includes('health_check_type: "cmd"') && coolifyHelper.includes('custom_healthcheck_found: "false"') && /await coolify\(`\$\{resourcePath\}\/stop`, \{ method: "POST" \}\);/u.test(coolifyHelper) && !/if \(!STOPPED_RUNTIME_STATUSES\.test\([^)]*controller\?\.status/u.test(coolifyHelper) && coolifyHelper.includes("STOPPED_RUNTIME_STATUSES") && coolifyHelper.includes("controllerHealthUrlForEnvironment") && coolifyHelper.includes("probeControllerHealth") && coolifyHelper.includes("beforeStop.reachable") && coolifyHelper.includes("beforeStop.revisionBound") && coolifyHelper.includes("!afterStop.reachable") && coolifyHelper.includes("CONTROLLER_DRAIN_CONFIRMATIONS") && coolifyHelper.includes("CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 31_000") && coolifyHelper.includes("performance.now() - drainEvidenceSinceMs >= requiredDrainFenceMs") && coolifyHelper.includes("snapshotRelease") && coolifyHelper.includes("FULL_GIT_SHA_PATTERN"), "Coolify helper fail-closes live mount shapes, mesh-only metrics, least-privilege env, CNAME, HMAC, worker drain, revision snapshot, command health and full-SHA release contracts", "Coolify helper must enforce live narrow mounts, mesh-only metrics, least-privilege env, matching web/controller CNAME, worker-general HMAC, revision-bound stop plus continuous legacy-safe drain, revision snapshot, command health and full-SHA pinning" ); const ciJob = yamlJobBody(ciWorkflow, "portal-domain-controller"); const ciCommands = [ "pnpm --filter @hivegrid/portal-domain-controller typecheck", "pnpm --filter @hivegrid/portal-domain-controller test", "pnpm --filter @hivegrid/portal-domain-controller build", "pnpm run verify:runtime-closures -- --runtime=portal-domain-controller", "pnpm run verify:portal-domain-controller-deploy", ]; check( Boolean(ciJob) && ciCommands.every((command) => ciJob.includes(command)), "controller CI job typechecks, tests, builds and verifies its runtime/deploy contracts", "Gitea CI controller job is missing a required focused command" ); check( ciWorkflow.includes("bash scripts/ci/docker-smoke-suite.sh all") && /PORTAL_CONTROLLER_IMAGE="hivegrid-portal-domain-controller:smoke-\$\{DOCKER_SMOKE_RUN_ID\}"/u.test(dockerSmokeSuite) && /build_root_target portal-domain-controller "\$PORTAL_CONTROLLER_IMAGE"/u.test(dockerSmokeSuite) && /bash scripts\/docker-portal-domain-controller-runtime-smoke\.sh "\$PORTAL_CONTROLLER_IMAGE"/u.test(dockerSmokeSuite), "Docker smoke builds and exercises the controller image", "Gitea Docker smoke must build and run the portal-domain controller image" ); const smokeCleanupIndex = runtimeSmoke.indexOf("docker_smoke_install_cleanup_traps cleanup\ncleanup"); const smokeReloadVolumeIndex = runtimeSmoke.indexOf('docker volume create "$RELOAD_TRIGGER_VOLUME"'); check( smokeCleanupIndex >= 0 && smokeReloadVolumeIndex > smokeCleanupIndex && runtimeSmoke.includes('-v "$RELOAD_TRIGGER_VOLUME:/sentinel"') && runtimeSmoke.includes('fs.writeFileSync(file, "ready\\n", { mode: 0o600, flag: "wx" })') && runtimeSmoke.includes('-v "$CONFIG_VOLUME:/data/coolify/proxy/dynamic/hivegrid-portal-domains"') && runtimeSmoke.includes( '--mount "type=volume,source=$RELOAD_TRIGGER_VOLUME,destination=/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload,volume-subpath=.hivegrid-portal-domains.reload"' ) && runtimeSmoke.includes('-e PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE="hivegrid-portal@docker"') && runtimeSmoke.includes('-e PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT="https"') && runtimeSmoke.includes('node apps/portal-domain-controller/dist/deploy-health.js "$EXPECTED_REVISION"') && runtimeSmoke.includes('docker_smoke_expect_docker_exec_rejection "portal-controller:reject-foreign-revision"') && !runtimeSmoke.includes("$SMOKE_TEMP_DIR:/data/coolify/proxy/dynamic") && !runtimeSmoke.includes("$RELOAD_TRIGGER_FILE:/data/coolify/proxy/dynamic"), "Docker smoke creates a daemon-visible volume sentinel, mounts it narrowly and proves the revision-bound health CLI", "Docker smoke must create the sibling reload sentinel in a daemon-visible volume after initial cleanup, mount only the narrow file subpath, set validated routing and prove the health CLI" ); const targetLoops = [...deployWorkflow.matchAll(DEPLOY_TARGET_LOOP_PATTERN)].map((match) => match[1]?.trim().split(/\s+/u) ?? []); check( targetLoops.length >= 5 && targetLoops.every((targets) => targets.includes("portal-domain-controller")), "staging provenance, build, signing and alias loops include the controller image", "every deploy image-target loop must include portal-domain-controller" ); check( deployWorkflow.includes("node scripts/verify-portal-domain-controller-deploy.mjs"), "deploy preflight runs the controller-specific verifier", "deploy preflight must run scripts/verify-portal-domain-controller-deploy.mjs" ); check( deployWorkflow.includes("node --check scripts/coolify-portal-domain-controller.mjs") && deployWorkflow.includes("COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID: ${{ vars.COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID }}") && !deployWorkflow.includes("COOLIFY_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID") && !deployWorkflow.includes("PRODUCTION_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"), "deploy preflight requires the explicit staging controller resource without activating production", "deploy workflow must syntax-check the helper, source the staging controller UUID and leave production inactive until its resource is provisioned" ); check( countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs assert-runtime") === 1 && countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs snapshot-release") === 1 && countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs stop-and-drain") === 1 && countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs set-release") === 2 && countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs assert-release") === 2, "staging preflights, snapshots, drains, pins, verifies and can recover the controller release", "the staging rollout must run controller assert-runtime, snapshot-release and stop-and-drain once plus set-release/assert-release for recovery and acceptance" ); const environmentPinnedCommands = ["assert-runtime", "snapshot-release", "stop-and-drain", "set-release", "assert-release"]; check( environmentPinnedCommands.every((command) => { const expected = new Set(["set-release", "assert-release"]).has(command) ? ["staging", "staging"] : ["staging"]; return JSON.stringify(controllerInvocationEnvironments(deployWorkflow, command)) === JSON.stringify(expected); }), "controller preflight, release mutation and release proof pin the exact staging CNAME environment", "every active controller command must take staging as its explicit fail-closed first argument" ); check( deployWorkflow.includes('coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"') && /coolify-deploy\.mjs wait\s+\\?\s*"\$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"/u.test(deployWorkflow) && !deployWorkflow.includes('coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"') && !deployWorkflow.includes('coolify-deploy.mjs wait "$PRODUCTION_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"'), "staging controller deploy uses an explicit trigger and wait chain without production activation", "the staging controller must be triggered and waited independently while production remains inactive" ); const webReleaseProofIndex = deployWorkflow.indexOf("coolify-deploy.mjs assert-web-surface-release"); const controllerDrainIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs stop-and-drain"); const webTriggerIndex = deployWorkflow.indexOf('coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID"'); const workerGeneralProofIndex = deployWorkflow.indexOf('"$STAGING_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general"'); const dependentHealthGateIndex = deployWorkflow.indexOf("One or more Coolify staging dependent resources failed health checks"); const controllerReleaseIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs set-release"); const controllerTriggerIndex = deployWorkflow.indexOf( 'coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"', controllerReleaseIndex ); check( controllerDrainIndex >= 0 && webTriggerIndex > controllerDrainIndex && webReleaseProofIndex > webTriggerIndex && workerGeneralProofIndex > webReleaseProofIndex && dependentHealthGateIndex > workerGeneralProofIndex && controllerReleaseIndex > dependentHealthGateIndex && controllerTriggerIndex > controllerReleaseIndex, "staging drains the old controller before web cutover and starts it only after the paired worker-general revision is healthy", "the staging rollout must drain the old controller before web cutover, prove both web surfaces and worker-general, then pin and start the exact controller revision" ); const snapshotIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs snapshot-release"); const recoveryArmedIndex = deployWorkflow.indexOf("STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true"); const controllerAcceptedIndex = deployWorkflow.indexOf("STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true"); const normalControllerProofIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs assert-release", controllerReleaseIndex); const recoveryStepIndex = deployWorkflow.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure"); const recoveryReleaseIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs set-release", recoveryStepIndex); const recoveryTriggerIndex = deployWorkflow.indexOf('coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"', recoveryStepIndex); const recoveryWaitIndex = deployWorkflow.indexOf('coolify-deploy.mjs wait', recoveryStepIndex); const recoveryProofIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs assert-release", recoveryStepIndex); const nextStepIndex = deployWorkflow.indexOf("\n - name:", recoveryStepIndex + 1); const recoveryStep = recoveryStepIndex >= 0 ? deployWorkflow.slice(recoveryStepIndex, nextStepIndex >= 0 ? nextStepIndex : undefined) : ""; const recoveryReleaseBlock = recoveryReleaseIndex >= 0 && recoveryTriggerIndex > recoveryReleaseIndex ? deployWorkflow.slice(recoveryReleaseIndex, recoveryTriggerIndex) : ""; check( snapshotIndex >= 0 && snapshotIndex < controllerDrainIndex && recoveryArmedIndex > snapshotIndex && recoveryArmedIndex < controllerDrainIndex && normalControllerProofIndex > controllerTriggerIndex && controllerAcceptedIndex > normalControllerProofIndex && recoveryStepIndex > controllerAcceptedIndex && recoveryReleaseIndex > recoveryStepIndex && recoveryTriggerIndex > recoveryReleaseIndex && recoveryWaitIndex > recoveryTriggerIndex && recoveryProofIndex > recoveryWaitIndex && /^Restore previous Coolify staging portal-domain-controller release on rollout failure\s*$/mu.test(recoveryStep) && /^\s*if:\s*always\(\)\s*$/mu.test(recoveryStep) && /DEPLOY_ACTION" != "stage/.test(recoveryStep) && /STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED:-false/.test(recoveryStep) && /STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED:-false/.test(recoveryStep) && /"\$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"/.test(recoveryReleaseBlock) && !/"\$IMAGE_TAG"/.test(recoveryReleaseBlock) && /coolify-deploy\.mjs\s+wait\s+\\?\s*"\$RECOVERY_DEPLOYMENT_UUID"/.test(recoveryStep) && !/^\s*continue-on-error:\s*true\s*$/mu.test(recoveryStep) && !/\bset\s+(?:\+e|\+o\s+errexit)\b/.test(recoveryStep) && !/(^|[^|])\|\|\s*(?:true|:|exit\s+0)(?=[\s;)"']|$)/m.test(recoveryStep), "staging always restores the validated pre-drain controller release unless the new controller release is accepted", "staging must arm before drain, accept only after controller proof, then run an always-guarded previous-release set/trigger/wait/assert recovery" ); return { passes, failures }; } export function formatPortalDomainControllerDeployResult(result) { return [ ...result.passes.map((message) => `PASS ${message}`), ...result.failures.map((message) => `FAIL ${message}`), `Summary: ${result.passes.length} passed, ${result.failures.length} failed`, ].join("\n"); } function dockerTargetBody(dockerfile, target) { const match = new RegExp(`^FROM\\s+[^\\n]+\\s+AS\\s+${target}\\s*$`, "imu").exec(dockerfile); if (!match || match.index === undefined) return ""; const remainder = dockerfile.slice(match.index + match[0].length); const nextStage = remainder.search(/^FROM\s+/imu); return nextStage >= 0 ? remainder.slice(0, nextStage) : remainder; } function dockerRunCommands(stageBody) { return stageBody .replace(/\\\r?\n\s*/gu, " ") .split(/\r?\n/u) .map((line) => normalizeCommand(line)) .filter((line) => line.startsWith("RUN ")); } function shellFunctionBody(source, functionName) { const match = new RegExp(`^${escapeRegExp(functionName)}\\(\\)\\s*\\{\\s*$`, "mu").exec(source); if (!match || match.index === undefined) return ""; const remainder = source.slice(match.index + match[0].length); const closingBrace = remainder.search(/^\}\s*$/mu); return closingBrace >= 0 ? remainder.slice(0, closingBrace) : ""; } function shellArrayItems(source, variableName) { const match = new RegExp(`^${escapeRegExp(variableName)}=\\(([^)]*)\\)\\s*$`, "mu").exec(source); return match?.[1]?.trim().split(/\s+/u).filter(Boolean) ?? []; } function normalizeCommand(command) { return command.replace(/\\\r?\n\s*/gu, " ").replace(/\s+/gu, " ").trim(); } function hasFailOpenBypass(source) { return /\bset\s+\+e\b|\|\|\s*(?:true|:|exit\s+0)(?=[\s;)}"']|$)|;\s*(?:true|exit\s+0)(?=[\s;)}"']|$)/mu.test(source); } function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } function yamlJobBody(workflow, jobName) { const match = new RegExp(`^ ${jobName}:\\s*$`, "mu").exec(workflow); if (!match || match.index === undefined) return ""; const remainder = workflow.slice(match.index + match[0].length); const nextJob = remainder.search(/^ [A-Za-z0-9_-]+:\s*$/mu); return nextJob >= 0 ? remainder.slice(0, nextJob) : remainder; } function listFiles(root) { if (!fs.existsSync(root)) return []; return fs.readdirSync(root, { withFileTypes: true }).flatMap((entry) => { const absolute = path.join(root, entry.name); return entry.isDirectory() ? listFiles(absolute) : [absolute]; }); } function countOccurrences(value, search) { return value.split(search).length - 1; } function controllerInvocationEnvironments(workflow, command) { const lines = workflow.split(/\r?\n/u); const marker = `coolify-portal-domain-controller.mjs ${command}`; return lines.flatMap((line, index) => { if (!line.includes(marker)) return []; const nextLine = lines[index + 1]?.trim().replace(/\\$/u, "").trim(); return nextLine ? [nextLine] : []; }); } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const result = verifyPortalDomainControllerDeploy(); console.log(formatPortalDomainControllerDeployResult(result)); if (result.failures.length > 0) process.exitCode = 1; }