// @vitest-environment node import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, describe, it } from "vitest"; import { verifyPortalDomainControllerDeploy } from "./verify-portal-domain-controller-deploy.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const fixtures = []; const requiredPaths = [ "Dockerfile", ".gitea/workflows/ci.yml", ".gitea/workflows/deploy.yml", "scripts/ci/deploy-production-closures.sh", "scripts/ci/docker-smoke-suite.sh", "scripts/coolify-portal-domain-controller.mjs", "scripts/docker-portal-domain-controller-runtime-smoke.sh", "apps/portal-domain-controller/package.json", "apps/portal-domain-controller/scripts/build.mjs", "apps/portal-domain-controller/src", "runtime/portal-domain-controller/package.json", ]; afterEach(() => { for (const fixture of fixtures.splice(0)) fs.rmSync(fixture, { recursive: true, force: true }); }); describe("portal-domain controller deploy verifier", () => { it("accepts the reviewed repository contract", () => { const result = verifyPortalDomainControllerDeploy(repoRoot); assert.deepEqual(result.failures, []); }); it("fails closed when the controller bypasses the common bundle or exact helper deploy", () => { const fixture = createFixture(); const dockerfilePath = path.join(fixture, "Dockerfile"); fs.writeFileSync( dockerfilePath, fs .readFileSync(dockerfilePath, "utf8") .replace("bash scripts/ci/deploy-production-closures.sh", "true") .replace( "FROM production-deps-bundle AS portal-domain-controller-production-deps", "FROM build AS portal-domain-controller-production-deps" ) ); const helperPath = path.join(fixture, "scripts/ci/deploy-production-closures.sh"); fs.writeFileSync( helperPath, fs .readFileSync(helperPath, "utf8") .replace("--filter @hivegrid/runtime-portal-domain-controller deploy --prod --no-optional", "--filter @hivegrid/runtime-worker deploy --prod") .replace("deploy_mcp deploy_portal_controller deploy_worker", "deploy_mcp deploy_worker") .replace("mcp portal-domain-controller worker", "mcp worker") .replace('return "$status"', "return 0") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /only one locked hivegrid-pnpm-store-v2 invocation of scripts\/ci\/deploy-production-closures\.sh/); assert.match(result.failures.join("\n"), /inherit production-deps-bundle and the helper must deploy the exact filtered closure without locked metadata/); assert.match(result.failures.join("\n"), /include its task and target once, collect its PID in run_wave/); }); it("fails closed when the warm-store fallback is broad, repeated or sentinel-based", () => { const fixture = createFixture(); const helperPath = path.join(fixture, "scripts/ci/deploy-production-closures.sh"); fs.writeFileSync( helperPath, fs .readFileSync(helperPath, "utf8") .replace( 'for target in "${targets[@]}"; do\n rm -rf -- "/prod/${target}"\ndone', "rm -rf -- /prod/*", ) .replace("if pnpm fetch --frozen-lockfile; then", "lockfile_hash=sentinel\npnpm fetch --frozen-lockfile\nif pnpm fetch --frozen-lockfile || true; then"), ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /clean only all five exact \/prod targets, fetch once, retry one full wave/); }); it("fails closed when controller metadata exclusion or SBOM generation becomes best-effort", () => { const fixture = createFixture(); const dockerfilePath = path.join(fixture, "Dockerfile"); fs.writeFileSync( dockerfilePath, fs .readFileSync(dockerfilePath, "utf8") .replace( "COPY --exclude=**/*.map --exclude=**/*.d.ts --from=portal-domain-controller-production-deps", "COPY --from=portal-domain-controller-production-deps" ) .replace( "node /myapp/scripts/generate-sbom.mjs\n\nFROM production-deps-bundle AS worker-production-deps", "node /myapp/scripts/generate-sbom.mjs || true\n\nFROM production-deps-bundle AS worker-production-deps" ) ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /final dependency COPY must use both --exclude/); assert.match(result.failures.join("\n"), /contain only its target-specific installed-tree SBOM/); }); it("fails closed when the controller HEALTHCHECK polling cadence regresses", () => { const fixture = createFixture(); const dockerfilePath = path.join(fixture, "Dockerfile"); fs.writeFileSync( dockerfilePath, fs .readFileSync(dockerfilePath, "utf8") .replace( "HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --start-interval=2s --retries=3 \\\n CMD node apps/portal-domain-controller/dist/deploy-health.js", "HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 \\\n CMD node apps/portal-domain-controller/dist/deploy-health.js" ) ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /use a 2s start interval with the 30s\/3 steady budget/); }); it("fails closed when provider-qualified routing is bypassed or image publication omits the controller", () => { const fixture = createFixture(); const traefikPath = path.join(fixture, "apps/portal-domain-controller/src/traefik.ts"); fs.writeFileSync( traefikPath, fs.readFileSync(traefikPath, "utf8").replace("` service: ${JSON.stringify(backendService)}`", '` url: "http://attacker:8080"`') ); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); fs.writeFileSync(workflowPath, fs.readFileSync(workflowPath, "utf8").replaceAll(" hocuspocus portal-domain-controller; do", " hocuspocus; do")); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /provider-qualified backend service/); assert.match(result.failures.join("\n"), /image-target loop/); }); it("fails closed when persisted route reconciliation no longer precedes queue startup", () => { const fixture = createFixture(); const indexPath = path.join(fixture, "apps/portal-domain-controller/src/index.ts"); fs.writeFileSync( indexPath, fs .readFileSync(indexPath, "utf8") .replace("await writer.reconcilePersistedDomainConfigs();", "void writer.reconcilePersistedDomainConfigs();") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /trust only known active config/); }); it("fails closed when startup reconciliation trusts active state without known config", () => { const fixture = createFixture(); const traefikPath = path.join(fixture, "apps/portal-domain-controller/src/traefik.ts"); fs.writeFileSync( traefikPath, fs .readFileSync(traefikPath, "utf8") .replace("if (await fileMatches(configPath, legacyActive)) expected = active;", "expected = active;") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /trust only known active config/); }); it("fails closed when tombstone priority can outrank a rollback active route", () => { const fixture = createFixture(); const traefikPath = path.join(fixture, "apps/portal-domain-controller/src/traefik.ts"); fs.writeFileSync( traefikPath, fs .readFileSync(traefikPath, "utf8") .replace('Buffer.byteLength(exactHostRule, "utf8") - 1', 'Buffer.byteLength(exactHostRule, "utf8") + 1') ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /below legacy exact-rule priority/); }); it("fails closed when workflow identity or helper runtime checks are removed", () => { const fixture = createFixture(); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); fs.writeFileSync( workflowPath, fs.readFileSync(workflowPath, "utf8").replaceAll("COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID", "REMOVED_STAGING_CONTROLLER_UUID") ); const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs"); fs.writeFileSync( helperPath, fs .readFileSync(helperPath, "utf8") .replaceAll("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE", "REMOVED_BACKEND_SERVICE") .replaceAll("/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload", "/unsafe/reload") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /source the staging controller UUID/); assert.match(result.failures.join("\n"), /live narrow mounts, mesh-only metrics, least-privilege env, matching web\/controller CNAME/); }); it("fails closed when a controller command loses its explicit environment CNAME pin", () => { const fixture = createFixture(); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); const workflowLines = fs.readFileSync(workflowPath, "utf8").split("\n"); const invocationLine = workflowLines.findIndex((line) => line.includes("coolify-portal-domain-controller.mjs assert-runtime")); assert.notEqual(invocationLine, -1); workflowLines[invocationLine + 1] = workflowLines[invocationLine + 1].replace("staging", "production"); fs.writeFileSync(workflowPath, workflowLines.join("\n")); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /staging as its explicit fail-closed first argument/); }); it("fails closed when the Docker smoke omits the narrow reload sentinel contract", () => { const fixture = createFixture(); const smokePath = path.join(fixture, "scripts/docker-portal-domain-controller-runtime-smoke.sh"); fs.writeFileSync( smokePath, fs .readFileSync(smokePath, "utf8") .replaceAll("/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload", "/unsafe/reload") .replace('-e PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE="hivegrid-portal@docker"', "") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /sibling reload sentinel in a daemon-visible volume after initial cleanup/); }); it("fails closed when the orchestrated Docker smoke omits the controller runtime", () => { const fixture = createFixture(); const suitePath = path.join(fixture, "scripts/ci/docker-smoke-suite.sh"); fs.writeFileSync( suitePath, fs .readFileSync(suitePath, "utf8") .replace('bash scripts/docker-portal-domain-controller-runtime-smoke.sh "$PORTAL_CONTROLLER_IMAGE"', "true") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /build and run the portal-domain controller image/); }); it("allows only the reviewed flock process boundary and requires flock in the image", () => { const processFixture = createFixture(); const indexPath = path.join(processFixture, "apps/portal-domain-controller/src/index.ts"); fs.appendFileSync(indexPath, '\nimport { exec } from "node:child_process";\nvoid exec;\n'); const processResult = verifyPortalDomainControllerDeploy(processFixture); assert.match(processResult.failures.join("\n"), /non-flock process-execution capability/); const imageFixture = createFixture(); const dockerfilePath = path.join(imageFixture, "Dockerfile"); fs.writeFileSync( dockerfilePath, fs .readFileSync(dockerfilePath, "utf8") .replace("apt-get install -y --no-install-recommends util-linux", "apt-get install -y --no-install-recommends openssl") ); const imageResult = verifyPortalDomainControllerDeploy(imageFixture); assert.match(imageResult.failures.join("\n"), /install util-linux/); }); it("fails closed when the controller is not drained before web cutover", () => { const fixture = createFixture(); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); const workflow = fs.readFileSync(workflowPath, "utf8"); const drain = "node scripts/coolify-portal-domain-controller.mjs stop-and-drain"; const webTrigger = 'node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID"'; fs.writeFileSync( workflowPath, workflow.replace(drain, "CONTROLLER_DRAIN_PLACEHOLDER").replace(webTrigger, drain).replace("CONTROLLER_DRAIN_PLACEHOLDER", webTrigger) ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /drain the old controller before web cutover/); }); it("fails closed when stale stopped status can skip the controller stop request", () => { const fixture = createFixture(); const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs"); fs.writeFileSync( helperPath, fs .readFileSync(helperPath, "utf8") .replace( ' await coolify(`${resourcePath}/stop`, { method: "POST" });', ' if (!STOPPED_RUNTIME_STATUSES.test(normalizeValue(controller?.status))) {\n await coolify(`${resourcePath}/stop`, { method: "POST" });\n }' ) ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /revision-bound stop plus continuous legacy-safe drain/); }); it("fails closed when stopped status is accepted while mesh health remains reachable", () => { const fixture = createFixture(); const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs"); fs.writeFileSync( helperPath, fs.readFileSync(helperPath, "utf8").replace( 'STOPPED_RUNTIME_STATUSES.test(normalizeValue(current?.status)) && !afterStop.reachable', 'STOPPED_RUNTIME_STATUSES.test(normalizeValue(current?.status))' ) ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /revision-bound stop plus continuous legacy-safe drain/); }); it("fails closed when shutdown removes mesh health before queue and lock drainage", () => { const fixture = createFixture(); const indexPath = path.join(fixture, "apps/portal-domain-controller/src/index.ts"); fs.writeFileSync( indexPath, fs .readFileSync(indexPath, "utf8") .replace(" await queue.close();\n await claimLock.close();\n await http.close();", " await http.close();\n await queue.close();\n await claimLock.close();") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /close queue, then claim lock, then the mesh health server/); }); it("fails closed when the bootstrap drain fence is shorter than the legacy hard shutdown", () => { const fixture = createFixture(); const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs"); fs.writeFileSync( helperPath, fs.readFileSync(helperPath, "utf8").replace("CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 31_000", "CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 29_000") ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /continuous legacy-safe drain/); }); it("fails closed when the controller release is pinned before paired web proof", () => { const fixture = createFixture(); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); const workflow = fs.readFileSync(workflowPath, "utf8"); const setRelease = "node scripts/coolify-portal-domain-controller.mjs set-release"; const webProof = "node scripts/coolify-deploy.mjs assert-web-surface-release"; fs.writeFileSync( workflowPath, workflow.replace(setRelease, "SET_RELEASE_PLACEHOLDER").replace(webProof, setRelease).replace("SET_RELEASE_PLACEHOLDER", webProof) ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /prove both web surfaces and worker-general, then pin and start/); }); it("fails closed when the controller release is pinned before worker-general health proof", () => { const fixture = createFixture(); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); const workflow = fs.readFileSync(workflowPath, "utf8"); const setRelease = "node scripts/coolify-portal-domain-controller.mjs set-release"; const dependentHealthGate = "One or more Coolify staging dependent resources failed health checks"; fs.writeFileSync( workflowPath, workflow.replace(setRelease, "SET_RELEASE_PLACEHOLDER").replace(dependentHealthGate, setRelease).replace("SET_RELEASE_PLACEHOLDER", dependentHealthGate) ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /prove both web surfaces and worker-general, then pin and start/); }); it("fails closed when pre-drain recovery is not armed from a validated release snapshot", () => { const fixture = createFixture(); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); fs.writeFileSync( workflowPath, fs .readFileSync(workflowPath, "utf8") .replace("coolify-portal-domain-controller.mjs snapshot-release", "coolify-portal-domain-controller.mjs removed-release-snapshot") .replace("STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true", "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=false"), ); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /snapshot-release/); assert.match(result.failures.join("\n"), /arm before drain/); }); it("fails closed when recovery masks failures or reuses the rejected image tag", () => { const fixture = createFixture(); const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml"); const workflow = fs.readFileSync(workflowPath, "utf8"); const recoveryStepIndex = workflow.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure"); assert.notEqual(recoveryStepIndex, -1); const beforeRecovery = workflow.slice(0, recoveryStepIndex); const recovery = workflow .slice(recoveryStepIndex) .replace('"$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"', '"$IMAGE_TAG"') .replace( 'RECOVERY_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")"', 'RECOVERY_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" || true)"', ); fs.writeFileSync(workflowPath, beforeRecovery + recovery); const result = verifyPortalDomainControllerDeploy(fixture); assert.match(result.failures.join("\n"), /always-guarded previous-release set\/trigger\/wait\/assert recovery/); }); }); function createFixture() { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "portal-domain-controller-deploy-")); fixtures.push(fixture); for (const relativePath of requiredPaths) { const source = path.join(repoRoot, relativePath); const target = path.join(fixture, relativePath); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.cpSync(source, target, { recursive: true }); } return fixture; }