// @vitest-environment node import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import http from "node:http"; import path from "node:path"; import { describe, it } from "vitest"; const scriptPath = path.resolve("scripts/coolify-portal-domain-controller.mjs"); const bearerToken = "controller-coolify-token-sentinel"; const controllerUuid = "c".repeat(24); const workerGeneralUuid = "g".repeat(24); const workerMailUuid = "m".repeat(24); const webCoreUuid = "w".repeat(24); const webPortalUuid = "p".repeat(24); const deploymentUuid = "d".repeat(24); const revision = "a".repeat(40); const controllerImage = "registry.example.test/toolctrl-labs/hivegrid/portal-domain-controller"; const workerImage = "registry.example.test/toolctrl-labs/hivegrid/worker"; const webImage = "registry.example.test/toolctrl-labs/hivegrid/web"; const commandSecret = "controller-command-secret-sentinel-value"; const otherCommandSecret = "different-controller-command-secret-value"; const metricsToken = "controller-metrics-token-sentinel-value"; const configDirectory = "/data/coolify/proxy/dynamic/hivegrid-portal-domains"; const reloadTriggerFile = "/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload"; const stagingCname = "staging-domains.portal.hivegrid.de"; const productionCname = "domains.portal.hivegrid.de"; const stagingMetricsBinding = "10.42.0.3:19120:8080"; const productionMetricsBinding = "10.42.0.8:19120:8080"; function expectedControllerHealthConfig(expectedRevision = revision) { return { health_check_enabled: true, health_check_type: "cmd", health_check_command: `node apps/portal-domain-controller/dist/deploy-health.js ${expectedRevision}`, health_check_interval: 30, health_check_timeout: 10, health_check_retries: 3, health_check_start_period: 20, }; } function controllerEnv(overrides = {}) { return envEntries({ PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: commandSecret, PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: metricsToken, PORTAL_DOMAIN_CONTROLLER_REDIS_URL: "rediss://controller:secret@redis.example.test:6380/7", PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: stagingCname, PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: `https-0-${webPortalUuid}@docker`, PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "https", PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: configDirectory, PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: reloadTriggerFile, ...overrides, }); } function workerEnv(group, overrides = {}) { return envEntries({ WORKER_QUEUE_GROUP: group, BULLMQ_REDIS_URL: "rediss://worker:other-secret@redis.example.test:6380/7", ...(group === "general" ? { PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: commandSecret, PORTAL_DOMAIN_ORCHESTRATION_REQUIRED: "true", } : {}), ...overrides, }); } function envEntries(values) { return Object.entries(values) .filter(([, value]) => value !== undefined) .map(([key, value]) => ({ key, value, is_preview: false })); } function exactStorages() { return { file_storages: [ { uuid: "storage-config", is_directory: true, fs_path: configDirectory, mount_path: configDirectory, }, ], persistent_storages: [ { uuid: "storage-reload", host_path: reloadTriggerFile, mount_path: reloadTriggerFile, }, ], }; } function fixture(overrides = {}) { const controller = { id: 101, uuid: controllerUuid, name: "hivegrid-portal-domain-controller", build_pack: "dockerimage", docker_registry_image_name: controllerImage, docker_registry_image_tag: "previous", pre_deployment_command: "", post_deployment_command: "", fqdn: "", ports_mappings: stagingMetricsBinding, status: "stopped", custom_healthcheck_found: false, health_check_enabled: true, health_check_type: "http", health_check_command: "", health_check_interval: 30, health_check_timeout: 10, health_check_retries: 3, health_check_start_period: 20, ...overrides.controller, }; const workers = [ { uuid: workerGeneralUuid, name: "worker-general", docker_registry_image_name: workerImage }, { uuid: workerMailUuid, name: "worker-mail", docker_registry_image_name: workerImage }, ]; const webCore = { uuid: webCoreUuid, name: "hivegrid-web", build_pack: "dockerimage", docker_registry_image_name: webImage, }; const webPortal = { uuid: webPortalUuid, name: "hivegrid-portal", build_pack: "dockerimage", docker_registry_image_name: webImage, }; return { controller, apps: [controller, webCore, webPortal, ...workers], envByUuid: new Map([ [controllerUuid, controllerEnv(overrides.controllerEnv)], [ webCoreUuid, envEntries({ WEB_RUNTIME_SURFACE: "core", PORTAL_CUSTOM_DOMAIN_CNAME_TARGET: overrides.webCnameTarget ?? stagingCname, PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: overrides.webCoreRuntimeEnabled ?? "true", ...overrides.webCoreEnv, }), ], [ webPortalUuid, envEntries({ WEB_RUNTIME_SURFACE: "portal", PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: overrides.webPortalRuntimeEnabled ?? "true", ...overrides.webPortalEnv, }), ], [workerGeneralUuid, workerEnv("general", overrides.workerGeneralEnv)], [workerMailUuid, workerEnv("mail")], ]), storages: overrides.storages ?? exactStorages(), stopDoesNotTransition: overrides.stopDoesNotTransition ?? false, controllerProcessRunning: overrides.controllerProcessRunning ?? true, controllerHealthRevision: overrides.controllerHealthRevision ?? controller.docker_registry_image_tag, postStopHealthSequence: overrides.postStopHealthSequence ? [...overrides.postStopHealthSequence] : null, stopRequested: false, releaseMutationDrifts: overrides.releaseMutationDrifts ?? false, compensationDoesNotRestore: overrides.compensationDoesNotRestore ?? false, releasePatchCount: 0, statusSequence: overrides.statusSequence ?? null, allowedCsv: [controllerUuid, webCoreUuid, webPortalUuid, workerGeneralUuid, workerMailUuid].join(","), }; } function startFixtureServer(state) { return startServer((request) => { if (request.url === "/controller-healthz") { const reachable = state.stopRequested && state.postStopHealthSequence?.length ? state.postStopHealthSequence.shift() : state.controllerProcessRunning; if (!reachable) return { destroy: true }; return { payload: { status: "ok", service: "hivegrid-portal-domain-controller", revision: state.controllerHealthRevision, }, }; } if (request.url === "/applications") return { payload: state.apps }; if (request.url === `/applications/${controllerUuid}/storages`) return { payload: state.storages }; const envMatch = /^\/applications\/([a-z0-9]{24})\/envs$/u.exec(request.url); if (envMatch) return { payload: state.envByUuid.get(envMatch[1]) ?? [] }; if (request.url === `/deployments/${deploymentUuid}`) { return { payload: { deployment_uuid: deploymentUuid, application_id: state.controller.id, application_name: "hivegrid-portal-domain-controller", status: "finished", }, }; } if (request.url === `/applications/${controllerUuid}` && request.method === "PATCH") { state.releasePatchCount += 1; const patch = JSON.parse(request.body); if (state.releaseMutationDrifts && state.releasePatchCount === 1) { Object.assign(state.controller, patch, { health_check_command: "drifted-command" }); } else if (!(state.compensationDoesNotRestore && state.releasePatchCount > 1)) { Object.assign(state.controller, patch); } return { payload: state.controller }; } if (request.url === `/applications/${controllerUuid}/stop` && request.method === "POST") { state.stopRequested = true; if (!state.stopDoesNotTransition) { state.controller.status = "exited:unhealthy"; state.controllerProcessRunning = false; } return { payload: { message: "Application stopping request queued." } }; } if (request.url === `/applications/${controllerUuid}`) { if (state.statusSequence?.length) state.controller.status = state.statusSequence.shift(); return { payload: state.controller }; } return { status: 404, payload: {} }; }); } function runHelper(serverUrl, ...args) { const envOverrides = typeof args[0] === "object" && args[0] !== null ? args.shift() : {}; return new Promise((resolve) => { const child = spawn(process.execPath, [scriptPath, ...args], { env: { PATH: process.env.PATH || "", NODE_ENV: "test", SELECTED_COOLIFY_API_URL: serverUrl, SELECTED_COOLIFY_TOKEN: bearerToken, COOLIFY_DEPLOYMENT_POLL_MS: "1", COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "2", COOLIFY_CONTROLLER_DRAIN_HEALTH_URL: `${serverUrl}/controller-healthz`, COOLIFY_CONTROLLER_DRAIN_FENCE_MS: "1", ...envOverrides, }, stdio: ["ignore", "pipe", "pipe"], }); let stdout = ""; let stderr = ""; child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); child.stdout.on("data", (chunk) => { stdout += chunk; }); child.stderr.on("data", (chunk) => { stderr += chunk; }); child.on("close", (status) => resolve({ status, stdout, stderr })); }); } async function readBody(request) { let body = ""; request.setEncoding("utf8"); for await (const chunk of request) body += chunk; return body; } function startServer(responder) { const requests = []; const serverErrors = []; const server = http.createServer(async (request, response) => { const recorded = { method: request.method, url: request.url, authorization: request.headers.authorization, body: await readBody(request), }; requests.push(recorded); try { const result = await responder(recorded); if (result?.destroy) { response.destroy(); return; } response.writeHead(result?.status || 200, { "content-type": "application/json" }); response.end(JSON.stringify(result?.payload ?? {})); } catch (error) { serverErrors.push(error); response.writeHead(500, { "content-type": "application/json" }); response.end("{}"); } }); return new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => resolve({ url: `http://127.0.0.1:${server.address().port}`, requests, serverErrors, close: () => new Promise((done, failed) => server.close((error) => (error ? failed(error) : done()))), }) ); }); } function assertSecretSafe(result) { for (const secret of [bearerToken, commandSecret, otherCommandSecret, metricsToken, "controller:secret", "worker:other-secret"]) { assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(secret.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"))); } } describe("portal-domain controller Coolify deployment helper", () => { it("accepts only the private two-mount runtime with matched worker-general HMAC wiring", async () => { const state = fixture(); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /runtime, mounts, and worker-general HMAC wiring verified/); assert.equal(result.stderr, ""); assertSecretSafe(result); assert.deepEqual(server.serverErrors, []); assert.ok(server.requests.some(({ url }) => url === `/applications/${controllerUuid}/storages`)); assert.ok(server.requests.some(({ url }) => url === `/applications/${workerGeneralUuid}/envs`)); assert.ok(server.requests.some(({ url }) => url === `/applications/${webPortalUuid}/envs`)); assert.ok(server.requests.every(({ authorization }) => authorization === `Bearer ${bearerToken}`)); } finally { await server.close(); } }); it("accepts only Coolify's exact automatic metadata keys", async () => { const accepted = fixture({ controllerEnv: { COOLIFY_BRANCH: "main", COOLIFY_CONTAINER_NAME: "controller-container", COOLIFY_FQDN: "", COOLIFY_RESOURCE_UUID: controllerUuid, COOLIFY_URL: "", }, }); const acceptedServer = await startFixtureServer(accepted); try { const result = await runHelper(acceptedServer.url, "assert-runtime", "staging", controllerUuid, accepted.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 0, result.stderr); assertSecretSafe(result); } finally { await acceptedServer.close(); } const rejected = fixture({ controllerEnv: { COOLIFY_DEPLOYMENT_TOKEN: "not-allowed" } }); const rejectedServer = await startFixtureServer(rejected); try { const result = await runHelper(rejectedServer.url, "assert-runtime", "staging", controllerUuid, rejected.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /forbidden privileged environment keys/); assertSecretSafe(result); } finally { await rejectedServer.close(); } }); it("keeps the controller probe on the exact internal port behind the mesh metrics binding", async () => { const accepted = fixture({ controllerEnv: { PORTAL_DOMAIN_CONTROLLER_PORT: "8080" } }); const acceptedServer = await startFixtureServer(accepted); try { const result = await runHelper(acceptedServer.url, "assert-runtime", "staging", controllerUuid, accepted.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 0, result.stderr); assertSecretSafe(result); } finally { await acceptedServer.close(); } const rejected = fixture({ controllerEnv: { PORTAL_DOMAIN_CONTROLLER_PORT: "8081" } }); const rejectedServer = await startFixtureServer(rejected); try { const result = await runHelper(rejectedServer.url, "assert-runtime", "staging", controllerUuid, rejected.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /must be exactly 8080/); assertSecretSafe(result); } finally { await rejectedServer.close(); } }); it("rejects a worker-general HMAC mismatch before any mutation", async () => { const state = fixture({ workerGeneralEnv: { PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: otherCommandSecret } }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 1); assert.match(result.stderr, /HMAC secrets must match/); assert.ok(server.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await server.close(); } }); it("rejects a broad Traefik root mount or missing reload sentinel", async () => { for (const storages of [ [{ type: "file", is_directory: true, fs_path: "/data/coolify/proxy/dynamic", mount_path: "/data/coolify/proxy/dynamic" }], { file_storages: exactStorages().file_storages, persistent_storages: [ { ...exactStorages().persistent_storages[0], host_path: "/data/coolify/proxy/dynamic/not-the-reload-sentinel", }, ], }, ]) { const state = fixture({ storages }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /exactly two narrow bind mounts|config subdirectory and RW reload sentinel/); assertSecretSafe(result); } finally { await server.close(); } } }); it("rejects missing or malformed environment-specific Traefik routing identities", async () => { for (const controllerEnvOverride of [ { PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: undefined }, { PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "http://hivegrid-portal:8080" }, { PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "https;malformed" }, { PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "websecure" }, ]) { const state = fixture({ controllerEnv: controllerEnvOverride }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /BACKEND_SERVICE|ENTRYPOINT/); assertSecretSafe(result); } finally { await server.close(); } } }); it("rejects a missing or ambiguous allowlisted portal surface before any mutation", async () => { const duplicatePortalUuid = "q".repeat(24); const missing = fixture({ webPortalEnv: { WEB_RUNTIME_SURFACE: undefined } }); const ambiguous = fixture(); ambiguous.apps.push({ uuid: duplicatePortalUuid, name: "hivegrid-portal-shadow", build_pack: "dockerimage", docker_registry_image_name: webImage, }); ambiguous.envByUuid.set(duplicatePortalUuid, envEntries({ WEB_RUNTIME_SURFACE: "portal" })); ambiguous.allowedCsv = `${ambiguous.allowedCsv},${duplicatePortalUuid}`; for (const state of [missing, ambiguous]) { const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 1); assert.match(result.stderr, /exactly one allowlisted hivegrid-portal surface/); assert.ok(server.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await server.close(); } } }); it("rejects a valid Docker service name that does not identify the resolved portal resource", async () => { const state = fixture({ controllerEnv: { PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: `https-0-${webCoreUuid}@docker`, }, }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 1); assert.match(result.stderr, /must target the exact allowlisted hivegrid-portal Docker service/); assert.ok(server.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await server.close(); } }); it("rejects CNAME drift and unexpected controller credentials before mutation", async () => { for (const state of [ fixture({ webCnameTarget: "production-ingress.example.test" }), fixture({ controllerEnv: { STORAGE_SECRET_ACCESS_KEY: "must-never-reach-controller" } }), ]) { const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 1); assert.match(result.stderr, /hivegrid-web must use exactly|least-privilege environment allowlist/); assert.ok(server.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await server.close(); } } }); it("requires the exact custom-domain runtime opt-in on both staging web surfaces", async () => { for (const state of [ fixture({ webCoreEnv: { PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: undefined } }), fixture({ webCoreRuntimeEnabled: "false" }), fixture({ webCoreRuntimeEnabled: "1" }), fixture({ webPortalEnv: { PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: undefined } }), fixture({ webPortalRuntimeEnabled: "false" }), fixture({ webPortalRuntimeEnabled: "TRUE" }), ]) { const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 1); assert.match(result.stderr, /PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED exactly to true in staging/); assert.ok(server.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await server.close(); } } }); it("pins each deployment environment to its exact CNAME target before mutation", async () => { const cases = [ { environment: "production", state: fixture({ controller: { ports_mappings: productionMetricsBinding } }), expected: new RegExp(`EXPECTED_CNAME must be exactly ${productionCname}`), }, { environment: "staging", state: fixture({ controllerEnv: { PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: productionCname }, webCnameTarget: productionCname, }), expected: new RegExp(`EXPECTED_CNAME must be exactly ${stagingCname}`), }, { environment: "preview", state: fixture(), expected: /deployment environment must be exactly staging or production/, }, ]; for (const { environment, state, expected } of cases) { const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", environment, controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 1); assert.match(result.stderr, expected); assert.ok(server.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await server.close(); } } }); it("accepts the production environment only with the production CNAME target", async () => { const state = fixture({ controller: { ports_mappings: productionMetricsBinding }, controllerEnv: { PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: productionCname }, webCnameTarget: productionCname, webCoreRuntimeEnabled: "false", webPortalRuntimeEnabled: "false", }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "assert-runtime", "production", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 0, result.stderr); assertSecretSafe(result); } finally { await server.close(); } }); it("pins the explicit resource to the full SHA and clears deployment hooks", async () => { const state = fixture(); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /immutable release configured/); const patches = server.requests.filter(({ method }) => method === "PATCH"); assert.equal(patches.length, 1); assert.deepEqual(JSON.parse(patches[0].body), { pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: revision, ...expectedControllerHealthConfig(), }); assertSecretSafe(result); } finally { await server.close(); } }); it("verifies compensation after a rejected release mutation", async () => { for (const compensationDoesNotRestore of [false, true]) { const state = fixture({ controller: { health_check_interval: 0, health_check_start_period: 0 }, releaseMutationDrifts: true, compensationDoesNotRestore, }); const previousTag = state.controller.docker_registry_image_tag; const previousHealthType = state.controller.health_check_type; const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision); assert.equal(result.status, 1); assert.match( result.stderr, compensationDoesNotRestore ? /compensation was incomplete/ : /previous configuration was restored/ ); assert.equal(server.requests.filter(({ method }) => method === "PATCH").length, 2); if (!compensationDoesNotRestore) { assert.equal(state.controller.docker_registry_image_tag, previousTag); assert.equal(state.controller.health_check_type, previousHealthType); assert.equal(state.controller.health_check_interval, 0); assert.equal(state.controller.health_check_start_period, 0); } assertSecretSafe(result); } finally { await server.close(); } } }); it("stops and drains the old controller before a paired web rollout", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "running:healthy" } }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /stopped and command worker drained/); assert.deepEqual( server.requests.filter(({ method, url }) => method === "POST" && url.endsWith("/stop")).map(({ url }) => url), [`/applications/${controllerUuid}/stop`] ); assertSecretSafe(result); } finally { await server.close(); } }); it("fails closed for operator recovery when pre-stop health is unreachable despite exited status", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "exited:unhealthy" }, controllerProcessRunning: false, }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /pre-stop health is unreachable or not revision-bound; operator recovery is required/); assert.equal(server.requests.some(({ method, url }) => method === "POST" && url.endsWith("/stop")), false); assertSecretSafe(result); } finally { await server.close(); } }); it("does not trust stale exited status while the queued stop and live health lag", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "exited:unhealthy" }, statusSequence: ["exited:unhealthy", "exited:unhealthy", "running:healthy", "exited:unhealthy", "exited:unhealthy"], postStopHealthSequence: [true, true, false, false], }); const server = await startFixtureServer(state); try { const result = await runHelper( server.url, { COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "4" }, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage ); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /stopped and command worker drained/); assert.deepEqual( server.requests.filter(({ method, url }) => method === "POST" && url.endsWith("/stop")).map(({ url }) => url), [`/applications/${controllerUuid}/stop`] ); assert.equal(server.requests.filter(({ method, url }) => method === "GET" && url === `/applications/${controllerUuid}`).length, 5); assert.equal(server.requests.filter(({ url }) => url === "/controller-healthz").length, 5); assertSecretSafe(result); } finally { await server.close(); } }); it("fails closed when Coolify cannot prove the controller stopped", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "running:healthy" }, stopDoesNotTransition: true, }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /did not stop and drain/); assertSecretSafe(result); } finally { await server.close(); } }); it("fails closed before stop when mesh health belongs to another revision", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "running:healthy" }, controllerHealthRevision: "b".repeat(40), }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /pre-stop health is unreachable or not revision-bound/); assert.equal(server.requests.some(({ method, url }) => method === "POST" && url.endsWith("/stop")), false); assertSecretSafe(result); } finally { await server.close(); } }); it("snapshots only a validated immutable controller release without mutation", async () => { const valid = fixture({ controller: { docker_registry_image_tag: revision, custom_healthcheck_found: false, ...expectedControllerHealthConfig(), }, }); const validServer = await startFixtureServer(valid); try { const result = await runHelper(validServer.url, "snapshot-release", "staging", controllerUuid, valid.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 0, result.stderr); assert.equal(result.stdout, revision); assert.equal(result.stderr, ""); assert.ok(validServer.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await validServer.close(); } for (const controller of [ { docker_registry_image_tag: "staging" }, { docker_registry_image_tag: revision, custom_healthcheck_found: false, ...expectedControllerHealthConfig(), health_check_command: "drifted" }, ]) { const invalid = fixture({ controller }); const invalidServer = await startFixtureServer(invalid); try { const result = await runHelper( invalidServer.url, "snapshot-release", "staging", controllerUuid, invalid.allowedCsv, controllerImage, workerImage, ); assert.equal(result.status, 1); assert.match(result.stderr, /full lowercase Git SHA|revision-bound command health configuration drifted/); assert.ok(invalidServer.requests.every(({ method }) => method === "GET")); assertSecretSafe(result); } finally { await invalidServer.close(); } } }); it("rejects a mutable release tag before contacting Coolify", async () => { const state = fixture(); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, "staging"); assert.equal(result.status, 1); assert.match(result.stderr, /full lowercase Git SHA/); assert.deepEqual(server.requests, []); assertSecretSafe(result); } finally { await server.close(); } }); it("binds the completed deployment to a revision-bound command healthz and readyz contract", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "running:healthy", custom_healthcheck_found: false, ...expectedControllerHealthConfig(), }, }); const server = await startFixtureServer(state); try { const result = await runHelper( server.url, "assert-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision, deploymentUuid, ); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /healthz, readyz, and immutable revision verified/); assert.ok(server.requests.some(({ url }) => url === `/deployments/${deploymentUuid}`)); assertSecretSafe(result); } finally { await server.close(); } }); it("tolerates at least eight stale exited statuses within the freshly deployed controller grace period", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "exited:unhealthy", custom_healthcheck_found: false, ...expectedControllerHealthConfig(), }, // First entry feeds the runtime-contract lookup, the rest the poll loop. statusSequence: [ "exited:unhealthy", "exited:unhealthy", "exited:unhealthy", "exited:unhealthy", "exited:unhealthy", "exited:unhealthy", "exited:unhealthy", "exited:unhealthy", "exited:unhealthy", "running:healthy", ], }); const server = await startFixtureServer(state); try { const result = await runHelper( server.url, { COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "10", COOLIFY_FAILED_STATUS_GRACE_MS: "60000" }, "assert-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision, deploymentUuid ); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /healthz, readyz, and immutable revision verified/); assertSecretSafe(result); } finally { await server.close(); } }); it("still fails closed when the controller stays in a failed runtime state", async () => { const state = fixture({ controller: { docker_registry_image_tag: revision, status: "exited:unhealthy", custom_healthcheck_found: false, ...expectedControllerHealthConfig(), }, }); const server = await startFixtureServer(state); try { const result = await runHelper( server.url, { COOLIFY_DEPLOYMENT_POLL_MS: "10", COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "6", COOLIFY_FAILED_STATUS_GRACE_MS: "5", }, "assert-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision, deploymentUuid ); assert.equal(result.status, 1); assert.match(result.stderr, /entered a terminal runtime state/); assertSecretSafe(result); } finally { await server.close(); } }); it("rejects an invalid failed-status grace before contacting Coolify", async () => { const state = fixture(); const server = await startFixtureServer(state); try { const result = await runHelper( server.url, { COOLIFY_FAILED_STATUS_GRACE_MS: "-1" }, "assert-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision, deploymentUuid ); assert.equal(result.status, 1); assert.match(result.stderr, /COOLIFY_FAILED_STATUS_GRACE_MS must be a non-negative safe integer/); assert.deepEqual(server.requests, []); assertSecretSafe(result); } finally { await server.close(); } }); it("rejects a public controller route, an image-healthcheck override, or command-health drift", async () => { for (const controller of [ { fqdn: "https://controller.example.test" }, { docker_registry_image_tag: revision, status: "running:healthy", custom_healthcheck_found: true, ...expectedControllerHealthConfig(), }, { docker_registry_image_tag: revision, status: "running:healthy", custom_healthcheck_found: false, ...expectedControllerHealthConfig(), health_check_command: "curl http://127.0.0.1:8080/readyz", }, ]) { const state = fixture({ controller }); const server = await startFixtureServer(state); try { const args = controller.fqdn ? ["assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage] : ["assert-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision, deploymentUuid]; const result = await runHelper(server.url, ...args); assert.equal(result.status, 1); assert.match(result.stderr, /must remain private|command health configuration drifted/); assertSecretSafe(result); } finally { await server.close(); } } }); it("rejects a public or missing metrics port instead of exposing the controller beyond the mesh", async () => { for (const ports_mappings of [null, "0.0.0.0:19120:8080", "10.42.0.3:19120:8080,10.42.0.3:19121:8080"]) { const state = fixture({ controller: { ports_mappings } }); const server = await startFixtureServer(state); try { const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage); assert.equal(result.status, 1); assert.match(result.stderr, /metrics only through the exact mesh binding/); assertSecretSafe(result); } finally { await server.close(); } } }); });