#!/usr/bin/env node import { timingSafeEqual } from "node:crypto"; import { domainToASCII } from "node:url"; const [, , command, ...args] = process.argv; const apiUrl = (process.env.SELECTED_COOLIFY_API_URL || process.env.COOLIFY_STAGING_API_URL || process.env.COOLIFY_API_URL || "").replace(/\/+$/, ""); const token = process.env.SELECTED_COOLIFY_TOKEN || process.env.COOLIFY_STAGING_TOKEN || process.env.COOLIFY_TOKEN || ""; const apiTimeoutMs = Number(process.env.COOLIFY_API_TIMEOUT_MS || "20000"); const pollDelayMs = Number(process.env.COOLIFY_DEPLOYMENT_POLL_MS || "5000"); const pollAttempts = Number(process.env.COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS || "90"); const failedStatusGraceMs = Number(process.env.COOLIFY_FAILED_STATUS_GRACE_MS || "60000"); const drainHealthUrlOverride = (process.env.COOLIFY_CONTROLLER_DRAIN_HEALTH_URL || "").trim(); const drainFenceMsOverride = (process.env.COOLIFY_CONTROLLER_DRAIN_FENCE_MS || "").trim(); const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/u; const COOLIFY_CUID_PATTERN = /^[a-z][a-z0-9]{23}$/u; const CONTROLLER_NAME = "hivegrid-portal-domain-controller"; const CONTROLLER_DEPLOY_HEALTH_PATH = "apps/portal-domain-controller/dist/deploy-health.js"; const CONTROLLER_HEALTH_INTERVAL_SECONDS = 30; const CONTROLLER_HEALTH_TIMEOUT_SECONDS = 10; const CONTROLLER_HEALTH_RETRIES = 3; const CONTROLLER_HEALTH_START_PERIOD_SECONDS = 20; const CONTROLLER_DRAIN_CONFIRMATIONS = 2; const CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 31_000; const CONTROLLER_HEALTH_PROBE_TIMEOUT_MS = 5_000; const CONTROLLER_INTERNAL_PORT = "8080"; const WEB_SURFACE_IDENTITIES = Object.freeze({ core: "hivegrid-web", portal: "hivegrid-portal", }); const PORTAL_CNAME_BY_ENVIRONMENT = Object.freeze({ staging: "staging-domains.portal.hivegrid.de", production: "domains.portal.hivegrid.de", }); const CONTROLLER_METRICS_BINDING_BY_ENVIRONMENT = Object.freeze({ staging: "10.42.0.3:19120:8080", production: "10.42.0.8:19120:8080", }); const PORTAL_TRAEFIK_ENTRYPOINT = "https"; const CONTROLLER_CONFIG_DIR = "/data/coolify/proxy/dynamic/hivegrid-portal-domains"; const CONTROLLER_RELOAD_TRIGGER_FILE = "/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload"; const COMPLETED_DEPLOYMENT_STATUSES = new Set(["finished", "success", "succeeded", "completed", "done"]); const FAILED_RUNTIME_STATUSES = /failed|exited|dead|cancelled|canceled/iu; const STOPPED_RUNTIME_STATUSES = /^(?:stopped|exited(?::[^:]*)?)$/iu; const CONTROLLER_REQUIRED_ENV_KEYS = Object.freeze([ "PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET", "PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN", "PORTAL_DOMAIN_CONTROLLER_REDIS_URL", "PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME", "PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE", "PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR", "PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE", ]); const CONTROLLER_ALLOWED_ENV_KEYS = new Set([ ...CONTROLLER_REQUIRED_ENV_KEYS, "PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT", "PORTAL_DOMAIN_CONTROLLER_HOST", "PORTAL_DOMAIN_CONTROLLER_PORT", "PORTAL_DOMAIN_CONTROLLER_CONCURRENCY", ]); const CONTROLLER_COOLIFY_AUTOMATIC_ENV_KEYS = new Set(["COOLIFY_BRANCH", "COOLIFY_CONTAINER_NAME", "COOLIFY_FQDN", "COOLIFY_RESOURCE_UUID", "COOLIFY_URL"]); const CONTROLLER_FORBIDDEN_ENV_KEYS = new Set([ "APP_REVISION", "DATABASE_URL", "DATABASE_READ_URL", "DIRECT_URL", "DOCKER_HOST", "PGHOST", "PGPORT", "PGDATABASE", "PGUSER", "PGPASSWORD", "CLOUDFLARE_API_TOKEN", "CLOUDFLARE_API_KEY", "COOLIFY_TOKEN", "SSH_AUTH_SOCK", ]); const CONTROLLER_FORBIDDEN_ENV_PREFIXES = Object.freeze(["MIGRATOR_", "SEED_", "COOLIFY_", "CLOUDFLARE_", "SSH_"]); function fail(message) { throw new Error(message); } async function coolify(path, options = {}) { if (!apiUrl || !token) fail("Coolify API URL and token are required"); if (!Number.isInteger(apiTimeoutMs) || apiTimeoutMs < 10 || apiTimeoutMs > 120_000) { fail("COOLIFY_API_TIMEOUT_MS must be an integer between 10 and 120000"); } let response; try { response = await fetch(`${apiUrl}${path}`, { ...options, signal: options.signal ?? AbortSignal.timeout(apiTimeoutMs), headers: { Authorization: `Bearer ${token}`, Accept: "application/json", ...(options.body ? { "Content-Type": "application/json" } : {}), ...options.headers, }, }); } catch { fail("Coolify API request failed or timed out"); } const body = await response.text(); let payload = {}; if (body.trim()) { try { payload = JSON.parse(body); } catch { payload = {}; } } if (!response.ok) fail(`Coolify API request failed with HTTP ${response.status}`); return payload; } function normalizeValue(value) { return value === null || value === undefined ? "" : String(value).trim(); } function normalizeBoolean(value) { if (value === true || value === "true" || value === 1 || value === "1") return "true"; if (value === false || value === "false" || value === 0 || value === "0") return "false"; return ""; } function normalizeEnvValue(entry) { let value = normalizeValue(entry?.value ?? entry?.real_value); while (value.length >= 2 && ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"')))) { value = value.slice(1, -1).trim(); } return value; } function requireCoolifyCuid(value, label) { if (!COOLIFY_CUID_PATTERN.test(value)) fail(`${label} must be a valid Coolify CUID`); } function allowedResourceSet(csv) { const values = csv .split(",") .map((value) => value.trim()) .filter(Boolean); if (values.length === 0) fail("Portal-domain controller deployment requires an explicit Coolify resource allowlist"); const resources = new Set(values); if (resources.size !== values.length) fail("Coolify resource allowlist must not contain duplicates"); for (const value of resources) requireCoolifyCuid(value, "Coolify resource allowlist entry"); return resources; } function arrayPayload(payload, label) { if (Array.isArray(payload)) return payload; if (Array.isArray(payload?.data)) return payload.data; fail(`Coolify ${label} response was malformed`); } function runtimeEnvEntries(payload) { const entries = []; const keys = new Set(); for (const entry of arrayPayload(payload, "environment")) { if (normalizeBoolean(entry?.is_preview) === "true") continue; const key = normalizeValue(entry?.key); if (!key) continue; if (keys.has(key)) fail("Coolify runtime environment contains duplicate non-preview keys"); keys.add(key); entries.push(entry); } return entries; } function runtimeStorageEntries(payload) { let entries; if (Array.isArray(payload) || Array.isArray(payload?.data)) { entries = arrayPayload(payload, "storage"); } else if (Array.isArray(payload?.file_storages) && Array.isArray(payload?.persistent_storages)) { entries = [ ...payload.file_storages.map((entry) => ({ ...entry, type: "file" })), ...payload.persistent_storages.map((entry) => ({ ...entry, type: "persistent" })), ]; } else { fail("Coolify storage response was malformed"); } return entries.filter((entry) => normalizeBoolean(entry?.is_preview) !== "true"); } function valuesByKey(entries) { return new Map(entries.map((entry) => [normalizeValue(entry?.key), normalizeEnvValue(entry)])); } function normalizedImage(value) { return normalizeValue(value).replace(/\/+$/, ""); } function assertPrivateControllerIdentity(app, resourceUuid, expectedImage, expectedMetricsBinding) { if ( app?.uuid !== resourceUuid || normalizeValue(app?.name) !== CONTROLLER_NAME || normalizedImage(app?.docker_registry_image_name) !== expectedImage || normalizeValue(app?.build_pack) !== "dockerimage" ) { fail("Portal-domain controller UUID, name, image, or dockerimage build-pack identity does not match"); } const routeValues = [app?.fqdn, app?.domain, app?.domains] .flatMap((value) => (Array.isArray(value) ? value : [value])) .flatMap((value) => normalizeValue(value).split(/[\s,]+/u)) .filter(Boolean); if (routeValues.length > 0) fail("Portal-domain controller must remain private without a public Coolify route"); if (normalizeValue(app?.ports_mappings) !== expectedMetricsBinding) { fail(`Portal-domain controller must expose metrics only through the exact mesh binding ${expectedMetricsBinding}`); } } function storagePath(entry, ...keys) { for (const key of keys) { const value = normalizeValue(entry?.[key]); if (value) return value; } return ""; } function assertControllerStorageContract(entries) { if (entries.length !== 2) { fail("Portal-domain controller requires exactly two narrow bind mounts"); } const normalized = entries.map((entry) => ({ type: normalizeValue(entry?.type), directory: normalizeBoolean(entry?.is_directory), source: storagePath(entry, "fs_path", "host_path", "source"), destination: storagePath(entry, "mount_path", "destination", "target"), readOnly: normalizeBoolean(entry?.is_read_only ?? entry?.read_only), })); const configDirectory = normalized.filter( (entry) => entry.type === "file" && entry.directory === "true" && entry.source === CONTROLLER_CONFIG_DIR && entry.destination === CONTROLLER_CONFIG_DIR && entry.readOnly !== "true" ); const reloadSentinel = normalized.filter( (entry) => (entry.type === "file" || entry.type === "persistent") && entry.directory !== "true" && entry.source === CONTROLLER_RELOAD_TRIGGER_FILE && entry.destination === CONTROLLER_RELOAD_TRIGGER_FILE && entry.readOnly !== "true" ); if (configDirectory.length !== 1 || reloadSentinel.length !== 1) { fail("Portal-domain controller bind mounts must expose only its RW config subdirectory and RW reload sentinel file"); } } function requireSecret(values, key) { const value = values.get(key) || ""; const bytes = Buffer.byteLength(value, "utf8"); if (bytes < 32 || bytes > 512) fail(`${key} must contain between 32 and 512 bytes`); return value; } function equalSecret(left, right) { const leftBuffer = Buffer.from(left, "utf8"); const rightBuffer = Buffer.from(right, "utf8"); return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer); } function redisQueueIdentity(value, label) { let parsed; try { parsed = new URL(value); } catch { fail(`${label} must be a valid Redis connection URL`); } if (!new Set(["redis:", "rediss:"]).has(parsed.protocol) || !parsed.hostname || parsed.hash) { fail(`${label} must be a valid Redis connection URL`); } const database = parsed.pathname.replace(/^\//u, "") || "0"; if (!/^\d+$/u.test(database)) fail(`${label} must select a numeric Redis database`); return `${parsed.protocol}//${parsed.hostname.toLowerCase()}:${parsed.port || "6379"}/${database}`; } function assertHostname(value) { const normalized = value.toLowerCase().replace(/\.$/u, ""); const ascii = domainToASCII(normalized); if ( !ascii || ascii.length > 253 || ascii.includes("*") || !ascii.includes(".") || ascii.split(".").some((label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u.test(label)) ) { fail("PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME must be an exact DNS hostname"); } } function assertBackendService(value) { if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,126}@docker$/u.test(value)) { fail("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE must be a provider-qualified Docker service name"); } } function assertEntryPoint(value) { const entryPoint = value || PORTAL_TRAEFIK_ENTRYPOINT; if (entryPoint !== PORTAL_TRAEFIK_ENTRYPOINT) { fail("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT must be exactly https"); } return entryPoint; } function assertWebSurfacePair(core, portal) { const expected = [ [core, WEB_SURFACE_IDENTITIES.core], [portal, WEB_SURFACE_IDENTITIES.portal], ]; for (const [app, expectedName] of expected) { requireCoolifyCuid(app?.uuid || "", `${expectedName} resource UUID`); if (normalizeValue(app?.name) !== expectedName || normalizeValue(app?.build_pack) !== "dockerimage" || !normalizedImage(app?.docker_registry_image_name)) { fail("Coolify web surface identity must match its explicit name, dockerimage build pack, and image"); } } if (core.uuid === portal.uuid || normalizedImage(core.docker_registry_image_name) !== normalizedImage(portal.docker_registry_image_name)) { fail("hivegrid-web and hivegrid-portal must be distinct resources using the same immutable web image"); } } function portalBackendService(portal) { requireCoolifyCuid(portal?.uuid || "", "hivegrid-portal resource UUID"); return `https-0-${portal.uuid}@docker`; } function expectedCnameForEnvironment(environment) { const expectedCname = PORTAL_CNAME_BY_ENVIRONMENT[environment]; if (!expectedCname) { fail("Portal-domain controller deployment environment must be exactly staging or production"); } return expectedCname; } function expectedMetricsBindingForEnvironment(environment) { const binding = CONTROLLER_METRICS_BINDING_BY_ENVIRONMENT[environment]; if (!binding) { fail("Portal-domain controller deployment environment must be exactly staging or production"); } return binding; } function controllerHealthUrlForEnvironment(environment) { const [host, publishedPort, internalPort, ...rest] = expectedMetricsBindingForEnvironment(environment).split(":"); if (rest.length > 0 || !/^10\.42\.0\.\d+$/u.test(host || "") || !/^\d{2,5}$/u.test(publishedPort || "") || internalPort !== CONTROLLER_INTERNAL_PORT) { fail("Portal-domain controller mesh health binding is invalid"); } if (!drainHealthUrlOverride) return `http://${host}:${publishedPort}/healthz`; if (process.env.NODE_ENV !== "test") fail("Portal-domain controller drain health override is test-only"); let override; let selectedApi; try { override = new URL(drainHealthUrlOverride); selectedApi = new URL(apiUrl); } catch { fail("Portal-domain controller test drain health URL is invalid"); } const loopbackHosts = new Set(["127.0.0.1", "localhost", "::1"]); if ( override.protocol !== "http:" || !loopbackHosts.has(override.hostname) || !loopbackHosts.has(selectedApi.hostname) || override.origin !== selectedApi.origin || override.username || override.password || override.search || override.hash ) { fail("Portal-domain controller test drain health URL must be loopback-only HTTP"); } return override.href; } function controllerDrainFenceMs() { if (!drainFenceMsOverride) return CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS; if (process.env.NODE_ENV !== "test" || !drainHealthUrlOverride) { fail("Portal-domain controller drain fence override is test-only"); } const value = Number(drainFenceMsOverride); if (!Number.isSafeInteger(value) || value < 1 || value > 1_000) { fail("Portal-domain controller test drain fence must be an integer between 1 and 1000"); } return value; } function assertControllerEnv(entries, expectedCname) { const values = valuesByKey(entries); for (const key of CONTROLLER_REQUIRED_ENV_KEYS) { if (!values.get(key)) fail(`Portal-domain controller runtime is missing ${key}`); } const forbidden = [...values.keys()].filter( (key) => !CONTROLLER_COOLIFY_AUTOMATIC_ENV_KEYS.has(key) && (CONTROLLER_FORBIDDEN_ENV_KEYS.has(key) || CONTROLLER_FORBIDDEN_ENV_PREFIXES.some((prefix) => key.startsWith(prefix))) ); if (forbidden.length > 0) fail("Portal-domain controller runtime contains forbidden privileged environment keys"); const unexpected = [...values.keys()].filter((key) => !CONTROLLER_ALLOWED_ENV_KEYS.has(key) && !CONTROLLER_COOLIFY_AUTOMATIC_ENV_KEYS.has(key)); if (unexpected.length > 0) { fail("Portal-domain controller runtime must contain only its explicit least-privilege environment allowlist"); } const configuredPort = values.get("PORTAL_DOMAIN_CONTROLLER_PORT") || CONTROLLER_INTERNAL_PORT; if (configuredPort !== CONTROLLER_INTERNAL_PORT) { fail(`PORTAL_DOMAIN_CONTROLLER_PORT must be exactly ${CONTROLLER_INTERNAL_PORT} for the mesh metrics binding`); } const commandSecret = requireSecret(values, "PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET"); const metricsToken = requireSecret(values, "PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN"); if (equalSecret(commandSecret, metricsToken)) fail("Portal-domain controller command and metrics secrets must differ"); const redisIdentity = redisQueueIdentity(values.get("PORTAL_DOMAIN_CONTROLLER_REDIS_URL") || "", "PORTAL_DOMAIN_CONTROLLER_REDIS_URL"); const configuredCname = values.get("PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME") || ""; assertHostname(configuredCname); if (configuredCname !== expectedCname) { fail(`PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME must be exactly ${expectedCname} for this deployment environment`); } const backendService = values.get("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE") || ""; assertBackendService(backendService); const entryPoint = assertEntryPoint(values.get("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT") || ""); if (values.get("PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR") !== CONTROLLER_CONFIG_DIR) { fail("PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR must use the dedicated Traefik subdirectory"); } if (values.get("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE") !== CONTROLLER_RELOAD_TRIGGER_FILE) { fail("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must use the single watched-root sentinel file"); } return { commandSecret, redisIdentity, expectedCname: configuredCname, backendService, entryPoint }; } function assertWebCnameEnv(entries, expectedCname) { const values = valuesByKey(entries); if (values.get("PORTAL_CUSTOM_DOMAIN_CNAME_TARGET") !== expectedCname) { fail(`hivegrid-web must use exactly ${expectedCname} for this deployment environment`); } } function assertStagingWebCustomDomainRuntime(entries, environment, surfaceName) { if (environment !== "staging") return; if (valuesByKey(entries).get("PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED") !== "true") { fail(`${surfaceName} must set PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED exactly to true in staging`); } } function assertWorkerGeneralEnv(entries, controllerContract) { const values = valuesByKey(entries); if (values.get("WORKER_QUEUE_GROUP") !== "general") fail("Portal-domain orchestration must use worker-general"); if (values.get("PORTAL_DOMAIN_ORCHESTRATION_REQUIRED") !== "true") { fail("worker-general must set PORTAL_DOMAIN_ORCHESTRATION_REQUIRED=true"); } if (values.has("PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN")) { fail("worker-general must not receive the portal-domain controller metrics token"); } const commandSecret = requireSecret(values, "PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET"); if (!equalSecret(commandSecret, controllerContract.commandSecret)) { fail("worker-general and portal-domain controller command HMAC secrets must match"); } const workerRedisUrl = values.get("BULLMQ_REDIS_URL") || values.get("REDIS_URL") || ""; const workerRedisIdentity = redisQueueIdentity(workerRedisUrl, "worker-general Redis URL"); if (workerRedisIdentity !== controllerContract.redisIdentity) { fail("worker-general and portal-domain controller must use the same Redis queue endpoint and database"); } } async function resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage) { const expectedCname = expectedCnameForEnvironment(environment); const expectedMetricsBinding = expectedMetricsBindingForEnvironment(environment); requireCoolifyCuid(resourceUuid, "Portal-domain controller resource UUID"); const allowed = allowedResourceSet(allowedCsv); if (!allowed.has(resourceUuid)) fail("Portal-domain controller resource UUID is outside the Coolify allowlist"); const expectedControllerImage = normalizedImage(controllerImage); const expectedWorkerImage = normalizedImage(workerImage); if (!expectedControllerImage || !expectedWorkerImage) fail("Controller and worker image identities are required"); const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`; const [appsPayload, controller, controllerEnvPayload, storagePayload] = await Promise.all([ coolify("/applications"), coolify(resourcePath), coolify(`${resourcePath}/envs`), coolify(`${resourcePath}/storages`), ]); const apps = arrayPayload(appsPayload, "applications"); if (!apps.some((app) => app?.uuid === resourceUuid)) { fail("Portal-domain controller resource is absent from the Coolify application inventory"); } assertPrivateControllerIdentity(controller, resourceUuid, expectedControllerImage, expectedMetricsBinding); assertControllerStorageContract(runtimeStorageEntries(storagePayload)); const controllerContract = assertControllerEnv(runtimeEnvEntries(controllerEnvPayload), expectedCname); const runtimeCandidatesWithEnv = await Promise.all( apps .filter((app) => allowed.has(app?.uuid) && app?.uuid !== resourceUuid) .map(async (app) => ({ app, entries: runtimeEnvEntries(await coolify(`/applications/${encodeURIComponent(app.uuid)}/envs`)), })) ); const workerGeneralMatches = runtimeCandidatesWithEnv.filter( ({ app, entries }) => normalizedImage(app?.docker_registry_image_name) === expectedWorkerImage && valuesByKey(entries).get("WORKER_QUEUE_GROUP") === "general" ); if (workerGeneralMatches.length !== 1) { fail("Coolify must expose exactly one allowlisted worker-general for portal-domain orchestration"); } assertWorkerGeneralEnv(workerGeneralMatches[0].entries, controllerContract); const webCoreMatches = runtimeCandidatesWithEnv.filter(({ entries }) => valuesByKey(entries).get("WEB_RUNTIME_SURFACE") === "core"); if (webCoreMatches.length !== 1) { fail("Coolify must expose exactly one allowlisted hivegrid-web core surface for CNAME verification"); } assertWebCnameEnv(webCoreMatches[0].entries, expectedCname); assertStagingWebCustomDomainRuntime(webCoreMatches[0].entries, environment, WEB_SURFACE_IDENTITIES.core); const webPortalMatches = runtimeCandidatesWithEnv.filter(({ entries }) => valuesByKey(entries).get("WEB_RUNTIME_SURFACE") === "portal"); if (webPortalMatches.length !== 1) { fail("Coolify must expose exactly one allowlisted hivegrid-portal surface for custom-domain routing"); } assertStagingWebCustomDomainRuntime(webPortalMatches[0].entries, environment, WEB_SURFACE_IDENTITIES.portal); assertWebSurfacePair(webCoreMatches[0].app, webPortalMatches[0].app); if (controllerContract.backendService !== portalBackendService(webPortalMatches[0].app)) { fail("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE must target the exact allowlisted hivegrid-portal Docker service"); } return { controller, expectedControllerImage }; } async function assertRuntime(environment, resourceUuid, allowedCsv, controllerImage, workerImage) { await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage); process.stdout.write("Portal-domain controller runtime, mounts, and worker-general HMAC wiring verified\n"); } async function snapshotRelease(environment, resourceUuid, allowedCsv, controllerImage, workerImage) { const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage); const revision = normalizeValue(controller?.docker_registry_image_tag); if (!FULL_GIT_SHA_PATTERN.test(revision)) { fail("Portal-domain controller previous release must be a full lowercase Git SHA"); } assertReleaseConfig(controller, resourceUuid, expectedControllerImage, revision, expectedMetricsBindingForEnvironment(environment)); process.stdout.write(revision); } function assertPollingContract() { if (!Number.isInteger(pollDelayMs) || pollDelayMs < 1 || pollDelayMs > 60_000) { fail("COOLIFY_DEPLOYMENT_POLL_MS must be an integer between 1 and 60000"); } if (!Number.isInteger(pollAttempts) || pollAttempts < 1 || pollAttempts > 180) { fail("COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS must be an integer between 1 and 180"); } if (!Number.isSafeInteger(failedStatusGraceMs) || failedStatusGraceMs < 0) { fail("COOLIFY_FAILED_STATUS_GRACE_MS must be a non-negative safe integer"); } } async function probeControllerHealth(healthUrl, expectedRevision) { let response; try { response = await fetch(healthUrl, { cache: "no-store", redirect: "error", signal: AbortSignal.timeout(CONTROLLER_HEALTH_PROBE_TIMEOUT_MS), }); } catch { return { reachable: false, revisionBound: false }; } if (!response.ok) return { reachable: true, revisionBound: false }; let payload; try { payload = await response.json(); } catch { return { reachable: true, revisionBound: false }; } return { reachable: true, revisionBound: payload?.status === "ok" && payload?.service === CONTROLLER_NAME && normalizeValue(payload?.revision) === expectedRevision, }; } async function stopAndDrain(environment, resourceUuid, allowedCsv, controllerImage, workerImage) { assertPollingContract(); if (pollAttempts < CONTROLLER_DRAIN_CONFIRMATIONS) { fail(`Portal-domain controller drain requires at least ${CONTROLLER_DRAIN_CONFIRMATIONS} polling attempts`); } const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage); const expectedMetricsBinding = expectedMetricsBindingForEnvironment(environment); const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`; const expectedRevision = normalizeValue(controller?.docker_registry_image_tag); if (!FULL_GIT_SHA_PATTERN.test(expectedRevision)) { fail("Portal-domain controller drain requires the current immutable full-SHA release"); } const healthUrl = controllerHealthUrlForEnvironment(environment); const requiredDrainFenceMs = controllerDrainFenceMs(); const beforeStop = await probeControllerHealth(healthUrl, expectedRevision); if (!beforeStop.reachable || !beforeStop.revisionBound) { fail("Portal-domain controller pre-stop health is unreachable or not revision-bound; operator recovery is required"); } // Coolify queues this action and its application status may be stale. The // pre-stop revision-bound mesh probe proves which worker is live. After the // request, both Coolify's stopped state and health-port disappearance must be // observed twice. The first rollout also keeps that evidence continuous for // 31 seconds, beyond the legacy controller's 30-second hard shutdown; future // releases additionally close /healthz only after queue and lock drainage. await coolify(`${resourcePath}/stop`, { method: "POST" }); let drainConfirmations = 0; let drainEvidenceSinceMs; for (let attempt = 1; attempt <= pollAttempts; attempt += 1) { const current = await coolify(resourcePath); assertPrivateControllerIdentity(current, resourceUuid, expectedControllerImage, expectedMetricsBinding); const afterStop = await probeControllerHealth(healthUrl, expectedRevision); if (STOPPED_RUNTIME_STATUSES.test(normalizeValue(current?.status)) && !afterStop.reachable) { drainConfirmations += 1; drainEvidenceSinceMs ??= performance.now(); } else { drainConfirmations = 0; drainEvidenceSinceMs = undefined; } if ( drainConfirmations >= CONTROLLER_DRAIN_CONFIRMATIONS && drainEvidenceSinceMs !== undefined && performance.now() - drainEvidenceSinceMs >= requiredDrainFenceMs ) { process.stdout.write("Portal-domain controller stopped and command worker drained before web rollout\n"); return; } if (attempt < pollAttempts) await new Promise((resolve) => setTimeout(resolve, pollDelayMs)); } fail("Portal-domain controller did not stop and drain before the web rollout"); } function controllerHealthCommand(revision) { return `node ${CONTROLLER_DEPLOY_HEALTH_PATH} ${revision}`; } function releasePayload(revision) { return { pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: revision, health_check_enabled: true, health_check_type: "cmd", health_check_command: controllerHealthCommand(revision), health_check_interval: CONTROLLER_HEALTH_INTERVAL_SECONDS, health_check_timeout: CONTROLLER_HEALTH_TIMEOUT_SECONDS, health_check_retries: CONTROLLER_HEALTH_RETRIES, health_check_start_period: CONTROLLER_HEALTH_START_PERIOD_SECONDS, }; } function snapshotReleaseNumber(value, key) { if (value === null || value === undefined || normalizeValue(value) === "") { fail(`Portal-domain controller existing ${key} configuration is missing`); } const number = Number(value); if (!Number.isFinite(number)) { fail(`Portal-domain controller existing ${key} configuration is invalid`); } return number; } function previousReleasePayload(app) { const healthCheckEnabled = normalizeBoolean(app?.health_check_enabled); if (!healthCheckEnabled) { fail("Portal-domain controller existing health_check_enabled configuration is invalid"); } return { pre_deployment_command: normalizeValue(app?.pre_deployment_command), post_deployment_command: normalizeValue(app?.post_deployment_command), docker_registry_image_tag: normalizeValue(app?.docker_registry_image_tag), health_check_enabled: healthCheckEnabled === "true", health_check_type: normalizeValue(app?.health_check_type), health_check_command: normalizeValue(app?.health_check_command), health_check_interval: snapshotReleaseNumber(app?.health_check_interval, "health_check_interval"), health_check_timeout: snapshotReleaseNumber(app?.health_check_timeout, "health_check_timeout"), health_check_retries: snapshotReleaseNumber(app?.health_check_retries, "health_check_retries"), health_check_start_period: snapshotReleaseNumber(app?.health_check_start_period, "health_check_start_period"), }; } function normalizedMutableReleaseConfig(app) { return { pre_deployment_command: normalizeValue(app?.pre_deployment_command), post_deployment_command: normalizeValue(app?.post_deployment_command), docker_registry_image_tag: normalizeValue(app?.docker_registry_image_tag), health_check_enabled: normalizeBoolean(app?.health_check_enabled), health_check_type: normalizeValue(app?.health_check_type), health_check_command: normalizeValue(app?.health_check_command), health_check_interval: Number(app?.health_check_interval), health_check_timeout: Number(app?.health_check_timeout), health_check_retries: Number(app?.health_check_retries), health_check_start_period: Number(app?.health_check_start_period), }; } function mutableReleaseConfigDrift(app, expected) { const actualConfig = normalizedMutableReleaseConfig(app); const expectedConfig = normalizedMutableReleaseConfig(expected); return Object.keys(expectedConfig).filter((key) => actualConfig[key] !== expectedConfig[key]); } function assertReleaseConfig(app, resourceUuid, image, revision, expectedMetricsBinding) { assertPrivateControllerIdentity(app, resourceUuid, image, expectedMetricsBinding); const actual = { ...normalizedMutableReleaseConfig(app), custom_healthcheck_found: normalizeBoolean(app?.custom_healthcheck_found), }; const expected = { ...releasePayload(revision), health_check_enabled: "true", custom_healthcheck_found: "false", }; const drift = Object.keys(expected).filter((key) => actual[key] !== expected[key]); if (drift.length > 0) { fail("Portal-domain controller release or revision-bound command health configuration drifted"); } } async function setRelease(environment, resourceUuid, allowedCsv, controllerImage, workerImage, revision) { if (!FULL_GIT_SHA_PATTERN.test(revision)) { fail("Portal-domain controller release requires a full lowercase Git SHA"); } const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage); const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`; const previous = previousReleasePayload(controller); try { await coolify(resourcePath, { method: "PATCH", body: JSON.stringify(releasePayload(revision)) }); const updated = await coolify(resourcePath); assertReleaseConfig(updated, resourceUuid, expectedControllerImage, revision, expectedMetricsBindingForEnvironment(environment)); } catch { let restored = false; try { await coolify(resourcePath, { method: "PATCH", body: JSON.stringify(previous) }); const restoredApp = await coolify(resourcePath); restored = mutableReleaseConfigDrift(restoredApp, previous).length === 0; } catch { restored = false; } fail( restored ? "Portal-domain controller release mutation failed and previous configuration was restored" : "Portal-domain controller release mutation failed and compensation was incomplete" ); } process.stdout.write("Portal-domain controller immutable release configured\n"); } function deploymentStatus(payload) { return normalizeValue(payload?.status || payload?.deployment?.status || payload?.data?.status).toLowerCase(); } function assertCompletedDeployment(payload, deploymentUuid, app) { if (normalizeValue(payload?.deployment_uuid) !== deploymentUuid) { fail("Portal-domain controller deployment identity does not match the triggered deployment"); } if (!COMPLETED_DEPLOYMENT_STATUSES.has(deploymentStatus(payload))) { fail("Portal-domain controller deployment is not completed"); } if (normalizeValue(payload?.application_name) !== CONTROLLER_NAME) { fail("Portal-domain controller deployment belongs to an unexpected application"); } const applicationId = normalizeValue(app?.id); const deploymentApplicationId = normalizeValue(payload?.application_id); if (applicationId && deploymentApplicationId && applicationId !== deploymentApplicationId) { fail("Portal-domain controller deployment application identity does not match the live resource"); } } async function assertRelease(environment, resourceUuid, allowedCsv, controllerImage, workerImage, revision, deploymentUuid) { if (!FULL_GIT_SHA_PATTERN.test(revision)) { fail("Portal-domain controller release verification requires a full lowercase Git SHA"); } requireCoolifyCuid(deploymentUuid, "Portal-domain controller deployment UUID"); assertPollingContract(); const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage); const deployment = await coolify(`/deployments/${encodeURIComponent(deploymentUuid)}`); assertCompletedDeployment(deployment, deploymentUuid, controller); const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`; // The controller is deliberately stopped-and-drained before the rollout, so Coolify's // cached application status can still read "exited" for a while after the deployment // reports finished. Keep the fail-closed boundary time-based so a polling-frequency // change cannot shorten the stale-status grace period. let failedStatusSinceMs; for (let attempt = 1; attempt <= pollAttempts; attempt += 1) { const current = await coolify(resourcePath); assertReleaseConfig(current, resourceUuid, expectedControllerImage, revision, expectedMetricsBindingForEnvironment(environment)); const status = normalizeValue(current?.status).toLowerCase(); if (status === "running:healthy") { process.stdout.write("Portal-domain controller revision-bound healthz, readyz, and immutable revision verified\n"); return; } if (FAILED_RUNTIME_STATUSES.test(status)) { const nowMs = performance.now(); failedStatusSinceMs ??= nowMs; if (nowMs - failedStatusSinceMs >= failedStatusGraceMs) { fail("Portal-domain controller entered a terminal runtime state"); } } else { failedStatusSinceMs = undefined; } if (attempt < pollAttempts) await new Promise((resolve) => setTimeout(resolve, pollDelayMs)); } fail("Portal-domain controller did not become running:healthy in time"); } try { if (command === "assert-runtime") { await assertRuntime(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || ""); } else if (command === "snapshot-release") { await snapshotRelease(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || ""); } else if (command === "stop-and-drain") { await stopAndDrain(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || ""); } else if (command === "set-release") { await setRelease(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || "", args[5] || ""); } else if (command === "assert-release") { await assertRelease(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || "", args[5] || "", args[6] || ""); } else { fail( "Usage: coolify-portal-domain-controller.mjs assert-runtime|snapshot-release|stop-and-drain | set-release | assert-release " ); } } catch (error) { process.stderr.write(`${error instanceof Error ? error.message : "Portal-domain controller Coolify helper failed"}\n`); process.exitCode = 1; }