825 lines
37 KiB
JavaScript
825 lines
37 KiB
JavaScript
#!/usr/bin/env node
|
|
import { timingSafeEqual } from "node:crypto";
|
|
import { domainToASCII } from "node:url";
|
|
|
|
const [, , command, ...args] = process.argv;
|
|
|
|
const apiUrl = (process.env.SELECTED_COOLIFY_API_URL || process.env.COOLIFY_STAGING_API_URL || process.env.COOLIFY_API_URL || "").replace(/\/+$/, "");
|
|
const token = process.env.SELECTED_COOLIFY_TOKEN || process.env.COOLIFY_STAGING_TOKEN || process.env.COOLIFY_TOKEN || "";
|
|
const apiTimeoutMs = Number(process.env.COOLIFY_API_TIMEOUT_MS || "20000");
|
|
const pollDelayMs = Number(process.env.COOLIFY_DEPLOYMENT_POLL_MS || "5000");
|
|
const pollAttempts = Number(process.env.COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS || "90");
|
|
const failedStatusGraceMs = Number(process.env.COOLIFY_FAILED_STATUS_GRACE_MS || "60000");
|
|
const drainHealthUrlOverride = (process.env.COOLIFY_CONTROLLER_DRAIN_HEALTH_URL || "").trim();
|
|
const drainFenceMsOverride = (process.env.COOLIFY_CONTROLLER_DRAIN_FENCE_MS || "").trim();
|
|
|
|
const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/u;
|
|
const COOLIFY_CUID_PATTERN = /^[a-z][a-z0-9]{23}$/u;
|
|
const CONTROLLER_NAME = "hivegrid-portal-domain-controller";
|
|
const CONTROLLER_DEPLOY_HEALTH_PATH = "apps/portal-domain-controller/dist/deploy-health.js";
|
|
const CONTROLLER_HEALTH_INTERVAL_SECONDS = 30;
|
|
const CONTROLLER_HEALTH_TIMEOUT_SECONDS = 10;
|
|
const CONTROLLER_HEALTH_RETRIES = 3;
|
|
const CONTROLLER_HEALTH_START_PERIOD_SECONDS = 20;
|
|
const CONTROLLER_DRAIN_CONFIRMATIONS = 2;
|
|
const CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 31_000;
|
|
const CONTROLLER_HEALTH_PROBE_TIMEOUT_MS = 5_000;
|
|
const CONTROLLER_INTERNAL_PORT = "8080";
|
|
const WEB_SURFACE_IDENTITIES = Object.freeze({
|
|
core: "hivegrid-web",
|
|
portal: "hivegrid-portal",
|
|
});
|
|
const PORTAL_CNAME_BY_ENVIRONMENT = Object.freeze({
|
|
staging: "staging-domains.portal.hivegrid.de",
|
|
production: "domains.portal.hivegrid.de",
|
|
});
|
|
const CONTROLLER_METRICS_BINDING_BY_ENVIRONMENT = Object.freeze({
|
|
staging: "10.42.0.3:19120:8080",
|
|
production: "10.42.0.8:19120:8080",
|
|
});
|
|
const PORTAL_TRAEFIK_ENTRYPOINT = "https";
|
|
const CONTROLLER_CONFIG_DIR = "/data/coolify/proxy/dynamic/hivegrid-portal-domains";
|
|
const CONTROLLER_RELOAD_TRIGGER_FILE = "/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload";
|
|
const COMPLETED_DEPLOYMENT_STATUSES = new Set(["finished", "success", "succeeded", "completed", "done"]);
|
|
const FAILED_RUNTIME_STATUSES = /failed|exited|dead|cancelled|canceled/iu;
|
|
const STOPPED_RUNTIME_STATUSES = /^(?:stopped|exited(?::[^:]*)?)$/iu;
|
|
const CONTROLLER_REQUIRED_ENV_KEYS = Object.freeze([
|
|
"PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET",
|
|
"PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN",
|
|
"PORTAL_DOMAIN_CONTROLLER_REDIS_URL",
|
|
"PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME",
|
|
"PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE",
|
|
"PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR",
|
|
"PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE",
|
|
]);
|
|
const CONTROLLER_ALLOWED_ENV_KEYS = new Set([
|
|
...CONTROLLER_REQUIRED_ENV_KEYS,
|
|
"PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT",
|
|
"PORTAL_DOMAIN_CONTROLLER_HOST",
|
|
"PORTAL_DOMAIN_CONTROLLER_PORT",
|
|
"PORTAL_DOMAIN_CONTROLLER_CONCURRENCY",
|
|
]);
|
|
const CONTROLLER_COOLIFY_AUTOMATIC_ENV_KEYS = new Set(["COOLIFY_BRANCH", "COOLIFY_CONTAINER_NAME", "COOLIFY_FQDN", "COOLIFY_RESOURCE_UUID", "COOLIFY_URL"]);
|
|
const CONTROLLER_FORBIDDEN_ENV_KEYS = new Set([
|
|
"APP_REVISION",
|
|
"DATABASE_URL",
|
|
"DATABASE_READ_URL",
|
|
"DIRECT_URL",
|
|
"DOCKER_HOST",
|
|
"PGHOST",
|
|
"PGPORT",
|
|
"PGDATABASE",
|
|
"PGUSER",
|
|
"PGPASSWORD",
|
|
"CLOUDFLARE_API_TOKEN",
|
|
"CLOUDFLARE_API_KEY",
|
|
"COOLIFY_TOKEN",
|
|
"SSH_AUTH_SOCK",
|
|
]);
|
|
const CONTROLLER_FORBIDDEN_ENV_PREFIXES = Object.freeze(["MIGRATOR_", "SEED_", "COOLIFY_", "CLOUDFLARE_", "SSH_"]);
|
|
|
|
function fail(message) {
|
|
throw new Error(message);
|
|
}
|
|
|
|
async function coolify(path, options = {}) {
|
|
if (!apiUrl || !token) fail("Coolify API URL and token are required");
|
|
if (!Number.isInteger(apiTimeoutMs) || apiTimeoutMs < 10 || apiTimeoutMs > 120_000) {
|
|
fail("COOLIFY_API_TIMEOUT_MS must be an integer between 10 and 120000");
|
|
}
|
|
|
|
let response;
|
|
try {
|
|
response = await fetch(`${apiUrl}${path}`, {
|
|
...options,
|
|
signal: options.signal ?? AbortSignal.timeout(apiTimeoutMs),
|
|
headers: {
|
|
Authorization: `Bearer ${token}`,
|
|
Accept: "application/json",
|
|
...(options.body ? { "Content-Type": "application/json" } : {}),
|
|
...options.headers,
|
|
},
|
|
});
|
|
} catch {
|
|
fail("Coolify API request failed or timed out");
|
|
}
|
|
|
|
const body = await response.text();
|
|
let payload = {};
|
|
if (body.trim()) {
|
|
try {
|
|
payload = JSON.parse(body);
|
|
} catch {
|
|
payload = {};
|
|
}
|
|
}
|
|
if (!response.ok) fail(`Coolify API request failed with HTTP ${response.status}`);
|
|
return payload;
|
|
}
|
|
|
|
function normalizeValue(value) {
|
|
return value === null || value === undefined ? "" : String(value).trim();
|
|
}
|
|
|
|
function normalizeBoolean(value) {
|
|
if (value === true || value === "true" || value === 1 || value === "1") return "true";
|
|
if (value === false || value === "false" || value === 0 || value === "0") return "false";
|
|
return "";
|
|
}
|
|
|
|
function normalizeEnvValue(entry) {
|
|
let value = normalizeValue(entry?.value ?? entry?.real_value);
|
|
while (value.length >= 2 && ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"')))) {
|
|
value = value.slice(1, -1).trim();
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function requireCoolifyCuid(value, label) {
|
|
if (!COOLIFY_CUID_PATTERN.test(value)) fail(`${label} must be a valid Coolify CUID`);
|
|
}
|
|
|
|
function allowedResourceSet(csv) {
|
|
const values = csv
|
|
.split(",")
|
|
.map((value) => value.trim())
|
|
.filter(Boolean);
|
|
if (values.length === 0) fail("Portal-domain controller deployment requires an explicit Coolify resource allowlist");
|
|
const resources = new Set(values);
|
|
if (resources.size !== values.length) fail("Coolify resource allowlist must not contain duplicates");
|
|
for (const value of resources) requireCoolifyCuid(value, "Coolify resource allowlist entry");
|
|
return resources;
|
|
}
|
|
|
|
function arrayPayload(payload, label) {
|
|
if (Array.isArray(payload)) return payload;
|
|
if (Array.isArray(payload?.data)) return payload.data;
|
|
fail(`Coolify ${label} response was malformed`);
|
|
}
|
|
|
|
function runtimeEnvEntries(payload) {
|
|
const entries = [];
|
|
const keys = new Set();
|
|
for (const entry of arrayPayload(payload, "environment")) {
|
|
if (normalizeBoolean(entry?.is_preview) === "true") continue;
|
|
const key = normalizeValue(entry?.key);
|
|
if (!key) continue;
|
|
if (keys.has(key)) fail("Coolify runtime environment contains duplicate non-preview keys");
|
|
keys.add(key);
|
|
entries.push(entry);
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
function runtimeStorageEntries(payload) {
|
|
let entries;
|
|
if (Array.isArray(payload) || Array.isArray(payload?.data)) {
|
|
entries = arrayPayload(payload, "storage");
|
|
} else if (Array.isArray(payload?.file_storages) && Array.isArray(payload?.persistent_storages)) {
|
|
entries = [
|
|
...payload.file_storages.map((entry) => ({ ...entry, type: "file" })),
|
|
...payload.persistent_storages.map((entry) => ({ ...entry, type: "persistent" })),
|
|
];
|
|
} else {
|
|
fail("Coolify storage response was malformed");
|
|
}
|
|
return entries.filter((entry) => normalizeBoolean(entry?.is_preview) !== "true");
|
|
}
|
|
|
|
function valuesByKey(entries) {
|
|
return new Map(entries.map((entry) => [normalizeValue(entry?.key), normalizeEnvValue(entry)]));
|
|
}
|
|
|
|
function normalizedImage(value) {
|
|
return normalizeValue(value).replace(/\/+$/, "");
|
|
}
|
|
|
|
function assertPrivateControllerIdentity(app, resourceUuid, expectedImage, expectedMetricsBinding) {
|
|
if (
|
|
app?.uuid !== resourceUuid ||
|
|
normalizeValue(app?.name) !== CONTROLLER_NAME ||
|
|
normalizedImage(app?.docker_registry_image_name) !== expectedImage ||
|
|
normalizeValue(app?.build_pack) !== "dockerimage"
|
|
) {
|
|
fail("Portal-domain controller UUID, name, image, or dockerimage build-pack identity does not match");
|
|
}
|
|
const routeValues = [app?.fqdn, app?.domain, app?.domains]
|
|
.flatMap((value) => (Array.isArray(value) ? value : [value]))
|
|
.flatMap((value) => normalizeValue(value).split(/[\s,]+/u))
|
|
.filter(Boolean);
|
|
if (routeValues.length > 0) fail("Portal-domain controller must remain private without a public Coolify route");
|
|
if (normalizeValue(app?.ports_mappings) !== expectedMetricsBinding) {
|
|
fail(`Portal-domain controller must expose metrics only through the exact mesh binding ${expectedMetricsBinding}`);
|
|
}
|
|
}
|
|
|
|
function storagePath(entry, ...keys) {
|
|
for (const key of keys) {
|
|
const value = normalizeValue(entry?.[key]);
|
|
if (value) return value;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function assertControllerStorageContract(entries) {
|
|
if (entries.length !== 2) {
|
|
fail("Portal-domain controller requires exactly two narrow bind mounts");
|
|
}
|
|
const normalized = entries.map((entry) => ({
|
|
type: normalizeValue(entry?.type),
|
|
directory: normalizeBoolean(entry?.is_directory),
|
|
source: storagePath(entry, "fs_path", "host_path", "source"),
|
|
destination: storagePath(entry, "mount_path", "destination", "target"),
|
|
readOnly: normalizeBoolean(entry?.is_read_only ?? entry?.read_only),
|
|
}));
|
|
const configDirectory = normalized.filter(
|
|
(entry) =>
|
|
entry.type === "file" &&
|
|
entry.directory === "true" &&
|
|
entry.source === CONTROLLER_CONFIG_DIR &&
|
|
entry.destination === CONTROLLER_CONFIG_DIR &&
|
|
entry.readOnly !== "true"
|
|
);
|
|
const reloadSentinel = normalized.filter(
|
|
(entry) =>
|
|
(entry.type === "file" || entry.type === "persistent") &&
|
|
entry.directory !== "true" &&
|
|
entry.source === CONTROLLER_RELOAD_TRIGGER_FILE &&
|
|
entry.destination === CONTROLLER_RELOAD_TRIGGER_FILE &&
|
|
entry.readOnly !== "true"
|
|
);
|
|
if (configDirectory.length !== 1 || reloadSentinel.length !== 1) {
|
|
fail("Portal-domain controller bind mounts must expose only its RW config subdirectory and RW reload sentinel file");
|
|
}
|
|
}
|
|
|
|
function requireSecret(values, key) {
|
|
const value = values.get(key) || "";
|
|
const bytes = Buffer.byteLength(value, "utf8");
|
|
if (bytes < 32 || bytes > 512) fail(`${key} must contain between 32 and 512 bytes`);
|
|
return value;
|
|
}
|
|
|
|
function equalSecret(left, right) {
|
|
const leftBuffer = Buffer.from(left, "utf8");
|
|
const rightBuffer = Buffer.from(right, "utf8");
|
|
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
|
|
}
|
|
|
|
function redisQueueIdentity(value, label) {
|
|
let parsed;
|
|
try {
|
|
parsed = new URL(value);
|
|
} catch {
|
|
fail(`${label} must be a valid Redis connection URL`);
|
|
}
|
|
if (!new Set(["redis:", "rediss:"]).has(parsed.protocol) || !parsed.hostname || parsed.hash) {
|
|
fail(`${label} must be a valid Redis connection URL`);
|
|
}
|
|
const database = parsed.pathname.replace(/^\//u, "") || "0";
|
|
if (!/^\d+$/u.test(database)) fail(`${label} must select a numeric Redis database`);
|
|
return `${parsed.protocol}//${parsed.hostname.toLowerCase()}:${parsed.port || "6379"}/${database}`;
|
|
}
|
|
|
|
function assertHostname(value) {
|
|
const normalized = value.toLowerCase().replace(/\.$/u, "");
|
|
const ascii = domainToASCII(normalized);
|
|
if (
|
|
!ascii ||
|
|
ascii.length > 253 ||
|
|
ascii.includes("*") ||
|
|
!ascii.includes(".") ||
|
|
ascii.split(".").some((label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/u.test(label))
|
|
) {
|
|
fail("PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME must be an exact DNS hostname");
|
|
}
|
|
}
|
|
|
|
function assertBackendService(value) {
|
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,126}@docker$/u.test(value)) {
|
|
fail("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE must be a provider-qualified Docker service name");
|
|
}
|
|
}
|
|
|
|
function assertEntryPoint(value) {
|
|
const entryPoint = value || PORTAL_TRAEFIK_ENTRYPOINT;
|
|
if (entryPoint !== PORTAL_TRAEFIK_ENTRYPOINT) {
|
|
fail("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT must be exactly https");
|
|
}
|
|
return entryPoint;
|
|
}
|
|
|
|
function assertWebSurfacePair(core, portal) {
|
|
const expected = [
|
|
[core, WEB_SURFACE_IDENTITIES.core],
|
|
[portal, WEB_SURFACE_IDENTITIES.portal],
|
|
];
|
|
for (const [app, expectedName] of expected) {
|
|
requireCoolifyCuid(app?.uuid || "", `${expectedName} resource UUID`);
|
|
if (normalizeValue(app?.name) !== expectedName || normalizeValue(app?.build_pack) !== "dockerimage" || !normalizedImage(app?.docker_registry_image_name)) {
|
|
fail("Coolify web surface identity must match its explicit name, dockerimage build pack, and image");
|
|
}
|
|
}
|
|
if (core.uuid === portal.uuid || normalizedImage(core.docker_registry_image_name) !== normalizedImage(portal.docker_registry_image_name)) {
|
|
fail("hivegrid-web and hivegrid-portal must be distinct resources using the same immutable web image");
|
|
}
|
|
}
|
|
|
|
function portalBackendService(portal) {
|
|
requireCoolifyCuid(portal?.uuid || "", "hivegrid-portal resource UUID");
|
|
return `https-0-${portal.uuid}@docker`;
|
|
}
|
|
|
|
function expectedCnameForEnvironment(environment) {
|
|
const expectedCname = PORTAL_CNAME_BY_ENVIRONMENT[environment];
|
|
if (!expectedCname) {
|
|
fail("Portal-domain controller deployment environment must be exactly staging or production");
|
|
}
|
|
return expectedCname;
|
|
}
|
|
|
|
function expectedMetricsBindingForEnvironment(environment) {
|
|
const binding = CONTROLLER_METRICS_BINDING_BY_ENVIRONMENT[environment];
|
|
if (!binding) {
|
|
fail("Portal-domain controller deployment environment must be exactly staging or production");
|
|
}
|
|
return binding;
|
|
}
|
|
|
|
function controllerHealthUrlForEnvironment(environment) {
|
|
const [host, publishedPort, internalPort, ...rest] = expectedMetricsBindingForEnvironment(environment).split(":");
|
|
if (rest.length > 0 || !/^10\.42\.0\.\d+$/u.test(host || "") || !/^\d{2,5}$/u.test(publishedPort || "") || internalPort !== CONTROLLER_INTERNAL_PORT) {
|
|
fail("Portal-domain controller mesh health binding is invalid");
|
|
}
|
|
if (!drainHealthUrlOverride) return `http://${host}:${publishedPort}/healthz`;
|
|
if (process.env.NODE_ENV !== "test") fail("Portal-domain controller drain health override is test-only");
|
|
let override;
|
|
let selectedApi;
|
|
try {
|
|
override = new URL(drainHealthUrlOverride);
|
|
selectedApi = new URL(apiUrl);
|
|
} catch {
|
|
fail("Portal-domain controller test drain health URL is invalid");
|
|
}
|
|
const loopbackHosts = new Set(["127.0.0.1", "localhost", "::1"]);
|
|
if (
|
|
override.protocol !== "http:" ||
|
|
!loopbackHosts.has(override.hostname) ||
|
|
!loopbackHosts.has(selectedApi.hostname) ||
|
|
override.origin !== selectedApi.origin ||
|
|
override.username ||
|
|
override.password ||
|
|
override.search ||
|
|
override.hash
|
|
) {
|
|
fail("Portal-domain controller test drain health URL must be loopback-only HTTP");
|
|
}
|
|
return override.href;
|
|
}
|
|
|
|
function controllerDrainFenceMs() {
|
|
if (!drainFenceMsOverride) return CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS;
|
|
if (process.env.NODE_ENV !== "test" || !drainHealthUrlOverride) {
|
|
fail("Portal-domain controller drain fence override is test-only");
|
|
}
|
|
const value = Number(drainFenceMsOverride);
|
|
if (!Number.isSafeInteger(value) || value < 1 || value > 1_000) {
|
|
fail("Portal-domain controller test drain fence must be an integer between 1 and 1000");
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function assertControllerEnv(entries, expectedCname) {
|
|
const values = valuesByKey(entries);
|
|
for (const key of CONTROLLER_REQUIRED_ENV_KEYS) {
|
|
if (!values.get(key)) fail(`Portal-domain controller runtime is missing ${key}`);
|
|
}
|
|
const forbidden = [...values.keys()].filter(
|
|
(key) =>
|
|
!CONTROLLER_COOLIFY_AUTOMATIC_ENV_KEYS.has(key) &&
|
|
(CONTROLLER_FORBIDDEN_ENV_KEYS.has(key) || CONTROLLER_FORBIDDEN_ENV_PREFIXES.some((prefix) => key.startsWith(prefix)))
|
|
);
|
|
if (forbidden.length > 0) fail("Portal-domain controller runtime contains forbidden privileged environment keys");
|
|
const unexpected = [...values.keys()].filter((key) => !CONTROLLER_ALLOWED_ENV_KEYS.has(key) && !CONTROLLER_COOLIFY_AUTOMATIC_ENV_KEYS.has(key));
|
|
if (unexpected.length > 0) {
|
|
fail("Portal-domain controller runtime must contain only its explicit least-privilege environment allowlist");
|
|
}
|
|
const configuredPort = values.get("PORTAL_DOMAIN_CONTROLLER_PORT") || CONTROLLER_INTERNAL_PORT;
|
|
if (configuredPort !== CONTROLLER_INTERNAL_PORT) {
|
|
fail(`PORTAL_DOMAIN_CONTROLLER_PORT must be exactly ${CONTROLLER_INTERNAL_PORT} for the mesh metrics binding`);
|
|
}
|
|
|
|
const commandSecret = requireSecret(values, "PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET");
|
|
const metricsToken = requireSecret(values, "PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN");
|
|
if (equalSecret(commandSecret, metricsToken)) fail("Portal-domain controller command and metrics secrets must differ");
|
|
const redisIdentity = redisQueueIdentity(values.get("PORTAL_DOMAIN_CONTROLLER_REDIS_URL") || "", "PORTAL_DOMAIN_CONTROLLER_REDIS_URL");
|
|
const configuredCname = values.get("PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME") || "";
|
|
assertHostname(configuredCname);
|
|
if (configuredCname !== expectedCname) {
|
|
fail(`PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME must be exactly ${expectedCname} for this deployment environment`);
|
|
}
|
|
const backendService = values.get("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE") || "";
|
|
assertBackendService(backendService);
|
|
const entryPoint = assertEntryPoint(values.get("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT") || "");
|
|
if (values.get("PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR") !== CONTROLLER_CONFIG_DIR) {
|
|
fail("PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR must use the dedicated Traefik subdirectory");
|
|
}
|
|
if (values.get("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE") !== CONTROLLER_RELOAD_TRIGGER_FILE) {
|
|
fail("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must use the single watched-root sentinel file");
|
|
}
|
|
return { commandSecret, redisIdentity, expectedCname: configuredCname, backendService, entryPoint };
|
|
}
|
|
|
|
function assertWebCnameEnv(entries, expectedCname) {
|
|
const values = valuesByKey(entries);
|
|
if (values.get("PORTAL_CUSTOM_DOMAIN_CNAME_TARGET") !== expectedCname) {
|
|
fail(`hivegrid-web must use exactly ${expectedCname} for this deployment environment`);
|
|
}
|
|
}
|
|
|
|
function assertStagingWebCustomDomainRuntime(entries, environment, surfaceName) {
|
|
if (environment !== "staging") return;
|
|
if (valuesByKey(entries).get("PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED") !== "true") {
|
|
fail(`${surfaceName} must set PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED exactly to true in staging`);
|
|
}
|
|
}
|
|
|
|
function assertWorkerGeneralEnv(entries, controllerContract) {
|
|
const values = valuesByKey(entries);
|
|
if (values.get("WORKER_QUEUE_GROUP") !== "general") fail("Portal-domain orchestration must use worker-general");
|
|
if (values.get("PORTAL_DOMAIN_ORCHESTRATION_REQUIRED") !== "true") {
|
|
fail("worker-general must set PORTAL_DOMAIN_ORCHESTRATION_REQUIRED=true");
|
|
}
|
|
if (values.has("PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN")) {
|
|
fail("worker-general must not receive the portal-domain controller metrics token");
|
|
}
|
|
const commandSecret = requireSecret(values, "PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET");
|
|
if (!equalSecret(commandSecret, controllerContract.commandSecret)) {
|
|
fail("worker-general and portal-domain controller command HMAC secrets must match");
|
|
}
|
|
const workerRedisUrl = values.get("BULLMQ_REDIS_URL") || values.get("REDIS_URL") || "";
|
|
const workerRedisIdentity = redisQueueIdentity(workerRedisUrl, "worker-general Redis URL");
|
|
if (workerRedisIdentity !== controllerContract.redisIdentity) {
|
|
fail("worker-general and portal-domain controller must use the same Redis queue endpoint and database");
|
|
}
|
|
}
|
|
|
|
async function resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage) {
|
|
const expectedCname = expectedCnameForEnvironment(environment);
|
|
const expectedMetricsBinding = expectedMetricsBindingForEnvironment(environment);
|
|
requireCoolifyCuid(resourceUuid, "Portal-domain controller resource UUID");
|
|
const allowed = allowedResourceSet(allowedCsv);
|
|
if (!allowed.has(resourceUuid)) fail("Portal-domain controller resource UUID is outside the Coolify allowlist");
|
|
const expectedControllerImage = normalizedImage(controllerImage);
|
|
const expectedWorkerImage = normalizedImage(workerImage);
|
|
if (!expectedControllerImage || !expectedWorkerImage) fail("Controller and worker image identities are required");
|
|
|
|
const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`;
|
|
const [appsPayload, controller, controllerEnvPayload, storagePayload] = await Promise.all([
|
|
coolify("/applications"),
|
|
coolify(resourcePath),
|
|
coolify(`${resourcePath}/envs`),
|
|
coolify(`${resourcePath}/storages`),
|
|
]);
|
|
const apps = arrayPayload(appsPayload, "applications");
|
|
if (!apps.some((app) => app?.uuid === resourceUuid)) {
|
|
fail("Portal-domain controller resource is absent from the Coolify application inventory");
|
|
}
|
|
assertPrivateControllerIdentity(controller, resourceUuid, expectedControllerImage, expectedMetricsBinding);
|
|
assertControllerStorageContract(runtimeStorageEntries(storagePayload));
|
|
const controllerContract = assertControllerEnv(runtimeEnvEntries(controllerEnvPayload), expectedCname);
|
|
|
|
const runtimeCandidatesWithEnv = await Promise.all(
|
|
apps
|
|
.filter((app) => allowed.has(app?.uuid) && app?.uuid !== resourceUuid)
|
|
.map(async (app) => ({
|
|
app,
|
|
entries: runtimeEnvEntries(await coolify(`/applications/${encodeURIComponent(app.uuid)}/envs`)),
|
|
}))
|
|
);
|
|
const workerGeneralMatches = runtimeCandidatesWithEnv.filter(
|
|
({ app, entries }) =>
|
|
normalizedImage(app?.docker_registry_image_name) === expectedWorkerImage && valuesByKey(entries).get("WORKER_QUEUE_GROUP") === "general"
|
|
);
|
|
if (workerGeneralMatches.length !== 1) {
|
|
fail("Coolify must expose exactly one allowlisted worker-general for portal-domain orchestration");
|
|
}
|
|
assertWorkerGeneralEnv(workerGeneralMatches[0].entries, controllerContract);
|
|
const webCoreMatches = runtimeCandidatesWithEnv.filter(({ entries }) => valuesByKey(entries).get("WEB_RUNTIME_SURFACE") === "core");
|
|
if (webCoreMatches.length !== 1) {
|
|
fail("Coolify must expose exactly one allowlisted hivegrid-web core surface for CNAME verification");
|
|
}
|
|
assertWebCnameEnv(webCoreMatches[0].entries, expectedCname);
|
|
assertStagingWebCustomDomainRuntime(webCoreMatches[0].entries, environment, WEB_SURFACE_IDENTITIES.core);
|
|
const webPortalMatches = runtimeCandidatesWithEnv.filter(({ entries }) => valuesByKey(entries).get("WEB_RUNTIME_SURFACE") === "portal");
|
|
if (webPortalMatches.length !== 1) {
|
|
fail("Coolify must expose exactly one allowlisted hivegrid-portal surface for custom-domain routing");
|
|
}
|
|
assertStagingWebCustomDomainRuntime(webPortalMatches[0].entries, environment, WEB_SURFACE_IDENTITIES.portal);
|
|
assertWebSurfacePair(webCoreMatches[0].app, webPortalMatches[0].app);
|
|
if (controllerContract.backendService !== portalBackendService(webPortalMatches[0].app)) {
|
|
fail("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE must target the exact allowlisted hivegrid-portal Docker service");
|
|
}
|
|
return { controller, expectedControllerImage };
|
|
}
|
|
|
|
async function assertRuntime(environment, resourceUuid, allowedCsv, controllerImage, workerImage) {
|
|
await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage);
|
|
process.stdout.write("Portal-domain controller runtime, mounts, and worker-general HMAC wiring verified\n");
|
|
}
|
|
|
|
async function snapshotRelease(environment, resourceUuid, allowedCsv, controllerImage, workerImage) {
|
|
const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage);
|
|
const revision = normalizeValue(controller?.docker_registry_image_tag);
|
|
if (!FULL_GIT_SHA_PATTERN.test(revision)) {
|
|
fail("Portal-domain controller previous release must be a full lowercase Git SHA");
|
|
}
|
|
assertReleaseConfig(controller, resourceUuid, expectedControllerImage, revision, expectedMetricsBindingForEnvironment(environment));
|
|
process.stdout.write(revision);
|
|
}
|
|
|
|
function assertPollingContract() {
|
|
if (!Number.isInteger(pollDelayMs) || pollDelayMs < 1 || pollDelayMs > 60_000) {
|
|
fail("COOLIFY_DEPLOYMENT_POLL_MS must be an integer between 1 and 60000");
|
|
}
|
|
if (!Number.isInteger(pollAttempts) || pollAttempts < 1 || pollAttempts > 180) {
|
|
fail("COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS must be an integer between 1 and 180");
|
|
}
|
|
if (!Number.isSafeInteger(failedStatusGraceMs) || failedStatusGraceMs < 0) {
|
|
fail("COOLIFY_FAILED_STATUS_GRACE_MS must be a non-negative safe integer");
|
|
}
|
|
}
|
|
|
|
async function probeControllerHealth(healthUrl, expectedRevision) {
|
|
let response;
|
|
try {
|
|
response = await fetch(healthUrl, {
|
|
cache: "no-store",
|
|
redirect: "error",
|
|
signal: AbortSignal.timeout(CONTROLLER_HEALTH_PROBE_TIMEOUT_MS),
|
|
});
|
|
} catch {
|
|
return { reachable: false, revisionBound: false };
|
|
}
|
|
if (!response.ok) return { reachable: true, revisionBound: false };
|
|
let payload;
|
|
try {
|
|
payload = await response.json();
|
|
} catch {
|
|
return { reachable: true, revisionBound: false };
|
|
}
|
|
return {
|
|
reachable: true,
|
|
revisionBound:
|
|
payload?.status === "ok" && payload?.service === CONTROLLER_NAME && normalizeValue(payload?.revision) === expectedRevision,
|
|
};
|
|
}
|
|
|
|
async function stopAndDrain(environment, resourceUuid, allowedCsv, controllerImage, workerImage) {
|
|
assertPollingContract();
|
|
if (pollAttempts < CONTROLLER_DRAIN_CONFIRMATIONS) {
|
|
fail(`Portal-domain controller drain requires at least ${CONTROLLER_DRAIN_CONFIRMATIONS} polling attempts`);
|
|
}
|
|
const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage);
|
|
const expectedMetricsBinding = expectedMetricsBindingForEnvironment(environment);
|
|
const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`;
|
|
const expectedRevision = normalizeValue(controller?.docker_registry_image_tag);
|
|
if (!FULL_GIT_SHA_PATTERN.test(expectedRevision)) {
|
|
fail("Portal-domain controller drain requires the current immutable full-SHA release");
|
|
}
|
|
const healthUrl = controllerHealthUrlForEnvironment(environment);
|
|
const requiredDrainFenceMs = controllerDrainFenceMs();
|
|
const beforeStop = await probeControllerHealth(healthUrl, expectedRevision);
|
|
if (!beforeStop.reachable || !beforeStop.revisionBound) {
|
|
fail("Portal-domain controller pre-stop health is unreachable or not revision-bound; operator recovery is required");
|
|
}
|
|
|
|
// Coolify queues this action and its application status may be stale. The
|
|
// pre-stop revision-bound mesh probe proves which worker is live. After the
|
|
// request, both Coolify's stopped state and health-port disappearance must be
|
|
// observed twice. The first rollout also keeps that evidence continuous for
|
|
// 31 seconds, beyond the legacy controller's 30-second hard shutdown; future
|
|
// releases additionally close /healthz only after queue and lock drainage.
|
|
await coolify(`${resourcePath}/stop`, { method: "POST" });
|
|
let drainConfirmations = 0;
|
|
let drainEvidenceSinceMs;
|
|
for (let attempt = 1; attempt <= pollAttempts; attempt += 1) {
|
|
const current = await coolify(resourcePath);
|
|
assertPrivateControllerIdentity(current, resourceUuid, expectedControllerImage, expectedMetricsBinding);
|
|
const afterStop = await probeControllerHealth(healthUrl, expectedRevision);
|
|
if (STOPPED_RUNTIME_STATUSES.test(normalizeValue(current?.status)) && !afterStop.reachable) {
|
|
drainConfirmations += 1;
|
|
drainEvidenceSinceMs ??= performance.now();
|
|
} else {
|
|
drainConfirmations = 0;
|
|
drainEvidenceSinceMs = undefined;
|
|
}
|
|
if (
|
|
drainConfirmations >= CONTROLLER_DRAIN_CONFIRMATIONS &&
|
|
drainEvidenceSinceMs !== undefined &&
|
|
performance.now() - drainEvidenceSinceMs >= requiredDrainFenceMs
|
|
) {
|
|
process.stdout.write("Portal-domain controller stopped and command worker drained before web rollout\n");
|
|
return;
|
|
}
|
|
if (attempt < pollAttempts) await new Promise((resolve) => setTimeout(resolve, pollDelayMs));
|
|
}
|
|
fail("Portal-domain controller did not stop and drain before the web rollout");
|
|
}
|
|
|
|
function controllerHealthCommand(revision) {
|
|
return `node ${CONTROLLER_DEPLOY_HEALTH_PATH} ${revision}`;
|
|
}
|
|
|
|
function releasePayload(revision) {
|
|
return {
|
|
pre_deployment_command: "",
|
|
post_deployment_command: "",
|
|
docker_registry_image_tag: revision,
|
|
health_check_enabled: true,
|
|
health_check_type: "cmd",
|
|
health_check_command: controllerHealthCommand(revision),
|
|
health_check_interval: CONTROLLER_HEALTH_INTERVAL_SECONDS,
|
|
health_check_timeout: CONTROLLER_HEALTH_TIMEOUT_SECONDS,
|
|
health_check_retries: CONTROLLER_HEALTH_RETRIES,
|
|
health_check_start_period: CONTROLLER_HEALTH_START_PERIOD_SECONDS,
|
|
};
|
|
}
|
|
|
|
function snapshotReleaseNumber(value, key) {
|
|
if (value === null || value === undefined || normalizeValue(value) === "") {
|
|
fail(`Portal-domain controller existing ${key} configuration is missing`);
|
|
}
|
|
const number = Number(value);
|
|
if (!Number.isFinite(number)) {
|
|
fail(`Portal-domain controller existing ${key} configuration is invalid`);
|
|
}
|
|
return number;
|
|
}
|
|
|
|
function previousReleasePayload(app) {
|
|
const healthCheckEnabled = normalizeBoolean(app?.health_check_enabled);
|
|
if (!healthCheckEnabled) {
|
|
fail("Portal-domain controller existing health_check_enabled configuration is invalid");
|
|
}
|
|
return {
|
|
pre_deployment_command: normalizeValue(app?.pre_deployment_command),
|
|
post_deployment_command: normalizeValue(app?.post_deployment_command),
|
|
docker_registry_image_tag: normalizeValue(app?.docker_registry_image_tag),
|
|
health_check_enabled: healthCheckEnabled === "true",
|
|
health_check_type: normalizeValue(app?.health_check_type),
|
|
health_check_command: normalizeValue(app?.health_check_command),
|
|
health_check_interval: snapshotReleaseNumber(app?.health_check_interval, "health_check_interval"),
|
|
health_check_timeout: snapshotReleaseNumber(app?.health_check_timeout, "health_check_timeout"),
|
|
health_check_retries: snapshotReleaseNumber(app?.health_check_retries, "health_check_retries"),
|
|
health_check_start_period: snapshotReleaseNumber(app?.health_check_start_period, "health_check_start_period"),
|
|
};
|
|
}
|
|
|
|
function normalizedMutableReleaseConfig(app) {
|
|
return {
|
|
pre_deployment_command: normalizeValue(app?.pre_deployment_command),
|
|
post_deployment_command: normalizeValue(app?.post_deployment_command),
|
|
docker_registry_image_tag: normalizeValue(app?.docker_registry_image_tag),
|
|
health_check_enabled: normalizeBoolean(app?.health_check_enabled),
|
|
health_check_type: normalizeValue(app?.health_check_type),
|
|
health_check_command: normalizeValue(app?.health_check_command),
|
|
health_check_interval: Number(app?.health_check_interval),
|
|
health_check_timeout: Number(app?.health_check_timeout),
|
|
health_check_retries: Number(app?.health_check_retries),
|
|
health_check_start_period: Number(app?.health_check_start_period),
|
|
};
|
|
}
|
|
|
|
function mutableReleaseConfigDrift(app, expected) {
|
|
const actualConfig = normalizedMutableReleaseConfig(app);
|
|
const expectedConfig = normalizedMutableReleaseConfig(expected);
|
|
return Object.keys(expectedConfig).filter((key) => actualConfig[key] !== expectedConfig[key]);
|
|
}
|
|
|
|
function assertReleaseConfig(app, resourceUuid, image, revision, expectedMetricsBinding) {
|
|
assertPrivateControllerIdentity(app, resourceUuid, image, expectedMetricsBinding);
|
|
const actual = {
|
|
...normalizedMutableReleaseConfig(app),
|
|
custom_healthcheck_found: normalizeBoolean(app?.custom_healthcheck_found),
|
|
};
|
|
const expected = {
|
|
...releasePayload(revision),
|
|
health_check_enabled: "true",
|
|
custom_healthcheck_found: "false",
|
|
};
|
|
const drift = Object.keys(expected).filter((key) => actual[key] !== expected[key]);
|
|
if (drift.length > 0) {
|
|
fail("Portal-domain controller release or revision-bound command health configuration drifted");
|
|
}
|
|
}
|
|
|
|
async function setRelease(environment, resourceUuid, allowedCsv, controllerImage, workerImage, revision) {
|
|
if (!FULL_GIT_SHA_PATTERN.test(revision)) {
|
|
fail("Portal-domain controller release requires a full lowercase Git SHA");
|
|
}
|
|
const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage);
|
|
const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`;
|
|
const previous = previousReleasePayload(controller);
|
|
try {
|
|
await coolify(resourcePath, { method: "PATCH", body: JSON.stringify(releasePayload(revision)) });
|
|
const updated = await coolify(resourcePath);
|
|
assertReleaseConfig(updated, resourceUuid, expectedControllerImage, revision, expectedMetricsBindingForEnvironment(environment));
|
|
} catch {
|
|
let restored = false;
|
|
try {
|
|
await coolify(resourcePath, { method: "PATCH", body: JSON.stringify(previous) });
|
|
const restoredApp = await coolify(resourcePath);
|
|
restored = mutableReleaseConfigDrift(restoredApp, previous).length === 0;
|
|
} catch {
|
|
restored = false;
|
|
}
|
|
fail(
|
|
restored
|
|
? "Portal-domain controller release mutation failed and previous configuration was restored"
|
|
: "Portal-domain controller release mutation failed and compensation was incomplete"
|
|
);
|
|
}
|
|
process.stdout.write("Portal-domain controller immutable release configured\n");
|
|
}
|
|
|
|
function deploymentStatus(payload) {
|
|
return normalizeValue(payload?.status || payload?.deployment?.status || payload?.data?.status).toLowerCase();
|
|
}
|
|
|
|
function assertCompletedDeployment(payload, deploymentUuid, app) {
|
|
if (normalizeValue(payload?.deployment_uuid) !== deploymentUuid) {
|
|
fail("Portal-domain controller deployment identity does not match the triggered deployment");
|
|
}
|
|
if (!COMPLETED_DEPLOYMENT_STATUSES.has(deploymentStatus(payload))) {
|
|
fail("Portal-domain controller deployment is not completed");
|
|
}
|
|
if (normalizeValue(payload?.application_name) !== CONTROLLER_NAME) {
|
|
fail("Portal-domain controller deployment belongs to an unexpected application");
|
|
}
|
|
const applicationId = normalizeValue(app?.id);
|
|
const deploymentApplicationId = normalizeValue(payload?.application_id);
|
|
if (applicationId && deploymentApplicationId && applicationId !== deploymentApplicationId) {
|
|
fail("Portal-domain controller deployment application identity does not match the live resource");
|
|
}
|
|
}
|
|
|
|
async function assertRelease(environment, resourceUuid, allowedCsv, controllerImage, workerImage, revision, deploymentUuid) {
|
|
if (!FULL_GIT_SHA_PATTERN.test(revision)) {
|
|
fail("Portal-domain controller release verification requires a full lowercase Git SHA");
|
|
}
|
|
requireCoolifyCuid(deploymentUuid, "Portal-domain controller deployment UUID");
|
|
assertPollingContract();
|
|
const { controller, expectedControllerImage } = await resolveRuntimeContract(environment, resourceUuid, allowedCsv, controllerImage, workerImage);
|
|
const deployment = await coolify(`/deployments/${encodeURIComponent(deploymentUuid)}`);
|
|
assertCompletedDeployment(deployment, deploymentUuid, controller);
|
|
|
|
const resourcePath = `/applications/${encodeURIComponent(resourceUuid)}`;
|
|
// The controller is deliberately stopped-and-drained before the rollout, so Coolify's
|
|
// cached application status can still read "exited" for a while after the deployment
|
|
// reports finished. Keep the fail-closed boundary time-based so a polling-frequency
|
|
// change cannot shorten the stale-status grace period.
|
|
let failedStatusSinceMs;
|
|
for (let attempt = 1; attempt <= pollAttempts; attempt += 1) {
|
|
const current = await coolify(resourcePath);
|
|
assertReleaseConfig(current, resourceUuid, expectedControllerImage, revision, expectedMetricsBindingForEnvironment(environment));
|
|
const status = normalizeValue(current?.status).toLowerCase();
|
|
if (status === "running:healthy") {
|
|
process.stdout.write("Portal-domain controller revision-bound healthz, readyz, and immutable revision verified\n");
|
|
return;
|
|
}
|
|
if (FAILED_RUNTIME_STATUSES.test(status)) {
|
|
const nowMs = performance.now();
|
|
failedStatusSinceMs ??= nowMs;
|
|
if (nowMs - failedStatusSinceMs >= failedStatusGraceMs) {
|
|
fail("Portal-domain controller entered a terminal runtime state");
|
|
}
|
|
} else {
|
|
failedStatusSinceMs = undefined;
|
|
}
|
|
if (attempt < pollAttempts) await new Promise((resolve) => setTimeout(resolve, pollDelayMs));
|
|
}
|
|
fail("Portal-domain controller did not become running:healthy in time");
|
|
}
|
|
|
|
try {
|
|
if (command === "assert-runtime") {
|
|
await assertRuntime(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || "");
|
|
} else if (command === "snapshot-release") {
|
|
await snapshotRelease(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || "");
|
|
} else if (command === "stop-and-drain") {
|
|
await stopAndDrain(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || "");
|
|
} else if (command === "set-release") {
|
|
await setRelease(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || "", args[5] || "");
|
|
} else if (command === "assert-release") {
|
|
await assertRelease(args[0] || "", args[1] || "", args[2] || "", args[3] || "", args[4] || "", args[5] || "", args[6] || "");
|
|
} else {
|
|
fail(
|
|
"Usage: coolify-portal-domain-controller.mjs assert-runtime|snapshot-release|stop-and-drain <staging|production> <controller-resource-uuid> <allowed-resource-csv> <controller-image> <worker-image> | set-release <staging|production> <controller-resource-uuid> <allowed-resource-csv> <controller-image> <worker-image> <full-git-sha> | assert-release <staging|production> <controller-resource-uuid> <allowed-resource-csv> <controller-image> <worker-image> <full-git-sha> <deployment-uuid>"
|
|
);
|
|
}
|
|
} catch (error) {
|
|
process.stderr.write(`${error instanceof Error ? error.message : "Portal-domain controller Coolify helper failed"}\n`);
|
|
process.exitCode = 1;
|
|
}
|