#!/usr/bin/env node import path from "node:path"; import { fileURLToPath } from "node:url"; export const GROUPS = [ { name: "general", healthPort: "9101", adminPort: "9201", concurrencyKey: "WORKER_CONCURRENCY_GENERAL", concurrency: "10" }, { name: "mail", healthPort: "9102", adminPort: "9202", concurrencyKey: "WORKER_CONCURRENCY_MAIL", concurrency: "5" }, { name: "import", healthPort: "9103", adminPort: "9203", concurrencyKey: "WORKER_CONCURRENCY_IMPORT", concurrency: "1" }, { name: "webhook", healthPort: "9104", adminPort: "9204", concurrencyKey: "WORKER_CONCURRENCY_WEBHOOK", concurrency: "20" }, { name: "plugin", healthPort: "9105", adminPort: "9205", concurrencyKey: "WORKER_CONCURRENCY_PLUGIN", concurrency: "5" }, ]; const [, , environment = "staging"] = process.argv; const prefix = environment === "production" ? "COOLIFY" : "COOLIFY_STAGING"; const apiUrl = (process.env.SELECTED_COOLIFY_API_URL || process.env[`${prefix}_API_URL`] || "").replace(/\/+$/, ""); const token = process.env.SELECTED_COOLIFY_TOKEN || process.env[`${prefix}_TOKEN`] || ""; const apiTimeoutMs = Number(process.env.COOLIFY_API_TIMEOUT_MS || "20000"); const imageTag = process.env.IMAGE_TAG || (environment === "production" ? "main" : "staging"); const registry = process.env.REGISTRY || ""; const imageNamespace = process.env.IMAGE_NAMESPACE || ""; const imagePrefix = process.env.IMAGE_PREFIX || ""; const workerImage = process.env.WORKER_IMAGE || [registry, imageNamespace, imagePrefix, "worker"].filter(Boolean).join("/"); const existingResourceUuids = process.env[`${prefix}_RESOURCE_UUIDS`] || ""; const resolvedEnvName = process.env.COOLIFY_RESOLVED_RESOURCE_UUIDS_ENV_NAME || (environment === "production" ? "PRODUCTION_RESOLVED_RESOURCE_UUIDS" : "STAGING_RESOLVED_RESOURCE_UUIDS"); const provisionEnabled = process.env[`${prefix}_PROVISION_WORKER_GROUPS`] === "true"; const configuredWorkerEnvSourceUuid = process.env[`${prefix}_WORKER_ENV_SOURCE_UUID`] || process.env.WORKER_ENV_SOURCE_UUID || ""; const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/; const COOLIFY_CUID_PATTERN = /^[a-z][a-z0-9]{23}$/; const WORKER_DATABASE_ROLE = "hivegrid_worker"; class CoolifyRuntimeEnvConflictError extends Error {} export const WORKER_ENV_OVERRIDE_KEYS = new Set([ "APP_REVISION", "DATABASE_RUNTIME_ROLE", "RLS_RUNTIME_MODE", "WORKER_ENABLED", "WORKER_QUEUE_GROUP", "WORKER_CONCURRENCY", "WORKER_HEALTH_PORT", "WORKER_ADMIN_PORT", "WORKER_CONCURRENCY_GENERAL", "WORKER_CONCURRENCY_MAIL", "WORKER_CONCURRENCY_IMPORT", "WORKER_CONCURRENCY_WEBHOOK", "WORKER_CONCURRENCY_PLUGIN", "MARKETING_SEQUENCES_ENABLED", "TRANSACTIONAL_MAIL_OUTBOX_REQUIRED", "OUTBOX_DISPATCHER_ENABLED", "OUTBOX_DISPATCHER_REQUIRED", "PLUGIN_HOOK_FANOUT_CONCURRENCY", "PLUGIN_HOOK_FANOUT_ENABLED", "PLUGIN_HOOK_FANOUT_REQUIRED", "PLUGIN_RUNTIME_PRODUCTION_ENABLED", "WORKFLOW_TRIGGER_EXECUTOR_MODE", "WORKFLOW_TRIGGER_OUTBOX_ENABLED", "WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY", "WORKFLOW_TRIGGER_OUTBOX_REQUIRED", ]); export const WORKER_PRIVILEGED_ENV_KEYS = new Set(["DIRECT_URL", "MIGRATOR_DATABASE_URL", "MIGRATE_ON_BOOT", "SEED_ON_BOOT", "SEED_ONCE"]); export const WORKER_GENERAL_PROVIDER_ENV_KEYS = new Set([ "PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED", "LAGO_API_URL", "LAGO_API_KEY", "LAGO_TIMEOUT_MS", "MOLLIE_API_KEY", "MOLLIE_PROFILE_ID", "MOLLIE_TIMEOUT_MS", "BILLING_MOLLIE_WEBHOOK_URL", ]); export const KWP_WORKER_ENV_KEY = "KWP_HIVEGRID_TENANT_ID"; const KWP_WORKER_GROUP_NAMES = new Set(["general", "import"]); const WORKER_MAIL_ONLY_ENV_KEYS = new Set(["CRYPTO_SECRET"]); function isWorkerPrivilegedEnvKey(key) { return WORKER_PRIVILEGED_ENV_KEYS.has(key) || key.startsWith("MIGRATOR_") || key.startsWith("SEED_"); } function canWorkerGroupReceiveProviderEnv(group, key) { if (key === KWP_WORKER_ENV_KEY) return KWP_WORKER_GROUP_NAMES.has(group.name); if (WORKER_MAIL_ONLY_ENV_KEYS.has(key)) return group.name === "mail"; return group.name === "general" || !WORKER_GENERAL_PROVIDER_ENV_KEYS.has(key); } function fail(message) { process.stderr.write(`${message}\n`); process.exit(1); } function requireEnv(name) { const value = process.env[name] || ""; if (!value) { fail(`${name} is required to provision missing Coolify worker applications`); } return value; } function requireCoolifyCuid(value, label) { if (!COOLIFY_CUID_PATTERN.test(value)) { fail(`${label} must be a valid Coolify CUID`); } } async function coolify(path, options = {}) { if (!apiUrl || !token) { fail("Coolify API URL and token are required"); } if (!Number.isInteger(apiTimeoutMs) || apiTimeoutMs < 10 || apiTimeoutMs > 120_000) { fail("COOLIFY_API_TIMEOUT_MS must be an integer between 10 and 120000"); } let response; try { response = await fetch(`${apiUrl}${path}`, { ...options, signal: options.signal ?? AbortSignal.timeout(apiTimeoutMs), headers: { Authorization: `Bearer ${token}`, Accept: "application/json", ...(options.body ? { "Content-Type": "application/json" } : {}), ...options.headers, }, }); } catch { fail("Coolify API request failed or timed out"); } const text = await response.text(); let payload = {}; if (text.trim()) { try { payload = JSON.parse(text); } catch { payload = { message: text.trim() }; } } if (!response.ok) { fail(`Coolify API request failed with HTTP ${response.status}`); } return payload; } function asArray(payload) { if (Array.isArray(payload)) return payload; if (Array.isArray(payload?.data)) return payload.data; fail("Coolify list response was malformed"); } function appText(app) { return [ app.name, app.description, app.docker_registry_image_name, app.fqdn, app.domain, app.domains, JSON.stringify(app.environment_variables || app.envs || app.environment || ""), ] .filter(Boolean) .join(" "); } function matchesWorkerGroup(app, group) { if (app.docker_registry_image_name !== workerImage) return false; const text = appText(app); const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["']?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); return envPattern.test(text) || namePattern.test(text); } function isGroupWorker(app) { return GROUPS.some((group) => matchesWorkerGroup(app, group.name)); } function managedWorkerDescription(group) { return `Managed by deploy workflow. WORKER_QUEUE_GROUP=${group.name}`; } function hasManagedWorkerMarker(app, group) { return app?.name === `worker-${group.name}` && app?.description === managedWorkerDescription(group); } function hasManagedWorkerIdentity(app, group) { return hasManagedWorkerMarker(app, group) && app?.docker_registry_image_name === workerImage; } function hasManagedWorkerRetryProvenance(app, group) { const deployedImageTag = normalizeCoolifyConfigValue(app?.docker_registry_image_tag); if (!hasManagedWorkerIdentity(app, group) || !FULL_GIT_SHA_PATTERN.test(deployedImageTag)) { return false; } return normalizeCoolifyConfigValue(app?.health_check_command) === buildWorkerDeploymentHealthCommand(deployedImageTag, group.name); } function isInProvisioningScope(app, scope) { if (!scope) return false; return ( Number(app?.environment_id) === scope.environmentId && app?.destination?.uuid === scope.destinationUuid && app?.destination?.server?.uuid === scope.serverUuid ); } function isInProvisioningEnvironment(app, scope) { return Boolean(scope) && Number(app?.environment_id) === scope.environmentId; } async function resolveProvisioningScope() { const projectUuid = requireEnv(`${prefix}_PROJECT_UUID`); const serverUuid = requireEnv(`${prefix}_SERVER_UUID`); const destinationUuid = requireEnv(`${prefix}_DESTINATION_UUID`); const environmentUuid = process.env[`${prefix}_ENVIRONMENT_UUID`] || ""; const environmentName = process.env[`${prefix}_ENVIRONMENT_NAME`] || environment; requireCoolifyCuid(projectUuid, `${prefix}_PROJECT_UUID`); requireCoolifyCuid(serverUuid, `${prefix}_SERVER_UUID`); requireCoolifyCuid(destinationUuid, `${prefix}_DESTINATION_UUID`); if (environmentUuid) requireCoolifyCuid(environmentUuid, `${prefix}_ENVIRONMENT_UUID`); const projectPayload = await coolify(`/projects/${encodeURIComponent(projectUuid)}`); const project = projectPayload?.data && !Array.isArray(projectPayload.data) ? projectPayload.data : projectPayload; if (project?.uuid !== projectUuid || !Array.isArray(project?.environments)) { fail(`${prefix}_PROJECT_UUID response did not expose a verifiable environment scope`); } const matchingEnvironments = project.environments.filter((candidate) => environmentUuid ? candidate?.uuid === environmentUuid : candidate?.name === environmentName ); if (matchingEnvironments.length !== 1 || !Number.isInteger(Number(matchingEnvironments[0]?.id))) { fail(`${prefix} worker provisioning environment scope could not be resolved uniquely`); } return { projectUuid, serverUuid, destinationUuid, environmentUuid, environmentName, environmentId: Number(matchingEnvironments[0].id), }; } function resolveWorkerEnvSourceUuid(apps, allowed) { if (configuredWorkerEnvSourceUuid) { const app = apps.find((candidate) => candidate.uuid === configuredWorkerEnvSourceUuid); if (!app) { fail(`${prefix}_WORKER_ENV_SOURCE_UUID does not match an existing Coolify application`); } if (allowed.size > 0 && !allowed.has(configuredWorkerEnvSourceUuid)) { fail(`${prefix}_WORKER_ENV_SOURCE_UUID must be included in ${prefix}_RESOURCE_UUIDS`); } if (app.docker_registry_image_name !== workerImage) { fail(`${prefix}_WORKER_ENV_SOURCE_UUID must reference the exact worker image`); } return configuredWorkerEnvSourceUuid; } const candidates = apps .filter((candidate) => candidate.docker_registry_image_name === workerImage) .filter((candidate) => allowed.size === 0 || allowed.has(candidate.uuid)) .filter((candidate) => !isGroupWorker(candidate)); candidates.sort((a, b) => { const aName = String(a.name || ""); const bName = String(b.name || ""); if (aName === "hivegrid-worker") return -1; if (bName === "hivegrid-worker") return 1; if (/worker/i.test(aName) && !/worker/i.test(bName)) return -1; if (/worker/i.test(bName) && !/worker/i.test(aName)) return 1; return aName.localeCompare(bName); }); return candidates[0]?.uuid || ""; } function envKey(entry) { return typeof entry?.key === "string" ? entry.key : ""; } function envValue(entry) { const rawValue = entry?.value ?? entry?.real_value; let value = typeof rawValue === "string" ? rawValue : ""; while (value.length >= 2 && ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith('"') && value.endsWith('"')))) { value = value.slice(1, -1); } return value; } export function canonicalRuntimeEnvEntries(entries) { const canonical = []; const runtimeKeys = new Set(); for (const entry of entries || []) { if (normalizeCoolifyBoolean(entry?.is_preview) === "true") continue; const key = envKey(entry); if (!key) continue; if (runtimeKeys.has(key)) { throw new CoolifyRuntimeEnvConflictError("Coolify runtime environment response contained duplicate non-preview keys"); } runtimeKeys.add(key); canonical.push(entry); } return canonical; } export function desiredWorkerEnv(group) { const env = { DATABASE_RUNTIME_ROLE: WORKER_DATABASE_ROLE, RLS_RUNTIME_MODE: "required", WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: group.name, WORKER_HEALTH_PORT: group.healthPort, WORKER_ADMIN_PORT: group.adminPort, [group.concurrencyKey]: group.concurrency, }; if (group.name === "general") { Object.assign(env, { OUTBOX_DISPATCHER_ENABLED: "true", OUTBOX_DISPATCHER_REQUIRED: "true", WORKFLOW_TRIGGER_EXECUTOR_MODE: "http", WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true", WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY: "5", WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true", }); } if (group.name === "mail") { Object.assign(env, { MARKETING_SEQUENCES_ENABLED: "false", TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true", }); } return env; } export function envEntriesToRecord(entries) { const record = {}; for (const entry of canonicalRuntimeEnvEntries(entries)) { const key = envKey(entry); if (!key) continue; record[key] = envValue(entry); } return record; } export function normalizeWorkflowTriggerExecutorMode(value) { const normalized = typeof value === "string" ? value.trim().toLowerCase() : ""; return normalized === "http" || normalized === "hybrid" || normalized === "native" ? normalized : ""; } export function getWorkerEnvDrift(app, group, entries) { const actual = envEntriesToRecord(entries); const exactDrift = Object.entries(desiredWorkerEnv(group)) .filter(([key, expected]) => { if (key === "WORKFLOW_TRIGGER_EXECUTOR_MODE") { return !normalizeWorkflowTriggerExecutorMode(actual[key]); } return actual[key] !== expected; }) .map(([key, expected]) => ({ key, expected: key === "WORKFLOW_TRIGGER_EXECUTOR_MODE" ? "http|hybrid|native" : expected, actual: actual[key] ?? "", appUuid: app?.uuid ?? "", appName: app?.name ?? `worker-${group.name}`, group: group.name, })); const sharedRuntimeDrift = getSharedWorkerRuntimeDrift(app, group, actual); const groupRuntimeDrift = group.name === "general" ? getGeneralWorkerRuntimeDrift(app, group, actual) : group.name === "mail" ? getMailWorkerRuntimeDrift(app, group, actual) : []; const revisionOverrideDrift = Object.hasOwn(actual, "APP_REVISION") ? [ { key: "APP_REVISION", expected: "unset; image-baked revision is authoritative", actual: "", appUuid: app?.uuid ?? "", appName: app?.name ?? `worker-${group.name}`, group: group.name, }, ] : []; const privilegedEnvDrift = Object.keys(actual) .filter(isWorkerPrivilegedEnvKey) .map((key) => ({ key, expected: "unset; migration and seed credentials are forbidden in worker runtimes", actual: "", appUuid: app?.uuid ?? "", appName: app?.name ?? `worker-${group.name}`, group: group.name, })); const providerBoundaryDrift = Object.keys(actual) .filter( (key) => (group.name !== "general" && WORKER_GENERAL_PROVIDER_ENV_KEYS.has(key)) || (key === KWP_WORKER_ENV_KEY && !KWP_WORKER_GROUP_NAMES.has(group.name)) ) .map((key) => ({ key, expected: key === KWP_WORKER_ENV_KEY ? "unset; the KWP pilot tenant belongs only to worker-general and worker-import" : "unset; billing provider credentials belong only to worker-general", actual: "", appUuid: app?.uuid ?? "", appName: app?.name ?? `worker-${group.name}`, group: group.name, })); return [...exactDrift, ...revisionOverrideDrift, ...privilegedEnvDrift, ...providerBoundaryDrift, ...sharedRuntimeDrift, ...groupRuntimeDrift]; } export function buildWorkerDeploymentHealthCommand(deployedImageTag, workerGroup) { if (!FULL_GIT_SHA_PATTERN.test(deployedImageTag)) { throw new Error("Worker deployment health requires a full lowercase Git SHA image tag."); } if (!GROUPS.some((group) => group.name === workerGroup)) { throw new Error("Worker deployment health requires a valid worker group."); } return `node apps/worker/dist/deploy-health.js ${deployedImageTag} ${workerGroup}`; } function desiredWorkerAppConfig(group, deployedImageTag) { return { docker_registry_image_tag: deployedImageTag, ports_exposes: group.healthPort, health_check_enabled: true, health_check_type: "cmd", health_check_command: buildWorkerDeploymentHealthCommand(deployedImageTag, group.name), health_check_interval: 30, health_check_timeout: 10, health_check_retries: 3, health_check_start_period: 20, }; } export function getWorkerAppConfigDrift(app, group, deployedImageTag) { const appName = app?.name ?? `worker-${group.name}`; const appUuid = app?.uuid ?? ""; const item = (key, expected, actual) => ({ key, expected, actual, appUuid, appName, group: group.name, }); const desired = desiredWorkerAppConfig(group, deployedImageTag); const expected = { ...desired, health_check_enabled: "true", custom_healthcheck_found: "false" }; const actual = { docker_registry_image_tag: normalizeCoolifyConfigValue(app?.docker_registry_image_tag), ports_exposes: normalizeCoolifyConfigValue(app?.ports_exposes), health_check_enabled: normalizeCoolifyBoolean(app?.health_check_enabled), health_check_type: normalizeCoolifyConfigValue(app?.health_check_type), health_check_command: normalizeCoolifyConfigValue(app?.health_check_command), health_check_interval: Number(app?.health_check_interval), health_check_timeout: Number(app?.health_check_timeout), health_check_retries: Number(app?.health_check_retries), health_check_start_period: Number(app?.health_check_start_period), custom_healthcheck_found: normalizeCoolifyBoolean(app?.custom_healthcheck_found), }; return Object.entries(expected) .filter(([key, expectedValue]) => actual[key] !== expectedValue) .map(([key, expectedValue]) => item(key, expectedValue, actual[key] || "")); } function normalizeCoolifyBoolean(value) { return value === true || value === "true" || value === 1 || value === "1" ? "true" : value === false || value === "false" || value === 0 || value === "0" ? "false" : ""; } function normalizeCoolifyConfigValue(value) { if (value === null || value === undefined) return ""; return String(value).trim(); } function hasEnvValue(actual, key) { return typeof actual[key] === "string" && actual[key].trim().length > 0; } function getSharedWorkerRuntimeDrift(app, group, actual) { const drift = []; const item = (key, expected, actualValue) => ({ key, expected, actual: actualValue, appUuid: app?.uuid ?? "", appName: app?.name ?? `worker-${group.name}`, group: group.name, }); if (!hasEnvValue(actual, "DATABASE_URL")) { drift.push(item("DATABASE_URL", "configured DATABASE_URL", "")); } else { let databaseRole = ""; try { const databaseUrl = new URL(actual.DATABASE_URL); if (databaseUrl.protocol === "postgres:" || databaseUrl.protocol === "postgresql:") { databaseRole = decodeURIComponent(databaseUrl.username); } } catch { // Invalid URLs are reported below without echoing the credential. } if (databaseRole !== WORKER_DATABASE_ROLE) { drift.push( item( "DATABASE_URL", `PostgreSQL DSN authenticated as ${WORKER_DATABASE_ROLE}`, databaseRole ? "" : "" ) ); } } if (!hasEnvValue(actual, "BULLMQ_REDIS_URL")) { drift.push( item("BULLMQ_REDIS_URL", "configured BULLMQ_REDIS_URL", actual.REDIS_URL ? "" : "") ); } return drift; } function getMailWorkerRuntimeDrift(app, group, actual) { if (hasEnvValue(actual, "CRYPTO_SECRET")) return []; return [ { key: "CRYPTO_SECRET", expected: "configured shared tenant-envelope key matching hivegrid-web", actual: "", appUuid: app?.uuid ?? "", appName: app?.name ?? `worker-${group.name}`, group: group.name, }, ]; } function getGeneralWorkerRuntimeDrift(app, group, actual) { const drift = []; const item = (key, expected, actualValue) => ({ key, expected, actual: actualValue, appUuid: app?.uuid ?? "", appName: app?.name ?? `worker-${group.name}`, group: group.name, }); const mode = normalizeWorkflowTriggerExecutorMode(actual.WORKFLOW_TRIGGER_EXECUTOR_MODE); if (mode === "http" || mode === "hybrid") { if (!hasEnvValue(actual, "WORKFLOW_TRIGGER_EXECUTOR_URL") && !hasEnvValue(actual, "APP_URL") && !hasEnvValue(actual, "WEB_ORIGIN")) { drift.push(item("WORKFLOW_TRIGGER_EXECUTOR_URL", "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN", "")); } if (!hasEnvValue(actual, "API_ACCESS_TOKEN")) { drift.push(item("API_ACCESS_TOKEN", "configured API_ACCESS_TOKEN matching the web runtime", "")); } else if (hasEnvValue(actual, "WORKFLOW_TRIGGER_EXECUTOR_TOKEN")) { drift.push(item("WORKFLOW_TRIGGER_EXECUTOR_TOKEN", "unset; API_ACCESS_TOKEN is the supported workflow bridge credential", "")); } } const providerGate = actual.PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED?.trim(); if (providerGate && providerGate !== "true" && providerGate !== "false") { drift.push(item("PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED", "true, false, or unset", "")); } const providerSideEffectRequired = providerGate !== "false" && (providerGate === "true" || actual.RLS_RUNTIME_MODE === "required"); if (providerSideEffectRequired) { for (const key of ["LAGO_API_URL", "LAGO_API_KEY", "MOLLIE_API_KEY", "BILLING_MOLLIE_WEBHOOK_URL"]) { if (!hasEnvValue(actual, key)) { drift.push(item(key, `configured ${key}`, "")); } } if (hasEnvValue(actual, "LAGO_API_URL")) { try { const url = new URL(actual.LAGO_API_URL); if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) { throw new Error("invalid provider URL"); } } catch { drift.push(item("LAGO_API_URL", "credential-free absolute HTTP(S) URL", "")); } } if (hasEnvValue(actual, "BILLING_MOLLIE_WEBHOOK_URL")) { try { const url = new URL(actual.BILLING_MOLLIE_WEBHOOK_URL); if ( url.protocol !== "https:" || url.username || url.password || url.search || url.hash || url.pathname !== "/v1/billing/mollie/webhooks" ) { throw new Error("invalid Mollie webhook URL"); } } catch { drift.push(item("BILLING_MOLLIE_WEBHOOK_URL", "canonical HTTPS Mollie webhook URL", "")); } } for (const key of ["LAGO_TIMEOUT_MS", "MOLLIE_TIMEOUT_MS"]) { const value = actual[key]; const parsed = Number(value); if (hasEnvValue(actual, key) && (!Number.isSafeInteger(parsed) || parsed <= 0)) { drift.push(item(key, "positive integer milliseconds when configured", "")); } } } if (actual.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true") { if (actual.PLUGIN_HOOK_FANOUT_ENABLED === "false") { drift.push(item("PLUGIN_HOOK_FANOUT_ENABLED", "unset or true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true", "false")); } if (actual.PLUGIN_HOOK_FANOUT_REQUIRED !== "true") { drift.push(item("PLUGIN_HOOK_FANOUT_REQUIRED", "true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true", actual.PLUGIN_HOOK_FANOUT_REQUIRED ?? "")); } } return drift; } async function cloneWorkerEnv(sourceUuid, targetUuid, group, { sourceEnvs: providedSourceEnvs, targetEnvs: providedTargetEnvs } = {}) { if (!sourceUuid) return providedTargetEnvs || []; const [sourceEnvs, targetEnvs] = await Promise.all([ providedSourceEnvs ? Promise.resolve(providedSourceEnvs) : coolify(`/applications/${encodeURIComponent(sourceUuid)}/envs`).then(asArray), providedTargetEnvs ? Promise.resolve(providedTargetEnvs) : coolify(`/applications/${encodeURIComponent(targetUuid)}/envs`).then(asArray), ]); const runtimeSourceEnvs = canonicalRuntimeEnvEntries(sourceEnvs); const runtimeTargetEnvs = canonicalRuntimeEnvEntries(targetEnvs); const targetKeys = new Set(runtimeTargetEnvs.map(envKey).filter(Boolean)); const reconciledTargetEnvs = [...runtimeTargetEnvs]; for (const entry of runtimeSourceEnvs) { const key = envKey(entry); if (!key || targetKeys.has(key) || WORKER_ENV_OVERRIDE_KEYS.has(key) || isWorkerPrivilegedEnvKey(key) || !canWorkerGroupReceiveProviderEnv(group, key)) { continue; } await coolify(`/applications/${encodeURIComponent(targetUuid)}/envs`, { method: "POST", body: JSON.stringify({ key, value: envValue(entry), is_literal: entry.is_literal ?? true, is_preview: false, is_multiline: entry.is_multiline ?? false, is_shown_once: entry.is_shown_once ?? false, }), }); targetKeys.add(key); reconciledTargetEnvs.push({ key, value: envValue(entry), is_preview: false }); } return reconciledTargetEnvs; } async function createWorkerApplication(group, workerEnvSourceUuid, sourceEnvs, provisioningScope) { if (!provisionEnabled) { fail( `Missing Coolify worker-${group.name} application. Set ${prefix}_PROVISION_WORKER_GROUPS=true with project/server/destination variables to create it automatically.` ); } const body = buildWorkerApplicationBody({ group, projectUuid: provisioningScope.projectUuid, serverUuid: provisioningScope.serverUuid, destinationUuid: provisioningScope.destinationUuid, environmentName: provisioningScope.environmentName, environmentUuid: provisioningScope.environmentUuid, workerImage, imageTag, }); const created = await coolify("/applications/dockerimage", { method: "POST", body: JSON.stringify(body), }); const uuid = typeof created.uuid === "string" ? created.uuid : ""; if (!uuid) { fail(`Coolify did not return a UUID for worker-${group.name}`); } requireCoolifyCuid(uuid, `Coolify worker-${group.name} UUID`); const createdApp = await coolify(`/applications/${encodeURIComponent(uuid)}`); if (createdApp?.uuid !== uuid || !hasManagedWorkerRetryProvenance(createdApp, group) || !isInProvisioningScope(createdApp, provisioningScope)) { fail(`Created Coolify worker-${group.name} application identity could not be verified`); } assertWorkerCommandHealthCanBeAuthoritative(createdApp, group); const targetEnvs = asArray(await coolify(`/applications/${encodeURIComponent(uuid)}/envs`)); const clonedEnvs = await cloneWorkerEnv(workerEnvSourceUuid, uuid, group, { sourceEnvs, targetEnvs }); await ensureMissingWorkerEnv(uuid, group, clonedEnvs); process.stdout.write(`Created Coolify worker-${group.name} application\n`); return createdApp; } export function buildWorkerApplicationBody({ group, projectUuid, serverUuid, destinationUuid, environmentName, environmentUuid = "", workerImage, imageTag }) { const body = { project_uuid: projectUuid, server_uuid: serverUuid, environment_name: environmentName, docker_registry_image_name: workerImage, docker_registry_image_tag: imageTag, destination_uuid: destinationUuid, name: `worker-${group.name}`, description: managedWorkerDescription(group), ...desiredWorkerAppConfig(group, imageTag), instant_deploy: false, connect_to_docker_network: true, }; if (environmentUuid) { body.environment_uuid = environmentUuid; } return body; } async function ensureMissingWorkerEnv(uuid, group, entries) { const actual = envEntriesToRecord(entries); for (const [key, value] of Object.entries(desiredWorkerEnv(group))) { if (Object.hasOwn(actual, key)) continue; await coolify(`/applications/${encodeURIComponent(uuid)}/envs`, { method: "POST", body: JSON.stringify({ key, value, is_literal: true, is_preview: false, is_multiline: false, is_shown_once: false, }), }); } } function getUnsafeManagedWorkerEnvKeys(group, entries) { const actual = envEntriesToRecord(entries); const desired = desiredWorkerEnv(group); const unsafe = []; if (Object.hasOwn(actual, "APP_REVISION")) unsafe.push("APP_REVISION"); unsafe.push(...Object.keys(actual).filter(isWorkerPrivilegedEnvKey)); if (!KWP_WORKER_GROUP_NAMES.has(group.name) && Object.hasOwn(actual, KWP_WORKER_ENV_KEY)) unsafe.push(KWP_WORKER_ENV_KEY); for (const key of Object.keys(desired)) { if (!Object.hasOwn(actual, key)) continue; if (key === "WORKFLOW_TRIGGER_EXECUTOR_MODE") { if (!normalizeWorkflowTriggerExecutorMode(actual[key])) unsafe.push(key); } else if (actual[key] !== desired[key]) { unsafe.push(key); } } return [...new Set(unsafe)].sort(); } function predictedManagedWorkerEnv(group, targetEnvs, sourceEnvs) { const predicted = envEntriesToRecord(targetEnvs); if (group.name !== "general") { for (const key of WORKER_GENERAL_PROVIDER_ENV_KEYS) { delete predicted[key]; } } if (!KWP_WORKER_GROUP_NAMES.has(group.name)) delete predicted[KWP_WORKER_ENV_KEY]; for (const entry of sourceEnvs) { const key = envKey(entry); if ( !key || Object.hasOwn(predicted, key) || WORKER_ENV_OVERRIDE_KEYS.has(key) || isWorkerPrivilegedEnvKey(key) || !canWorkerGroupReceiveProviderEnv(group, key) ) { continue; } predicted[key] = envValue(entry); } for (const [key, value] of Object.entries(desiredWorkerEnv(group))) { if (!Object.hasOwn(predicted, key)) predicted[key] = value; } return entriesFromRecord(predicted); } function buildProviderEnvSanitizationPlan(group, entries) { const removals = []; const sanitizedEnvs = []; for (const entry of entries) { const key = envKey(entry); const disallowedBillingProvider = group.name !== "general" && WORKER_GENERAL_PROVIDER_ENV_KEYS.has(key); const disallowedKwpPilot = key === KWP_WORKER_ENV_KEY && !KWP_WORKER_GROUP_NAMES.has(group.name); if (!disallowedBillingProvider && !disallowedKwpPilot) { sanitizedEnvs.push(entry); continue; } const uuid = typeof entry?.uuid === "string" ? entry.uuid.trim() : ""; if (!uuid) { fail(`Coolify worker-${group.name} provider environment cannot be removed safely because its env UUID is missing`); } removals.push({ uuid }); } return { sanitizedEnvs, removals }; } async function reconcileKwpWorkerEnv(resolvedGroups) { const general = resolvedGroups.find((entry) => entry.group.name === "general"); const importWorker = resolvedGroups.find((entry) => entry.group.name === "import"); if (!general?.app || !importWorker?.app) { fail("Coolify KWP worker environment requires worker-general and worker-import"); } const generalEntry = canonicalRuntimeEnvEntries(general.envs).find((entry) => envKey(entry) === KWP_WORKER_ENV_KEY); const importEntry = canonicalRuntimeEnvEntries(importWorker.envs).find((entry) => envKey(entry) === KWP_WORKER_ENV_KEY); const desiredValue = generalEntry ? envValue(generalEntry).trim() : ""; const actualValue = importEntry ? envValue(importEntry).trim() : ""; if (!desiredValue) { if (!importEntry) return; const envUuid = typeof importEntry.uuid === "string" ? importEntry.uuid.trim() : ""; if (!envUuid) { fail("Coolify worker-import KWP environment cannot be removed safely because its env UUID is missing"); } await coolify(`/applications/${encodeURIComponent(importWorker.app.uuid)}/envs/${encodeURIComponent(envUuid)}`, { method: "DELETE" }); importWorker.envs = importWorker.envs.filter((entry) => entry !== importEntry); process.stdout.write("Removed stale KWP pilot environment from Coolify worker-import\n"); return; } if (actualValue === desiredValue) return; await coolify(`/applications/${encodeURIComponent(importWorker.app.uuid)}/envs`, { method: importEntry ? "PATCH" : "POST", body: JSON.stringify({ key: KWP_WORKER_ENV_KEY, value: desiredValue, is_literal: generalEntry.is_literal ?? true, is_preview: false, is_multiline: generalEntry.is_multiline ?? false, is_shown_once: generalEntry.is_shown_once ?? false, }), }); importWorker.envs = [ ...importWorker.envs.filter((entry) => envKey(entry) !== KWP_WORKER_ENV_KEY), { ...generalEntry, key: KWP_WORKER_ENV_KEY, value: desiredValue, is_preview: false }, ]; process.stdout.write("Reconciled KWP pilot environment for Coolify worker-import\n"); } async function removeForbiddenProviderEnv(uuid, group, removals) { for (const removal of removals) { await coolify(`/applications/${encodeURIComponent(uuid)}/envs/${encodeURIComponent(removal.uuid)}`, { method: "DELETE", }); } if (removals.length > 0) { process.stdout.write(`Removed disallowed provider environment from Coolify worker-${group.name}\n`); } } function entriesFromRecord(record) { return Object.entries(record).map(([key, value]) => ({ key, value })); } async function ensureWorkerAppConfig(uuid, app, group) { assertWorkerCommandHealthCanBeAuthoritative(app, group); const drift = getWorkerAppConfigDrift(app, group, imageTag); if (drift.length === 0) { return app; } await coolify(`/applications/${encodeURIComponent(uuid)}`, { method: "PATCH", body: JSON.stringify(desiredWorkerAppConfig(group, imageTag)), }); const updated = await coolify(`/applications/${encodeURIComponent(uuid)}`); const remainingDrift = getWorkerAppConfigDrift(updated, group, imageTag); if (remainingDrift.length > 0) { fail(`Coolify worker-${group.name} app config drift remains after reconciliation`); } process.stdout.write(`Reconciled Coolify worker-${group.name} deployment health\n`); return updated; } function assertWorkerCommandHealthCanBeAuthoritative(app, group) { if (normalizeCoolifyBoolean(app?.custom_healthcheck_found) !== "false") { fail(`Coolify worker-${group.name} image healthcheck must be absent so command health can be authoritative`); } } function assertWorkerEnvEntriesMatch(app, group, envs, { includeAppConfig = true } = {}) { const drift = [...(includeAppConfig ? getWorkerAppConfigDrift(app, group, imageTag) : []), ...getWorkerEnvDrift(app, group, envs)]; if (drift.length > 0) { const keys = [...new Set(drift.map((item) => item.key))].sort().join(", "); fail(`Coolify worker-${group.name} drift detected in: ${keys}`); } } async function assertWorkerEnvMatches(uuid, app, group, options = {}) { const envs = asArray(await coolify(`/applications/${encodeURIComponent(uuid)}/envs`)); assertWorkerEnvEntriesMatch(app, group, envs, options); } async function hydrateManagedWorkerApps(apps, allowed, provisioningScope) { return Promise.all( apps.map(async (app) => { const markerGroup = GROUPS.find((group) => hasManagedWorkerMarker(app, group)); const matchedGroup = markerGroup || GROUPS.find((group) => matchesWorkerGroup(app, group.name)); if (provisionEnabled && !isInProvisioningEnvironment(app, provisioningScope)) return app; const candidateIsInScope = matchedGroup && (provisionEnabled || allowed.size === 0 || allowed.has(app.uuid)); if (!candidateIsInScope) return app; const detailed = await coolify(`/applications/${encodeURIComponent(app.uuid)}`); if (detailed?.uuid !== app.uuid) { fail(`Coolify worker-${matchedGroup.name} application identity could not be verified`); } return detailed; }) ); } function resolveWorkerGroups(apps, allowed, provisioningScope) { const resolvedGroups = GROUPS.map((group) => { const isTargetScope = (app) => !provisionEnabled || isInProvisioningScope(app, provisioningScope); const markedApps = apps.filter((app) => hasManagedWorkerMarker(app, group) && isTargetScope(app)); for (const app of markedApps) { const trustedByAllowlist = allowed.has(app.uuid); if ((provisionEnabled || allowed.size === 0 || trustedByAllowlist) && !hasManagedWorkerIdentity(app, group)) { fail(`Managed Coolify worker-${group.name} application identity is invalid`); } if (provisionEnabled && !trustedByAllowlist) { if (!hasManagedWorkerRetryProvenance(app, group)) { fail(`Managed Coolify worker-${group.name} retry provenance is invalid`); } if (!isInProvisioningScope(app, provisioningScope)) { fail(`Managed Coolify worker-${group.name} application is outside the configured provisioning scope`); } } } const allowedMatches = apps.filter((app) => matchesWorkerGroup(app, group.name) && isTargetScope(app) && (allowed.size === 0 || allowed.has(app.uuid))); const retryMatches = provisionEnabled ? markedApps.filter( (app) => hasManagedWorkerIdentity(app, group) && (allowed.has(app.uuid) || (hasManagedWorkerRetryProvenance(app, group) && isInProvisioningScope(app, provisioningScope))) ) : []; const candidates = [...new Map([...allowedMatches, ...retryMatches].map((app) => [app.uuid, app])).values()]; if (candidates.length > 1) { fail(`Multiple Coolify worker-${group.name} applications matched the managed scope`); } const app = candidates[0]; return { group, app, envs: [], managed: Boolean( provisionEnabled && app && hasManagedWorkerIdentity(app, group) && (allowed.has(app.uuid) || (hasManagedWorkerRetryProvenance(app, group) && isInProvisioningScope(app, provisioningScope))) ), needsRepair: false, }; }); return resolvedGroups; } async function main() { if (!["staging", "production"].includes(environment)) { fail("Usage: coolify-ensure-worker-groups.mjs staging|production"); } if (!workerImage || workerImage.split("/").length < 4) { fail("Worker image path could not be resolved from REGISTRY, IMAGE_NAMESPACE, IMAGE_PREFIX, and worker target"); } if (!/^[A-Z][A-Z0-9_]*$/.test(resolvedEnvName)) { fail("Resolved Coolify resource environment variable name is invalid"); } const allowed = new Set(existingResourceUuids.split(",").filter(Boolean)); for (const uuid of allowed) { requireCoolifyCuid(uuid, `${prefix}_RESOURCE_UUIDS entry`); } if (configuredWorkerEnvSourceUuid) { requireCoolifyCuid(configuredWorkerEnvSourceUuid, `${prefix}_WORKER_ENV_SOURCE_UUID`); } const listedApps = asArray(await coolify("/applications")); for (const app of listedApps) { requireCoolifyCuid(typeof app?.uuid === "string" ? app.uuid : "", "Coolify application UUID"); } const provisioningScope = provisionEnabled ? await resolveProvisioningScope() : null; const apps = await hydrateManagedWorkerApps(listedApps, allowed, provisioningScope); const resolvedGroups = resolveWorkerGroups(apps, allowed, provisioningScope); for (const entry of resolvedGroups) { if (!entry.app && !provisionEnabled) { fail(`Missing Coolify worker-${entry.group.name} application`); } if (entry.app) { assertWorkerCommandHealthCanBeAuthoritative(entry.app, entry.group); entry.envs = asArray(await coolify(`/applications/${encodeURIComponent(entry.app.uuid)}/envs`)); const providerEnvSanitization = buildProviderEnvSanitizationPlan(entry.group, entry.envs); entry.sanitizedEnvs = providerEnvSanitization.sanitizedEnvs; entry.providerEnvRemovals = providerEnvSanitization.removals; if (entry.managed) { const unsafeKeys = getUnsafeManagedWorkerEnvKeys(entry.group, entry.sanitizedEnvs); if (unsafeKeys.length > 0) { fail(`Managed Coolify worker-${entry.group.name} partial environment is unsafe: ${unsafeKeys.join(", ")}`); } entry.needsRepair = getWorkerEnvDrift(entry.app, entry.group, entry.sanitizedEnvs).length > 0; } else { assertWorkerEnvEntriesMatch(entry.app, entry.group, entry.sanitizedEnvs, { includeAppConfig: false }); } } else { entry.sanitizedEnvs = []; entry.providerEnvRemovals = []; } } const needsProvisioningOrRepair = resolvedGroups.some((entry) => !entry.app || entry.needsRepair); const workerEnvSourceUuid = needsProvisioningOrRepair ? resolveWorkerEnvSourceUuid(apps, allowed) : ""; const sourceEnvs = needsProvisioningOrRepair && workerEnvSourceUuid ? canonicalRuntimeEnvEntries(asArray(await coolify(`/applications/${encodeURIComponent(workerEnvSourceUuid)}/envs`))) : []; for (const entry of resolvedGroups) { if (!entry.app || entry.needsRepair) { const predicted = predictedManagedWorkerEnv(entry.group, entry.sanitizedEnvs, sourceEnvs); const drift = getWorkerEnvDrift(entry.app, entry.group, predicted); if (drift.length > 0) { const keys = [...new Set(drift.map((item) => item.key))].sort().join(", "); fail(`Coolify worker-${entry.group.name} cannot be safely provisioned or repaired; predicted drift: ${keys}`); } } } for (const entry of resolvedGroups) { if (entry.app && entry.providerEnvRemovals.length > 0) { await removeForbiddenProviderEnv(entry.app.uuid, entry.group, entry.providerEnvRemovals); entry.envs = entry.sanitizedEnvs; } } for (const entry of resolvedGroups) { if (!entry.app) { entry.app = await createWorkerApplication(entry.group, workerEnvSourceUuid, sourceEnvs, provisioningScope); await assertWorkerEnvMatches(entry.app.uuid, entry.app, entry.group, { includeAppConfig: false }); } else if (entry.needsRepair) { const clonedEnvs = await cloneWorkerEnv(workerEnvSourceUuid, entry.app.uuid, entry.group, { sourceEnvs, targetEnvs: entry.sanitizedEnvs, }); await ensureMissingWorkerEnv(entry.app.uuid, entry.group, clonedEnvs); await assertWorkerEnvMatches(entry.app.uuid, entry.app, entry.group, { includeAppConfig: false }); process.stdout.write(`Repaired partial Coolify worker-${entry.group.name} application\n`); } } for (const entry of resolvedGroups) { entry.envs = asArray(await coolify(`/applications/${encodeURIComponent(entry.app.uuid)}/envs`)); } await reconcileKwpWorkerEnv(resolvedGroups); for (const { app, group } of resolvedGroups) { assertWorkerCommandHealthCanBeAuthoritative(app, group); } for (const entry of resolvedGroups) { entry.app = await ensureWorkerAppConfig(entry.app.uuid, entry.app, entry.group); } for (const { app, group } of resolvedGroups) { await assertWorkerEnvMatches(app.uuid, app, group); } const resolvedWorkers = resolvedGroups.map(({ app }) => app.uuid); const resolved = [...new Set([...allowed, ...resolvedWorkers].filter(Boolean))].join(","); if (process.env.GITHUB_ENV) { const fs = await import("node:fs"); fs.appendFileSync(process.env.GITHUB_ENV, `${resolvedEnvName}=${resolved}\n`); } process.stdout.write(`Coolify ${environment} worker group applications are present\n`); } const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : ""; if (invokedPath && fileURLToPath(import.meta.url) === invokedPath) { try { await main(); } catch (error) { if (error instanceof CoolifyRuntimeEnvConflictError) { fail(error.message); } throw error; } }