import { spawn } from "node:child_process"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import http from "node:http"; import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { GROUPS, KWP_WORKER_ENV_KEY, WORKER_ENV_OVERRIDE_KEYS, WORKER_GENERAL_PROVIDER_ENV_KEYS, WORKER_PRIVILEGED_ENV_KEYS, buildWorkerDeploymentHealthCommand, buildWorkerApplicationBody, desiredWorkerEnv, envEntriesToRecord, getWorkerAppConfigDrift, getWorkerEnvDrift, } from "./coolify-ensure-worker-groups.mjs"; const DEPLOY_REVISION = "a".repeat(40); const PROVISIONER_PATH = path.resolve("scripts/coolify-ensure-worker-groups.mjs"); const PROVISIONER_TOKEN = "coolify-worker-provisioner-token-sentinel"; const WORKER_IMAGE = "registry.example.test/toolctrl-labs/hivegrid/worker"; const WORKER_DATABASE_URL = "postgres://hivegrid_worker:redacted@db.example.test/hivegrid"; const SHARED_CRYPTO_SECRET = "shared-tenant-envelope-secret-sentinel"; const WORKER_UUIDS = Object.fromEntries( GROUPS.map((workerGroup, index) => [workerGroup.name, `${String.fromCharCode(97 + index)}${String(index).repeat(23)}`]) ); const WORKER_ENV_SOURCE_UUID = `s${"8".repeat(23)}`; const RETRY_GENERAL_UUID = `r${"9".repeat(23)}`; const PROJECT_UUID = `p${"1".repeat(23)}`; const SERVER_UUID = `v${"2".repeat(23)}`; const DESTINATION_UUID = `d${"3".repeat(23)}`; const ENVIRONMENT_UUID = `n${"4".repeat(23)}`; const ENVIRONMENT_ID = 41; const FOREIGN_SERVER_UUID = `q${"5".repeat(23)}`; const FOREIGN_DESTINATION_UUID = `x${"6".repeat(23)}`; const FOREIGN_ENVIRONMENT_ID = 84; const FOREIGN_WORKER_UUIDS = Object.fromEntries( GROUPS.map((workerGroup, index) => [workerGroup.name, `${String.fromCharCode(107 + index)}${String(index + 5).repeat(23)}`]) ); function group(name) { const found = GROUPS.find((item) => item.name === name); if (!found) { throw new Error(`Missing test group ${name}`); } return found; } function entriesFromRecord(record) { return Object.entries(record).map(([key, value]) => ({ key, value })); } function productAcquisitionProviderEnv() { return { PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED: "true", LAGO_API_URL: "http://lago.internal.test:3108", LAGO_API_KEY: "redacted-lago-key", LAGO_TIMEOUT_MS: "10000", MOLLIE_API_KEY: "redacted-mollie-key", MOLLIE_PROFILE_ID: "redacted-mollie-profile", MOLLIE_TIMEOUT_MS: "10000", BILLING_MOLLIE_WEBHOOK_URL: "https://staging-api.hivegrid.de/v1/billing/mollie/webhooks", }; } const KWP_PILOT_TENANT = "kwp-pilot-tenant-sentinel"; function desiredWorkerEnvWithRuntime(workerGroup, overrides = {}) { return { ...desiredWorkerEnv(workerGroup), DATABASE_URL: WORKER_DATABASE_URL, BULLMQ_REDIS_URL: "redis://redacted", ...(workerGroup.name === "general" ? productAcquisitionProviderEnv() : {}), ...(workerGroup.name === "mail" ? { CRYPTO_SECRET: SHARED_CRYPTO_SECRET } : {}), ...overrides, }; } function provisioningResourceUuids(missingGroup = "general") { return [WORKER_ENV_SOURCE_UUID, ...GROUPS.filter((workerGroup) => workerGroup.name !== missingGroup).map((workerGroup) => WORKER_UUIDS[workerGroup.name])]; } function provisioningEnvOverrides() { return { COOLIFY_STAGING_PROVISION_WORKER_GROUPS: "true", COOLIFY_STAGING_PROJECT_UUID: PROJECT_UUID, COOLIFY_STAGING_SERVER_UUID: SERVER_UUID, COOLIFY_STAGING_DESTINATION_UUID: DESTINATION_UUID, COOLIFY_STAGING_ENVIRONMENT_UUID: ENVIRONMENT_UUID, COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID: WORKER_ENV_SOURCE_UUID, }; } function runProvisioner(serverUrl, resourceUuids, envOverrides = {}) { return new Promise((resolve) => { const child = spawn(process.execPath, [PROVISIONER_PATH, "staging"], { env: { PATH: process.env.PATH || "", NODE_ENV: "test", SELECTED_COOLIFY_API_URL: serverUrl, SELECTED_COOLIFY_TOKEN: PROVISIONER_TOKEN, IMAGE_TAG: DEPLOY_REVISION, WORKER_IMAGE, COOLIFY_STAGING_RESOURCE_UUIDS: resourceUuids.join(","), ...envOverrides, }, stdio: ["ignore", "pipe", "pipe"], }); let stdout = ""; let stderr = ""; child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); child.stdout.on("data", (chunk) => { stdout += chunk; }); child.stderr.on("data", (chunk) => { stderr += chunk; }); child.on("close", (status) => { resolve({ status, stdout, stderr }); }); }); } async function requestBody(request) { let body = ""; request.setEncoding("utf8"); for await (const chunk of request) { body += chunk; } return body; } function startProvisionerServer({ appRevisionOverride = false, customHealthcheckFound = false, invalidEnvGroup = "", malformedEnvGroup = "", persistPatch = true, } = {}) { const apps = new Map( GROUPS.map((workerGroup) => [ `worker-${workerGroup.name}`, { uuid: WORKER_UUIDS[workerGroup.name], name: `worker-${workerGroup.name}`, description: `WORKER_QUEUE_GROUP=${workerGroup.name}`, docker_registry_image_name: WORKER_IMAGE, docker_registry_image_tag: "previous-revision", ports_exposes: workerGroup.healthPort, health_check_enabled: false, health_check_type: "http", health_check_path: "/readyz", custom_healthcheck_found: customHealthcheckFound, }, ]) ); const requests = []; const server = http.createServer(async (request, response) => { const body = await requestBody(request); requests.push({ method: request.method, url: request.url, authorization: request.headers.authorization, body }); const segments = String(request.url || "") .split("/") .filter(Boolean); let status = 200; let payload = {}; if (request.method === "GET" && request.url === "/applications") { payload = [...apps.values()]; } else if (segments[0] === "applications" && segments[2] === "envs" && request.method === "GET") { const requestedUuid = decodeURIComponent(segments[1] || ""); const app = [...apps.values()].find((candidate) => candidate.uuid === requestedUuid); const workerGroup = GROUPS.find((candidate) => app?.name === `worker-${candidate.name}`); if (!workerGroup) { status = 404; } else if (workerGroup.name === malformedEnvGroup) { payload = { unexpected: [] }; } else { payload = entriesFromRecord( desiredWorkerEnvWithRuntime(workerGroup, { ...(appRevisionOverride || workerGroup.name === invalidEnvGroup ? { APP_REVISION: "" } : {}), ...(workerGroup.name === "general" ? { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token" } : {}), }) ); } } else if (segments[0] === "applications" && segments.length === 2 && request.method === "GET") { const requestedUuid = decodeURIComponent(segments[1] || ""); payload = [...apps.values()].find((candidate) => candidate.uuid === requestedUuid) || {}; } else if (segments[0] === "applications" && segments.length === 2 && request.method === "PATCH") { const uuid = decodeURIComponent(segments[1] || ""); const app = [...apps.values()].find((candidate) => candidate.uuid === uuid); if (!app) { status = 404; } else if (persistPatch) { Object.assign(app, JSON.parse(body)); payload = app; } } else { status = 404; } response.writeHead(status, { "content-type": "application/json" }); response.end(JSON.stringify(payload)); }); return new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { resolve({ url: `http://127.0.0.1:${server.address().port}`, apps, requests, close: () => new Promise((closeResolve, closeReject) => server.close((error) => (error ? closeReject(error) : closeResolve()))), }); }); }); } function buildProvisionedWorkerApp(workerGroup, uuid, { managed = false } = {}) { const body = buildWorkerApplicationBody({ group: workerGroup, projectUuid: PROJECT_UUID, serverUuid: SERVER_UUID, destinationUuid: DESTINATION_UUID, environmentName: "staging", environmentUuid: ENVIRONMENT_UUID, workerImage: WORKER_IMAGE, imageTag: DEPLOY_REVISION, }); return { ...body, uuid, description: managed ? body.description : `WORKER_QUEUE_GROUP=${workerGroup.name}`, custom_healthcheck_found: false, environment_id: ENVIRONMENT_ID, destination: { uuid: DESTINATION_UUID, server: { uuid: SERVER_UUID }, }, }; } function startProvisioningRetryServer({ preexistingPartialEnv = null, preexistingPartialAppOverrides = {}, sourceEnvEntries = null, failEnvKeyOnce = "", allTargetWorkersManaged = false, includeForeignFleet = false, missingGroup = "general", targetEnvOverrides = {}, } = {}) { let envUuidSequence = 0; const attachEnvUuids = (entries) => entries.map((entry) => ({ ...entry, uuid: typeof entry?.uuid === "string" && entry.uuid ? entry.uuid : `env-${++envUuidSequence}`, })); const sourceApp = { uuid: WORKER_ENV_SOURCE_UUID, name: "hivegrid-worker", description: "Managed worker environment source", docker_registry_image_name: WORKER_IMAGE, }; const apps = new Map([[sourceApp.uuid, sourceApp]]); const envs = new Map([ [ sourceApp.uuid, attachEnvUuids( sourceEnvEntries || entriesFromRecord({ DATABASE_URL: WORKER_DATABASE_URL, BULLMQ_REDIS_URL: "redis://redacted", ...productAcquisitionProviderEnv(), WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token", CRYPTO_SECRET: SHARED_CRYPTO_SECRET, }) ), ], ]); for (const workerGroup of GROUPS.filter((candidate) => candidate.name !== missingGroup)) { const app = buildProvisionedWorkerApp(workerGroup, WORKER_UUIDS[workerGroup.name], { managed: allTargetWorkersManaged }); apps.set(app.uuid, app); envs.set( app.uuid, attachEnvUuids( entriesFromRecord( desiredWorkerEnvWithRuntime(workerGroup, { ...(workerGroup.name === "general" ? { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token" } : {}), ...(targetEnvOverrides[workerGroup.name] || {}), }) ) ) ); } if (preexistingPartialEnv !== null) { if (missingGroup !== "general") { throw new Error("preexistingPartialEnv is only supported for the general worker test fixture"); } const partial = { ...buildProvisionedWorkerApp(group("general"), RETRY_GENERAL_UUID, { managed: true }), ...preexistingPartialAppOverrides, }; apps.set(partial.uuid, partial); envs.set(partial.uuid, attachEnvUuids(entriesFromRecord(preexistingPartialEnv))); } if (includeForeignFleet) { for (const workerGroup of GROUPS) { const foreign = { ...buildProvisionedWorkerApp(workerGroup, FOREIGN_WORKER_UUIDS[workerGroup.name], { managed: true }), environment_id: FOREIGN_ENVIRONMENT_ID, destination: { uuid: FOREIGN_DESTINATION_UUID, server: { uuid: FOREIGN_SERVER_UUID }, }, }; apps.set(foreign.uuid, foreign); envs.set( foreign.uuid, attachEnvUuids( entriesFromRecord( desiredWorkerEnvWithRuntime(workerGroup, { ...(workerGroup.name === "general" ? { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://foreign.example.test", API_ACCESS_TOKEN: "foreign-token" } : {}), }) ) ) ); } } const requests = []; let pendingFailureKey = failEnvKeyOnce; const server = http.createServer(async (request, response) => { const body = await requestBody(request); requests.push({ method: request.method, url: request.url, authorization: request.headers.authorization, body }); const segments = String(request.url || "") .split("/") .filter(Boolean); let status = 200; let payload = {}; if (request.method === "GET" && request.url === `/projects/${PROJECT_UUID}`) { payload = { uuid: PROJECT_UUID, environments: [{ id: ENVIRONMENT_ID, uuid: ENVIRONMENT_UUID, name: "staging" }], }; } else if (request.method === "GET" && request.url === "/applications") { payload = [...apps.values()].map(({ destination: _destination, ...app }) => app); } else if (request.method === "POST" && request.url === "/applications/dockerimage") { const input = JSON.parse(body); const createdGroup = GROUPS.find((candidate) => input.name === `worker-${candidate.name}`); if (!createdGroup) { status = 400; payload = { message: "unknown worker group" }; response.writeHead(status, { "content-type": "application/json" }); response.end(JSON.stringify(payload)); return; } const app = { ...input, uuid: createdGroup.name === "general" ? RETRY_GENERAL_UUID : WORKER_UUIDS[createdGroup.name], custom_healthcheck_found: false, environment_id: ENVIRONMENT_ID, destination: { uuid: DESTINATION_UUID, server: { uuid: SERVER_UUID }, }, }; apps.set(app.uuid, app); envs.set(app.uuid, []); payload = { uuid: app.uuid }; } else if (segments[0] === "applications" && segments[2] === "envs" && request.method === "GET") { const requestedUuid = decodeURIComponent(segments[1] || ""); if (!envs.has(requestedUuid)) { status = 404; } else { payload = envs.get(requestedUuid); } } else if (segments[0] === "applications" && segments[2] === "envs" && ["POST", "PATCH"].includes(request.method)) { const requestedUuid = decodeURIComponent(segments[1] || ""); const input = JSON.parse(body); if (!envs.has(requestedUuid)) { status = 404; } else if (input.key === pendingFailureKey) { pendingFailureKey = ""; status = 500; } else { const existing = envs.get(requestedUuid).filter((entry) => entry.key !== input.key); const [stored] = attachEnvUuids([input]); existing.push(stored); envs.set(requestedUuid, existing); payload = stored; } } else if (segments[0] === "applications" && segments[2] === "envs" && segments.length === 4 && request.method === "DELETE") { const requestedUuid = decodeURIComponent(segments[1] || ""); const requestedEnvUuid = decodeURIComponent(segments[3] || ""); const existing = envs.get(requestedUuid); if (!existing || !existing.some((entry) => entry.uuid === requestedEnvUuid)) { status = 404; } else { envs.set( requestedUuid, existing.filter((entry) => entry.uuid !== requestedEnvUuid) ); payload = { message: "Environment variable deleted." }; } } else if (segments[0] === "applications" && segments.length === 2 && request.method === "GET") { const requestedUuid = decodeURIComponent(segments[1] || ""); const app = apps.get(requestedUuid); if (!app) { status = 404; } else { payload = app; } } else if (segments[0] === "applications" && segments.length === 2 && request.method === "PATCH") { const requestedUuid = decodeURIComponent(segments[1] || ""); const app = apps.get(requestedUuid); if (!app) { status = 404; } else { Object.assign(app, JSON.parse(body)); payload = app; } } else { status = 404; } response.writeHead(status, { "content-type": "application/json" }); response.end(JSON.stringify(payload)); }); return new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { resolve({ url: `http://127.0.0.1:${server.address().port}`, apps, envs, requests, close: () => new Promise((closeResolve, closeReject) => server.close((error) => (error ? closeReject(error) : closeResolve()))), }); }); }); } describe("coolify worker group env drift", () => { it.each(["mail", "import", "webhook", "plugin"])("treats provider credentials on worker-%s as redacted least-privilege drift", (groupName) => { const workerGroup = group(groupName); const entries = entriesFromRecord({ ...desiredWorkerEnvWithRuntime(workerGroup), ...productAcquisitionProviderEnv(), }); const providerDrift = getWorkerEnvDrift({ uuid: `worker-${groupName}` }, workerGroup, entries).filter((item) => WORKER_GENERAL_PROVIDER_ENV_KEYS.has(item.key) ); expect(providerDrift).toHaveLength(WORKER_GENERAL_PROVIDER_ENV_KEYS.size); expect(providerDrift).toEqual( expect.arrayContaining( [...WORKER_GENERAL_PROVIDER_ENV_KEYS].map((key) => expect.objectContaining({ key, expected: "unset; billing provider credentials belong only to worker-general", actual: "", group: groupName, }) ) ) ); expect(JSON.stringify(providerDrift)).not.toContain("redacted-lago-key"); expect(JSON.stringify(providerDrift)).not.toContain("redacted-mollie-key"); }); it("reconciles and verifies revision-bound health config through the Coolify API", async () => { const server = await startProvisionerServer(); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(result.stdout).toContain("Coolify staging worker group applications are present"); expect(result.stdout).not.toContain(PROVISIONER_TOKEN); expect(result.stderr).not.toContain(PROVISIONER_TOKEN); expect(server.requests.every((request) => request.authorization === `Bearer ${PROVISIONER_TOKEN}`)).toBe(true); const patches = server.requests.filter((request) => request.method === "PATCH"); expect(patches).toHaveLength(GROUPS.length); for (const workerGroup of GROUPS) { const app = server.apps.get(`worker-${workerGroup.name}`); expect(app).toMatchObject({ docker_registry_image_tag: DEPLOY_REVISION, health_check_enabled: true, health_check_type: "cmd", health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} ${workerGroup.name}`, health_check_interval: 30, health_check_timeout: 10, health_check_retries: 3, health_check_start_period: 20, custom_healthcheck_found: false, }); } } finally { await server.close(); } }); it("deduplicates workers already present in the resolved resource allowlist", async () => { const server = await startProvisionerServer(); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); const tempDirectory = await mkdtemp(path.join(tmpdir(), "hivegrid-coolify-resources-")); const githubEnv = path.join(tempDirectory, "github-env"); try { const result = await runProvisioner(server.url, resourceUuids, { GITHUB_ENV: githubEnv }); expect(result.status).toBe(0); expect(await readFile(githubEnv, "utf8")).toBe(`STAGING_RESOLVED_RESOURCE_UUIDS=${resourceUuids.join(",")}\n`); } finally { await server.close(); await rm(tempDirectory, { recursive: true, force: true }); } }); it("fails closed when Coolify does not persist worker health config", async () => { const server = await startProvisionerServer({ persistPatch: false }); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids); expect(result.status).toBe(1); expect(result.stdout).toBe(""); expect(result.stderr).toContain("app config drift remains after reconciliation"); expect(result.stdout).not.toContain(PROVISIONER_TOKEN); expect(result.stderr).not.toContain(PROVISIONER_TOKEN); } finally { await server.close(); } }); it("does not mutate worker app config when runtime env validation fails", async () => { const server = await startProvisionerServer({ appRevisionOverride: true }); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids); expect(result.status).toBe(1); expect(result.stderr).toContain("APP_REVISION"); expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("prevalidates every worker group before mutating any app config", async () => { const server = await startProvisionerServer({ invalidEnvGroup: "plugin" }); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids); expect(result.status).toBe(1); expect(result.stderr).toContain("APP_REVISION"); expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]); expect(server.requests.filter((request) => request.method === "GET" && request.url.endsWith("/envs"))).toHaveLength(GROUPS.length); } finally { await server.close(); } }); it("fails closed on malformed worker environment responses before mutation", async () => { const server = await startProvisionerServer({ malformedEnvGroup: "webhook" }); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids); expect(result.status).toBe(1); expect(result.stderr).toContain("list response was malformed"); expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("rejects worker images whose built-in healthcheck would override command health", async () => { const server = await startProvisionerServer({ customHealthcheckFound: true }); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids); expect(result.status).toBe(1); expect(result.stdout).toBe(""); expect(result.stderr).toContain("image healthcheck must be absent"); expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("rejects invalid Coolify request timeout configuration before making a request", async () => { const server = await startProvisionerServer(); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids, { COOLIFY_API_TIMEOUT_MS: "invalid" }); expect(result.status).toBe(1); expect(result.stderr).toContain("COOLIFY_API_TIMEOUT_MS"); expect(server.requests).toEqual([]); } finally { await server.close(); } }); it("rejects remote application UUIDs that could inject the resolved environment file", async () => { const server = await startProvisionerServer(); server.apps.get("worker-general").uuid = `${WORKER_UUIDS.general}\nINJECTED=value`; try { const result = await runProvisioner(server.url, []); expect(result.status).toBe(1); expect(result.stdout).toBe(""); expect(result.stderr).toContain("valid Coolify CUID"); expect(result.stderr).not.toContain("INJECTED"); expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it.each(["DATABASE_URL", "WORKER_ENABLED"])( "repairs a safely identified partial worker after %s provisioning fails without creating a duplicate app", async (failedKey) => { const server = await startProvisioningRetryServer({ failEnvKeyOnce: failedKey }); try { const first = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(first.status).toBe(1); expect(first.stderr).toContain("Coolify API request failed with HTTP 500"); expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toHaveLength(1); const second = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(second.status).toBe(0); expect(second.stderr).toBe(""); expect(second.stdout).toContain("Repaired partial Coolify worker-general application"); expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toHaveLength(1); expect(envEntriesToRecord(server.envs.get(RETRY_GENERAL_UUID))).toMatchObject({ ...desiredWorkerEnv(group("general")), DATABASE_URL: WORKER_DATABASE_URL, BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token", }); } finally { await server.close(); } } ); it.each([ [ "preview-first", [ { key: "DATABASE_URL", value: "", is_preview: true }, { key: "DATABASE_URL", value: WORKER_DATABASE_URL, is_preview: false }, ], ], [ "production-first", [ { key: "DATABASE_URL", value: WORKER_DATABASE_URL, is_preview: false }, { key: "DATABASE_URL", value: "", is_preview: true }, ], ], ])("clones the non-preview runtime value independent of %s Coolify env ordering", async (_order, databaseEntries) => { const server = await startProvisioningRetryServer({ sourceEnvEntries: [ ...databaseEntries, { key: "BULLMQ_REDIS_URL", value: "redis://redacted", is_preview: false }, ...entriesFromRecord(productAcquisitionProviderEnv()).map((entry) => ({ ...entry, is_preview: false })), { key: "WORKFLOW_TRIGGER_EXECUTOR_URL", value: "https://app.example.test", is_preview: false }, { key: "API_ACCESS_TOKEN", value: "redacted-worker-token", is_preview: false }, ], }); try { const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(envEntriesToRecord(server.envs.get(RETRY_GENERAL_UUID))).toMatchObject({ DATABASE_URL: WORKER_DATABASE_URL, BULLMQ_REDIS_URL: "redis://redacted", ...productAcquisitionProviderEnv(), }); const clonedDatabaseRequests = server.requests .filter((request) => request.method === "POST" && request.url === `/applications/${RETRY_GENERAL_UUID}/envs`) .map((request) => JSON.parse(request.body)) .filter((body) => body.key === "DATABASE_URL"); expect(clonedDatabaseRequests).toEqual([expect.objectContaining({ value: WORKER_DATABASE_URL, is_preview: false })]); } finally { await server.close(); } }); it("fails before mutation when a misplaced provider variable has no deletable Coolify env UUID", async () => { const server = await startProvisioningRetryServer({ missingGroup: "none", targetEnvOverrides: { mail: { LAGO_API_KEY: "undeletable-lago-secret-sentinel", }, }, }); const mailEnvs = server.envs.get(WORKER_UUIDS.mail); for (const entry of mailEnvs) { if (entry.key === "LAGO_API_KEY") { delete entry.uuid; } } try { const result = await runProvisioner(server.url, provisioningResourceUuids("none"), provisioningEnvOverrides()); expect(result.status).toBe(1); expect(result.stderr).toContain("provider environment cannot be removed safely because its env UUID is missing"); expect(result.stdout).not.toContain("undeletable-lago-secret-sentinel"); expect(result.stderr).not.toContain("undeletable-lago-secret-sentinel"); expect(server.requests.filter((request) => ["POST", "PATCH", "DELETE"].includes(request.method))).toEqual([]); } finally { await server.close(); } }); it("never clones migrator or seed credentials from the worker env source", async () => { const server = await startProvisioningRetryServer({ sourceEnvEntries: entriesFromRecord({ DATABASE_URL: WORKER_DATABASE_URL, BULLMQ_REDIS_URL: "redis://redacted", ...productAcquisitionProviderEnv(), WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token", DIRECT_URL: "postgres://owner:direct-secret-sentinel@db.example.test/hivegrid", MIGRATOR_DATABASE_URL: "postgres://migrator:migrator-secret-sentinel@db.example.test/hivegrid", SEED_ADMIN_PASSWORD: "seed-secret-sentinel", }), }); try { const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(result.status).toBe(0); const target = envEntriesToRecord(server.envs.get(RETRY_GENERAL_UUID)); expect(target).not.toHaveProperty("DIRECT_URL"); expect(target).not.toHaveProperty("MIGRATOR_DATABASE_URL"); expect(target).not.toHaveProperty("SEED_ADMIN_PASSWORD"); const clonedKeys = server.requests .filter((request) => request.method === "POST" && request.url === `/applications/${RETRY_GENERAL_UUID}/envs`) .map((request) => JSON.parse(request.body).key); expect(clonedKeys).not.toEqual(expect.arrayContaining(["DIRECT_URL", "MIGRATOR_DATABASE_URL", "SEED_ADMIN_PASSWORD"])); expect(result.stdout).not.toContain("secret-sentinel"); expect(result.stderr).not.toContain("secret-sentinel"); } finally { await server.close(); } }); it("provisions a non-general worker without cloning provider credentials from the approved source", async () => { const server = await startProvisioningRetryServer({ missingGroup: "mail" }); try { const result = await runProvisioner(server.url, provisioningResourceUuids("mail"), provisioningEnvOverrides()); expect(result.status).toBe(0); expect(result.stderr).toBe(""); const target = envEntriesToRecord(server.envs.get(WORKER_UUIDS.mail)); for (const key of WORKER_GENERAL_PROVIDER_ENV_KEYS) { expect(target).not.toHaveProperty(key); } expect(target.CRYPTO_SECRET).toBe(SHARED_CRYPTO_SECRET); const clonedKeys = server.requests .filter((request) => request.method === "POST" && request.url === `/applications/${WORKER_UUIDS.mail}/envs`) .map((request) => JSON.parse(request.body).key); expect(clonedKeys).not.toEqual(expect.arrayContaining([...WORKER_GENERAL_PROVIDER_ENV_KEYS])); expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.general))).toMatchObject(productAcquisitionProviderEnv()); } finally { await server.close(); } }); it.each([ ["managed", true], ["legacy allowlisted", false], ])("removes provider credentials from an existing %s non-general worker without logging values", async (_kind, managed) => { const misplacedProviderEnv = { ...productAcquisitionProviderEnv(), LAGO_API_KEY: "lago-secret-sentinel", MOLLIE_API_KEY: "mollie-secret-sentinel", }; const server = await startProvisioningRetryServer({ allTargetWorkersManaged: managed, missingGroup: "none", targetEnvOverrides: { mail: misplacedProviderEnv }, }); try { const result = await runProvisioner(server.url, provisioningResourceUuids("none"), provisioningEnvOverrides()); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(result.stdout).toContain("Removed disallowed provider environment from Coolify worker-mail"); for (const key of WORKER_GENERAL_PROVIDER_ENV_KEYS) { expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.mail))).not.toHaveProperty(key); } const deletionRequests = server.requests.filter( (request) => request.method === "DELETE" && request.url.startsWith(`/applications/${WORKER_UUIDS.mail}/envs/`) ); expect(deletionRequests).toHaveLength(WORKER_GENERAL_PROVIDER_ENV_KEYS.size); expect(result.stdout).not.toContain("lago-secret-sentinel"); expect(result.stderr).not.toContain("lago-secret-sentinel"); expect(result.stdout).not.toContain("mollie-secret-sentinel"); expect(result.stderr).not.toContain("mollie-secret-sentinel"); expect(JSON.stringify(server.requests)).not.toContain("lago-secret-sentinel"); expect(JSON.stringify(server.requests)).not.toContain("mollie-secret-sentinel"); expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.general))).toMatchObject(productAcquisitionProviderEnv()); } finally { await server.close(); } }); it("mirrors the KWP pilot tenant from worker-general to worker-import only", async () => { const server = await startProvisioningRetryServer({ missingGroup: "none", targetEnvOverrides: { general: { [KWP_WORKER_ENV_KEY]: KWP_PILOT_TENANT }, }, }); try { const result = await runProvisioner(server.url, provisioningResourceUuids("none"), provisioningEnvOverrides()); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(result.stdout).toContain("Reconciled KWP pilot environment for Coolify worker-import"); expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.import))).toMatchObject({ [KWP_WORKER_ENV_KEY]: KWP_PILOT_TENANT, }); for (const groupName of ["mail", "webhook", "plugin"]) { expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS[groupName]))).not.toHaveProperty(KWP_WORKER_ENV_KEY); } expect(result.stdout).not.toContain(KWP_PILOT_TENANT); expect(result.stderr).not.toContain(KWP_PILOT_TENANT); } finally { await server.close(); } }); it("fails before provisioning when the worker env source uses the application database role", async () => { const unsafeUrl = "postgres://hivegrid_app:source-secret-sentinel@db.example.test/hivegrid"; const server = await startProvisioningRetryServer({ sourceEnvEntries: entriesFromRecord({ DATABASE_URL: unsafeUrl, BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token", }), }); try { const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(result.status).toBe(1); expect(result.stderr).toMatch(/predicted drift: .*DATABASE_URL/); expect(result.stdout).not.toContain(unsafeUrl); expect(result.stderr).not.toContain(unsafeUrl); expect(result.stdout).not.toContain("source-secret-sentinel"); expect(result.stderr).not.toContain("source-secret-sentinel"); expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("fails before app creation or secret cloning when runtime source keys have duplicate non-preview entries", async () => { const server = await startProvisioningRetryServer({ sourceEnvEntries: [ { key: "DATABASE_URL", value: "postgres://hivegrid_worker:first-redacted@db.example.test/hivegrid", is_preview: false }, { key: "DATABASE_URL", value: "postgres://hivegrid_worker:second-redacted@db.example.test/hivegrid", is_preview: false }, { key: "BULLMQ_REDIS_URL", value: "redis://redacted", is_preview: false }, { key: "WORKFLOW_TRIGGER_EXECUTOR_URL", value: "https://app.example.test", is_preview: false }, { key: "API_ACCESS_TOKEN", value: "redacted-worker-token", is_preview: false }, ], }); try { const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(result.status).toBe(1); expect(result.stderr).toContain("duplicate non-preview keys"); expect(result.stdout).not.toContain("first-redacted"); expect(result.stderr).not.toContain("first-redacted"); expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("fails before mutation when a marked partial worker has conflicting managed environment values", async () => { const server = await startProvisioningRetryServer({ preexistingPartialEnv: { DATABASE_URL: WORKER_DATABASE_URL, BULLMQ_REDIS_URL: "redis://redacted", WORKER_ENABLED: "false", }, }); try { const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(result.status).toBe(1); expect(result.stderr).toContain("partial environment is unsafe: WORKER_ENABLED"); expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("reconciles only the target environment when a complete foreign managed fleet shares names and images", async () => { const server = await startProvisioningRetryServer({ preexistingPartialEnv: desiredWorkerEnvWithRuntime(group("general"), { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token", }), preexistingPartialAppOverrides: { docker_registry_image_tag: "staging", health_check_enabled: false, health_check_type: "http", health_check_command: "", }, allTargetWorkersManaged: true, includeForeignFleet: true, }); try { const result = await runProvisioner(server.url, [...provisioningResourceUuids(), RETRY_GENERAL_UUID], provisioningEnvOverrides()); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toEqual([]); expect(server.requests.filter((request) => request.method === "PATCH" && request.url === `/applications/${RETRY_GENERAL_UUID}`)).toHaveLength(1); expect(server.apps.get(RETRY_GENERAL_UUID)).toMatchObject({ docker_registry_image_tag: DEPLOY_REVISION, health_check_type: "cmd", health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`, }); const foreignUuids = new Set(Object.values(FOREIGN_WORKER_UUIDS)); expect(server.requests.filter((request) => [...foreignUuids].some((uuid) => String(request.url || "").includes(uuid)))).toEqual([]); } finally { await server.close(); } }); it("reconciles desired health config drift without weakening managed retry identity", async () => { const previousRevision = "b".repeat(40); const server = await startProvisioningRetryServer({ preexistingPartialEnv: desiredWorkerEnvWithRuntime(group("general"), { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token", }), preexistingPartialAppOverrides: { docker_registry_image_tag: previousRevision, health_check_command: `node apps/worker/dist/deploy-health.js ${previousRevision} general`, health_check_interval: 5, }, }); try { const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toEqual([]); expect(server.requests.filter((request) => request.method === "PATCH" && request.url === `/applications/${RETRY_GENERAL_UUID}`)).toHaveLength(1); expect(server.apps.get(RETRY_GENERAL_UUID)).toMatchObject({ docker_registry_image_tag: DEPLOY_REVISION, health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`, health_check_interval: 30, }); } finally { await server.close(); } }); it("reconciles legacy health config on an explicitly allowlisted managed worker", async () => { const server = await startProvisioningRetryServer({ preexistingPartialEnv: desiredWorkerEnvWithRuntime(group("general"), { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token", }), preexistingPartialAppOverrides: { docker_registry_image_tag: "staging", health_check_enabled: false, health_check_type: "http", health_check_command: "", }, }); try { const result = await runProvisioner(server.url, [...provisioningResourceUuids(), RETRY_GENERAL_UUID], provisioningEnvOverrides()); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toEqual([]); expect(server.requests.filter((request) => request.method === "PATCH" && request.url === `/applications/${RETRY_GENERAL_UUID}`)).toHaveLength(1); expect(server.apps.get(RETRY_GENERAL_UUID)).toMatchObject({ docker_registry_image_tag: DEPLOY_REVISION, health_check_enabled: true, health_check_type: "cmd", health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`, }); } finally { await server.close(); } }); it("rejects an explicit worker env source with the wrong image before mutation", async () => { const server = await startProvisioningRetryServer(); server.apps.get(WORKER_ENV_SOURCE_UUID).docker_registry_image_name = "registry.example.test/toolctrl-labs/hivegrid/web"; try { const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides()); expect(result.status).toBe(1); expect(result.stderr).toContain("must reference the exact worker image"); expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("rejects an explicit worker env source outside a non-empty allowlist before mutation", async () => { const server = await startProvisioningRetryServer(); const resourceUuids = provisioningResourceUuids().filter((uuid) => uuid !== WORKER_ENV_SOURCE_UUID); try { const result = await runProvisioner(server.url, resourceUuids, provisioningEnvOverrides()); expect(result.status).toBe(1); expect(result.stderr).toContain("must be included in COOLIFY_STAGING_RESOURCE_UUIDS"); expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]); } finally { await server.close(); } }); it("ignores an unused legacy env source when all worker groups already exist", async () => { const server = await startProvisionerServer(); const source = { uuid: WORKER_ENV_SOURCE_UUID, name: "hivegrid-web", docker_registry_image_name: "registry.example.test/toolctrl-labs/hivegrid/web", }; server.apps.set(source.uuid, source); const resourceUuids = [...server.apps.values()].map((app) => app.uuid); try { const result = await runProvisioner(server.url, resourceUuids, { COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID: source.uuid, }); expect(result.status).toBe(0); expect(result.stderr).toBe(""); expect(server.requests.filter((request) => String(request.url || "").includes(source.uuid))).toEqual([]); } finally { await server.close(); } }); it("provisions worker apps with revision-bound command health checks", () => { expect( buildWorkerApplicationBody({ group: group("mail"), projectUuid: "project_1", serverUuid: "server_1", destinationUuid: "destination_1", environmentName: "production", environmentUuid: "environment_1", workerImage: "registry.example.test/hivegrid/worker", imageTag: DEPLOY_REVISION, }) ).toMatchObject({ project_uuid: "project_1", server_uuid: "server_1", destination_uuid: "destination_1", environment_name: "production", environment_uuid: "environment_1", docker_registry_image_name: "registry.example.test/hivegrid/worker", docker_registry_image_tag: DEPLOY_REVISION, name: "worker-mail", ports_exposes: "9102", health_check_enabled: true, health_check_type: "cmd", health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} mail`, connect_to_docker_network: true, }); }); it("rejects mutable or malformed worker deployment health tags", () => { expect(() => buildWorkerDeploymentHealthCommand("staging", "general")).toThrow(/full lowercase Git SHA/); expect(() => buildWorkerDeploymentHealthCommand(DEPLOY_REVISION, "unknown")).toThrow(/valid worker group/); expect(buildWorkerDeploymentHealthCommand(DEPLOY_REVISION, "general")).toBe(`node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`); }); it("builds the expected mail worker environment", () => { expect(desiredWorkerEnv(group("mail"))).toEqual({ DATABASE_RUNTIME_ROLE: "hivegrid_worker", RLS_RUNTIME_MODE: "required", WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "mail", WORKER_HEALTH_PORT: "9102", WORKER_ADMIN_PORT: "9202", WORKER_CONCURRENCY_MAIL: "5", MARKETING_SEQUENCES_ENABLED: "false", TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true", }); }); it("builds the expected general worker production outbox environment", () => { expect(desiredWorkerEnv(group("general"))).toMatchObject({ DATABASE_RUNTIME_ROLE: "hivegrid_worker", RLS_RUNTIME_MODE: "required", WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "general", WORKER_HEALTH_PORT: "9101", WORKER_ADMIN_PORT: "9201", WORKER_CONCURRENCY_GENERAL: "10", OUTBOX_DISPATCHER_ENABLED: "true", OUTBOX_DISPATCHER_REQUIRED: "true", WORKFLOW_TRIGGER_EXECUTOR_MODE: "http", WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true", WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY: "5", WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true", }); }); it("protects every managed worker env key from baseline-env cloning", () => { expect(WORKER_ENV_OVERRIDE_KEYS.has("APP_REVISION")).toBe(true); for (const item of GROUPS) { for (const key of Object.keys(desiredWorkerEnv(item))) { expect(WORKER_ENV_OVERRIDE_KEYS.has(key)).toBe(true); } } }); it("classifies migration and seed credentials as forbidden worker runtime env", () => { expect([...WORKER_PRIVILEGED_ENV_KEYS].sort()).toEqual(["DIRECT_URL", "MIGRATE_ON_BOOT", "MIGRATOR_DATABASE_URL", "SEED_ONCE", "SEED_ON_BOOT"]); }); it("normalizes quoted Coolify env values", () => { expect( envEntriesToRecord([ { key: "WORKER_ENABLED", value: '"true"' }, { key: "WORKER_QUEUE_GROUP", value: "'mail'" }, { key: "APP_REVISION", real_value: "'configured-revision'" }, ]) ).toEqual({ APP_REVISION: "configured-revision", WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "mail", }); }); it.each([ [ { key: "MAILJET_API_KEY", value: "", is_preview: true }, { key: "MAILJET_API_KEY", value: "runtime-key", is_preview: false }, ], [ { key: "MAILJET_API_KEY", value: "runtime-key", is_preview: false }, { key: "MAILJET_API_KEY", value: "", is_preview: true }, ], ])("uses the non-preview target value regardless of duplicate API ordering", (...entries) => { expect(envEntriesToRecord(entries)).toEqual({ MAILJET_API_KEY: "runtime-key" }); }); it("rejects APP_REVISION overrides that could spoof stale worker images", () => { const mail = group("mail"); const entries = desiredWorkerEnvWithRuntime(mail, { APP_REVISION: "" }); expect(getWorkerEnvDrift({ uuid: "worker-mail", name: "worker-mail" }, mail, entriesFromRecord(entries))).toContainEqual( expect.objectContaining({ key: "APP_REVISION", expected: "unset; image-baked revision is authoritative", actual: "", group: "mail", }) ); }); it("returns no drift when an existing worker app matches the desired env", () => { const mail = group("mail"); const drift = getWorkerEnvDrift({ uuid: "worker-mail", name: "worker-mail" }, mail, entriesFromRecord(desiredWorkerEnvWithRuntime(mail))); expect(drift).toEqual([]); }); it("returns no app config drift for the revision-bound command healthcheck", () => { const mail = group("mail"); expect( getWorkerAppConfigDrift( { uuid: "worker-mail", name: "worker-mail", docker_registry_image_tag: DEPLOY_REVISION, ports_exposes: "9102", health_check_enabled: true, health_check_type: "cmd", health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} mail`, health_check_interval: 30, health_check_timeout: 10, health_check_retries: 3, health_check_start_period: 20, custom_healthcheck_found: false, }, mail, DEPLOY_REVISION ) ).toEqual([]); }); it("reports existing worker app health-check drift", () => { const mail = group("mail"); expect( getWorkerAppConfigDrift( { uuid: "worker-mail", name: "worker-mail", docker_registry_image_tag: "previous-revision", ports_exposes: "9101", health_check_enabled: false, health_check_type: "http", health_check_command: null, health_check_interval: 10, health_check_timeout: 5, health_check_retries: 2, health_check_start_period: 5, custom_healthcheck_found: true, }, mail, DEPLOY_REVISION ) ).toEqual([ expect.objectContaining({ key: "docker_registry_image_tag", expected: DEPLOY_REVISION, actual: "previous-revision", group: "mail", }), expect.objectContaining({ key: "ports_exposes", expected: "9102", actual: "9101", group: "mail" }), expect.objectContaining({ key: "health_check_enabled", expected: "true", actual: "false", group: "mail" }), expect.objectContaining({ key: "health_check_type", expected: "cmd", actual: "http", group: "mail" }), expect.objectContaining({ key: "health_check_command", expected: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} mail`, actual: "", group: "mail", }), expect.objectContaining({ key: "health_check_interval", expected: 30, actual: 10, group: "mail" }), expect.objectContaining({ key: "health_check_timeout", expected: 10, actual: 5, group: "mail" }), expect.objectContaining({ key: "health_check_retries", expected: 3, actual: 2, group: "mail" }), expect.objectContaining({ key: "health_check_start_period", expected: 20, actual: 5, group: "mail" }), expect.objectContaining({ key: "custom_healthcheck_found", expected: "false", actual: "true", group: "mail" }), ]); }); it("accepts a fully configured general worker HTTP bridge", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("accepts WEB_ORIGIN as the general worker HTTP bridge fallback", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WEB_ORIGIN: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("fails closed when a general worker lacks required Lago or Mollie runtime configuration", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", LAGO_API_KEY: "", MOLLIE_PROFILE_ID: "", MOLLIE_TIMEOUT_MS: "0", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual( expect.arrayContaining([ expect.objectContaining({ key: "LAGO_API_KEY", actual: "" }), expect.objectContaining({ key: "MOLLIE_TIMEOUT_MS", actual: "" }), ]) ); expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).not.toEqual( expect.arrayContaining([expect.objectContaining({ key: "MOLLIE_PROFILE_ID" })]) ); }); it("allows an explicitly disabled provider subscriber without Lago or Mollie runtime configuration", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED: "false", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("rejects ambiguous provider subscriber gate values", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), ...productAcquisitionProviderEnv(), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED: "disabled", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual( expect.arrayContaining([ expect.objectContaining({ key: "PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED", actual: "", }), ]) ); }); it("does not require plugin hook fanout before the production plugin runtime gate flips", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", PLUGIN_RUNTIME_PRODUCTION_ENABLED: "false", PLUGIN_HOOK_FANOUT_ENABLED: "false", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("requires plugin hook fanout fail-closed when production plugin runtime is enabled", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true", PLUGIN_HOOK_FANOUT_ENABLED: "false", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual( expect.arrayContaining([ expect.objectContaining({ key: "PLUGIN_HOOK_FANOUT_ENABLED", expected: "unset or true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true", actual: "false", group: "general", }), expect.objectContaining({ key: "PLUGIN_HOOK_FANOUT_REQUIRED", expected: "true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true", actual: "", group: "general", }), ]) ); }); it("accepts production plugin runtime when general worker hook fanout is required", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true", PLUGIN_HOOK_FANOUT_REQUIRED: "true", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("accepts native workflow execution as an explicit general worker opt-in", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_MODE: "native", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("normalizes native workflow execution mode like the worker runtime", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_MODE: " NATIVE ", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("accepts hybrid workflow execution when the HTTP fallback is configured", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_MODE: "hybrid", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]); }); it("normalizes hybrid workflow execution mode before checking HTTP fallback dependencies", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_MODE: " HYBRID ", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual( expect.arrayContaining([ expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN", actual: "", group: "general", }), expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "", group: "general", }), ]) ); }); it("requires the HTTP fallback dependencies for hybrid workflow execution", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_MODE: "hybrid", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN", actual: "", group: "general", }), expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "", group: "general", }), ]); }); it("rejects legacy-only workflow executor token configuration", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", WORKFLOW_TRIGGER_EXECUTOR_TOKEN: "legacy-token", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "", group: "general", }), ]); }); it("rejects legacy workflow executor token even when API_ACCESS_TOKEN is configured", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", WORKFLOW_TRIGGER_EXECUTOR_TOKEN: "legacy-token", }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_TOKEN", expected: "unset; API_ACCESS_TOKEN is the supported workflow bridge credential", actual: "", group: "general", }), ]); }); it("reports disabled existing worker apps", () => { const mail = group("mail"); const entries = { ...desiredWorkerEnvWithRuntime(mail), WORKER_ENABLED: "false" }; expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "WORKER_ENABLED", expected: "true", actual: "false", group: "mail" }), ]); }); it("reports wrong queue groups", () => { const mail = group("mail"); const entries = { ...desiredWorkerEnvWithRuntime(mail), WORKER_QUEUE_GROUP: "general" }; expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "WORKER_QUEUE_GROUP", expected: "mail", actual: "general" }), ]); }); it("reports mail workers without the marketing sequence killswitch", () => { const mail = group("mail"); const entries = { ...desiredWorkerEnvWithRuntime(mail) }; delete entries.MARKETING_SEQUENCES_ENABLED; expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "MARKETING_SEQUENCES_ENABLED", expected: "false", actual: "", group: "mail" }), ]); }); it("reports mail workers without fail-loud transactional outbox bootstrap", () => { const mail = group("mail"); const entries = { ...desiredWorkerEnvWithRuntime(mail) }; delete entries.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED; expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "TRANSACTIONAL_MAIL_OUTBOX_REQUIRED", expected: "true", actual: "", group: "mail" }), ]); }); it("reports mail workers without the shared tenant-envelope key without exposing its value", () => { const mail = group("mail"); const entries = { ...desiredWorkerEnvWithRuntime(mail) }; delete entries.CRYPTO_SECRET; const drift = getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries)); expect(drift).toEqual([ expect.objectContaining({ key: "CRYPTO_SECRET", expected: "configured shared tenant-envelope key matching hivegrid-web", actual: "", group: "mail", }), ]); expect(JSON.stringify(drift)).not.toContain(SHARED_CRYPTO_SECRET); }); it("requires DATABASE_URL and BULLMQ_REDIS_URL on every managed worker group", () => { for (const workerGroup of GROUPS) { const missingDatabase = desiredWorkerEnvWithRuntime(workerGroup); delete missingDatabase.DATABASE_URL; const missingRedis = desiredWorkerEnvWithRuntime(workerGroup); delete missingRedis.BULLMQ_REDIS_URL; missingRedis.REDIS_URL = "redis://fallback-not-accepted-for-coolify-workers"; expect(getWorkerEnvDrift({ uuid: `worker-${workerGroup.name}` }, workerGroup, entriesFromRecord(missingDatabase))).toEqual( expect.arrayContaining([ expect.objectContaining({ key: "DATABASE_URL", expected: "configured DATABASE_URL", actual: "", group: workerGroup.name }), ]) ); expect(getWorkerEnvDrift({ uuid: `worker-${workerGroup.name}` }, workerGroup, entriesFromRecord(missingRedis))).toEqual( expect.arrayContaining([ expect.objectContaining({ key: "BULLMQ_REDIS_URL", expected: "configured BULLMQ_REDIS_URL", actual: "", group: workerGroup.name, }), ]) ); } }); it("rejects worker DSNs authenticated as an application or owner role without leaking the URL", () => { const workerGroup = group("mail"); for (const databaseUrl of [ "postgres://hivegrid_app:secret-sentinel@db.example.test/hivegrid", "postgres://postgres:secret-sentinel@db.example.test/hivegrid", "not-a-postgres-url", ]) { const entries = desiredWorkerEnvWithRuntime(workerGroup, { DATABASE_URL: databaseUrl }); const drift = getWorkerEnvDrift({ uuid: "worker-mail" }, workerGroup, entriesFromRecord(entries)); expect(drift).toContainEqual( expect.objectContaining({ key: "DATABASE_URL", expected: "PostgreSQL DSN authenticated as hivegrid_worker", }) ); expect(JSON.stringify(drift)).not.toContain("secret-sentinel"); expect(JSON.stringify(drift)).not.toContain(databaseUrl); } }); it("rejects migration or seed credentials already present on a worker", () => { const workerGroup = group("mail"); const entries = desiredWorkerEnvWithRuntime(workerGroup, { DIRECT_URL: "postgres://owner:secret-sentinel@db.example.test/hivegrid", MIGRATOR_DATABASE_URL: "postgres://migrator:secret-sentinel@db.example.test/hivegrid", SEED_ADMIN_PASSWORD: "seed-secret-sentinel", }); const drift = getWorkerEnvDrift({ uuid: "worker-mail" }, workerGroup, entriesFromRecord(entries)); expect(drift.map(({ key }) => key)).toEqual(expect.arrayContaining(["DIRECT_URL", "MIGRATOR_DATABASE_URL", "SEED_ADMIN_PASSWORD"])); expect(JSON.stringify(drift)).not.toContain("secret-sentinel"); }); it("reports missing required general-worker outbox activation", () => { const general = group("general"); const entries = { ...desiredWorkerEnvWithRuntime(general), BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", OUTBOX_DISPATCHER_REQUIRED: "false", }; delete entries.WORKFLOW_TRIGGER_EXECUTOR_MODE; entries.WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY = "1"; delete entries.WORKFLOW_TRIGGER_OUTBOX_REQUIRED; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "OUTBOX_DISPATCHER_REQUIRED", expected: "true", actual: "false", group: "general" }), expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_MODE", expected: "http|hybrid|native", actual: "", group: "general" }), expect.objectContaining({ key: "WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY", expected: "5", actual: "1", group: "general" }), expect.objectContaining({ key: "WORKFLOW_TRIGGER_OUTBOX_REQUIRED", expected: "true", actual: "", group: "general" }), ]); }); it("reports missing general worker Redis and HTTP bridge dependencies", () => { const general = group("general"); const entries = { ...desiredWorkerEnv(general) }; expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual( expect.arrayContaining([ expect.objectContaining({ key: "DATABASE_URL", expected: "configured DATABASE_URL", actual: "", group: "general" }), expect.objectContaining({ key: "BULLMQ_REDIS_URL", expected: "configured BULLMQ_REDIS_URL", actual: "", group: "general" }), expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN", actual: "", group: "general", }), expect.objectContaining({ key: "API_ACCESS_TOKEN", expected: "configured API_ACCESS_TOKEN matching the web runtime", actual: "", group: "general", }), ]) ); }); it("reports swapped or missing worker ports", () => { const plugin = group("plugin"); const entries = { ...desiredWorkerEnvWithRuntime(plugin), WORKER_HEALTH_PORT: "9205" }; delete entries.WORKER_ADMIN_PORT; expect(getWorkerEnvDrift({ uuid: "worker-plugin" }, plugin, entriesFromRecord(entries))).toEqual([ expect.objectContaining({ key: "WORKER_HEALTH_PORT", expected: "9105", actual: "9205" }), expect.objectContaining({ key: "WORKER_ADMIN_PORT", expected: "9205", actual: "" }), ]); }); it("reports missing or wrong group concurrency", () => { const webhook = group("webhook"); const wrong = { ...desiredWorkerEnvWithRuntime(webhook), WORKER_CONCURRENCY_WEBHOOK: "5" }; const missing = { ...desiredWorkerEnvWithRuntime(webhook) }; delete missing.WORKER_CONCURRENCY_WEBHOOK; expect(getWorkerEnvDrift({ uuid: "worker-webhook" }, webhook, entriesFromRecord(wrong))).toEqual([ expect.objectContaining({ key: "WORKER_CONCURRENCY_WEBHOOK", expected: "20", actual: "5" }), ]); expect(getWorkerEnvDrift({ uuid: "worker-webhook" }, webhook, entriesFromRecord(missing))).toEqual([ expect.objectContaining({ key: "WORKER_CONCURRENCY_WEBHOOK", expected: "20", actual: "" }), ]); }); });