1735 lines
69 KiB
JavaScript
1735 lines
69 KiB
JavaScript
import { spawn } from "node:child_process";
|
|
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
import http from "node:http";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { describe, expect, it } from "vitest";
|
|
import {
|
|
GROUPS,
|
|
KWP_WORKER_ENV_KEY,
|
|
WORKER_ENV_OVERRIDE_KEYS,
|
|
WORKER_GENERAL_PROVIDER_ENV_KEYS,
|
|
WORKER_PRIVILEGED_ENV_KEYS,
|
|
buildWorkerDeploymentHealthCommand,
|
|
buildWorkerApplicationBody,
|
|
desiredWorkerEnv,
|
|
envEntriesToRecord,
|
|
getWorkerAppConfigDrift,
|
|
getWorkerEnvDrift,
|
|
} from "./coolify-ensure-worker-groups.mjs";
|
|
|
|
const DEPLOY_REVISION = "a".repeat(40);
|
|
const PROVISIONER_PATH = path.resolve("scripts/coolify-ensure-worker-groups.mjs");
|
|
const PROVISIONER_TOKEN = "coolify-worker-provisioner-token-sentinel";
|
|
const WORKER_IMAGE = "registry.example.test/toolctrl-labs/hivegrid/worker";
|
|
const WORKER_DATABASE_URL = "postgres://hivegrid_worker:redacted@db.example.test/hivegrid";
|
|
const SHARED_CRYPTO_SECRET = "shared-tenant-envelope-secret-sentinel";
|
|
const WORKER_UUIDS = Object.fromEntries(
|
|
GROUPS.map((workerGroup, index) => [workerGroup.name, `${String.fromCharCode(97 + index)}${String(index).repeat(23)}`])
|
|
);
|
|
const WORKER_ENV_SOURCE_UUID = `s${"8".repeat(23)}`;
|
|
const RETRY_GENERAL_UUID = `r${"9".repeat(23)}`;
|
|
const PROJECT_UUID = `p${"1".repeat(23)}`;
|
|
const SERVER_UUID = `v${"2".repeat(23)}`;
|
|
const DESTINATION_UUID = `d${"3".repeat(23)}`;
|
|
const ENVIRONMENT_UUID = `n${"4".repeat(23)}`;
|
|
const ENVIRONMENT_ID = 41;
|
|
const FOREIGN_SERVER_UUID = `q${"5".repeat(23)}`;
|
|
const FOREIGN_DESTINATION_UUID = `x${"6".repeat(23)}`;
|
|
const FOREIGN_ENVIRONMENT_ID = 84;
|
|
const FOREIGN_WORKER_UUIDS = Object.fromEntries(
|
|
GROUPS.map((workerGroup, index) => [workerGroup.name, `${String.fromCharCode(107 + index)}${String(index + 5).repeat(23)}`])
|
|
);
|
|
|
|
function group(name) {
|
|
const found = GROUPS.find((item) => item.name === name);
|
|
if (!found) {
|
|
throw new Error(`Missing test group ${name}`);
|
|
}
|
|
return found;
|
|
}
|
|
|
|
function entriesFromRecord(record) {
|
|
return Object.entries(record).map(([key, value]) => ({ key, value }));
|
|
}
|
|
|
|
function productAcquisitionProviderEnv() {
|
|
return {
|
|
PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED: "true",
|
|
LAGO_API_URL: "http://lago.internal.test:3108",
|
|
LAGO_API_KEY: "redacted-lago-key",
|
|
LAGO_TIMEOUT_MS: "10000",
|
|
MOLLIE_API_KEY: "redacted-mollie-key",
|
|
MOLLIE_PROFILE_ID: "redacted-mollie-profile",
|
|
MOLLIE_TIMEOUT_MS: "10000",
|
|
BILLING_MOLLIE_WEBHOOK_URL: "https://staging-api.hivegrid.de/v1/billing/mollie/webhooks",
|
|
};
|
|
}
|
|
|
|
const KWP_PILOT_TENANT = "kwp-pilot-tenant-sentinel";
|
|
|
|
function desiredWorkerEnvWithRuntime(workerGroup, overrides = {}) {
|
|
return {
|
|
...desiredWorkerEnv(workerGroup),
|
|
DATABASE_URL: WORKER_DATABASE_URL,
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
...(workerGroup.name === "general" ? productAcquisitionProviderEnv() : {}),
|
|
...(workerGroup.name === "mail" ? { CRYPTO_SECRET: SHARED_CRYPTO_SECRET } : {}),
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function provisioningResourceUuids(missingGroup = "general") {
|
|
return [WORKER_ENV_SOURCE_UUID, ...GROUPS.filter((workerGroup) => workerGroup.name !== missingGroup).map((workerGroup) => WORKER_UUIDS[workerGroup.name])];
|
|
}
|
|
|
|
function provisioningEnvOverrides() {
|
|
return {
|
|
COOLIFY_STAGING_PROVISION_WORKER_GROUPS: "true",
|
|
COOLIFY_STAGING_PROJECT_UUID: PROJECT_UUID,
|
|
COOLIFY_STAGING_SERVER_UUID: SERVER_UUID,
|
|
COOLIFY_STAGING_DESTINATION_UUID: DESTINATION_UUID,
|
|
COOLIFY_STAGING_ENVIRONMENT_UUID: ENVIRONMENT_UUID,
|
|
COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID: WORKER_ENV_SOURCE_UUID,
|
|
};
|
|
}
|
|
|
|
function runProvisioner(serverUrl, resourceUuids, envOverrides = {}) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(process.execPath, [PROVISIONER_PATH, "staging"], {
|
|
env: {
|
|
PATH: process.env.PATH || "",
|
|
NODE_ENV: "test",
|
|
SELECTED_COOLIFY_API_URL: serverUrl,
|
|
SELECTED_COOLIFY_TOKEN: PROVISIONER_TOKEN,
|
|
IMAGE_TAG: DEPLOY_REVISION,
|
|
WORKER_IMAGE,
|
|
COOLIFY_STAGING_RESOURCE_UUIDS: resourceUuids.join(","),
|
|
...envOverrides,
|
|
},
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
let stdout = "";
|
|
let stderr = "";
|
|
child.stdout.setEncoding("utf8");
|
|
child.stderr.setEncoding("utf8");
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += chunk;
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += chunk;
|
|
});
|
|
child.on("close", (status) => {
|
|
resolve({ status, stdout, stderr });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function requestBody(request) {
|
|
let body = "";
|
|
request.setEncoding("utf8");
|
|
for await (const chunk of request) {
|
|
body += chunk;
|
|
}
|
|
return body;
|
|
}
|
|
|
|
function startProvisionerServer({
|
|
appRevisionOverride = false,
|
|
customHealthcheckFound = false,
|
|
invalidEnvGroup = "",
|
|
malformedEnvGroup = "",
|
|
persistPatch = true,
|
|
} = {}) {
|
|
const apps = new Map(
|
|
GROUPS.map((workerGroup) => [
|
|
`worker-${workerGroup.name}`,
|
|
{
|
|
uuid: WORKER_UUIDS[workerGroup.name],
|
|
name: `worker-${workerGroup.name}`,
|
|
description: `WORKER_QUEUE_GROUP=${workerGroup.name}`,
|
|
docker_registry_image_name: WORKER_IMAGE,
|
|
docker_registry_image_tag: "previous-revision",
|
|
ports_exposes: workerGroup.healthPort,
|
|
health_check_enabled: false,
|
|
health_check_type: "http",
|
|
health_check_path: "/readyz",
|
|
custom_healthcheck_found: customHealthcheckFound,
|
|
},
|
|
])
|
|
);
|
|
const requests = [];
|
|
const server = http.createServer(async (request, response) => {
|
|
const body = await requestBody(request);
|
|
requests.push({ method: request.method, url: request.url, authorization: request.headers.authorization, body });
|
|
const segments = String(request.url || "")
|
|
.split("/")
|
|
.filter(Boolean);
|
|
let status = 200;
|
|
let payload = {};
|
|
|
|
if (request.method === "GET" && request.url === "/applications") {
|
|
payload = [...apps.values()];
|
|
} else if (segments[0] === "applications" && segments[2] === "envs" && request.method === "GET") {
|
|
const requestedUuid = decodeURIComponent(segments[1] || "");
|
|
const app = [...apps.values()].find((candidate) => candidate.uuid === requestedUuid);
|
|
const workerGroup = GROUPS.find((candidate) => app?.name === `worker-${candidate.name}`);
|
|
if (!workerGroup) {
|
|
status = 404;
|
|
} else if (workerGroup.name === malformedEnvGroup) {
|
|
payload = { unexpected: [] };
|
|
} else {
|
|
payload = entriesFromRecord(
|
|
desiredWorkerEnvWithRuntime(workerGroup, {
|
|
...(appRevisionOverride || workerGroup.name === invalidEnvGroup ? { APP_REVISION: "" } : {}),
|
|
...(workerGroup.name === "general" ? { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token" } : {}),
|
|
})
|
|
);
|
|
}
|
|
} else if (segments[0] === "applications" && segments.length === 2 && request.method === "GET") {
|
|
const requestedUuid = decodeURIComponent(segments[1] || "");
|
|
payload = [...apps.values()].find((candidate) => candidate.uuid === requestedUuid) || {};
|
|
} else if (segments[0] === "applications" && segments.length === 2 && request.method === "PATCH") {
|
|
const uuid = decodeURIComponent(segments[1] || "");
|
|
const app = [...apps.values()].find((candidate) => candidate.uuid === uuid);
|
|
if (!app) {
|
|
status = 404;
|
|
} else if (persistPatch) {
|
|
Object.assign(app, JSON.parse(body));
|
|
payload = app;
|
|
}
|
|
} else {
|
|
status = 404;
|
|
}
|
|
|
|
response.writeHead(status, { "content-type": "application/json" });
|
|
response.end(JSON.stringify(payload));
|
|
});
|
|
|
|
return new Promise((resolve, reject) => {
|
|
server.once("error", reject);
|
|
server.listen(0, "127.0.0.1", () => {
|
|
resolve({
|
|
url: `http://127.0.0.1:${server.address().port}`,
|
|
apps,
|
|
requests,
|
|
close: () => new Promise((closeResolve, closeReject) => server.close((error) => (error ? closeReject(error) : closeResolve()))),
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
function buildProvisionedWorkerApp(workerGroup, uuid, { managed = false } = {}) {
|
|
const body = buildWorkerApplicationBody({
|
|
group: workerGroup,
|
|
projectUuid: PROJECT_UUID,
|
|
serverUuid: SERVER_UUID,
|
|
destinationUuid: DESTINATION_UUID,
|
|
environmentName: "staging",
|
|
environmentUuid: ENVIRONMENT_UUID,
|
|
workerImage: WORKER_IMAGE,
|
|
imageTag: DEPLOY_REVISION,
|
|
});
|
|
return {
|
|
...body,
|
|
uuid,
|
|
description: managed ? body.description : `WORKER_QUEUE_GROUP=${workerGroup.name}`,
|
|
custom_healthcheck_found: false,
|
|
environment_id: ENVIRONMENT_ID,
|
|
destination: {
|
|
uuid: DESTINATION_UUID,
|
|
server: { uuid: SERVER_UUID },
|
|
},
|
|
};
|
|
}
|
|
|
|
function startProvisioningRetryServer({
|
|
preexistingPartialEnv = null,
|
|
preexistingPartialAppOverrides = {},
|
|
sourceEnvEntries = null,
|
|
failEnvKeyOnce = "",
|
|
allTargetWorkersManaged = false,
|
|
includeForeignFleet = false,
|
|
missingGroup = "general",
|
|
targetEnvOverrides = {},
|
|
} = {}) {
|
|
let envUuidSequence = 0;
|
|
const attachEnvUuids = (entries) =>
|
|
entries.map((entry) => ({
|
|
...entry,
|
|
uuid: typeof entry?.uuid === "string" && entry.uuid ? entry.uuid : `env-${++envUuidSequence}`,
|
|
}));
|
|
const sourceApp = {
|
|
uuid: WORKER_ENV_SOURCE_UUID,
|
|
name: "hivegrid-worker",
|
|
description: "Managed worker environment source",
|
|
docker_registry_image_name: WORKER_IMAGE,
|
|
};
|
|
const apps = new Map([[sourceApp.uuid, sourceApp]]);
|
|
const envs = new Map([
|
|
[
|
|
sourceApp.uuid,
|
|
attachEnvUuids(
|
|
sourceEnvEntries ||
|
|
entriesFromRecord({
|
|
DATABASE_URL: WORKER_DATABASE_URL,
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
...productAcquisitionProviderEnv(),
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-worker-token",
|
|
CRYPTO_SECRET: SHARED_CRYPTO_SECRET,
|
|
})
|
|
),
|
|
],
|
|
]);
|
|
for (const workerGroup of GROUPS.filter((candidate) => candidate.name !== missingGroup)) {
|
|
const app = buildProvisionedWorkerApp(workerGroup, WORKER_UUIDS[workerGroup.name], { managed: allTargetWorkersManaged });
|
|
apps.set(app.uuid, app);
|
|
envs.set(
|
|
app.uuid,
|
|
attachEnvUuids(
|
|
entriesFromRecord(
|
|
desiredWorkerEnvWithRuntime(workerGroup, {
|
|
...(workerGroup.name === "general" ? { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-worker-token" } : {}),
|
|
...(targetEnvOverrides[workerGroup.name] || {}),
|
|
})
|
|
)
|
|
)
|
|
);
|
|
}
|
|
if (preexistingPartialEnv !== null) {
|
|
if (missingGroup !== "general") {
|
|
throw new Error("preexistingPartialEnv is only supported for the general worker test fixture");
|
|
}
|
|
const partial = {
|
|
...buildProvisionedWorkerApp(group("general"), RETRY_GENERAL_UUID, { managed: true }),
|
|
...preexistingPartialAppOverrides,
|
|
};
|
|
apps.set(partial.uuid, partial);
|
|
envs.set(partial.uuid, attachEnvUuids(entriesFromRecord(preexistingPartialEnv)));
|
|
}
|
|
if (includeForeignFleet) {
|
|
for (const workerGroup of GROUPS) {
|
|
const foreign = {
|
|
...buildProvisionedWorkerApp(workerGroup, FOREIGN_WORKER_UUIDS[workerGroup.name], { managed: true }),
|
|
environment_id: FOREIGN_ENVIRONMENT_ID,
|
|
destination: {
|
|
uuid: FOREIGN_DESTINATION_UUID,
|
|
server: { uuid: FOREIGN_SERVER_UUID },
|
|
},
|
|
};
|
|
apps.set(foreign.uuid, foreign);
|
|
envs.set(
|
|
foreign.uuid,
|
|
attachEnvUuids(
|
|
entriesFromRecord(
|
|
desiredWorkerEnvWithRuntime(workerGroup, {
|
|
...(workerGroup.name === "general" ? { WORKFLOW_TRIGGER_EXECUTOR_URL: "https://foreign.example.test", API_ACCESS_TOKEN: "foreign-token" } : {}),
|
|
})
|
|
)
|
|
)
|
|
);
|
|
}
|
|
}
|
|
|
|
const requests = [];
|
|
let pendingFailureKey = failEnvKeyOnce;
|
|
const server = http.createServer(async (request, response) => {
|
|
const body = await requestBody(request);
|
|
requests.push({ method: request.method, url: request.url, authorization: request.headers.authorization, body });
|
|
const segments = String(request.url || "")
|
|
.split("/")
|
|
.filter(Boolean);
|
|
let status = 200;
|
|
let payload = {};
|
|
|
|
if (request.method === "GET" && request.url === `/projects/${PROJECT_UUID}`) {
|
|
payload = {
|
|
uuid: PROJECT_UUID,
|
|
environments: [{ id: ENVIRONMENT_ID, uuid: ENVIRONMENT_UUID, name: "staging" }],
|
|
};
|
|
} else if (request.method === "GET" && request.url === "/applications") {
|
|
payload = [...apps.values()].map(({ destination: _destination, ...app }) => app);
|
|
} else if (request.method === "POST" && request.url === "/applications/dockerimage") {
|
|
const input = JSON.parse(body);
|
|
const createdGroup = GROUPS.find((candidate) => input.name === `worker-${candidate.name}`);
|
|
if (!createdGroup) {
|
|
status = 400;
|
|
payload = { message: "unknown worker group" };
|
|
response.writeHead(status, { "content-type": "application/json" });
|
|
response.end(JSON.stringify(payload));
|
|
return;
|
|
}
|
|
const app = {
|
|
...input,
|
|
uuid: createdGroup.name === "general" ? RETRY_GENERAL_UUID : WORKER_UUIDS[createdGroup.name],
|
|
custom_healthcheck_found: false,
|
|
environment_id: ENVIRONMENT_ID,
|
|
destination: {
|
|
uuid: DESTINATION_UUID,
|
|
server: { uuid: SERVER_UUID },
|
|
},
|
|
};
|
|
apps.set(app.uuid, app);
|
|
envs.set(app.uuid, []);
|
|
payload = { uuid: app.uuid };
|
|
} else if (segments[0] === "applications" && segments[2] === "envs" && request.method === "GET") {
|
|
const requestedUuid = decodeURIComponent(segments[1] || "");
|
|
if (!envs.has(requestedUuid)) {
|
|
status = 404;
|
|
} else {
|
|
payload = envs.get(requestedUuid);
|
|
}
|
|
} else if (segments[0] === "applications" && segments[2] === "envs" && ["POST", "PATCH"].includes(request.method)) {
|
|
const requestedUuid = decodeURIComponent(segments[1] || "");
|
|
const input = JSON.parse(body);
|
|
if (!envs.has(requestedUuid)) {
|
|
status = 404;
|
|
} else if (input.key === pendingFailureKey) {
|
|
pendingFailureKey = "";
|
|
status = 500;
|
|
} else {
|
|
const existing = envs.get(requestedUuid).filter((entry) => entry.key !== input.key);
|
|
const [stored] = attachEnvUuids([input]);
|
|
existing.push(stored);
|
|
envs.set(requestedUuid, existing);
|
|
payload = stored;
|
|
}
|
|
} else if (segments[0] === "applications" && segments[2] === "envs" && segments.length === 4 && request.method === "DELETE") {
|
|
const requestedUuid = decodeURIComponent(segments[1] || "");
|
|
const requestedEnvUuid = decodeURIComponent(segments[3] || "");
|
|
const existing = envs.get(requestedUuid);
|
|
if (!existing || !existing.some((entry) => entry.uuid === requestedEnvUuid)) {
|
|
status = 404;
|
|
} else {
|
|
envs.set(
|
|
requestedUuid,
|
|
existing.filter((entry) => entry.uuid !== requestedEnvUuid)
|
|
);
|
|
payload = { message: "Environment variable deleted." };
|
|
}
|
|
} else if (segments[0] === "applications" && segments.length === 2 && request.method === "GET") {
|
|
const requestedUuid = decodeURIComponent(segments[1] || "");
|
|
const app = apps.get(requestedUuid);
|
|
if (!app) {
|
|
status = 404;
|
|
} else {
|
|
payload = app;
|
|
}
|
|
} else if (segments[0] === "applications" && segments.length === 2 && request.method === "PATCH") {
|
|
const requestedUuid = decodeURIComponent(segments[1] || "");
|
|
const app = apps.get(requestedUuid);
|
|
if (!app) {
|
|
status = 404;
|
|
} else {
|
|
Object.assign(app, JSON.parse(body));
|
|
payload = app;
|
|
}
|
|
} else {
|
|
status = 404;
|
|
}
|
|
|
|
response.writeHead(status, { "content-type": "application/json" });
|
|
response.end(JSON.stringify(payload));
|
|
});
|
|
|
|
return new Promise((resolve, reject) => {
|
|
server.once("error", reject);
|
|
server.listen(0, "127.0.0.1", () => {
|
|
resolve({
|
|
url: `http://127.0.0.1:${server.address().port}`,
|
|
apps,
|
|
envs,
|
|
requests,
|
|
close: () => new Promise((closeResolve, closeReject) => server.close((error) => (error ? closeReject(error) : closeResolve()))),
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
describe("coolify worker group env drift", () => {
|
|
it.each(["mail", "import", "webhook", "plugin"])("treats provider credentials on worker-%s as redacted least-privilege drift", (groupName) => {
|
|
const workerGroup = group(groupName);
|
|
const entries = entriesFromRecord({
|
|
...desiredWorkerEnvWithRuntime(workerGroup),
|
|
...productAcquisitionProviderEnv(),
|
|
});
|
|
|
|
const providerDrift = getWorkerEnvDrift({ uuid: `worker-${groupName}` }, workerGroup, entries).filter((item) =>
|
|
WORKER_GENERAL_PROVIDER_ENV_KEYS.has(item.key)
|
|
);
|
|
|
|
expect(providerDrift).toHaveLength(WORKER_GENERAL_PROVIDER_ENV_KEYS.size);
|
|
expect(providerDrift).toEqual(
|
|
expect.arrayContaining(
|
|
[...WORKER_GENERAL_PROVIDER_ENV_KEYS].map((key) =>
|
|
expect.objectContaining({
|
|
key,
|
|
expected: "unset; billing provider credentials belong only to worker-general",
|
|
actual: "<configured>",
|
|
group: groupName,
|
|
})
|
|
)
|
|
)
|
|
);
|
|
expect(JSON.stringify(providerDrift)).not.toContain("redacted-lago-key");
|
|
expect(JSON.stringify(providerDrift)).not.toContain("redacted-mollie-key");
|
|
});
|
|
|
|
it("reconciles and verifies revision-bound health config through the Coolify API", async () => {
|
|
const server = await startProvisionerServer();
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids);
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(result.stdout).toContain("Coolify staging worker group applications are present");
|
|
expect(result.stdout).not.toContain(PROVISIONER_TOKEN);
|
|
expect(result.stderr).not.toContain(PROVISIONER_TOKEN);
|
|
expect(server.requests.every((request) => request.authorization === `Bearer ${PROVISIONER_TOKEN}`)).toBe(true);
|
|
const patches = server.requests.filter((request) => request.method === "PATCH");
|
|
expect(patches).toHaveLength(GROUPS.length);
|
|
for (const workerGroup of GROUPS) {
|
|
const app = server.apps.get(`worker-${workerGroup.name}`);
|
|
expect(app).toMatchObject({
|
|
docker_registry_image_tag: DEPLOY_REVISION,
|
|
health_check_enabled: true,
|
|
health_check_type: "cmd",
|
|
health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} ${workerGroup.name}`,
|
|
health_check_interval: 30,
|
|
health_check_timeout: 10,
|
|
health_check_retries: 3,
|
|
health_check_start_period: 20,
|
|
custom_healthcheck_found: false,
|
|
});
|
|
}
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("deduplicates workers already present in the resolved resource allowlist", async () => {
|
|
const server = await startProvisionerServer();
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
const tempDirectory = await mkdtemp(path.join(tmpdir(), "hivegrid-coolify-resources-"));
|
|
const githubEnv = path.join(tempDirectory, "github-env");
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids, { GITHUB_ENV: githubEnv });
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(await readFile(githubEnv, "utf8")).toBe(`STAGING_RESOLVED_RESOURCE_UUIDS=${resourceUuids.join(",")}\n`);
|
|
} finally {
|
|
await server.close();
|
|
await rm(tempDirectory, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("fails closed when Coolify does not persist worker health config", async () => {
|
|
const server = await startProvisionerServer({ persistPatch: false });
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids);
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stdout).toBe("");
|
|
expect(result.stderr).toContain("app config drift remains after reconciliation");
|
|
expect(result.stdout).not.toContain(PROVISIONER_TOKEN);
|
|
expect(result.stderr).not.toContain(PROVISIONER_TOKEN);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("does not mutate worker app config when runtime env validation fails", async () => {
|
|
const server = await startProvisionerServer({ appRevisionOverride: true });
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids);
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("APP_REVISION");
|
|
expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("prevalidates every worker group before mutating any app config", async () => {
|
|
const server = await startProvisionerServer({ invalidEnvGroup: "plugin" });
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids);
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("APP_REVISION");
|
|
expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]);
|
|
expect(server.requests.filter((request) => request.method === "GET" && request.url.endsWith("/envs"))).toHaveLength(GROUPS.length);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("fails closed on malformed worker environment responses before mutation", async () => {
|
|
const server = await startProvisionerServer({ malformedEnvGroup: "webhook" });
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids);
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("list response was malformed");
|
|
expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("rejects worker images whose built-in healthcheck would override command health", async () => {
|
|
const server = await startProvisionerServer({ customHealthcheckFound: true });
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids);
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stdout).toBe("");
|
|
expect(result.stderr).toContain("image healthcheck must be absent");
|
|
expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("rejects invalid Coolify request timeout configuration before making a request", async () => {
|
|
const server = await startProvisionerServer();
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids, { COOLIFY_API_TIMEOUT_MS: "invalid" });
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("COOLIFY_API_TIMEOUT_MS");
|
|
expect(server.requests).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("rejects remote application UUIDs that could inject the resolved environment file", async () => {
|
|
const server = await startProvisionerServer();
|
|
server.apps.get("worker-general").uuid = `${WORKER_UUIDS.general}\nINJECTED=value`;
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, []);
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stdout).toBe("");
|
|
expect(result.stderr).toContain("valid Coolify CUID");
|
|
expect(result.stderr).not.toContain("INJECTED");
|
|
expect(server.requests.filter((request) => request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it.each(["DATABASE_URL", "WORKER_ENABLED"])(
|
|
"repairs a safely identified partial worker after %s provisioning fails without creating a duplicate app",
|
|
async (failedKey) => {
|
|
const server = await startProvisioningRetryServer({ failEnvKeyOnce: failedKey });
|
|
|
|
try {
|
|
const first = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
expect(first.status).toBe(1);
|
|
expect(first.stderr).toContain("Coolify API request failed with HTTP 500");
|
|
expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toHaveLength(1);
|
|
|
|
const second = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
expect(second.status).toBe(0);
|
|
expect(second.stderr).toBe("");
|
|
expect(second.stdout).toContain("Repaired partial Coolify worker-general application");
|
|
expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toHaveLength(1);
|
|
expect(envEntriesToRecord(server.envs.get(RETRY_GENERAL_UUID))).toMatchObject({
|
|
...desiredWorkerEnv(group("general")),
|
|
DATABASE_URL: WORKER_DATABASE_URL,
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-worker-token",
|
|
});
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
}
|
|
);
|
|
|
|
it.each([
|
|
[
|
|
"preview-first",
|
|
[
|
|
{ key: "DATABASE_URL", value: "", is_preview: true },
|
|
{ key: "DATABASE_URL", value: WORKER_DATABASE_URL, is_preview: false },
|
|
],
|
|
],
|
|
[
|
|
"production-first",
|
|
[
|
|
{ key: "DATABASE_URL", value: WORKER_DATABASE_URL, is_preview: false },
|
|
{ key: "DATABASE_URL", value: "", is_preview: true },
|
|
],
|
|
],
|
|
])("clones the non-preview runtime value independent of %s Coolify env ordering", async (_order, databaseEntries) => {
|
|
const server = await startProvisioningRetryServer({
|
|
sourceEnvEntries: [
|
|
...databaseEntries,
|
|
{ key: "BULLMQ_REDIS_URL", value: "redis://redacted", is_preview: false },
|
|
...entriesFromRecord(productAcquisitionProviderEnv()).map((entry) => ({ ...entry, is_preview: false })),
|
|
{ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", value: "https://app.example.test", is_preview: false },
|
|
{ key: "API_ACCESS_TOKEN", value: "redacted-worker-token", is_preview: false },
|
|
],
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(envEntriesToRecord(server.envs.get(RETRY_GENERAL_UUID))).toMatchObject({
|
|
DATABASE_URL: WORKER_DATABASE_URL,
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
...productAcquisitionProviderEnv(),
|
|
});
|
|
const clonedDatabaseRequests = server.requests
|
|
.filter((request) => request.method === "POST" && request.url === `/applications/${RETRY_GENERAL_UUID}/envs`)
|
|
.map((request) => JSON.parse(request.body))
|
|
.filter((body) => body.key === "DATABASE_URL");
|
|
expect(clonedDatabaseRequests).toEqual([expect.objectContaining({ value: WORKER_DATABASE_URL, is_preview: false })]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("fails before mutation when a misplaced provider variable has no deletable Coolify env UUID", async () => {
|
|
const server = await startProvisioningRetryServer({
|
|
missingGroup: "none",
|
|
targetEnvOverrides: {
|
|
mail: {
|
|
LAGO_API_KEY: "undeletable-lago-secret-sentinel",
|
|
},
|
|
},
|
|
});
|
|
const mailEnvs = server.envs.get(WORKER_UUIDS.mail);
|
|
for (const entry of mailEnvs) {
|
|
if (entry.key === "LAGO_API_KEY") {
|
|
delete entry.uuid;
|
|
}
|
|
}
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids("none"), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("provider environment cannot be removed safely because its env UUID is missing");
|
|
expect(result.stdout).not.toContain("undeletable-lago-secret-sentinel");
|
|
expect(result.stderr).not.toContain("undeletable-lago-secret-sentinel");
|
|
expect(server.requests.filter((request) => ["POST", "PATCH", "DELETE"].includes(request.method))).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("never clones migrator or seed credentials from the worker env source", async () => {
|
|
const server = await startProvisioningRetryServer({
|
|
sourceEnvEntries: entriesFromRecord({
|
|
DATABASE_URL: WORKER_DATABASE_URL,
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
...productAcquisitionProviderEnv(),
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-worker-token",
|
|
DIRECT_URL: "postgres://owner:direct-secret-sentinel@db.example.test/hivegrid",
|
|
MIGRATOR_DATABASE_URL: "postgres://migrator:migrator-secret-sentinel@db.example.test/hivegrid",
|
|
SEED_ADMIN_PASSWORD: "seed-secret-sentinel",
|
|
}),
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
const target = envEntriesToRecord(server.envs.get(RETRY_GENERAL_UUID));
|
|
expect(target).not.toHaveProperty("DIRECT_URL");
|
|
expect(target).not.toHaveProperty("MIGRATOR_DATABASE_URL");
|
|
expect(target).not.toHaveProperty("SEED_ADMIN_PASSWORD");
|
|
const clonedKeys = server.requests
|
|
.filter((request) => request.method === "POST" && request.url === `/applications/${RETRY_GENERAL_UUID}/envs`)
|
|
.map((request) => JSON.parse(request.body).key);
|
|
expect(clonedKeys).not.toEqual(expect.arrayContaining(["DIRECT_URL", "MIGRATOR_DATABASE_URL", "SEED_ADMIN_PASSWORD"]));
|
|
expect(result.stdout).not.toContain("secret-sentinel");
|
|
expect(result.stderr).not.toContain("secret-sentinel");
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("provisions a non-general worker without cloning provider credentials from the approved source", async () => {
|
|
const server = await startProvisioningRetryServer({ missingGroup: "mail" });
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids("mail"), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
const target = envEntriesToRecord(server.envs.get(WORKER_UUIDS.mail));
|
|
for (const key of WORKER_GENERAL_PROVIDER_ENV_KEYS) {
|
|
expect(target).not.toHaveProperty(key);
|
|
}
|
|
expect(target.CRYPTO_SECRET).toBe(SHARED_CRYPTO_SECRET);
|
|
const clonedKeys = server.requests
|
|
.filter((request) => request.method === "POST" && request.url === `/applications/${WORKER_UUIDS.mail}/envs`)
|
|
.map((request) => JSON.parse(request.body).key);
|
|
expect(clonedKeys).not.toEqual(expect.arrayContaining([...WORKER_GENERAL_PROVIDER_ENV_KEYS]));
|
|
expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.general))).toMatchObject(productAcquisitionProviderEnv());
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it.each([
|
|
["managed", true],
|
|
["legacy allowlisted", false],
|
|
])("removes provider credentials from an existing %s non-general worker without logging values", async (_kind, managed) => {
|
|
const misplacedProviderEnv = {
|
|
...productAcquisitionProviderEnv(),
|
|
LAGO_API_KEY: "lago-secret-sentinel",
|
|
MOLLIE_API_KEY: "mollie-secret-sentinel",
|
|
};
|
|
const server = await startProvisioningRetryServer({
|
|
allTargetWorkersManaged: managed,
|
|
missingGroup: "none",
|
|
targetEnvOverrides: { mail: misplacedProviderEnv },
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids("none"), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(result.stdout).toContain("Removed disallowed provider environment from Coolify worker-mail");
|
|
for (const key of WORKER_GENERAL_PROVIDER_ENV_KEYS) {
|
|
expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.mail))).not.toHaveProperty(key);
|
|
}
|
|
const deletionRequests = server.requests.filter(
|
|
(request) => request.method === "DELETE" && request.url.startsWith(`/applications/${WORKER_UUIDS.mail}/envs/`)
|
|
);
|
|
expect(deletionRequests).toHaveLength(WORKER_GENERAL_PROVIDER_ENV_KEYS.size);
|
|
expect(result.stdout).not.toContain("lago-secret-sentinel");
|
|
expect(result.stderr).not.toContain("lago-secret-sentinel");
|
|
expect(result.stdout).not.toContain("mollie-secret-sentinel");
|
|
expect(result.stderr).not.toContain("mollie-secret-sentinel");
|
|
expect(JSON.stringify(server.requests)).not.toContain("lago-secret-sentinel");
|
|
expect(JSON.stringify(server.requests)).not.toContain("mollie-secret-sentinel");
|
|
expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.general))).toMatchObject(productAcquisitionProviderEnv());
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("mirrors the KWP pilot tenant from worker-general to worker-import only", async () => {
|
|
const server = await startProvisioningRetryServer({
|
|
missingGroup: "none",
|
|
targetEnvOverrides: {
|
|
general: { [KWP_WORKER_ENV_KEY]: KWP_PILOT_TENANT },
|
|
},
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids("none"), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(result.stdout).toContain("Reconciled KWP pilot environment for Coolify worker-import");
|
|
expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS.import))).toMatchObject({
|
|
[KWP_WORKER_ENV_KEY]: KWP_PILOT_TENANT,
|
|
});
|
|
for (const groupName of ["mail", "webhook", "plugin"]) {
|
|
expect(envEntriesToRecord(server.envs.get(WORKER_UUIDS[groupName]))).not.toHaveProperty(KWP_WORKER_ENV_KEY);
|
|
}
|
|
expect(result.stdout).not.toContain(KWP_PILOT_TENANT);
|
|
expect(result.stderr).not.toContain(KWP_PILOT_TENANT);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("fails before provisioning when the worker env source uses the application database role", async () => {
|
|
const unsafeUrl = "postgres://hivegrid_app:source-secret-sentinel@db.example.test/hivegrid";
|
|
const server = await startProvisioningRetryServer({
|
|
sourceEnvEntries: entriesFromRecord({
|
|
DATABASE_URL: unsafeUrl,
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-worker-token",
|
|
}),
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toMatch(/predicted drift: .*DATABASE_URL/);
|
|
expect(result.stdout).not.toContain(unsafeUrl);
|
|
expect(result.stderr).not.toContain(unsafeUrl);
|
|
expect(result.stdout).not.toContain("source-secret-sentinel");
|
|
expect(result.stderr).not.toContain("source-secret-sentinel");
|
|
expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("fails before app creation or secret cloning when runtime source keys have duplicate non-preview entries", async () => {
|
|
const server = await startProvisioningRetryServer({
|
|
sourceEnvEntries: [
|
|
{ key: "DATABASE_URL", value: "postgres://hivegrid_worker:first-redacted@db.example.test/hivegrid", is_preview: false },
|
|
{ key: "DATABASE_URL", value: "postgres://hivegrid_worker:second-redacted@db.example.test/hivegrid", is_preview: false },
|
|
{ key: "BULLMQ_REDIS_URL", value: "redis://redacted", is_preview: false },
|
|
{ key: "WORKFLOW_TRIGGER_EXECUTOR_URL", value: "https://app.example.test", is_preview: false },
|
|
{ key: "API_ACCESS_TOKEN", value: "redacted-worker-token", is_preview: false },
|
|
],
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("duplicate non-preview keys");
|
|
expect(result.stdout).not.toContain("first-redacted");
|
|
expect(result.stderr).not.toContain("first-redacted");
|
|
expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("fails before mutation when a marked partial worker has conflicting managed environment values", async () => {
|
|
const server = await startProvisioningRetryServer({
|
|
preexistingPartialEnv: {
|
|
DATABASE_URL: WORKER_DATABASE_URL,
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKER_ENABLED: "false",
|
|
},
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("partial environment is unsafe: WORKER_ENABLED");
|
|
expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("reconciles only the target environment when a complete foreign managed fleet shares names and images", async () => {
|
|
const server = await startProvisioningRetryServer({
|
|
preexistingPartialEnv: desiredWorkerEnvWithRuntime(group("general"), {
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-worker-token",
|
|
}),
|
|
preexistingPartialAppOverrides: {
|
|
docker_registry_image_tag: "staging",
|
|
health_check_enabled: false,
|
|
health_check_type: "http",
|
|
health_check_command: "",
|
|
},
|
|
allTargetWorkersManaged: true,
|
|
includeForeignFleet: true,
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, [...provisioningResourceUuids(), RETRY_GENERAL_UUID], provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toEqual([]);
|
|
expect(server.requests.filter((request) => request.method === "PATCH" && request.url === `/applications/${RETRY_GENERAL_UUID}`)).toHaveLength(1);
|
|
expect(server.apps.get(RETRY_GENERAL_UUID)).toMatchObject({
|
|
docker_registry_image_tag: DEPLOY_REVISION,
|
|
health_check_type: "cmd",
|
|
health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`,
|
|
});
|
|
const foreignUuids = new Set(Object.values(FOREIGN_WORKER_UUIDS));
|
|
expect(server.requests.filter((request) => [...foreignUuids].some((uuid) => String(request.url || "").includes(uuid)))).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("reconciles desired health config drift without weakening managed retry identity", async () => {
|
|
const previousRevision = "b".repeat(40);
|
|
const server = await startProvisioningRetryServer({
|
|
preexistingPartialEnv: desiredWorkerEnvWithRuntime(group("general"), {
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-worker-token",
|
|
}),
|
|
preexistingPartialAppOverrides: {
|
|
docker_registry_image_tag: previousRevision,
|
|
health_check_command: `node apps/worker/dist/deploy-health.js ${previousRevision} general`,
|
|
health_check_interval: 5,
|
|
},
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toEqual([]);
|
|
expect(server.requests.filter((request) => request.method === "PATCH" && request.url === `/applications/${RETRY_GENERAL_UUID}`)).toHaveLength(1);
|
|
expect(server.apps.get(RETRY_GENERAL_UUID)).toMatchObject({
|
|
docker_registry_image_tag: DEPLOY_REVISION,
|
|
health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`,
|
|
health_check_interval: 30,
|
|
});
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("reconciles legacy health config on an explicitly allowlisted managed worker", async () => {
|
|
const server = await startProvisioningRetryServer({
|
|
preexistingPartialEnv: desiredWorkerEnvWithRuntime(group("general"), {
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-worker-token",
|
|
}),
|
|
preexistingPartialAppOverrides: {
|
|
docker_registry_image_tag: "staging",
|
|
health_check_enabled: false,
|
|
health_check_type: "http",
|
|
health_check_command: "",
|
|
},
|
|
});
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, [...provisioningResourceUuids(), RETRY_GENERAL_UUID], provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(server.requests.filter((request) => request.method === "POST" && request.url === "/applications/dockerimage")).toEqual([]);
|
|
expect(server.requests.filter((request) => request.method === "PATCH" && request.url === `/applications/${RETRY_GENERAL_UUID}`)).toHaveLength(1);
|
|
expect(server.apps.get(RETRY_GENERAL_UUID)).toMatchObject({
|
|
docker_registry_image_tag: DEPLOY_REVISION,
|
|
health_check_enabled: true,
|
|
health_check_type: "cmd",
|
|
health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`,
|
|
});
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("rejects an explicit worker env source with the wrong image before mutation", async () => {
|
|
const server = await startProvisioningRetryServer();
|
|
server.apps.get(WORKER_ENV_SOURCE_UUID).docker_registry_image_name = "registry.example.test/toolctrl-labs/hivegrid/web";
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, provisioningResourceUuids(), provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("must reference the exact worker image");
|
|
expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("rejects an explicit worker env source outside a non-empty allowlist before mutation", async () => {
|
|
const server = await startProvisioningRetryServer();
|
|
const resourceUuids = provisioningResourceUuids().filter((uuid) => uuid !== WORKER_ENV_SOURCE_UUID);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids, provisioningEnvOverrides());
|
|
|
|
expect(result.status).toBe(1);
|
|
expect(result.stderr).toContain("must be included in COOLIFY_STAGING_RESOURCE_UUIDS");
|
|
expect(server.requests.filter((request) => request.method === "POST" || request.method === "PATCH")).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("ignores an unused legacy env source when all worker groups already exist", async () => {
|
|
const server = await startProvisionerServer();
|
|
const source = {
|
|
uuid: WORKER_ENV_SOURCE_UUID,
|
|
name: "hivegrid-web",
|
|
docker_registry_image_name: "registry.example.test/toolctrl-labs/hivegrid/web",
|
|
};
|
|
server.apps.set(source.uuid, source);
|
|
const resourceUuids = [...server.apps.values()].map((app) => app.uuid);
|
|
|
|
try {
|
|
const result = await runProvisioner(server.url, resourceUuids, {
|
|
COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID: source.uuid,
|
|
});
|
|
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toBe("");
|
|
expect(server.requests.filter((request) => String(request.url || "").includes(source.uuid))).toEqual([]);
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
});
|
|
|
|
it("provisions worker apps with revision-bound command health checks", () => {
|
|
expect(
|
|
buildWorkerApplicationBody({
|
|
group: group("mail"),
|
|
projectUuid: "project_1",
|
|
serverUuid: "server_1",
|
|
destinationUuid: "destination_1",
|
|
environmentName: "production",
|
|
environmentUuid: "environment_1",
|
|
workerImage: "registry.example.test/hivegrid/worker",
|
|
imageTag: DEPLOY_REVISION,
|
|
})
|
|
).toMatchObject({
|
|
project_uuid: "project_1",
|
|
server_uuid: "server_1",
|
|
destination_uuid: "destination_1",
|
|
environment_name: "production",
|
|
environment_uuid: "environment_1",
|
|
docker_registry_image_name: "registry.example.test/hivegrid/worker",
|
|
docker_registry_image_tag: DEPLOY_REVISION,
|
|
name: "worker-mail",
|
|
ports_exposes: "9102",
|
|
health_check_enabled: true,
|
|
health_check_type: "cmd",
|
|
health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} mail`,
|
|
connect_to_docker_network: true,
|
|
});
|
|
});
|
|
|
|
it("rejects mutable or malformed worker deployment health tags", () => {
|
|
expect(() => buildWorkerDeploymentHealthCommand("staging", "general")).toThrow(/full lowercase Git SHA/);
|
|
expect(() => buildWorkerDeploymentHealthCommand(DEPLOY_REVISION, "unknown")).toThrow(/valid worker group/);
|
|
expect(buildWorkerDeploymentHealthCommand(DEPLOY_REVISION, "general")).toBe(`node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} general`);
|
|
});
|
|
|
|
it("builds the expected mail worker environment", () => {
|
|
expect(desiredWorkerEnv(group("mail"))).toEqual({
|
|
DATABASE_RUNTIME_ROLE: "hivegrid_worker",
|
|
RLS_RUNTIME_MODE: "required",
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "mail",
|
|
WORKER_HEALTH_PORT: "9102",
|
|
WORKER_ADMIN_PORT: "9202",
|
|
WORKER_CONCURRENCY_MAIL: "5",
|
|
MARKETING_SEQUENCES_ENABLED: "false",
|
|
TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true",
|
|
});
|
|
});
|
|
|
|
it("builds the expected general worker production outbox environment", () => {
|
|
expect(desiredWorkerEnv(group("general"))).toMatchObject({
|
|
DATABASE_RUNTIME_ROLE: "hivegrid_worker",
|
|
RLS_RUNTIME_MODE: "required",
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "general",
|
|
WORKER_HEALTH_PORT: "9101",
|
|
WORKER_ADMIN_PORT: "9201",
|
|
WORKER_CONCURRENCY_GENERAL: "10",
|
|
OUTBOX_DISPATCHER_ENABLED: "true",
|
|
OUTBOX_DISPATCHER_REQUIRED: "true",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "http",
|
|
WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
|
|
WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY: "5",
|
|
WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
|
|
});
|
|
});
|
|
|
|
it("protects every managed worker env key from baseline-env cloning", () => {
|
|
expect(WORKER_ENV_OVERRIDE_KEYS.has("APP_REVISION")).toBe(true);
|
|
for (const item of GROUPS) {
|
|
for (const key of Object.keys(desiredWorkerEnv(item))) {
|
|
expect(WORKER_ENV_OVERRIDE_KEYS.has(key)).toBe(true);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("classifies migration and seed credentials as forbidden worker runtime env", () => {
|
|
expect([...WORKER_PRIVILEGED_ENV_KEYS].sort()).toEqual(["DIRECT_URL", "MIGRATE_ON_BOOT", "MIGRATOR_DATABASE_URL", "SEED_ONCE", "SEED_ON_BOOT"]);
|
|
});
|
|
|
|
it("normalizes quoted Coolify env values", () => {
|
|
expect(
|
|
envEntriesToRecord([
|
|
{ key: "WORKER_ENABLED", value: '"true"' },
|
|
{ key: "WORKER_QUEUE_GROUP", value: "'mail'" },
|
|
{ key: "APP_REVISION", real_value: "'configured-revision'" },
|
|
])
|
|
).toEqual({
|
|
APP_REVISION: "configured-revision",
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "mail",
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
[
|
|
{ key: "MAILJET_API_KEY", value: "", is_preview: true },
|
|
{ key: "MAILJET_API_KEY", value: "runtime-key", is_preview: false },
|
|
],
|
|
[
|
|
{ key: "MAILJET_API_KEY", value: "runtime-key", is_preview: false },
|
|
{ key: "MAILJET_API_KEY", value: "", is_preview: true },
|
|
],
|
|
])("uses the non-preview target value regardless of duplicate API ordering", (...entries) => {
|
|
expect(envEntriesToRecord(entries)).toEqual({ MAILJET_API_KEY: "runtime-key" });
|
|
});
|
|
|
|
it("rejects APP_REVISION overrides that could spoof stale worker images", () => {
|
|
const mail = group("mail");
|
|
const entries = desiredWorkerEnvWithRuntime(mail, { APP_REVISION: "" });
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail", name: "worker-mail" }, mail, entriesFromRecord(entries))).toContainEqual(
|
|
expect.objectContaining({
|
|
key: "APP_REVISION",
|
|
expected: "unset; image-baked revision is authoritative",
|
|
actual: "<configured>",
|
|
group: "mail",
|
|
})
|
|
);
|
|
});
|
|
|
|
it("returns no drift when an existing worker app matches the desired env", () => {
|
|
const mail = group("mail");
|
|
const drift = getWorkerEnvDrift({ uuid: "worker-mail", name: "worker-mail" }, mail, entriesFromRecord(desiredWorkerEnvWithRuntime(mail)));
|
|
|
|
expect(drift).toEqual([]);
|
|
});
|
|
|
|
it("returns no app config drift for the revision-bound command healthcheck", () => {
|
|
const mail = group("mail");
|
|
|
|
expect(
|
|
getWorkerAppConfigDrift(
|
|
{
|
|
uuid: "worker-mail",
|
|
name: "worker-mail",
|
|
docker_registry_image_tag: DEPLOY_REVISION,
|
|
ports_exposes: "9102",
|
|
health_check_enabled: true,
|
|
health_check_type: "cmd",
|
|
health_check_command: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} mail`,
|
|
health_check_interval: 30,
|
|
health_check_timeout: 10,
|
|
health_check_retries: 3,
|
|
health_check_start_period: 20,
|
|
custom_healthcheck_found: false,
|
|
},
|
|
mail,
|
|
DEPLOY_REVISION
|
|
)
|
|
).toEqual([]);
|
|
});
|
|
|
|
it("reports existing worker app health-check drift", () => {
|
|
const mail = group("mail");
|
|
|
|
expect(
|
|
getWorkerAppConfigDrift(
|
|
{
|
|
uuid: "worker-mail",
|
|
name: "worker-mail",
|
|
docker_registry_image_tag: "previous-revision",
|
|
ports_exposes: "9101",
|
|
health_check_enabled: false,
|
|
health_check_type: "http",
|
|
health_check_command: null,
|
|
health_check_interval: 10,
|
|
health_check_timeout: 5,
|
|
health_check_retries: 2,
|
|
health_check_start_period: 5,
|
|
custom_healthcheck_found: true,
|
|
},
|
|
mail,
|
|
DEPLOY_REVISION
|
|
)
|
|
).toEqual([
|
|
expect.objectContaining({
|
|
key: "docker_registry_image_tag",
|
|
expected: DEPLOY_REVISION,
|
|
actual: "previous-revision",
|
|
group: "mail",
|
|
}),
|
|
expect.objectContaining({ key: "ports_exposes", expected: "9102", actual: "9101", group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_enabled", expected: "true", actual: "false", group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_type", expected: "cmd", actual: "http", group: "mail" }),
|
|
expect.objectContaining({
|
|
key: "health_check_command",
|
|
expected: `node apps/worker/dist/deploy-health.js ${DEPLOY_REVISION} mail`,
|
|
actual: "<missing>",
|
|
group: "mail",
|
|
}),
|
|
expect.objectContaining({ key: "health_check_interval", expected: 30, actual: 10, group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_timeout", expected: 10, actual: 5, group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_retries", expected: 3, actual: 2, group: "mail" }),
|
|
expect.objectContaining({ key: "health_check_start_period", expected: 20, actual: 5, group: "mail" }),
|
|
expect.objectContaining({ key: "custom_healthcheck_found", expected: "false", actual: "true", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("accepts a fully configured general worker HTTP bridge", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("accepts WEB_ORIGIN as the general worker HTTP bridge fallback", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WEB_ORIGIN: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("fails closed when a general worker lacks required Lago or Mollie runtime configuration", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
LAGO_API_KEY: "",
|
|
MOLLIE_PROFILE_ID: "",
|
|
MOLLIE_TIMEOUT_MS: "0",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ key: "LAGO_API_KEY", actual: "<missing>" }),
|
|
expect.objectContaining({ key: "MOLLIE_TIMEOUT_MS", actual: "<invalid>" }),
|
|
])
|
|
);
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).not.toEqual(
|
|
expect.arrayContaining([expect.objectContaining({ key: "MOLLIE_PROFILE_ID" })])
|
|
);
|
|
});
|
|
|
|
it("allows an explicitly disabled provider subscriber without Lago or Mollie runtime configuration", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED: "false",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("rejects ambiguous provider subscriber gate values", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
...productAcquisitionProviderEnv(),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED: "disabled",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
key: "PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_REQUIRED",
|
|
actual: "<invalid>",
|
|
}),
|
|
])
|
|
);
|
|
});
|
|
|
|
it("does not require plugin hook fanout before the production plugin runtime gate flips", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PLUGIN_RUNTIME_PRODUCTION_ENABLED: "false",
|
|
PLUGIN_HOOK_FANOUT_ENABLED: "false",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("requires plugin hook fanout fail-closed when production plugin runtime is enabled", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
|
|
PLUGIN_HOOK_FANOUT_ENABLED: "false",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
key: "PLUGIN_HOOK_FANOUT_ENABLED",
|
|
expected: "unset or true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true",
|
|
actual: "false",
|
|
group: "general",
|
|
}),
|
|
expect.objectContaining({
|
|
key: "PLUGIN_HOOK_FANOUT_REQUIRED",
|
|
expected: "true when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
])
|
|
);
|
|
});
|
|
|
|
it("accepts production plugin runtime when general worker hook fanout is required", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
|
|
PLUGIN_HOOK_FANOUT_REQUIRED: "true",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("accepts native workflow execution as an explicit general worker opt-in", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "native",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("normalizes native workflow execution mode like the worker runtime", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: " NATIVE ",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("accepts hybrid workflow execution when the HTTP fallback is configured", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "hybrid",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([]);
|
|
});
|
|
|
|
it("normalizes hybrid workflow execution mode before checking HTTP fallback dependencies", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: " HYBRID ",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
key: "WORKFLOW_TRIGGER_EXECUTOR_URL",
|
|
expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
expect.objectContaining({
|
|
key: "API_ACCESS_TOKEN",
|
|
expected: "configured API_ACCESS_TOKEN matching the web runtime",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
])
|
|
);
|
|
});
|
|
|
|
it("requires the HTTP fallback dependencies for hybrid workflow execution", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_MODE: "hybrid",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({
|
|
key: "WORKFLOW_TRIGGER_EXECUTOR_URL",
|
|
expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
expect.objectContaining({
|
|
key: "API_ACCESS_TOKEN",
|
|
expected: "configured API_ACCESS_TOKEN matching the web runtime",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("rejects legacy-only workflow executor token configuration", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
WORKFLOW_TRIGGER_EXECUTOR_TOKEN: "legacy-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({
|
|
key: "API_ACCESS_TOKEN",
|
|
expected: "configured API_ACCESS_TOKEN matching the web runtime",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("rejects legacy workflow executor token even when API_ACCESS_TOKEN is configured", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
WORKFLOW_TRIGGER_EXECUTOR_TOKEN: "legacy-token",
|
|
};
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({
|
|
key: "WORKFLOW_TRIGGER_EXECUTOR_TOKEN",
|
|
expected: "unset; API_ACCESS_TOKEN is the supported workflow bridge credential",
|
|
actual: "<configured>",
|
|
group: "general",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("reports disabled existing worker apps", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail), WORKER_ENABLED: "false" };
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_ENABLED", expected: "true", actual: "false", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("reports wrong queue groups", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail), WORKER_QUEUE_GROUP: "general" };
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_QUEUE_GROUP", expected: "mail", actual: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("reports mail workers without the marketing sequence killswitch", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail) };
|
|
delete entries.MARKETING_SEQUENCES_ENABLED;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "MARKETING_SEQUENCES_ENABLED", expected: "false", actual: "<missing>", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("reports mail workers without fail-loud transactional outbox bootstrap", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail) };
|
|
delete entries.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "TRANSACTIONAL_MAIL_OUTBOX_REQUIRED", expected: "true", actual: "<missing>", group: "mail" }),
|
|
]);
|
|
});
|
|
|
|
it("reports mail workers without the shared tenant-envelope key without exposing its value", () => {
|
|
const mail = group("mail");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(mail) };
|
|
delete entries.CRYPTO_SECRET;
|
|
|
|
const drift = getWorkerEnvDrift({ uuid: "worker-mail" }, mail, entriesFromRecord(entries));
|
|
expect(drift).toEqual([
|
|
expect.objectContaining({
|
|
key: "CRYPTO_SECRET",
|
|
expected: "configured shared tenant-envelope key matching hivegrid-web",
|
|
actual: "<missing>",
|
|
group: "mail",
|
|
}),
|
|
]);
|
|
expect(JSON.stringify(drift)).not.toContain(SHARED_CRYPTO_SECRET);
|
|
});
|
|
|
|
it("requires DATABASE_URL and BULLMQ_REDIS_URL on every managed worker group", () => {
|
|
for (const workerGroup of GROUPS) {
|
|
const missingDatabase = desiredWorkerEnvWithRuntime(workerGroup);
|
|
delete missingDatabase.DATABASE_URL;
|
|
const missingRedis = desiredWorkerEnvWithRuntime(workerGroup);
|
|
delete missingRedis.BULLMQ_REDIS_URL;
|
|
missingRedis.REDIS_URL = "redis://fallback-not-accepted-for-coolify-workers";
|
|
|
|
expect(getWorkerEnvDrift({ uuid: `worker-${workerGroup.name}` }, workerGroup, entriesFromRecord(missingDatabase))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ key: "DATABASE_URL", expected: "configured DATABASE_URL", actual: "<missing>", group: workerGroup.name }),
|
|
])
|
|
);
|
|
expect(getWorkerEnvDrift({ uuid: `worker-${workerGroup.name}` }, workerGroup, entriesFromRecord(missingRedis))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
key: "BULLMQ_REDIS_URL",
|
|
expected: "configured BULLMQ_REDIS_URL",
|
|
actual: "<missing; REDIS_URL fallback not accepted for managed workers>",
|
|
group: workerGroup.name,
|
|
}),
|
|
])
|
|
);
|
|
}
|
|
});
|
|
|
|
it("rejects worker DSNs authenticated as an application or owner role without leaking the URL", () => {
|
|
const workerGroup = group("mail");
|
|
for (const databaseUrl of [
|
|
"postgres://hivegrid_app:secret-sentinel@db.example.test/hivegrid",
|
|
"postgres://postgres:secret-sentinel@db.example.test/hivegrid",
|
|
"not-a-postgres-url",
|
|
]) {
|
|
const entries = desiredWorkerEnvWithRuntime(workerGroup, { DATABASE_URL: databaseUrl });
|
|
const drift = getWorkerEnvDrift({ uuid: "worker-mail" }, workerGroup, entriesFromRecord(entries));
|
|
|
|
expect(drift).toContainEqual(
|
|
expect.objectContaining({
|
|
key: "DATABASE_URL",
|
|
expected: "PostgreSQL DSN authenticated as hivegrid_worker",
|
|
})
|
|
);
|
|
expect(JSON.stringify(drift)).not.toContain("secret-sentinel");
|
|
expect(JSON.stringify(drift)).not.toContain(databaseUrl);
|
|
}
|
|
});
|
|
|
|
it("rejects migration or seed credentials already present on a worker", () => {
|
|
const workerGroup = group("mail");
|
|
const entries = desiredWorkerEnvWithRuntime(workerGroup, {
|
|
DIRECT_URL: "postgres://owner:secret-sentinel@db.example.test/hivegrid",
|
|
MIGRATOR_DATABASE_URL: "postgres://migrator:secret-sentinel@db.example.test/hivegrid",
|
|
SEED_ADMIN_PASSWORD: "seed-secret-sentinel",
|
|
});
|
|
const drift = getWorkerEnvDrift({ uuid: "worker-mail" }, workerGroup, entriesFromRecord(entries));
|
|
|
|
expect(drift.map(({ key }) => key)).toEqual(expect.arrayContaining(["DIRECT_URL", "MIGRATOR_DATABASE_URL", "SEED_ADMIN_PASSWORD"]));
|
|
expect(JSON.stringify(drift)).not.toContain("secret-sentinel");
|
|
});
|
|
|
|
it("reports missing required general-worker outbox activation", () => {
|
|
const general = group("general");
|
|
const entries = {
|
|
...desiredWorkerEnvWithRuntime(general),
|
|
BULLMQ_REDIS_URL: "redis://redacted",
|
|
WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test",
|
|
API_ACCESS_TOKEN: "redacted-token",
|
|
OUTBOX_DISPATCHER_REQUIRED: "false",
|
|
};
|
|
delete entries.WORKFLOW_TRIGGER_EXECUTOR_MODE;
|
|
entries.WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY = "1";
|
|
delete entries.WORKFLOW_TRIGGER_OUTBOX_REQUIRED;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "OUTBOX_DISPATCHER_REQUIRED", expected: "true", actual: "false", group: "general" }),
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_EXECUTOR_MODE", expected: "http|hybrid|native", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY", expected: "5", actual: "1", group: "general" }),
|
|
expect.objectContaining({ key: "WORKFLOW_TRIGGER_OUTBOX_REQUIRED", expected: "true", actual: "<missing>", group: "general" }),
|
|
]);
|
|
});
|
|
|
|
it("reports missing general worker Redis and HTTP bridge dependencies", () => {
|
|
const general = group("general");
|
|
const entries = { ...desiredWorkerEnv(general) };
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-general" }, general, entriesFromRecord(entries))).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ key: "DATABASE_URL", expected: "configured DATABASE_URL", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({ key: "BULLMQ_REDIS_URL", expected: "configured BULLMQ_REDIS_URL", actual: "<missing>", group: "general" }),
|
|
expect.objectContaining({
|
|
key: "WORKFLOW_TRIGGER_EXECUTOR_URL",
|
|
expected: "configured WORKFLOW_TRIGGER_EXECUTOR_URL, APP_URL, or WEB_ORIGIN",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
expect.objectContaining({
|
|
key: "API_ACCESS_TOKEN",
|
|
expected: "configured API_ACCESS_TOKEN matching the web runtime",
|
|
actual: "<missing>",
|
|
group: "general",
|
|
}),
|
|
])
|
|
);
|
|
});
|
|
|
|
it("reports swapped or missing worker ports", () => {
|
|
const plugin = group("plugin");
|
|
const entries = { ...desiredWorkerEnvWithRuntime(plugin), WORKER_HEALTH_PORT: "9205" };
|
|
delete entries.WORKER_ADMIN_PORT;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-plugin" }, plugin, entriesFromRecord(entries))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_HEALTH_PORT", expected: "9105", actual: "9205" }),
|
|
expect.objectContaining({ key: "WORKER_ADMIN_PORT", expected: "9205", actual: "<missing>" }),
|
|
]);
|
|
});
|
|
|
|
it("reports missing or wrong group concurrency", () => {
|
|
const webhook = group("webhook");
|
|
const wrong = { ...desiredWorkerEnvWithRuntime(webhook), WORKER_CONCURRENCY_WEBHOOK: "5" };
|
|
const missing = { ...desiredWorkerEnvWithRuntime(webhook) };
|
|
delete missing.WORKER_CONCURRENCY_WEBHOOK;
|
|
|
|
expect(getWorkerEnvDrift({ uuid: "worker-webhook" }, webhook, entriesFromRecord(wrong))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_CONCURRENCY_WEBHOOK", expected: "20", actual: "5" }),
|
|
]);
|
|
expect(getWorkerEnvDrift({ uuid: "worker-webhook" }, webhook, entriesFromRecord(missing))).toEqual([
|
|
expect.objectContaining({ key: "WORKER_CONCURRENCY_WEBHOOK", expected: "20", actual: "<missing>" }),
|
|
]);
|
|
});
|
|
});
|