410 lines
19 KiB
JavaScript
410 lines
19 KiB
JavaScript
// @vitest-environment node
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { afterEach, describe, it } from "vitest";
|
|
import { verifyPortalDomainControllerDeploy } from "./verify-portal-domain-controller-deploy.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const fixtures = [];
|
|
const requiredPaths = [
|
|
"Dockerfile",
|
|
".gitea/workflows/ci.yml",
|
|
".gitea/workflows/deploy.yml",
|
|
"scripts/ci/deploy-production-closures.sh",
|
|
"scripts/ci/docker-smoke-suite.sh",
|
|
"scripts/coolify-portal-domain-controller.mjs",
|
|
"scripts/docker-portal-domain-controller-runtime-smoke.sh",
|
|
"apps/portal-domain-controller/package.json",
|
|
"apps/portal-domain-controller/scripts/build.mjs",
|
|
"apps/portal-domain-controller/src",
|
|
"runtime/portal-domain-controller/package.json",
|
|
];
|
|
|
|
afterEach(() => {
|
|
for (const fixture of fixtures.splice(0)) fs.rmSync(fixture, { recursive: true, force: true });
|
|
});
|
|
|
|
describe("portal-domain controller deploy verifier", () => {
|
|
it("accepts the reviewed repository contract", () => {
|
|
const result = verifyPortalDomainControllerDeploy(repoRoot);
|
|
assert.deepEqual(result.failures, []);
|
|
});
|
|
|
|
it("fails closed when the controller bypasses the common bundle or exact helper deploy", () => {
|
|
const fixture = createFixture();
|
|
const dockerfilePath = path.join(fixture, "Dockerfile");
|
|
fs.writeFileSync(
|
|
dockerfilePath,
|
|
fs
|
|
.readFileSync(dockerfilePath, "utf8")
|
|
.replace("bash scripts/ci/deploy-production-closures.sh", "true")
|
|
.replace(
|
|
"FROM production-deps-bundle AS portal-domain-controller-production-deps",
|
|
"FROM build AS portal-domain-controller-production-deps"
|
|
)
|
|
);
|
|
const helperPath = path.join(fixture, "scripts/ci/deploy-production-closures.sh");
|
|
fs.writeFileSync(
|
|
helperPath,
|
|
fs
|
|
.readFileSync(helperPath, "utf8")
|
|
.replace("--filter @hivegrid/runtime-portal-domain-controller deploy --prod --no-optional", "--filter @hivegrid/runtime-worker deploy --prod")
|
|
.replace("deploy_mcp deploy_portal_controller deploy_worker", "deploy_mcp deploy_worker")
|
|
.replace("mcp portal-domain-controller worker", "mcp worker")
|
|
.replace('return "$status"', "return 0")
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /only one locked hivegrid-pnpm-store-v2 invocation of scripts\/ci\/deploy-production-closures\.sh/);
|
|
assert.match(result.failures.join("\n"), /inherit production-deps-bundle and the helper must deploy the exact filtered closure without locked metadata/);
|
|
assert.match(result.failures.join("\n"), /include its task and target once, collect its PID in run_wave/);
|
|
});
|
|
|
|
it("fails closed when the warm-store fallback is broad, repeated or sentinel-based", () => {
|
|
const fixture = createFixture();
|
|
const helperPath = path.join(fixture, "scripts/ci/deploy-production-closures.sh");
|
|
fs.writeFileSync(
|
|
helperPath,
|
|
fs
|
|
.readFileSync(helperPath, "utf8")
|
|
.replace(
|
|
'for target in "${targets[@]}"; do\n rm -rf -- "/prod/${target}"\ndone',
|
|
"rm -rf -- /prod/*",
|
|
)
|
|
.replace("if pnpm fetch --frozen-lockfile; then", "lockfile_hash=sentinel\npnpm fetch --frozen-lockfile\nif pnpm fetch --frozen-lockfile || true; then"),
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /clean only all five exact \/prod targets, fetch once, retry one full wave/);
|
|
});
|
|
|
|
it("fails closed when controller metadata exclusion or SBOM generation becomes best-effort", () => {
|
|
const fixture = createFixture();
|
|
const dockerfilePath = path.join(fixture, "Dockerfile");
|
|
fs.writeFileSync(
|
|
dockerfilePath,
|
|
fs
|
|
.readFileSync(dockerfilePath, "utf8")
|
|
.replace(
|
|
"COPY --exclude=**/*.map --exclude=**/*.d.ts --from=portal-domain-controller-production-deps",
|
|
"COPY --from=portal-domain-controller-production-deps"
|
|
)
|
|
.replace(
|
|
"node /myapp/scripts/generate-sbom.mjs\n\nFROM production-deps-bundle AS worker-production-deps",
|
|
"node /myapp/scripts/generate-sbom.mjs || true\n\nFROM production-deps-bundle AS worker-production-deps"
|
|
)
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /final dependency COPY must use both --exclude/);
|
|
assert.match(result.failures.join("\n"), /contain only its target-specific installed-tree SBOM/);
|
|
});
|
|
|
|
it("fails closed when the controller HEALTHCHECK polling cadence regresses", () => {
|
|
const fixture = createFixture();
|
|
const dockerfilePath = path.join(fixture, "Dockerfile");
|
|
fs.writeFileSync(
|
|
dockerfilePath,
|
|
fs
|
|
.readFileSync(dockerfilePath, "utf8")
|
|
.replace(
|
|
"HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --start-interval=2s --retries=3 \\\n CMD node apps/portal-domain-controller/dist/deploy-health.js",
|
|
"HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 \\\n CMD node apps/portal-domain-controller/dist/deploy-health.js"
|
|
)
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /use a 2s start interval with the 30s\/3 steady budget/);
|
|
});
|
|
|
|
it("fails closed when provider-qualified routing is bypassed or image publication omits the controller", () => {
|
|
const fixture = createFixture();
|
|
const traefikPath = path.join(fixture, "apps/portal-domain-controller/src/traefik.ts");
|
|
fs.writeFileSync(
|
|
traefikPath,
|
|
fs.readFileSync(traefikPath, "utf8").replace("` service: ${JSON.stringify(backendService)}`", '` url: "http://attacker:8080"`')
|
|
);
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
fs.writeFileSync(workflowPath, fs.readFileSync(workflowPath, "utf8").replaceAll(" hocuspocus portal-domain-controller; do", " hocuspocus; do"));
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /provider-qualified backend service/);
|
|
assert.match(result.failures.join("\n"), /image-target loop/);
|
|
});
|
|
|
|
it("fails closed when persisted route reconciliation no longer precedes queue startup", () => {
|
|
const fixture = createFixture();
|
|
const indexPath = path.join(fixture, "apps/portal-domain-controller/src/index.ts");
|
|
fs.writeFileSync(
|
|
indexPath,
|
|
fs
|
|
.readFileSync(indexPath, "utf8")
|
|
.replace("await writer.reconcilePersistedDomainConfigs();", "void writer.reconcilePersistedDomainConfigs();")
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /trust only known active config/);
|
|
});
|
|
|
|
it("fails closed when startup reconciliation trusts active state without known config", () => {
|
|
const fixture = createFixture();
|
|
const traefikPath = path.join(fixture, "apps/portal-domain-controller/src/traefik.ts");
|
|
fs.writeFileSync(
|
|
traefikPath,
|
|
fs
|
|
.readFileSync(traefikPath, "utf8")
|
|
.replace("if (await fileMatches(configPath, legacyActive)) expected = active;", "expected = active;")
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /trust only known active config/);
|
|
});
|
|
|
|
it("fails closed when tombstone priority can outrank a rollback active route", () => {
|
|
const fixture = createFixture();
|
|
const traefikPath = path.join(fixture, "apps/portal-domain-controller/src/traefik.ts");
|
|
fs.writeFileSync(
|
|
traefikPath,
|
|
fs
|
|
.readFileSync(traefikPath, "utf8")
|
|
.replace('Buffer.byteLength(exactHostRule, "utf8") - 1', 'Buffer.byteLength(exactHostRule, "utf8") + 1')
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /below legacy exact-rule priority/);
|
|
});
|
|
|
|
it("fails closed when workflow identity or helper runtime checks are removed", () => {
|
|
const fixture = createFixture();
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
fs.writeFileSync(
|
|
workflowPath,
|
|
fs.readFileSync(workflowPath, "utf8").replaceAll("COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID", "REMOVED_STAGING_CONTROLLER_UUID")
|
|
);
|
|
const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs");
|
|
fs.writeFileSync(
|
|
helperPath,
|
|
fs
|
|
.readFileSync(helperPath, "utf8")
|
|
.replaceAll("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE", "REMOVED_BACKEND_SERVICE")
|
|
.replaceAll("/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload", "/unsafe/reload")
|
|
);
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /source the staging controller UUID/);
|
|
assert.match(result.failures.join("\n"), /live narrow mounts, mesh-only metrics, least-privilege env, matching web\/controller CNAME/);
|
|
});
|
|
|
|
it("fails closed when a controller command loses its explicit environment CNAME pin", () => {
|
|
const fixture = createFixture();
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
const workflowLines = fs.readFileSync(workflowPath, "utf8").split("\n");
|
|
const invocationLine = workflowLines.findIndex((line) => line.includes("coolify-portal-domain-controller.mjs assert-runtime"));
|
|
assert.notEqual(invocationLine, -1);
|
|
workflowLines[invocationLine + 1] = workflowLines[invocationLine + 1].replace("staging", "production");
|
|
fs.writeFileSync(workflowPath, workflowLines.join("\n"));
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /staging as its explicit fail-closed first argument/);
|
|
});
|
|
|
|
it("fails closed when the Docker smoke omits the narrow reload sentinel contract", () => {
|
|
const fixture = createFixture();
|
|
const smokePath = path.join(fixture, "scripts/docker-portal-domain-controller-runtime-smoke.sh");
|
|
fs.writeFileSync(
|
|
smokePath,
|
|
fs
|
|
.readFileSync(smokePath, "utf8")
|
|
.replaceAll("/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload", "/unsafe/reload")
|
|
.replace('-e PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE="hivegrid-portal@docker"', "")
|
|
);
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /sibling reload sentinel in a daemon-visible volume after initial cleanup/);
|
|
});
|
|
|
|
it("fails closed when the orchestrated Docker smoke omits the controller runtime", () => {
|
|
const fixture = createFixture();
|
|
const suitePath = path.join(fixture, "scripts/ci/docker-smoke-suite.sh");
|
|
fs.writeFileSync(
|
|
suitePath,
|
|
fs
|
|
.readFileSync(suitePath, "utf8")
|
|
.replace('bash scripts/docker-portal-domain-controller-runtime-smoke.sh "$PORTAL_CONTROLLER_IMAGE"', "true")
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /build and run the portal-domain controller image/);
|
|
});
|
|
|
|
it("allows only the reviewed flock process boundary and requires flock in the image", () => {
|
|
const processFixture = createFixture();
|
|
const indexPath = path.join(processFixture, "apps/portal-domain-controller/src/index.ts");
|
|
fs.appendFileSync(indexPath, '\nimport { exec } from "node:child_process";\nvoid exec;\n');
|
|
const processResult = verifyPortalDomainControllerDeploy(processFixture);
|
|
assert.match(processResult.failures.join("\n"), /non-flock process-execution capability/);
|
|
|
|
const imageFixture = createFixture();
|
|
const dockerfilePath = path.join(imageFixture, "Dockerfile");
|
|
fs.writeFileSync(
|
|
dockerfilePath,
|
|
fs
|
|
.readFileSync(dockerfilePath, "utf8")
|
|
.replace("apt-get install -y --no-install-recommends util-linux", "apt-get install -y --no-install-recommends openssl")
|
|
);
|
|
const imageResult = verifyPortalDomainControllerDeploy(imageFixture);
|
|
assert.match(imageResult.failures.join("\n"), /install util-linux/);
|
|
});
|
|
|
|
it("fails closed when the controller is not drained before web cutover", () => {
|
|
const fixture = createFixture();
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
const workflow = fs.readFileSync(workflowPath, "utf8");
|
|
const drain = "node scripts/coolify-portal-domain-controller.mjs stop-and-drain";
|
|
const webTrigger = 'node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID"';
|
|
fs.writeFileSync(
|
|
workflowPath,
|
|
workflow.replace(drain, "CONTROLLER_DRAIN_PLACEHOLDER").replace(webTrigger, drain).replace("CONTROLLER_DRAIN_PLACEHOLDER", webTrigger)
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /drain the old controller before web cutover/);
|
|
});
|
|
|
|
it("fails closed when stale stopped status can skip the controller stop request", () => {
|
|
const fixture = createFixture();
|
|
const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs");
|
|
fs.writeFileSync(
|
|
helperPath,
|
|
fs
|
|
.readFileSync(helperPath, "utf8")
|
|
.replace(
|
|
' await coolify(`${resourcePath}/stop`, { method: "POST" });',
|
|
' if (!STOPPED_RUNTIME_STATUSES.test(normalizeValue(controller?.status))) {\n await coolify(`${resourcePath}/stop`, { method: "POST" });\n }'
|
|
)
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /revision-bound stop plus continuous legacy-safe drain/);
|
|
});
|
|
|
|
it("fails closed when stopped status is accepted while mesh health remains reachable", () => {
|
|
const fixture = createFixture();
|
|
const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs");
|
|
fs.writeFileSync(
|
|
helperPath,
|
|
fs.readFileSync(helperPath, "utf8").replace(
|
|
'STOPPED_RUNTIME_STATUSES.test(normalizeValue(current?.status)) && !afterStop.reachable',
|
|
'STOPPED_RUNTIME_STATUSES.test(normalizeValue(current?.status))'
|
|
)
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /revision-bound stop plus continuous legacy-safe drain/);
|
|
});
|
|
|
|
it("fails closed when shutdown removes mesh health before queue and lock drainage", () => {
|
|
const fixture = createFixture();
|
|
const indexPath = path.join(fixture, "apps/portal-domain-controller/src/index.ts");
|
|
fs.writeFileSync(
|
|
indexPath,
|
|
fs
|
|
.readFileSync(indexPath, "utf8")
|
|
.replace(" await queue.close();\n await claimLock.close();\n await http.close();", " await http.close();\n await queue.close();\n await claimLock.close();")
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /close queue, then claim lock, then the mesh health server/);
|
|
});
|
|
|
|
it("fails closed when the bootstrap drain fence is shorter than the legacy hard shutdown", () => {
|
|
const fixture = createFixture();
|
|
const helperPath = path.join(fixture, "scripts/coolify-portal-domain-controller.mjs");
|
|
fs.writeFileSync(
|
|
helperPath,
|
|
fs.readFileSync(helperPath, "utf8").replace("CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 31_000", "CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 29_000")
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /continuous legacy-safe drain/);
|
|
});
|
|
|
|
it("fails closed when the controller release is pinned before paired web proof", () => {
|
|
const fixture = createFixture();
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
const workflow = fs.readFileSync(workflowPath, "utf8");
|
|
const setRelease = "node scripts/coolify-portal-domain-controller.mjs set-release";
|
|
const webProof = "node scripts/coolify-deploy.mjs assert-web-surface-release";
|
|
fs.writeFileSync(
|
|
workflowPath,
|
|
workflow.replace(setRelease, "SET_RELEASE_PLACEHOLDER").replace(webProof, setRelease).replace("SET_RELEASE_PLACEHOLDER", webProof)
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /prove both web surfaces and worker-general, then pin and start/);
|
|
});
|
|
|
|
it("fails closed when the controller release is pinned before worker-general health proof", () => {
|
|
const fixture = createFixture();
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
const workflow = fs.readFileSync(workflowPath, "utf8");
|
|
const setRelease = "node scripts/coolify-portal-domain-controller.mjs set-release";
|
|
const dependentHealthGate = "One or more Coolify staging dependent resources failed health checks";
|
|
fs.writeFileSync(
|
|
workflowPath,
|
|
workflow.replace(setRelease, "SET_RELEASE_PLACEHOLDER").replace(dependentHealthGate, setRelease).replace("SET_RELEASE_PLACEHOLDER", dependentHealthGate)
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /prove both web surfaces and worker-general, then pin and start/);
|
|
});
|
|
|
|
it("fails closed when pre-drain recovery is not armed from a validated release snapshot", () => {
|
|
const fixture = createFixture();
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
fs.writeFileSync(
|
|
workflowPath,
|
|
fs
|
|
.readFileSync(workflowPath, "utf8")
|
|
.replace("coolify-portal-domain-controller.mjs snapshot-release", "coolify-portal-domain-controller.mjs removed-release-snapshot")
|
|
.replace("STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true", "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=false"),
|
|
);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /snapshot-release/);
|
|
assert.match(result.failures.join("\n"), /arm before drain/);
|
|
});
|
|
|
|
it("fails closed when recovery masks failures or reuses the rejected image tag", () => {
|
|
const fixture = createFixture();
|
|
const workflowPath = path.join(fixture, ".gitea/workflows/deploy.yml");
|
|
const workflow = fs.readFileSync(workflowPath, "utf8");
|
|
const recoveryStepIndex = workflow.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure");
|
|
assert.notEqual(recoveryStepIndex, -1);
|
|
const beforeRecovery = workflow.slice(0, recoveryStepIndex);
|
|
const recovery = workflow
|
|
.slice(recoveryStepIndex)
|
|
.replace('"$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"', '"$IMAGE_TAG"')
|
|
.replace(
|
|
'RECOVERY_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")"',
|
|
'RECOVERY_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" || true)"',
|
|
);
|
|
fs.writeFileSync(workflowPath, beforeRecovery + recovery);
|
|
|
|
const result = verifyPortalDomainControllerDeploy(fixture);
|
|
assert.match(result.failures.join("\n"), /always-guarded previous-release set\/trigger\/wait\/assert recovery/);
|
|
});
|
|
});
|
|
|
|
function createFixture() {
|
|
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "portal-domain-controller-deploy-"));
|
|
fixtures.push(fixture);
|
|
for (const relativePath of requiredPaths) {
|
|
const source = path.join(repoRoot, relativePath);
|
|
const target = path.join(fixture, relativePath);
|
|
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
fs.cpSync(source, target, { recursive: true });
|
|
}
|
|
return fixture;
|
|
}
|