533 lines
34 KiB
JavaScript
533 lines
34 KiB
JavaScript
#!/usr/bin/env node
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const REQUIRED_RUNTIME_DEPENDENCIES = ["bullmq", "ioredis", "pino", "prom-client"];
|
|
const FORBIDDEN_RUNTIME_DEPENDENCIES = ["@prisma/client", "pg", "prisma", "ssh2", "dockerode", "@aws-sdk/client-s3"];
|
|
const DEPLOY_TARGET_LOOP_PATTERN = /for TARGET in ([^;\n]+); do/gu;
|
|
|
|
export function verifyPortalDomainControllerDeploy(rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")) {
|
|
const passes = [];
|
|
const failures = [];
|
|
const check = (condition, success, failure) => {
|
|
(condition ? passes : failures).push(condition ? success : failure);
|
|
};
|
|
const read = (relativePath) => fs.readFileSync(path.join(rootDir, relativePath), "utf8");
|
|
const parse = (relativePath) => JSON.parse(read(relativePath));
|
|
|
|
let dockerfile;
|
|
let deployWorkflow;
|
|
let ciWorkflow;
|
|
let dockerSmokeSuite;
|
|
let productionClosureHelper;
|
|
let coolifyHelper;
|
|
let runtimeSmoke;
|
|
let appManifest;
|
|
let runtimeManifest;
|
|
let controllerBuild;
|
|
let deployHealthSource;
|
|
try {
|
|
dockerfile = read("Dockerfile");
|
|
deployWorkflow = read(".gitea/workflows/deploy.yml");
|
|
ciWorkflow = read(".gitea/workflows/ci.yml");
|
|
dockerSmokeSuite = read("scripts/ci/docker-smoke-suite.sh");
|
|
productionClosureHelper = read("scripts/ci/deploy-production-closures.sh");
|
|
coolifyHelper = read("scripts/coolify-portal-domain-controller.mjs");
|
|
runtimeSmoke = read("scripts/docker-portal-domain-controller-runtime-smoke.sh");
|
|
appManifest = parse("apps/portal-domain-controller/package.json");
|
|
runtimeManifest = parse("runtime/portal-domain-controller/package.json");
|
|
controllerBuild = read("apps/portal-domain-controller/scripts/build.mjs");
|
|
deployHealthSource = read("apps/portal-domain-controller/src/deploy-health.ts");
|
|
} catch (error) {
|
|
return { passes, failures: [`controller deploy input missing or malformed: ${error instanceof Error ? error.message : String(error)}`] };
|
|
}
|
|
|
|
check(
|
|
appManifest.name === "@hivegrid/portal-domain-controller" && appManifest.private === true,
|
|
"controller uses a private @hivegrid workspace package",
|
|
"apps/portal-domain-controller must be the private @hivegrid/portal-domain-controller package"
|
|
);
|
|
|
|
const runtimeDependencies = Object.keys(runtimeManifest.dependencies ?? {}).sort();
|
|
check(
|
|
REQUIRED_RUNTIME_DEPENDENCIES.every((name) => runtimeDependencies.includes(name)),
|
|
"runtime manifest declares the complete external dependency closure",
|
|
`runtime/portal-domain-controller is missing required dependencies: ${REQUIRED_RUNTIME_DEPENDENCIES.filter((name) => !runtimeDependencies.includes(name)).join(", ")}`
|
|
);
|
|
check(
|
|
FORBIDDEN_RUNTIME_DEPENDENCIES.every((name) => !runtimeDependencies.includes(name)),
|
|
"runtime manifest excludes database, SSH, Docker and storage clients",
|
|
`controller runtime contains forbidden dependencies: ${FORBIDDEN_RUNTIME_DEPENDENCIES.filter((name) => runtimeDependencies.includes(name)).join(", ")}`
|
|
);
|
|
|
|
const runtimeBody = dockerTargetBody(dockerfile, "portal-domain-controller");
|
|
const productionBundleBody = dockerTargetBody(dockerfile, "production-deps-bundle");
|
|
const controllerProductionDepsBody = dockerTargetBody(dockerfile, "portal-domain-controller-production-deps");
|
|
const productionBundleCommands = dockerRunCommands(productionBundleBody);
|
|
const controllerProductionDepsCommands = dockerRunCommands(controllerProductionDepsBody);
|
|
const expectedBundleCommand =
|
|
"RUN --mount=type=cache,id=hivegrid-pnpm-store-v2,target=/pnpm/store,sharing=locked bash scripts/ci/deploy-production-closures.sh";
|
|
const expectedControllerDeploy =
|
|
"pnpm --offline --config.inject-workspace-packages=true --filter @hivegrid/runtime-portal-domain-controller deploy --prod --no-optional /prod/portal-domain-controller";
|
|
const controllerDeployBody = normalizeCommand(shellFunctionBody(productionClosureHelper, "deploy_portal_controller"));
|
|
const helperTasks = shellArrayItems(productionClosureHelper, "tasks");
|
|
const helperTargets = shellArrayItems(productionClosureHelper, "targets");
|
|
const runWaveBody = normalizeCommand(shellFunctionBody(productionClosureHelper, "run_wave"));
|
|
const normalizedProductionClosureHelper = normalizeCommand(productionClosureHelper);
|
|
const expectedControllerSbom =
|
|
"RUN SBOM_TARGET=portal-domain-controller SBOM_ROOT_PATH=/prod/portal-domain-controller SBOM_NODE_MODULES_PATH=/prod/portal-domain-controller/node_modules SBOM_OUTPUT_PATH=/prod/portal-domain-controller/sbom-pnpm.json node /myapp/scripts/generate-sbom.mjs";
|
|
check(Boolean(runtimeBody), "Dockerfile defines the controller image", "Dockerfile is missing FROM base AS portal-domain-controller");
|
|
check(
|
|
/^FROM\s+build\s+AS\s+production-deps-bundle\s*$/imu.test(dockerfile) &&
|
|
productionBundleCommands.length === 1 &&
|
|
productionBundleCommands[0] === expectedBundleCommand,
|
|
"Docker build delegates the common locked warm store directly to the closure helper",
|
|
"Dockerfile must build production-deps-bundle from build with only one locked hivegrid-pnpm-store-v2 invocation of scripts/ci/deploy-production-closures.sh"
|
|
);
|
|
check(
|
|
/^FROM\s+production-deps-bundle\s+AS\s+portal-domain-controller-production-deps\s*$/imu.test(dockerfile) &&
|
|
controllerDeployBody === expectedControllerDeploy &&
|
|
!/prune_runtime_metadata|find\s+[^\n]*node_modules[^\n]*(?:\*\.map|\*\.d\.ts)/u.test(productionClosureHelper),
|
|
"controller production closure inherits the common bundle without metadata work under the store lock",
|
|
"controller production dependency stage must inherit production-deps-bundle and the helper must deploy the exact filtered closure without locked metadata find/delete work"
|
|
);
|
|
check(
|
|
helperTasks.filter((task) => task === "deploy_portal_controller").length === 1 &&
|
|
helperTargets.filter((target) => target === "portal-domain-controller").length === 1 &&
|
|
/local -a pids=\(\)/u.test(runWaveBody) &&
|
|
/"\$task"\s*&/u.test(runWaveBody) &&
|
|
/pids\+=\("\$!"\)/u.test(runWaveBody) &&
|
|
/if wait "\$\{pids\[\$index\]\}"; then/u.test(runWaveBody) &&
|
|
/if \[\[ "\$status" -eq 0 \]\]; then status="\$child_status"/u.test(runWaveBody) &&
|
|
/return "\$status"/u.test(runWaveBody) &&
|
|
!hasFailOpenBypass(productionClosureHelper),
|
|
"controller closure participates in every fully reaped concurrent wave and propagates failure",
|
|
"controller production closure helper must include its task and target once, collect its PID in run_wave, await every PID and return the first failure"
|
|
);
|
|
const initialWaveIndex = normalizedProductionClosureHelper.indexOf("run_wave offline-first || initial_status=$?");
|
|
const cleanupIndex = normalizedProductionClosureHelper.indexOf('for target in "${targets[@]}"; do rm -rf -- "/prod/${target}" done');
|
|
const fetchIndex = normalizedProductionClosureHelper.indexOf("if pnpm fetch --frozen-lockfile; then");
|
|
const retryWaveIndex = normalizedProductionClosureHelper.indexOf("run_wave after-fetch || retry_status=$?");
|
|
check(
|
|
initialWaveIndex >= 0 &&
|
|
cleanupIndex > initialWaveIndex &&
|
|
fetchIndex > cleanupIndex &&
|
|
retryWaveIndex > fetchIndex &&
|
|
(normalizedProductionClosureHelper.match(/\brun_wave\s+(?:offline-first|after-fetch)\b/gu) ?? []).length === 2 &&
|
|
(normalizedProductionClosureHelper.match(/\bpnpm\s+fetch\s+--frozen-lockfile\b/gu) ?? []).length === 1 &&
|
|
(normalizedProductionClosureHelper.match(/\brm\s+-rf\b/gu) ?? []).length === 1 &&
|
|
/if \[\[ "\$initial_status" -eq 0 \]\]; then[\s\S]*exit 0[\s\S]*fi/u.test(normalizedProductionClosureHelper) &&
|
|
/else fetch_status=\$\?[\s\S]*exit "\$fetch_status"[\s\S]*fi/u.test(normalizedProductionClosureHelper) &&
|
|
/if \[\[ "\$retry_status" -ne 0 \]\]; then[\s\S]*exit "\$retry_status"[\s\S]*fi/u.test(normalizedProductionClosureHelper) &&
|
|
!/sentinel|sha256sum|lockfile[_-]?hash/iu.test(productionClosureHelper) &&
|
|
!/\brm\s+-rf\s+(?:--\s+)?["']?\/prod["']?(?:\s|;|$)/u.test(productionClosureHelper) &&
|
|
!/\brm\s+-rf\s+(?:--\s+)?["']?\/prod\/\*/u.test(productionClosureHelper) &&
|
|
!hasFailOpenBypass(productionClosureHelper),
|
|
"controller closure uses the one-fetch exact-target fallback and one complete retry without a sentinel",
|
|
"controller production closure helper must try offline first, clean only all five exact /prod targets, fetch once, retry one full wave and propagate fetch or retry failure without a sentinel"
|
|
);
|
|
check(
|
|
controllerProductionDepsCommands.length === 1 && controllerProductionDepsCommands[0] === expectedControllerSbom,
|
|
"controller dependency stage generates only its target-specific installed-tree SBOM",
|
|
"controller production dependency stage must contain only its target-specific installed-tree SBOM"
|
|
);
|
|
check(
|
|
runtimeBody?.includes(
|
|
"COPY --exclude=**/*.map --exclude=**/*.d.ts --from=portal-domain-controller-production-deps /prod/portal-domain-controller/node_modules /myapp/node_modules"
|
|
),
|
|
"controller final dependency COPY excludes maps and declarations",
|
|
"controller final dependency COPY must use both --exclude=**/*.map and --exclude=**/*.d.ts"
|
|
);
|
|
check(
|
|
Boolean(runtimeBody) &&
|
|
/^USER\s+node\s*$/imu.test(runtimeBody) &&
|
|
/^CMD\s+\["node",\s*"apps\/portal-domain-controller\/dist\/index\.js"\]\s*$/imu.test(runtimeBody),
|
|
"controller image runs the compiled app as the non-root node user",
|
|
"controller image must run its compiled entry point as USER node"
|
|
);
|
|
check(
|
|
Boolean(runtimeBody) &&
|
|
/PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR="\/data\/coolify\/proxy\/dynamic\/hivegrid-portal-domains"/u.test(runtimeBody) &&
|
|
/PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE="\/data\/coolify\/proxy\/dynamic\/\.hivegrid-portal-domains\.reload"/u.test(runtimeBody) &&
|
|
!/ENV\s+PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE=/u.test(runtimeBody),
|
|
"controller image fixes only the narrow Traefik paths and leaves routing environment-specific",
|
|
"controller image must fix the dedicated config/sentinel paths without baking an environment-specific backend service"
|
|
);
|
|
check(
|
|
Boolean(runtimeBody) &&
|
|
/HEALTHCHECK\s+--interval=30s\b[^\n]*--start-interval=2s\b[^\n]*--retries=3\b[\s\S]*CMD node apps\/portal-domain-controller\/dist\/deploy-health\.js "\$APP_REVISION"/u.test(runtimeBody) &&
|
|
/src\/deploy-health\.ts/u.test(controllerBuild) &&
|
|
/\/healthz/u.test(deployHealthSource) &&
|
|
/\/readyz/u.test(deployHealthSource) &&
|
|
/health\.revision !== expectedRevision/u.test(deployHealthSource) &&
|
|
/\/myapp\/\.image-revision/u.test(deployHealthSource) &&
|
|
/hivegrid-portal-domain-controller/u.test(deployHealthSource) &&
|
|
/org\.opencontainers\.image\.sbom="\/myapp\/sbom-pnpm\.json"/u.test(runtimeBody),
|
|
"controller image carries a fast-start compiled healthz, readyz, immutable revision and SBOM contract",
|
|
"controller image must use a 2s start interval with the 30s/3 steady budget and run its compiled healthz, readyz and baked APP_REVISION probe with the target-specific SBOM"
|
|
);
|
|
check(
|
|
Boolean(runtimeBody) && !/(?:DATABASE_URL|DIRECT_URL|COOLIFY_TOKEN|CLOUDFLARE_API|DOCKER_HOST|SSH_PRIVATE_KEY)\s*=/u.test(runtimeBody),
|
|
"controller image declares no privileged runtime credentials",
|
|
"controller final image must not declare database, Coolify, Cloudflare, Docker or SSH credentials"
|
|
);
|
|
|
|
const sourceRoot = path.join(rootDir, "apps/portal-domain-controller/src");
|
|
const productionSources = listFiles(sourceRoot).filter((file) => file.endsWith(".ts") && !file.endsWith(".test.ts"));
|
|
const productionSourceEntries = productionSources.map((file) => ({ file, source: fs.readFileSync(file, "utf8") }));
|
|
const productionSource = productionSourceEntries.map(({ source }) => source).join("\n");
|
|
const claimLockSource = productionSourceEntries.find(({ file }) => path.basename(file) === "claim-lock.ts")?.source ?? "";
|
|
const indexSource = productionSourceEntries.find(({ file }) => path.basename(file) === "index.ts")?.source ?? "";
|
|
const processExecutionOutsideClaimLock = productionSourceEntries.some(
|
|
({ file, source }) => path.basename(file) !== "claim-lock.ts" && /node:child_process/u.test(source)
|
|
);
|
|
const reviewedFlockBoundary =
|
|
/^import \{ spawn \} from "node:child_process";$/mu.test(claimLockSource) &&
|
|
/^const FLOCK_BINARY = "\/usr\/bin\/flock";$/mu.test(claimLockSource) &&
|
|
/spawn\(binary, \["--exclusive", "--wait", timeoutSeconds, "3"\], \{\s*stdio: \["ignore", "ignore", "ignore", fd\],\s*\}\)/u.test(claimLockSource) &&
|
|
!/\b(?:exec|execFile|fork|spawnSync)\s*\(/u.test(claimLockSource) &&
|
|
!/\bshell\s*:/u.test(claimLockSource);
|
|
check(
|
|
!/(?:from\s+["'](?:@hivegrid\/db|@prisma\/client|pg|ssh2|dockerode))/u.test(productionSource) && !processExecutionOutsideClaimLock && reviewedFlockBoundary,
|
|
"controller source has no database, SSH or Docker client boundary and process execution is limited to reviewed /usr/bin/flock fencing",
|
|
"controller source imports a forbidden database, SSH, Docker or non-flock process-execution capability"
|
|
);
|
|
check(
|
|
Boolean(runtimeBody) && /apt-get install -y --no-install-recommends util-linux/u.test(runtimeBody),
|
|
"controller image installs the kernel flock binary explicitly",
|
|
"controller final image must install util-linux for the reviewed /usr/bin/flock fence"
|
|
);
|
|
check(
|
|
/createHmac/u.test(productionSource) && /timingSafeEqual/u.test(productionSource) && /contentHash/u.test(productionSource),
|
|
"controller queue protocol is content-hashed and HMAC authenticated",
|
|
"controller contracts must bind a content hash to a timing-safe HMAC signature"
|
|
);
|
|
check(
|
|
/O_NOFOLLOW/u.test(productionSource) && /await rename\(/u.test(productionSource) && /await handle\.sync\(\)/u.test(productionSource),
|
|
"Traefik writes reject symlinks and use fsync plus atomic rename",
|
|
"Traefik writer must use O_NOFOLLOW, fsync and same-directory atomic rename"
|
|
);
|
|
check(
|
|
/normalizePortalDomainBackendService/u.test(productionSource) &&
|
|
/PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE/u.test(productionSource) &&
|
|
/service:\s*\$\{JSON\.stringify\(backendService\)\}/u.test(productionSource) &&
|
|
/Host\(\\`\$\{hostname\}\\`\)/u.test(productionSource),
|
|
"Traefik renderer uses an exact Host matcher and provider-qualified environment routing",
|
|
"Traefik renderer must use exact Host() and a validated provider-qualified backend service"
|
|
);
|
|
const persistedConfigReconcileIndex = indexSource.indexOf("await writer.reconcilePersistedDomainConfigs()");
|
|
const queueStartupIndex = indexSource.indexOf("await createDomainQueueRuntime(");
|
|
check(
|
|
/service: "noop@internal"/u.test(productionSource) &&
|
|
/"418": 404/u.test(productionSource) &&
|
|
/tls: \{\}/u.test(productionSource) &&
|
|
/Math\.max\(1, Buffer\.byteLength\(exactHostRule, "utf8"\) - 1\)/u.test(productionSource) &&
|
|
/let expected = tombstone/u.test(productionSource) &&
|
|
/candidate\.state\.action !== "deprovision"/u.test(productionSource) &&
|
|
/if \(await fileMatches\(configPath, active\)\) continue/u.test(productionSource) &&
|
|
/if \(await fileMatches\(configPath, legacyActive\)\) expected = active/u.test(productionSource) &&
|
|
/renderTraefikDomainConfigForKey/u.test(productionSource) &&
|
|
/if \(persisted\.length > 0\) await this\.#triggerReload\(guard\)/u.test(productionSource) &&
|
|
persistedConfigReconcileIndex >= 0 &&
|
|
queueStartupIndex > persistedConfigReconcileIndex,
|
|
"controller renders rollback-compatible exact-host backend-free 404 tombstones and safely reconciles every persisted route before queue startup",
|
|
"controller must render the reviewed noop 418-to-404 TLS tombstone below legacy exact-rule priority, trust only known active config, reconcile deprovisioned states, and replay one reload before consuming commands"
|
|
);
|
|
const shutdownQueueCloseIndex = indexSource.lastIndexOf("await queue.close()");
|
|
const shutdownClaimLockCloseIndex = indexSource.lastIndexOf("await claimLock.close()");
|
|
const shutdownHttpCloseIndex = indexSource.lastIndexOf("await http.close()");
|
|
check(
|
|
shutdownQueueCloseIndex >= 0 &&
|
|
shutdownClaimLockCloseIndex > shutdownQueueCloseIndex &&
|
|
shutdownHttpCloseIndex > shutdownClaimLockCloseIndex,
|
|
"controller shutdown removes mesh health only after queue and claim-lock drainage",
|
|
"controller shutdown must close queue, then claim lock, then the mesh health server"
|
|
);
|
|
check(
|
|
/PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME/u.test(productionSource) &&
|
|
/#openSafeReloadTrigger/u.test(productionSource) &&
|
|
/handle\.truncate\(0\)/u.test(productionSource) &&
|
|
/handle\.sync\(\)/u.test(productionSource),
|
|
"Traefik mutations touch the narrow watched-root sentinel through a safe file handle",
|
|
"Traefik mutations must safely truncate, write and fsync the dedicated reload sentinel"
|
|
);
|
|
|
|
check(
|
|
coolifyHelper.includes("/data/coolify/proxy/dynamic/hivegrid-portal-domains") &&
|
|
coolifyHelper.includes("/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload") &&
|
|
coolifyHelper.includes("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE") &&
|
|
coolifyHelper.includes("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT") &&
|
|
coolifyHelper.includes("PORTAL_CUSTOM_DOMAIN_CNAME_TARGET") &&
|
|
coolifyHelper.includes('staging: "staging-domains.portal.hivegrid.de"') &&
|
|
coolifyHelper.includes('production: "domains.portal.hivegrid.de"') &&
|
|
coolifyHelper.includes('staging: "10.42.0.3:19120:8080"') &&
|
|
coolifyHelper.includes('production: "10.42.0.8:19120:8080"') &&
|
|
coolifyHelper.includes("ports_mappings") &&
|
|
coolifyHelper.includes("PORTAL_DOMAIN_ORCHESTRATION_REQUIRED") &&
|
|
coolifyHelper.includes("CONTROLLER_ALLOWED_ENV_KEYS") &&
|
|
coolifyHelper.includes("file_storages") &&
|
|
coolifyHelper.includes("persistent_storages") &&
|
|
coolifyHelper.includes("/storages") &&
|
|
coolifyHelper.includes("timingSafeEqual") &&
|
|
coolifyHelper.includes("running:healthy") &&
|
|
coolifyHelper.includes("custom_healthcheck_found") &&
|
|
coolifyHelper.includes("apps/portal-domain-controller/dist/deploy-health.js") &&
|
|
coolifyHelper.includes('health_check_type: "cmd"') &&
|
|
coolifyHelper.includes('custom_healthcheck_found: "false"') &&
|
|
/await coolify\(`\$\{resourcePath\}\/stop`, \{ method: "POST" \}\);/u.test(coolifyHelper) &&
|
|
!/if \(!STOPPED_RUNTIME_STATUSES\.test\([^)]*controller\?\.status/u.test(coolifyHelper) &&
|
|
coolifyHelper.includes("STOPPED_RUNTIME_STATUSES") &&
|
|
coolifyHelper.includes("controllerHealthUrlForEnvironment") &&
|
|
coolifyHelper.includes("probeControllerHealth") &&
|
|
coolifyHelper.includes("beforeStop.reachable") &&
|
|
coolifyHelper.includes("beforeStop.revisionBound") &&
|
|
coolifyHelper.includes("!afterStop.reachable") &&
|
|
coolifyHelper.includes("CONTROLLER_DRAIN_CONFIRMATIONS") &&
|
|
coolifyHelper.includes("CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 31_000") &&
|
|
coolifyHelper.includes("performance.now() - drainEvidenceSinceMs >= requiredDrainFenceMs") &&
|
|
coolifyHelper.includes("snapshotRelease") &&
|
|
coolifyHelper.includes("FULL_GIT_SHA_PATTERN"),
|
|
"Coolify helper fail-closes live mount shapes, mesh-only metrics, least-privilege env, CNAME, HMAC, worker drain, revision snapshot, command health and full-SHA release contracts",
|
|
"Coolify helper must enforce live narrow mounts, mesh-only metrics, least-privilege env, matching web/controller CNAME, worker-general HMAC, revision-bound stop plus continuous legacy-safe drain, revision snapshot, command health and full-SHA pinning"
|
|
);
|
|
|
|
const ciJob = yamlJobBody(ciWorkflow, "portal-domain-controller");
|
|
const ciCommands = [
|
|
"pnpm --filter @hivegrid/portal-domain-controller typecheck",
|
|
"pnpm --filter @hivegrid/portal-domain-controller test",
|
|
"pnpm --filter @hivegrid/portal-domain-controller build",
|
|
"pnpm run verify:runtime-closures -- --runtime=portal-domain-controller",
|
|
"pnpm run verify:portal-domain-controller-deploy",
|
|
];
|
|
check(
|
|
Boolean(ciJob) && ciCommands.every((command) => ciJob.includes(command)),
|
|
"controller CI job typechecks, tests, builds and verifies its runtime/deploy contracts",
|
|
"Gitea CI controller job is missing a required focused command"
|
|
);
|
|
check(
|
|
ciWorkflow.includes("bash scripts/ci/docker-smoke-suite.sh all") &&
|
|
/PORTAL_CONTROLLER_IMAGE="hivegrid-portal-domain-controller:smoke-\$\{DOCKER_SMOKE_RUN_ID\}"/u.test(dockerSmokeSuite) &&
|
|
/build_root_target portal-domain-controller "\$PORTAL_CONTROLLER_IMAGE"/u.test(dockerSmokeSuite) &&
|
|
/bash scripts\/docker-portal-domain-controller-runtime-smoke\.sh "\$PORTAL_CONTROLLER_IMAGE"/u.test(dockerSmokeSuite),
|
|
"Docker smoke builds and exercises the controller image",
|
|
"Gitea Docker smoke must build and run the portal-domain controller image"
|
|
);
|
|
const smokeCleanupIndex = runtimeSmoke.indexOf("docker_smoke_install_cleanup_traps cleanup\ncleanup");
|
|
const smokeReloadVolumeIndex = runtimeSmoke.indexOf('docker volume create "$RELOAD_TRIGGER_VOLUME"');
|
|
check(
|
|
smokeCleanupIndex >= 0 &&
|
|
smokeReloadVolumeIndex > smokeCleanupIndex &&
|
|
runtimeSmoke.includes('-v "$RELOAD_TRIGGER_VOLUME:/sentinel"') &&
|
|
runtimeSmoke.includes('fs.writeFileSync(file, "ready\\n", { mode: 0o600, flag: "wx" })') &&
|
|
runtimeSmoke.includes('-v "$CONFIG_VOLUME:/data/coolify/proxy/dynamic/hivegrid-portal-domains"') &&
|
|
runtimeSmoke.includes(
|
|
'--mount "type=volume,source=$RELOAD_TRIGGER_VOLUME,destination=/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload,volume-subpath=.hivegrid-portal-domains.reload"'
|
|
) &&
|
|
runtimeSmoke.includes('-e PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE="hivegrid-portal@docker"') &&
|
|
runtimeSmoke.includes('-e PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT="https"') &&
|
|
runtimeSmoke.includes('node apps/portal-domain-controller/dist/deploy-health.js "$EXPECTED_REVISION"') &&
|
|
runtimeSmoke.includes('docker_smoke_expect_docker_exec_rejection "portal-controller:reject-foreign-revision"') &&
|
|
!runtimeSmoke.includes("$SMOKE_TEMP_DIR:/data/coolify/proxy/dynamic") &&
|
|
!runtimeSmoke.includes("$RELOAD_TRIGGER_FILE:/data/coolify/proxy/dynamic"),
|
|
"Docker smoke creates a daemon-visible volume sentinel, mounts it narrowly and proves the revision-bound health CLI",
|
|
"Docker smoke must create the sibling reload sentinel in a daemon-visible volume after initial cleanup, mount only the narrow file subpath, set validated routing and prove the health CLI"
|
|
);
|
|
|
|
const targetLoops = [...deployWorkflow.matchAll(DEPLOY_TARGET_LOOP_PATTERN)].map((match) => match[1]?.trim().split(/\s+/u) ?? []);
|
|
check(
|
|
targetLoops.length >= 5 && targetLoops.every((targets) => targets.includes("portal-domain-controller")),
|
|
"staging provenance, build, signing and alias loops include the controller image",
|
|
"every deploy image-target loop must include portal-domain-controller"
|
|
);
|
|
check(
|
|
deployWorkflow.includes("node scripts/verify-portal-domain-controller-deploy.mjs"),
|
|
"deploy preflight runs the controller-specific verifier",
|
|
"deploy preflight must run scripts/verify-portal-domain-controller-deploy.mjs"
|
|
);
|
|
check(
|
|
deployWorkflow.includes("node --check scripts/coolify-portal-domain-controller.mjs") &&
|
|
deployWorkflow.includes("COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID: ${{ vars.COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID }}") &&
|
|
!deployWorkflow.includes("COOLIFY_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID") &&
|
|
!deployWorkflow.includes("PRODUCTION_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"),
|
|
"deploy preflight requires the explicit staging controller resource without activating production",
|
|
"deploy workflow must syntax-check the helper, source the staging controller UUID and leave production inactive until its resource is provisioned"
|
|
);
|
|
check(
|
|
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs assert-runtime") === 1 &&
|
|
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs snapshot-release") === 1 &&
|
|
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs stop-and-drain") === 1 &&
|
|
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs set-release") === 2 &&
|
|
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs assert-release") === 2,
|
|
"staging preflights, snapshots, drains, pins, verifies and can recover the controller release",
|
|
"the staging rollout must run controller assert-runtime, snapshot-release and stop-and-drain once plus set-release/assert-release for recovery and acceptance"
|
|
);
|
|
const environmentPinnedCommands = ["assert-runtime", "snapshot-release", "stop-and-drain", "set-release", "assert-release"];
|
|
check(
|
|
environmentPinnedCommands.every((command) => {
|
|
const expected = new Set(["set-release", "assert-release"]).has(command) ? ["staging", "staging"] : ["staging"];
|
|
return JSON.stringify(controllerInvocationEnvironments(deployWorkflow, command)) === JSON.stringify(expected);
|
|
}),
|
|
"controller preflight, release mutation and release proof pin the exact staging CNAME environment",
|
|
"every active controller command must take staging as its explicit fail-closed first argument"
|
|
);
|
|
check(
|
|
deployWorkflow.includes('coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"') &&
|
|
/coolify-deploy\.mjs wait\s+\\?\s*"\$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"/u.test(deployWorkflow) &&
|
|
!deployWorkflow.includes('coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"') &&
|
|
!deployWorkflow.includes('coolify-deploy.mjs wait "$PRODUCTION_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"'),
|
|
"staging controller deploy uses an explicit trigger and wait chain without production activation",
|
|
"the staging controller must be triggered and waited independently while production remains inactive"
|
|
);
|
|
const webReleaseProofIndex = deployWorkflow.indexOf("coolify-deploy.mjs assert-web-surface-release");
|
|
const controllerDrainIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs stop-and-drain");
|
|
const webTriggerIndex = deployWorkflow.indexOf('coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID"');
|
|
const workerGeneralProofIndex = deployWorkflow.indexOf('"$STAGING_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general"');
|
|
const dependentHealthGateIndex = deployWorkflow.indexOf("One or more Coolify staging dependent resources failed health checks");
|
|
const controllerReleaseIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs set-release");
|
|
const controllerTriggerIndex = deployWorkflow.indexOf(
|
|
'coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"',
|
|
controllerReleaseIndex
|
|
);
|
|
check(
|
|
controllerDrainIndex >= 0 &&
|
|
webTriggerIndex > controllerDrainIndex &&
|
|
webReleaseProofIndex > webTriggerIndex &&
|
|
workerGeneralProofIndex > webReleaseProofIndex &&
|
|
dependentHealthGateIndex > workerGeneralProofIndex &&
|
|
controllerReleaseIndex > dependentHealthGateIndex &&
|
|
controllerTriggerIndex > controllerReleaseIndex,
|
|
"staging drains the old controller before web cutover and starts it only after the paired worker-general revision is healthy",
|
|
"the staging rollout must drain the old controller before web cutover, prove both web surfaces and worker-general, then pin and start the exact controller revision"
|
|
);
|
|
|
|
const snapshotIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs snapshot-release");
|
|
const recoveryArmedIndex = deployWorkflow.indexOf("STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true");
|
|
const controllerAcceptedIndex = deployWorkflow.indexOf("STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true");
|
|
const normalControllerProofIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs assert-release", controllerReleaseIndex);
|
|
const recoveryStepIndex = deployWorkflow.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure");
|
|
const recoveryReleaseIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs set-release", recoveryStepIndex);
|
|
const recoveryTriggerIndex = deployWorkflow.indexOf('coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"', recoveryStepIndex);
|
|
const recoveryWaitIndex = deployWorkflow.indexOf('coolify-deploy.mjs wait', recoveryStepIndex);
|
|
const recoveryProofIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs assert-release", recoveryStepIndex);
|
|
const nextStepIndex = deployWorkflow.indexOf("\n - name:", recoveryStepIndex + 1);
|
|
const recoveryStep = recoveryStepIndex >= 0 ? deployWorkflow.slice(recoveryStepIndex, nextStepIndex >= 0 ? nextStepIndex : undefined) : "";
|
|
const recoveryReleaseBlock =
|
|
recoveryReleaseIndex >= 0 && recoveryTriggerIndex > recoveryReleaseIndex
|
|
? deployWorkflow.slice(recoveryReleaseIndex, recoveryTriggerIndex)
|
|
: "";
|
|
check(
|
|
snapshotIndex >= 0 &&
|
|
snapshotIndex < controllerDrainIndex &&
|
|
recoveryArmedIndex > snapshotIndex &&
|
|
recoveryArmedIndex < controllerDrainIndex &&
|
|
normalControllerProofIndex > controllerTriggerIndex &&
|
|
controllerAcceptedIndex > normalControllerProofIndex &&
|
|
recoveryStepIndex > controllerAcceptedIndex &&
|
|
recoveryReleaseIndex > recoveryStepIndex &&
|
|
recoveryTriggerIndex > recoveryReleaseIndex &&
|
|
recoveryWaitIndex > recoveryTriggerIndex &&
|
|
recoveryProofIndex > recoveryWaitIndex &&
|
|
/^Restore previous Coolify staging portal-domain-controller release on rollout failure\s*$/mu.test(recoveryStep) &&
|
|
/^\s*if:\s*always\(\)\s*$/mu.test(recoveryStep) &&
|
|
/DEPLOY_ACTION" != "stage/.test(recoveryStep) &&
|
|
/STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED:-false/.test(recoveryStep) &&
|
|
/STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED:-false/.test(recoveryStep) &&
|
|
/"\$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"/.test(recoveryReleaseBlock) &&
|
|
!/"\$IMAGE_TAG"/.test(recoveryReleaseBlock) &&
|
|
/coolify-deploy\.mjs\s+wait\s+\\?\s*"\$RECOVERY_DEPLOYMENT_UUID"/.test(recoveryStep) &&
|
|
!/^\s*continue-on-error:\s*true\s*$/mu.test(recoveryStep) &&
|
|
!/\bset\s+(?:\+e|\+o\s+errexit)\b/.test(recoveryStep) &&
|
|
!/(^|[^|])\|\|\s*(?:true|:|exit\s+0)(?=[\s;)"']|$)/m.test(recoveryStep),
|
|
"staging always restores the validated pre-drain controller release unless the new controller release is accepted",
|
|
"staging must arm before drain, accept only after controller proof, then run an always-guarded previous-release set/trigger/wait/assert recovery"
|
|
);
|
|
|
|
return { passes, failures };
|
|
}
|
|
|
|
export function formatPortalDomainControllerDeployResult(result) {
|
|
return [
|
|
...result.passes.map((message) => `PASS ${message}`),
|
|
...result.failures.map((message) => `FAIL ${message}`),
|
|
`Summary: ${result.passes.length} passed, ${result.failures.length} failed`,
|
|
].join("\n");
|
|
}
|
|
|
|
function dockerTargetBody(dockerfile, target) {
|
|
const match = new RegExp(`^FROM\\s+[^\\n]+\\s+AS\\s+${target}\\s*$`, "imu").exec(dockerfile);
|
|
if (!match || match.index === undefined) return "";
|
|
const remainder = dockerfile.slice(match.index + match[0].length);
|
|
const nextStage = remainder.search(/^FROM\s+/imu);
|
|
return nextStage >= 0 ? remainder.slice(0, nextStage) : remainder;
|
|
}
|
|
|
|
function dockerRunCommands(stageBody) {
|
|
return stageBody
|
|
.replace(/\\\r?\n\s*/gu, " ")
|
|
.split(/\r?\n/u)
|
|
.map((line) => normalizeCommand(line))
|
|
.filter((line) => line.startsWith("RUN "));
|
|
}
|
|
|
|
function shellFunctionBody(source, functionName) {
|
|
const match = new RegExp(`^${escapeRegExp(functionName)}\\(\\)\\s*\\{\\s*$`, "mu").exec(source);
|
|
if (!match || match.index === undefined) return "";
|
|
const remainder = source.slice(match.index + match[0].length);
|
|
const closingBrace = remainder.search(/^\}\s*$/mu);
|
|
return closingBrace >= 0 ? remainder.slice(0, closingBrace) : "";
|
|
}
|
|
|
|
function shellArrayItems(source, variableName) {
|
|
const match = new RegExp(`^${escapeRegExp(variableName)}=\\(([^)]*)\\)\\s*$`, "mu").exec(source);
|
|
return match?.[1]?.trim().split(/\s+/u).filter(Boolean) ?? [];
|
|
}
|
|
|
|
function normalizeCommand(command) {
|
|
return command.replace(/\\\r?\n\s*/gu, " ").replace(/\s+/gu, " ").trim();
|
|
}
|
|
|
|
function hasFailOpenBypass(source) {
|
|
return /\bset\s+\+e\b|\|\|\s*(?:true|:|exit\s+0)(?=[\s;)}"']|$)|;\s*(?:true|exit\s+0)(?=[\s;)}"']|$)/mu.test(source);
|
|
}
|
|
|
|
function escapeRegExp(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|
|
|
|
function yamlJobBody(workflow, jobName) {
|
|
const match = new RegExp(`^ ${jobName}:\\s*$`, "mu").exec(workflow);
|
|
if (!match || match.index === undefined) return "";
|
|
const remainder = workflow.slice(match.index + match[0].length);
|
|
const nextJob = remainder.search(/^ [A-Za-z0-9_-]+:\s*$/mu);
|
|
return nextJob >= 0 ? remainder.slice(0, nextJob) : remainder;
|
|
}
|
|
|
|
function listFiles(root) {
|
|
if (!fs.existsSync(root)) return [];
|
|
return fs.readdirSync(root, { withFileTypes: true }).flatMap((entry) => {
|
|
const absolute = path.join(root, entry.name);
|
|
return entry.isDirectory() ? listFiles(absolute) : [absolute];
|
|
});
|
|
}
|
|
|
|
function countOccurrences(value, search) {
|
|
return value.split(search).length - 1;
|
|
}
|
|
|
|
function controllerInvocationEnvironments(workflow, command) {
|
|
const lines = workflow.split(/\r?\n/u);
|
|
const marker = `coolify-portal-domain-controller.mjs ${command}`;
|
|
return lines.flatMap((line, index) => {
|
|
if (!line.includes(marker)) return [];
|
|
const nextLine = lines[index + 1]?.trim().replace(/\\$/u, "").trim();
|
|
return nextLine ? [nextLine] : [];
|
|
});
|
|
}
|
|
|
|
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
const result = verifyPortalDomainControllerDeploy();
|
|
console.log(formatPortalDomainControllerDeployResult(result));
|
|
if (result.failures.length > 0) process.exitCode = 1;
|
|
}
|