Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+532
@@ -0,0 +1,532 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const REQUIRED_RUNTIME_DEPENDENCIES = ["bullmq", "ioredis", "pino", "prom-client"];
|
||||
const FORBIDDEN_RUNTIME_DEPENDENCIES = ["@prisma/client", "pg", "prisma", "ssh2", "dockerode", "@aws-sdk/client-s3"];
|
||||
const DEPLOY_TARGET_LOOP_PATTERN = /for TARGET in ([^;\n]+); do/gu;
|
||||
|
||||
export function verifyPortalDomainControllerDeploy(rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")) {
|
||||
const passes = [];
|
||||
const failures = [];
|
||||
const check = (condition, success, failure) => {
|
||||
(condition ? passes : failures).push(condition ? success : failure);
|
||||
};
|
||||
const read = (relativePath) => fs.readFileSync(path.join(rootDir, relativePath), "utf8");
|
||||
const parse = (relativePath) => JSON.parse(read(relativePath));
|
||||
|
||||
let dockerfile;
|
||||
let deployWorkflow;
|
||||
let ciWorkflow;
|
||||
let dockerSmokeSuite;
|
||||
let productionClosureHelper;
|
||||
let coolifyHelper;
|
||||
let runtimeSmoke;
|
||||
let appManifest;
|
||||
let runtimeManifest;
|
||||
let controllerBuild;
|
||||
let deployHealthSource;
|
||||
try {
|
||||
dockerfile = read("Dockerfile");
|
||||
deployWorkflow = read(".gitea/workflows/deploy.yml");
|
||||
ciWorkflow = read(".gitea/workflows/ci.yml");
|
||||
dockerSmokeSuite = read("scripts/ci/docker-smoke-suite.sh");
|
||||
productionClosureHelper = read("scripts/ci/deploy-production-closures.sh");
|
||||
coolifyHelper = read("scripts/coolify-portal-domain-controller.mjs");
|
||||
runtimeSmoke = read("scripts/docker-portal-domain-controller-runtime-smoke.sh");
|
||||
appManifest = parse("apps/portal-domain-controller/package.json");
|
||||
runtimeManifest = parse("runtime/portal-domain-controller/package.json");
|
||||
controllerBuild = read("apps/portal-domain-controller/scripts/build.mjs");
|
||||
deployHealthSource = read("apps/portal-domain-controller/src/deploy-health.ts");
|
||||
} catch (error) {
|
||||
return { passes, failures: [`controller deploy input missing or malformed: ${error instanceof Error ? error.message : String(error)}`] };
|
||||
}
|
||||
|
||||
check(
|
||||
appManifest.name === "@hivegrid/portal-domain-controller" && appManifest.private === true,
|
||||
"controller uses a private @hivegrid workspace package",
|
||||
"apps/portal-domain-controller must be the private @hivegrid/portal-domain-controller package"
|
||||
);
|
||||
|
||||
const runtimeDependencies = Object.keys(runtimeManifest.dependencies ?? {}).sort();
|
||||
check(
|
||||
REQUIRED_RUNTIME_DEPENDENCIES.every((name) => runtimeDependencies.includes(name)),
|
||||
"runtime manifest declares the complete external dependency closure",
|
||||
`runtime/portal-domain-controller is missing required dependencies: ${REQUIRED_RUNTIME_DEPENDENCIES.filter((name) => !runtimeDependencies.includes(name)).join(", ")}`
|
||||
);
|
||||
check(
|
||||
FORBIDDEN_RUNTIME_DEPENDENCIES.every((name) => !runtimeDependencies.includes(name)),
|
||||
"runtime manifest excludes database, SSH, Docker and storage clients",
|
||||
`controller runtime contains forbidden dependencies: ${FORBIDDEN_RUNTIME_DEPENDENCIES.filter((name) => runtimeDependencies.includes(name)).join(", ")}`
|
||||
);
|
||||
|
||||
const runtimeBody = dockerTargetBody(dockerfile, "portal-domain-controller");
|
||||
const productionBundleBody = dockerTargetBody(dockerfile, "production-deps-bundle");
|
||||
const controllerProductionDepsBody = dockerTargetBody(dockerfile, "portal-domain-controller-production-deps");
|
||||
const productionBundleCommands = dockerRunCommands(productionBundleBody);
|
||||
const controllerProductionDepsCommands = dockerRunCommands(controllerProductionDepsBody);
|
||||
const expectedBundleCommand =
|
||||
"RUN --mount=type=cache,id=hivegrid-pnpm-store-v2,target=/pnpm/store,sharing=locked bash scripts/ci/deploy-production-closures.sh";
|
||||
const expectedControllerDeploy =
|
||||
"pnpm --offline --config.inject-workspace-packages=true --filter @hivegrid/runtime-portal-domain-controller deploy --prod --no-optional /prod/portal-domain-controller";
|
||||
const controllerDeployBody = normalizeCommand(shellFunctionBody(productionClosureHelper, "deploy_portal_controller"));
|
||||
const helperTasks = shellArrayItems(productionClosureHelper, "tasks");
|
||||
const helperTargets = shellArrayItems(productionClosureHelper, "targets");
|
||||
const runWaveBody = normalizeCommand(shellFunctionBody(productionClosureHelper, "run_wave"));
|
||||
const normalizedProductionClosureHelper = normalizeCommand(productionClosureHelper);
|
||||
const expectedControllerSbom =
|
||||
"RUN SBOM_TARGET=portal-domain-controller SBOM_ROOT_PATH=/prod/portal-domain-controller SBOM_NODE_MODULES_PATH=/prod/portal-domain-controller/node_modules SBOM_OUTPUT_PATH=/prod/portal-domain-controller/sbom-pnpm.json node /myapp/scripts/generate-sbom.mjs";
|
||||
check(Boolean(runtimeBody), "Dockerfile defines the controller image", "Dockerfile is missing FROM base AS portal-domain-controller");
|
||||
check(
|
||||
/^FROM\s+build\s+AS\s+production-deps-bundle\s*$/imu.test(dockerfile) &&
|
||||
productionBundleCommands.length === 1 &&
|
||||
productionBundleCommands[0] === expectedBundleCommand,
|
||||
"Docker build delegates the common locked warm store directly to the closure helper",
|
||||
"Dockerfile must build production-deps-bundle from build with only one locked hivegrid-pnpm-store-v2 invocation of scripts/ci/deploy-production-closures.sh"
|
||||
);
|
||||
check(
|
||||
/^FROM\s+production-deps-bundle\s+AS\s+portal-domain-controller-production-deps\s*$/imu.test(dockerfile) &&
|
||||
controllerDeployBody === expectedControllerDeploy &&
|
||||
!/prune_runtime_metadata|find\s+[^\n]*node_modules[^\n]*(?:\*\.map|\*\.d\.ts)/u.test(productionClosureHelper),
|
||||
"controller production closure inherits the common bundle without metadata work under the store lock",
|
||||
"controller production dependency stage must inherit production-deps-bundle and the helper must deploy the exact filtered closure without locked metadata find/delete work"
|
||||
);
|
||||
check(
|
||||
helperTasks.filter((task) => task === "deploy_portal_controller").length === 1 &&
|
||||
helperTargets.filter((target) => target === "portal-domain-controller").length === 1 &&
|
||||
/local -a pids=\(\)/u.test(runWaveBody) &&
|
||||
/"\$task"\s*&/u.test(runWaveBody) &&
|
||||
/pids\+=\("\$!"\)/u.test(runWaveBody) &&
|
||||
/if wait "\$\{pids\[\$index\]\}"; then/u.test(runWaveBody) &&
|
||||
/if \[\[ "\$status" -eq 0 \]\]; then status="\$child_status"/u.test(runWaveBody) &&
|
||||
/return "\$status"/u.test(runWaveBody) &&
|
||||
!hasFailOpenBypass(productionClosureHelper),
|
||||
"controller closure participates in every fully reaped concurrent wave and propagates failure",
|
||||
"controller production closure helper must include its task and target once, collect its PID in run_wave, await every PID and return the first failure"
|
||||
);
|
||||
const initialWaveIndex = normalizedProductionClosureHelper.indexOf("run_wave offline-first || initial_status=$?");
|
||||
const cleanupIndex = normalizedProductionClosureHelper.indexOf('for target in "${targets[@]}"; do rm -rf -- "/prod/${target}" done');
|
||||
const fetchIndex = normalizedProductionClosureHelper.indexOf("if pnpm fetch --frozen-lockfile; then");
|
||||
const retryWaveIndex = normalizedProductionClosureHelper.indexOf("run_wave after-fetch || retry_status=$?");
|
||||
check(
|
||||
initialWaveIndex >= 0 &&
|
||||
cleanupIndex > initialWaveIndex &&
|
||||
fetchIndex > cleanupIndex &&
|
||||
retryWaveIndex > fetchIndex &&
|
||||
(normalizedProductionClosureHelper.match(/\brun_wave\s+(?:offline-first|after-fetch)\b/gu) ?? []).length === 2 &&
|
||||
(normalizedProductionClosureHelper.match(/\bpnpm\s+fetch\s+--frozen-lockfile\b/gu) ?? []).length === 1 &&
|
||||
(normalizedProductionClosureHelper.match(/\brm\s+-rf\b/gu) ?? []).length === 1 &&
|
||||
/if \[\[ "\$initial_status" -eq 0 \]\]; then[\s\S]*exit 0[\s\S]*fi/u.test(normalizedProductionClosureHelper) &&
|
||||
/else fetch_status=\$\?[\s\S]*exit "\$fetch_status"[\s\S]*fi/u.test(normalizedProductionClosureHelper) &&
|
||||
/if \[\[ "\$retry_status" -ne 0 \]\]; then[\s\S]*exit "\$retry_status"[\s\S]*fi/u.test(normalizedProductionClosureHelper) &&
|
||||
!/sentinel|sha256sum|lockfile[_-]?hash/iu.test(productionClosureHelper) &&
|
||||
!/\brm\s+-rf\s+(?:--\s+)?["']?\/prod["']?(?:\s|;|$)/u.test(productionClosureHelper) &&
|
||||
!/\brm\s+-rf\s+(?:--\s+)?["']?\/prod\/\*/u.test(productionClosureHelper) &&
|
||||
!hasFailOpenBypass(productionClosureHelper),
|
||||
"controller closure uses the one-fetch exact-target fallback and one complete retry without a sentinel",
|
||||
"controller production closure helper must try offline first, clean only all five exact /prod targets, fetch once, retry one full wave and propagate fetch or retry failure without a sentinel"
|
||||
);
|
||||
check(
|
||||
controllerProductionDepsCommands.length === 1 && controllerProductionDepsCommands[0] === expectedControllerSbom,
|
||||
"controller dependency stage generates only its target-specific installed-tree SBOM",
|
||||
"controller production dependency stage must contain only its target-specific installed-tree SBOM"
|
||||
);
|
||||
check(
|
||||
runtimeBody?.includes(
|
||||
"COPY --exclude=**/*.map --exclude=**/*.d.ts --from=portal-domain-controller-production-deps /prod/portal-domain-controller/node_modules /myapp/node_modules"
|
||||
),
|
||||
"controller final dependency COPY excludes maps and declarations",
|
||||
"controller final dependency COPY must use both --exclude=**/*.map and --exclude=**/*.d.ts"
|
||||
);
|
||||
check(
|
||||
Boolean(runtimeBody) &&
|
||||
/^USER\s+node\s*$/imu.test(runtimeBody) &&
|
||||
/^CMD\s+\["node",\s*"apps\/portal-domain-controller\/dist\/index\.js"\]\s*$/imu.test(runtimeBody),
|
||||
"controller image runs the compiled app as the non-root node user",
|
||||
"controller image must run its compiled entry point as USER node"
|
||||
);
|
||||
check(
|
||||
Boolean(runtimeBody) &&
|
||||
/PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR="\/data\/coolify\/proxy\/dynamic\/hivegrid-portal-domains"/u.test(runtimeBody) &&
|
||||
/PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE="\/data\/coolify\/proxy\/dynamic\/\.hivegrid-portal-domains\.reload"/u.test(runtimeBody) &&
|
||||
!/ENV\s+PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE=/u.test(runtimeBody),
|
||||
"controller image fixes only the narrow Traefik paths and leaves routing environment-specific",
|
||||
"controller image must fix the dedicated config/sentinel paths without baking an environment-specific backend service"
|
||||
);
|
||||
check(
|
||||
Boolean(runtimeBody) &&
|
||||
/HEALTHCHECK\s+--interval=30s\b[^\n]*--start-interval=2s\b[^\n]*--retries=3\b[\s\S]*CMD node apps\/portal-domain-controller\/dist\/deploy-health\.js "\$APP_REVISION"/u.test(runtimeBody) &&
|
||||
/src\/deploy-health\.ts/u.test(controllerBuild) &&
|
||||
/\/healthz/u.test(deployHealthSource) &&
|
||||
/\/readyz/u.test(deployHealthSource) &&
|
||||
/health\.revision !== expectedRevision/u.test(deployHealthSource) &&
|
||||
/\/myapp\/\.image-revision/u.test(deployHealthSource) &&
|
||||
/hivegrid-portal-domain-controller/u.test(deployHealthSource) &&
|
||||
/org\.opencontainers\.image\.sbom="\/myapp\/sbom-pnpm\.json"/u.test(runtimeBody),
|
||||
"controller image carries a fast-start compiled healthz, readyz, immutable revision and SBOM contract",
|
||||
"controller image must use a 2s start interval with the 30s/3 steady budget and run its compiled healthz, readyz and baked APP_REVISION probe with the target-specific SBOM"
|
||||
);
|
||||
check(
|
||||
Boolean(runtimeBody) && !/(?:DATABASE_URL|DIRECT_URL|COOLIFY_TOKEN|CLOUDFLARE_API|DOCKER_HOST|SSH_PRIVATE_KEY)\s*=/u.test(runtimeBody),
|
||||
"controller image declares no privileged runtime credentials",
|
||||
"controller final image must not declare database, Coolify, Cloudflare, Docker or SSH credentials"
|
||||
);
|
||||
|
||||
const sourceRoot = path.join(rootDir, "apps/portal-domain-controller/src");
|
||||
const productionSources = listFiles(sourceRoot).filter((file) => file.endsWith(".ts") && !file.endsWith(".test.ts"));
|
||||
const productionSourceEntries = productionSources.map((file) => ({ file, source: fs.readFileSync(file, "utf8") }));
|
||||
const productionSource = productionSourceEntries.map(({ source }) => source).join("\n");
|
||||
const claimLockSource = productionSourceEntries.find(({ file }) => path.basename(file) === "claim-lock.ts")?.source ?? "";
|
||||
const indexSource = productionSourceEntries.find(({ file }) => path.basename(file) === "index.ts")?.source ?? "";
|
||||
const processExecutionOutsideClaimLock = productionSourceEntries.some(
|
||||
({ file, source }) => path.basename(file) !== "claim-lock.ts" && /node:child_process/u.test(source)
|
||||
);
|
||||
const reviewedFlockBoundary =
|
||||
/^import \{ spawn \} from "node:child_process";$/mu.test(claimLockSource) &&
|
||||
/^const FLOCK_BINARY = "\/usr\/bin\/flock";$/mu.test(claimLockSource) &&
|
||||
/spawn\(binary, \["--exclusive", "--wait", timeoutSeconds, "3"\], \{\s*stdio: \["ignore", "ignore", "ignore", fd\],\s*\}\)/u.test(claimLockSource) &&
|
||||
!/\b(?:exec|execFile|fork|spawnSync)\s*\(/u.test(claimLockSource) &&
|
||||
!/\bshell\s*:/u.test(claimLockSource);
|
||||
check(
|
||||
!/(?:from\s+["'](?:@hivegrid\/db|@prisma\/client|pg|ssh2|dockerode))/u.test(productionSource) && !processExecutionOutsideClaimLock && reviewedFlockBoundary,
|
||||
"controller source has no database, SSH or Docker client boundary and process execution is limited to reviewed /usr/bin/flock fencing",
|
||||
"controller source imports a forbidden database, SSH, Docker or non-flock process-execution capability"
|
||||
);
|
||||
check(
|
||||
Boolean(runtimeBody) && /apt-get install -y --no-install-recommends util-linux/u.test(runtimeBody),
|
||||
"controller image installs the kernel flock binary explicitly",
|
||||
"controller final image must install util-linux for the reviewed /usr/bin/flock fence"
|
||||
);
|
||||
check(
|
||||
/createHmac/u.test(productionSource) && /timingSafeEqual/u.test(productionSource) && /contentHash/u.test(productionSource),
|
||||
"controller queue protocol is content-hashed and HMAC authenticated",
|
||||
"controller contracts must bind a content hash to a timing-safe HMAC signature"
|
||||
);
|
||||
check(
|
||||
/O_NOFOLLOW/u.test(productionSource) && /await rename\(/u.test(productionSource) && /await handle\.sync\(\)/u.test(productionSource),
|
||||
"Traefik writes reject symlinks and use fsync plus atomic rename",
|
||||
"Traefik writer must use O_NOFOLLOW, fsync and same-directory atomic rename"
|
||||
);
|
||||
check(
|
||||
/normalizePortalDomainBackendService/u.test(productionSource) &&
|
||||
/PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE/u.test(productionSource) &&
|
||||
/service:\s*\$\{JSON\.stringify\(backendService\)\}/u.test(productionSource) &&
|
||||
/Host\(\\`\$\{hostname\}\\`\)/u.test(productionSource),
|
||||
"Traefik renderer uses an exact Host matcher and provider-qualified environment routing",
|
||||
"Traefik renderer must use exact Host() and a validated provider-qualified backend service"
|
||||
);
|
||||
const persistedConfigReconcileIndex = indexSource.indexOf("await writer.reconcilePersistedDomainConfigs()");
|
||||
const queueStartupIndex = indexSource.indexOf("await createDomainQueueRuntime(");
|
||||
check(
|
||||
/service: "noop@internal"/u.test(productionSource) &&
|
||||
/"418": 404/u.test(productionSource) &&
|
||||
/tls: \{\}/u.test(productionSource) &&
|
||||
/Math\.max\(1, Buffer\.byteLength\(exactHostRule, "utf8"\) - 1\)/u.test(productionSource) &&
|
||||
/let expected = tombstone/u.test(productionSource) &&
|
||||
/candidate\.state\.action !== "deprovision"/u.test(productionSource) &&
|
||||
/if \(await fileMatches\(configPath, active\)\) continue/u.test(productionSource) &&
|
||||
/if \(await fileMatches\(configPath, legacyActive\)\) expected = active/u.test(productionSource) &&
|
||||
/renderTraefikDomainConfigForKey/u.test(productionSource) &&
|
||||
/if \(persisted\.length > 0\) await this\.#triggerReload\(guard\)/u.test(productionSource) &&
|
||||
persistedConfigReconcileIndex >= 0 &&
|
||||
queueStartupIndex > persistedConfigReconcileIndex,
|
||||
"controller renders rollback-compatible exact-host backend-free 404 tombstones and safely reconciles every persisted route before queue startup",
|
||||
"controller must render the reviewed noop 418-to-404 TLS tombstone below legacy exact-rule priority, trust only known active config, reconcile deprovisioned states, and replay one reload before consuming commands"
|
||||
);
|
||||
const shutdownQueueCloseIndex = indexSource.lastIndexOf("await queue.close()");
|
||||
const shutdownClaimLockCloseIndex = indexSource.lastIndexOf("await claimLock.close()");
|
||||
const shutdownHttpCloseIndex = indexSource.lastIndexOf("await http.close()");
|
||||
check(
|
||||
shutdownQueueCloseIndex >= 0 &&
|
||||
shutdownClaimLockCloseIndex > shutdownQueueCloseIndex &&
|
||||
shutdownHttpCloseIndex > shutdownClaimLockCloseIndex,
|
||||
"controller shutdown removes mesh health only after queue and claim-lock drainage",
|
||||
"controller shutdown must close queue, then claim lock, then the mesh health server"
|
||||
);
|
||||
check(
|
||||
/PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME/u.test(productionSource) &&
|
||||
/#openSafeReloadTrigger/u.test(productionSource) &&
|
||||
/handle\.truncate\(0\)/u.test(productionSource) &&
|
||||
/handle\.sync\(\)/u.test(productionSource),
|
||||
"Traefik mutations touch the narrow watched-root sentinel through a safe file handle",
|
||||
"Traefik mutations must safely truncate, write and fsync the dedicated reload sentinel"
|
||||
);
|
||||
|
||||
check(
|
||||
coolifyHelper.includes("/data/coolify/proxy/dynamic/hivegrid-portal-domains") &&
|
||||
coolifyHelper.includes("/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload") &&
|
||||
coolifyHelper.includes("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE") &&
|
||||
coolifyHelper.includes("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT") &&
|
||||
coolifyHelper.includes("PORTAL_CUSTOM_DOMAIN_CNAME_TARGET") &&
|
||||
coolifyHelper.includes('staging: "staging-domains.portal.hivegrid.de"') &&
|
||||
coolifyHelper.includes('production: "domains.portal.hivegrid.de"') &&
|
||||
coolifyHelper.includes('staging: "10.42.0.3:19120:8080"') &&
|
||||
coolifyHelper.includes('production: "10.42.0.8:19120:8080"') &&
|
||||
coolifyHelper.includes("ports_mappings") &&
|
||||
coolifyHelper.includes("PORTAL_DOMAIN_ORCHESTRATION_REQUIRED") &&
|
||||
coolifyHelper.includes("CONTROLLER_ALLOWED_ENV_KEYS") &&
|
||||
coolifyHelper.includes("file_storages") &&
|
||||
coolifyHelper.includes("persistent_storages") &&
|
||||
coolifyHelper.includes("/storages") &&
|
||||
coolifyHelper.includes("timingSafeEqual") &&
|
||||
coolifyHelper.includes("running:healthy") &&
|
||||
coolifyHelper.includes("custom_healthcheck_found") &&
|
||||
coolifyHelper.includes("apps/portal-domain-controller/dist/deploy-health.js") &&
|
||||
coolifyHelper.includes('health_check_type: "cmd"') &&
|
||||
coolifyHelper.includes('custom_healthcheck_found: "false"') &&
|
||||
/await coolify\(`\$\{resourcePath\}\/stop`, \{ method: "POST" \}\);/u.test(coolifyHelper) &&
|
||||
!/if \(!STOPPED_RUNTIME_STATUSES\.test\([^)]*controller\?\.status/u.test(coolifyHelper) &&
|
||||
coolifyHelper.includes("STOPPED_RUNTIME_STATUSES") &&
|
||||
coolifyHelper.includes("controllerHealthUrlForEnvironment") &&
|
||||
coolifyHelper.includes("probeControllerHealth") &&
|
||||
coolifyHelper.includes("beforeStop.reachable") &&
|
||||
coolifyHelper.includes("beforeStop.revisionBound") &&
|
||||
coolifyHelper.includes("!afterStop.reachable") &&
|
||||
coolifyHelper.includes("CONTROLLER_DRAIN_CONFIRMATIONS") &&
|
||||
coolifyHelper.includes("CONTROLLER_LEGACY_SHUTDOWN_FENCE_MS = 31_000") &&
|
||||
coolifyHelper.includes("performance.now() - drainEvidenceSinceMs >= requiredDrainFenceMs") &&
|
||||
coolifyHelper.includes("snapshotRelease") &&
|
||||
coolifyHelper.includes("FULL_GIT_SHA_PATTERN"),
|
||||
"Coolify helper fail-closes live mount shapes, mesh-only metrics, least-privilege env, CNAME, HMAC, worker drain, revision snapshot, command health and full-SHA release contracts",
|
||||
"Coolify helper must enforce live narrow mounts, mesh-only metrics, least-privilege env, matching web/controller CNAME, worker-general HMAC, revision-bound stop plus continuous legacy-safe drain, revision snapshot, command health and full-SHA pinning"
|
||||
);
|
||||
|
||||
const ciJob = yamlJobBody(ciWorkflow, "portal-domain-controller");
|
||||
const ciCommands = [
|
||||
"pnpm --filter @hivegrid/portal-domain-controller typecheck",
|
||||
"pnpm --filter @hivegrid/portal-domain-controller test",
|
||||
"pnpm --filter @hivegrid/portal-domain-controller build",
|
||||
"pnpm run verify:runtime-closures -- --runtime=portal-domain-controller",
|
||||
"pnpm run verify:portal-domain-controller-deploy",
|
||||
];
|
||||
check(
|
||||
Boolean(ciJob) && ciCommands.every((command) => ciJob.includes(command)),
|
||||
"controller CI job typechecks, tests, builds and verifies its runtime/deploy contracts",
|
||||
"Gitea CI controller job is missing a required focused command"
|
||||
);
|
||||
check(
|
||||
ciWorkflow.includes("bash scripts/ci/docker-smoke-suite.sh all") &&
|
||||
/PORTAL_CONTROLLER_IMAGE="hivegrid-portal-domain-controller:smoke-\$\{DOCKER_SMOKE_RUN_ID\}"/u.test(dockerSmokeSuite) &&
|
||||
/build_root_target portal-domain-controller "\$PORTAL_CONTROLLER_IMAGE"/u.test(dockerSmokeSuite) &&
|
||||
/bash scripts\/docker-portal-domain-controller-runtime-smoke\.sh "\$PORTAL_CONTROLLER_IMAGE"/u.test(dockerSmokeSuite),
|
||||
"Docker smoke builds and exercises the controller image",
|
||||
"Gitea Docker smoke must build and run the portal-domain controller image"
|
||||
);
|
||||
const smokeCleanupIndex = runtimeSmoke.indexOf("docker_smoke_install_cleanup_traps cleanup\ncleanup");
|
||||
const smokeReloadVolumeIndex = runtimeSmoke.indexOf('docker volume create "$RELOAD_TRIGGER_VOLUME"');
|
||||
check(
|
||||
smokeCleanupIndex >= 0 &&
|
||||
smokeReloadVolumeIndex > smokeCleanupIndex &&
|
||||
runtimeSmoke.includes('-v "$RELOAD_TRIGGER_VOLUME:/sentinel"') &&
|
||||
runtimeSmoke.includes('fs.writeFileSync(file, "ready\\n", { mode: 0o600, flag: "wx" })') &&
|
||||
runtimeSmoke.includes('-v "$CONFIG_VOLUME:/data/coolify/proxy/dynamic/hivegrid-portal-domains"') &&
|
||||
runtimeSmoke.includes(
|
||||
'--mount "type=volume,source=$RELOAD_TRIGGER_VOLUME,destination=/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload,volume-subpath=.hivegrid-portal-domains.reload"'
|
||||
) &&
|
||||
runtimeSmoke.includes('-e PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE="hivegrid-portal@docker"') &&
|
||||
runtimeSmoke.includes('-e PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT="https"') &&
|
||||
runtimeSmoke.includes('node apps/portal-domain-controller/dist/deploy-health.js "$EXPECTED_REVISION"') &&
|
||||
runtimeSmoke.includes('docker_smoke_expect_docker_exec_rejection "portal-controller:reject-foreign-revision"') &&
|
||||
!runtimeSmoke.includes("$SMOKE_TEMP_DIR:/data/coolify/proxy/dynamic") &&
|
||||
!runtimeSmoke.includes("$RELOAD_TRIGGER_FILE:/data/coolify/proxy/dynamic"),
|
||||
"Docker smoke creates a daemon-visible volume sentinel, mounts it narrowly and proves the revision-bound health CLI",
|
||||
"Docker smoke must create the sibling reload sentinel in a daemon-visible volume after initial cleanup, mount only the narrow file subpath, set validated routing and prove the health CLI"
|
||||
);
|
||||
|
||||
const targetLoops = [...deployWorkflow.matchAll(DEPLOY_TARGET_LOOP_PATTERN)].map((match) => match[1]?.trim().split(/\s+/u) ?? []);
|
||||
check(
|
||||
targetLoops.length >= 5 && targetLoops.every((targets) => targets.includes("portal-domain-controller")),
|
||||
"staging provenance, build, signing and alias loops include the controller image",
|
||||
"every deploy image-target loop must include portal-domain-controller"
|
||||
);
|
||||
check(
|
||||
deployWorkflow.includes("node scripts/verify-portal-domain-controller-deploy.mjs"),
|
||||
"deploy preflight runs the controller-specific verifier",
|
||||
"deploy preflight must run scripts/verify-portal-domain-controller-deploy.mjs"
|
||||
);
|
||||
check(
|
||||
deployWorkflow.includes("node --check scripts/coolify-portal-domain-controller.mjs") &&
|
||||
deployWorkflow.includes("COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID: ${{ vars.COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID }}") &&
|
||||
!deployWorkflow.includes("COOLIFY_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID") &&
|
||||
!deployWorkflow.includes("PRODUCTION_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"),
|
||||
"deploy preflight requires the explicit staging controller resource without activating production",
|
||||
"deploy workflow must syntax-check the helper, source the staging controller UUID and leave production inactive until its resource is provisioned"
|
||||
);
|
||||
check(
|
||||
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs assert-runtime") === 1 &&
|
||||
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs snapshot-release") === 1 &&
|
||||
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs stop-and-drain") === 1 &&
|
||||
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs set-release") === 2 &&
|
||||
countOccurrences(deployWorkflow, "coolify-portal-domain-controller.mjs assert-release") === 2,
|
||||
"staging preflights, snapshots, drains, pins, verifies and can recover the controller release",
|
||||
"the staging rollout must run controller assert-runtime, snapshot-release and stop-and-drain once plus set-release/assert-release for recovery and acceptance"
|
||||
);
|
||||
const environmentPinnedCommands = ["assert-runtime", "snapshot-release", "stop-and-drain", "set-release", "assert-release"];
|
||||
check(
|
||||
environmentPinnedCommands.every((command) => {
|
||||
const expected = new Set(["set-release", "assert-release"]).has(command) ? ["staging", "staging"] : ["staging"];
|
||||
return JSON.stringify(controllerInvocationEnvironments(deployWorkflow, command)) === JSON.stringify(expected);
|
||||
}),
|
||||
"controller preflight, release mutation and release proof pin the exact staging CNAME environment",
|
||||
"every active controller command must take staging as its explicit fail-closed first argument"
|
||||
);
|
||||
check(
|
||||
deployWorkflow.includes('coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"') &&
|
||||
/coolify-deploy\.mjs wait\s+\\?\s*"\$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"/u.test(deployWorkflow) &&
|
||||
!deployWorkflow.includes('coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"') &&
|
||||
!deployWorkflow.includes('coolify-deploy.mjs wait "$PRODUCTION_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"'),
|
||||
"staging controller deploy uses an explicit trigger and wait chain without production activation",
|
||||
"the staging controller must be triggered and waited independently while production remains inactive"
|
||||
);
|
||||
const webReleaseProofIndex = deployWorkflow.indexOf("coolify-deploy.mjs assert-web-surface-release");
|
||||
const controllerDrainIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs stop-and-drain");
|
||||
const webTriggerIndex = deployWorkflow.indexOf('coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID"');
|
||||
const workerGeneralProofIndex = deployWorkflow.indexOf('"$STAGING_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general"');
|
||||
const dependentHealthGateIndex = deployWorkflow.indexOf("One or more Coolify staging dependent resources failed health checks");
|
||||
const controllerReleaseIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs set-release");
|
||||
const controllerTriggerIndex = deployWorkflow.indexOf(
|
||||
'coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"',
|
||||
controllerReleaseIndex
|
||||
);
|
||||
check(
|
||||
controllerDrainIndex >= 0 &&
|
||||
webTriggerIndex > controllerDrainIndex &&
|
||||
webReleaseProofIndex > webTriggerIndex &&
|
||||
workerGeneralProofIndex > webReleaseProofIndex &&
|
||||
dependentHealthGateIndex > workerGeneralProofIndex &&
|
||||
controllerReleaseIndex > dependentHealthGateIndex &&
|
||||
controllerTriggerIndex > controllerReleaseIndex,
|
||||
"staging drains the old controller before web cutover and starts it only after the paired worker-general revision is healthy",
|
||||
"the staging rollout must drain the old controller before web cutover, prove both web surfaces and worker-general, then pin and start the exact controller revision"
|
||||
);
|
||||
|
||||
const snapshotIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs snapshot-release");
|
||||
const recoveryArmedIndex = deployWorkflow.indexOf("STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true");
|
||||
const controllerAcceptedIndex = deployWorkflow.indexOf("STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true");
|
||||
const normalControllerProofIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs assert-release", controllerReleaseIndex);
|
||||
const recoveryStepIndex = deployWorkflow.indexOf("Restore previous Coolify staging portal-domain-controller release on rollout failure");
|
||||
const recoveryReleaseIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs set-release", recoveryStepIndex);
|
||||
const recoveryTriggerIndex = deployWorkflow.indexOf('coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"', recoveryStepIndex);
|
||||
const recoveryWaitIndex = deployWorkflow.indexOf('coolify-deploy.mjs wait', recoveryStepIndex);
|
||||
const recoveryProofIndex = deployWorkflow.indexOf("coolify-portal-domain-controller.mjs assert-release", recoveryStepIndex);
|
||||
const nextStepIndex = deployWorkflow.indexOf("\n - name:", recoveryStepIndex + 1);
|
||||
const recoveryStep = recoveryStepIndex >= 0 ? deployWorkflow.slice(recoveryStepIndex, nextStepIndex >= 0 ? nextStepIndex : undefined) : "";
|
||||
const recoveryReleaseBlock =
|
||||
recoveryReleaseIndex >= 0 && recoveryTriggerIndex > recoveryReleaseIndex
|
||||
? deployWorkflow.slice(recoveryReleaseIndex, recoveryTriggerIndex)
|
||||
: "";
|
||||
check(
|
||||
snapshotIndex >= 0 &&
|
||||
snapshotIndex < controllerDrainIndex &&
|
||||
recoveryArmedIndex > snapshotIndex &&
|
||||
recoveryArmedIndex < controllerDrainIndex &&
|
||||
normalControllerProofIndex > controllerTriggerIndex &&
|
||||
controllerAcceptedIndex > normalControllerProofIndex &&
|
||||
recoveryStepIndex > controllerAcceptedIndex &&
|
||||
recoveryReleaseIndex > recoveryStepIndex &&
|
||||
recoveryTriggerIndex > recoveryReleaseIndex &&
|
||||
recoveryWaitIndex > recoveryTriggerIndex &&
|
||||
recoveryProofIndex > recoveryWaitIndex &&
|
||||
/^Restore previous Coolify staging portal-domain-controller release on rollout failure\s*$/mu.test(recoveryStep) &&
|
||||
/^\s*if:\s*always\(\)\s*$/mu.test(recoveryStep) &&
|
||||
/DEPLOY_ACTION" != "stage/.test(recoveryStep) &&
|
||||
/STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED:-false/.test(recoveryStep) &&
|
||||
/STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED:-false/.test(recoveryStep) &&
|
||||
/"\$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"/.test(recoveryReleaseBlock) &&
|
||||
!/"\$IMAGE_TAG"/.test(recoveryReleaseBlock) &&
|
||||
/coolify-deploy\.mjs\s+wait\s+\\?\s*"\$RECOVERY_DEPLOYMENT_UUID"/.test(recoveryStep) &&
|
||||
!/^\s*continue-on-error:\s*true\s*$/mu.test(recoveryStep) &&
|
||||
!/\bset\s+(?:\+e|\+o\s+errexit)\b/.test(recoveryStep) &&
|
||||
!/(^|[^|])\|\|\s*(?:true|:|exit\s+0)(?=[\s;)"']|$)/m.test(recoveryStep),
|
||||
"staging always restores the validated pre-drain controller release unless the new controller release is accepted",
|
||||
"staging must arm before drain, accept only after controller proof, then run an always-guarded previous-release set/trigger/wait/assert recovery"
|
||||
);
|
||||
|
||||
return { passes, failures };
|
||||
}
|
||||
|
||||
export function formatPortalDomainControllerDeployResult(result) {
|
||||
return [
|
||||
...result.passes.map((message) => `PASS ${message}`),
|
||||
...result.failures.map((message) => `FAIL ${message}`),
|
||||
`Summary: ${result.passes.length} passed, ${result.failures.length} failed`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function dockerTargetBody(dockerfile, target) {
|
||||
const match = new RegExp(`^FROM\\s+[^\\n]+\\s+AS\\s+${target}\\s*$`, "imu").exec(dockerfile);
|
||||
if (!match || match.index === undefined) return "";
|
||||
const remainder = dockerfile.slice(match.index + match[0].length);
|
||||
const nextStage = remainder.search(/^FROM\s+/imu);
|
||||
return nextStage >= 0 ? remainder.slice(0, nextStage) : remainder;
|
||||
}
|
||||
|
||||
function dockerRunCommands(stageBody) {
|
||||
return stageBody
|
||||
.replace(/\\\r?\n\s*/gu, " ")
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => normalizeCommand(line))
|
||||
.filter((line) => line.startsWith("RUN "));
|
||||
}
|
||||
|
||||
function shellFunctionBody(source, functionName) {
|
||||
const match = new RegExp(`^${escapeRegExp(functionName)}\\(\\)\\s*\\{\\s*$`, "mu").exec(source);
|
||||
if (!match || match.index === undefined) return "";
|
||||
const remainder = source.slice(match.index + match[0].length);
|
||||
const closingBrace = remainder.search(/^\}\s*$/mu);
|
||||
return closingBrace >= 0 ? remainder.slice(0, closingBrace) : "";
|
||||
}
|
||||
|
||||
function shellArrayItems(source, variableName) {
|
||||
const match = new RegExp(`^${escapeRegExp(variableName)}=\\(([^)]*)\\)\\s*$`, "mu").exec(source);
|
||||
return match?.[1]?.trim().split(/\s+/u).filter(Boolean) ?? [];
|
||||
}
|
||||
|
||||
function normalizeCommand(command) {
|
||||
return command.replace(/\\\r?\n\s*/gu, " ").replace(/\s+/gu, " ").trim();
|
||||
}
|
||||
|
||||
function hasFailOpenBypass(source) {
|
||||
return /\bset\s+\+e\b|\|\|\s*(?:true|:|exit\s+0)(?=[\s;)}"']|$)|;\s*(?:true|exit\s+0)(?=[\s;)}"']|$)/mu.test(source);
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
function yamlJobBody(workflow, jobName) {
|
||||
const match = new RegExp(`^ ${jobName}:\\s*$`, "mu").exec(workflow);
|
||||
if (!match || match.index === undefined) return "";
|
||||
const remainder = workflow.slice(match.index + match[0].length);
|
||||
const nextJob = remainder.search(/^ [A-Za-z0-9_-]+:\s*$/mu);
|
||||
return nextJob >= 0 ? remainder.slice(0, nextJob) : remainder;
|
||||
}
|
||||
|
||||
function listFiles(root) {
|
||||
if (!fs.existsSync(root)) return [];
|
||||
return fs.readdirSync(root, { withFileTypes: true }).flatMap((entry) => {
|
||||
const absolute = path.join(root, entry.name);
|
||||
return entry.isDirectory() ? listFiles(absolute) : [absolute];
|
||||
});
|
||||
}
|
||||
|
||||
function countOccurrences(value, search) {
|
||||
return value.split(search).length - 1;
|
||||
}
|
||||
|
||||
function controllerInvocationEnvironments(workflow, command) {
|
||||
const lines = workflow.split(/\r?\n/u);
|
||||
const marker = `coolify-portal-domain-controller.mjs ${command}`;
|
||||
return lines.flatMap((line, index) => {
|
||||
if (!line.includes(marker)) return [];
|
||||
const nextLine = lines[index + 1]?.trim().replace(/\\$/u, "").trim();
|
||||
return nextLine ? [nextLine] : [];
|
||||
});
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
const result = verifyPortalDomainControllerDeploy();
|
||||
console.log(formatPortalDomainControllerDeployResult(result));
|
||||
if (result.failures.length > 0) process.exitCode = 1;
|
||||
}
|
||||
Reference in new issue
Block a user