#!/usr/bin/env node import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { GROUPS, WORKER_PRIVILEGED_ENV_KEYS, buildWorkerDeploymentHealthCommand, desiredWorkerEnv, getWorkerAppConfigDrift, getWorkerEnvDrift, } from "./coolify-ensure-worker-groups.mjs"; const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const DEPLOY_HEALTH_EXPECTED_REVISION = "a".repeat(40); const WORKER_DEPLOY_HEALTH_COMMAND_EXAMPLE = "node apps/worker/dist/deploy-health.js <40-character lowercase Git SHA> "; const MCP_DEPLOY_HEALTH_COMMAND_EXAMPLE = "node apps/mcp/dist/deploy-health.js <40-character lowercase Git SHA>"; const REQUIRED_PACKAGE_SCRIPTS = [ "verify:deploy-workflow", "verify:tenant-policy-contract", "verify:rls-release-readiness", "verify:rls-live", "verify:migrator-role", "verify:docker-baseline", "verify:docker-non-root", "verify:health-baseline", "verify:api-inventory", "verify:openapi-security", "verify:i18n", "verify:phase-c-boundaries", "test:hot-swap", "verify:ui-visual", "verify:docker-runtime-smoke", "verify:docker-api-runtime-smoke", "verify:docker-mcp-runtime-smoke", "verify:docker-worker-runtime-smoke", "verify:docker-hocuspocus-runtime-smoke", "verify:docker-portal-domain-controller-runtime-smoke", "verify:portal-domain-controller-deploy", "verify:production-seed-artifact", "verify:workflow-outbox-native", "verify:plugin-runtime-live", "db:roles:provision", "db:migrate:deploy", ]; const WORKFLOW_OUTBOX_NATIVE_SCRIPT = "verify:workflow-outbox-native"; const REQUIRED_WORKFLOW_OUTBOX_NATIVE_TEST_TARGETS = [ "apps/worker/src/outbox/event-router.test.ts", "apps/worker/src/jobs/workflow-trigger.test.ts", "apps/worker/src/jobs/workflow-trigger-bootstrap.test.ts", "apps/worker/src/jobs/workflow-trigger-boundary.test.ts", "apps/worker/src/jobs/workflow-trigger-store.test.ts", "apps/worker/src/health.test.ts", "apps/web/app/modules/workflowEngine/routes/workflow-engine/workflow-producer-inventory.test.ts", ]; const REQUIRED_DEPLOY_WORKFLOW_MARKERS = [ "stage-or-promote", "runs-on: toolctrl-deploy", "name: Verify deploy wiring", "name: Resolve live staging release", "workflow_dispatch", 'docker buildx build --builder "$DEPLOY_BUILDER_NAME" --push --target', "COOLIFY_STAGING_API_URL", "COOLIFY_API_URL", "/healthz", 'if [ "$RESOURCE_KIND" = "mcp" ]; then', 'HEALTH_PATH="/readyz"', '"$HEALTH_URL$HEALTH_PATH"', "payload.revision", "payload.service", 'git checkout --detach "$IMAGE_TAG"', "pnpm run verify:tenant-policy-contract", "pnpm run verify:rls-release-readiness", "pnpm run verify:rls-live", "for TARGET in web api worker mcp hocuspocus portal-domain-controller; do", "node scripts/verify-portal-domain-controller-deploy.mjs", 'DOCKERFILE="apps/hocuspocus/Dockerfile"', "HOCUSPOCUS_RESOURCE_UUID", 'assert-runtime-db-env "$HOCUSPOCUS_RESOURCE_UUID" hivegrid_app', 'if [ "$RESOURCE_KIND" = "hocuspocus" ]; then', ]; const REQUIRED_CI_COMMANDS = [ "pnpm run verify:deploy-workflow", "pnpm run verify:deploy-readiness", "pnpm run verify:docker-non-root", "pnpm run verify:health-baseline", "pnpm run verify:api-inventory", "pnpm run verify:openapi-security", "pnpm run verify:i18n", "pnpm run test:hot-swap", "pnpm run verify:portal-domain-controller-deploy", "pnpm run verify:workflow-outbox-native", "pnpm run verify:tenant-policy-contract", "pnpm run verify:rls-live", "pnpm --filter @hivegrid/mcp typecheck", "pnpm --filter @hivegrid/mcp test", "pnpm run build:ci", ]; const REQUIRED_DOC_MARKERS = [ "stage-and-promote", "web`, `api`, `mcp`, `worker`, and `hocuspocus`", "GET /healthz", "GET /readyz", "workflow_dispatch", "input-free", "main branch protection", "direct pushes to `main` must be disabled", "pull-request CI workflow to pass", "Do not set `SEED_DEMO_DATA=true`", "exact deployment SHA", "pnpm run verify:tenant-policy-contract", "pnpm run verify:rls-release-readiness", "pnpm run verify:rls-live", "hivegrid_migrator", ]; const REQUIRED_WORKER_MATRIX_DOCS = ["docs/worker.md", "docs/ops/worker-queues.md"]; const REQUIRED_WORKER_RUNBOOK_MARKERS = [ "OUTBOX_DISPATCHER_REQUIRED=true", "WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true", "TRANSACTIONAL_MAIL_OUTBOX_REQUIRED", WORKER_DEPLOY_HEALTH_COMMAND_EXAMPLE, ]; const REQUIRED_CANONICAL_WORKER_RUNBOOK_MARKERS = [ "scripts/coolify-ensure-worker-groups.mjs", "OUTBOX_DISPATCHER_REQUIRED=true", "WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true", "TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true", "pnpm run verify:plugin-runtime-live", WORKER_DEPLOY_HEALTH_COMMAND_EXAMPLE, "configure-legacy-rollback-healthcheck", ]; const REQUIRED_DEPLOY_DOC_MARKERS = [ "B-012 is not a repository code blocker", "operator-owned host hardening", "direct pushes to `main` must be disabled", "pull-request CI workflow to pass", WORKER_DEPLOY_HEALTH_COMMAND_EXAMPLE, MCP_DEPLOY_HEALTH_COMMAND_EXAMPLE, "configure-legacy-rollback-healthcheck", "Hocuspocus", "running:healthy", ]; const REQUIRED_MIGRATOR_ENTRYPOINT_MARKERS = [ "node /myapp/scripts/verify-migrator-role.mjs", "node /myapp/scripts/adopt-prisma-baseline.mjs --adopt", "node /myapp/scripts/adopt-prisma-baseline.mjs --verify-current-schema", "node /myapp/packages/db/dist/deploy-permission-catalog-sync.server.js", ]; const REQUIRED_MIGRATOR_ROLE_PROBE_MARKERS = [ 'export const REQUIRED_MIGRATOR_ROLE = "hivegrid_migrator";', "roles[0].isSuperuser !== false", "roles[0].bypassesRls !== false", 'memberships[0].grantedRole !== "hivegrid_rls_definer"', "memberships[0].adminOption !== false", "memberships[0].inheritOption !== false", "memberships[0].setOption !== true", ]; const REQUIRED_MIGRATOR_DOCKER_MARKERS = [ "COPY --from=build /myapp/scripts/verify-migrator-role.mjs /myapp/scripts/verify-migrator-role.mjs", "COPY --from=build /myapp/scripts/adopt-prisma-baseline.mjs /myapp/scripts/adopt-prisma-baseline.mjs", ]; const REQUIRED_WORKER_BUILD_MARKERS = ['"src/deploy-health.ts"']; const REQUIRED_WORKER_PROVISIONER_MARKERS = [ "const COOLIFY_CUID_PATTERN = /^[a-z][a-z0-9]{23}$/;", "Coolify list response was malformed", "const resolvedGroups = GROUPS.map", "for (const entry of resolvedGroups)", "assertWorkerCommandHealthCanBeAuthoritative", 'const workerEnvSourceUuid = needsProvisioningOrRepair ? resolveWorkerEnvSourceUuid(apps, allowed) : "";', ]; const REQUIRED_WORKER_DEPLOY_HEALTH_MARKERS = [ "const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/;", 'const IMAGE_REVISION_PATH = "/myapp/.image-revision";', "imageRevision !== expectedRevision || env.APP_REVISION !== expectedRevision", 'env.WORKER_ENABLED !== "true"', "env.WORKER_QUEUE_GROUP !== expectedGroup", "/readyz", "payload.workerGroup !== expectedGroup", 'readinessChecks.database === "ok"', 'readinessChecks.redis === "ok"', ]; const REQUIRED_MCP_BUILD_MARKERS = ['"src/deploy-health.ts"']; const REQUIRED_MCP_DEPLOY_HEALTH_MARKERS = [ "const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/;", 'const IMAGE_REVISION_PATH = "/myapp/.image-revision";', "imageRevision !== expectedRevision || env.APP_REVISION !== expectedRevision", "/readyz", 'payload.service !== "mcp"', "payload.revision !== expectedRevision", 'readinessChecks.database === "ok"', 'readinessChecks.redis === "ok"', ]; const REQUIRED_ASSET_CDN_RUNBOOK_MARKERS = ["Rollout status", "remain parked", "Bunny purge", "reactivation guide"]; const REQUIRED_SENTRY_RUNBOOK_MARKERS = ["B-008 is closed", "operator alerting item"]; const REQUIRED_GENERAL_WORKER_ENV = { DATABASE_RUNTIME_ROLE: "hivegrid_worker", RLS_RUNTIME_MODE: "required", OUTBOX_DISPATCHER_ENABLED: "true", OUTBOX_DISPATCHER_REQUIRED: "true", WORKFLOW_TRIGGER_EXECUTOR_MODE: "http", WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true", WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true", }; const REQUIRED_GENERAL_WORKER_DRIFT_KEYS = [ "DATABASE_URL", "BULLMQ_REDIS_URL", "WORKFLOW_TRIGGER_EXECUTOR_URL", "API_ACCESS_TOKEN", "LAGO_API_URL", "LAGO_API_KEY", "MOLLIE_API_KEY", "BILLING_MOLLIE_WEBHOOK_URL", ]; function read(rootDir, relativePath, failures) { const absolutePath = path.join(rootDir, relativePath); if (!fs.existsSync(absolutePath)) { failures.push(`${relativePath} missing required file`); return ""; } return fs.readFileSync(absolutePath, "utf8"); } function checkIncludes(content, markers, label, result) { for (const marker of markers) { if (!content.includes(marker)) { result.failures.push(`${label} missing ${marker}`); } else { result.passes.push(`${label} contains ${marker}`); } } } function checkMinimumOccurrences(content, marker, minimum, label, result) { const occurrences = content.split(marker).length - 1; if (occurrences >= minimum) { result.passes.push(`${label} contains ${marker} at least ${minimum} times`); } else { result.failures.push(`${label} requires ${marker} at least ${minimum} times (found ${occurrences})`); } } function checkWorkflowOutboxNativeScript(packageJson, result) { const command = packageJson.scripts?.[WORKFLOW_OUTBOX_NATIVE_SCRIPT]; if (!command) { return; } for (const target of REQUIRED_WORKFLOW_OUTBOX_NATIVE_TEST_TARGETS) { if (command.includes(target)) { result.passes.push(`package.json ${WORKFLOW_OUTBOX_NATIVE_SCRIPT} includes ${target}`); } else { result.failures.push(`package.json ${WORKFLOW_OUTBOX_NATIVE_SCRIPT} missing ${target}`); } } } function hasWorkerMatrixRow(content, group) { const appName = `worker-${group.name}`; const groupEnv = `WORKER_QUEUE_GROUP=${group.name}`; return content.split(/\r?\n/).some((line) => { if (!line.includes("|") || !line.includes(`\`${appName}\``) || !line.includes(`\`${groupEnv}\``)) { return false; } return line.includes(`\`${group.healthPort}\``) && line.includes(`\`${group.adminPort}\``); }); } function checkWorkerMatrixDocs(rootDir, result) { for (const relativePath of REQUIRED_WORKER_MATRIX_DOCS) { const content = read(rootDir, relativePath, result.failures); if (!content) continue; for (const group of GROUPS) { if (hasWorkerMatrixRow(content, group)) { result.passes.push(`${relativePath} documents worker-${group.name} as ${group.name} on health ${group.healthPort} and admin ${group.adminPort}`); } else { result.failures.push( `${relativePath} missing worker-${group.name} row with WORKER_QUEUE_GROUP=${group.name}, health port ${group.healthPort}, and admin port ${group.adminPort}` ); } } } const workerRunbook = read(rootDir, "docs/worker.md", result.failures); if (workerRunbook) { checkIncludes(workerRunbook, REQUIRED_WORKER_RUNBOOK_MARKERS, "worker docs", result); } const canonicalWorkerRunbook = read(rootDir, "docs/ops/worker-queues.md", result.failures); if (canonicalWorkerRunbook) { checkIncludes(canonicalWorkerRunbook, REQUIRED_CANONICAL_WORKER_RUNBOOK_MARKERS, "canonical worker queue docs", result); } } function entriesFromRecord(record) { return Object.entries(record).map(([key, value]) => ({ key, value })); } function checkCoolifyWorkerProvisionerContract(result) { const general = GROUPS.find((group) => group.name === "general"); if (!general) { result.failures.push("Coolify worker provisioner missing worker-general group"); return; } const desiredGeneralEnv = desiredWorkerEnv(general); for (const [key, expected] of Object.entries(REQUIRED_GENERAL_WORKER_ENV)) { if (desiredGeneralEnv[key] === expected) { result.passes.push(`Coolify worker provisioner sets worker-general ${key}=${expected}`); } else { result.failures.push(`Coolify worker provisioner must set worker-general ${key}=${expected}`); } } const runtimeDrift = getWorkerEnvDrift({ uuid: "worker-general", name: "worker-general" }, general, entriesFromRecord(desiredGeneralEnv)); const driftKeys = new Set(runtimeDrift.map((item) => item.key)); for (const key of REQUIRED_GENERAL_WORKER_DRIFT_KEYS) { if (driftKeys.has(key)) { result.passes.push(`Coolify worker provisioner detects missing worker-general ${key}`); } else { result.failures.push(`Coolify worker provisioner must detect missing worker-general ${key}`); } } const wrongDatabaseRoleDrift = getWorkerEnvDrift( { uuid: "worker-general", name: "worker-general" }, general, entriesFromRecord({ ...desiredGeneralEnv, DATABASE_URL: "postgres://hivegrid_app:redacted@db.example.test/hivegrid", BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", }) ); if (wrongDatabaseRoleDrift.some((item) => item.key === "DATABASE_URL")) { result.passes.push("Coolify worker provisioner rejects a non-worker DATABASE_URL role"); } else { result.failures.push("Coolify worker provisioner must reject a non-worker DATABASE_URL role"); } const privilegedEnv = Object.fromEntries([...WORKER_PRIVILEGED_ENV_KEYS].map((key) => [key, "redacted"])); const privilegedDrift = getWorkerEnvDrift( { uuid: "worker-general", name: "worker-general" }, general, entriesFromRecord({ ...desiredGeneralEnv, DATABASE_URL: "postgres://hivegrid_worker:redacted@db.example.test/hivegrid", BULLMQ_REDIS_URL: "redis://redacted", WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.test", API_ACCESS_TOKEN: "redacted-token", ...privilegedEnv, }) ); const privilegedDriftKeys = new Set(privilegedDrift.map((item) => item.key)); for (const key of WORKER_PRIVILEGED_ENV_KEYS) { if (privilegedDriftKeys.has(key)) { result.passes.push(`Coolify worker provisioner rejects privileged worker env ${key}`); } else { result.failures.push(`Coolify worker provisioner must reject privileged worker env ${key}`); } } let expectedHealthCommand = ""; try { expectedHealthCommand = buildWorkerDeploymentHealthCommand(DEPLOY_HEALTH_EXPECTED_REVISION, "general"); } catch { result.failures.push("Coolify worker provisioner must accept a full lowercase Git SHA for deployment health"); } const expectedCommand = `node apps/worker/dist/deploy-health.js ${DEPLOY_HEALTH_EXPECTED_REVISION} general`; if (expectedHealthCommand === expectedCommand) { result.passes.push("Coolify worker provisioner binds worker deployment health to the immutable image SHA"); } else { result.failures.push(`Coolify worker provisioner must build ${expectedCommand}`); } for (const invalidRevision of ["staging", DEPLOY_HEALTH_EXPECTED_REVISION.toUpperCase()]) { try { buildWorkerDeploymentHealthCommand(invalidRevision, "general"); result.failures.push("Coolify worker provisioner must reject non-immutable or non-lowercase deployment revisions"); } catch { result.passes.push("Coolify worker provisioner rejects a non-immutable deployment revision"); } } const matchingConfig = { uuid: "worker-general", name: "worker-general", docker_registry_image_tag: DEPLOY_HEALTH_EXPECTED_REVISION, ports_exposes: general.healthPort, health_check_enabled: true, health_check_type: "cmd", health_check_command: expectedCommand, health_check_interval: 30, health_check_timeout: 10, health_check_retries: 3, health_check_start_period: 20, custom_healthcheck_found: false, }; const matchingConfigDrift = getWorkerAppConfigDrift(matchingConfig, general, DEPLOY_HEALTH_EXPECTED_REVISION); if (matchingConfigDrift.length === 0) { result.passes.push("Coolify worker provisioner accepts enabled SHA-bound cmd health configuration"); } else { result.failures.push("Coolify worker provisioner rejects its required SHA-bound cmd health configuration"); } const staleConfigDrift = getWorkerAppConfigDrift( { ...matchingConfig, health_check_enabled: false, health_check_type: "http", health_check_command: "node apps/worker/dist/health.js", custom_healthcheck_found: true, }, general, DEPLOY_HEALTH_EXPECTED_REVISION ); const staleConfigDriftKeys = new Set(staleConfigDrift.map((item) => item.key)); for (const key of ["health_check_enabled", "health_check_type", "health_check_command", "custom_healthcheck_found"]) { if (staleConfigDriftKeys.has(key)) { result.passes.push(`Coolify worker provisioner detects stale worker ${key}`); } else { result.failures.push(`Coolify worker provisioner must detect stale worker ${key}`); } } } export function verifyDeployReadiness(rootDir = ROOT_DIR) { const result = { passes: [], failures: [] }; const packageJsonContent = read(rootDir, "package.json", result.failures); if (packageJsonContent) { const packageJson = JSON.parse(packageJsonContent); for (const scriptName of REQUIRED_PACKAGE_SCRIPTS) { if (!packageJson.scripts?.[scriptName]) { result.failures.push(`package.json missing script ${scriptName}`); } else { result.passes.push(`package.json script ${scriptName}`); } } checkWorkflowOutboxNativeScript(packageJson, result); } const deployWorkflow = read(rootDir, ".gitea/workflows/deploy.yml", result.failures); checkIncludes(deployWorkflow, REQUIRED_DEPLOY_WORKFLOW_MARKERS, ".gitea deploy workflow", result); checkMinimumOccurrences(deployWorkflow, 'if [ "$RESOURCE_KIND" = "mcp" ]; then', 2, ".gitea deploy workflow", result); checkMinimumOccurrences(deployWorkflow, 'if [ "$RESOURCE_KIND" = "hocuspocus" ]; then', 2, ".gitea deploy workflow", result); checkMinimumOccurrences(deployWorkflow, 'assert-runtime-db-env "$HOCUSPOCUS_RESOURCE_UUID" hivegrid_app', 2, ".gitea deploy workflow", result); checkMinimumOccurrences( deployWorkflow, 'EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP"', 2, ".gitea deploy workflow", result ); checkMinimumOccurrences( deployWorkflow, '[ "$HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_HEALTH_TYPE" != "cmd" ]', 2, ".gitea deploy workflow", result ); checkMinimumOccurrences(deployWorkflow, '[ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]', 2, ".gitea deploy workflow", result); checkMinimumOccurrences(deployWorkflow, 'EXPECTED_MCP_HEALTH_COMMAND="node apps/mcp/dist/deploy-health.js $IMAGE_TAG"', 2, ".gitea deploy workflow", result); checkMinimumOccurrences(deployWorkflow, '[ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]', 2, ".gitea deploy workflow", result); checkIncludes(read(rootDir, ".gitea/workflows/ci.yml", result.failures), REQUIRED_CI_COMMANDS, ".gitea CI workflow", result); checkIncludes(read(rootDir, "docs/deployment.md", result.failures), REQUIRED_DOC_MARKERS, "deployment docs", result); checkIncludes(read(rootDir, "docs/deployment.md", result.failures), REQUIRED_DEPLOY_DOC_MARKERS, "deployment docs", result); checkIncludes(read(rootDir, "docs/ops/asset-cdn.md", result.failures), REQUIRED_ASSET_CDN_RUNBOOK_MARKERS, "asset CDN runbook", result); checkIncludes(read(rootDir, "docs/observability/sentry-selfhost.md", result.failures), REQUIRED_SENTRY_RUNBOOK_MARKERS, "Sentry self-host runbook", result); checkWorkerMatrixDocs(rootDir, result); checkCoolifyWorkerProvisionerContract(result); checkIncludes( read(rootDir, "scripts/coolify-ensure-worker-groups.mjs", result.failures), REQUIRED_WORKER_PROVISIONER_MARKERS, "Coolify worker provisioner", result ); checkIncludes(read(rootDir, "apps/worker/scripts/build.mjs", result.failures), REQUIRED_WORKER_BUILD_MARKERS, "worker build", result); checkIncludes( read(rootDir, "apps/worker/src/deploy-health.ts", result.failures), REQUIRED_WORKER_DEPLOY_HEALTH_MARKERS, "worker deployment health entrypoint", result ); checkIncludes(read(rootDir, "apps/mcp/scripts/build.mjs", result.failures), REQUIRED_MCP_BUILD_MARKERS, "MCP build", result); checkIncludes( read(rootDir, "apps/mcp/src/deploy-health.ts", result.failures), REQUIRED_MCP_DEPLOY_HEALTH_MARKERS, "MCP deployment health entrypoint", result ); checkIncludes(read(rootDir, "start.sh", result.failures), REQUIRED_MIGRATOR_ENTRYPOINT_MARKERS, "migration entrypoint", result); checkIncludes(read(rootDir, "scripts/verify-migrator-role.mjs", result.failures), REQUIRED_MIGRATOR_ROLE_PROBE_MARKERS, "migrator role probe", result); checkIncludes(read(rootDir, "Dockerfile", result.failures), REQUIRED_MIGRATOR_DOCKER_MARKERS, "Dockerfile", result); checkMinimumOccurrences( read(rootDir, "Dockerfile", result.failures), "RUN printf '%s\\n' \"$APP_REVISION\" > /myapp/.image-revision", 3, "Dockerfile", result ); return result; } function printResult(result) { for (const message of result.passes) { console.log(`PASS ${message}`); } if (result.failures.length > 0) { for (const message of result.failures) { console.error(`FAIL ${message}`); } process.exitCode = 1; } else { console.log("Deploy readiness contract verified."); } } const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : ""; if (invokedPath && fileURLToPath(import.meta.url) === invokedPath) { try { printResult(verifyDeployReadiness()); } catch (error) { if (error instanceof SyntaxError) { console.error(`FAIL package.json could not be parsed: ${error.message}`); } else { console.error(`FAIL deploy readiness verifier crashed: ${error instanceof Error ? error.message : String(error)}`); } process.exitCode = 1; } }