import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type WorkerConfigRedisParseStatus } from "@hivegrid/config"; import { getDbClient } from "@hivegrid/db/client.server"; import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@hivegrid/queue"; import { Queue } from "bullmq"; import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js"; import { getMissingWorkerGdprExportStorageEnv, getMissingWorkerStorageEnv } from "./storage.js"; export type WorkerHealthEnv = EnvRecord; export type WorkerHealthStatus = "disabled" | "ready" | "misconfigured"; export type WorkerHealthRedisSource = "BULLMQ_REDIS_URL" | "REDIS_URL" | "none"; export type WorkerHealthDatabaseStatus = "skipped" | "ready" | "misconfigured"; export type WorkerHealthRedisStatus = "skipped" | "ready" | "misconfigured"; export type WorkerHealthStorageStatus = "skipped" | "ready" | "misconfigured"; export type WorkerHealthImportStatus = "skipped" | "ready" | "misconfigured"; export type WorkerHealthMailStatus = "skipped" | "ready" | "misconfigured"; export type WorkerHealthPluginRuntimeStatus = "skipped" | "ready" | "misconfigured"; export type WorkerHealthRequiredModeStatus = "skipped" | "ready" | "misconfigured"; export type WorkerHealthReport = { status: WorkerHealthStatus; revision: string; workerEnabled: boolean; workerGroup: WorkerQueueGroup | null; redisConfigured: boolean; redisSource: WorkerHealthRedisSource; redisParseStatus: "not_configured" | "valid" | "invalid"; redisStatus: WorkerHealthRedisStatus; redisErrorCode?: string; databaseConfigured: boolean; databaseStatus: WorkerHealthDatabaseStatus; databaseErrorCode?: string; storageConfigured: boolean; storageStatus: WorkerHealthStorageStatus; mailjetConfigured: boolean; mailjetStatus: WorkerHealthMailStatus; importConfigured: boolean; importStatus: WorkerHealthImportStatus; pluginRuntimeConfigured: boolean; pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus; queues: readonly QueueName[]; errors: readonly string[]; }; export type WorkerHealthCliIo = { stderr?: Pick; stdout: Pick; }; export type WorkerDatabaseProbe = () => Promise | void; export type WorkerRedisProbe = (redisUrl: string) => Promise | void; export type WorkerHealthDependencies = { databaseProbe?: WorkerDatabaseProbe; redisProbe?: WorkerRedisProbe; }; export async function buildWorkerHealthReport( env: WorkerHealthEnv = process.env as WorkerHealthEnv, dependencies: WorkerHealthDependencies = {} ): Promise { const result = loadWorkerConfig(env); const { config } = result; const shouldProbeDatabase = config.enabled && result.ok; const shouldProbeRedis = config.enabled && result.ok; const databaseConfigured = Boolean(env.DATABASE_URL); const [redisReadiness, databaseReadiness] = await Promise.all([ checkRedisReadiness(shouldProbeRedis, config.redis.url, dependencies.redisProbe), checkDatabaseReadiness(shouldProbeDatabase, dependencies.databaseProbe), ]); const storageReadiness = checkStorageReadiness( config.enabled && (env.EXPORT_RETENTION_QUEUE_ENABLED === "true" || ["full", "portal-gdpr-export"].includes(resolveImportQueueRuntimeMode(env))), env ); const mailReadiness = checkMailReadiness(config.enabled, config.mailjet.configured, isMailProviderRequired(env), env); const importReadiness = checkImportReadiness(config.enabled && result.ok && getWorkerGroup(env) === "import", env); const pluginRuntimeReadiness = checkPluginRuntimeReadiness(config.enabled && result.ok, getWorkerGroup(env), env); const requiredModeReadiness = checkRequiredGeneralWorkerReadiness(config.enabled && result.ok && getWorkerGroup(env) === "general", env); const status = toWorkerHealthStatus( config.enabled, result.ok, redisReadiness.status, databaseReadiness.status, storageReadiness.status, mailReadiness.status, importReadiness.status, pluginRuntimeReadiness.status, requiredModeReadiness.status ); return { status, revision: env.APP_REVISION || "unknown", workerEnabled: config.enabled, workerGroup: getWorkerGroup(env), redisConfigured: config.redis.configured, redisSource: config.redis.source, redisParseStatus: toHealthRedisParseStatus(config.redis.parseStatus), redisStatus: redisReadiness.status, ...(redisReadiness.errorCode ? { redisErrorCode: redisReadiness.errorCode } : {}), databaseConfigured, databaseStatus: databaseReadiness.status, ...(databaseReadiness.errorCode ? { databaseErrorCode: databaseReadiness.errorCode } : {}), storageConfigured: storageReadiness.configured, storageStatus: storageReadiness.status, mailjetConfigured: mailReadiness.configured, mailjetStatus: mailReadiness.status, importConfigured: importReadiness.configured, importStatus: importReadiness.status, pluginRuntimeConfigured: pluginRuntimeReadiness.configured, pluginRuntimeStatus: pluginRuntimeReadiness.status, queues: getWorkerQueueNames(env), errors: [ ...result.diagnostics.map(formatWorkerConfigDiagnostic), ...storageReadiness.errors, ...mailReadiness.errors, ...importReadiness.errors, ...pluginRuntimeReadiness.errors, ...requiredModeReadiness.errors, ], }; } function getWorkerGroup(env: WorkerHealthEnv): WorkerQueueGroup | null { const group = env.WORKER_QUEUE_GROUP; return typeof group === "string" && isWorkerQueueGroup(group) ? group : null; } export async function runWorkerHealthCli( env: WorkerHealthEnv = process.env as WorkerHealthEnv, io: WorkerHealthCliIo = process, dependencies: WorkerHealthDependencies = {} ): Promise { const report = await buildWorkerHealthReport(env, dependencies); io.stdout.write(`${JSON.stringify(report)}\n`); return report.status === "misconfigured" ? 1 : 0; } function toWorkerHealthStatus( workerEnabled: boolean, redisReady: boolean, redisStatus: WorkerHealthRedisStatus, databaseStatus: WorkerHealthDatabaseStatus, storageStatus: WorkerHealthStorageStatus, mailStatus: WorkerHealthMailStatus, importStatus: WorkerHealthImportStatus, pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus, requiredModeStatus: WorkerHealthRequiredModeStatus ): WorkerHealthStatus { if (!workerEnabled) { return "disabled"; } if ( !redisReady || redisStatus === "misconfigured" || databaseStatus === "misconfigured" || storageStatus === "misconfigured" || mailStatus === "misconfigured" || importStatus === "misconfigured" || pluginRuntimeStatus === "misconfigured" || requiredModeStatus === "misconfigured" ) { return "misconfigured"; } return "ready"; } function checkPluginRuntimeReadiness( shouldCheck: boolean, workerGroup: WorkerQueueGroup | null, env: WorkerHealthEnv ): { configured: boolean; errors: string[]; status: WorkerHealthPluginRuntimeStatus } { const productionRuntimeEnabled = env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true"; if (!shouldCheck || !productionRuntimeEnabled || (workerGroup !== "general" && workerGroup !== "plugin")) { return { configured: false, errors: [], status: "skipped" }; } const errors: string[] = []; if (!hasTrimmedValue(env.PLUGIN_TRUSTED_PUBLIC_KEYS)) { errors.push("PLUGIN_TRUSTED_PUBLIC_KEYS:missing-plugin-trust-root:Required when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."); } if (workerGroup === "general") { if (!hasTrimmedValue(env.BULLMQ_REDIS_URL) && !hasTrimmedValue(env.REDIS_URL)) { errors.push( "BULLMQ_REDIS_URL:missing-plugin-runtime-redis:Required for plugin hook fanout and capability-use subscribers when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true." ); } if (env.OUTBOX_DISPATCHER_ENABLED !== "true") { errors.push( "OUTBOX_DISPATCHER_ENABLED:missing-plugin-runtime-dispatcher:Required on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true." ); } if (env.PLUGIN_HOOK_FANOUT_ENABLED === "false") { errors.push( "PLUGIN_HOOK_FANOUT_ENABLED:disabled-plugin-runtime-fanout:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true." ); } if (env.PLUGIN_CAPABILITY_USE_ENABLED === "false") { errors.push( "PLUGIN_CAPABILITY_USE_ENABLED:disabled-plugin-capability-use:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true." ); } } return { configured: true, errors, status: errors.length > 0 ? "misconfigured" : "ready" }; } function isMailProviderRequired(env: WorkerHealthEnv): boolean { return env.WORKER_QUEUE_GROUP === "mail" && env.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED === "true"; } function checkRequiredGeneralWorkerReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { errors: string[]; status: WorkerHealthRequiredModeStatus } { if (!shouldCheck) { return { errors: [], status: "skipped" }; } const errors: string[] = []; if (env.OUTBOX_DISPATCHER_REQUIRED === "true" && env.OUTBOX_DISPATCHER_ENABLED !== "true") { errors.push("OUTBOX_DISPATCHER_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and OUTBOX_DISPATCHER_REQUIRED=true."); } const workflowTriggerRequired = env.WORKFLOW_TRIGGER_OUTBOX_REQUIRED === "true"; const workflowTriggerEnabled = env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true"; const workflowTriggerModeLabel = workflowTriggerRequired ? "WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true" : "WORKFLOW_TRIGGER_OUTBOX_ENABLED=true"; if (workflowTriggerRequired || workflowTriggerEnabled) { if (env.OUTBOX_DISPATCHER_ENABLED !== "true" && env.OUTBOX_DISPATCHER_REQUIRED !== "true") { errors.push(`OUTBOX_DISPATCHER_ENABLED:missing-workflow-trigger-dispatcher:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`); } if (workflowTriggerRequired && !workflowTriggerEnabled) { errors.push( "WORKFLOW_TRIGGER_OUTBOX_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true." ); } const executorMode = parseWorkflowTriggerExecutorModeForHealth(env.WORKFLOW_TRIGGER_EXECUTOR_MODE); if (!executorMode) { errors.push( `WORKFLOW_TRIGGER_EXECUTOR_MODE:invalid-workflow-executor-mode:Expected http, native, or hybrid when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.` ); } else if (workflowTriggerExecutorNeedsHttpBridge(executorMode)) { if (!hasTrimmedValue(env.API_ACCESS_TOKEN)) { errors.push( `API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.` ); } if (!hasTrimmedValue(env.WORKFLOW_TRIGGER_EXECUTOR_URL) && !hasTrimmedValue(env.APP_URL) && !hasTrimmedValue(env.WEB_ORIGIN)) { errors.push( `WORKFLOW_TRIGGER_EXECUTOR_URL:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.` ); } } } return { errors, status: errors.length > 0 ? "misconfigured" : "ready" }; } function parseWorkflowTriggerExecutorModeForHealth(value: unknown): "http" | "native" | "hybrid" | null { if (typeof value !== "string" || value.trim().length === 0) { return "http"; } const normalized = value.trim().toLowerCase(); return normalized === "http" || normalized === "native" || normalized === "hybrid" ? normalized : null; } function workflowTriggerExecutorNeedsHttpBridge(mode: "http" | "native" | "hybrid"): boolean { return mode !== "native"; } function hasTrimmedValue(value: string | undefined): boolean { return typeof value === "string" && value.trim().length > 0; } function checkMailReadiness( shouldCheck: boolean, configured: boolean, required: boolean, env: WorkerHealthEnv ): { configured: boolean; errors: string[]; status: WorkerHealthMailStatus } { if (!shouldCheck) { return { configured: false, errors: [], status: "skipped" }; } if (!configured) { if (required) { const missing = ["MAILJET_API_KEY", "MAILJET_SECRET_KEY", "MAILJET_FROM_EMAIL"].filter((name) => !env[name]?.trim()); return { configured: false, errors: missing.map( (name) => `${name}:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.` ), status: "misconfigured", }; } return { configured: false, errors: [], status: "skipped" }; } return { configured: true, errors: [], status: "ready" }; } function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthImportStatus } { if (!shouldCheck) { return { configured: false, errors: [], status: "skipped" }; } const importEnabled = env.IMPORT_QUEUE_ENABLED === "true"; const runtimeReady = env.IMPORT_QUEUE_RUNTIME_READY === "true"; const mode = resolveImportQueueRuntimeMode(env); if (mode === "idle") { return { configured: false, errors: [], status: "skipped" }; } if (mode === "portal-gdpr-export" || mode === "full") { return { configured: true, errors: [], status: "ready" }; } if (!importEnabled || !runtimeReady) { const errors: string[] = []; if (!importEnabled) { errors.push( "IMPORT_QUEUE_ENABLED:missing-import-runtime-gate:Required with IMPORT_QUEUE_RUNTIME_READY=true before WORKER_QUEUE_GROUP=import can consume hivegrid.imports/hivegrid.exports." ); } if (!runtimeReady) { errors.push( "IMPORT_QUEUE_RUNTIME_READY:missing-import-runtime-gate:Required with IMPORT_QUEUE_ENABLED=true after import, export, and scan services are wired." ); } return { configured: false, errors, status: "misconfigured", }; } return { configured: false, errors: ["WORKER_QUEUE_GROUP:invalid-import-runtime-mode:Import worker queue gates do not resolve to an allowed runtime mode."], status: "misconfigured", }; } function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthStorageStatus } { if (!shouldCheck) { return { configured: false, errors: [], status: "skipped" }; } const importMode = resolveImportQueueRuntimeMode(env); const ownsGdprExports = importMode === "full" || importMode === "portal-gdpr-export"; const missing = ownsGdprExports ? getMissingWorkerGdprExportStorageEnv(env) : getMissingWorkerStorageEnv(env); if (missing.length > 0) { const requiredBy = [ ...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? ["EXPORT_RETENTION_QUEUE_ENABLED=true"] : []), ...(importMode === "portal-gdpr-export" ? ["PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true"] : []), ...(importMode === "full" ? ["the full import runtime owns GDPR export jobs"] : []), ].join(" or "); return { configured: false, errors: missing.map((name) => `${name}:missing-storage-config:Required when ${requiredBy}.`), status: "misconfigured", }; } return { configured: true, errors: [], status: "ready" }; } async function checkRedisReadiness( shouldProbe: boolean, redisUrl: string | undefined, redisProbe: WorkerRedisProbe = defaultRedisProbe ): Promise<{ status: WorkerHealthRedisStatus; errorCode?: string }> { if (!shouldProbe || !redisUrl) { return { status: "skipped" }; } try { await redisProbe(redisUrl); return { status: "ready" }; } catch (error) { return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error, "REDIS_READINESS_FAILED") }; } } async function checkDatabaseReadiness( shouldProbe: boolean, databaseProbe: WorkerDatabaseProbe = defaultDatabaseProbe ): Promise<{ status: WorkerHealthDatabaseStatus; errorCode?: string }> { if (!shouldProbe) { return { status: "skipped" }; } try { await databaseProbe(); return { status: "ready" }; } catch (error) { return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error) }; } } async function defaultDatabaseProbe(): Promise { const db = getDbClient(); try { await db.$queryRaw`SELECT 1`; } finally { await db.$disconnect(); } } async function defaultRedisProbe(redisUrl: string): Promise { const queue = new Queue("__hivegrid_worker_health__", { connection: parseRedisUrl(redisUrl), }); try { await withTimeout(queue.waitUntilReady(), 2_000); } finally { await queue.close(); } } function toHealthRedisParseStatus(status: WorkerConfigRedisParseStatus): WorkerHealthReport["redisParseStatus"] { if (status === "valid") { return "valid"; } if (status === "not-provided") { return "not_configured"; } return "invalid"; } function toRedactedDependencyErrorCode(error: unknown, fallback = "DATABASE_READINESS_FAILED"): string { if (typeof error === "object" && error !== null && "code" in error) { const code = (error as { code?: unknown }).code; if (typeof code === "string" && /^[A-Z0-9_-]{1,64}$/i.test(code)) { return code; } } return fallback; } async function withTimeout(promise: Promise, timeoutMs: number): Promise { let timeout: NodeJS.Timeout | undefined; try { return await Promise.race([ promise, new Promise((_, reject) => { timeout = setTimeout(() => reject(Object.assign(new Error("Redis readiness timed out."), { code: "REDIS_TIMEOUT" })), timeoutMs); }), ]); } finally { if (timeout) { clearTimeout(timeout); } } } function formatWorkerConfigDiagnostic(diagnostic: WorkerConfigDiagnostic): string { return `${diagnostic.name}:${diagnostic.code}:${diagnostic.message}`; } if (import.meta.url === `file://${process.argv[1]}`) { runWorkerHealthCli() .then((exitCode) => { process.exitCode = exitCode; }) .catch(() => { process.exitCode = 1; }); }