/** * Per-worker HTTP health server. * * Exposes two Kubernetes/Coolify-style probes on a dedicated port so a * platform can verify the worker process is alive (`/healthz`) and that * its critical dependencies are reachable (`/readyz`). * * Separate from `admin-server.ts` (which serves /status and /metrics * gated by METRICS_ENABLED + a bearer token). Health probes must work * even when metrics are off and even before a bearer is provisioned. * * - `/healthz` — process liveness. Returns 200 as long as the event * loop is responsive. No external dependency calls. * - `/readyz` — readiness. Issues a BullMQ Redis ping and a Prisma * `SELECT 1` to confirm both backends are reachable. * Returns 200 only when every probe succeeds; 503 * otherwise. Each check is bounded by a timeout so a * hung dependency cannot stall the probe. * * The handler is intentionally framework-free — node's built-in * `http.createServer` keeps the bundle size and startup cost minimal. */ import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; import type { WorkerQueueGroup } from "@hivegrid/queue"; export const DEFAULT_WORKER_HEALTH_PORT = 9100; export type HealthProbeResult = { ok: true } | { ok: false; error: string }; export type HealthProbe = () => Promise | HealthProbeResult; export type ReadyResponseBody = { status: "ready" | "not_ready"; workerGroup?: WorkerQueueGroup; checks: Record; }; export type WorkerHealthServerOptions = { port?: number; host?: string; workerGroup?: WorkerQueueGroup; redisProbe: HealthProbe; databaseProbe: HealthProbe; readinessProbes?: Record; probeTimeoutMs?: number; logger?: { error?: (fields: Record, msg: string) => void; info?: (fields: Record, msg: string) => void; }; }; export type WorkerHealthServerHandle = { port: number; close: () => Promise; }; const DEFAULT_PROBE_TIMEOUT_MS = 2_000; /** * Pure handler factory — exposed so tests can hit the request handler * directly without binding a TCP port. */ export function createWorkerHealthHandler(deps: Omit) { const timeoutMs = deps.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; return async function handler(req: IncomingMessage, res: ServerResponse): Promise { const url = req.url ?? "/"; if (url === "/healthz" || url === "/healthz/") { respondJson(res, 200, { status: "alive", ...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}) }); return; } if (url === "/readyz" || url === "/readyz/") { const additionalProbes = Object.entries(deps.readinessProbes ?? {}); const [redis, database, ...additionalResults] = await Promise.all([ runWithTimeout(deps.redisProbe, timeoutMs, "redis"), runWithTimeout(deps.databaseProbe, timeoutMs, "database"), ...additionalProbes.map(([label, probe]) => runWithTimeout(probe, timeoutMs, label)), ]); const checks: ReadyResponseBody["checks"] = { redis: redis.ok ? "ok" : "error", database: database.ok ? "ok" : "error", }; for (const [index, [label]] of additionalProbes.entries()) { checks[label] = additionalResults[index]?.ok ? "ok" : "error"; } const allOk = redis.ok && database.ok && additionalResults.every((result) => result.ok); respondJson(res, allOk ? 200 : 503, { status: allOk ? "ready" : "not_ready", ...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}), checks, }); return; } respondJson(res, 404, { error: { code: "not_found", message: "Endpoint not found." } }); }; } /** * Binds the health handler to a TCP port and returns a handle whose * `close()` shuts the server cleanly during SIGTERM/SIGINT. */ export function startWorkerHealthServer(options: WorkerHealthServerOptions): WorkerHealthServerHandle { const handler = createWorkerHealthHandler({ redisProbe: options.redisProbe, databaseProbe: options.databaseProbe, ...(options.readinessProbes ? { readinessProbes: options.readinessProbes } : {}), ...(options.workerGroup ? { workerGroup: options.workerGroup } : {}), ...(options.probeTimeoutMs !== undefined ? { probeTimeoutMs: options.probeTimeoutMs } : {}), }); const port = options.port ?? DEFAULT_WORKER_HEALTH_PORT; const host = options.host ?? "0.0.0.0"; const server: Server = createServer((req, res) => { void handler(req, res).catch((error: unknown) => { options.logger?.error?.({ error: serialiseError(error) }, "worker health server handler failed"); if (!res.headersSent) { respondJson(res, 500, { error: { code: "internal", message: "Internal server error." } }); } else { res.end(); } }); }); server.listen(port, host); options.logger?.info?.({ port, host, workerGroup: options.workerGroup }, "worker health server listening"); return { port, close: () => new Promise((resolve) => { server.close(() => resolve()); }), }; } async function runWithTimeout(probe: HealthProbe, timeoutMs: number, label: string): Promise { let timeout: NodeJS.Timeout | undefined; try { const result = await Promise.race([ Promise.resolve().then(() => probe()), new Promise((resolve) => { timeout = setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs); }), ]); return result; } catch (error) { return { ok: false, error: serialiseError(error).message }; } finally { if (timeout) { clearTimeout(timeout); } } } function respondJson(res: ServerResponse, statusCode: number, body: unknown): void { res.statusCode = statusCode; res.setHeader("content-type", "application/json"); res.setHeader("cache-control", "no-store"); res.end(JSON.stringify(body)); } function serialiseError(error: unknown): { name: string; message: string } { if (error instanceof Error) { return { name: error.name, message: error.message.slice(0, 200) }; } return { name: "UnknownError", message: String(error).slice(0, 200) }; }