168 lines
6.3 KiB
TypeScript
168 lines
6.3 KiB
TypeScript
/**
|
|
* Per-worker HTTP health server.
|
|
*
|
|
* Exposes two Kubernetes/Coolify-style probes on a dedicated port so a
|
|
* platform can verify the worker process is alive (`/healthz`) and that
|
|
* its critical dependencies are reachable (`/readyz`).
|
|
*
|
|
* Separate from `admin-server.ts` (which serves /status and /metrics
|
|
* gated by METRICS_ENABLED + a bearer token). Health probes must work
|
|
* even when metrics are off and even before a bearer is provisioned.
|
|
*
|
|
* - `/healthz` — process liveness. Returns 200 as long as the event
|
|
* loop is responsive. No external dependency calls.
|
|
* - `/readyz` — readiness. Issues a BullMQ Redis ping and a Prisma
|
|
* `SELECT 1` to confirm both backends are reachable.
|
|
* Returns 200 only when every probe succeeds; 503
|
|
* otherwise. Each check is bounded by a timeout so a
|
|
* hung dependency cannot stall the probe.
|
|
*
|
|
* The handler is intentionally framework-free — node's built-in
|
|
* `http.createServer` keeps the bundle size and startup cost minimal.
|
|
*/
|
|
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
|
import type { WorkerQueueGroup } from "@hivegrid/queue";
|
|
|
|
export const DEFAULT_WORKER_HEALTH_PORT = 9100;
|
|
|
|
export type HealthProbeResult = { ok: true } | { ok: false; error: string };
|
|
|
|
export type HealthProbe = () => Promise<HealthProbeResult> | HealthProbeResult;
|
|
|
|
export type ReadyResponseBody = {
|
|
status: "ready" | "not_ready";
|
|
workerGroup?: WorkerQueueGroup;
|
|
checks: Record<string, "ok" | "error">;
|
|
};
|
|
|
|
export type WorkerHealthServerOptions = {
|
|
port?: number;
|
|
host?: string;
|
|
workerGroup?: WorkerQueueGroup;
|
|
redisProbe: HealthProbe;
|
|
databaseProbe: HealthProbe;
|
|
readinessProbes?: Record<string, HealthProbe>;
|
|
probeTimeoutMs?: number;
|
|
logger?: {
|
|
error?: (fields: Record<string, unknown>, msg: string) => void;
|
|
info?: (fields: Record<string, unknown>, msg: string) => void;
|
|
};
|
|
};
|
|
|
|
export type WorkerHealthServerHandle = {
|
|
port: number;
|
|
close: () => Promise<void>;
|
|
};
|
|
|
|
const DEFAULT_PROBE_TIMEOUT_MS = 2_000;
|
|
|
|
/**
|
|
* Pure handler factory — exposed so tests can hit the request handler
|
|
* directly without binding a TCP port.
|
|
*/
|
|
export function createWorkerHealthHandler(deps: Omit<WorkerHealthServerOptions, "port" | "host" | "logger">) {
|
|
const timeoutMs = deps.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
|
|
return async function handler(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
const url = req.url ?? "/";
|
|
if (url === "/healthz" || url === "/healthz/") {
|
|
respondJson(res, 200, { status: "alive", ...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}) });
|
|
return;
|
|
}
|
|
if (url === "/readyz" || url === "/readyz/") {
|
|
const additionalProbes = Object.entries(deps.readinessProbes ?? {});
|
|
const [redis, database, ...additionalResults] = await Promise.all([
|
|
runWithTimeout(deps.redisProbe, timeoutMs, "redis"),
|
|
runWithTimeout(deps.databaseProbe, timeoutMs, "database"),
|
|
...additionalProbes.map(([label, probe]) => runWithTimeout(probe, timeoutMs, label)),
|
|
]);
|
|
const checks: ReadyResponseBody["checks"] = {
|
|
redis: redis.ok ? "ok" : "error",
|
|
database: database.ok ? "ok" : "error",
|
|
};
|
|
for (const [index, [label]] of additionalProbes.entries()) {
|
|
checks[label] = additionalResults[index]?.ok ? "ok" : "error";
|
|
}
|
|
const allOk = redis.ok && database.ok && additionalResults.every((result) => result.ok);
|
|
respondJson(res, allOk ? 200 : 503, {
|
|
status: allOk ? "ready" : "not_ready",
|
|
...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}),
|
|
checks,
|
|
});
|
|
return;
|
|
}
|
|
respondJson(res, 404, { error: { code: "not_found", message: "Endpoint not found." } });
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Binds the health handler to a TCP port and returns a handle whose
|
|
* `close()` shuts the server cleanly during SIGTERM/SIGINT.
|
|
*/
|
|
export function startWorkerHealthServer(options: WorkerHealthServerOptions): WorkerHealthServerHandle {
|
|
const handler = createWorkerHealthHandler({
|
|
redisProbe: options.redisProbe,
|
|
databaseProbe: options.databaseProbe,
|
|
...(options.readinessProbes ? { readinessProbes: options.readinessProbes } : {}),
|
|
...(options.workerGroup ? { workerGroup: options.workerGroup } : {}),
|
|
...(options.probeTimeoutMs !== undefined ? { probeTimeoutMs: options.probeTimeoutMs } : {}),
|
|
});
|
|
|
|
const port = options.port ?? DEFAULT_WORKER_HEALTH_PORT;
|
|
const host = options.host ?? "0.0.0.0";
|
|
|
|
const server: Server = createServer((req, res) => {
|
|
void handler(req, res).catch((error: unknown) => {
|
|
options.logger?.error?.({ error: serialiseError(error) }, "worker health server handler failed");
|
|
if (!res.headersSent) {
|
|
respondJson(res, 500, { error: { code: "internal", message: "Internal server error." } });
|
|
} else {
|
|
res.end();
|
|
}
|
|
});
|
|
});
|
|
|
|
server.listen(port, host);
|
|
options.logger?.info?.({ port, host, workerGroup: options.workerGroup }, "worker health server listening");
|
|
|
|
return {
|
|
port,
|
|
close: () =>
|
|
new Promise<void>((resolve) => {
|
|
server.close(() => resolve());
|
|
}),
|
|
};
|
|
}
|
|
|
|
async function runWithTimeout(probe: HealthProbe, timeoutMs: number, label: string): Promise<HealthProbeResult> {
|
|
let timeout: NodeJS.Timeout | undefined;
|
|
try {
|
|
const result = await Promise.race([
|
|
Promise.resolve().then(() => probe()),
|
|
new Promise<HealthProbeResult>((resolve) => {
|
|
timeout = setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs);
|
|
}),
|
|
]);
|
|
return result;
|
|
} catch (error) {
|
|
return { ok: false, error: serialiseError(error).message };
|
|
} finally {
|
|
if (timeout) {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
}
|
|
|
|
function respondJson(res: ServerResponse, statusCode: number, body: unknown): void {
|
|
res.statusCode = statusCode;
|
|
res.setHeader("content-type", "application/json");
|
|
res.setHeader("cache-control", "no-store");
|
|
res.end(JSON.stringify(body));
|
|
}
|
|
|
|
function serialiseError(error: unknown): { name: string; message: string } {
|
|
if (error instanceof Error) {
|
|
return { name: error.name, message: error.message.slice(0, 200) };
|
|
}
|
|
return { name: "UnknownError", message: String(error).slice(0, 200) };
|
|
}
|