Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* Per-worker HTTP health server.
|
||||
*
|
||||
* Exposes two Kubernetes/Coolify-style probes on a dedicated port so a
|
||||
* platform can verify the worker process is alive (`/healthz`) and that
|
||||
* its critical dependencies are reachable (`/readyz`).
|
||||
*
|
||||
* Separate from `admin-server.ts` (which serves /status and /metrics
|
||||
* gated by METRICS_ENABLED + a bearer token). Health probes must work
|
||||
* even when metrics are off and even before a bearer is provisioned.
|
||||
*
|
||||
* - `/healthz` — process liveness. Returns 200 as long as the event
|
||||
* loop is responsive. No external dependency calls.
|
||||
* - `/readyz` — readiness. Issues a BullMQ Redis ping and a Prisma
|
||||
* `SELECT 1` to confirm both backends are reachable.
|
||||
* Returns 200 only when every probe succeeds; 503
|
||||
* otherwise. Each check is bounded by a timeout so a
|
||||
* hung dependency cannot stall the probe.
|
||||
*
|
||||
* The handler is intentionally framework-free — node's built-in
|
||||
* `http.createServer` keeps the bundle size and startup cost minimal.
|
||||
*/
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import type { WorkerQueueGroup } from "@toolctrl/queue";
|
||||
|
||||
export const DEFAULT_WORKER_HEALTH_PORT = 9100;
|
||||
|
||||
export type HealthProbeResult = { ok: true } | { ok: false; error: string };
|
||||
|
||||
export type HealthProbe = () => Promise<HealthProbeResult> | HealthProbeResult;
|
||||
|
||||
export type ReadyResponseBody = {
|
||||
status: "ready" | "not_ready";
|
||||
workerGroup?: WorkerQueueGroup;
|
||||
checks: Record<string, "ok" | "error">;
|
||||
};
|
||||
|
||||
export type WorkerHealthServerOptions = {
|
||||
port?: number;
|
||||
host?: string;
|
||||
workerGroup?: WorkerQueueGroup;
|
||||
redisProbe: HealthProbe;
|
||||
databaseProbe: HealthProbe;
|
||||
readinessProbes?: Record<string, HealthProbe>;
|
||||
probeTimeoutMs?: number;
|
||||
logger?: {
|
||||
error?: (fields: Record<string, unknown>, msg: string) => void;
|
||||
info?: (fields: Record<string, unknown>, msg: string) => void;
|
||||
};
|
||||
};
|
||||
|
||||
export type WorkerHealthServerHandle = {
|
||||
port: number;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
const DEFAULT_PROBE_TIMEOUT_MS = 2_000;
|
||||
|
||||
/**
|
||||
* Pure handler factory — exposed so tests can hit the request handler
|
||||
* directly without binding a TCP port.
|
||||
*/
|
||||
export function createWorkerHealthHandler(deps: Omit<WorkerHealthServerOptions, "port" | "host" | "logger">) {
|
||||
const timeoutMs = deps.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
|
||||
return async function handler(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
||||
const url = req.url ?? "/";
|
||||
if (url === "/healthz" || url === "/healthz/") {
|
||||
respondJson(res, 200, { status: "alive", ...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}) });
|
||||
return;
|
||||
}
|
||||
if (url === "/readyz" || url === "/readyz/") {
|
||||
const additionalProbes = Object.entries(deps.readinessProbes ?? {});
|
||||
const [redis, database, ...additionalResults] = await Promise.all([
|
||||
runWithTimeout(deps.redisProbe, timeoutMs, "redis"),
|
||||
runWithTimeout(deps.databaseProbe, timeoutMs, "database"),
|
||||
...additionalProbes.map(([label, probe]) => runWithTimeout(probe, timeoutMs, label)),
|
||||
]);
|
||||
const checks: ReadyResponseBody["checks"] = {
|
||||
redis: redis.ok ? "ok" : "error",
|
||||
database: database.ok ? "ok" : "error",
|
||||
};
|
||||
for (const [index, [label]] of additionalProbes.entries()) {
|
||||
checks[label] = additionalResults[index]?.ok ? "ok" : "error";
|
||||
}
|
||||
const allOk = redis.ok && database.ok && additionalResults.every((result) => result.ok);
|
||||
respondJson(res, allOk ? 200 : 503, {
|
||||
status: allOk ? "ready" : "not_ready",
|
||||
...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}),
|
||||
checks,
|
||||
});
|
||||
return;
|
||||
}
|
||||
respondJson(res, 404, { error: { code: "not_found", message: "Endpoint not found." } });
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds the health handler to a TCP port and returns a handle whose
|
||||
* `close()` shuts the server cleanly during SIGTERM/SIGINT.
|
||||
*/
|
||||
export function startWorkerHealthServer(options: WorkerHealthServerOptions): WorkerHealthServerHandle {
|
||||
const handler = createWorkerHealthHandler({
|
||||
redisProbe: options.redisProbe,
|
||||
databaseProbe: options.databaseProbe,
|
||||
...(options.readinessProbes ? { readinessProbes: options.readinessProbes } : {}),
|
||||
...(options.workerGroup ? { workerGroup: options.workerGroup } : {}),
|
||||
...(options.probeTimeoutMs !== undefined ? { probeTimeoutMs: options.probeTimeoutMs } : {}),
|
||||
});
|
||||
|
||||
const port = options.port ?? DEFAULT_WORKER_HEALTH_PORT;
|
||||
const host = options.host ?? "0.0.0.0";
|
||||
|
||||
const server: Server = createServer((req, res) => {
|
||||
void handler(req, res).catch((error: unknown) => {
|
||||
options.logger?.error?.({ error: serialiseError(error) }, "worker health server handler failed");
|
||||
if (!res.headersSent) {
|
||||
respondJson(res, 500, { error: { code: "internal", message: "Internal server error." } });
|
||||
} else {
|
||||
res.end();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
server.listen(port, host);
|
||||
options.logger?.info?.({ port, host, workerGroup: options.workerGroup }, "worker health server listening");
|
||||
|
||||
return {
|
||||
port,
|
||||
close: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function runWithTimeout(probe: HealthProbe, timeoutMs: number, label: string): Promise<HealthProbeResult> {
|
||||
try {
|
||||
const result = await Promise.race([
|
||||
Promise.resolve().then(() => probe()),
|
||||
new Promise<HealthProbeResult>((resolve) =>
|
||||
setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs)
|
||||
),
|
||||
]);
|
||||
return result;
|
||||
} catch (error) {
|
||||
return { ok: false, error: serialiseError(error).message };
|
||||
}
|
||||
}
|
||||
|
||||
function respondJson(res: ServerResponse, statusCode: number, body: unknown): void {
|
||||
res.statusCode = statusCode;
|
||||
res.setHeader("content-type", "application/json");
|
||||
res.setHeader("cache-control", "no-store");
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
function serialiseError(error: unknown): { name: string; message: string } {
|
||||
if (error instanceof Error) {
|
||||
return { name: error.name, message: error.message.slice(0, 200) };
|
||||
}
|
||||
return { name: "UnknownError", message: String(error).slice(0, 200) };
|
||||
}
|
||||
Reference in new issue
Block a user