Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/worker/src/health.ts
T

481 lines
18 KiB
TypeScript

import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type WorkerConfigRedisParseStatus } from "@hivegrid/config";
import { getDbClient } from "@hivegrid/db/client.server";
import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@hivegrid/queue";
import { Queue } from "bullmq";
import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js";
import { getMissingWorkerGdprExportStorageEnv, getMissingWorkerStorageEnv } from "./storage.js";
export type WorkerHealthEnv = EnvRecord;
export type WorkerHealthStatus = "disabled" | "ready" | "misconfigured";
export type WorkerHealthRedisSource = "BULLMQ_REDIS_URL" | "REDIS_URL" | "none";
export type WorkerHealthDatabaseStatus = "skipped" | "ready" | "misconfigured";
export type WorkerHealthRedisStatus = "skipped" | "ready" | "misconfigured";
export type WorkerHealthStorageStatus = "skipped" | "ready" | "misconfigured";
export type WorkerHealthImportStatus = "skipped" | "ready" | "misconfigured";
export type WorkerHealthMailStatus = "skipped" | "ready" | "misconfigured";
export type WorkerHealthPluginRuntimeStatus = "skipped" | "ready" | "misconfigured";
export type WorkerHealthRequiredModeStatus = "skipped" | "ready" | "misconfigured";
export type WorkerHealthReport = {
status: WorkerHealthStatus;
revision: string;
workerEnabled: boolean;
workerGroup: WorkerQueueGroup | null;
redisConfigured: boolean;
redisSource: WorkerHealthRedisSource;
redisParseStatus: "not_configured" | "valid" | "invalid";
redisStatus: WorkerHealthRedisStatus;
redisErrorCode?: string;
databaseConfigured: boolean;
databaseStatus: WorkerHealthDatabaseStatus;
databaseErrorCode?: string;
storageConfigured: boolean;
storageStatus: WorkerHealthStorageStatus;
mailjetConfigured: boolean;
mailjetStatus: WorkerHealthMailStatus;
importConfigured: boolean;
importStatus: WorkerHealthImportStatus;
pluginRuntimeConfigured: boolean;
pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus;
queues: readonly QueueName[];
errors: readonly string[];
};
export type WorkerHealthCliIo = {
stderr?: Pick<NodeJS.WriteStream, "write">;
stdout: Pick<NodeJS.WriteStream, "write">;
};
export type WorkerDatabaseProbe = () => Promise<void> | void;
export type WorkerRedisProbe = (redisUrl: string) => Promise<void> | void;
export type WorkerHealthDependencies = {
databaseProbe?: WorkerDatabaseProbe;
redisProbe?: WorkerRedisProbe;
};
export async function buildWorkerHealthReport(
env: WorkerHealthEnv = process.env as WorkerHealthEnv,
dependencies: WorkerHealthDependencies = {}
): Promise<WorkerHealthReport> {
const result = loadWorkerConfig(env);
const { config } = result;
const shouldProbeDatabase = config.enabled && result.ok;
const shouldProbeRedis = config.enabled && result.ok;
const databaseConfigured = Boolean(env.DATABASE_URL);
const [redisReadiness, databaseReadiness] = await Promise.all([
checkRedisReadiness(shouldProbeRedis, config.redis.url, dependencies.redisProbe),
checkDatabaseReadiness(shouldProbeDatabase, dependencies.databaseProbe),
]);
const storageReadiness = checkStorageReadiness(
config.enabled && (env.EXPORT_RETENTION_QUEUE_ENABLED === "true" || ["full", "portal-gdpr-export"].includes(resolveImportQueueRuntimeMode(env))),
env
);
const mailReadiness = checkMailReadiness(config.enabled, config.mailjet.configured, isMailProviderRequired(env), env);
const importReadiness = checkImportReadiness(config.enabled && result.ok && getWorkerGroup(env) === "import", env);
const pluginRuntimeReadiness = checkPluginRuntimeReadiness(config.enabled && result.ok, getWorkerGroup(env), env);
const requiredModeReadiness = checkRequiredGeneralWorkerReadiness(config.enabled && result.ok && getWorkerGroup(env) === "general", env);
const status = toWorkerHealthStatus(
config.enabled,
result.ok,
redisReadiness.status,
databaseReadiness.status,
storageReadiness.status,
mailReadiness.status,
importReadiness.status,
pluginRuntimeReadiness.status,
requiredModeReadiness.status
);
return {
status,
revision: env.APP_REVISION || "unknown",
workerEnabled: config.enabled,
workerGroup: getWorkerGroup(env),
redisConfigured: config.redis.configured,
redisSource: config.redis.source,
redisParseStatus: toHealthRedisParseStatus(config.redis.parseStatus),
redisStatus: redisReadiness.status,
...(redisReadiness.errorCode ? { redisErrorCode: redisReadiness.errorCode } : {}),
databaseConfigured,
databaseStatus: databaseReadiness.status,
...(databaseReadiness.errorCode ? { databaseErrorCode: databaseReadiness.errorCode } : {}),
storageConfigured: storageReadiness.configured,
storageStatus: storageReadiness.status,
mailjetConfigured: mailReadiness.configured,
mailjetStatus: mailReadiness.status,
importConfigured: importReadiness.configured,
importStatus: importReadiness.status,
pluginRuntimeConfigured: pluginRuntimeReadiness.configured,
pluginRuntimeStatus: pluginRuntimeReadiness.status,
queues: getWorkerQueueNames(env),
errors: [
...result.diagnostics.map(formatWorkerConfigDiagnostic),
...storageReadiness.errors,
...mailReadiness.errors,
...importReadiness.errors,
...pluginRuntimeReadiness.errors,
...requiredModeReadiness.errors,
],
};
}
function getWorkerGroup(env: WorkerHealthEnv): WorkerQueueGroup | null {
const group = env.WORKER_QUEUE_GROUP;
return typeof group === "string" && isWorkerQueueGroup(group) ? group : null;
}
export async function runWorkerHealthCli(
env: WorkerHealthEnv = process.env as WorkerHealthEnv,
io: WorkerHealthCliIo = process,
dependencies: WorkerHealthDependencies = {}
): Promise<number> {
const report = await buildWorkerHealthReport(env, dependencies);
io.stdout.write(`${JSON.stringify(report)}\n`);
return report.status === "misconfigured" ? 1 : 0;
}
function toWorkerHealthStatus(
workerEnabled: boolean,
redisReady: boolean,
redisStatus: WorkerHealthRedisStatus,
databaseStatus: WorkerHealthDatabaseStatus,
storageStatus: WorkerHealthStorageStatus,
mailStatus: WorkerHealthMailStatus,
importStatus: WorkerHealthImportStatus,
pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus,
requiredModeStatus: WorkerHealthRequiredModeStatus
): WorkerHealthStatus {
if (!workerEnabled) {
return "disabled";
}
if (
!redisReady ||
redisStatus === "misconfigured" ||
databaseStatus === "misconfigured" ||
storageStatus === "misconfigured" ||
mailStatus === "misconfigured" ||
importStatus === "misconfigured" ||
pluginRuntimeStatus === "misconfigured" ||
requiredModeStatus === "misconfigured"
) {
return "misconfigured";
}
return "ready";
}
function checkPluginRuntimeReadiness(
shouldCheck: boolean,
workerGroup: WorkerQueueGroup | null,
env: WorkerHealthEnv
): { configured: boolean; errors: string[]; status: WorkerHealthPluginRuntimeStatus } {
const productionRuntimeEnabled = env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true";
if (!shouldCheck || !productionRuntimeEnabled || (workerGroup !== "general" && workerGroup !== "plugin")) {
return { configured: false, errors: [], status: "skipped" };
}
const errors: string[] = [];
if (!hasTrimmedValue(env.PLUGIN_TRUSTED_PUBLIC_KEYS)) {
errors.push("PLUGIN_TRUSTED_PUBLIC_KEYS:missing-plugin-trust-root:Required when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.");
}
if (workerGroup === "general") {
if (!hasTrimmedValue(env.BULLMQ_REDIS_URL) && !hasTrimmedValue(env.REDIS_URL)) {
errors.push(
"BULLMQ_REDIS_URL:missing-plugin-runtime-redis:Required for plugin hook fanout and capability-use subscribers when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
);
}
if (env.OUTBOX_DISPATCHER_ENABLED !== "true") {
errors.push(
"OUTBOX_DISPATCHER_ENABLED:missing-plugin-runtime-dispatcher:Required on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
);
}
if (env.PLUGIN_HOOK_FANOUT_ENABLED === "false") {
errors.push(
"PLUGIN_HOOK_FANOUT_ENABLED:disabled-plugin-runtime-fanout:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
);
}
if (env.PLUGIN_CAPABILITY_USE_ENABLED === "false") {
errors.push(
"PLUGIN_CAPABILITY_USE_ENABLED:disabled-plugin-capability-use:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
);
}
}
return { configured: true, errors, status: errors.length > 0 ? "misconfigured" : "ready" };
}
function isMailProviderRequired(env: WorkerHealthEnv): boolean {
return env.WORKER_QUEUE_GROUP === "mail" && env.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED === "true";
}
function checkRequiredGeneralWorkerReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { errors: string[]; status: WorkerHealthRequiredModeStatus } {
if (!shouldCheck) {
return { errors: [], status: "skipped" };
}
const errors: string[] = [];
if (env.OUTBOX_DISPATCHER_REQUIRED === "true" && env.OUTBOX_DISPATCHER_ENABLED !== "true") {
errors.push("OUTBOX_DISPATCHER_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and OUTBOX_DISPATCHER_REQUIRED=true.");
}
const workflowTriggerRequired = env.WORKFLOW_TRIGGER_OUTBOX_REQUIRED === "true";
const workflowTriggerEnabled = env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true";
const workflowTriggerModeLabel = workflowTriggerRequired ? "WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true" : "WORKFLOW_TRIGGER_OUTBOX_ENABLED=true";
if (workflowTriggerRequired || workflowTriggerEnabled) {
if (env.OUTBOX_DISPATCHER_ENABLED !== "true" && env.OUTBOX_DISPATCHER_REQUIRED !== "true") {
errors.push(`OUTBOX_DISPATCHER_ENABLED:missing-workflow-trigger-dispatcher:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`);
}
if (workflowTriggerRequired && !workflowTriggerEnabled) {
errors.push(
"WORKFLOW_TRIGGER_OUTBOX_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true."
);
}
const executorMode = parseWorkflowTriggerExecutorModeForHealth(env.WORKFLOW_TRIGGER_EXECUTOR_MODE);
if (!executorMode) {
errors.push(
`WORKFLOW_TRIGGER_EXECUTOR_MODE:invalid-workflow-executor-mode:Expected http, native, or hybrid when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`
);
} else if (workflowTriggerExecutorNeedsHttpBridge(executorMode)) {
if (!hasTrimmedValue(env.API_ACCESS_TOKEN)) {
errors.push(
`API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
);
}
if (!hasTrimmedValue(env.WORKFLOW_TRIGGER_EXECUTOR_URL) && !hasTrimmedValue(env.APP_URL) && !hasTrimmedValue(env.WEB_ORIGIN)) {
errors.push(
`WORKFLOW_TRIGGER_EXECUTOR_URL:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
);
}
}
}
return { errors, status: errors.length > 0 ? "misconfigured" : "ready" };
}
function parseWorkflowTriggerExecutorModeForHealth(value: unknown): "http" | "native" | "hybrid" | null {
if (typeof value !== "string" || value.trim().length === 0) {
return "http";
}
const normalized = value.trim().toLowerCase();
return normalized === "http" || normalized === "native" || normalized === "hybrid" ? normalized : null;
}
function workflowTriggerExecutorNeedsHttpBridge(mode: "http" | "native" | "hybrid"): boolean {
return mode !== "native";
}
function hasTrimmedValue(value: string | undefined): boolean {
return typeof value === "string" && value.trim().length > 0;
}
function checkMailReadiness(
shouldCheck: boolean,
configured: boolean,
required: boolean,
env: WorkerHealthEnv
): { configured: boolean; errors: string[]; status: WorkerHealthMailStatus } {
if (!shouldCheck) {
return { configured: false, errors: [], status: "skipped" };
}
if (!configured) {
if (required) {
const missing = ["MAILJET_API_KEY", "MAILJET_SECRET_KEY", "MAILJET_FROM_EMAIL"].filter((name) => !env[name]?.trim());
return {
configured: false,
errors: missing.map(
(name) => `${name}:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.`
),
status: "misconfigured",
};
}
return { configured: false, errors: [], status: "skipped" };
}
return { configured: true, errors: [], status: "ready" };
}
function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthImportStatus } {
if (!shouldCheck) {
return { configured: false, errors: [], status: "skipped" };
}
const importEnabled = env.IMPORT_QUEUE_ENABLED === "true";
const runtimeReady = env.IMPORT_QUEUE_RUNTIME_READY === "true";
const mode = resolveImportQueueRuntimeMode(env);
if (mode === "idle") {
return { configured: false, errors: [], status: "skipped" };
}
if (mode === "portal-gdpr-export" || mode === "full") {
return { configured: true, errors: [], status: "ready" };
}
if (!importEnabled || !runtimeReady) {
const errors: string[] = [];
if (!importEnabled) {
errors.push(
"IMPORT_QUEUE_ENABLED:missing-import-runtime-gate:Required with IMPORT_QUEUE_RUNTIME_READY=true before WORKER_QUEUE_GROUP=import can consume hivegrid.imports/hivegrid.exports."
);
}
if (!runtimeReady) {
errors.push(
"IMPORT_QUEUE_RUNTIME_READY:missing-import-runtime-gate:Required with IMPORT_QUEUE_ENABLED=true after import, export, and scan services are wired."
);
}
return {
configured: false,
errors,
status: "misconfigured",
};
}
return {
configured: false,
errors: ["WORKER_QUEUE_GROUP:invalid-import-runtime-mode:Import worker queue gates do not resolve to an allowed runtime mode."],
status: "misconfigured",
};
}
function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthStorageStatus } {
if (!shouldCheck) {
return { configured: false, errors: [], status: "skipped" };
}
const importMode = resolveImportQueueRuntimeMode(env);
const ownsGdprExports = importMode === "full" || importMode === "portal-gdpr-export";
const missing = ownsGdprExports ? getMissingWorkerGdprExportStorageEnv(env) : getMissingWorkerStorageEnv(env);
if (missing.length > 0) {
const requiredBy = [
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? ["EXPORT_RETENTION_QUEUE_ENABLED=true"] : []),
...(importMode === "portal-gdpr-export" ? ["PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true"] : []),
...(importMode === "full" ? ["the full import runtime owns GDPR export jobs"] : []),
].join(" or ");
return {
configured: false,
errors: missing.map((name) => `${name}:missing-storage-config:Required when ${requiredBy}.`),
status: "misconfigured",
};
}
return { configured: true, errors: [], status: "ready" };
}
async function checkRedisReadiness(
shouldProbe: boolean,
redisUrl: string | undefined,
redisProbe: WorkerRedisProbe = defaultRedisProbe
): Promise<{ status: WorkerHealthRedisStatus; errorCode?: string }> {
if (!shouldProbe || !redisUrl) {
return { status: "skipped" };
}
try {
await redisProbe(redisUrl);
return { status: "ready" };
} catch (error) {
return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error, "REDIS_READINESS_FAILED") };
}
}
async function checkDatabaseReadiness(
shouldProbe: boolean,
databaseProbe: WorkerDatabaseProbe = defaultDatabaseProbe
): Promise<{ status: WorkerHealthDatabaseStatus; errorCode?: string }> {
if (!shouldProbe) {
return { status: "skipped" };
}
try {
await databaseProbe();
return { status: "ready" };
} catch (error) {
return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error) };
}
}
async function defaultDatabaseProbe(): Promise<void> {
const db = getDbClient();
try {
await db.$queryRaw`SELECT 1`;
} finally {
await db.$disconnect();
}
}
async function defaultRedisProbe(redisUrl: string): Promise<void> {
const queue = new Queue("__hivegrid_worker_health__", {
connection: parseRedisUrl(redisUrl),
});
try {
await withTimeout(queue.waitUntilReady(), 2_000);
} finally {
await queue.close();
}
}
function toHealthRedisParseStatus(status: WorkerConfigRedisParseStatus): WorkerHealthReport["redisParseStatus"] {
if (status === "valid") {
return "valid";
}
if (status === "not-provided") {
return "not_configured";
}
return "invalid";
}
function toRedactedDependencyErrorCode(error: unknown, fallback = "DATABASE_READINESS_FAILED"): string {
if (typeof error === "object" && error !== null && "code" in error) {
const code = (error as { code?: unknown }).code;
if (typeof code === "string" && /^[A-Z0-9_-]{1,64}$/i.test(code)) {
return code;
}
}
return fallback;
}
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
let timeout: NodeJS.Timeout | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timeout = setTimeout(() => reject(Object.assign(new Error("Redis readiness timed out."), { code: "REDIS_TIMEOUT" })), timeoutMs);
}),
]);
} finally {
if (timeout) {
clearTimeout(timeout);
}
}
}
function formatWorkerConfigDiagnostic(diagnostic: WorkerConfigDiagnostic): string {
return `${diagnostic.name}:${diagnostic.code}:${diagnostic.message}`;
}
if (import.meta.url === `file://${process.argv[1]}`) {
runWorkerHealthCli()
.then((exitCode) => {
process.exitCode = exitCode;
})
.catch(() => {
process.exitCode = 1;
});
}