481 lines
18 KiB
TypeScript
481 lines
18 KiB
TypeScript
import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type WorkerConfigRedisParseStatus } from "@hivegrid/config";
|
|
import { getDbClient } from "@hivegrid/db/client.server";
|
|
import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@hivegrid/queue";
|
|
import { Queue } from "bullmq";
|
|
import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js";
|
|
import { getMissingWorkerGdprExportStorageEnv, getMissingWorkerStorageEnv } from "./storage.js";
|
|
|
|
export type WorkerHealthEnv = EnvRecord;
|
|
|
|
export type WorkerHealthStatus = "disabled" | "ready" | "misconfigured";
|
|
export type WorkerHealthRedisSource = "BULLMQ_REDIS_URL" | "REDIS_URL" | "none";
|
|
|
|
export type WorkerHealthDatabaseStatus = "skipped" | "ready" | "misconfigured";
|
|
export type WorkerHealthRedisStatus = "skipped" | "ready" | "misconfigured";
|
|
export type WorkerHealthStorageStatus = "skipped" | "ready" | "misconfigured";
|
|
export type WorkerHealthImportStatus = "skipped" | "ready" | "misconfigured";
|
|
export type WorkerHealthMailStatus = "skipped" | "ready" | "misconfigured";
|
|
export type WorkerHealthPluginRuntimeStatus = "skipped" | "ready" | "misconfigured";
|
|
export type WorkerHealthRequiredModeStatus = "skipped" | "ready" | "misconfigured";
|
|
|
|
export type WorkerHealthReport = {
|
|
status: WorkerHealthStatus;
|
|
revision: string;
|
|
workerEnabled: boolean;
|
|
workerGroup: WorkerQueueGroup | null;
|
|
redisConfigured: boolean;
|
|
redisSource: WorkerHealthRedisSource;
|
|
redisParseStatus: "not_configured" | "valid" | "invalid";
|
|
redisStatus: WorkerHealthRedisStatus;
|
|
redisErrorCode?: string;
|
|
databaseConfigured: boolean;
|
|
databaseStatus: WorkerHealthDatabaseStatus;
|
|
databaseErrorCode?: string;
|
|
storageConfigured: boolean;
|
|
storageStatus: WorkerHealthStorageStatus;
|
|
mailjetConfigured: boolean;
|
|
mailjetStatus: WorkerHealthMailStatus;
|
|
importConfigured: boolean;
|
|
importStatus: WorkerHealthImportStatus;
|
|
pluginRuntimeConfigured: boolean;
|
|
pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus;
|
|
queues: readonly QueueName[];
|
|
errors: readonly string[];
|
|
};
|
|
|
|
export type WorkerHealthCliIo = {
|
|
stderr?: Pick<NodeJS.WriteStream, "write">;
|
|
stdout: Pick<NodeJS.WriteStream, "write">;
|
|
};
|
|
|
|
export type WorkerDatabaseProbe = () => Promise<void> | void;
|
|
export type WorkerRedisProbe = (redisUrl: string) => Promise<void> | void;
|
|
|
|
export type WorkerHealthDependencies = {
|
|
databaseProbe?: WorkerDatabaseProbe;
|
|
redisProbe?: WorkerRedisProbe;
|
|
};
|
|
|
|
export async function buildWorkerHealthReport(
|
|
env: WorkerHealthEnv = process.env as WorkerHealthEnv,
|
|
dependencies: WorkerHealthDependencies = {}
|
|
): Promise<WorkerHealthReport> {
|
|
const result = loadWorkerConfig(env);
|
|
const { config } = result;
|
|
const shouldProbeDatabase = config.enabled && result.ok;
|
|
const shouldProbeRedis = config.enabled && result.ok;
|
|
const databaseConfigured = Boolean(env.DATABASE_URL);
|
|
const [redisReadiness, databaseReadiness] = await Promise.all([
|
|
checkRedisReadiness(shouldProbeRedis, config.redis.url, dependencies.redisProbe),
|
|
checkDatabaseReadiness(shouldProbeDatabase, dependencies.databaseProbe),
|
|
]);
|
|
const storageReadiness = checkStorageReadiness(
|
|
config.enabled && (env.EXPORT_RETENTION_QUEUE_ENABLED === "true" || ["full", "portal-gdpr-export"].includes(resolveImportQueueRuntimeMode(env))),
|
|
env
|
|
);
|
|
const mailReadiness = checkMailReadiness(config.enabled, config.mailjet.configured, isMailProviderRequired(env), env);
|
|
const importReadiness = checkImportReadiness(config.enabled && result.ok && getWorkerGroup(env) === "import", env);
|
|
const pluginRuntimeReadiness = checkPluginRuntimeReadiness(config.enabled && result.ok, getWorkerGroup(env), env);
|
|
const requiredModeReadiness = checkRequiredGeneralWorkerReadiness(config.enabled && result.ok && getWorkerGroup(env) === "general", env);
|
|
const status = toWorkerHealthStatus(
|
|
config.enabled,
|
|
result.ok,
|
|
redisReadiness.status,
|
|
databaseReadiness.status,
|
|
storageReadiness.status,
|
|
mailReadiness.status,
|
|
importReadiness.status,
|
|
pluginRuntimeReadiness.status,
|
|
requiredModeReadiness.status
|
|
);
|
|
|
|
return {
|
|
status,
|
|
revision: env.APP_REVISION || "unknown",
|
|
workerEnabled: config.enabled,
|
|
workerGroup: getWorkerGroup(env),
|
|
redisConfigured: config.redis.configured,
|
|
redisSource: config.redis.source,
|
|
redisParseStatus: toHealthRedisParseStatus(config.redis.parseStatus),
|
|
redisStatus: redisReadiness.status,
|
|
...(redisReadiness.errorCode ? { redisErrorCode: redisReadiness.errorCode } : {}),
|
|
databaseConfigured,
|
|
databaseStatus: databaseReadiness.status,
|
|
...(databaseReadiness.errorCode ? { databaseErrorCode: databaseReadiness.errorCode } : {}),
|
|
storageConfigured: storageReadiness.configured,
|
|
storageStatus: storageReadiness.status,
|
|
mailjetConfigured: mailReadiness.configured,
|
|
mailjetStatus: mailReadiness.status,
|
|
importConfigured: importReadiness.configured,
|
|
importStatus: importReadiness.status,
|
|
pluginRuntimeConfigured: pluginRuntimeReadiness.configured,
|
|
pluginRuntimeStatus: pluginRuntimeReadiness.status,
|
|
queues: getWorkerQueueNames(env),
|
|
errors: [
|
|
...result.diagnostics.map(formatWorkerConfigDiagnostic),
|
|
...storageReadiness.errors,
|
|
...mailReadiness.errors,
|
|
...importReadiness.errors,
|
|
...pluginRuntimeReadiness.errors,
|
|
...requiredModeReadiness.errors,
|
|
],
|
|
};
|
|
}
|
|
|
|
function getWorkerGroup(env: WorkerHealthEnv): WorkerQueueGroup | null {
|
|
const group = env.WORKER_QUEUE_GROUP;
|
|
return typeof group === "string" && isWorkerQueueGroup(group) ? group : null;
|
|
}
|
|
|
|
export async function runWorkerHealthCli(
|
|
env: WorkerHealthEnv = process.env as WorkerHealthEnv,
|
|
io: WorkerHealthCliIo = process,
|
|
dependencies: WorkerHealthDependencies = {}
|
|
): Promise<number> {
|
|
const report = await buildWorkerHealthReport(env, dependencies);
|
|
io.stdout.write(`${JSON.stringify(report)}\n`);
|
|
return report.status === "misconfigured" ? 1 : 0;
|
|
}
|
|
|
|
function toWorkerHealthStatus(
|
|
workerEnabled: boolean,
|
|
redisReady: boolean,
|
|
redisStatus: WorkerHealthRedisStatus,
|
|
databaseStatus: WorkerHealthDatabaseStatus,
|
|
storageStatus: WorkerHealthStorageStatus,
|
|
mailStatus: WorkerHealthMailStatus,
|
|
importStatus: WorkerHealthImportStatus,
|
|
pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus,
|
|
requiredModeStatus: WorkerHealthRequiredModeStatus
|
|
): WorkerHealthStatus {
|
|
if (!workerEnabled) {
|
|
return "disabled";
|
|
}
|
|
|
|
if (
|
|
!redisReady ||
|
|
redisStatus === "misconfigured" ||
|
|
databaseStatus === "misconfigured" ||
|
|
storageStatus === "misconfigured" ||
|
|
mailStatus === "misconfigured" ||
|
|
importStatus === "misconfigured" ||
|
|
pluginRuntimeStatus === "misconfigured" ||
|
|
requiredModeStatus === "misconfigured"
|
|
) {
|
|
return "misconfigured";
|
|
}
|
|
|
|
return "ready";
|
|
}
|
|
|
|
function checkPluginRuntimeReadiness(
|
|
shouldCheck: boolean,
|
|
workerGroup: WorkerQueueGroup | null,
|
|
env: WorkerHealthEnv
|
|
): { configured: boolean; errors: string[]; status: WorkerHealthPluginRuntimeStatus } {
|
|
const productionRuntimeEnabled = env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true";
|
|
if (!shouldCheck || !productionRuntimeEnabled || (workerGroup !== "general" && workerGroup !== "plugin")) {
|
|
return { configured: false, errors: [], status: "skipped" };
|
|
}
|
|
|
|
const errors: string[] = [];
|
|
if (!hasTrimmedValue(env.PLUGIN_TRUSTED_PUBLIC_KEYS)) {
|
|
errors.push("PLUGIN_TRUSTED_PUBLIC_KEYS:missing-plugin-trust-root:Required when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.");
|
|
}
|
|
|
|
if (workerGroup === "general") {
|
|
if (!hasTrimmedValue(env.BULLMQ_REDIS_URL) && !hasTrimmedValue(env.REDIS_URL)) {
|
|
errors.push(
|
|
"BULLMQ_REDIS_URL:missing-plugin-runtime-redis:Required for plugin hook fanout and capability-use subscribers when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
|
);
|
|
}
|
|
if (env.OUTBOX_DISPATCHER_ENABLED !== "true") {
|
|
errors.push(
|
|
"OUTBOX_DISPATCHER_ENABLED:missing-plugin-runtime-dispatcher:Required on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
|
);
|
|
}
|
|
if (env.PLUGIN_HOOK_FANOUT_ENABLED === "false") {
|
|
errors.push(
|
|
"PLUGIN_HOOK_FANOUT_ENABLED:disabled-plugin-runtime-fanout:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
|
);
|
|
}
|
|
if (env.PLUGIN_CAPABILITY_USE_ENABLED === "false") {
|
|
errors.push(
|
|
"PLUGIN_CAPABILITY_USE_ENABLED:disabled-plugin-capability-use:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
|
|
);
|
|
}
|
|
}
|
|
|
|
return { configured: true, errors, status: errors.length > 0 ? "misconfigured" : "ready" };
|
|
}
|
|
|
|
function isMailProviderRequired(env: WorkerHealthEnv): boolean {
|
|
return env.WORKER_QUEUE_GROUP === "mail" && env.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED === "true";
|
|
}
|
|
|
|
function checkRequiredGeneralWorkerReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { errors: string[]; status: WorkerHealthRequiredModeStatus } {
|
|
if (!shouldCheck) {
|
|
return { errors: [], status: "skipped" };
|
|
}
|
|
|
|
const errors: string[] = [];
|
|
|
|
if (env.OUTBOX_DISPATCHER_REQUIRED === "true" && env.OUTBOX_DISPATCHER_ENABLED !== "true") {
|
|
errors.push("OUTBOX_DISPATCHER_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and OUTBOX_DISPATCHER_REQUIRED=true.");
|
|
}
|
|
|
|
const workflowTriggerRequired = env.WORKFLOW_TRIGGER_OUTBOX_REQUIRED === "true";
|
|
const workflowTriggerEnabled = env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true";
|
|
const workflowTriggerModeLabel = workflowTriggerRequired ? "WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true" : "WORKFLOW_TRIGGER_OUTBOX_ENABLED=true";
|
|
|
|
if (workflowTriggerRequired || workflowTriggerEnabled) {
|
|
if (env.OUTBOX_DISPATCHER_ENABLED !== "true" && env.OUTBOX_DISPATCHER_REQUIRED !== "true") {
|
|
errors.push(`OUTBOX_DISPATCHER_ENABLED:missing-workflow-trigger-dispatcher:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`);
|
|
}
|
|
|
|
if (workflowTriggerRequired && !workflowTriggerEnabled) {
|
|
errors.push(
|
|
"WORKFLOW_TRIGGER_OUTBOX_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true."
|
|
);
|
|
}
|
|
|
|
const executorMode = parseWorkflowTriggerExecutorModeForHealth(env.WORKFLOW_TRIGGER_EXECUTOR_MODE);
|
|
if (!executorMode) {
|
|
errors.push(
|
|
`WORKFLOW_TRIGGER_EXECUTOR_MODE:invalid-workflow-executor-mode:Expected http, native, or hybrid when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`
|
|
);
|
|
} else if (workflowTriggerExecutorNeedsHttpBridge(executorMode)) {
|
|
if (!hasTrimmedValue(env.API_ACCESS_TOKEN)) {
|
|
errors.push(
|
|
`API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
|
|
);
|
|
}
|
|
|
|
if (!hasTrimmedValue(env.WORKFLOW_TRIGGER_EXECUTOR_URL) && !hasTrimmedValue(env.APP_URL) && !hasTrimmedValue(env.WEB_ORIGIN)) {
|
|
errors.push(
|
|
`WORKFLOW_TRIGGER_EXECUTOR_URL:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
return { errors, status: errors.length > 0 ? "misconfigured" : "ready" };
|
|
}
|
|
|
|
function parseWorkflowTriggerExecutorModeForHealth(value: unknown): "http" | "native" | "hybrid" | null {
|
|
if (typeof value !== "string" || value.trim().length === 0) {
|
|
return "http";
|
|
}
|
|
const normalized = value.trim().toLowerCase();
|
|
return normalized === "http" || normalized === "native" || normalized === "hybrid" ? normalized : null;
|
|
}
|
|
|
|
function workflowTriggerExecutorNeedsHttpBridge(mode: "http" | "native" | "hybrid"): boolean {
|
|
return mode !== "native";
|
|
}
|
|
|
|
function hasTrimmedValue(value: string | undefined): boolean {
|
|
return typeof value === "string" && value.trim().length > 0;
|
|
}
|
|
|
|
function checkMailReadiness(
|
|
shouldCheck: boolean,
|
|
configured: boolean,
|
|
required: boolean,
|
|
env: WorkerHealthEnv
|
|
): { configured: boolean; errors: string[]; status: WorkerHealthMailStatus } {
|
|
if (!shouldCheck) {
|
|
return { configured: false, errors: [], status: "skipped" };
|
|
}
|
|
|
|
if (!configured) {
|
|
if (required) {
|
|
const missing = ["MAILJET_API_KEY", "MAILJET_SECRET_KEY", "MAILJET_FROM_EMAIL"].filter((name) => !env[name]?.trim());
|
|
return {
|
|
configured: false,
|
|
errors: missing.map(
|
|
(name) => `${name}:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.`
|
|
),
|
|
status: "misconfigured",
|
|
};
|
|
}
|
|
return { configured: false, errors: [], status: "skipped" };
|
|
}
|
|
|
|
return { configured: true, errors: [], status: "ready" };
|
|
}
|
|
|
|
function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthImportStatus } {
|
|
if (!shouldCheck) {
|
|
return { configured: false, errors: [], status: "skipped" };
|
|
}
|
|
|
|
const importEnabled = env.IMPORT_QUEUE_ENABLED === "true";
|
|
const runtimeReady = env.IMPORT_QUEUE_RUNTIME_READY === "true";
|
|
const mode = resolveImportQueueRuntimeMode(env);
|
|
if (mode === "idle") {
|
|
return { configured: false, errors: [], status: "skipped" };
|
|
}
|
|
|
|
if (mode === "portal-gdpr-export" || mode === "full") {
|
|
return { configured: true, errors: [], status: "ready" };
|
|
}
|
|
|
|
if (!importEnabled || !runtimeReady) {
|
|
const errors: string[] = [];
|
|
if (!importEnabled) {
|
|
errors.push(
|
|
"IMPORT_QUEUE_ENABLED:missing-import-runtime-gate:Required with IMPORT_QUEUE_RUNTIME_READY=true before WORKER_QUEUE_GROUP=import can consume hivegrid.imports/hivegrid.exports."
|
|
);
|
|
}
|
|
if (!runtimeReady) {
|
|
errors.push(
|
|
"IMPORT_QUEUE_RUNTIME_READY:missing-import-runtime-gate:Required with IMPORT_QUEUE_ENABLED=true after import, export, and scan services are wired."
|
|
);
|
|
}
|
|
return {
|
|
configured: false,
|
|
errors,
|
|
status: "misconfigured",
|
|
};
|
|
}
|
|
|
|
return {
|
|
configured: false,
|
|
errors: ["WORKER_QUEUE_GROUP:invalid-import-runtime-mode:Import worker queue gates do not resolve to an allowed runtime mode."],
|
|
status: "misconfigured",
|
|
};
|
|
}
|
|
|
|
function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthStorageStatus } {
|
|
if (!shouldCheck) {
|
|
return { configured: false, errors: [], status: "skipped" };
|
|
}
|
|
|
|
const importMode = resolveImportQueueRuntimeMode(env);
|
|
const ownsGdprExports = importMode === "full" || importMode === "portal-gdpr-export";
|
|
const missing = ownsGdprExports ? getMissingWorkerGdprExportStorageEnv(env) : getMissingWorkerStorageEnv(env);
|
|
if (missing.length > 0) {
|
|
const requiredBy = [
|
|
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? ["EXPORT_RETENTION_QUEUE_ENABLED=true"] : []),
|
|
...(importMode === "portal-gdpr-export" ? ["PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true"] : []),
|
|
...(importMode === "full" ? ["the full import runtime owns GDPR export jobs"] : []),
|
|
].join(" or ");
|
|
return {
|
|
configured: false,
|
|
errors: missing.map((name) => `${name}:missing-storage-config:Required when ${requiredBy}.`),
|
|
status: "misconfigured",
|
|
};
|
|
}
|
|
|
|
return { configured: true, errors: [], status: "ready" };
|
|
}
|
|
|
|
async function checkRedisReadiness(
|
|
shouldProbe: boolean,
|
|
redisUrl: string | undefined,
|
|
redisProbe: WorkerRedisProbe = defaultRedisProbe
|
|
): Promise<{ status: WorkerHealthRedisStatus; errorCode?: string }> {
|
|
if (!shouldProbe || !redisUrl) {
|
|
return { status: "skipped" };
|
|
}
|
|
|
|
try {
|
|
await redisProbe(redisUrl);
|
|
return { status: "ready" };
|
|
} catch (error) {
|
|
return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error, "REDIS_READINESS_FAILED") };
|
|
}
|
|
}
|
|
|
|
async function checkDatabaseReadiness(
|
|
shouldProbe: boolean,
|
|
databaseProbe: WorkerDatabaseProbe = defaultDatabaseProbe
|
|
): Promise<{ status: WorkerHealthDatabaseStatus; errorCode?: string }> {
|
|
if (!shouldProbe) {
|
|
return { status: "skipped" };
|
|
}
|
|
|
|
try {
|
|
await databaseProbe();
|
|
return { status: "ready" };
|
|
} catch (error) {
|
|
return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error) };
|
|
}
|
|
}
|
|
|
|
async function defaultDatabaseProbe(): Promise<void> {
|
|
const db = getDbClient();
|
|
|
|
try {
|
|
await db.$queryRaw`SELECT 1`;
|
|
} finally {
|
|
await db.$disconnect();
|
|
}
|
|
}
|
|
|
|
async function defaultRedisProbe(redisUrl: string): Promise<void> {
|
|
const queue = new Queue("__hivegrid_worker_health__", {
|
|
connection: parseRedisUrl(redisUrl),
|
|
});
|
|
|
|
try {
|
|
await withTimeout(queue.waitUntilReady(), 2_000);
|
|
} finally {
|
|
await queue.close();
|
|
}
|
|
}
|
|
|
|
function toHealthRedisParseStatus(status: WorkerConfigRedisParseStatus): WorkerHealthReport["redisParseStatus"] {
|
|
if (status === "valid") {
|
|
return "valid";
|
|
}
|
|
|
|
if (status === "not-provided") {
|
|
return "not_configured";
|
|
}
|
|
|
|
return "invalid";
|
|
}
|
|
|
|
function toRedactedDependencyErrorCode(error: unknown, fallback = "DATABASE_READINESS_FAILED"): string {
|
|
if (typeof error === "object" && error !== null && "code" in error) {
|
|
const code = (error as { code?: unknown }).code;
|
|
if (typeof code === "string" && /^[A-Z0-9_-]{1,64}$/i.test(code)) {
|
|
return code;
|
|
}
|
|
}
|
|
|
|
return fallback;
|
|
}
|
|
|
|
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
|
let timeout: NodeJS.Timeout | undefined;
|
|
try {
|
|
return await Promise.race([
|
|
promise,
|
|
new Promise<never>((_, reject) => {
|
|
timeout = setTimeout(() => reject(Object.assign(new Error("Redis readiness timed out."), { code: "REDIS_TIMEOUT" })), timeoutMs);
|
|
}),
|
|
]);
|
|
} finally {
|
|
if (timeout) {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
}
|
|
|
|
function formatWorkerConfigDiagnostic(diagnostic: WorkerConfigDiagnostic): string {
|
|
return `${diagnostic.name}:${diagnostic.code}:${diagnostic.message}`;
|
|
}
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
runWorkerHealthCli()
|
|
.then((exitCode) => {
|
|
process.exitCode = exitCode;
|
|
})
|
|
.catch(() => {
|
|
process.exitCode = 1;
|
|
});
|
|
}
|