import { loadRequiredWorkerConfig, type EnvRecord } from "@hivegrid/config"; import { recordQueueFailedJob, type QueueFailedJobRecordInput } from "@hivegrid/db/queue-failed-jobs.server"; import { createNoopErrorReporter, reportError, type ErrorReporter, type ErrorReportSeverity } from "@hivegrid/error-reporter"; import type { DefaultMetrics } from "@hivegrid/metrics"; import { parseRedisUrl, queueNames, type QueueName, type TypedProcessor } from "@hivegrid/queue"; import { createWorker } from "@hivegrid/queue/runtime"; import { createRegisteredProcessor, type HandlerDependencies } from "./handlers.js"; import { wrapProcessorWithRlsContext } from "./rls-context.js"; import { redactCompletedWorkspaceOwnerInvitationJob, redactTerminalFailedWorkspaceOwnerInvitationJob } from "./security/workspace-owner-invitation-secret.js"; import { redactCompletedPortalSetupInvitationJob, redactTerminalFailedPortalSetupInvitationJob } from "./security/portal-setup-invitation-secret.js"; import { defaultWorkerQueueNames, getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js"; import { createWorkerStorageDeleteService } from "./storage.js"; import { createWorkerScanService } from "./scanService.js"; /** * Subset of the shared metrics bundle the queue runtime actually writes to. * Keeping this narrow in the public signature lets tests inject a single * counter without instantiating the entire prom-client registry. */ export type WorkerRuntimeMetrics = Pick< DefaultMetrics, "queueJobsCompletedTotal" | "queueJobsFailedTotal" | "queueJobDurationSeconds" | "mailSentTotal" | "webhookDeliveryAttemptsTotal" >; export const workerQueueNames = defaultWorkerQueueNames; export { getWorkerQueueNames }; type WorkerBackoffOptions = { delay?: number; type?: string; }; type WorkerJob = { attemptsMade?: number; data?: unknown; id?: string | number; name: string; opts?: { attempts?: number; backoff?: number | WorkerBackoffOptions; }; queueName?: string; updateData?: (data: never) => Promise; }; type WorkerLogBindings = { attemptsLimit?: number | null; attemptsMade?: number; backoffDelay?: number; backoffType?: string; jobId?: string | null; jobName?: string; queueName?: string; requestId?: string; }; export type WorkerRuntimeLogger = { child?: (bindings: Record) => WorkerRuntimeLogger; error: (message?: unknown, ...optionalParams: unknown[]) => void; info?: (message?: unknown, ...optionalParams: unknown[]) => void; log?: (message?: unknown, ...optionalParams: unknown[]) => void; warn?: (message?: unknown, ...optionalParams: unknown[]) => void; }; export type WorkerRuntimeWorker = { close(): Promise; name: string; on(event: "completed", listener: (job: WorkerJob) => void): WorkerRuntimeWorker; on(event: "failed", listener: (job: WorkerJob | undefined, error: Error) => void): WorkerRuntimeWorker; }; export type WorkerRuntimeStatus = | { configured: false; enabled: false; queueNames: readonly QueueName[]; status: "disabled"; workers: readonly []; } | { configured: true; enabled: true; queueNames: readonly QueueName[]; status: "started"; workers: readonly WorkerRuntimeWorker[]; }; export type SentryStartSpanArgs = { op: string; name: string; attributes?: Record; }; export type SentrySpan = { setAttribute(key: string, value: string | number | boolean | undefined): void; setStatus?(status: { code: number; message?: string }): void; }; export type SentryLikeForWorkerTracing = { startSpan: (args: SentryStartSpanArgs, callback: (span: SentrySpan) => Promise | T) => Promise | T; captureException?: (error: unknown, captureContext?: Record) => void; }; export type StartWorkerRuntimeOptions = { createProcessor?: (queueName: Name, dependencies: HandlerDependencies) => TypedProcessor; createWorkerImpl?: ( queueName: Name, processor: TypedProcessor, options: Parameters>[2] ) => WorkerRuntimeWorker; env?: EnvRecord; logger?: WorkerRuntimeLogger; queueNames?: readonly QueueName[]; recordFailedJob?: (data: QueueFailedJobRecordInput) => Promise; reporter?: ErrorReporter; /** * Optional Sentry-like module for queue.process span tracing. When omitted, * the runtime falls back to a pass-through processor. The production caller * lazy-loads `@sentry/node`; tests inject a deterministic fake. */ sentry?: SentryLikeForWorkerTracing; /** * Optional Prometheus metrics. When set, the runtime records * `queue_jobs_completed_total` on success or `queue_jobs_failed_total` * on throw, and observes `queue_job_duration_seconds`. Queue-depth gauges * such as `queue_jobs_active` are owned by the metrics scheduler. * Omitting this is a no-op so existing callers stay byte-identical. */ metrics?: WorkerRuntimeMetrics; /** * Optional time source for duration histograms. Defaults to * `performance.now()`; tests inject a deterministic clock. */ now?: () => number; /** * Optional extra HandlerDependencies merged into the runtime-built * dependency object. The worker bootstrap uses this to inject * Prisma-backed services that the runtime layer cannot construct on * its own. */ extraHandlerDependencies?: Partial; }; export type ShutdownWorkerRuntimeResult = { closedWorkers: readonly string[]; }; export function startWorkerRuntime(options: StartWorkerRuntimeOptions = {}): WorkerRuntimeStatus { const env = options.env ?? (process.env as EnvRecord); const logger = options.logger ?? console; const queues = options.queueNames ?? getWorkerQueueNames(env); const config = loadRequiredWorkerConfig(env); if (!config.enabled) { logInfo(logger, { queueNames: queues }, "Worker disabled; queues not started."); return { configured: false, enabled: false, queueNames: queues, status: "disabled", workers: [], }; } if (!config.redis.url) { throw new Error("Worker Redis URL is required when worker runtime is enabled."); } const connection = parseRedisUrl(config.redis.url); const createWorkerImpl = options.createWorkerImpl ?? createWorker; const createProcessor = options.createProcessor ?? ((queueName, dependencies) => createRegisteredProcessor(queueName, { dependencies })); const recordFailedJob = options.recordFailedJob ?? recordQueueFailedJob; const reporter = options.reporter ?? createNoopErrorReporter(); const sentry = options.sentry; const scanService = env.SCAN_QUEUE_ENABLED === "true" ? createWorkerScanService(env) : undefined; const metrics = options.metrics; const mailMetricsHook = metrics ? ({ provider, status }: { provider: "mailjet"; status: "delivered" | "failed" }) => { try { metrics.mailSentTotal.labels({ provider, status }).inc(); } catch { // Metric writes never block mail. } } : undefined; const webhookDeliveryMetricsHook = metrics ? ({ status }: { status: "success" | "retry" | "dlq" | "rejected" }) => { try { metrics.webhookDeliveryAttemptsTotal.labels({ status }).inc(); } catch { // Metric writes never block delivery. } } : undefined; const handlerDependencies: HandlerDependencies = { env, // Handlers emit the provider-stage diagnostics that are required to // distinguish a recoverable KWP outage from a reconnect condition. Keep // the narrow handler logger on the same structured runtime sink instead // of silently falling back to the no-op logger. logger: { info: (fields, message) => logInfo(logger, fields, message), error: (fields, message) => logError(logger, fields, message), }, ...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? { storageDeleteService: createWorkerStorageDeleteService(env) } : {}), ...(scanService ? { scanService } : {}), ...(mailMetricsHook ? { mailMetricsHook } : {}), ...(webhookDeliveryMetricsHook ? { webhookDeliveryMetricsHook } : {}), ...(options.extraHandlerDependencies ?? {}), }; assertQueueServicesWired(queues, handlerDependencies, env); const now = options.now ?? (() => performance.now()); const workers = queues.map((queueName) => { const baseProcessor = createProcessor(queueName, handlerDependencies); const tenantScopedProcessor = wrapProcessorWithRlsContext(baseProcessor); const instrumented = metrics ? wrapProcessorWithMetrics(queueName, tenantScopedProcessor, metrics, now) : tenantScopedProcessor; const processor = createLoggedProcessor(queueName, instrumented, logger, reporter, sentry); const worker = createWorkerImpl(queueName, processor, { connection, concurrency: config.concurrency }); worker.on("completed", (job: WorkerJob) => { const jobLogger = createJobLogger(logger, queueName, job); logInfo(jobLogger, {}, "Worker job completed."); void Promise.all([redactCompletedWorkspaceOwnerInvitationJob(job), redactCompletedPortalSetupInvitationJob(job)]).catch((error) => { const redactedError = redactWorkerError(error); logError(jobLogger, { error: redactedError }, "Completed capability-bearing mail job payload redaction failed."); reportWorkerError(reporter, "job.secret_redaction_failed", "error", queueName, job, redactedError); }); }); worker.on("failed", (job: WorkerJob | undefined, error: Error) => { const jobLogger = createJobLogger(logger, queueName, job); const redactedError = redactWorkerError(error); logError(jobLogger, { error: redactedError }, "Worker job failed."); reportWorkerError(reporter, "job.failed", "error", queueName, job, redactedError); if (isTerminalFailure(job)) { void Promise.all([redactTerminalFailedWorkspaceOwnerInvitationJob(job ?? {}), redactTerminalFailedPortalSetupInvitationJob(job ?? {})]).catch( (redactionError) => { const safeRedactionError = redactWorkerError(redactionError); logError(jobLogger, { error: safeRedactionError }, "Terminal capability-bearing mail job payload redaction failed."); reportWorkerError(reporter, "job.secret_redaction_failed", "error", queueName, job, safeRedactionError); } ); } void persistTerminalFailedJob({ queueName, job, error: redactedError, logger: jobLogger, recordFailedJob, reporter }); }); return worker; }); logInfo(logger, { workerNames: workers.map((worker) => worker.name) }, "Worker started."); return { configured: true, enabled: true, queueNames: queues, status: "started", workers, }; } function assertQueueServicesWired(queues: readonly QueueName[], dependencies: HandlerDependencies, env: EnvRecord): void { const ownsImports = queues.includes(queueNames.imports); const ownsExports = queues.includes(queueNames.exports); const ownsKwpImports = queues.includes(queueNames.kwpImports); const importMode = resolveImportQueueRuntimeMode(env); const missing: string[] = []; if (importMode === "portal-gdpr-export" && (!ownsKwpImports || !ownsExports || ownsImports || queues.length !== 2)) { throw new Error("Portal GDPR export queue mode must own hivegrid.kwp-imports and hivegrid.exports."); } if (ownsKwpImports) { if (!dependencies.tenantMaintenanceLeaseStore) missing.push("tenantMaintenanceLeaseStore"); if (!dependencies.syncKwpProjectBinding) missing.push("syncKwpProjectBinding"); if (!dependencies.syncKwpProjectCatalog) missing.push("syncKwpProjectCatalog"); if (!dependencies.enqueueKwpProjectCatalogContinuation) missing.push("enqueueKwpProjectCatalogContinuation"); if (!dependencies.syncKwpProjectInbound) missing.push("syncKwpProjectInbound"); if (!dependencies.enqueueKwpArchiveFileJobs) missing.push("enqueueKwpArchiveFileJobs"); if (!dependencies.syncKwpArchiveFile) missing.push("syncKwpArchiveFile"); if (!dependencies.runKwpContractSentinel) missing.push("runKwpContractSentinel"); } if (ownsImports) { if (!dependencies.importService) missing.push("importService"); if (!dependencies.scanService) missing.push("scanService"); if (!dependencies.adminSalesDataService) missing.push("adminSalesDataService"); } if (ownsExports) { const privacyServiceWired = typeof dependencies.privacyGdprExportService?.resolveReady === "function" && typeof dependencies.privacyGdprExportService?.resolveReclaimableProcessing === "function" && typeof dependencies.privacyGdprExportService?.createAttempt === "function" && typeof dependencies.privacyGdprExportService?.runExport === "function" && typeof dependencies.privacyGdprExportService?.deleteArtifact === "function"; if (importMode === "portal-gdpr-export") { if (!dependencies.portalGdprExportService) missing.push("portalGdprExportService"); if (!privacyServiceWired) missing.push("privacyGdprExportService"); } else { if (!dependencies.exportService) missing.push("exportService"); if (!dependencies.portalGdprExportService) missing.push("portalGdprExportService"); if (!privacyServiceWired) missing.push("privacyGdprExportService"); } } if (isRequiredTenantHardDeleteRuntime(env)) { if (!env.MEILI_URL?.trim()) missing.push("MEILI_URL"); if (!env.MEILI_API_KEY?.trim()) missing.push("MEILI_API_KEY"); if (!dependencies.privacyTenantDeletionService) missing.push("privacyTenantDeletionService"); } if (missing.length > 0) { const runtimeLabel = importMode === "portal-gdpr-export" ? "KWP and portal/privacy GDPR queues" : importMode === "idle" ? "KWP provider queue" : "Worker KWP/import/export queues"; throw new Error(`${runtimeLabel} requires wired production services: missing ${missing.join(", ")}.`); } } function isRequiredTenantHardDeleteRuntime(env: EnvRecord): boolean { return ( env.WORKER_QUEUE_GROUP === "general" && (env.NODE_ENV === "production" || env.RLS_RUNTIME_MODE === "required" || env.SENTRY_ENVIRONMENT === "staging" || env.SENTRY_ENVIRONMENT === "production") ); } function createJobLogger(logger: WorkerRuntimeLogger, fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerRuntimeLogger { const bindings = buildWorkerJobBindings(fallbackQueueName, job); return typeof logger.child === "function" ? logger.child(bindings) : createBoundFallbackLogger(logger, bindings); } function createBoundFallbackLogger(logger: WorkerRuntimeLogger, bindings: WorkerLogBindings): WorkerRuntimeLogger { return { child: (childBindings) => createBoundFallbackLogger(logger, { ...bindings, ...childBindings }), error: (record?: unknown, ...optionalParams: unknown[]) => logger.error(mergeLogRecord(bindings, record), ...optionalParams), info: (record?: unknown, ...optionalParams: unknown[]) => { if (logger.info) { logger.info(mergeLogRecord(bindings, record), ...optionalParams); return; } logger.log?.(mergeLogRecord(bindings, record), ...optionalParams); }, log: (record?: unknown, ...optionalParams: unknown[]) => logger.log?.(mergeLogRecord(bindings, record), ...optionalParams), warn: (record?: unknown, ...optionalParams: unknown[]) => { if (logger.warn) { logger.warn(mergeLogRecord(bindings, record), ...optionalParams); return; } logger.log?.(mergeLogRecord(bindings, record), ...optionalParams); }, }; } function buildWorkerJobBindings(fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerLogBindings { const backoff = parseWorkerBackoff(job?.opts?.backoff); return { ...extractRequestIdBinding(job), queueName: job?.queueName ?? fallbackQueueName, jobName: job?.name ?? "unknown", jobId: job?.id === undefined ? null : String(job.id), attemptsMade: job?.attemptsMade ?? 0, attemptsLimit: job?.opts?.attempts ?? null, ...backoff, }; } function extractRequestIdBinding(job: WorkerJob | undefined): Pick { if (!job || !isRecord(job.data)) { return {}; } const metadata = job.data.metadata; if (!isRecord(metadata) || typeof metadata.requestId !== "string") { return {}; } return { requestId: metadata.requestId }; } async function persistTerminalFailedJob({ queueName, job, error, logger, recordFailedJob, reporter, }: { queueName: QueueName; job: WorkerJob | undefined; error: Error; logger: WorkerRuntimeLogger; recordFailedJob: (data: QueueFailedJobRecordInput) => Promise; reporter: ErrorReporter; }): Promise { if (!isTerminalFailure(job)) { return; } try { await recordFailedJob({ queueName: job?.queueName ?? queueName, jobName: job?.name ?? "unknown", jobId: job?.id === undefined ? null : String(job.id), attemptsMade: job?.attemptsMade ?? 0, attemptsLimit: job?.opts?.attempts ?? null, errorName: error.name, errorMessage: error.message, failedAt: new Date(), }); } catch (persistenceError) { const redactedPersistenceError = redactWorkerError(persistenceError); logError(logger, { error: redactedPersistenceError }, "Failed to persist terminal worker failure."); reportWorkerError(reporter, "job.persistence_failed", "error", queueName, job, redactedPersistenceError); } } function isTerminalFailure(job: WorkerJob | undefined): boolean { const attemptsLimit = job?.opts?.attempts; if (attemptsLimit === undefined) { return true; } return (job?.attemptsMade ?? 0) >= attemptsLimit; } function parseWorkerBackoff(backoff: NonNullable["backoff"]): Pick { if (typeof backoff === "number") { return { backoffDelay: backoff, backoffType: "fixed" }; } if (backoff && typeof backoff === "object") { return { ...(typeof backoff.delay === "number" ? { backoffDelay: backoff.delay } : {}), ...(typeof backoff.type === "string" ? { backoffType: backoff.type } : {}), }; } return {}; } export async function shutdownWorkerRuntime( runtime: Pick, options: { logger?: WorkerRuntimeLogger; reason?: string; reporter?: ErrorReporter } = {} ): Promise { const logger = options.logger ?? console; const reason = options.reason ?? "shutdown"; const reporter = options.reporter ?? createNoopErrorReporter(); const closeResults = await Promise.allSettled(runtime.workers.map((worker) => worker.close())); const failedCloses = closeResults.flatMap((result, index) => { if (result.status === "fulfilled") { return []; } const worker = runtime.workers[index]; const workerName = worker?.name ?? "unknown"; const redactedError = redactWorkerError(result.reason); logError(logger, { error: redactedError, reason, workerName }, "Failed to close worker."); reportError(reporter, { component: "worker", operation: "shutdown.close_failed", severity: "error", queueName: workerName, workerName, reason, error: redactedError, }); return [workerName]; }); if (failedCloses.length > 0) { throw new Error(`Failed to close workers: ${failedCloses.join(", ")}.`); } const closedWorkers = runtime.workers.map((worker) => worker.name); logInfo(logger, { closedWorkers, reason }, "Worker shutdown complete."); return { closedWorkers }; } function createLoggedProcessor( queueName: Name, processor: TypedProcessor, logger: WorkerRuntimeLogger, reporter: ErrorReporter, sentry: SentryLikeForWorkerTracing | undefined ): TypedProcessor { const tracedCore: TypedProcessor = async (job) => { try { return await processor(job); } catch (error) { const redactedError = redactWorkerError(error); logError(createJobLogger(logger, queueName, job), { error: redactedError }, "Worker handler failed."); reportWorkerError(reporter, "handler.failed", "error", queueName, job, redactedError); throw error; } }; if (!sentry) { return tracedCore; } return wrapProcessorWithSentryTracing(queueName, tracedCore, sentry); } /** * Wraps the BullMQ processor in a `Sentry.startSpan({ op: 'queue.process' })` * span tagged with queue + job + tenant context. Sentry's @sentry/node v8/v10 * has no BullMQ auto-instrumentation, so we do this manually. */ function wrapProcessorWithSentryTracing( queueName: Name, inner: TypedProcessor, sentry: SentryLikeForWorkerTracing ): TypedProcessor { return async (job) => { const jobShape = (job ?? {}) as Partial & { data?: unknown }; const jobName = jobShape.name ?? "unknown"; const jobId = jobShape.id !== undefined && jobShape.id !== null ? String(jobShape.id) : undefined; const attemptsMade = jobShape.attemptsMade ?? 0; const attemptsLimit = jobShape.opts?.attempts; const tenantId = extractTenantId(jobShape.data); return sentry.startSpan( { op: "queue.process", name: `${queueName}/${jobName}`, attributes: { "messaging.system": "bullmq", "messaging.destination.name": queueName, "messaging.operation": "process", "messaging.message.id": jobId, "messaging.bullmq.job.name": jobName, "messaging.bullmq.attempts_made": attemptsMade, ...(typeof attemptsLimit === "number" ? { "messaging.bullmq.attempts_limit": attemptsLimit } : {}), ...(tenantId ? { "hivegrid.tenant_id": tenantId } : {}), }, }, async (span) => { try { const result = await inner(job); span.setStatus?.({ code: 1, message: "ok" }); return result; } catch (error) { span.setStatus?.({ code: 2, message: "internal_error" }); if (typeof sentry.captureException === "function") { try { sentry.captureException(error, { tags: { component: "worker", queue: queueName, job_name: jobName, ...(jobId ? { job_id: jobId } : {}), ...(tenantId ? { tenant_id: tenantId } : {}), }, }); } catch { // Sentry transport failures must never mask the runtime failure. } } throw error; } } ); }; } /** * Wraps a BullMQ processor in a Prometheus-instrumented closure. * * Side effects per invocation: * - `queue_jobs_completed_total{queue,job_name}` +1 on success. * - `queue_jobs_failed_total{queue,job_name,error_type}` +1 on throw. * `error_type` is the redacted `error.name` or `"Error"` to keep * cardinality bounded (no message strings). * - `queue_job_duration_seconds{queue,job_name}` observation in seconds. * * Metric writes are wrapped in try/catch so a registry failure can never * mask a queue handler error. */ function wrapProcessorWithMetrics( queueName: Name, inner: TypedProcessor, metrics: WorkerRuntimeMetrics, now: () => number ): TypedProcessor { return async (job) => { const jobShape = (job ?? {}) as Partial; const jobName = jobShape.name ?? "unknown"; const started = now(); let failed = false; let errorType = "Error"; try { const result = await inner(job); try { metrics.queueJobsCompletedTotal.labels({ queue: queueName, job_name: jobName }).inc(); } catch { // ignore } return result; } catch (error) { failed = true; errorType = pickErrorType(error); throw error; } finally { try { metrics.queueJobDurationSeconds.labels({ queue: queueName, job_name: jobName }).observe((now() - started) / 1000); } catch { // ignore } if (failed) { try { metrics.queueJobsFailedTotal.labels({ queue: queueName, job_name: jobName, error_type: errorType }).inc(); } catch { // ignore } } } }; } function pickErrorType(error: unknown): string { if (error instanceof Error) { const name = error.name.trim(); return name.length > 0 ? name : "Error"; } return "Error"; } function extractTenantId(data: unknown): string | undefined { if (!isRecord(data)) return undefined; const direct = data.tenantId; if (typeof direct === "string" && direct) return direct; const metadata = data.metadata; if (isRecord(metadata) && typeof metadata.tenantId === "string" && metadata.tenantId) { return metadata.tenantId; } return undefined; } function reportWorkerError( reporter: ErrorReporter, operation: string, severity: ErrorReportSeverity, fallbackQueueName: QueueName, job: WorkerJob | undefined, error: unknown ): void { const bindings = buildWorkerJobBindings(fallbackQueueName, job); reportError(reporter, { component: "worker", operation, severity, queueName: bindings.queueName, jobName: bindings.jobName, jobId: bindings.jobId ?? undefined, requestId: bindings.requestId, attemptsMade: bindings.attemptsMade, attemptsLimit: bindings.attemptsLimit, retryCount: bindings.attemptsMade, backoffDelay: bindings.backoffDelay, backoffType: bindings.backoffType, error, }); } function logInfo(logger: WorkerRuntimeLogger, record: Record, message: string): void { if (logger.info) { logger.info(record, message); return; } logger.log?.(record, message); } function logError(logger: WorkerRuntimeLogger, record: Record, message: string): void { logger.error(record, message); } function mergeLogRecord(bindings: WorkerLogBindings, record: unknown): Record { if (isRecord(record)) { return { ...bindings, ...record }; } return bindings; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function redactWorkerError(error: unknown): Error { const source = error instanceof Error ? error : new Error(String(error)); const redacted = new Error(redactWorkerDiagnosticText(source.message || source.name)); redacted.name = source.name || "Error"; return redacted; } function redactWorkerDiagnosticText(value: string): string { return value .replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]") .replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]") .replace( /(authorization|access[_-]?key(?:id)?|api[-_]?key|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]" ) .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]") .slice(0, 1_000); }