// IMPORTANT: `./otel.js` MUST stay the very first import in this file. // The OpenTelemetry NodeSDK can only patch BullMQ, Prisma, Redis and // friends when it loads before those modules are required for the first // time. The module is a no-op when OTEL_EXPORTER_OTLP_ENDPOINT is unset. import "./otel.js"; import { ensureAndSyncProjectChannelIfEnabledWithTx } from "@hivegrid/chat/project-sync.server"; import { DEFAULT_WORKER_ADMIN_PORT, isWorkerMetricsEnabled, startWorkerAdminServer, type WorkerAdminServerHandle, type WorkerStatusReport, } from "./admin-server.js"; import { createWorkerConnection } from "./connection.js"; import { bootstrapWorkerGroup, InvalidWorkerQueueGroupError } from "./group-runtime.js"; import { DEFAULT_WORKER_HEALTH_PORT, startWorkerHealthServer, type HealthProbe, type WorkerHealthServerHandle } from "./health-server.js"; import { bootstrapOutboxDispatcher, type OutboxDispatcherBootstrapResult } from "./outbox/bootstrap.js"; import { isOutboxDispatcherRequired } from "./outbox/dispatcher.js"; import { bootstrapSearchIndexer, type SearchIndexerBootstrapResult, type SearchIndexerWorkerHandle } from "./jobs/search-indexer-bootstrap.js"; import { bootstrapFormSubmissionRoute, type FormSubmissionRouteBootstrapResult, type FormSubmissionRouteWorkerHandle, } from "./jobs/form-submission-route-bootstrap.js"; import { runStartDoubleOptInFormAction, type StartDoubleOptInPrisma } from "./jobs/form-submission-actions.js"; import { bootstrapFormSubmissionWebhookDelivery, type FormSubmissionWebhookDeliveryBootstrapResult, type FormSubmissionWebhookDeliveryWorkerHandle, } from "./jobs/form-submission-webhook-delivery-bootstrap.js"; import { bootstrapTransactionalMailOutbox, isTransactionalMailOutboxRequired, type TransactionalMailOutboxBootstrapResult, type TransactionalMailOutboxWorkerHandle, } from "./jobs/transactional-mail-outbox-bootstrap.js"; import { cancelCampaignScheduleWithPrisma, isCampaignDeliveryAllowedWithPrisma, type CampaignScheduleCancellationPrisma, } from "./jobs/campaign-schedule-cancellation.js"; import { bootstrapPortalStatusMail, type PortalStatusMailBootstrapResult, type PortalStatusMailWorkerHandle } from "./jobs/portal-status-mail-bootstrap.js"; import { bootstrapPortalDomainOrchestration, type PortalDomainOrchestrationBootstrapResult } from "./jobs/portal-domain-orchestration-bootstrap.js"; import { bootstrapProductAcquisitionProviderSideEffect, type ProductAcquisitionProviderSideEffectBootstrapResult, } from "./jobs/product-acquisition-provider-side-effect-bootstrap.js"; import { bootstrapRecurringInvoiceCollection, type RecurringInvoiceCollectionBootstrapResult } from "./jobs/recurring-invoice-collection-bootstrap.js"; import { claimProductProviderSideEffectOutboxEvent, releaseProductProviderSideEffectOutboxClaim, runProductProviderSideEffectWithReceipt, } from "./jobs/product-acquisition-provider-side-effect-receipt.js"; import { assertProductAcquisitionRuntimeEnv, resolveProductAcquisitionProviderSideEffectMode } from "./jobs/product-acquisition-runtime-env.js"; import { createPrismaPortalStatusMailStore, resolvePortalStatusMailBaseUrl, type PortalStatusMailPrisma } from "./jobs/portal-status-mail-store.js"; import { bootstrapWorkflowTrigger, isWorkflowTriggerOutboxRequired, type WorkflowTriggerBootstrapResult, type WorkflowTriggerWorkerHandle, } from "./jobs/workflow-trigger-bootstrap.js"; import type { QueueJsonValue, TelephonyWebhookProcessJob, TypedQueue } from "@hivegrid/queue"; import { createWorkflowTriggerExecutorFromEnv, parseWorkflowTriggerExecutorMode, type WorkflowTriggerExecutorMode, } from "./jobs/workflow-trigger-executor-factory.js"; import type { WorkflowTriggerNativePrisma } from "./jobs/workflow-trigger-native-executor.js"; import { createPrismaWorkflowTriggerStore, type WorkflowTriggerStorePrisma } from "./jobs/workflow-trigger-store.js"; import { runPortalAutomationTriggerJob } from "./jobs/portal-automation.js"; import { AUTOMATION_EXECUTOR_OUTBOX_QUEUE, createAutomationExecutorProcessor, createRunAutomationForwardTrigger } from "./jobs/automation-forward.js"; import { bootstrapEmailSequenceWorkers, type EmailSequenceBootstrapResult, type EmailSequenceWorkerHandle } from "./jobs/email-sequence-bootstrap.js"; import { bootstrapAuditEventPersist, type AuditEventPersistBootstrapResult, type AuditEventPersistWorkerHandle } from "./jobs/audit-event-persist-bootstrap.js"; import { bootstrapChatRealtimeWakeup, type ChatRealtimeWakeupBootstrapResult, type ChatRealtimeWakeupWorkerHandle, } from "./jobs/chat-realtime-wakeup-bootstrap.js"; import { bootstrapChatLinkPreview, type ChatLinkPreviewBootstrapResult, type ChatLinkPreviewWorkerHandle } from "./jobs/chat-link-preview-bootstrap.js"; import { bootstrapLegacyEventReceiver, type LegacyEventReceiverBootstrapResult, type LegacyEventReceiverWorkerHandle, } from "./jobs/legacy-event-receiver-bootstrap.js"; import { bootstrapPluginHookFanout, isPluginHookFanoutRequired, type PluginHookFanoutBootstrapResult, type PluginHookFanoutWorkerHandle, } from "./jobs/plugin-hook-fanout-bootstrap.js"; import { bootstrapPluginCapabilityUse, type PluginCapabilityUseBootstrapResult, type PluginCapabilityUseWorkerHandle, } from "./jobs/plugin-capability-use-bootstrap.js"; import { createPrismaPluginCapabilityUseStore, type PluginCapabilityUseStorePrisma } from "./jobs/plugin-capability-use.js"; import { registerDsgvoGraceCron } from "./jobs/dsgvo-grace-cron.js"; import { registerOutboxDoneRetentionCron } from "./jobs/outbox-done-retention-cron.js"; import { registerPrivacyGdprExportRetentionCron } from "./jobs/privacy-gdpr-export-retention-cron.js"; import { registerPortalGdprExportRetentionCron } from "./jobs/portal-gdpr-export-retention-cron.js"; import { registerWorkflowExecutionStalledReconcilerCron } from "./jobs/workflow-execution-stalled-reconciler-cron.js"; import { registerAutomationSchedulerCron } from "./jobs/automation-scheduler-cron.js"; import { registerTaskArchiveRetentionCron } from "./jobs/task-archive-retention-cron.js"; import { registerLeadRetentionCron } from "./jobs/lead-retention-cron.js"; import { registerProjectAutoArchiveCron } from "./jobs/project-auto-archive-cron.js"; import { registerBillingDunningSweepCron } from "./jobs/billing-dunning-cron.js"; import { registerIntegrationsHealthSweepCron, registerIntegrationsTokenRefreshCron, registerKwpProjectMaintenanceCron, } from "./jobs/integrations-maintenance-cron.js"; import { runAllTenantKwpProjectMaintenance } from "./jobs/kwp-project-maintenance.js"; import { registerCalendarInternalWriteSyncCron, registerCalendarTokenRefreshCron, registerCalendarWebhookDrainCron, registerCalendarWebhookRenewalCron, } from "./jobs/calendar-maintenance-cron.js"; import { registerEmailInboxEventCleanupCron, registerEmailInboxInitialSyncCron, registerEmailInboxStalePollCron, registerEmailInboxTokenRefreshCron, registerEmailInboxWebhookDrainCron, registerEmailInboxWebhookRenewalCron, } from "./jobs/email-inbox-maintenance-cron.js"; import { registerTaskReminderSweepCron } from "./jobs/task-reminder-cron.js"; import { registerAdminSalesTaskReminderSweepCron } from "./jobs/admin-sales-task-reminder-cron.js"; import { registerAdminEmailDeliveryCron } from "./jobs/admin-email-delivery-cron.js"; import { registerAdminEmailEventRetentionCron } from "./jobs/admin-email-event-retention-cron.js"; import { registerAdminEmailSyncCron } from "./jobs/admin-email-sync-cron.js"; import { registerAdminEmailDomainCron } from "./jobs/admin-email-domain-cron.js"; import { registerAdminCallRecordingCrons } from "./jobs/admin-call-recording-cron.js"; import { registerAdminOutboundCallCron } from "./jobs/admin-outbound-call-cron.js"; import { registerAdminBookingReminderSweepCron } from "./jobs/admin-booking-reminder-cron.js"; import { registerAdminCalendarSyncCron } from "./jobs/admin-calendar-sync-cron.js"; import { bootstrapAdminBookingOutbox, type AdminBookingOutboxBootstrapResult, type AdminBookingOutboxWorkerHandle, } from "./jobs/admin-booking-outbox-bootstrap.js"; import { registerHrUploadReminderDispatchCron } from "./jobs/hr-upload-reminder-cron.js"; import { registerEmailDomainProbeCron } from "./jobs/email-domain-probe-cron.js"; import { registerTelephonyWebhookMaintenanceCron, runAllTenantTelephonyWebhookMaintenance } from "./jobs/telephony-webhook-maintenance.js"; import { registerProductAcquisitionReconcilerCron } from "./jobs/product-acquisition-reconciler-cron.js"; import { PRODUCT_ACQUISITION_RECONCILE_JOB_ID, asProductAcquisitionReconcileClient, runAllTenantProductAcquisitionReconciliation, } from "./jobs/product-acquisition-reconciler.js"; import { PLUGIN_AUDIT_RETENTION_QUEUE, registerPluginAuditRetentionCron, resolvePluginAuditRetentionConfig, runPluginAuditRetention, } from "./jobs/plugin-audit-retention.js"; import { AUDIT_RETENTION_QUEUE, registerAuditRetentionCron, runAuditRetentionJob } from "./jobs/audit-retention.js"; import { createWorkflowExecutionStalledReconciler, type WorkflowExecutionStalledReconcilerPrisma } from "./jobs/workflow-execution-stalled-reconciler.js"; import type { HandlerDependencies } from "./handlers.js"; import { DelayedError as BullMQDelayedError, Queue as BullMQQueue, Worker as BullMQWorker } from "bullmq"; import { TELEPHONY_WEBHOOK_QUEUE_NAME, buildFilePreviewJob, buildFileScanJob, buildKwpProjectCatalogSyncJob, buildPlatformNotificationJob, buildTelephonyWebhookJob, enqueueJob, enqueueTelephonyWebhookJob, parseRedisUrl, queueNames, recoverFailedTelephonyWebhookJob, } from "@hivegrid/queue"; import { createIdempotencyStore } from "@hivegrid/shared-store"; import { resolvePlatformLeadNotificationRecipient } from "@hivegrid/db/platform-lead-notifications.server"; import type { SentryLikeForWorkerTracing } from "./runtime.js"; import { shutdownWorkerRuntime, startWorkerRuntime, type WorkerRuntimeStatus } from "./runtime.js"; import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js"; import { runWorkerPayloadWithRlsContext } from "./rls-context.js"; import { createQueue, createWorker } from "./queue-runtime.js"; import type { Prisma } from "@hivegrid/db/prisma-browser"; import { workerLogger } from "./logger.js"; import { getWorkerMetrics } from "./metrics.js"; import { workerErrorReporter } from "./sentry.js"; import { closeSharedStore, getSharedStore } from "./shared-store.js"; import { createWorkerStorageDeleteService } from "./storage.js"; import { buildKwpArchiveDiscoveryEnqueue, buildKwpArchiveFileEnqueue } from "./kwp-archive-queue.js"; const SCHEDULED_JOB_DEFAULT_OPTIONS = { removeOnComplete: 100, removeOnFail: 1_000, } as const; function createRlsAwareBullMqProcessor(processor: (job: Job) => Promise) { return async (job: Job) => runWorkerPayloadWithRlsContext(job.data, () => processor(job)); } type ShutdownState = { promise?: Promise; signal?: NodeJS.Signals; adminServer?: WorkerAdminServerHandle | undefined; healthServer?: WorkerHealthServerHandle | undefined; redisHealthProbe?: RedisHealthProbe | undefined; outbox?: OutboxDispatcherBootstrapResult | undefined; adminBookingOutbox?: AdminBookingOutboxBootstrapResult | undefined; workflowTrigger?: WorkflowTriggerBootstrapResult | undefined; searchIndexer?: SearchIndexerBootstrapResult | undefined; formSubmissionRoute?: FormSubmissionRouteBootstrapResult | undefined; formSubmissionWebhookDelivery?: FormSubmissionWebhookDeliveryBootstrapResult | undefined; formSubmissionWebhookQueue?: BullMQQueue | undefined; transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult | undefined; portalStatusMail?: PortalStatusMailBootstrapResult | undefined; portalDomainOrchestration?: PortalDomainOrchestrationBootstrapResult | undefined; productAcquisitionProviderSideEffect?: ProductAcquisitionProviderSideEffectBootstrapResult | undefined; recurringInvoiceCollection?: RecurringInvoiceCollectionBootstrapResult | undefined; emailSequences?: EmailSequenceBootstrapResult | undefined; auditEventPersist?: AuditEventPersistBootstrapResult | undefined; chatRealtimeWakeup?: ChatRealtimeWakeupBootstrapResult | undefined; chatLinkPreview?: ChatLinkPreviewBootstrapResult | undefined; legacyEventReceiver?: LegacyEventReceiverBootstrapResult | undefined; platformNotificationQueue?: TypedQueue | undefined; pluginHookFanout?: PluginHookFanoutBootstrapResult | undefined; pluginCapabilityUse?: PluginCapabilityUseBootstrapResult | undefined; pluginHookInvokeQueue?: BullMQQueue | undefined; pluginAuditRetentionWorker?: BullMQWorker | undefined; auditRetentionWorker?: BullMQWorker | undefined; filePreviewQueue?: TypedQueue | undefined; telephonyWebhookQueue?: BullMQQueue | undefined; tenantDeletionExportQueue?: BullMQQueue | undefined; telephonyWebhookWorker?: BullMQWorker | undefined; automationExecutorWorker?: BullMQWorker | undefined; }; type RedisHealthProbeResult = { ok: true } | { ok: false; error: string }; type RedisHealthProbe = (() => Promise) & { close: () => Promise }; function buildWorkerStatus( runtime: WorkerRuntimeStatus, options?: { outboxDispatcher?: OutboxDispatcherBootstrapResult; outboxDispatcherExpected?: boolean; workflowTrigger?: WorkflowTriggerBootstrapResult; workflowTriggerExpected?: boolean; workflowTriggerExecutorMode?: WorkflowTriggerExecutorMode; pluginHookFanout?: PluginHookFanoutBootstrapResult; pluginHookFanoutExpected?: boolean; transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult; transactionalMailOutboxExpected?: boolean; portalStatusMail?: PortalStatusMailBootstrapResult; portalStatusMailExpected?: boolean; emailSequences?: EmailSequenceBootstrapResult; emailSequencesExpected?: boolean; chatLinkPreview?: ChatLinkPreviewBootstrapResult; chatLinkPreviewExpected?: boolean; } ): WorkerStatusReport { const runtimeCheck = runtime.status === "started" ? "ok" : "down"; const outboxDispatcherCheck = options?.outboxDispatcher?.status === "started" ? "ok" : options?.outboxDispatcherExpected ? "down" : "disabled"; const workflowTriggerCheck = options?.workflowTrigger?.status === "started" ? "ok" : options?.workflowTriggerExpected ? "down" : "disabled"; const pluginHookFanoutCheck = options?.pluginHookFanout?.status === "started" ? "ok" : options?.pluginHookFanoutExpected ? "down" : "disabled"; const transactionalMailOutboxCheck = options?.transactionalMailOutbox?.status === "started" ? "ok" : options?.transactionalMailOutboxExpected ? "down" : "disabled"; const portalStatusMailCheck = options?.portalStatusMail?.status === "started" ? "ok" : options?.portalStatusMailExpected ? "down" : "disabled"; const emailSequencesCheck = options?.emailSequences?.status === "started" ? "ok" : options?.emailSequencesExpected ? "down" : "disabled"; const chatLinkPreviewCheck = options?.chatLinkPreview?.status === "started" ? "ok" : options?.chatLinkPreviewExpected ? "down" : "disabled"; const status = runtimeCheck === "ok" && outboxDispatcherCheck !== "down" && workflowTriggerCheck !== "down" && pluginHookFanoutCheck !== "down" && transactionalMailOutboxCheck !== "down" && portalStatusMailCheck !== "down" && emailSequencesCheck !== "down" && chatLinkPreviewCheck !== "down" ? "ok" : "down"; return { status, service: "worker", checks: { runtime: runtimeCheck, outboxDispatcher: outboxDispatcherCheck, workflowTrigger: workflowTriggerCheck, pluginHookFanout: pluginHookFanoutCheck, transactionalMailOutbox: transactionalMailOutboxCheck, portalStatusMail: portalStatusMailCheck, emailSequences: emailSequencesCheck, chatLinkPreview: chatLinkPreviewCheck, }, ...(workflowTriggerCheck !== "disabled" && options?.workflowTriggerExecutorMode ? { workflowTriggerExecutorMode: options.workflowTriggerExecutorMode } : {}), version: process.env.APP_REVISION?.trim() || "unknown", uptime_seconds: Math.round(process.uptime()), region: process.env.APP_REGION?.trim() || process.env.FLY_REGION?.trim() || "unknown", }; } function createRedisHealthProbe(env: NodeJS.ProcessEnv): RedisHealthProbe { const redisUrl = env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim(); let queue: import("bullmq").Queue | undefined; let readyQueue: Promise | undefined; const probe = async (): Promise => { if (!redisUrl) { return { ok: false, error: "Redis URL not configured" }; } try { readyQueue ??= (async () => { const { Queue } = await import("bullmq"); const nextQueue = new Queue("__hivegrid_worker_health__", { connection: parseRedisUrl(redisUrl) }); queue = nextQueue; await nextQueue.waitUntilReady(); return nextQueue; })(); await readyQueue; return { ok: true }; } catch (error) { const failedQueue = queue; queue = undefined; readyQueue = undefined; await failedQueue?.close().catch(() => undefined); return { ok: false, error: error instanceof Error ? error.name : "RedisProbeFailed" }; } }; probe.close = async () => { const queueToClose = queue; queue = undefined; readyQueue = undefined; await queueToClose?.close(); }; return probe; } function createDatabaseHealthProbe(): () => Promise<{ ok: true } | { ok: false; error: string }> { return async () => { try { const { getDbClient } = await import("@hivegrid/db/client.server"); const db = getDbClient(); await db.$queryRaw`SELECT 1`; return { ok: true }; } catch (error) { return { ok: false, error: error instanceof Error ? error.name : "DatabaseProbeFailed" }; } }; } function createRuntimeReadinessProbes({ workerGroup, outboxDispatcherExpected, workflowTriggerExpected, pluginHookFanoutExpected, transactionalMailOutboxExpected, portalStatusMailExpected, emailSequencesExpected, chatLinkPreviewExpected, getOutboxDispatcherStatus, getWorkflowTriggerStatus, getPluginHookFanoutStatus, getTransactionalMailOutboxStatus, getPortalStatusMailStatus, getEmailSequencesStatus, getChatLinkPreviewStatus, }: { workerGroup?: string; outboxDispatcherExpected: boolean; workflowTriggerExpected: boolean; pluginHookFanoutExpected: boolean; transactionalMailOutboxExpected: boolean; portalStatusMailExpected: boolean; emailSequencesExpected: boolean; chatLinkPreviewExpected: boolean; getOutboxDispatcherStatus: () => OutboxDispatcherBootstrapResult | undefined; getWorkflowTriggerStatus: () => WorkflowTriggerBootstrapResult | undefined; getPluginHookFanoutStatus: () => PluginHookFanoutBootstrapResult | undefined; getTransactionalMailOutboxStatus: () => TransactionalMailOutboxBootstrapResult | undefined; getPortalStatusMailStatus: () => PortalStatusMailBootstrapResult | undefined; getEmailSequencesStatus: () => EmailSequenceBootstrapResult | undefined; getChatLinkPreviewStatus: () => ChatLinkPreviewBootstrapResult | undefined; }): Record | undefined { if (workerGroup !== "general" && workerGroup !== "mail") { return undefined; } const probes: Record = {}; if (outboxDispatcherExpected) { probes.outboxDispatcher = () => getOutboxDispatcherStatus()?.status === "started" ? { ok: true } : { ok: false, error: "outbox dispatcher is not started" }; } if (workflowTriggerExpected) { probes.workflowTrigger = () => getWorkflowTriggerStatus()?.status === "started" ? { ok: true } : { ok: false, error: "workflow trigger subscriber is not started" }; } if (pluginHookFanoutExpected) { probes.pluginHookFanout = () => getPluginHookFanoutStatus()?.status === "started" ? { ok: true } : { ok: false, error: "plugin hook fanout subscriber is not started" }; } if (transactionalMailOutboxExpected) { probes.transactionalMailOutbox = () => getTransactionalMailOutboxStatus()?.status === "started" ? { ok: true } : { ok: false, error: "transactional mail outbox subscriber is not started" }; } if (portalStatusMailExpected) { probes.portalStatusMail = () => getPortalStatusMailStatus()?.status === "started" ? { ok: true } : { ok: false, error: "portal status mail subscriber is not started" }; } if (emailSequencesExpected) { probes.emailSequences = () => getEmailSequencesStatus()?.status === "started" ? { ok: true } : { ok: false, error: "email-sequence workers are not started" }; } if (chatLinkPreviewExpected) { probes.chatLinkPreview = () => getChatLinkPreviewStatus()?.status === "started" ? { ok: true } : { ok: false, error: "chat link preview subscriber is not started" }; } return Object.keys(probes).length > 0 ? probes : undefined; } function hasWorkflowTriggerOutboxRedis(env: NodeJS.ProcessEnv): boolean { return Boolean(env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim()); } function isChatLinkPreviewExpected(env: NodeJS.ProcessEnv, workerGroup: string | undefined): boolean { return workerGroup === "general" && env.CHAT_LINK_PREVIEW_ENABLED !== "false"; } function isWorkflowTriggerOutboxExpected(env: NodeJS.ProcessEnv): boolean { if (isWorkflowTriggerOutboxRequired(env) || env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true") { return true; } return env.WORKFLOW_TRIGGER_OUTBOX_ENABLED !== "false" && hasWorkflowTriggerOutboxRedis(env); } function isOutboxDispatcherExpected(env: NodeJS.ProcessEnv): boolean { return env.OUTBOX_DISPATCHER_ENABLED === "true" || isOutboxDispatcherRequired(env) || isWorkflowTriggerOutboxExpected(env); } function isPluginHookFanoutExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean { if (workerGroup !== "general") { return false; } return env.PLUGIN_HOOK_FANOUT_ENABLED === "true" || isPluginHookFanoutRequired(env, workerGroup); } function isTransactionalMailOutboxExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean { if (workerGroup !== "mail") { return false; } return env.TRANSACTIONAL_MAIL_OUTBOX_ENABLED !== "false" || isTransactionalMailOutboxRequired(env, workerGroup); } function isPortalStatusMailExpected(workerGroup?: string): boolean { return workerGroup === "mail"; } function isEmailSequencesExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean { return workerGroup === "mail" && env.MARKETING_SEQUENCES_ENABLED === "true"; } function isPluginInvokeRuntimeEnabled(env: NodeJS.ProcessEnv): boolean { return env.NODE_ENV !== "production" || env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true"; } function resolvePositiveInteger(value: string | undefined, fallback: number): number { const parsed = Number(value); return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback; } /** * Lazy-load @sentry/node for queue.process span tracing. When the SDK is * not installed or SENTRY_DSN_WORKER is empty, the wrapper degrades to a * pass-through processor automatically. */ async function loadSentryForWorker(): Promise { if (!process.env.SENTRY_DSN_WORKER) return undefined; try { const moduleName = "@sentry/node"; const loaded = (await import(/* @vite-ignore */ moduleName)) as SentryLikeForWorkerTracing & { default?: SentryLikeForWorkerTracing; }; if (typeof loaded?.startSpan === "function") return loaded; if (typeof loaded?.default?.startSpan === "function") return loaded.default; return undefined; } catch { return undefined; } } try { const sentryForTracing = await loadSentryForWorker(); const { metrics, registry } = getWorkerMetrics(); // Validate WORKER_QUEUE_GROUP before we start anything else. An invalid // value must abort boot loudly — we don't want a half-started worker // silently subscribing to the wrong queues. Disabled workers (eg local // dev where WORKER_ENABLED=false) skip the group check. // Only require WORKER_QUEUE_GROUP when the worker is actually enabled. // Disabled workers (eg local dev where WORKER_ENABLED!=='true') skip // group resolution so they can boot a no-op process for testing // adjacent code paths. let groupResolution: ReturnType | undefined; if (process.env.WORKER_ENABLED === "true") { try { groupResolution = bootstrapWorkerGroup(process.env); } catch (error) { if (error instanceof InvalidWorkerQueueGroupError) { workerLogger.error({ error: { message: error.message } }, "Worker boot rejected: invalid WORKER_QUEUE_GROUP."); process.exit(1); } throw error; } const { assertRlsRuntimeReady } = await import("@hivegrid/db/rls-runtime-readiness.server"); await assertRlsRuntimeReady(undefined, { ...process.env, DATABASE_RUNTIME_ROLE: process.env.DATABASE_RUNTIME_ROLE?.trim() || "hivegrid_worker", }); assertProductAcquisitionRuntimeEnv(process.env); } // Incoming-call browser pushes are part of the webhook worker contract. // Validate the provider before starting the queue runtime or readiness // server so a broken VAPID configuration cannot produce a ready worker // that silently drops ringing notifications. let telephonyWebPushProvider: import("./jobs/push-notification.js").PushProviderSeam | undefined; if (groupResolution?.group === "webhook") { const { createWebPushProvider } = await import("./jobs/web-push-provider.js"); telephonyWebPushProvider = createWebPushProvider({ env: process.env }); } // The legacy runtime entrypoint reads WORKER_CONCURRENCY from env. We // funnel the resolved per-group concurrency through that same channel so // the existing config validation + diagnostics path keeps working. const runtimeEnv: NodeJS.ProcessEnv = groupResolution ? { ...process.env, WORKER_CONCURRENCY: String(groupResolution.concurrency) } : process.env; // When a group is resolved we override the BullMQ worker factory so the // ioredis connection is tagged `worker--` (visible in Redis // CLIENT LIST for operator attribution) and so the per-group pool size // is applied. The legacy path (no group) keeps the original factory. const groupAwareCreateWorker = groupResolution ? [0]>( queueName: Name, processor: Parameters>[1], options: Parameters>[2] ) => { const augmented = createWorkerConnection(groupResolution.group, { env: process.env as NodeJS.ProcessEnv }); // BullMQ's WorkerOptions.connection union accepts plain connection // options OR a Redis instance. We always pass plain options here // (parseRedisUrl output), so a cast keeps the spread narrowed. const mergedConnection = { ...(options.connection as Record), ...augmented } as Parameters>[2]["connection"]; return createWorker(queueName, processor, { ...options, connection: mergedConnection, }); } : undefined; const extraHandlerDependencies: Partial = {}; extraHandlerDependencies.resolvePlatformLeadNotificationRecipient = resolvePlatformLeadNotificationRecipient; if (groupResolution?.group === "general" || groupResolution?.group === "import") { extraHandlerDependencies.tenantMaintenanceLeaseStore = getSharedStore(); } let filePreviewQueue: TypedQueue | undefined; let telephonyWebhookQueue: BullMQQueue | undefined; let tenantDeletionExportQueue: BullMQQueue | undefined; if (groupResolution?.group === "general") { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (redisUrl) { const { getDbClient } = await import("@hivegrid/db/client.server"); extraHandlerDependencies.runKwpProjectMaintenance = async () => { const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) }); try { return await runAllTenantKwpProjectMaintenance({ prisma: getDbClient() as unknown as Parameters[0]["prisma"], queue, }); } finally { await queue.close(); } }; workerLogger.info({}, "KWP project maintenance scheduler service bootstrapped (general group)."); } } if (groupResolution?.group === "import") { const [{ syncKwpProjectBinding }, { syncKwpProjectInbound }, { discoverKwpProjectArchiveBatch }, { syncKwpArchiveFile }, { syncKwpProjectCatalog }, { runKwpAllContractSentinels }] = await Promise.all([ import("@hivegrid/integrations/kwp-project-sync.server"), import("@hivegrid/integrations/kwp-inbound-sync.server"), import("@hivegrid/integrations/kwp-archive-discovery.server"), import("@hivegrid/integrations/kwp-archive-file.server"), import("@hivegrid/integrations/kwp-project-catalog.server"), import("@hivegrid/integrations/kwp-contract-sentinel.server"), ]); extraHandlerDependencies.syncKwpProjectBinding = (payload) => syncKwpProjectBinding({ bindingId: payload.bindingId }); extraHandlerDependencies.syncKwpProjectInbound = (payload) => syncKwpProjectInbound({ bindingId: payload.bindingId }); extraHandlerDependencies.syncKwpArchiveDiscovery = async (payload) => { const result = await discoverKwpProjectArchiveBatch({ bindingId: payload.bindingId, ...(payload.cursor ? { cursor: payload.cursor } : {}), }); return { ...result, cursor: result.cursor ? { pendingFolderIds: [...result.cursor.pendingFolderIds], visitedFolderIds: [...result.cursor.visitedFolderIds], } : null, }; }; extraHandlerDependencies.syncKwpProjectCatalog = (payload) => syncKwpProjectCatalog({ connectionId: payload.dataConnectionId }); extraHandlerDependencies.enqueueKwpProjectCatalogContinuation = async (payload) => { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (!redisUrl) { throw new Error("KWP project catalog continuation requires BULLMQ_REDIS_URL or REDIS_URL."); } const continuationBucket = `${payload.cadenceBucket}-${Date.now()}`; const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) }); try { const descriptor = buildKwpProjectCatalogSyncJob({ tenantId: payload.tenantId, dataConnectionId: payload.dataConnectionId, cadenceBucket: continuationBucket, }); await enqueueJob(queue, descriptor, { idempotencyKey: `kwp-project-catalog:${payload.tenantId}:${payload.dataConnectionId}:${continuationBucket}`, tenantId: payload.tenantId, delay: 60_000, attempts: 5, backoff: { type: "exponential", delay: 30_000 }, }); } finally { await queue.close(); } }; extraHandlerDependencies.enqueueKwpArchiveDiscovery = async (payload, cursor) => { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (!redisUrl) { throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL."); } const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) }); try { const { descriptor, options } = buildKwpArchiveDiscoveryEnqueue({ payload, cursor, now: new Date() }); await enqueueJob(queue, descriptor, options); } finally { await queue.close(); } }; extraHandlerDependencies.enqueueKwpArchiveFileJobs = async (payload, files) => { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (!redisUrl) { throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL."); } const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) }); try { for (const archiveFile of files) { const { descriptor, options } = buildKwpArchiveFileEnqueue(payload, archiveFile); await enqueueJob(queue, descriptor, options); } } finally { await queue.close(); } return files.length; }; extraHandlerDependencies.syncKwpArchiveFile = (payload) => { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (!redisUrl) { throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL."); } const storageBucket = process.env.STORAGE_BUCKET?.trim(); if (!storageBucket) { throw new Error("KWP archive import requires STORAGE_BUCKET."); } return syncKwpArchiveFile({ bindingId: payload.bindingId, archiveFile: payload.archiveFile, storage: createWorkerStorageDeleteService(process.env), storageBucket, enqueueScan: async (input) => { const queue = createQueue(queueNames.fileScan, { connection: parseRedisUrl(redisUrl) }); try { const descriptor = buildFileScanJob({ bucket: input.bucket, key: input.key, tenantId: input.tenantId, fileId: input.fileId, contentType: input.contentType, metadata: { source: "kwp" }, }); await enqueueJob(queue, descriptor, { idempotencyKey: `kwp-archive-scan:${input.tenantId}:${input.fileId}:${payload.cadenceBucket}`, tenantId: input.tenantId, attempts: 5, backoff: { type: "exponential", delay: 30_000 }, }); } finally { await queue.close(); } }, }); }; extraHandlerDependencies.runKwpContractSentinel = (payload) => runKwpAllContractSentinels({ connectionId: payload.dataConnectionId }); workerLogger.info({}, "KWP provider import services bootstrapped (import group)."); } if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (!redisUrl) { throw new Error("Telephony webhook maintenance requires BULLMQ_REDIS_URL or REDIS_URL."); } const { getDbClient } = await import("@hivegrid/db/client.server"); const queue = new BullMQQueue(TELEPHONY_WEBHOOK_QUEUE_NAME, { connection: parseRedisUrl(redisUrl), }); telephonyWebhookQueue = queue; extraHandlerDependencies.runTelephonyWebhookMaintenance = () => runAllTenantTelephonyWebhookMaintenance(getDbClient(), async (input, source) => { const descriptor = buildTelephonyWebhookJob(input); if (source === "failed") { await recoverFailedTelephonyWebhookJob(queue, descriptor); } else { await enqueueTelephonyWebhookJob(queue, descriptor); } }); workerLogger.info({ queue: TELEPHONY_WEBHOOK_QUEUE_NAME }, "telephony webhook maintenance service bootstrapped (general group)."); } catch (error) { workerLogger.error({ error }, "telephony webhook maintenance bootstrap failed; maintenance jobs will retry until fixed."); } } if (groupResolution?.group === "general") { const providerSideEffectMode = resolveProductAcquisitionProviderSideEffectMode(process.env); if (providerSideEffectMode === "disabled") { workerLogger.info({}, "product acquisition reconciler disabled with provider side effects."); } else { try { const apiKey = process.env.MOLLIE_API_KEY?.trim(); const lagoApiUrl = process.env.LAGO_API_URL?.trim(); const lagoApiKey = process.env.LAGO_API_KEY?.trim(); if (!apiKey) throw new Error("Product acquisition reconciliation requires MOLLIE_API_KEY."); if (!lagoApiUrl || !lagoApiKey) throw new Error("Product acquisition reconciliation requires Lago configuration."); const [{ createLagoBillingGateway, createMolliePaymentGateway }, { applyMolliePaymentSync }, { getDbClient }] = await Promise.all([ import("@hivegrid/billing"), import("@hivegrid/db/billing-sync.server"), import("@hivegrid/db/client.server"), ]); const prisma = asProductAcquisitionReconcileClient(getDbClient()); const mollie = createMolliePaymentGateway({ apiKey, profileId: process.env.MOLLIE_PROFILE_ID?.trim() || null, timeoutMs: resolvePositiveInteger(process.env.MOLLIE_TIMEOUT_MS, 10_000), }); const lago = createLagoBillingGateway({ apiUrl: lagoApiUrl, apiKey: lagoApiKey, timeoutMs: resolvePositiveInteger(process.env.LAGO_TIMEOUT_MS, 10_000), }); extraHandlerDependencies.runProductAcquisitionReconcile = ({ take } = {}) => runAllTenantProductAcquisitionReconciliation( { prisma, mollie, lago, applyPaymentSync: (input) => applyMolliePaymentSync(input, getDbClient() as unknown as NonNullable[1]>), alert: async (alert) => { const details = { code: alert.code, tenantId: alert.tenantId, acquisitionId: alert.acquisitionId, status: alert.status, providerPaymentId: alert.providerPaymentId, }; workerLogger.error(details, "billing product acquisition reconciliation requires operator attention"); await workerErrorReporter.report({ component: "worker", operation: "billing.product-acquisition-reconcile.quarantined", severity: "error", ...details, }); }, }, { batchSize: take } ); workerLogger.info({}, "product acquisition reconciler bootstrapped (general group)."); } catch (error) { workerLogger.error({ error }, "product acquisition reconciler bootstrap failed; maintenance jobs will retry until fixed."); } } } if (groupResolution?.group === "general") { try { const { getDbClient } = await import("@hivegrid/db/client.server"); extraHandlerDependencies.reconcileWorkflowExecutionStalls = createWorkflowExecutionStalledReconciler({ prisma: getDbClient() as unknown as WorkflowExecutionStalledReconcilerPrisma, }); workerLogger.info({}, "workflow-execution-stalled-reconciler service bootstrapped (general group)."); } catch (error) { workerLogger.error({ error }, "workflow-execution-stalled-reconciler bootstrap failed; maintenance handler will use DB fallback."); } } // AK-P4a §4 — file-preview generator service on the `general` group // (the group that owns the hivegrid.file-preview queue). Composed with a // Prisma client + STORAGE_* S3 seams; injected via extraHandlerDependencies. // When STORAGE_* is unset the factory returns undefined and the handler // dead-letters preview jobs. if (groupResolution?.group === "general") { try { const { getDbClient } = await import("@hivegrid/db/client.server"); const { createWorkerFilePreviewService } = await import("./jobs/file-preview.js"); const previewService = createWorkerFilePreviewService(process.env as Parameters[0], { prisma: getDbClient() as unknown as Parameters[1]["prisma"], logger: workerLogger, }); if (previewService) { extraHandlerDependencies.filePreviewService = previewService; workerLogger.info({}, "file-preview service bootstrapped (general group)."); } else { workerLogger.info({}, "file-preview service skipped: STORAGE_* not configured."); } // AK-P4a §5 — scan-verdict write-back. The general group owns the // hivegrid.file-scan queue, so wire the File.scanStatus writer here with // the same Prisma seam the preview service uses. const { createWorkerFileScanStatusWriter } = await import("./jobs/file-scan-status.js"); extraHandlerDependencies.fileScanStatusWriter = createWorkerFileScanStatusWriter({ prisma: getDbClient() as unknown as Parameters[0]["prisma"], logger: workerLogger, }); const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (!redisUrl) { throw new Error("Redis is required to enqueue previews after clean file scans."); } filePreviewQueue = createQueue(queueNames.filePreview, { connection: parseRedisUrl(redisUrl) }); extraHandlerDependencies.filePreviewEnqueuer = { enqueueAfterCleanScan: async (input) => { const descriptor = buildFilePreviewJob({ bucket: input.bucket, fileId: input.fileId, mimeType: input.contentType, storageKey: input.storageKey, tenantId: input.tenantId, }); await enqueueJob(filePreviewQueue!, descriptor, { jobId: `file-preview-after-scan-${input.fileId}`, tenantId: input.tenantId, }); }, }; workerLogger.info({}, "file-scan status writer bootstrapped (general group)."); } catch (error) { workerLogger.error({ error }, "file-preview bootstrap failed; preview jobs will dead-letter until fixed."); } } // Phase 06 #21 / F-015 — wire the real plugin-invoke dependencies on // the `plugin` worker group. Without this, `handlers.ts` throws when a // plugin-invoke job arrives — which is the intentional fail-loud // behaviour that F-015 introduced to replace the prior silent no-op // stub. The composition lives in jobs/plugin-invoke-bootstrap.ts. const pluginInvokeRuntimeEnabled = groupResolution?.group === "plugin" ? isPluginInvokeRuntimeEnabled(process.env) : true; if (groupResolution?.group === "plugin" && pluginInvokeRuntimeEnabled) { try { const { getDbClient } = await import("@hivegrid/db/client.server"); const { bootstrapPluginInvokeDeps } = await import("./jobs/plugin-invoke-bootstrap.js"); const { getSharedStore } = await import("./shared-store.js"); const prisma = getDbClient() as unknown as Parameters[0]["prisma"]; extraHandlerDependencies.pluginInvokeDeps = bootstrapPluginInvokeDeps({ env: process.env, prisma, store: getSharedStore(), logger: workerLogger, }); workerLogger.info({}, "plugin-invoke production deps bootstrapped (plugin group)."); } catch (error) { workerLogger.error({ error }, "plugin-invoke bootstrap failed in plugin worker group; exiting."); throw error; } if (!extraHandlerDependencies.pluginInvokeDeps) { workerLogger.error({}, "plugin-invoke bootstrap produced no dependencies in plugin worker group; exiting."); process.exit(1); throw new Error("plugin-invoke bootstrap produced no dependencies"); } } else if (groupResolution?.group === "plugin") { workerLogger.warn( { workerGroup: groupResolution.group, queueNames: groupResolution.queueNames }, "plugin-invoke runtime disabled before production signoff; starting plugin worker idle without queue consumers." ); } // The `import` group always owns the dedicated KWP queue. Generic imports and // exports remain separately gated: full mode adds both, while the internal // portal-GDPR mode adds only exports and wires only the two GDPR services. const importQueueRuntimeMode = resolveImportQueueRuntimeMode(process.env); const importQueueRuntimeEnabled = importQueueRuntimeMode === "full" || importQueueRuntimeMode === "portal-gdpr-export"; if (groupResolution?.group === "import" && importQueueRuntimeMode === "idle") { workerLogger.warn( { workerGroup: groupResolution.group, queueNames: groupResolution.queueNames }, "Generic import/export runtime gated off; starting the import worker with the dedicated KWP queue only." ); } if (groupResolution?.group === "import" && importQueueRuntimeEnabled) { try { const { getDbClient } = await import("@hivegrid/db/client.server"); if (importQueueRuntimeMode === "full") { const { createAdminSalesDataWorkerService } = await import("./jobs/admin-sales-data.js"); const adminSalesDataService = createAdminSalesDataWorkerService(process.env, { prisma: getDbClient() as unknown as Parameters[1]["prisma"], }); if (adminSalesDataService) { extraHandlerDependencies.adminSalesDataService = adminSalesDataService; workerLogger.info({}, "admin sales import/export service bootstrapped (import group)."); } } const { createWorkerPrivacyGdprExportService } = await import("./jobs/privacy-gdpr-export-bootstrap.js"); const privacyGdprExportService = createWorkerPrivacyGdprExportService(process.env, { prisma: getDbClient() as unknown as Parameters[1]["prisma"], sharedStore: getSharedStore(), }); if (privacyGdprExportService) { extraHandlerDependencies.privacyGdprExportService = privacyGdprExportService; workerLogger.info({}, "privacy GDPR export service bootstrapped (import group)."); // AK-P21 — best-effort "export ready" mail via the email queue. Both // import runtime modes own the privacy GDPR export handler, so the // mailer must not stay gated behind the `full` mode: in // `portal-gdpr-export` mode the very same job finished silently. const mailRedisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (mailRedisUrl) { const { createGdprExportReadyMailer } = await import("./jobs/privacy-gdpr-export-bootstrap.js"); const { buildNotificationJob, buildRetryJobOptions } = await import("@hivegrid/queue"); // AGENTS.md DoD C15 — go through the M011 notification queue, not the // raw transactional mail queue: the notification handler owns the // preference check, the NotificationDelivery record and the provider // call. const notificationQueue = new BullMQQueue(queueNames.notifications, { connection: parseRedisUrl(mailRedisUrl) }); extraHandlerDependencies.sendGdprExportReadyMail = createGdprExportReadyMailer({ prisma: getDbClient() as unknown as Parameters[0]["prisma"], enqueueNotification: async (input) => { const descriptor = buildNotificationJob(input); await notificationQueue.add(descriptor.jobName, descriptor.payload, buildRetryJobOptions(descriptor)); }, ...(process.env.APP_URL ? { appUrl: process.env.APP_URL } : {}), logger: workerLogger, }); workerLogger.info({}, "privacy GDPR export ready-mailer bootstrapped (import group)."); } } else { workerLogger.info({}, "privacy GDPR export service skipped: STORAGE_* not configured."); } const { createWorkerPortalGdprExportService } = await import("./jobs/portal-gdpr-export-bootstrap.js"); const portalGdprExportService = createWorkerPortalGdprExportService(process.env, { prisma: getDbClient() as unknown as Parameters[1]["prisma"], sharedStore: getSharedStore(), logger: workerLogger, }); if (portalGdprExportService) { extraHandlerDependencies.portalGdprExportService = portalGdprExportService; workerLogger.info({}, "portal GDPR JSON export service bootstrapped (import group)."); } else { workerLogger.info({}, "portal GDPR JSON export service skipped: STORAGE_* not configured."); } } catch (error) { workerLogger.error({ error }, "privacy GDPR export bootstrap failed; GDPR export jobs will dead-letter until fixed."); } } // AK-P21 — tenant hard-delete service (general group owns the maintenance // queue). Local dev/test may explicitly skip external cleanup, while // required staging/production runtimes fail closed before queue ownership. if (groupResolution?.group === "general") { const tenantHardDeleteRequired = isRequiredTenantHardDeleteRuntime(process.env); try { const { getDbClient } = await import("@hivegrid/db/client.server"); const { createWorkerPrivacyTenantDeletionService } = await import("./jobs/privacy-tenant-deletion-bootstrap.js"); if (tenantHardDeleteRequired && (!process.env.MEILI_URL?.trim() || !process.env.MEILI_API_KEY?.trim())) { throw new Error("Required tenant hard-delete search cleanup configuration is missing."); } let searchClient; if (process.env.MEILI_URL) { const { createSearchClient } = await import("@hivegrid/search"); searchClient = createSearchClient({ url: process.env.MEILI_URL, apiKey: process.env.MEILI_API_KEY ?? undefined }); } const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (redisUrl) { tenantDeletionExportQueue = new BullMQQueue(queueNames.exports, { connection: parseRedisUrl(redisUrl) }); } extraHandlerDependencies.privacyTenantDeletionService = createWorkerPrivacyTenantDeletionService(process.env, { prisma: getDbClient() as unknown as Parameters[1]["prisma"], ...(searchClient ? { searchClient } : {}), ...(tenantDeletionExportQueue ? { exportQueue: tenantDeletionExportQueue } : {}), ...(redisUrl ? { sharedStore: getSharedStore() } : {}), logger: workerLogger, }); workerLogger.info({}, "privacy tenant-deletion service bootstrapped (general group)."); } catch (error) { if (tenantHardDeleteRequired) { workerLogger.error( { errorName: error instanceof Error ? error.name : "UnknownError" }, "privacy tenant-deletion bootstrap failed in required mode; exiting." ); process.exit(1); throw new Error("Worker startup aborted after required tenant hard-delete bootstrap failure."); } workerLogger.error({ errorName: error instanceof Error ? error.name : "UnknownError" }, "privacy tenant-deletion bootstrap failed; continuing."); } } // R-P2b — DSGVO grace-period cron service. Without this block the daily // scan acks every tick as "not configured" and confirmed user deletion // requests are never executed (found in the 2026-07-17 ticket audit). if (groupResolution?.group === "general") { try { const { getDbClient } = await import("@hivegrid/db/client.server"); const { createWorkerDsgvoGraceCronService } = await import("./jobs/dsgvo-grace-bootstrap.js"); extraHandlerDependencies.dsgvoGraceCronService = createWorkerDsgvoGraceCronService({ prisma: getDbClient() as unknown as Parameters[0]["prisma"], logger: workerLogger, }); workerLogger.info({}, "dsgvo grace-cron service bootstrapped (general group)."); } catch (error) { workerLogger.error({ error }, "dsgvo grace-cron bootstrap failed; grace scans will skip until fixed."); } } // VA-P26b — PortalUser lifecycle is isolated from the internal User/TenantUser // privacy path. The general group owns the bounded daily maintenance scan. if (groupResolution?.group === "general") { try { const { getDbClient } = await import("@hivegrid/db/client.server"); const { createWorkerPortalLifecycleService } = await import("./jobs/portal-lifecycle-bootstrap.js"); extraHandlerDependencies.portalLifecycleService = createWorkerPortalLifecycleService(process.env, { prisma: getDbClient() as unknown as Parameters[1]["prisma"], logger: workerLogger, }); workerLogger.info({}, "portal lifecycle service bootstrapped (general group)."); } catch (error) { workerLogger.error({ error }, "portal lifecycle bootstrap failed; maintenance jobs will retry until fixed."); } } const runtimeQueueNames = groupResolution ? !pluginInvokeRuntimeEnabled ? [] : groupResolution.group === "import" ? getWorkerQueueNames(process.env) : groupResolution.queueNames : undefined; const runtime = startWorkerRuntime({ logger: workerLogger, reporter: workerErrorReporter, sentry: sentryForTracing, metrics, env: runtimeEnv, extraHandlerDependencies, ...(runtimeQueueNames ? { queueNames: runtimeQueueNames } : {}), ...(groupAwareCreateWorker ? { createWorkerImpl: groupAwareCreateWorker } : {}), }); if (groupResolution) { workerLogger.info( { workerGroup: groupResolution.group, queueNames: runtime.queueNames, concurrency: groupResolution.concurrency, }, "Worker queue group resolved." ); } let outboxStatus: OutboxDispatcherBootstrapResult | undefined; let workflowTriggerStatus: WorkflowTriggerBootstrapResult | undefined; let pluginHookFanoutStatus: PluginHookFanoutBootstrapResult | undefined; let transactionalMailOutboxStatus: TransactionalMailOutboxBootstrapResult | undefined; let portalStatusMailStatus: PortalStatusMailBootstrapResult | undefined; let emailSequencesStatus: EmailSequenceBootstrapResult | undefined; let chatLinkPreviewStatus: ChatLinkPreviewBootstrapResult | undefined; const outboxDispatcherExpected = groupResolution?.group === "general" && isOutboxDispatcherExpected(process.env); const workflowTriggerExpected = groupResolution?.group === "general" && isWorkflowTriggerOutboxExpected(process.env); const pluginHookFanoutExpected = isPluginHookFanoutExpected(process.env, groupResolution?.group); const transactionalMailOutboxExpected = isTransactionalMailOutboxExpected(process.env, groupResolution?.group); const portalStatusMailExpected = isPortalStatusMailExpected(groupResolution?.group); const emailSequencesExpected = isEmailSequencesExpected(process.env, groupResolution?.group); const chatLinkPreviewExpected = isChatLinkPreviewExpected(process.env, groupResolution?.group); const adminServer = isWorkerMetricsEnabled(process.env) ? startWorkerAdminServer({ enabled: true, token: process.env.METRICS_TOKEN, registry, port: Number.parseInt(process.env.WORKER_ADMIN_PORT ?? "", 10) || DEFAULT_WORKER_ADMIN_PORT, host: process.env.WORKER_ADMIN_HOST, logger: workerLogger, buildStatus: () => buildWorkerStatus(runtime, { outboxDispatcher: outboxStatus, outboxDispatcherExpected, workflowTrigger: workflowTriggerStatus, workflowTriggerExpected, workflowTriggerExecutorMode: parseWorkflowTriggerExecutorMode(process.env.WORKFLOW_TRIGGER_EXECUTOR_MODE), pluginHookFanout: pluginHookFanoutStatus, pluginHookFanoutExpected, transactionalMailOutbox: transactionalMailOutboxStatus, transactionalMailOutboxExpected, portalStatusMail: portalStatusMailStatus, portalStatusMailExpected, emailSequences: emailSequencesStatus, emailSequencesExpected, chatLinkPreview: chatLinkPreviewStatus, chatLinkPreviewExpected, }), }) : undefined; // Health server: always on when a runtime started. Bound to a separate // port from the admin server so liveness probes still work when // METRICS_ENABLED is false (eg early-stage staging). const redisHealthProbe = runtime.status === "started" ? createRedisHealthProbe(process.env) : undefined; const healthServer = runtime.status === "started" ? startWorkerHealthServer({ port: Number.parseInt(process.env.WORKER_HEALTH_PORT ?? "", 10) || DEFAULT_WORKER_HEALTH_PORT, host: process.env.WORKER_HEALTH_HOST, ...(groupResolution ? { workerGroup: groupResolution.group } : {}), redisProbe: redisHealthProbe!, databaseProbe: createDatabaseHealthProbe(), ...(() => { const readinessProbes = createRuntimeReadinessProbes({ workerGroup: groupResolution?.group, outboxDispatcherExpected, workflowTriggerExpected, pluginHookFanoutExpected, transactionalMailOutboxExpected, portalStatusMailExpected, emailSequencesExpected, chatLinkPreviewExpected, getOutboxDispatcherStatus: () => outboxStatus, getWorkflowTriggerStatus: () => workflowTriggerStatus, getPluginHookFanoutStatus: () => pluginHookFanoutStatus, getTransactionalMailOutboxStatus: () => transactionalMailOutboxStatus, getPortalStatusMailStatus: () => portalStatusMailStatus, getEmailSequencesStatus: () => emailSequencesStatus, getChatLinkPreviewStatus: () => chatLinkPreviewStatus, }); return readinessProbes ? { readinessProbes } : {}; })(), logger: workerLogger, }) : undefined; if (runtime.status === "disabled") { if (telephonyWebhookQueue) { await telephonyWebhookQueue.close(); } if (tenantDeletionExportQueue) { await tenantDeletionExportQueue.close(); } if (adminServer) { await adminServer.close(); } if (healthServer) { await healthServer.close(); } process.exit(0); } else { let outbox: OutboxDispatcherBootstrapResult | undefined; let adminBookingOutbox: AdminBookingOutboxBootstrapResult | undefined; let portalDomainOrchestration: PortalDomainOrchestrationBootstrapResult | undefined; let productAcquisitionProviderSideEffect: ProductAcquisitionProviderSideEffectBootstrapResult | undefined; let recurringInvoiceCollection: RecurringInvoiceCollectionBootstrapResult | undefined; let telephonyWebhookWorker: BullMQWorker | undefined; let automationExecutorWorker: BullMQWorker | undefined; if (groupResolution?.group === "webhook") { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (!redisUrl) { throw new Error("Telephony webhook worker requires BULLMQ_REDIS_URL or REDIS_URL."); } const [ { getDbClient }, { listActiveDeviceTokens, revokeDeviceToken }, { createTelephonyWebhookJobProcessor }, { dispatchPushNotification }, { withTenantRLS }, { buildTelephonyPushHref }, ] = await Promise.all([ import("@hivegrid/db/client.server"), import("@hivegrid/db/device-tokens.server"), import("./jobs/telephony-webhook-bootstrap.js"), import("./jobs/push-notification.js"), import("./db/tenantRls.js"), import("./jobs/telephony-push-target.js"), ]); const prisma = getDbClient(); const webPushProvider = telephonyWebPushProvider; if (!webPushProvider) { throw new Error("Telephony webhook worker started without a validated Web Push provider."); } const processor = createTelephonyWebhookJobProcessor(prisma, { pushIncomingCall: async (input) => { const tenantSlug = await withTenantRLS(prisma, input.tenantId, async (tx) => { const tenant = await tx.tenant.findFirst({ where: { id: input.tenantId }, select: { slug: true }, }); if (!tenant) throw new Error("Telephony push tenant is unavailable."); return tenant.slug; }); await dispatchPushNotification({ payload: { tenantId: input.tenantId, userId: input.userId, title: "HiveGrid", body: input.externalNumber, data: { callId: input.callId, href: buildTelephonyPushHref({ tenantSlug, callId: input.callId, target: input.target, }), messageKey: "telephony.notifications.incomingCall", }, }, tokenLister: ({ userId, tenantId }) => withTenantRLS(prisma, input.tenantId, (tx) => listActiveDeviceTokens( { userId, ...(tenantId ? { tenantId } : {}), }, tx as never ) ), tokenRevoker: async (token) => { await withTenantRLS(prisma, input.tenantId, (tx) => revokeDeviceToken( { token, userId: input.userId, tenantId: input.tenantId, }, tx as never ) ); }, providers: webPushProvider, logger: workerLogger, }); }, }); telephonyWebhookWorker = new BullMQWorker(TELEPHONY_WEBHOOK_QUEUE_NAME, createRlsAwareBullMqProcessor(processor), { connection: parseRedisUrl(redisUrl), concurrency: groupResolution.concurrency, }); workerLogger.info({ queue: TELEPHONY_WEBHOOK_QUEUE_NAME, concurrency: groupResolution.concurrency }, "telephony webhook worker started"); } // The outbox dispatcher is a cron-style polling job that lives on the // `general` group. Running it from every worker group would either (a) // multiply Redis/Postgres load by N replicas or (b) race for the same // FOR UPDATE SKIP LOCKED rows. When the group is undefined (legacy // WORKER_ENABLED=false path) we leave the dispatcher off too. const shouldRunOutbox = groupResolution?.group === "general"; const outboxRequiredForGroup = shouldRunOutbox && (isOutboxDispatcherRequired(process.env) || isWorkflowTriggerOutboxRequired(process.env)); if (shouldRunOutbox) { try { const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient(); outbox = await bootstrapOutboxDispatcher(process.env, { prisma, logger: workerLogger, reporter: workerErrorReporter, }); outboxStatus = outbox; if (outbox.status === "disabled" && outboxRequiredForGroup) { workerLogger.error({ reason: "OUTBOX_DISPATCHER_ENABLED!=true" }, "Outbox dispatcher disabled in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required outbox dispatcher disabled."); } } catch (error) { outboxStatus = { status: "disabled" }; if (outboxRequiredForGroup) { workerLogger.error({ error }, "Outbox dispatcher bootstrap failed in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required outbox dispatcher failure."); } workerLogger.error({ error }, "Outbox dispatcher bootstrap failed; continuing without dispatcher."); } } else if (groupResolution) { workerLogger.info({ workerGroup: groupResolution.group }, "Outbox dispatcher skipped on this worker group; it runs only on the general group."); } if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); if (redisUrl) { const [{ getDbClient }, { createAdminBookingOutboxDependencies }] = await Promise.all([ import("@hivegrid/db/client.server"), import("@hivegrid/integrations/admin-booking-worker.server"), ]); const connection = createWorkerConnection("general", { env: process.env as NodeJS.ProcessEnv, }); adminBookingOutbox = bootstrapAdminBookingOutbox(process.env, { dependencies: createAdminBookingOutboxDependencies({ client: getDbClient(), env: process.env, }), createWorker: (queueName, processor, options) => new BullMQWorker(queueName, processor, { connection, concurrency: options.concurrency, }) as unknown as AdminBookingOutboxWorkerHandle, logger: workerLogger, }); } else { adminBookingOutbox = { status: "disabled", reason: "no Redis URL configured", }; if (process.env.ADMIN_BOOKING_OUTBOX_REQUIRED === "true" || process.env.RLS_RUNTIME_MODE === "required") { throw new Error("Admin booking provider worker requires Redis in required mode."); } } } catch (error) { adminBookingOutbox = { status: "disabled", reason: "bootstrap failed", }; workerLogger.error({ error }, "Admin booking provider worker bootstrap failed."); if (process.env.ADMIN_BOOKING_OUTBOX_REQUIRED === "true" || process.env.RLS_RUNTIME_MODE === "required") { throw error; } } } if (groupResolution?.group === "general") { try { portalDomainOrchestration = await bootstrapPortalDomainOrchestration(process.env, { logger: workerLogger, reporter: workerErrorReporter, }); if (portalDomainOrchestration.status === "disabled") { if (process.env.PORTAL_DOMAIN_ORCHESTRATION_REQUIRED === "true") { throw new Error(`Portal-domain orchestration is required but disabled: ${portalDomainOrchestration.reason}`); } workerLogger.info({ reason: portalDomainOrchestration.reason }, "portal-domain orchestration disabled."); } } catch (error) { if (process.env.PORTAL_DOMAIN_ORCHESTRATION_REQUIRED === "true") { workerLogger.error({ error }, "Portal-domain orchestration failed in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required portal-domain orchestration failure."); } portalDomainOrchestration = { status: "disabled", reason: "bootstrap failed" }; workerLogger.error({ error }, "Portal-domain orchestration bootstrap failed; continuing while custom-domain activation stays unavailable."); } } if (groupResolution?.group === "general") { const providerSideEffectRequired = resolveProductAcquisitionProviderSideEffectMode(process.env) === "required"; try { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient(); const connection = redisUrl ? parseRedisUrl(redisUrl) : null; productAcquisitionProviderSideEffect = bootstrapProductAcquisitionProviderSideEffect(process.env, { logger: workerLogger, claimOutboxEvent: (envelope) => claimProductProviderSideEffectOutboxEvent(envelope, { store: prisma }), releaseOutboxClaim: (envelope, claim) => releaseProductProviderSideEffectOutboxClaim(envelope, claim, { store: prisma }), loadAcquisition: async ({ tenantId, acquisitionId }) => prisma.tenantProductAcquisition.findFirst({ where: { id: acquisitionId, tenantId }, select: { id: true, tenantId: true, productKind: true, status: true, providerAttempt: true, providerPaymentId: true, invoiceExternalId: true, lagoSubscriptionId: true, lagoProviderSubscriptionId: true, initialLagoInvoiceExternalId: true, }, }), createWorker: (queueName, processor, options) => { if (!connection) throw new Error("Product-acquisition provider side-effect worker requires Redis."); const worker = new BullMQWorker( queueName, async (job, token) => runWorkerPayloadWithRlsContext(job.data, () => runProductProviderSideEffectWithReceipt(job, processor, { store: prisma, logger: workerLogger, report: (event) => workerErrorReporter.report(event as never), deferReceiptFailure: async () => { await job.moveToDelayed(Date.now() + 5_000, token); throw new BullMQDelayedError(); }, }) ), { connection, concurrency: options.concurrency, } ); return worker; }, }); if (productAcquisitionProviderSideEffect.status === "disabled") { workerLogger.info({ reason: productAcquisitionProviderSideEffect.reason }, "product-acquisition provider side-effect worker disabled"); } } catch (error) { if (providerSideEffectRequired) { workerLogger.error({ error }, "Product-acquisition provider side-effect bootstrap failed in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required product-acquisition provider side-effect failure."); } productAcquisitionProviderSideEffect = { status: "disabled", reason: "bootstrap failed" }; workerLogger.error({ error }, "Product-acquisition provider side-effect bootstrap failed; continuing without subscriber."); } } if (groupResolution?.group === "general") { const providerSideEffectRequired = resolveProductAcquisitionProviderSideEffectMode(process.env) === "required"; try { const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim(); const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient(); const connection = redisUrl ? parseRedisUrl(redisUrl) : null; recurringInvoiceCollection = bootstrapRecurringInvoiceCollection(process.env, { logger: workerLogger, claimOutboxEvent: (envelope) => claimProductProviderSideEffectOutboxEvent(envelope, { store: prisma }), releaseOutboxClaim: (envelope, claim) => releaseProductProviderSideEffectOutboxClaim(envelope, claim, { store: prisma }), loadCollection: async ({ tenantId, collectionId, lagoSubscriptionId, billingProviderSubscriptionId }) => { const [ledger, subscription] = await Promise.all([ prisma.checkoutSessionStatus.findFirst({ where: { id: collectionId, tenantId }, select: { id: true, tenantId: true, pending: true, status: true, providerPaymentId: true, invoiceExternalId: true, lagoSubscriptionId: true, scopeKey: true, billingCadence: true, amountCents: true, currency: true, }, }), prisma.tenantSubscription.findUnique({ where: { tenantId }, select: { status: true, mollieCustomerId: true, products: { where: { lagoSubscriptionId, billingProviderSubscriptionId, }, select: { id: true }, take: 1, }, }, }), ]); if ( !ledger || !subscription?.mollieCustomerId?.trim() || subscription.products.length !== 1 || (subscription.status !== "active" && subscription.status !== "trial") ) { return null; } return { ...ledger, billingProviderSubscriptionId: ledger.scopeKey, mollieCustomerId: subscription.mollieCustomerId, }; }, updateCollection: async ({ tenantId, collectionId, providerPaymentId, status, pending }) => { const updated = await prisma.checkoutSessionStatus.updateMany({ where: { id: collectionId, tenantId, pending: true, providerPaymentId: null, }, data: { ...(providerPaymentId ? { providerPaymentId } : {}), status, pending, }, }); return updated.count === 1; }, createWorker: (queueName, processor, options) => { if (!connection) throw new Error("Recurring invoice collection worker requires Redis."); const worker = new BullMQWorker( queueName, async (job, token) => runWorkerPayloadWithRlsContext(job.data, () => runProductProviderSideEffectWithReceipt(job, processor, { store: prisma, logger: workerLogger, report: (event) => workerErrorReporter.report(event as never), deferReceiptFailure: async () => { await job.moveToDelayed(Date.now() + 5_000, token); throw new BullMQDelayedError(); }, }) ), { connection, concurrency: options.concurrency, } ); return worker; }, }); if (recurringInvoiceCollection.status === "disabled") { workerLogger.info({ reason: recurringInvoiceCollection.reason }, "recurring invoice collection worker disabled"); } } catch (error) { if (providerSideEffectRequired) { workerLogger.error({ error }, "Recurring invoice collection bootstrap failed in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required recurring invoice collection failure."); } recurringInvoiceCollection = { status: "disabled", reason: "bootstrap failed" }; workerLogger.error({ error }, "Recurring invoice collection bootstrap failed; continuing without subscriber."); } } let workflowTrigger: WorkflowTriggerBootstrapResult | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (process.env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "false") { workflowTrigger = bootstrapWorkflowTrigger(process.env, { logger: workerLogger }); workflowTriggerStatus = workflowTrigger; if (workflowTrigger.status === "disabled") { workerLogger.info({ reason: workflowTrigger.reason }, "workflow-trigger worker disabled by configuration."); } } else if (redisUrl) { const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient(); const executor = createWorkflowTriggerExecutorFromEnv(process.env, { logger: workerLogger, prisma: prisma as unknown as WorkflowTriggerNativePrisma, }); const connection = createWorkerConnection("general", { env: process.env as NodeJS.ProcessEnv }); workflowTrigger = bootstrapWorkflowTrigger(process.env, { logger: workerLogger, store: createPrismaWorkflowTriggerStore(prisma as unknown as WorkflowTriggerStorePrisma, { nativeCapabilityPreflight: parseWorkflowTriggerExecutorMode(process.env.WORKFLOW_TRIGGER_EXECUTOR_MODE) !== "http", }), ...(executor ? { executor } : {}), idempotency: createIdempotencyStore({ store: getSharedStore(), prefix: "workflow-trigger", ttlSeconds: 60 * 60 * 24, }), runPortalAutomation: (data) => runPortalAutomationTriggerJob(data, { prisma, logger: workerLogger, portalBaseUrl: resolvePortalStatusMailBaseUrl(process.env), }), runAutomationForward: createRunAutomationForwardTrigger({ prisma, logger: workerLogger }), createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as WorkflowTriggerWorkerHandle; }, }); if (workflowTrigger.status === "disabled") { if (isWorkflowTriggerOutboxRequired(process.env)) { workerLogger.error({ reason: workflowTrigger.reason }, "Workflow-trigger subscriber disabled in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required workflow-trigger subscriber disabled."); } workerLogger.warn({ reason: workflowTrigger.reason }, "workflow-trigger worker disabled."); } workflowTriggerStatus = workflowTrigger; if (workflowTrigger.status === "started") { // R-P12c: durable forward-executor consumer for // automation.execution.* outbox events (retry/delay continuations // arrive as scheduled outbox rows on the same queue). const automationExecutorProcessor = createAutomationExecutorProcessor({ prisma, logger: workerLogger }); automationExecutorWorker = new BullMQWorker( AUTOMATION_EXECUTOR_OUTBOX_QUEUE, createRlsAwareBullMqProcessor(async (job: { data: unknown }) => automationExecutorProcessor(job.data)), { connection, concurrency: 5 } ); workerLogger.info({ queue: AUTOMATION_EXECUTOR_OUTBOX_QUEUE }, "automation forward-executor worker started."); } } else { if (isWorkflowTriggerOutboxRequired(process.env)) { throw new Error("Workflow trigger outbox subscriber is required but disabled: no Redis URL configured."); } workflowTrigger = { status: "disabled", reason: "no Redis URL configured" }; workflowTriggerStatus = workflowTrigger; workerLogger.info({}, "workflow-trigger worker skipped: no Redis URL configured."); } } catch (error) { if (isWorkflowTriggerOutboxRequired(process.env)) { workerLogger.error({ error }, "Workflow-trigger bootstrap failed in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required workflow-trigger failure."); } workerLogger.error({ error }, "Workflow-trigger bootstrap failed; continuing without subscriber."); workflowTriggerStatus = { status: "disabled", reason: "bootstrap failed" }; } } else if (groupResolution && isWorkflowTriggerOutboxRequired(process.env)) { workerLogger.error({ workerGroup: groupResolution.group }, "Workflow-trigger subscriber required on a non-general worker group; exiting."); process.exit(1); throw new Error("Worker startup aborted after workflow-trigger required on a non-general worker group."); } // Phase 10 #68 — DSGVO grace-period cron. Like the outbox dispatcher // this is a single-instance scheduler and must therefore live on the // `general` group only (one replica per cluster). We register the // BullMQ job-scheduler entry; the actual processor is wired into the // maintenance queue by the maintenance handler, which inspects // job.name === DSGVO_GRACE_CRON_JOB_ID and calls runDsgvoGraceCron. if (groupResolution?.group === "general") { const exportRetentionRequired = process.env.EXPORT_RETENTION_QUEUE_ENABLED === "true"; const tenantHardDeleteRequired = isRequiredTenantHardDeleteRuntime(process.env); try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const connection = parseRedisUrl(redisUrl); const dsgvoCronQueue = new BullMQQueue(queueNames.maintenance, { connection, defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS, }); try { await registerDsgvoGraceCron(dsgvoCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "dsgvo-grace-cron scheduler registered (24h cadence)."); // AK-P21 — tenant hard-delete scan shares the maintenance queue // and the single-scheduled general-group invariant. const { registerPrivacyTenantDeletionCron } = await import("./jobs/privacy-tenant-deletion-bootstrap.js"); await registerPrivacyTenantDeletionCron(dsgvoCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "privacy-tenant-deletion scheduler registered (24h cadence)."); const { registerPortalLifecycleCron } = await import("./jobs/portal-lifecycle-bootstrap.js"); await registerPortalLifecycleCron(dsgvoCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "portal-lifecycle scheduler registered (24h cadence)."); if (exportRetentionRequired) { await registerPrivacyGdprExportRetentionCron(dsgvoCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "privacy GDPR export-retention scheduler registered (15m cadence)."); await registerPortalGdprExportRetentionCron(dsgvoCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "portal GDPR export-retention scheduler registered (15m cadence)."); } } finally { await dsgvoCronQueue.close(); } } else { if (exportRetentionRequired) { throw new Error("Required GDPR export-retention schedulers cannot start without Redis."); } workerLogger.info({}, "dsgvo-grace-cron scheduler skipped: no Redis URL configured."); } } catch (error) { if (exportRetentionRequired || tenantHardDeleteRequired) { workerLogger.error({}, "Required GDPR/tenant-deletion scheduler registration failed; exiting."); process.exit(1); throw new Error("Worker startup aborted after required GDPR/tenant-deletion scheduler failure."); } workerLogger.error({ error }, "dsgvo-grace-cron scheduler registration failed; continuing."); } } // DONE outbox rows are useful for short replay/debug windows but // should not grow forever. Keep retention single-scheduled on the // general worker so every deployment has one bounded daily sweeper. if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const connection = parseRedisUrl(redisUrl); const outboxRetentionQueue = new BullMQQueue(queueNames.maintenance, { connection, defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS, }); try { await registerOutboxDoneRetentionCron(outboxRetentionQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "outbox-done-retention scheduler registered (24h cadence)."); } finally { await outboxRetentionQueue.close(); } } else { workerLogger.info({}, "outbox-done-retention scheduler skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "outbox-done-retention scheduler registration failed; continuing."); } } // Workflow executions are inserted as `running` before their outbox // event is dispatched. If dispatch fails permanently, this singleton // sweep repairs only untouched manual/API executions with a matching // FAILED outbox row so they do not remain running forever. if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const connection = parseRedisUrl(redisUrl); const workflowStalledReconcileQueue = new BullMQQueue(queueNames.maintenance, { connection, defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS, }); try { await registerWorkflowExecutionStalledReconcilerCron(workflowStalledReconcileQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "workflow-execution-stalled-reconcile scheduler registered (5m cadence)."); } finally { await workflowStalledReconcileQueue.close(); } } else { workerLogger.info({}, "workflow-execution-stalled-reconcile scheduler skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "workflow-execution-stalled-reconcile scheduler registration failed; continuing."); } } // P4 Tasks (AK-P4b) — 30-day auto-archive (DoD 5) + hourly reminder sweep // (DoD 4). Single-instance schedulers on the `general` group (same // constraint as the other maintenance crons: must not multiply across // replicas). Both are no-ops without Redis configured. if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const connection = parseRedisUrl(redisUrl); const taskCronQueue = new BullMQQueue(queueNames.maintenance, { connection, defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS, }); try { await registerTaskArchiveRetentionCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "task-archive-retention scheduler registered (24h cadence)."); await registerLeadRetentionCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "lead-retention-sweep scheduler registered (24h cadence)."); await registerProjectAutoArchiveCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "project-auto-archive scheduler registered (24h cadence)."); await registerBillingDunningSweepCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "billing-dunning-sweep scheduler registered (24h cadence)."); await registerTaskReminderSweepCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "task-reminder-sweep scheduler registered (1h cadence)."); await registerAutomationSchedulerCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "automation-scheduler-sweep scheduler registered (1m cadence)."); await registerAdminSalesTaskReminderSweepCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "admin-sales-task-reminder-sweep scheduler registered (15m cadence)."); await registerAdminBookingReminderSweepCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "admin-booking-reminder-sweep scheduler registered (1m cadence)."); await registerAdminCalendarSyncCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "admin-calendar-sync scheduler registered (5m cadence)."); await registerAdminEmailDeliveryCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "admin-email-delivery scheduler registered (1m cadence)."); await registerAdminEmailEventRetentionCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "admin-email-event-retention scheduler registered (24h cadence)."); await registerAdminEmailSyncCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "admin-email-sync scheduler registered (2m cadence)."); await registerAdminEmailDomainCron(taskCronQueue, process.env); workerLogger.info({ queue: queueNames.maintenance }, "admin-email-domain-check scheduler registered (15m cadence)."); await registerAdminCallRecordingCrons(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "admin-call-recording schedulers registered (1m process, 1h retention)."); await registerAdminOutboundCallCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "admin-outbound-call scheduler registered (1m cadence)."); await registerIntegrationsTokenRefreshCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "integrations-token-refresh scheduler registered (1m cadence)."); await registerIntegrationsHealthSweepCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "integrations-health-sweep scheduler registered (15m cadence)."); await registerKwpProjectMaintenanceCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "kwp-project-maintenance scheduler registered (15m cadence)."); await registerCalendarTokenRefreshCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "calendar-token-refresh scheduler registered (5m cadence)."); await registerCalendarWebhookRenewalCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-renewal scheduler registered (6h cadence)."); await registerCalendarWebhookDrainCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-drain scheduler registered (1m cadence)."); await registerCalendarInternalWriteSyncCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "calendar-internal-write-sync scheduler registered (5m cadence)."); await registerEmailInboxTokenRefreshCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-token-refresh scheduler registered (5m cadence)."); await registerEmailInboxWebhookRenewalCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-renewal scheduler registered (6h cadence)."); await registerEmailInboxWebhookDrainCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-drain scheduler registered (1m cadence)."); await registerEmailInboxInitialSyncCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-initial-sync scheduler registered (2m cadence)."); await registerEmailInboxStalePollCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-stale-poll scheduler registered (30m cadence)."); await registerEmailInboxEventCleanupCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-event-cleanup scheduler registered (24h cadence)."); await registerHrUploadReminderDispatchCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "hr-upload-reminder-dispatch scheduler registered (24h cadence)."); await registerEmailDomainProbeCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "email-domain-probe scheduler registered (5m cadence)."); await registerTelephonyWebhookMaintenanceCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "telephony-webhook-maintenance scheduler registered (5m cadence)."); if (resolveProductAcquisitionProviderSideEffectMode(process.env) === "disabled") { await taskCronQueue.removeJobScheduler(PRODUCT_ACQUISITION_RECONCILE_JOB_ID); workerLogger.info({ queue: queueNames.maintenance }, "product-acquisition-reconcile scheduler removed while provider side effects are disabled."); } else { await registerProductAcquisitionReconcilerCron(taskCronQueue); workerLogger.info({ queue: queueNames.maintenance }, "product-acquisition-reconcile scheduler registered (5m cadence)."); } } finally { await taskCronQueue.close(); } } else { workerLogger.info({}, "task maintenance schedulers skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "task maintenance scheduler registration failed; continuing."); } } // Phase 06 #23 — PluginCapabilityUse audit-retention cron. Nightly // at 03:00 UTC (cron pattern overridable via PLUGIN_AUDIT_RETENTION_CRON). // Single-instance scheduler + worker on the `general` group, on a // dedicated queue (PLUGIN_AUDIT_RETENTION_QUEUE) so we don't have to // extend the maintenance CleanupJob discriminator. The worker is // attached to the SAME BullMQ Queue instance so the scheduler-fired // job lands here and gets processed. let pluginAuditRetentionWorker: import("bullmq").Worker | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const connection = parseRedisUrl(redisUrl); const pluginAuditCronQueue = new BullMQQueue(PLUGIN_AUDIT_RETENTION_QUEUE, { connection, defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS, }); try { const { pattern } = await registerPluginAuditRetentionCron(pluginAuditCronQueue, process.env); workerLogger.info({ queue: PLUGIN_AUDIT_RETENTION_QUEUE, pattern }, "plugin-audit-retention scheduler registered."); } finally { await pluginAuditCronQueue.close(); } const retentionConfig = resolvePluginAuditRetentionConfig(process.env); pluginAuditRetentionWorker = new BullMQWorker( PLUGIN_AUDIT_RETENTION_QUEUE, async () => { const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient() as unknown as { pluginCapabilityUse: { deleteMany: (args: { where: { id: { in: string[] } } }) => Promise<{ count: number }>; findMany: (args: { where: { timestamp: { lt: Date } }; select: { id: true }; take: number; orderBy: { timestamp: "asc" }; }) => Promise>; count: () => Promise; }; }; return runPluginAuditRetention({ config: retentionConfig, logger: workerLogger, deleteBatch: async (cutoff, batchSize) => { // Postgres does not support LIMIT on DELETE directly; pick // a bounded id-set first, then deleteMany by id. const candidates = await prisma.pluginCapabilityUse.findMany({ where: { timestamp: { lt: cutoff } }, select: { id: true }, take: batchSize, orderBy: { timestamp: "asc" }, }); if (candidates.length === 0) return 0; const ids = candidates.map((row) => row.id); const { count } = await prisma.pluginCapabilityUse.deleteMany({ where: { id: { in: ids } }, }); return count; }, // F-020 — surviving-rows count fed to the size-warning // step. Soft-fails inside runPluginAuditRetention if the // count() call throws, so a flaky DB doesn't poison the // retention result. countRows: () => prisma.pluginCapabilityUse.count(), }); }, { connection: parseRedisUrl(redisUrl), concurrency: 1 } ); workerLogger.info({ queue: PLUGIN_AUDIT_RETENTION_QUEUE, retentionDays: retentionConfig.retentionDays }, "plugin-audit-retention worker started."); } else { workerLogger.info({}, "plugin-audit-retention scheduler skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "plugin-audit-retention scheduler registration failed; continuing."); } } // AK-P3 AP-2 — AuditEvent retention cron. Nightly at 03:30 UTC // (AUDIT_RETENTION_CRON overridable). Single-instance scheduler + worker on // the `general` group, dedicated queue (AUDIT_RETENTION_QUEUE). Mirrors the // plugin-audit-retention pattern. The policy is fixed at ten years; every // bounded delete batch persists an authenticated chain checkpoint first. let auditRetentionWorker: import("bullmq").Worker | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const connection = parseRedisUrl(redisUrl); const auditRetentionQueue = new BullMQQueue(AUDIT_RETENTION_QUEUE, { connection, defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS, }); try { const { pattern } = await registerAuditRetentionCron(auditRetentionQueue, process.env); workerLogger.info({ queue: AUDIT_RETENTION_QUEUE, pattern }, "audit-retention scheduler registered."); } finally { await auditRetentionQueue.close(); } auditRetentionWorker = new BullMQWorker( AUDIT_RETENTION_QUEUE, async () => { const { getDbClient } = await import("@hivegrid/db/client.server"); const { createAuditEvent } = await import("@hivegrid/db/audit-events.server"); const { pruneAuditChainBatch } = await import("@hivegrid/db/audit-chain-checkpoint.server"); const prisma = getDbClient() as unknown as { auditEvent: { groupBy: (args: Record) => Promise>; }; }; return runAuditRetentionJob({ logger: workerLogger, // One aggregate query discovers all expired chains, including // the platform chain (tenantId=null); there is no per-tenant // regulatory-scope query fan-out. chainLister: async (threshold: Date): Promise> => { const rows = await prisma.auditEvent.groupBy({ by: ["tenantId"], where: { createdAt: { lt: threshold } }, }); return rows.map((row) => row.tenantId); }, batchPruner: (tenantId, threshold, batchSize) => pruneAuditChainBatch(tenantId, threshold, { batchSize }), // The cleanup is itself audited as a system action (traceable). auditWriter: async ({ tenantId, deletedCount, threshold, retentionYears }) => { await createAuditEvent({ tenantId, actorType: "system", action: "audit.retention.pruned", targetType: "AuditEvent", sourceModule: "audit", dataSensitivity: "internal", metadata: { deletedCount, threshold: threshold.toISOString(), retentionYears }, }); }, }); }, { connection: parseRedisUrl(redisUrl), concurrency: 1 } ); workerLogger.info({ queue: AUDIT_RETENTION_QUEUE }, "audit-retention worker started."); } else { workerLogger.info({}, "audit-retention scheduler skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "audit-retention scheduler registration failed; continuing."); } } let searchIndexer: SearchIndexerBootstrapResult | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; const { getDbClient } = await import("@hivegrid/db/client.server"); const { createSearchIndexerBulkPagers } = await import("./jobs/search-indexer-bulk-pagers.js"); const prisma = getDbClient(); searchIndexer = await bootstrapSearchIndexer(process.env, { logger: workerLogger, bulkPagers: createSearchIndexerBulkPagers({ prisma }), createWorkerImpl: redisUrl ? (queueName, processor, options) => { const connection = parseRedisUrl(redisUrl); const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as SearchIndexerWorkerHandle; } : undefined, }); } catch (error) { workerLogger.error({ error }, "Search-indexer bootstrap failed; stopping general worker."); throw error; } } let formSubmissionRoute: FormSubmissionRouteBootstrapResult | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient() as unknown as { formSubmission: { findUnique: (args: { where: { id: string }; select: { status: true; tenantId: true } }) => Promise<{ status: string; tenantId: string } | null>; updateMany: (args: { where: { id: string; tenantId: string }; data: Record }) => Promise<{ count: number }>; }; formAction: { findMany: (args: Record) => Promise< Array<{ id: string; actionType: string; name: string; fieldMapping: unknown; actionConfig: unknown; }> >; }; formActionLog: { create: (args: Record) => Promise<{ id: string; attempt: number }>; update: (args: Record) => Promise; }; inboxLead: { findFirst: (args: Record) => Promise<{ id: string } | null>; create: (args: Record) => Promise<{ id: string }>; }; kontakt: { findFirst: (args: Record) => Promise<{ id: string } | null>; create: (args: Record) => Promise<{ id: string }>; }; kontaktMarketingConsent: { upsert: (args: Record) => Promise; }; pipeline: { findFirst: (args: Record) => Promise<{ id: string } | null> }; pipelineStage: { findFirst: (args: Record) => Promise<{ id: string } | null> }; projekt: { create: (args: Record) => Promise<{ id: string }> }; task: { create: (args: Record) => Promise<{ id: string }> }; $transaction: ( callback: ( tx: Parameters[0] & { projekt: { create: (args: Record) => Promise<{ id: string }> }; } ) => Promise ) => Promise; }; const [{ publishOutboxEvent }, { planFormSubmissionFollowUps }, { buildWorkflowTriggerOutboxEvent }, { applyFieldMapping }] = await Promise.all([ import("@hivegrid/db/outbox"), import("./jobs/form-submission-followups.js"), import("@hivegrid/queue"), import("@hivegrid/form-widget"), ]); const connection = parseRedisUrl(redisUrl); function stringValue(values: Record, keys: readonly string[]): string | null { for (const key of keys) { const value = values[key]; if (typeof value === "string" && value.trim()) return value.trim(); } return null; } function objectValue(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; } formSubmissionRoute = bootstrapFormSubmissionRoute(process.env, { logger: workerLogger, handlerDeps: { store: { load: (submissionId) => prisma.formSubmission.findUnique({ where: { id: submissionId }, select: { status: true, tenantId: true }, }), markRouted: async (input) => { const { count } = await prisma.formSubmission.updateMany({ where: { id: input.submissionId, tenantId: input.tenantId }, data: { status: input.status, result: input.result }, }); if (count !== 1) { throw new Error("form-submission-route could not update the tenant-scoped submission row"); } }, }, emitFollowUps: async (followUp) => { const events = planFormSubmissionFollowUps(followUp); for (const event of events) { await publishOutboxEvent(prisma as never, { tenantId: event.tenantId, type: event.type, payload: event.payload as Prisma.InputJsonValue, dispatchKey: event.dispatchKey, }); } }, actionRunner: async ({ submissionId, formId, tenantId, values }) => { const actions = await prisma.formAction.findMany({ where: { formId, tenantId, enabled: true }, orderBy: { position: "asc" }, select: { id: true, actionType: true, name: true, fieldMapping: true, actionConfig: true }, }); for (const action of actions) { const mapped = applyFieldMapping({ payload: values, fieldMapping: objectValue(action.fieldMapping) as Record, }); const idempotencyKey = `form-action:${submissionId}:${action.id}`; const log = await prisma.formActionLog.create({ data: { formActionId: action.id, submissionId, formId, tenantId, actionType: action.actionType, status: "pending", inputSnapshot: { values: mapped }, idempotencyKey, }, select: { id: true, attempt: true }, }); try { const config = objectValue(action.actionConfig); let target: { type: string; id: string | null } = { type: action.actionType, id: null }; if (action.actionType === "auto_match") { const email = stringValue(mapped, ["email", "primaerEmail", "email_address"]); const phone = stringValue(mapped, ["phone", "telefon", "primaerTelefon"]); const contact = email || phone ? await prisma.kontakt.findFirst({ where: { tenantId, archivedAt: null, OR: [...(email ? [{ primaerEmail: email.toLowerCase() }] : []), ...(phone ? [{ primaerTelefon: phone }] : [])], }, select: { id: true }, }) : null; const lead = !contact && (email || phone) ? await prisma.inboxLead.findFirst({ where: { tenantId, archivedAt: null, OR: [...(email ? [{ email: email.toLowerCase() }] : []), ...(phone ? [{ telefon: phone }] : [])], }, select: { id: true }, }) : null; target = contact ? { type: "contact", id: contact.id } : lead ? { type: "lead", id: lead.id } : { type: "none", id: null }; if (target.id) { await prisma.formSubmission.updateMany({ where: { id: submissionId, tenantId }, data: { linkedEntityType: target.type, linkedEntityId: target.id }, }); } } else if (action.actionType === "create_lead") { const lead = await prisma.inboxLead.create({ data: { tenantId, createdByUserId: typeof config.userId === "string" ? config.userId : null, quelle: "form", quelleDetail: { formId, submissionId }, sourceOrigin: `/forms/${formId}`, vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]), nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]), email: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null, telefon: stringValue(mapped, ["telefon", "phone"]), message: stringValue(mapped, ["message", "nachricht", "details"]), rohdaten: { formId, submissionId, values: mapped }, weiterverarbeitungFreigabe: "ja", }, select: { id: true }, }); target = { type: "lead", id: lead.id }; } else if (action.actionType === "create_contact") { const contact = await prisma.kontakt.create({ data: { tenantId, createdByUserId: typeof config.userId === "string" ? config.userId : "system", kontaktTyp: "privatperson", vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]), nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]) ?? stringValue(mapped, ["name"]) ?? "Formular-Kontakt", primaerEmail: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null, primaerTelefon: stringValue(mapped, ["telefon", "phone"]), }, select: { id: true }, }); target = { type: "contact", id: contact.id }; } else if (action.actionType === "create_project") { const pipeline = await prisma.pipeline.findFirst({ where: { tenantId, archivedAt: null }, orderBy: { position: "asc" }, select: { id: true }, }); const stage = pipeline ? await prisma.pipelineStage.findFirst({ where: { tenantId, pipelineId: pipeline.id, archivedAt: null }, orderBy: { position: "asc" }, select: { id: true }, }) : null; if (!pipeline || !stage) throw new Error("No project pipeline/stage available for form action"); const actorUserId = typeof config.userId === "string" ? config.userId : null; const project = await prisma.$transaction(async (tx) => { const created = await tx.projekt.create({ data: { tenantId, createdByUserId: actorUserId, projektNummer: `FORM-${Date.now()}`, projektname: stringValue(mapped, ["projektname", "projectName", "name"]) ?? "Projekt aus Formular", beschreibung: stringValue(mapped, ["message", "description", "details"]), quelle: "form", pipelineId: pipeline.id, stageId: stage.id, }, select: { id: true }, }); await ensureAndSyncProjectChannelIfEnabledWithTx(tx, { tenantId, projectId: created.id, actorUserId, requestId: `form:${formId}:submission:${submissionId}:project:${created.id}:chat-created`, }); return created; }); target = { type: "project", id: project.id }; } else if (action.actionType === "create_task") { const task = await prisma.task.create({ data: { tenantId, createdByUserId: typeof config.userId === "string" ? config.userId : "system", title: stringValue(mapped, ["title", "taskTitle", "subject"]) ?? "Aufgabe aus Formular", description: stringValue(mapped, ["message", "description", "details"]), assignedToUserId: typeof config.assignedToUserId === "string" ? config.assignedToUserId : typeof config.userId === "string" ? config.userId : "system", source: "form_submission", sourceRef: submissionId, }, select: { id: true }, }); target = { type: "task", id: task.id }; } else if (action.actionType === "webhook") { const webhookUrl = typeof config.webhookUrl === "string" ? config.webhookUrl : null; if (!webhookUrl) throw new Error("Webhook action missing webhookUrl"); const webhookPayload = { submissionId, formId, tenantId, webhookUrl, values: mapped } as unknown as Prisma.InputJsonValue; await publishOutboxEvent(prisma as never, { tenantId, type: "webhook.form-submission.deliver", payload: webhookPayload, dispatchKey: `form-action-webhook:${submissionId}:${action.id}`, }); } else if (action.actionType === "send_email_to_team" || action.actionType === "send_email_to_submitter") { const recipients = action.actionType === "send_email_to_submitter" ? [stringValue(mapped, ["email", "email_address"])].filter((email): email is string => Boolean(email)) : Array.isArray(config.recipients) ? config.recipients.filter((email): email is string => typeof email === "string" && email.includes("@")) : []; if (recipients.length === 0) { await prisma.formActionLog.update({ where: { id: log.id }, data: { status: "skipped", outputSnapshot: { reason: "no_recipients" }, finishedAt: new Date() }, }); continue; } const mailPayload = { submissionId, formId, tenantId, recipients, values: mapped } as unknown as Prisma.InputJsonValue; await publishOutboxEvent(prisma as never, { tenantId, type: "mail.transactional.form-submission", payload: mailPayload, dispatchKey: `form-action-mail:${submissionId}:${action.id}`, }); } else if (action.actionType === "trigger_automation") { const automationData = { formId, submissionId, actionId: action.id, values: mapped } as unknown as QueueJsonValue; await publishOutboxEvent( prisma as never, buildWorkflowTriggerOutboxEvent({ tenantId, event: "form.submitted", data: automationData, userId: typeof config.userId === "string" ? config.userId : null, sourceId: `form:${formId}:submission:${submissionId}:action:${action.id}`, }) ); } else if (action.actionType === "start_double_opt_in") { const result = await runStartDoubleOptInFormAction({ prisma: prisma as unknown as StartDoubleOptInPrisma, publishOutboxEvent: (event) => publishOutboxEvent(prisma as never, { tenantId: event.tenantId, type: event.type, payload: event.payload as Prisma.InputJsonValue, dispatchKey: event.dispatchKey, }), tenantId, formId, submissionId, actionId: action.id, mapped, config, }); if (result.status === "skipped") { await prisma.formActionLog.update({ where: { id: log.id }, data: { status: "skipped", outputSnapshot: result, finishedAt: new Date() }, }); continue; } target = { type: "marketing_consent", id: result.kontaktId }; } else { await prisma.formActionLog.update({ where: { id: log.id }, data: { status: "skipped", outputSnapshot: { reason: "unsupported_action" }, finishedAt: new Date() }, }); continue; } await prisma.formActionLog.update({ where: { id: log.id }, data: { status: "success", targetType: target.type, targetId: target.id, outputSnapshot: { target }, finishedAt: new Date(), }, }); } catch (error) { await prisma.formActionLog.update({ where: { id: log.id }, data: { status: "failed", errorMessage: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500), finishedAt: new Date(), }, }); } } }, logger: workerLogger, }, createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as FormSubmissionRouteWorkerHandle; }, }); } else { formSubmissionRoute = { status: "disabled", reason: "no Redis URL configured" }; workerLogger.info({}, "form-submission-route worker skipped: no Redis URL configured."); } } catch (error) { formSubmissionRoute = { status: "disabled", reason: "bootstrap failed" }; workerLogger.error({ error }, "Form-submission-route bootstrap failed; continuing without subscriber."); } } let formSubmissionWebhookDelivery: FormSubmissionWebhookDeliveryBootstrapResult | undefined; let formSubmissionWebhookQueue: BullMQQueue | undefined; let transactionalMailOutbox: TransactionalMailOutboxBootstrapResult | undefined; let portalStatusMail: PortalStatusMailBootstrapResult | undefined; let emailSequences: EmailSequenceBootstrapResult | undefined; if (groupResolution?.group === "mail") { const transactionalMailOutboxRequired = isTransactionalMailOutboxRequired(process.env, groupResolution.group); try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const connection = parseRedisUrl(redisUrl); const { getDbClient } = await import("@hivegrid/db/client.server"); const campaignCancellationPrisma = getDbClient() as unknown as CampaignScheduleCancellationPrisma; formSubmissionWebhookQueue = new BullMQQueue(queueNames.email, { connection }); formSubmissionWebhookDelivery = bootstrapFormSubmissionWebhookDelivery(process.env, { logger: workerLogger, handlerDeps: { enqueueWebhook: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options), }, createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as FormSubmissionWebhookDeliveryWorkerHandle; }, }); try { transactionalMailOutbox = bootstrapTransactionalMailOutbox(process.env, { logger: workerLogger, handlerDeps: { enqueueMail: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options), cancelCampaignSchedule: (input) => cancelCampaignScheduleWithPrisma(campaignCancellationPrisma, input), isCampaignDeliveryAllowed: (input) => isCampaignDeliveryAllowedWithPrisma(campaignCancellationPrisma, input), }, createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as TransactionalMailOutboxWorkerHandle; }, }); transactionalMailOutboxStatus = transactionalMailOutbox; if (transactionalMailOutbox.status === "disabled") { if (transactionalMailOutboxRequired) { workerLogger.error({ reason: transactionalMailOutbox.reason }, "Transactional mail outbox disabled in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required transactional mail outbox disabled."); } workerLogger.warn({ reason: transactionalMailOutbox.reason }, "transactional mail outbox bridge disabled."); } } catch (error) { transactionalMailOutboxStatus = { status: "disabled", reason: "bootstrap failed" }; if (transactionalMailOutboxRequired) { workerLogger.error({ error }, "Transactional mail outbox bootstrap failed in required mode; exiting."); process.exit(1); throw new Error("Worker startup aborted after required transactional mail outbox failure."); } workerLogger.error({ error }, "Transactional mail outbox bootstrap failed; continuing without subscriber."); } try { const { getDbClient } = await import("@hivegrid/db/client.server"); const store = createPrismaPortalStatusMailStore(getDbClient() as unknown as PortalStatusMailPrisma, { portalBaseUrl: resolvePortalStatusMailBaseUrl(process.env), }); portalStatusMail = bootstrapPortalStatusMail(process.env, { logger: workerLogger, handlerDeps: { ...store, enqueueMail: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options), logger: workerLogger, }, createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as PortalStatusMailWorkerHandle; }, }); portalStatusMailStatus = portalStatusMail; if (portalStatusMail.status === "disabled") { workerLogger.error({ reason: portalStatusMail.reason }, "portal status mail worker disabled; queued deliveries will wait for recovery."); } } catch (error) { portalStatusMail = { status: "disabled", reason: "bootstrap failed" }; portalStatusMailStatus = portalStatusMail; workerLogger.error({ error }, "Portal status mail bootstrap failed; queued deliveries will wait for recovery."); } } else { formSubmissionWebhookDelivery = { status: "disabled", reason: "no Redis URL configured" }; transactionalMailOutbox = { status: "disabled", reason: "no Redis URL configured" }; portalStatusMail = { status: "disabled", reason: "no Redis URL configured" }; portalStatusMailStatus = portalStatusMail; transactionalMailOutboxStatus = transactionalMailOutbox; if (transactionalMailOutboxRequired) { workerLogger.error({}, "Transactional mail outbox required but Redis URL is not configured; exiting."); process.exit(1); throw new Error("Worker startup aborted after required transactional mail outbox missing Redis."); } workerLogger.info({}, "form-submission webhook bridge skipped: no Redis URL configured."); workerLogger.info({}, "transactional mail outbox bridge skipped: no Redis URL configured."); workerLogger.info({}, "portal status mail worker skipped: no Redis URL configured."); } const emailSequenceRedisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (!emailSequenceRedisUrl && process.env.MARKETING_SEQUENCES_ENABLED === "true") { emailSequences = { status: "disabled", reason: "no Redis URL configured" }; workerLogger.error({}, "Email-sequence workers enabled but Redis URL is not configured."); } else { const [{ runSequenceTriggerJob }, { runEmailSequenceStepJob }] = await Promise.all([ import("./jobs/email-sequence-trigger.js"), import("./jobs/email-sequence-scheduler.js"), ]); // Production deps (Prisma store + BullMQ scheduleStep producer + // template renderer + @hivegrid/mail Mailjet adapter + // unsubscribe-url builder) are loaded lazily inside the // processors so startup-time optional config issues do not // crash the mail worker. Missing Redis/Mailjet config still // fails closed at job time and is surfaced through worker logs. emailSequences = bootstrapEmailSequenceWorkers(process.env, { logger: workerLogger, triggerProcessor: async (job) => { const { createEmailSequenceTriggerDeps } = await import("./jobs/email-sequence-runtime.js"); const deps = await createEmailSequenceTriggerDeps({ logger: workerLogger }); if (!deps) return { matched: 0, started: 0, skipped: 0 }; return runSequenceTriggerJob(job.data, deps); }, schedulerProcessor: async (job) => { const { createEmailSequenceSchedulerDeps } = await import("./jobs/email-sequence-runtime.js"); const deps = await createEmailSequenceSchedulerDeps({ logger: workerLogger }); if (!deps) return { action: "cancelled" as const, completed: true }; return runEmailSequenceStepJob(job.data, deps); }, createWorkerImpl: emailSequenceRedisUrl ? (queueName, processor, options) => { const connection = parseRedisUrl(emailSequenceRedisUrl); const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as EmailSequenceWorkerHandle; } : undefined, }); } emailSequencesStatus = emailSequences; } catch (error) { emailSequences = { status: "disabled", reason: "bootstrap failed" }; emailSequencesStatus = emailSequences; workerLogger.error({ error }, "Email-sequence bootstrap failed; continuing without sequence workers."); } } let auditEventPersist: AuditEventPersistBootstrapResult | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient(); auditEventPersist = bootstrapAuditEventPersist(process.env, { logger: workerLogger, runInTransaction: async (fn) => // Serializable isolation guarantees concurrent dispatcher // deliveries for the same tenant either serialize into a // coherent chain or one of them retries on conflict. ( prisma as unknown as { $transaction: (runner: (tx: Parameters[0]) => Promise, opts?: { isolationLevel?: string }) => Promise; } ).$transaction(async (tx) => fn(tx as Parameters[0]), { isolationLevel: "Serializable" }) as Promise>, createWorkerImpl: redisUrl ? (queueName, processor, options) => { const connection = parseRedisUrl(redisUrl); const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as AuditEventPersistWorkerHandle; } : undefined, }); } catch (error) { workerLogger.error({ error }, "Audit-event-persist bootstrap failed; continuing without subscriber."); } } let chatRealtimeWakeup: ChatRealtimeWakeupBootstrapResult | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { chatRealtimeWakeup = bootstrapChatRealtimeWakeup(process.env, { logger: workerLogger, baseStore: getSharedStore(), createWorkerImpl: (queueName, processor, options) => { const connection = parseRedisUrl(redisUrl); const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as ChatRealtimeWakeupWorkerHandle; }, }); } else { workerLogger.info({}, "chat realtime wakeup worker skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "Chat realtime wakeup bootstrap failed; continuing without subscriber."); } } if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { chatLinkPreviewStatus = bootstrapChatLinkPreview(process.env, { logger: workerLogger, baseStore: getSharedStore(), createWorkerImpl: (queueName, processor, options) => { const connection = parseRedisUrl(redisUrl); const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as ChatLinkPreviewWorkerHandle; }, }); } else { chatLinkPreviewStatus = { status: "disabled", reason: "no Redis URL configured" }; workerLogger.info({}, "chat link preview worker skipped: no Redis URL configured."); } } catch (error) { chatLinkPreviewStatus = { status: "disabled", reason: "bootstrap failed" }; workerLogger.error({ error }, "Chat link preview bootstrap failed; continuing without subscriber."); } } let pluginHookInvokeQueue: BullMQQueue | undefined; if (groupResolution?.group === "general") { const pluginHookFanoutRequired = isPluginHookFanoutRequired(process.env, groupResolution.group); try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient(); const connection = parseRedisUrl(redisUrl); pluginHookInvokeQueue = new BullMQQueue(queueNames.pluginInvoke, { connection }); pluginHookFanoutStatus = bootstrapPluginHookFanout(process.env, { logger: workerLogger, prisma, pluginInvokeQueue: pluginHookInvokeQueue as never, idempotency: createIdempotencyStore({ store: getSharedStore(), prefix: "plugin-hook-fanout", ttlSeconds: 60 * 60 * 24, }), createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as PluginHookFanoutWorkerHandle; }, }); if (pluginHookFanoutStatus.status === "disabled" && pluginHookFanoutRequired) { workerLogger.error({ reason: pluginHookFanoutStatus.reason }, "Plugin-hook fanout disabled in required mode; exiting."); process.exit(1); } } else if (pluginHookFanoutRequired) { workerLogger.error({}, "Plugin-hook fanout required but Redis URL is not configured; exiting."); process.exit(1); } else { workerLogger.info({}, "plugin-hook fanout skipped: no Redis URL configured."); } } catch (error) { if (pluginHookFanoutRequired) { workerLogger.error({ error }, "Plugin-hook fanout bootstrap failed in required mode; exiting."); process.exit(1); } else { workerLogger.error({ error }, "Plugin-hook fanout bootstrap failed; continuing without subscriber."); } } } let pluginCapabilityUse: PluginCapabilityUseBootstrapResult | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const { getDbClient } = await import("@hivegrid/db/client.server"); const prisma = getDbClient(); const connection = parseRedisUrl(redisUrl); pluginCapabilityUse = bootstrapPluginCapabilityUse(process.env, { logger: workerLogger, store: createPrismaPluginCapabilityUseStore(prisma as unknown as PluginCapabilityUseStorePrisma), createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as PluginCapabilityUseWorkerHandle; }, }); } else { workerLogger.info({}, "plugin capability-use worker skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "Plugin capability-use bootstrap failed; continuing without subscriber."); } } // Phase 03 #42 — Legacy-event outbox receiver. Subscribes the // `outbox.notification` queue on the `general` group and writes // billing.subscription.* events into the legacy Event table on // behalf of the migrated Mollie webhook (apps/api/src/billing/ // mollie-webhook-route.ts). Disabled when the group is non-general // or no Redis URL is configured. let legacyEventReceiver: LegacyEventReceiverBootstrapResult | undefined; let platformNotificationQueue: TypedQueue | undefined; if (groupResolution?.group === "general") { try { const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL; if (redisUrl) { const { createLegacyEvent } = await import("@hivegrid/db/events.server"); const connection = parseRedisUrl(redisUrl); platformNotificationQueue = createQueue(queueNames.notifications, { connection }); const queue = platformNotificationQueue; legacyEventReceiver = bootstrapLegacyEventReceiver(process.env, { logger: workerLogger, store: { create: async (data) => { const row = await createLegacyEvent(data); return { id: row.id }; }, }, platformNotification: { enqueue: async (input, options) => { await enqueueJob(queue, buildPlatformNotificationJob(input), options); }, logger: workerLogger, }, createWorkerImpl: (queueName, processor, options) => { const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), { connection, concurrency: options.concurrency, }); return worker as unknown as LegacyEventReceiverWorkerHandle; }, }); } else { workerLogger.info({}, "legacy-event receiver skipped: no Redis URL configured."); } } catch (error) { workerLogger.error({ error }, "Legacy-event receiver bootstrap failed; continuing without subscriber."); } } const shutdownState: ShutdownState = { adminServer, healthServer, redisHealthProbe, outbox, adminBookingOutbox, workflowTrigger, searchIndexer, formSubmissionRoute, formSubmissionWebhookDelivery, formSubmissionWebhookQueue, transactionalMailOutbox, portalStatusMail, portalDomainOrchestration, productAcquisitionProviderSideEffect, recurringInvoiceCollection, emailSequences, auditEventPersist, chatRealtimeWakeup, chatLinkPreview: chatLinkPreviewStatus, legacyEventReceiver, platformNotificationQueue, pluginHookFanout: pluginHookFanoutStatus, pluginCapabilityUse, pluginHookInvokeQueue, pluginAuditRetentionWorker, auditRetentionWorker, filePreviewQueue, telephonyWebhookQueue, tenantDeletionExportQueue, telephonyWebhookWorker, automationExecutorWorker, }; process.on("SIGINT", () => { void shutdown(runtime, shutdownState, "SIGINT"); }); process.on("SIGTERM", () => { void shutdown(runtime, shutdownState, "SIGTERM"); }); } } catch (error) { workerLogger.error({ error }, "Worker startup failed; see worker health diagnostics for dependency details."); process.exit(1); } function isRequiredTenantHardDeleteRuntime(env: NodeJS.ProcessEnv): boolean { return ( env.WORKER_QUEUE_GROUP === "general" && (env.NODE_ENV === "production" || env.RLS_RUNTIME_MODE === "required" || env.SENTRY_ENVIRONMENT === "staging" || env.SENTRY_ENVIRONMENT === "production") ); } function shutdown(runtimeToClose: WorkerRuntimeStatus, state: ShutdownState, signal: NodeJS.Signals): Promise { state.signal ??= signal; state.promise ??= closeRuntime( runtimeToClose, state.signal, state.adminServer, state.healthServer, state.redisHealthProbe, state.outbox, state.adminBookingOutbox, state.workflowTrigger, state.searchIndexer, state.formSubmissionRoute, state.formSubmissionWebhookDelivery, state.formSubmissionWebhookQueue, state.transactionalMailOutbox, state.portalStatusMail, state.portalDomainOrchestration, state.productAcquisitionProviderSideEffect, state.recurringInvoiceCollection, state.emailSequences, state.auditEventPersist, state.chatRealtimeWakeup, state.chatLinkPreview, state.legacyEventReceiver, state.platformNotificationQueue, state.pluginHookFanout, state.pluginCapabilityUse, state.pluginHookInvokeQueue, state.pluginAuditRetentionWorker, state.auditRetentionWorker, state.filePreviewQueue, state.telephonyWebhookQueue, state.tenantDeletionExportQueue, state.telephonyWebhookWorker, state.automationExecutorWorker ); return state.promise; } async function closeRuntime( runtimeToClose: WorkerRuntimeStatus, signal: NodeJS.Signals, adminServer?: WorkerAdminServerHandle, healthServer?: WorkerHealthServerHandle, redisHealthProbe?: RedisHealthProbe, outbox?: OutboxDispatcherBootstrapResult, adminBookingOutbox?: AdminBookingOutboxBootstrapResult, workflowTrigger?: WorkflowTriggerBootstrapResult, searchIndexer?: SearchIndexerBootstrapResult, formSubmissionRoute?: FormSubmissionRouteBootstrapResult, formSubmissionWebhookDelivery?: FormSubmissionWebhookDeliveryBootstrapResult, formSubmissionWebhookQueue?: BullMQQueue, transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult, portalStatusMail?: PortalStatusMailBootstrapResult, portalDomainOrchestration?: PortalDomainOrchestrationBootstrapResult, productAcquisitionProviderSideEffect?: ProductAcquisitionProviderSideEffectBootstrapResult, recurringInvoiceCollection?: RecurringInvoiceCollectionBootstrapResult, emailSequences?: EmailSequenceBootstrapResult, auditEventPersist?: AuditEventPersistBootstrapResult, chatRealtimeWakeup?: ChatRealtimeWakeupBootstrapResult, chatLinkPreview?: ChatLinkPreviewBootstrapResult, legacyEventReceiver?: LegacyEventReceiverBootstrapResult, platformNotificationQueue?: TypedQueue, pluginHookFanout?: PluginHookFanoutBootstrapResult, pluginCapabilityUse?: PluginCapabilityUseBootstrapResult, pluginHookInvokeQueue?: BullMQQueue, pluginAuditRetentionWorker?: BullMQWorker, auditRetentionWorker?: BullMQWorker, filePreviewQueue?: TypedQueue, telephonyWebhookQueue?: BullMQQueue, tenantDeletionExportQueue?: BullMQQueue, telephonyWebhookWorker?: BullMQWorker, automationExecutorWorker?: BullMQWorker ): Promise { try { if (outbox?.handle) { try { await outbox.handle.close(); } catch (outboxError) { workerLogger.error({ error: outboxError, signal }, "Outbox dispatcher shutdown failed."); } } if (adminBookingOutbox?.status === "started") { try { await adminBookingOutbox.handle.close(); } catch (bookingError) { workerLogger.error({ error: bookingError, signal }, "Admin booking provider worker shutdown failed."); } } if (workflowTrigger?.status === "started") { try { await workflowTrigger.handle.close(); } catch (workflowTriggerError) { workerLogger.error({ error: workflowTriggerError, signal }, "Workflow-trigger worker shutdown failed."); } } if (searchIndexer?.status === "started") { try { await searchIndexer.handle.close(); } catch (searchError) { workerLogger.error({ error: searchError, signal }, "Search-indexer shutdown failed."); } } if (formSubmissionRoute?.status === "started") { try { await formSubmissionRoute.handle.close(); } catch (formSubmissionError) { workerLogger.error({ error: formSubmissionError, signal }, "Form-submission-route worker shutdown failed."); } } if (formSubmissionWebhookDelivery?.status === "started") { try { await formSubmissionWebhookDelivery.handle.close(); } catch (formSubmissionWebhookError) { workerLogger.error({ error: formSubmissionWebhookError, signal }, "Form-submission webhook bridge shutdown failed."); } } if (transactionalMailOutbox?.status === "started") { try { await transactionalMailOutbox.handle.close(); } catch (transactionalMailOutboxError) { workerLogger.error({ error: transactionalMailOutboxError, signal }, "Transactional mail outbox bridge shutdown failed."); } } if (portalStatusMail?.status === "started") { try { await portalStatusMail.handle.close(); } catch (portalStatusMailError) { workerLogger.error({ error: portalStatusMailError, signal }, "Portal status mail worker shutdown failed."); } } if (portalDomainOrchestration?.status === "started") { try { await portalDomainOrchestration.close(); } catch (portalDomainError) { workerLogger.error({ error: portalDomainError, signal }, "Portal-domain orchestration shutdown failed."); } } if (productAcquisitionProviderSideEffect?.status === "started") { try { await productAcquisitionProviderSideEffect.handle.close(); } catch (providerSideEffectError) { workerLogger.error({ error: providerSideEffectError, signal }, "Product-acquisition provider side-effect worker shutdown failed."); } } if (recurringInvoiceCollection?.status === "started") { try { await recurringInvoiceCollection.handle.close(); } catch (recurringInvoiceError) { workerLogger.error({ error: recurringInvoiceError, signal }, "Recurring invoice collection worker shutdown failed."); } } if (formSubmissionWebhookQueue) { try { await formSubmissionWebhookQueue.close(); } catch (formSubmissionWebhookQueueError) { workerLogger.error({ error: formSubmissionWebhookQueueError, signal }, "Form-submission webhook queue shutdown failed."); } } if (emailSequences?.status === "started") { try { await emailSequences.triggerHandle.close(); await emailSequences.schedulerHandle.close(); } catch (seqError) { workerLogger.error({ error: seqError, signal }, "Email-sequence workers shutdown failed."); } } if (auditEventPersist?.status === "started") { try { await auditEventPersist.handle.close(); } catch (auditError) { workerLogger.error({ error: auditError, signal }, "Audit-event-persist shutdown failed."); } } if (chatRealtimeWakeup?.status === "started") { try { await chatRealtimeWakeup.handle.close(); } catch (chatRealtimeError) { workerLogger.error({ error: chatRealtimeError, signal }, "Chat realtime wakeup worker shutdown failed."); } } if (chatLinkPreview?.status === "started") { try { await chatLinkPreview.handle.close(); } catch (chatLinkPreviewError) { workerLogger.error({ error: chatLinkPreviewError, signal }, "Chat link preview worker shutdown failed."); } } if (legacyEventReceiver?.status === "started") { try { await legacyEventReceiver.handle.close(); } catch (legacyError) { workerLogger.error({ error: legacyError, signal }, "Legacy-event receiver shutdown failed."); } } if (platformNotificationQueue) { try { await platformNotificationQueue.close(); } catch (platformNotificationQueueError) { workerLogger.error({ error: platformNotificationQueueError, signal }, "Platform notification queue shutdown failed."); } } if (pluginHookFanout?.status === "started") { try { await pluginHookFanout.handle.close(); } catch (pluginHookError) { workerLogger.error({ error: pluginHookError, signal }, "Plugin-hook fanout shutdown failed."); } } if (pluginCapabilityUse?.status === "started") { try { await pluginCapabilityUse.handle.close(); } catch (pluginCapabilityError) { workerLogger.error({ error: pluginCapabilityError, signal }, "Plugin capability-use worker shutdown failed."); } } if (pluginHookInvokeQueue) { try { await pluginHookInvokeQueue.close(); } catch (pluginHookQueueError) { workerLogger.error({ error: pluginHookQueueError, signal }, "Plugin-hook invoke queue shutdown failed."); } } if (pluginAuditRetentionWorker) { try { await pluginAuditRetentionWorker.close(); } catch (retentionError) { workerLogger.error({ error: retentionError, signal }, "plugin-audit-retention worker shutdown failed."); } } if (auditRetentionWorker) { try { await auditRetentionWorker.close(); } catch (retentionError) { workerLogger.error({ error: retentionError, signal }, "audit-retention worker shutdown failed."); } } if (filePreviewQueue) { try { await filePreviewQueue.close(); } catch (previewQueueError) { workerLogger.error({ error: previewQueueError, signal }, "File-preview producer queue shutdown failed."); } } if (telephonyWebhookQueue) { try { await telephonyWebhookQueue.close(); } catch (telephonyWebhookQueueError) { workerLogger.error({ error: telephonyWebhookQueueError, signal }, "Telephony webhook producer queue shutdown failed."); } } if (tenantDeletionExportQueue) { try { await tenantDeletionExportQueue.close(); } catch (tenantDeletionExportQueueError) { workerLogger.error({ error: tenantDeletionExportQueueError, signal }, "Tenant deletion export queue shutdown failed."); } } if (telephonyWebhookWorker) { try { await telephonyWebhookWorker.close(); } catch (telephonyWebhookError) { workerLogger.error({ error: telephonyWebhookError, signal }, "Telephony webhook worker shutdown failed."); } } if (automationExecutorWorker) { try { await automationExecutorWorker.close(); } catch (automationExecutorError) { workerLogger.error({ error: automationExecutorError, signal }, "Automation forward-executor worker shutdown failed."); } } await shutdownWorkerRuntime(runtimeToClose, { logger: workerLogger, reason: signal, reporter: workerErrorReporter }); if (adminServer) { await adminServer.close(); } if (healthServer) { await healthServer.close(); } if (redisHealthProbe) { await redisHealthProbe.close(); } try { await closeSharedStore(); } catch (sharedStoreError) { workerLogger.error({ error: sharedStoreError, signal }, "Worker shared-store shutdown failed."); } process.exit(0); } catch (error) { workerLogger.error({ error, signal }, "Worker shutdown failed."); process.exit(1); } }