Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/worker/src/index.ts
T

3070 lines
152 KiB
TypeScript

// IMPORTANT: `./otel.js` MUST stay the very first import in this file.
// The OpenTelemetry NodeSDK can only patch BullMQ, Prisma, Redis and
// friends when it loads before those modules are required for the first
// time. The module is a no-op when OTEL_EXPORTER_OTLP_ENDPOINT is unset.
import "./otel.js";
import { ensureAndSyncProjectChannelIfEnabledWithTx } from "@hivegrid/chat/project-sync.server";
import {
DEFAULT_WORKER_ADMIN_PORT,
isWorkerMetricsEnabled,
startWorkerAdminServer,
type WorkerAdminServerHandle,
type WorkerStatusReport,
} from "./admin-server.js";
import { createWorkerConnection } from "./connection.js";
import { bootstrapWorkerGroup, InvalidWorkerQueueGroupError } from "./group-runtime.js";
import { DEFAULT_WORKER_HEALTH_PORT, startWorkerHealthServer, type HealthProbe, type WorkerHealthServerHandle } from "./health-server.js";
import { bootstrapOutboxDispatcher, type OutboxDispatcherBootstrapResult } from "./outbox/bootstrap.js";
import { isOutboxDispatcherRequired } from "./outbox/dispatcher.js";
import { bootstrapSearchIndexer, type SearchIndexerBootstrapResult, type SearchIndexerWorkerHandle } from "./jobs/search-indexer-bootstrap.js";
import {
bootstrapFormSubmissionRoute,
type FormSubmissionRouteBootstrapResult,
type FormSubmissionRouteWorkerHandle,
} from "./jobs/form-submission-route-bootstrap.js";
import { runStartDoubleOptInFormAction, type StartDoubleOptInPrisma } from "./jobs/form-submission-actions.js";
import {
bootstrapFormSubmissionWebhookDelivery,
type FormSubmissionWebhookDeliveryBootstrapResult,
type FormSubmissionWebhookDeliveryWorkerHandle,
} from "./jobs/form-submission-webhook-delivery-bootstrap.js";
import {
bootstrapTransactionalMailOutbox,
isTransactionalMailOutboxRequired,
type TransactionalMailOutboxBootstrapResult,
type TransactionalMailOutboxWorkerHandle,
} from "./jobs/transactional-mail-outbox-bootstrap.js";
import {
cancelCampaignScheduleWithPrisma,
isCampaignDeliveryAllowedWithPrisma,
type CampaignScheduleCancellationPrisma,
} from "./jobs/campaign-schedule-cancellation.js";
import { bootstrapPortalStatusMail, type PortalStatusMailBootstrapResult, type PortalStatusMailWorkerHandle } from "./jobs/portal-status-mail-bootstrap.js";
import { bootstrapPortalDomainOrchestration, type PortalDomainOrchestrationBootstrapResult } from "./jobs/portal-domain-orchestration-bootstrap.js";
import {
bootstrapProductAcquisitionProviderSideEffect,
type ProductAcquisitionProviderSideEffectBootstrapResult,
} from "./jobs/product-acquisition-provider-side-effect-bootstrap.js";
import { bootstrapRecurringInvoiceCollection, type RecurringInvoiceCollectionBootstrapResult } from "./jobs/recurring-invoice-collection-bootstrap.js";
import {
claimProductProviderSideEffectOutboxEvent,
releaseProductProviderSideEffectOutboxClaim,
runProductProviderSideEffectWithReceipt,
} from "./jobs/product-acquisition-provider-side-effect-receipt.js";
import { assertProductAcquisitionRuntimeEnv, resolveProductAcquisitionProviderSideEffectMode } from "./jobs/product-acquisition-runtime-env.js";
import { createPrismaPortalStatusMailStore, resolvePortalStatusMailBaseUrl, type PortalStatusMailPrisma } from "./jobs/portal-status-mail-store.js";
import {
bootstrapWorkflowTrigger,
isWorkflowTriggerOutboxRequired,
type WorkflowTriggerBootstrapResult,
type WorkflowTriggerWorkerHandle,
} from "./jobs/workflow-trigger-bootstrap.js";
import type { QueueJsonValue, TelephonyWebhookProcessJob, TypedQueue } from "@hivegrid/queue";
import {
createWorkflowTriggerExecutorFromEnv,
parseWorkflowTriggerExecutorMode,
type WorkflowTriggerExecutorMode,
} from "./jobs/workflow-trigger-executor-factory.js";
import type { WorkflowTriggerNativePrisma } from "./jobs/workflow-trigger-native-executor.js";
import { createPrismaWorkflowTriggerStore, type WorkflowTriggerStorePrisma } from "./jobs/workflow-trigger-store.js";
import { runPortalAutomationTriggerJob } from "./jobs/portal-automation.js";
import { AUTOMATION_EXECUTOR_OUTBOX_QUEUE, createAutomationExecutorProcessor, createRunAutomationForwardTrigger } from "./jobs/automation-forward.js";
import { bootstrapEmailSequenceWorkers, type EmailSequenceBootstrapResult, type EmailSequenceWorkerHandle } from "./jobs/email-sequence-bootstrap.js";
import { bootstrapAuditEventPersist, type AuditEventPersistBootstrapResult, type AuditEventPersistWorkerHandle } from "./jobs/audit-event-persist-bootstrap.js";
import {
bootstrapChatRealtimeWakeup,
type ChatRealtimeWakeupBootstrapResult,
type ChatRealtimeWakeupWorkerHandle,
} from "./jobs/chat-realtime-wakeup-bootstrap.js";
import { bootstrapChatLinkPreview, type ChatLinkPreviewBootstrapResult, type ChatLinkPreviewWorkerHandle } from "./jobs/chat-link-preview-bootstrap.js";
import {
bootstrapLegacyEventReceiver,
type LegacyEventReceiverBootstrapResult,
type LegacyEventReceiverWorkerHandle,
} from "./jobs/legacy-event-receiver-bootstrap.js";
import {
bootstrapPluginHookFanout,
isPluginHookFanoutRequired,
type PluginHookFanoutBootstrapResult,
type PluginHookFanoutWorkerHandle,
} from "./jobs/plugin-hook-fanout-bootstrap.js";
import {
bootstrapPluginCapabilityUse,
type PluginCapabilityUseBootstrapResult,
type PluginCapabilityUseWorkerHandle,
} from "./jobs/plugin-capability-use-bootstrap.js";
import { createPrismaPluginCapabilityUseStore, type PluginCapabilityUseStorePrisma } from "./jobs/plugin-capability-use.js";
import { registerDsgvoGraceCron } from "./jobs/dsgvo-grace-cron.js";
import { registerOutboxDoneRetentionCron } from "./jobs/outbox-done-retention-cron.js";
import { registerPrivacyGdprExportRetentionCron } from "./jobs/privacy-gdpr-export-retention-cron.js";
import { registerPortalGdprExportRetentionCron } from "./jobs/portal-gdpr-export-retention-cron.js";
import { registerWorkflowExecutionStalledReconcilerCron } from "./jobs/workflow-execution-stalled-reconciler-cron.js";
import { registerAutomationSchedulerCron } from "./jobs/automation-scheduler-cron.js";
import { registerTaskArchiveRetentionCron } from "./jobs/task-archive-retention-cron.js";
import { registerLeadRetentionCron } from "./jobs/lead-retention-cron.js";
import { registerProjectAutoArchiveCron } from "./jobs/project-auto-archive-cron.js";
import { registerBillingDunningSweepCron } from "./jobs/billing-dunning-cron.js";
import {
registerIntegrationsHealthSweepCron,
registerIntegrationsTokenRefreshCron,
registerKwpProjectMaintenanceCron,
} from "./jobs/integrations-maintenance-cron.js";
import { runAllTenantKwpProjectMaintenance } from "./jobs/kwp-project-maintenance.js";
import {
registerCalendarInternalWriteSyncCron,
registerCalendarTokenRefreshCron,
registerCalendarWebhookDrainCron,
registerCalendarWebhookRenewalCron,
} from "./jobs/calendar-maintenance-cron.js";
import {
registerEmailInboxEventCleanupCron,
registerEmailInboxInitialSyncCron,
registerEmailInboxStalePollCron,
registerEmailInboxTokenRefreshCron,
registerEmailInboxWebhookDrainCron,
registerEmailInboxWebhookRenewalCron,
} from "./jobs/email-inbox-maintenance-cron.js";
import { registerTaskReminderSweepCron } from "./jobs/task-reminder-cron.js";
import { registerAdminSalesTaskReminderSweepCron } from "./jobs/admin-sales-task-reminder-cron.js";
import { registerAdminEmailDeliveryCron } from "./jobs/admin-email-delivery-cron.js";
import { registerAdminEmailEventRetentionCron } from "./jobs/admin-email-event-retention-cron.js";
import { registerAdminEmailSyncCron } from "./jobs/admin-email-sync-cron.js";
import { registerAdminEmailDomainCron } from "./jobs/admin-email-domain-cron.js";
import { registerAdminCallRecordingCrons } from "./jobs/admin-call-recording-cron.js";
import { registerAdminOutboundCallCron } from "./jobs/admin-outbound-call-cron.js";
import { registerAdminBookingReminderSweepCron } from "./jobs/admin-booking-reminder-cron.js";
import { registerAdminCalendarSyncCron } from "./jobs/admin-calendar-sync-cron.js";
import {
bootstrapAdminBookingOutbox,
type AdminBookingOutboxBootstrapResult,
type AdminBookingOutboxWorkerHandle,
} from "./jobs/admin-booking-outbox-bootstrap.js";
import { registerHrUploadReminderDispatchCron } from "./jobs/hr-upload-reminder-cron.js";
import { registerEmailDomainProbeCron } from "./jobs/email-domain-probe-cron.js";
import { registerTelephonyWebhookMaintenanceCron, runAllTenantTelephonyWebhookMaintenance } from "./jobs/telephony-webhook-maintenance.js";
import { registerProductAcquisitionReconcilerCron } from "./jobs/product-acquisition-reconciler-cron.js";
import {
PRODUCT_ACQUISITION_RECONCILE_JOB_ID,
asProductAcquisitionReconcileClient,
runAllTenantProductAcquisitionReconciliation,
} from "./jobs/product-acquisition-reconciler.js";
import {
PLUGIN_AUDIT_RETENTION_QUEUE,
registerPluginAuditRetentionCron,
resolvePluginAuditRetentionConfig,
runPluginAuditRetention,
} from "./jobs/plugin-audit-retention.js";
import { AUDIT_RETENTION_QUEUE, registerAuditRetentionCron, runAuditRetentionJob } from "./jobs/audit-retention.js";
import { createWorkflowExecutionStalledReconciler, type WorkflowExecutionStalledReconcilerPrisma } from "./jobs/workflow-execution-stalled-reconciler.js";
import type { HandlerDependencies } from "./handlers.js";
import { DelayedError as BullMQDelayedError, Queue as BullMQQueue, Worker as BullMQWorker } from "bullmq";
import {
TELEPHONY_WEBHOOK_QUEUE_NAME,
buildFilePreviewJob,
buildFileScanJob,
buildKwpProjectCatalogSyncJob,
buildPlatformNotificationJob,
buildTelephonyWebhookJob,
enqueueJob,
enqueueTelephonyWebhookJob,
parseRedisUrl,
queueNames,
recoverFailedTelephonyWebhookJob,
} from "@hivegrid/queue";
import { createIdempotencyStore } from "@hivegrid/shared-store";
import { resolvePlatformLeadNotificationRecipient } from "@hivegrid/db/platform-lead-notifications.server";
import type { SentryLikeForWorkerTracing } from "./runtime.js";
import { shutdownWorkerRuntime, startWorkerRuntime, type WorkerRuntimeStatus } from "./runtime.js";
import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js";
import { runWorkerPayloadWithRlsContext } from "./rls-context.js";
import { createQueue, createWorker } from "./queue-runtime.js";
import type { Prisma } from "@hivegrid/db/prisma-browser";
import { workerLogger } from "./logger.js";
import { getWorkerMetrics } from "./metrics.js";
import { workerErrorReporter } from "./sentry.js";
import { closeSharedStore, getSharedStore } from "./shared-store.js";
import { createWorkerStorageDeleteService } from "./storage.js";
import { buildKwpArchiveDiscoveryEnqueue, buildKwpArchiveFileEnqueue } from "./kwp-archive-queue.js";
const SCHEDULED_JOB_DEFAULT_OPTIONS = {
removeOnComplete: 100,
removeOnFail: 1_000,
} as const;
function createRlsAwareBullMqProcessor<Job extends { data: unknown }>(processor: (job: Job) => Promise<unknown>) {
return async (job: Job) => runWorkerPayloadWithRlsContext(job.data, () => processor(job));
}
type ShutdownState = {
promise?: Promise<void>;
signal?: NodeJS.Signals;
adminServer?: WorkerAdminServerHandle | undefined;
healthServer?: WorkerHealthServerHandle | undefined;
redisHealthProbe?: RedisHealthProbe | undefined;
outbox?: OutboxDispatcherBootstrapResult | undefined;
adminBookingOutbox?: AdminBookingOutboxBootstrapResult | undefined;
workflowTrigger?: WorkflowTriggerBootstrapResult | undefined;
searchIndexer?: SearchIndexerBootstrapResult | undefined;
formSubmissionRoute?: FormSubmissionRouteBootstrapResult | undefined;
formSubmissionWebhookDelivery?: FormSubmissionWebhookDeliveryBootstrapResult | undefined;
formSubmissionWebhookQueue?: BullMQQueue | undefined;
transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult | undefined;
portalStatusMail?: PortalStatusMailBootstrapResult | undefined;
portalDomainOrchestration?: PortalDomainOrchestrationBootstrapResult | undefined;
productAcquisitionProviderSideEffect?: ProductAcquisitionProviderSideEffectBootstrapResult | undefined;
recurringInvoiceCollection?: RecurringInvoiceCollectionBootstrapResult | undefined;
emailSequences?: EmailSequenceBootstrapResult | undefined;
auditEventPersist?: AuditEventPersistBootstrapResult | undefined;
chatRealtimeWakeup?: ChatRealtimeWakeupBootstrapResult | undefined;
chatLinkPreview?: ChatLinkPreviewBootstrapResult | undefined;
legacyEventReceiver?: LegacyEventReceiverBootstrapResult | undefined;
platformNotificationQueue?: TypedQueue<typeof queueNames.notifications> | undefined;
pluginHookFanout?: PluginHookFanoutBootstrapResult | undefined;
pluginCapabilityUse?: PluginCapabilityUseBootstrapResult | undefined;
pluginHookInvokeQueue?: BullMQQueue | undefined;
pluginAuditRetentionWorker?: BullMQWorker | undefined;
auditRetentionWorker?: BullMQWorker | undefined;
filePreviewQueue?: TypedQueue<typeof queueNames.filePreview> | undefined;
telephonyWebhookQueue?: BullMQQueue<TelephonyWebhookProcessJob> | undefined;
tenantDeletionExportQueue?: BullMQQueue | undefined;
telephonyWebhookWorker?: BullMQWorker | undefined;
automationExecutorWorker?: BullMQWorker | undefined;
};
type RedisHealthProbeResult = { ok: true } | { ok: false; error: string };
type RedisHealthProbe = (() => Promise<RedisHealthProbeResult>) & { close: () => Promise<void> };
function buildWorkerStatus(
runtime: WorkerRuntimeStatus,
options?: {
outboxDispatcher?: OutboxDispatcherBootstrapResult;
outboxDispatcherExpected?: boolean;
workflowTrigger?: WorkflowTriggerBootstrapResult;
workflowTriggerExpected?: boolean;
workflowTriggerExecutorMode?: WorkflowTriggerExecutorMode;
pluginHookFanout?: PluginHookFanoutBootstrapResult;
pluginHookFanoutExpected?: boolean;
transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult;
transactionalMailOutboxExpected?: boolean;
portalStatusMail?: PortalStatusMailBootstrapResult;
portalStatusMailExpected?: boolean;
emailSequences?: EmailSequenceBootstrapResult;
emailSequencesExpected?: boolean;
chatLinkPreview?: ChatLinkPreviewBootstrapResult;
chatLinkPreviewExpected?: boolean;
}
): WorkerStatusReport {
const runtimeCheck = runtime.status === "started" ? "ok" : "down";
const outboxDispatcherCheck = options?.outboxDispatcher?.status === "started" ? "ok" : options?.outboxDispatcherExpected ? "down" : "disabled";
const workflowTriggerCheck = options?.workflowTrigger?.status === "started" ? "ok" : options?.workflowTriggerExpected ? "down" : "disabled";
const pluginHookFanoutCheck = options?.pluginHookFanout?.status === "started" ? "ok" : options?.pluginHookFanoutExpected ? "down" : "disabled";
const transactionalMailOutboxCheck =
options?.transactionalMailOutbox?.status === "started" ? "ok" : options?.transactionalMailOutboxExpected ? "down" : "disabled";
const portalStatusMailCheck = options?.portalStatusMail?.status === "started" ? "ok" : options?.portalStatusMailExpected ? "down" : "disabled";
const emailSequencesCheck = options?.emailSequences?.status === "started" ? "ok" : options?.emailSequencesExpected ? "down" : "disabled";
const chatLinkPreviewCheck = options?.chatLinkPreview?.status === "started" ? "ok" : options?.chatLinkPreviewExpected ? "down" : "disabled";
const status =
runtimeCheck === "ok" &&
outboxDispatcherCheck !== "down" &&
workflowTriggerCheck !== "down" &&
pluginHookFanoutCheck !== "down" &&
transactionalMailOutboxCheck !== "down" &&
portalStatusMailCheck !== "down" &&
emailSequencesCheck !== "down" &&
chatLinkPreviewCheck !== "down"
? "ok"
: "down";
return {
status,
service: "worker",
checks: {
runtime: runtimeCheck,
outboxDispatcher: outboxDispatcherCheck,
workflowTrigger: workflowTriggerCheck,
pluginHookFanout: pluginHookFanoutCheck,
transactionalMailOutbox: transactionalMailOutboxCheck,
portalStatusMail: portalStatusMailCheck,
emailSequences: emailSequencesCheck,
chatLinkPreview: chatLinkPreviewCheck,
},
...(workflowTriggerCheck !== "disabled" && options?.workflowTriggerExecutorMode
? { workflowTriggerExecutorMode: options.workflowTriggerExecutorMode }
: {}),
version: process.env.APP_REVISION?.trim() || "unknown",
uptime_seconds: Math.round(process.uptime()),
region: process.env.APP_REGION?.trim() || process.env.FLY_REGION?.trim() || "unknown",
};
}
function createRedisHealthProbe(env: NodeJS.ProcessEnv): RedisHealthProbe {
const redisUrl = env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim();
let queue: import("bullmq").Queue | undefined;
let readyQueue: Promise<import("bullmq").Queue> | undefined;
const probe = async (): Promise<RedisHealthProbeResult> => {
if (!redisUrl) {
return { ok: false, error: "Redis URL not configured" };
}
try {
readyQueue ??= (async () => {
const { Queue } = await import("bullmq");
const nextQueue = new Queue("__hivegrid_worker_health__", { connection: parseRedisUrl(redisUrl) });
queue = nextQueue;
await nextQueue.waitUntilReady();
return nextQueue;
})();
await readyQueue;
return { ok: true };
} catch (error) {
const failedQueue = queue;
queue = undefined;
readyQueue = undefined;
await failedQueue?.close().catch(() => undefined);
return { ok: false, error: error instanceof Error ? error.name : "RedisProbeFailed" };
}
};
probe.close = async () => {
const queueToClose = queue;
queue = undefined;
readyQueue = undefined;
await queueToClose?.close();
};
return probe;
}
function createDatabaseHealthProbe(): () => Promise<{ ok: true } | { ok: false; error: string }> {
return async () => {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const db = getDbClient();
await db.$queryRaw`SELECT 1`;
return { ok: true };
} catch (error) {
return { ok: false, error: error instanceof Error ? error.name : "DatabaseProbeFailed" };
}
};
}
function createRuntimeReadinessProbes({
workerGroup,
outboxDispatcherExpected,
workflowTriggerExpected,
pluginHookFanoutExpected,
transactionalMailOutboxExpected,
portalStatusMailExpected,
emailSequencesExpected,
chatLinkPreviewExpected,
getOutboxDispatcherStatus,
getWorkflowTriggerStatus,
getPluginHookFanoutStatus,
getTransactionalMailOutboxStatus,
getPortalStatusMailStatus,
getEmailSequencesStatus,
getChatLinkPreviewStatus,
}: {
workerGroup?: string;
outboxDispatcherExpected: boolean;
workflowTriggerExpected: boolean;
pluginHookFanoutExpected: boolean;
transactionalMailOutboxExpected: boolean;
portalStatusMailExpected: boolean;
emailSequencesExpected: boolean;
chatLinkPreviewExpected: boolean;
getOutboxDispatcherStatus: () => OutboxDispatcherBootstrapResult | undefined;
getWorkflowTriggerStatus: () => WorkflowTriggerBootstrapResult | undefined;
getPluginHookFanoutStatus: () => PluginHookFanoutBootstrapResult | undefined;
getTransactionalMailOutboxStatus: () => TransactionalMailOutboxBootstrapResult | undefined;
getPortalStatusMailStatus: () => PortalStatusMailBootstrapResult | undefined;
getEmailSequencesStatus: () => EmailSequenceBootstrapResult | undefined;
getChatLinkPreviewStatus: () => ChatLinkPreviewBootstrapResult | undefined;
}): Record<string, HealthProbe> | undefined {
if (workerGroup !== "general" && workerGroup !== "mail") {
return undefined;
}
const probes: Record<string, HealthProbe> = {};
if (outboxDispatcherExpected) {
probes.outboxDispatcher = () =>
getOutboxDispatcherStatus()?.status === "started" ? { ok: true } : { ok: false, error: "outbox dispatcher is not started" };
}
if (workflowTriggerExpected) {
probes.workflowTrigger = () =>
getWorkflowTriggerStatus()?.status === "started" ? { ok: true } : { ok: false, error: "workflow trigger subscriber is not started" };
}
if (pluginHookFanoutExpected) {
probes.pluginHookFanout = () =>
getPluginHookFanoutStatus()?.status === "started" ? { ok: true } : { ok: false, error: "plugin hook fanout subscriber is not started" };
}
if (transactionalMailOutboxExpected) {
probes.transactionalMailOutbox = () =>
getTransactionalMailOutboxStatus()?.status === "started" ? { ok: true } : { ok: false, error: "transactional mail outbox subscriber is not started" };
}
if (portalStatusMailExpected) {
probes.portalStatusMail = () =>
getPortalStatusMailStatus()?.status === "started" ? { ok: true } : { ok: false, error: "portal status mail subscriber is not started" };
}
if (emailSequencesExpected) {
probes.emailSequences = () =>
getEmailSequencesStatus()?.status === "started" ? { ok: true } : { ok: false, error: "email-sequence workers are not started" };
}
if (chatLinkPreviewExpected) {
probes.chatLinkPreview = () =>
getChatLinkPreviewStatus()?.status === "started" ? { ok: true } : { ok: false, error: "chat link preview subscriber is not started" };
}
return Object.keys(probes).length > 0 ? probes : undefined;
}
function hasWorkflowTriggerOutboxRedis(env: NodeJS.ProcessEnv): boolean {
return Boolean(env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim());
}
function isChatLinkPreviewExpected(env: NodeJS.ProcessEnv, workerGroup: string | undefined): boolean {
return workerGroup === "general" && env.CHAT_LINK_PREVIEW_ENABLED !== "false";
}
function isWorkflowTriggerOutboxExpected(env: NodeJS.ProcessEnv): boolean {
if (isWorkflowTriggerOutboxRequired(env) || env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true") {
return true;
}
return env.WORKFLOW_TRIGGER_OUTBOX_ENABLED !== "false" && hasWorkflowTriggerOutboxRedis(env);
}
function isOutboxDispatcherExpected(env: NodeJS.ProcessEnv): boolean {
return env.OUTBOX_DISPATCHER_ENABLED === "true" || isOutboxDispatcherRequired(env) || isWorkflowTriggerOutboxExpected(env);
}
function isPluginHookFanoutExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean {
if (workerGroup !== "general") {
return false;
}
return env.PLUGIN_HOOK_FANOUT_ENABLED === "true" || isPluginHookFanoutRequired(env, workerGroup);
}
function isTransactionalMailOutboxExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean {
if (workerGroup !== "mail") {
return false;
}
return env.TRANSACTIONAL_MAIL_OUTBOX_ENABLED !== "false" || isTransactionalMailOutboxRequired(env, workerGroup);
}
function isPortalStatusMailExpected(workerGroup?: string): boolean {
return workerGroup === "mail";
}
function isEmailSequencesExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean {
return workerGroup === "mail" && env.MARKETING_SEQUENCES_ENABLED === "true";
}
function isPluginInvokeRuntimeEnabled(env: NodeJS.ProcessEnv): boolean {
return env.NODE_ENV !== "production" || env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true";
}
function resolvePositiveInteger(value: string | undefined, fallback: number): number {
const parsed = Number(value);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
}
/**
* Lazy-load @sentry/node for queue.process span tracing. When the SDK is
* not installed or SENTRY_DSN_WORKER is empty, the wrapper degrades to a
* pass-through processor automatically.
*/
async function loadSentryForWorker(): Promise<SentryLikeForWorkerTracing | undefined> {
if (!process.env.SENTRY_DSN_WORKER) return undefined;
try {
const moduleName = "@sentry/node";
const loaded = (await import(/* @vite-ignore */ moduleName)) as SentryLikeForWorkerTracing & {
default?: SentryLikeForWorkerTracing;
};
if (typeof loaded?.startSpan === "function") return loaded;
if (typeof loaded?.default?.startSpan === "function") return loaded.default;
return undefined;
} catch {
return undefined;
}
}
try {
const sentryForTracing = await loadSentryForWorker();
const { metrics, registry } = getWorkerMetrics();
// Validate WORKER_QUEUE_GROUP before we start anything else. An invalid
// value must abort boot loudly — we don't want a half-started worker
// silently subscribing to the wrong queues. Disabled workers (eg local
// dev where WORKER_ENABLED=false) skip the group check.
// Only require WORKER_QUEUE_GROUP when the worker is actually enabled.
// Disabled workers (eg local dev where WORKER_ENABLED!=='true') skip
// group resolution so they can boot a no-op process for testing
// adjacent code paths.
let groupResolution: ReturnType<typeof bootstrapWorkerGroup> | undefined;
if (process.env.WORKER_ENABLED === "true") {
try {
groupResolution = bootstrapWorkerGroup(process.env);
} catch (error) {
if (error instanceof InvalidWorkerQueueGroupError) {
workerLogger.error({ error: { message: error.message } }, "Worker boot rejected: invalid WORKER_QUEUE_GROUP.");
process.exit(1);
}
throw error;
}
const { assertRlsRuntimeReady } = await import("@hivegrid/db/rls-runtime-readiness.server");
await assertRlsRuntimeReady(undefined, {
...process.env,
DATABASE_RUNTIME_ROLE: process.env.DATABASE_RUNTIME_ROLE?.trim() || "hivegrid_worker",
});
assertProductAcquisitionRuntimeEnv(process.env);
}
// Incoming-call browser pushes are part of the webhook worker contract.
// Validate the provider before starting the queue runtime or readiness
// server so a broken VAPID configuration cannot produce a ready worker
// that silently drops ringing notifications.
let telephonyWebPushProvider: import("./jobs/push-notification.js").PushProviderSeam | undefined;
if (groupResolution?.group === "webhook") {
const { createWebPushProvider } = await import("./jobs/web-push-provider.js");
telephonyWebPushProvider = createWebPushProvider({ env: process.env });
}
// The legacy runtime entrypoint reads WORKER_CONCURRENCY from env. We
// funnel the resolved per-group concurrency through that same channel so
// the existing config validation + diagnostics path keeps working.
const runtimeEnv: NodeJS.ProcessEnv = groupResolution ? { ...process.env, WORKER_CONCURRENCY: String(groupResolution.concurrency) } : process.env;
// When a group is resolved we override the BullMQ worker factory so the
// ioredis connection is tagged `worker-<group>-<pid>` (visible in Redis
// CLIENT LIST for operator attribution) and so the per-group pool size
// is applied. The legacy path (no group) keeps the original factory.
const groupAwareCreateWorker = groupResolution
? <Name extends Parameters<typeof createWorker>[0]>(
queueName: Name,
processor: Parameters<typeof createWorker<Name>>[1],
options: Parameters<typeof createWorker<Name>>[2]
) => {
const augmented = createWorkerConnection(groupResolution.group, { env: process.env as NodeJS.ProcessEnv });
// BullMQ's WorkerOptions.connection union accepts plain connection
// options OR a Redis instance. We always pass plain options here
// (parseRedisUrl output), so a cast keeps the spread narrowed.
const mergedConnection = { ...(options.connection as Record<string, unknown>), ...augmented } as Parameters<typeof createWorker<Name>>[2]["connection"];
return createWorker(queueName, processor, {
...options,
connection: mergedConnection,
});
}
: undefined;
const extraHandlerDependencies: Partial<HandlerDependencies> = {};
extraHandlerDependencies.resolvePlatformLeadNotificationRecipient = resolvePlatformLeadNotificationRecipient;
if (groupResolution?.group === "general" || groupResolution?.group === "import") {
extraHandlerDependencies.tenantMaintenanceLeaseStore = getSharedStore();
}
let filePreviewQueue: TypedQueue<typeof queueNames.filePreview> | undefined;
let telephonyWebhookQueue: BullMQQueue<TelephonyWebhookProcessJob> | undefined;
let tenantDeletionExportQueue: BullMQQueue | undefined;
if (groupResolution?.group === "general") {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (redisUrl) {
const { getDbClient } = await import("@hivegrid/db/client.server");
extraHandlerDependencies.runKwpProjectMaintenance = async () => {
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
try {
return await runAllTenantKwpProjectMaintenance({
prisma: getDbClient() as unknown as Parameters<typeof runAllTenantKwpProjectMaintenance>[0]["prisma"],
queue,
});
} finally {
await queue.close();
}
};
workerLogger.info({}, "KWP project maintenance scheduler service bootstrapped (general group).");
}
}
if (groupResolution?.group === "import") {
const [{ syncKwpProjectBinding }, { syncKwpProjectInbound }, { discoverKwpProjectArchiveBatch }, { syncKwpArchiveFile }, { syncKwpProjectCatalog }, { runKwpAllContractSentinels }] =
await Promise.all([
import("@hivegrid/integrations/kwp-project-sync.server"),
import("@hivegrid/integrations/kwp-inbound-sync.server"),
import("@hivegrid/integrations/kwp-archive-discovery.server"),
import("@hivegrid/integrations/kwp-archive-file.server"),
import("@hivegrid/integrations/kwp-project-catalog.server"),
import("@hivegrid/integrations/kwp-contract-sentinel.server"),
]);
extraHandlerDependencies.syncKwpProjectBinding = (payload) => syncKwpProjectBinding({ bindingId: payload.bindingId });
extraHandlerDependencies.syncKwpProjectInbound = (payload) => syncKwpProjectInbound({ bindingId: payload.bindingId });
extraHandlerDependencies.syncKwpArchiveDiscovery = async (payload) => {
const result = await discoverKwpProjectArchiveBatch({
bindingId: payload.bindingId,
...(payload.cursor ? { cursor: payload.cursor } : {}),
});
return {
...result,
cursor: result.cursor
? {
pendingFolderIds: [...result.cursor.pendingFolderIds],
visitedFolderIds: [...result.cursor.visitedFolderIds],
}
: null,
};
};
extraHandlerDependencies.syncKwpProjectCatalog = (payload) => syncKwpProjectCatalog({ connectionId: payload.dataConnectionId });
extraHandlerDependencies.enqueueKwpProjectCatalogContinuation = async (payload) => {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (!redisUrl) {
throw new Error("KWP project catalog continuation requires BULLMQ_REDIS_URL or REDIS_URL.");
}
const continuationBucket = `${payload.cadenceBucket}-${Date.now()}`;
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
try {
const descriptor = buildKwpProjectCatalogSyncJob({
tenantId: payload.tenantId,
dataConnectionId: payload.dataConnectionId,
cadenceBucket: continuationBucket,
});
await enqueueJob(queue, descriptor, {
idempotencyKey: `kwp-project-catalog:${payload.tenantId}:${payload.dataConnectionId}:${continuationBucket}`,
tenantId: payload.tenantId,
delay: 60_000,
attempts: 5,
backoff: { type: "exponential", delay: 30_000 },
});
} finally {
await queue.close();
}
};
extraHandlerDependencies.enqueueKwpArchiveDiscovery = async (payload, cursor) => {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (!redisUrl) {
throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL.");
}
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
try {
const { descriptor, options } = buildKwpArchiveDiscoveryEnqueue({ payload, cursor, now: new Date() });
await enqueueJob(queue, descriptor, options);
} finally {
await queue.close();
}
};
extraHandlerDependencies.enqueueKwpArchiveFileJobs = async (payload, files) => {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (!redisUrl) {
throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL.");
}
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
try {
for (const archiveFile of files) {
const { descriptor, options } = buildKwpArchiveFileEnqueue(payload, archiveFile);
await enqueueJob(queue, descriptor, options);
}
} finally {
await queue.close();
}
return files.length;
};
extraHandlerDependencies.syncKwpArchiveFile = (payload) => {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (!redisUrl) {
throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL.");
}
const storageBucket = process.env.STORAGE_BUCKET?.trim();
if (!storageBucket) {
throw new Error("KWP archive import requires STORAGE_BUCKET.");
}
return syncKwpArchiveFile({
bindingId: payload.bindingId,
archiveFile: payload.archiveFile,
storage: createWorkerStorageDeleteService(process.env),
storageBucket,
enqueueScan: async (input) => {
const queue = createQueue(queueNames.fileScan, { connection: parseRedisUrl(redisUrl) });
try {
const descriptor = buildFileScanJob({
bucket: input.bucket,
key: input.key,
tenantId: input.tenantId,
fileId: input.fileId,
contentType: input.contentType,
metadata: { source: "kwp" },
});
await enqueueJob(queue, descriptor, {
idempotencyKey: `kwp-archive-scan:${input.tenantId}:${input.fileId}:${payload.cadenceBucket}`,
tenantId: input.tenantId,
attempts: 5,
backoff: { type: "exponential", delay: 30_000 },
});
} finally {
await queue.close();
}
},
});
};
extraHandlerDependencies.runKwpContractSentinel = (payload) => runKwpAllContractSentinels({ connectionId: payload.dataConnectionId });
workerLogger.info({}, "KWP provider import services bootstrapped (import group).");
}
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (!redisUrl) {
throw new Error("Telephony webhook maintenance requires BULLMQ_REDIS_URL or REDIS_URL.");
}
const { getDbClient } = await import("@hivegrid/db/client.server");
const queue = new BullMQQueue<TelephonyWebhookProcessJob>(TELEPHONY_WEBHOOK_QUEUE_NAME, {
connection: parseRedisUrl(redisUrl),
});
telephonyWebhookQueue = queue;
extraHandlerDependencies.runTelephonyWebhookMaintenance = () =>
runAllTenantTelephonyWebhookMaintenance(getDbClient(), async (input, source) => {
const descriptor = buildTelephonyWebhookJob(input);
if (source === "failed") {
await recoverFailedTelephonyWebhookJob(queue, descriptor);
} else {
await enqueueTelephonyWebhookJob(queue, descriptor);
}
});
workerLogger.info({ queue: TELEPHONY_WEBHOOK_QUEUE_NAME }, "telephony webhook maintenance service bootstrapped (general group).");
} catch (error) {
workerLogger.error({ error }, "telephony webhook maintenance bootstrap failed; maintenance jobs will retry until fixed.");
}
}
if (groupResolution?.group === "general") {
const providerSideEffectMode = resolveProductAcquisitionProviderSideEffectMode(process.env);
if (providerSideEffectMode === "disabled") {
workerLogger.info({}, "product acquisition reconciler disabled with provider side effects.");
} else {
try {
const apiKey = process.env.MOLLIE_API_KEY?.trim();
const lagoApiUrl = process.env.LAGO_API_URL?.trim();
const lagoApiKey = process.env.LAGO_API_KEY?.trim();
if (!apiKey) throw new Error("Product acquisition reconciliation requires MOLLIE_API_KEY.");
if (!lagoApiUrl || !lagoApiKey) throw new Error("Product acquisition reconciliation requires Lago configuration.");
const [{ createLagoBillingGateway, createMolliePaymentGateway }, { applyMolliePaymentSync }, { getDbClient }] = await Promise.all([
import("@hivegrid/billing"),
import("@hivegrid/db/billing-sync.server"),
import("@hivegrid/db/client.server"),
]);
const prisma = asProductAcquisitionReconcileClient(getDbClient());
const mollie = createMolliePaymentGateway({
apiKey,
profileId: process.env.MOLLIE_PROFILE_ID?.trim() || null,
timeoutMs: resolvePositiveInteger(process.env.MOLLIE_TIMEOUT_MS, 10_000),
});
const lago = createLagoBillingGateway({
apiUrl: lagoApiUrl,
apiKey: lagoApiKey,
timeoutMs: resolvePositiveInteger(process.env.LAGO_TIMEOUT_MS, 10_000),
});
extraHandlerDependencies.runProductAcquisitionReconcile = ({ take } = {}) =>
runAllTenantProductAcquisitionReconciliation(
{
prisma,
mollie,
lago,
applyPaymentSync: (input) => applyMolliePaymentSync(input, getDbClient() as unknown as NonNullable<Parameters<typeof applyMolliePaymentSync>[1]>),
alert: async (alert) => {
const details = {
code: alert.code,
tenantId: alert.tenantId,
acquisitionId: alert.acquisitionId,
status: alert.status,
providerPaymentId: alert.providerPaymentId,
};
workerLogger.error(details, "billing product acquisition reconciliation requires operator attention");
await workerErrorReporter.report({
component: "worker",
operation: "billing.product-acquisition-reconcile.quarantined",
severity: "error",
...details,
});
},
},
{ batchSize: take }
);
workerLogger.info({}, "product acquisition reconciler bootstrapped (general group).");
} catch (error) {
workerLogger.error({ error }, "product acquisition reconciler bootstrap failed; maintenance jobs will retry until fixed.");
}
}
}
if (groupResolution?.group === "general") {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
extraHandlerDependencies.reconcileWorkflowExecutionStalls = createWorkflowExecutionStalledReconciler({
prisma: getDbClient() as unknown as WorkflowExecutionStalledReconcilerPrisma,
});
workerLogger.info({}, "workflow-execution-stalled-reconciler service bootstrapped (general group).");
} catch (error) {
workerLogger.error({ error }, "workflow-execution-stalled-reconciler bootstrap failed; maintenance handler will use DB fallback.");
}
}
// AK-P4a §4 — file-preview generator service on the `general` group
// (the group that owns the hivegrid.file-preview queue). Composed with a
// Prisma client + STORAGE_* S3 seams; injected via extraHandlerDependencies.
// When STORAGE_* is unset the factory returns undefined and the handler
// dead-letters preview jobs.
if (groupResolution?.group === "general") {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createWorkerFilePreviewService } = await import("./jobs/file-preview.js");
const previewService = createWorkerFilePreviewService(process.env as Parameters<typeof createWorkerFilePreviewService>[0], {
prisma: getDbClient() as unknown as Parameters<typeof createWorkerFilePreviewService>[1]["prisma"],
logger: workerLogger,
});
if (previewService) {
extraHandlerDependencies.filePreviewService = previewService;
workerLogger.info({}, "file-preview service bootstrapped (general group).");
} else {
workerLogger.info({}, "file-preview service skipped: STORAGE_* not configured.");
}
// AK-P4a §5 — scan-verdict write-back. The general group owns the
// hivegrid.file-scan queue, so wire the File.scanStatus writer here with
// the same Prisma seam the preview service uses.
const { createWorkerFileScanStatusWriter } = await import("./jobs/file-scan-status.js");
extraHandlerDependencies.fileScanStatusWriter = createWorkerFileScanStatusWriter({
prisma: getDbClient() as unknown as Parameters<typeof createWorkerFileScanStatusWriter>[0]["prisma"],
logger: workerLogger,
});
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (!redisUrl) {
throw new Error("Redis is required to enqueue previews after clean file scans.");
}
filePreviewQueue = createQueue(queueNames.filePreview, { connection: parseRedisUrl(redisUrl) });
extraHandlerDependencies.filePreviewEnqueuer = {
enqueueAfterCleanScan: async (input) => {
const descriptor = buildFilePreviewJob({
bucket: input.bucket,
fileId: input.fileId,
mimeType: input.contentType,
storageKey: input.storageKey,
tenantId: input.tenantId,
});
await enqueueJob(filePreviewQueue!, descriptor, {
jobId: `file-preview-after-scan-${input.fileId}`,
tenantId: input.tenantId,
});
},
};
workerLogger.info({}, "file-scan status writer bootstrapped (general group).");
} catch (error) {
workerLogger.error({ error }, "file-preview bootstrap failed; preview jobs will dead-letter until fixed.");
}
}
// Phase 06 #21 / F-015 — wire the real plugin-invoke dependencies on
// the `plugin` worker group. Without this, `handlers.ts` throws when a
// plugin-invoke job arrives — which is the intentional fail-loud
// behaviour that F-015 introduced to replace the prior silent no-op
// stub. The composition lives in jobs/plugin-invoke-bootstrap.ts.
const pluginInvokeRuntimeEnabled = groupResolution?.group === "plugin" ? isPluginInvokeRuntimeEnabled(process.env) : true;
if (groupResolution?.group === "plugin" && pluginInvokeRuntimeEnabled) {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const { bootstrapPluginInvokeDeps } = await import("./jobs/plugin-invoke-bootstrap.js");
const { getSharedStore } = await import("./shared-store.js");
const prisma = getDbClient() as unknown as Parameters<typeof bootstrapPluginInvokeDeps>[0]["prisma"];
extraHandlerDependencies.pluginInvokeDeps = bootstrapPluginInvokeDeps({
env: process.env,
prisma,
store: getSharedStore(),
logger: workerLogger,
});
workerLogger.info({}, "plugin-invoke production deps bootstrapped (plugin group).");
} catch (error) {
workerLogger.error({ error }, "plugin-invoke bootstrap failed in plugin worker group; exiting.");
throw error;
}
if (!extraHandlerDependencies.pluginInvokeDeps) {
workerLogger.error({}, "plugin-invoke bootstrap produced no dependencies in plugin worker group; exiting.");
process.exit(1);
throw new Error("plugin-invoke bootstrap produced no dependencies");
}
} else if (groupResolution?.group === "plugin") {
workerLogger.warn(
{ workerGroup: groupResolution.group, queueNames: groupResolution.queueNames },
"plugin-invoke runtime disabled before production signoff; starting plugin worker idle without queue consumers."
);
}
// The `import` group always owns the dedicated KWP queue. Generic imports and
// exports remain separately gated: full mode adds both, while the internal
// portal-GDPR mode adds only exports and wires only the two GDPR services.
const importQueueRuntimeMode = resolveImportQueueRuntimeMode(process.env);
const importQueueRuntimeEnabled = importQueueRuntimeMode === "full" || importQueueRuntimeMode === "portal-gdpr-export";
if (groupResolution?.group === "import" && importQueueRuntimeMode === "idle") {
workerLogger.warn(
{ workerGroup: groupResolution.group, queueNames: groupResolution.queueNames },
"Generic import/export runtime gated off; starting the import worker with the dedicated KWP queue only."
);
}
if (groupResolution?.group === "import" && importQueueRuntimeEnabled) {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
if (importQueueRuntimeMode === "full") {
const { createAdminSalesDataWorkerService } = await import("./jobs/admin-sales-data.js");
const adminSalesDataService = createAdminSalesDataWorkerService(process.env, {
prisma: getDbClient() as unknown as Parameters<typeof createAdminSalesDataWorkerService>[1]["prisma"],
});
if (adminSalesDataService) {
extraHandlerDependencies.adminSalesDataService = adminSalesDataService;
workerLogger.info({}, "admin sales import/export service bootstrapped (import group).");
}
}
const { createWorkerPrivacyGdprExportService } = await import("./jobs/privacy-gdpr-export-bootstrap.js");
const privacyGdprExportService = createWorkerPrivacyGdprExportService(process.env, {
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPrivacyGdprExportService>[1]["prisma"],
sharedStore: getSharedStore(),
});
if (privacyGdprExportService) {
extraHandlerDependencies.privacyGdprExportService = privacyGdprExportService;
workerLogger.info({}, "privacy GDPR export service bootstrapped (import group).");
// AK-P21 — best-effort "export ready" mail via the email queue. Both
// import runtime modes own the privacy GDPR export handler, so the
// mailer must not stay gated behind the `full` mode: in
// `portal-gdpr-export` mode the very same job finished silently.
const mailRedisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (mailRedisUrl) {
const { createGdprExportReadyMailer } = await import("./jobs/privacy-gdpr-export-bootstrap.js");
const { buildNotificationJob, buildRetryJobOptions } = await import("@hivegrid/queue");
// AGENTS.md DoD C15 — go through the M011 notification queue, not the
// raw transactional mail queue: the notification handler owns the
// preference check, the NotificationDelivery record and the provider
// call.
const notificationQueue = new BullMQQueue(queueNames.notifications, { connection: parseRedisUrl(mailRedisUrl) });
extraHandlerDependencies.sendGdprExportReadyMail = createGdprExportReadyMailer({
prisma: getDbClient() as unknown as Parameters<typeof createGdprExportReadyMailer>[0]["prisma"],
enqueueNotification: async (input) => {
const descriptor = buildNotificationJob(input);
await notificationQueue.add(descriptor.jobName, descriptor.payload, buildRetryJobOptions(descriptor));
},
...(process.env.APP_URL ? { appUrl: process.env.APP_URL } : {}),
logger: workerLogger,
});
workerLogger.info({}, "privacy GDPR export ready-mailer bootstrapped (import group).");
}
} else {
workerLogger.info({}, "privacy GDPR export service skipped: STORAGE_* not configured.");
}
const { createWorkerPortalGdprExportService } = await import("./jobs/portal-gdpr-export-bootstrap.js");
const portalGdprExportService = createWorkerPortalGdprExportService(process.env, {
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPortalGdprExportService>[1]["prisma"],
sharedStore: getSharedStore(),
logger: workerLogger,
});
if (portalGdprExportService) {
extraHandlerDependencies.portalGdprExportService = portalGdprExportService;
workerLogger.info({}, "portal GDPR JSON export service bootstrapped (import group).");
} else {
workerLogger.info({}, "portal GDPR JSON export service skipped: STORAGE_* not configured.");
}
} catch (error) {
workerLogger.error({ error }, "privacy GDPR export bootstrap failed; GDPR export jobs will dead-letter until fixed.");
}
}
// AK-P21 — tenant hard-delete service (general group owns the maintenance
// queue). Local dev/test may explicitly skip external cleanup, while
// required staging/production runtimes fail closed before queue ownership.
if (groupResolution?.group === "general") {
const tenantHardDeleteRequired = isRequiredTenantHardDeleteRuntime(process.env);
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createWorkerPrivacyTenantDeletionService } = await import("./jobs/privacy-tenant-deletion-bootstrap.js");
if (tenantHardDeleteRequired && (!process.env.MEILI_URL?.trim() || !process.env.MEILI_API_KEY?.trim())) {
throw new Error("Required tenant hard-delete search cleanup configuration is missing.");
}
let searchClient;
if (process.env.MEILI_URL) {
const { createSearchClient } = await import("@hivegrid/search");
searchClient = createSearchClient({ url: process.env.MEILI_URL, apiKey: process.env.MEILI_API_KEY ?? undefined });
}
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (redisUrl) {
tenantDeletionExportQueue = new BullMQQueue(queueNames.exports, { connection: parseRedisUrl(redisUrl) });
}
extraHandlerDependencies.privacyTenantDeletionService = createWorkerPrivacyTenantDeletionService(process.env, {
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPrivacyTenantDeletionService>[1]["prisma"],
...(searchClient ? { searchClient } : {}),
...(tenantDeletionExportQueue ? { exportQueue: tenantDeletionExportQueue } : {}),
...(redisUrl ? { sharedStore: getSharedStore() } : {}),
logger: workerLogger,
});
workerLogger.info({}, "privacy tenant-deletion service bootstrapped (general group).");
} catch (error) {
if (tenantHardDeleteRequired) {
workerLogger.error(
{ errorName: error instanceof Error ? error.name : "UnknownError" },
"privacy tenant-deletion bootstrap failed in required mode; exiting."
);
process.exit(1);
throw new Error("Worker startup aborted after required tenant hard-delete bootstrap failure.");
}
workerLogger.error({ errorName: error instanceof Error ? error.name : "UnknownError" }, "privacy tenant-deletion bootstrap failed; continuing.");
}
}
// R-P2b — DSGVO grace-period cron service. Without this block the daily
// scan acks every tick as "not configured" and confirmed user deletion
// requests are never executed (found in the 2026-07-17 ticket audit).
if (groupResolution?.group === "general") {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createWorkerDsgvoGraceCronService } = await import("./jobs/dsgvo-grace-bootstrap.js");
extraHandlerDependencies.dsgvoGraceCronService = createWorkerDsgvoGraceCronService({
prisma: getDbClient() as unknown as Parameters<typeof createWorkerDsgvoGraceCronService>[0]["prisma"],
logger: workerLogger,
});
workerLogger.info({}, "dsgvo grace-cron service bootstrapped (general group).");
} catch (error) {
workerLogger.error({ error }, "dsgvo grace-cron bootstrap failed; grace scans will skip until fixed.");
}
}
// VA-P26b — PortalUser lifecycle is isolated from the internal User/TenantUser
// privacy path. The general group owns the bounded daily maintenance scan.
if (groupResolution?.group === "general") {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createWorkerPortalLifecycleService } = await import("./jobs/portal-lifecycle-bootstrap.js");
extraHandlerDependencies.portalLifecycleService = createWorkerPortalLifecycleService(process.env, {
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPortalLifecycleService>[1]["prisma"],
logger: workerLogger,
});
workerLogger.info({}, "portal lifecycle service bootstrapped (general group).");
} catch (error) {
workerLogger.error({ error }, "portal lifecycle bootstrap failed; maintenance jobs will retry until fixed.");
}
}
const runtimeQueueNames = groupResolution
? !pluginInvokeRuntimeEnabled
? []
: groupResolution.group === "import"
? getWorkerQueueNames(process.env)
: groupResolution.queueNames
: undefined;
const runtime = startWorkerRuntime({
logger: workerLogger,
reporter: workerErrorReporter,
sentry: sentryForTracing,
metrics,
env: runtimeEnv,
extraHandlerDependencies,
...(runtimeQueueNames ? { queueNames: runtimeQueueNames } : {}),
...(groupAwareCreateWorker ? { createWorkerImpl: groupAwareCreateWorker } : {}),
});
if (groupResolution) {
workerLogger.info(
{
workerGroup: groupResolution.group,
queueNames: runtime.queueNames,
concurrency: groupResolution.concurrency,
},
"Worker queue group resolved."
);
}
let outboxStatus: OutboxDispatcherBootstrapResult | undefined;
let workflowTriggerStatus: WorkflowTriggerBootstrapResult | undefined;
let pluginHookFanoutStatus: PluginHookFanoutBootstrapResult | undefined;
let transactionalMailOutboxStatus: TransactionalMailOutboxBootstrapResult | undefined;
let portalStatusMailStatus: PortalStatusMailBootstrapResult | undefined;
let emailSequencesStatus: EmailSequenceBootstrapResult | undefined;
let chatLinkPreviewStatus: ChatLinkPreviewBootstrapResult | undefined;
const outboxDispatcherExpected = groupResolution?.group === "general" && isOutboxDispatcherExpected(process.env);
const workflowTriggerExpected = groupResolution?.group === "general" && isWorkflowTriggerOutboxExpected(process.env);
const pluginHookFanoutExpected = isPluginHookFanoutExpected(process.env, groupResolution?.group);
const transactionalMailOutboxExpected = isTransactionalMailOutboxExpected(process.env, groupResolution?.group);
const portalStatusMailExpected = isPortalStatusMailExpected(groupResolution?.group);
const emailSequencesExpected = isEmailSequencesExpected(process.env, groupResolution?.group);
const chatLinkPreviewExpected = isChatLinkPreviewExpected(process.env, groupResolution?.group);
const adminServer = isWorkerMetricsEnabled(process.env)
? startWorkerAdminServer({
enabled: true,
token: process.env.METRICS_TOKEN,
registry,
port: Number.parseInt(process.env.WORKER_ADMIN_PORT ?? "", 10) || DEFAULT_WORKER_ADMIN_PORT,
host: process.env.WORKER_ADMIN_HOST,
logger: workerLogger,
buildStatus: () =>
buildWorkerStatus(runtime, {
outboxDispatcher: outboxStatus,
outboxDispatcherExpected,
workflowTrigger: workflowTriggerStatus,
workflowTriggerExpected,
workflowTriggerExecutorMode: parseWorkflowTriggerExecutorMode(process.env.WORKFLOW_TRIGGER_EXECUTOR_MODE),
pluginHookFanout: pluginHookFanoutStatus,
pluginHookFanoutExpected,
transactionalMailOutbox: transactionalMailOutboxStatus,
transactionalMailOutboxExpected,
portalStatusMail: portalStatusMailStatus,
portalStatusMailExpected,
emailSequences: emailSequencesStatus,
emailSequencesExpected,
chatLinkPreview: chatLinkPreviewStatus,
chatLinkPreviewExpected,
}),
})
: undefined;
// Health server: always on when a runtime started. Bound to a separate
// port from the admin server so liveness probes still work when
// METRICS_ENABLED is false (eg early-stage staging).
const redisHealthProbe = runtime.status === "started" ? createRedisHealthProbe(process.env) : undefined;
const healthServer =
runtime.status === "started"
? startWorkerHealthServer({
port: Number.parseInt(process.env.WORKER_HEALTH_PORT ?? "", 10) || DEFAULT_WORKER_HEALTH_PORT,
host: process.env.WORKER_HEALTH_HOST,
...(groupResolution ? { workerGroup: groupResolution.group } : {}),
redisProbe: redisHealthProbe!,
databaseProbe: createDatabaseHealthProbe(),
...(() => {
const readinessProbes = createRuntimeReadinessProbes({
workerGroup: groupResolution?.group,
outboxDispatcherExpected,
workflowTriggerExpected,
pluginHookFanoutExpected,
transactionalMailOutboxExpected,
portalStatusMailExpected,
emailSequencesExpected,
chatLinkPreviewExpected,
getOutboxDispatcherStatus: () => outboxStatus,
getWorkflowTriggerStatus: () => workflowTriggerStatus,
getPluginHookFanoutStatus: () => pluginHookFanoutStatus,
getTransactionalMailOutboxStatus: () => transactionalMailOutboxStatus,
getPortalStatusMailStatus: () => portalStatusMailStatus,
getEmailSequencesStatus: () => emailSequencesStatus,
getChatLinkPreviewStatus: () => chatLinkPreviewStatus,
});
return readinessProbes ? { readinessProbes } : {};
})(),
logger: workerLogger,
})
: undefined;
if (runtime.status === "disabled") {
if (telephonyWebhookQueue) {
await telephonyWebhookQueue.close();
}
if (tenantDeletionExportQueue) {
await tenantDeletionExportQueue.close();
}
if (adminServer) {
await adminServer.close();
}
if (healthServer) {
await healthServer.close();
}
process.exit(0);
} else {
let outbox: OutboxDispatcherBootstrapResult | undefined;
let adminBookingOutbox: AdminBookingOutboxBootstrapResult | undefined;
let portalDomainOrchestration: PortalDomainOrchestrationBootstrapResult | undefined;
let productAcquisitionProviderSideEffect: ProductAcquisitionProviderSideEffectBootstrapResult | undefined;
let recurringInvoiceCollection: RecurringInvoiceCollectionBootstrapResult | undefined;
let telephonyWebhookWorker: BullMQWorker | undefined;
let automationExecutorWorker: BullMQWorker | undefined;
if (groupResolution?.group === "webhook") {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (!redisUrl) {
throw new Error("Telephony webhook worker requires BULLMQ_REDIS_URL or REDIS_URL.");
}
const [
{ getDbClient },
{ listActiveDeviceTokens, revokeDeviceToken },
{ createTelephonyWebhookJobProcessor },
{ dispatchPushNotification },
{ withTenantRLS },
{ buildTelephonyPushHref },
] = await Promise.all([
import("@hivegrid/db/client.server"),
import("@hivegrid/db/device-tokens.server"),
import("./jobs/telephony-webhook-bootstrap.js"),
import("./jobs/push-notification.js"),
import("./db/tenantRls.js"),
import("./jobs/telephony-push-target.js"),
]);
const prisma = getDbClient();
const webPushProvider = telephonyWebPushProvider;
if (!webPushProvider) {
throw new Error("Telephony webhook worker started without a validated Web Push provider.");
}
const processor = createTelephonyWebhookJobProcessor(prisma, {
pushIncomingCall: async (input) => {
const tenantSlug = await withTenantRLS(prisma, input.tenantId, async (tx) => {
const tenant = await tx.tenant.findFirst({
where: { id: input.tenantId },
select: { slug: true },
});
if (!tenant) throw new Error("Telephony push tenant is unavailable.");
return tenant.slug;
});
await dispatchPushNotification({
payload: {
tenantId: input.tenantId,
userId: input.userId,
title: "HiveGrid",
body: input.externalNumber,
data: {
callId: input.callId,
href: buildTelephonyPushHref({
tenantSlug,
callId: input.callId,
target: input.target,
}),
messageKey: "telephony.notifications.incomingCall",
},
},
tokenLister: ({ userId, tenantId }) =>
withTenantRLS(prisma, input.tenantId, (tx) =>
listActiveDeviceTokens(
{
userId,
...(tenantId ? { tenantId } : {}),
},
tx as never
)
),
tokenRevoker: async (token) => {
await withTenantRLS(prisma, input.tenantId, (tx) =>
revokeDeviceToken(
{
token,
userId: input.userId,
tenantId: input.tenantId,
},
tx as never
)
);
},
providers: webPushProvider,
logger: workerLogger,
});
},
});
telephonyWebhookWorker = new BullMQWorker<TelephonyWebhookProcessJob>(TELEPHONY_WEBHOOK_QUEUE_NAME, createRlsAwareBullMqProcessor(processor), {
connection: parseRedisUrl(redisUrl),
concurrency: groupResolution.concurrency,
});
workerLogger.info({ queue: TELEPHONY_WEBHOOK_QUEUE_NAME, concurrency: groupResolution.concurrency }, "telephony webhook worker started");
}
// The outbox dispatcher is a cron-style polling job that lives on the
// `general` group. Running it from every worker group would either (a)
// multiply Redis/Postgres load by N replicas or (b) race for the same
// FOR UPDATE SKIP LOCKED rows. When the group is undefined (legacy
// WORKER_ENABLED=false path) we leave the dispatcher off too.
const shouldRunOutbox = groupResolution?.group === "general";
const outboxRequiredForGroup = shouldRunOutbox && (isOutboxDispatcherRequired(process.env) || isWorkflowTriggerOutboxRequired(process.env));
if (shouldRunOutbox) {
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient();
outbox = await bootstrapOutboxDispatcher(process.env, {
prisma,
logger: workerLogger,
reporter: workerErrorReporter,
});
outboxStatus = outbox;
if (outbox.status === "disabled" && outboxRequiredForGroup) {
workerLogger.error({ reason: "OUTBOX_DISPATCHER_ENABLED!=true" }, "Outbox dispatcher disabled in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required outbox dispatcher disabled.");
}
} catch (error) {
outboxStatus = { status: "disabled" };
if (outboxRequiredForGroup) {
workerLogger.error({ error }, "Outbox dispatcher bootstrap failed in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required outbox dispatcher failure.");
}
workerLogger.error({ error }, "Outbox dispatcher bootstrap failed; continuing without dispatcher.");
}
} else if (groupResolution) {
workerLogger.info({ workerGroup: groupResolution.group }, "Outbox dispatcher skipped on this worker group; it runs only on the general group.");
}
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
if (redisUrl) {
const [{ getDbClient }, { createAdminBookingOutboxDependencies }] = await Promise.all([
import("@hivegrid/db/client.server"),
import("@hivegrid/integrations/admin-booking-worker.server"),
]);
const connection = createWorkerConnection("general", {
env: process.env as NodeJS.ProcessEnv,
});
adminBookingOutbox = bootstrapAdminBookingOutbox(process.env, {
dependencies: createAdminBookingOutboxDependencies({
client: getDbClient(),
env: process.env,
}),
createWorker: (queueName, processor, options) =>
new BullMQWorker(queueName, processor, {
connection,
concurrency: options.concurrency,
}) as unknown as AdminBookingOutboxWorkerHandle,
logger: workerLogger,
});
} else {
adminBookingOutbox = {
status: "disabled",
reason: "no Redis URL configured",
};
if (process.env.ADMIN_BOOKING_OUTBOX_REQUIRED === "true" || process.env.RLS_RUNTIME_MODE === "required") {
throw new Error("Admin booking provider worker requires Redis in required mode.");
}
}
} catch (error) {
adminBookingOutbox = {
status: "disabled",
reason: "bootstrap failed",
};
workerLogger.error({ error }, "Admin booking provider worker bootstrap failed.");
if (process.env.ADMIN_BOOKING_OUTBOX_REQUIRED === "true" || process.env.RLS_RUNTIME_MODE === "required") {
throw error;
}
}
}
if (groupResolution?.group === "general") {
try {
portalDomainOrchestration = await bootstrapPortalDomainOrchestration(process.env, {
logger: workerLogger,
reporter: workerErrorReporter,
});
if (portalDomainOrchestration.status === "disabled") {
if (process.env.PORTAL_DOMAIN_ORCHESTRATION_REQUIRED === "true") {
throw new Error(`Portal-domain orchestration is required but disabled: ${portalDomainOrchestration.reason}`);
}
workerLogger.info({ reason: portalDomainOrchestration.reason }, "portal-domain orchestration disabled.");
}
} catch (error) {
if (process.env.PORTAL_DOMAIN_ORCHESTRATION_REQUIRED === "true") {
workerLogger.error({ error }, "Portal-domain orchestration failed in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required portal-domain orchestration failure.");
}
portalDomainOrchestration = { status: "disabled", reason: "bootstrap failed" };
workerLogger.error({ error }, "Portal-domain orchestration bootstrap failed; continuing while custom-domain activation stays unavailable.");
}
}
if (groupResolution?.group === "general") {
const providerSideEffectRequired = resolveProductAcquisitionProviderSideEffectMode(process.env) === "required";
try {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient();
const connection = redisUrl ? parseRedisUrl(redisUrl) : null;
productAcquisitionProviderSideEffect = bootstrapProductAcquisitionProviderSideEffect(process.env, {
logger: workerLogger,
claimOutboxEvent: (envelope) => claimProductProviderSideEffectOutboxEvent(envelope, { store: prisma }),
releaseOutboxClaim: (envelope, claim) => releaseProductProviderSideEffectOutboxClaim(envelope, claim, { store: prisma }),
loadAcquisition: async ({ tenantId, acquisitionId }) =>
prisma.tenantProductAcquisition.findFirst({
where: { id: acquisitionId, tenantId },
select: {
id: true,
tenantId: true,
productKind: true,
status: true,
providerAttempt: true,
providerPaymentId: true,
invoiceExternalId: true,
lagoSubscriptionId: true,
lagoProviderSubscriptionId: true,
initialLagoInvoiceExternalId: true,
},
}),
createWorker: (queueName, processor, options) => {
if (!connection) throw new Error("Product-acquisition provider side-effect worker requires Redis.");
const worker = new BullMQWorker(
queueName,
async (job, token) =>
runWorkerPayloadWithRlsContext(job.data, () =>
runProductProviderSideEffectWithReceipt(job, processor, {
store: prisma,
logger: workerLogger,
report: (event) => workerErrorReporter.report(event as never),
deferReceiptFailure: async () => {
await job.moveToDelayed(Date.now() + 5_000, token);
throw new BullMQDelayedError();
},
})
),
{
connection,
concurrency: options.concurrency,
}
);
return worker;
},
});
if (productAcquisitionProviderSideEffect.status === "disabled") {
workerLogger.info({ reason: productAcquisitionProviderSideEffect.reason }, "product-acquisition provider side-effect worker disabled");
}
} catch (error) {
if (providerSideEffectRequired) {
workerLogger.error({ error }, "Product-acquisition provider side-effect bootstrap failed in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required product-acquisition provider side-effect failure.");
}
productAcquisitionProviderSideEffect = { status: "disabled", reason: "bootstrap failed" };
workerLogger.error({ error }, "Product-acquisition provider side-effect bootstrap failed; continuing without subscriber.");
}
}
if (groupResolution?.group === "general") {
const providerSideEffectRequired = resolveProductAcquisitionProviderSideEffectMode(process.env) === "required";
try {
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient();
const connection = redisUrl ? parseRedisUrl(redisUrl) : null;
recurringInvoiceCollection = bootstrapRecurringInvoiceCollection(process.env, {
logger: workerLogger,
claimOutboxEvent: (envelope) => claimProductProviderSideEffectOutboxEvent(envelope, { store: prisma }),
releaseOutboxClaim: (envelope, claim) => releaseProductProviderSideEffectOutboxClaim(envelope, claim, { store: prisma }),
loadCollection: async ({ tenantId, collectionId, lagoSubscriptionId, billingProviderSubscriptionId }) => {
const [ledger, subscription] = await Promise.all([
prisma.checkoutSessionStatus.findFirst({
where: { id: collectionId, tenantId },
select: {
id: true,
tenantId: true,
pending: true,
status: true,
providerPaymentId: true,
invoiceExternalId: true,
lagoSubscriptionId: true,
scopeKey: true,
billingCadence: true,
amountCents: true,
currency: true,
},
}),
prisma.tenantSubscription.findUnique({
where: { tenantId },
select: {
status: true,
mollieCustomerId: true,
products: {
where: {
lagoSubscriptionId,
billingProviderSubscriptionId,
},
select: { id: true },
take: 1,
},
},
}),
]);
if (
!ledger ||
!subscription?.mollieCustomerId?.trim() ||
subscription.products.length !== 1 ||
(subscription.status !== "active" && subscription.status !== "trial")
) {
return null;
}
return {
...ledger,
billingProviderSubscriptionId: ledger.scopeKey,
mollieCustomerId: subscription.mollieCustomerId,
};
},
updateCollection: async ({ tenantId, collectionId, providerPaymentId, status, pending }) => {
const updated = await prisma.checkoutSessionStatus.updateMany({
where: {
id: collectionId,
tenantId,
pending: true,
providerPaymentId: null,
},
data: {
...(providerPaymentId ? { providerPaymentId } : {}),
status,
pending,
},
});
return updated.count === 1;
},
createWorker: (queueName, processor, options) => {
if (!connection) throw new Error("Recurring invoice collection worker requires Redis.");
const worker = new BullMQWorker(
queueName,
async (job, token) =>
runWorkerPayloadWithRlsContext(job.data, () =>
runProductProviderSideEffectWithReceipt(job, processor, {
store: prisma,
logger: workerLogger,
report: (event) => workerErrorReporter.report(event as never),
deferReceiptFailure: async () => {
await job.moveToDelayed(Date.now() + 5_000, token);
throw new BullMQDelayedError();
},
})
),
{
connection,
concurrency: options.concurrency,
}
);
return worker;
},
});
if (recurringInvoiceCollection.status === "disabled") {
workerLogger.info({ reason: recurringInvoiceCollection.reason }, "recurring invoice collection worker disabled");
}
} catch (error) {
if (providerSideEffectRequired) {
workerLogger.error({ error }, "Recurring invoice collection bootstrap failed in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required recurring invoice collection failure.");
}
recurringInvoiceCollection = { status: "disabled", reason: "bootstrap failed" };
workerLogger.error({ error }, "Recurring invoice collection bootstrap failed; continuing without subscriber.");
}
}
let workflowTrigger: WorkflowTriggerBootstrapResult | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (process.env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "false") {
workflowTrigger = bootstrapWorkflowTrigger(process.env, { logger: workerLogger });
workflowTriggerStatus = workflowTrigger;
if (workflowTrigger.status === "disabled") {
workerLogger.info({ reason: workflowTrigger.reason }, "workflow-trigger worker disabled by configuration.");
}
} else if (redisUrl) {
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient();
const executor = createWorkflowTriggerExecutorFromEnv(process.env, {
logger: workerLogger,
prisma: prisma as unknown as WorkflowTriggerNativePrisma,
});
const connection = createWorkerConnection("general", { env: process.env as NodeJS.ProcessEnv });
workflowTrigger = bootstrapWorkflowTrigger(process.env, {
logger: workerLogger,
store: createPrismaWorkflowTriggerStore(prisma as unknown as WorkflowTriggerStorePrisma, {
nativeCapabilityPreflight: parseWorkflowTriggerExecutorMode(process.env.WORKFLOW_TRIGGER_EXECUTOR_MODE) !== "http",
}),
...(executor ? { executor } : {}),
idempotency: createIdempotencyStore({
store: getSharedStore(),
prefix: "workflow-trigger",
ttlSeconds: 60 * 60 * 24,
}),
runPortalAutomation: (data) =>
runPortalAutomationTriggerJob(data, {
prisma,
logger: workerLogger,
portalBaseUrl: resolvePortalStatusMailBaseUrl(process.env),
}),
runAutomationForward: createRunAutomationForwardTrigger({ prisma, logger: workerLogger }),
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as WorkflowTriggerWorkerHandle;
},
});
if (workflowTrigger.status === "disabled") {
if (isWorkflowTriggerOutboxRequired(process.env)) {
workerLogger.error({ reason: workflowTrigger.reason }, "Workflow-trigger subscriber disabled in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required workflow-trigger subscriber disabled.");
}
workerLogger.warn({ reason: workflowTrigger.reason }, "workflow-trigger worker disabled.");
}
workflowTriggerStatus = workflowTrigger;
if (workflowTrigger.status === "started") {
// R-P12c: durable forward-executor consumer for
// automation.execution.* outbox events (retry/delay continuations
// arrive as scheduled outbox rows on the same queue).
const automationExecutorProcessor = createAutomationExecutorProcessor({ prisma, logger: workerLogger });
automationExecutorWorker = new BullMQWorker(
AUTOMATION_EXECUTOR_OUTBOX_QUEUE,
createRlsAwareBullMqProcessor(async (job: { data: unknown }) => automationExecutorProcessor(job.data)),
{ connection, concurrency: 5 }
);
workerLogger.info({ queue: AUTOMATION_EXECUTOR_OUTBOX_QUEUE }, "automation forward-executor worker started.");
}
} else {
if (isWorkflowTriggerOutboxRequired(process.env)) {
throw new Error("Workflow trigger outbox subscriber is required but disabled: no Redis URL configured.");
}
workflowTrigger = { status: "disabled", reason: "no Redis URL configured" };
workflowTriggerStatus = workflowTrigger;
workerLogger.info({}, "workflow-trigger worker skipped: no Redis URL configured.");
}
} catch (error) {
if (isWorkflowTriggerOutboxRequired(process.env)) {
workerLogger.error({ error }, "Workflow-trigger bootstrap failed in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required workflow-trigger failure.");
}
workerLogger.error({ error }, "Workflow-trigger bootstrap failed; continuing without subscriber.");
workflowTriggerStatus = { status: "disabled", reason: "bootstrap failed" };
}
} else if (groupResolution && isWorkflowTriggerOutboxRequired(process.env)) {
workerLogger.error({ workerGroup: groupResolution.group }, "Workflow-trigger subscriber required on a non-general worker group; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after workflow-trigger required on a non-general worker group.");
}
// Phase 10 #68 — DSGVO grace-period cron. Like the outbox dispatcher
// this is a single-instance scheduler and must therefore live on the
// `general` group only (one replica per cluster). We register the
// BullMQ job-scheduler entry; the actual processor is wired into the
// maintenance queue by the maintenance handler, which inspects
// job.name === DSGVO_GRACE_CRON_JOB_ID and calls runDsgvoGraceCron.
if (groupResolution?.group === "general") {
const exportRetentionRequired = process.env.EXPORT_RETENTION_QUEUE_ENABLED === "true";
const tenantHardDeleteRequired = isRequiredTenantHardDeleteRuntime(process.env);
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const connection = parseRedisUrl(redisUrl);
const dsgvoCronQueue = new BullMQQueue(queueNames.maintenance, {
connection,
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
});
try {
await registerDsgvoGraceCron(dsgvoCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "dsgvo-grace-cron scheduler registered (24h cadence).");
// AK-P21 — tenant hard-delete scan shares the maintenance queue
// and the single-scheduled general-group invariant.
const { registerPrivacyTenantDeletionCron } = await import("./jobs/privacy-tenant-deletion-bootstrap.js");
await registerPrivacyTenantDeletionCron(dsgvoCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "privacy-tenant-deletion scheduler registered (24h cadence).");
const { registerPortalLifecycleCron } = await import("./jobs/portal-lifecycle-bootstrap.js");
await registerPortalLifecycleCron(dsgvoCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "portal-lifecycle scheduler registered (24h cadence).");
if (exportRetentionRequired) {
await registerPrivacyGdprExportRetentionCron(dsgvoCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "privacy GDPR export-retention scheduler registered (15m cadence).");
await registerPortalGdprExportRetentionCron(dsgvoCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "portal GDPR export-retention scheduler registered (15m cadence).");
}
} finally {
await dsgvoCronQueue.close();
}
} else {
if (exportRetentionRequired) {
throw new Error("Required GDPR export-retention schedulers cannot start without Redis.");
}
workerLogger.info({}, "dsgvo-grace-cron scheduler skipped: no Redis URL configured.");
}
} catch (error) {
if (exportRetentionRequired || tenantHardDeleteRequired) {
workerLogger.error({}, "Required GDPR/tenant-deletion scheduler registration failed; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required GDPR/tenant-deletion scheduler failure.");
}
workerLogger.error({ error }, "dsgvo-grace-cron scheduler registration failed; continuing.");
}
}
// DONE outbox rows are useful for short replay/debug windows but
// should not grow forever. Keep retention single-scheduled on the
// general worker so every deployment has one bounded daily sweeper.
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const connection = parseRedisUrl(redisUrl);
const outboxRetentionQueue = new BullMQQueue(queueNames.maintenance, {
connection,
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
});
try {
await registerOutboxDoneRetentionCron(outboxRetentionQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "outbox-done-retention scheduler registered (24h cadence).");
} finally {
await outboxRetentionQueue.close();
}
} else {
workerLogger.info({}, "outbox-done-retention scheduler skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "outbox-done-retention scheduler registration failed; continuing.");
}
}
// Workflow executions are inserted as `running` before their outbox
// event is dispatched. If dispatch fails permanently, this singleton
// sweep repairs only untouched manual/API executions with a matching
// FAILED outbox row so they do not remain running forever.
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const connection = parseRedisUrl(redisUrl);
const workflowStalledReconcileQueue = new BullMQQueue(queueNames.maintenance, {
connection,
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
});
try {
await registerWorkflowExecutionStalledReconcilerCron(workflowStalledReconcileQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "workflow-execution-stalled-reconcile scheduler registered (5m cadence).");
} finally {
await workflowStalledReconcileQueue.close();
}
} else {
workerLogger.info({}, "workflow-execution-stalled-reconcile scheduler skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "workflow-execution-stalled-reconcile scheduler registration failed; continuing.");
}
}
// P4 Tasks (AK-P4b) — 30-day auto-archive (DoD 5) + hourly reminder sweep
// (DoD 4). Single-instance schedulers on the `general` group (same
// constraint as the other maintenance crons: must not multiply across
// replicas). Both are no-ops without Redis configured.
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const connection = parseRedisUrl(redisUrl);
const taskCronQueue = new BullMQQueue(queueNames.maintenance, {
connection,
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
});
try {
await registerTaskArchiveRetentionCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "task-archive-retention scheduler registered (24h cadence).");
await registerLeadRetentionCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "lead-retention-sweep scheduler registered (24h cadence).");
await registerProjectAutoArchiveCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "project-auto-archive scheduler registered (24h cadence).");
await registerBillingDunningSweepCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "billing-dunning-sweep scheduler registered (24h cadence).");
await registerTaskReminderSweepCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "task-reminder-sweep scheduler registered (1h cadence).");
await registerAutomationSchedulerCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "automation-scheduler-sweep scheduler registered (1m cadence).");
await registerAdminSalesTaskReminderSweepCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "admin-sales-task-reminder-sweep scheduler registered (15m cadence).");
await registerAdminBookingReminderSweepCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "admin-booking-reminder-sweep scheduler registered (1m cadence).");
await registerAdminCalendarSyncCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "admin-calendar-sync scheduler registered (5m cadence).");
await registerAdminEmailDeliveryCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-delivery scheduler registered (1m cadence).");
await registerAdminEmailEventRetentionCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-event-retention scheduler registered (24h cadence).");
await registerAdminEmailSyncCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-sync scheduler registered (2m cadence).");
await registerAdminEmailDomainCron(taskCronQueue, process.env);
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-domain-check scheduler registered (15m cadence).");
await registerAdminCallRecordingCrons(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "admin-call-recording schedulers registered (1m process, 1h retention).");
await registerAdminOutboundCallCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "admin-outbound-call scheduler registered (1m cadence).");
await registerIntegrationsTokenRefreshCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "integrations-token-refresh scheduler registered (1m cadence).");
await registerIntegrationsHealthSweepCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "integrations-health-sweep scheduler registered (15m cadence).");
await registerKwpProjectMaintenanceCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "kwp-project-maintenance scheduler registered (15m cadence).");
await registerCalendarTokenRefreshCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "calendar-token-refresh scheduler registered (5m cadence).");
await registerCalendarWebhookRenewalCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-renewal scheduler registered (6h cadence).");
await registerCalendarWebhookDrainCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-drain scheduler registered (1m cadence).");
await registerCalendarInternalWriteSyncCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "calendar-internal-write-sync scheduler registered (5m cadence).");
await registerEmailInboxTokenRefreshCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-token-refresh scheduler registered (5m cadence).");
await registerEmailInboxWebhookRenewalCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-renewal scheduler registered (6h cadence).");
await registerEmailInboxWebhookDrainCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-drain scheduler registered (1m cadence).");
await registerEmailInboxInitialSyncCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-initial-sync scheduler registered (2m cadence).");
await registerEmailInboxStalePollCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-stale-poll scheduler registered (30m cadence).");
await registerEmailInboxEventCleanupCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-event-cleanup scheduler registered (24h cadence).");
await registerHrUploadReminderDispatchCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "hr-upload-reminder-dispatch scheduler registered (24h cadence).");
await registerEmailDomainProbeCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "email-domain-probe scheduler registered (5m cadence).");
await registerTelephonyWebhookMaintenanceCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "telephony-webhook-maintenance scheduler registered (5m cadence).");
if (resolveProductAcquisitionProviderSideEffectMode(process.env) === "disabled") {
await taskCronQueue.removeJobScheduler(PRODUCT_ACQUISITION_RECONCILE_JOB_ID);
workerLogger.info({ queue: queueNames.maintenance }, "product-acquisition-reconcile scheduler removed while provider side effects are disabled.");
} else {
await registerProductAcquisitionReconcilerCron(taskCronQueue);
workerLogger.info({ queue: queueNames.maintenance }, "product-acquisition-reconcile scheduler registered (5m cadence).");
}
} finally {
await taskCronQueue.close();
}
} else {
workerLogger.info({}, "task maintenance schedulers skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "task maintenance scheduler registration failed; continuing.");
}
}
// Phase 06 #23 — PluginCapabilityUse audit-retention cron. Nightly
// at 03:00 UTC (cron pattern overridable via PLUGIN_AUDIT_RETENTION_CRON).
// Single-instance scheduler + worker on the `general` group, on a
// dedicated queue (PLUGIN_AUDIT_RETENTION_QUEUE) so we don't have to
// extend the maintenance CleanupJob discriminator. The worker is
// attached to the SAME BullMQ Queue instance so the scheduler-fired
// job lands here and gets processed.
let pluginAuditRetentionWorker: import("bullmq").Worker | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const connection = parseRedisUrl(redisUrl);
const pluginAuditCronQueue = new BullMQQueue(PLUGIN_AUDIT_RETENTION_QUEUE, {
connection,
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
});
try {
const { pattern } = await registerPluginAuditRetentionCron(pluginAuditCronQueue, process.env);
workerLogger.info({ queue: PLUGIN_AUDIT_RETENTION_QUEUE, pattern }, "plugin-audit-retention scheduler registered.");
} finally {
await pluginAuditCronQueue.close();
}
const retentionConfig = resolvePluginAuditRetentionConfig(process.env);
pluginAuditRetentionWorker = new BullMQWorker(
PLUGIN_AUDIT_RETENTION_QUEUE,
async () => {
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient() as unknown as {
pluginCapabilityUse: {
deleteMany: (args: { where: { id: { in: string[] } } }) => Promise<{ count: number }>;
findMany: (args: {
where: { timestamp: { lt: Date } };
select: { id: true };
take: number;
orderBy: { timestamp: "asc" };
}) => Promise<Array<{ id: string }>>;
count: () => Promise<number>;
};
};
return runPluginAuditRetention({
config: retentionConfig,
logger: workerLogger,
deleteBatch: async (cutoff, batchSize) => {
// Postgres does not support LIMIT on DELETE directly; pick
// a bounded id-set first, then deleteMany by id.
const candidates = await prisma.pluginCapabilityUse.findMany({
where: { timestamp: { lt: cutoff } },
select: { id: true },
take: batchSize,
orderBy: { timestamp: "asc" },
});
if (candidates.length === 0) return 0;
const ids = candidates.map((row) => row.id);
const { count } = await prisma.pluginCapabilityUse.deleteMany({
where: { id: { in: ids } },
});
return count;
},
// F-020 — surviving-rows count fed to the size-warning
// step. Soft-fails inside runPluginAuditRetention if the
// count() call throws, so a flaky DB doesn't poison the
// retention result.
countRows: () => prisma.pluginCapabilityUse.count(),
});
},
{ connection: parseRedisUrl(redisUrl), concurrency: 1 }
);
workerLogger.info({ queue: PLUGIN_AUDIT_RETENTION_QUEUE, retentionDays: retentionConfig.retentionDays }, "plugin-audit-retention worker started.");
} else {
workerLogger.info({}, "plugin-audit-retention scheduler skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "plugin-audit-retention scheduler registration failed; continuing.");
}
}
// AK-P3 AP-2 — AuditEvent retention cron. Nightly at 03:30 UTC
// (AUDIT_RETENTION_CRON overridable). Single-instance scheduler + worker on
// the `general` group, dedicated queue (AUDIT_RETENTION_QUEUE). Mirrors the
// plugin-audit-retention pattern. The policy is fixed at ten years; every
// bounded delete batch persists an authenticated chain checkpoint first.
let auditRetentionWorker: import("bullmq").Worker | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const connection = parseRedisUrl(redisUrl);
const auditRetentionQueue = new BullMQQueue(AUDIT_RETENTION_QUEUE, {
connection,
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
});
try {
const { pattern } = await registerAuditRetentionCron(auditRetentionQueue, process.env);
workerLogger.info({ queue: AUDIT_RETENTION_QUEUE, pattern }, "audit-retention scheduler registered.");
} finally {
await auditRetentionQueue.close();
}
auditRetentionWorker = new BullMQWorker(
AUDIT_RETENTION_QUEUE,
async () => {
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createAuditEvent } = await import("@hivegrid/db/audit-events.server");
const { pruneAuditChainBatch } = await import("@hivegrid/db/audit-chain-checkpoint.server");
const prisma = getDbClient() as unknown as {
auditEvent: {
groupBy: (args: Record<string, unknown>) => Promise<Array<{ tenantId: string | null }>>;
};
};
return runAuditRetentionJob({
logger: workerLogger,
// One aggregate query discovers all expired chains, including
// the platform chain (tenantId=null); there is no per-tenant
// regulatory-scope query fan-out.
chainLister: async (threshold: Date): Promise<ReadonlyArray<string | null>> => {
const rows = await prisma.auditEvent.groupBy({
by: ["tenantId"],
where: { createdAt: { lt: threshold } },
});
return rows.map((row) => row.tenantId);
},
batchPruner: (tenantId, threshold, batchSize) => pruneAuditChainBatch(tenantId, threshold, { batchSize }),
// The cleanup is itself audited as a system action (traceable).
auditWriter: async ({ tenantId, deletedCount, threshold, retentionYears }) => {
await createAuditEvent({
tenantId,
actorType: "system",
action: "audit.retention.pruned",
targetType: "AuditEvent",
sourceModule: "audit",
dataSensitivity: "internal",
metadata: { deletedCount, threshold: threshold.toISOString(), retentionYears },
});
},
});
},
{ connection: parseRedisUrl(redisUrl), concurrency: 1 }
);
workerLogger.info({ queue: AUDIT_RETENTION_QUEUE }, "audit-retention worker started.");
} else {
workerLogger.info({}, "audit-retention scheduler skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "audit-retention scheduler registration failed; continuing.");
}
}
let searchIndexer: SearchIndexerBootstrapResult | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createSearchIndexerBulkPagers } = await import("./jobs/search-indexer-bulk-pagers.js");
const prisma = getDbClient();
searchIndexer = await bootstrapSearchIndexer(process.env, {
logger: workerLogger,
bulkPagers: createSearchIndexerBulkPagers({ prisma }),
createWorkerImpl: redisUrl
? (queueName, processor, options) => {
const connection = parseRedisUrl(redisUrl);
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as SearchIndexerWorkerHandle;
}
: undefined,
});
} catch (error) {
workerLogger.error({ error }, "Search-indexer bootstrap failed; stopping general worker.");
throw error;
}
}
let formSubmissionRoute: FormSubmissionRouteBootstrapResult | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient() as unknown as {
formSubmission: {
findUnique: (args: { where: { id: string }; select: { status: true; tenantId: true } }) => Promise<{ status: string; tenantId: string } | null>;
updateMany: (args: { where: { id: string; tenantId: string }; data: Record<string, unknown> }) => Promise<{ count: number }>;
};
formAction: {
findMany: (args: Record<string, unknown>) => Promise<
Array<{
id: string;
actionType: string;
name: string;
fieldMapping: unknown;
actionConfig: unknown;
}>
>;
};
formActionLog: {
create: (args: Record<string, unknown>) => Promise<{ id: string; attempt: number }>;
update: (args: Record<string, unknown>) => Promise<unknown>;
};
inboxLead: {
findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null>;
create: (args: Record<string, unknown>) => Promise<{ id: string }>;
};
kontakt: {
findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null>;
create: (args: Record<string, unknown>) => Promise<{ id: string }>;
};
kontaktMarketingConsent: {
upsert: (args: Record<string, unknown>) => Promise<unknown>;
};
pipeline: { findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null> };
pipelineStage: { findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null> };
projekt: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
task: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
$transaction: <Result>(
callback: (
tx: Parameters<typeof ensureAndSyncProjectChannelIfEnabledWithTx>[0] & {
projekt: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
}
) => Promise<Result>
) => Promise<Result>;
};
const [{ publishOutboxEvent }, { planFormSubmissionFollowUps }, { buildWorkflowTriggerOutboxEvent }, { applyFieldMapping }] = await Promise.all([
import("@hivegrid/db/outbox"),
import("./jobs/form-submission-followups.js"),
import("@hivegrid/queue"),
import("@hivegrid/form-widget"),
]);
const connection = parseRedisUrl(redisUrl);
function stringValue(values: Record<string, unknown>, keys: readonly string[]): string | null {
for (const key of keys) {
const value = values[key];
if (typeof value === "string" && value.trim()) return value.trim();
}
return null;
}
function objectValue(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
}
formSubmissionRoute = bootstrapFormSubmissionRoute(process.env, {
logger: workerLogger,
handlerDeps: {
store: {
load: (submissionId) =>
prisma.formSubmission.findUnique({
where: { id: submissionId },
select: { status: true, tenantId: true },
}),
markRouted: async (input) => {
const { count } = await prisma.formSubmission.updateMany({
where: { id: input.submissionId, tenantId: input.tenantId },
data: { status: input.status, result: input.result },
});
if (count !== 1) {
throw new Error("form-submission-route could not update the tenant-scoped submission row");
}
},
},
emitFollowUps: async (followUp) => {
const events = planFormSubmissionFollowUps(followUp);
for (const event of events) {
await publishOutboxEvent(prisma as never, {
tenantId: event.tenantId,
type: event.type,
payload: event.payload as Prisma.InputJsonValue,
dispatchKey: event.dispatchKey,
});
}
},
actionRunner: async ({ submissionId, formId, tenantId, values }) => {
const actions = await prisma.formAction.findMany({
where: { formId, tenantId, enabled: true },
orderBy: { position: "asc" },
select: { id: true, actionType: true, name: true, fieldMapping: true, actionConfig: true },
});
for (const action of actions) {
const mapped = applyFieldMapping({
payload: values,
fieldMapping: objectValue(action.fieldMapping) as Record<string, string>,
});
const idempotencyKey = `form-action:${submissionId}:${action.id}`;
const log = await prisma.formActionLog.create({
data: {
formActionId: action.id,
submissionId,
formId,
tenantId,
actionType: action.actionType,
status: "pending",
inputSnapshot: { values: mapped },
idempotencyKey,
},
select: { id: true, attempt: true },
});
try {
const config = objectValue(action.actionConfig);
let target: { type: string; id: string | null } = { type: action.actionType, id: null };
if (action.actionType === "auto_match") {
const email = stringValue(mapped, ["email", "primaerEmail", "email_address"]);
const phone = stringValue(mapped, ["phone", "telefon", "primaerTelefon"]);
const contact =
email || phone
? await prisma.kontakt.findFirst({
where: {
tenantId,
archivedAt: null,
OR: [...(email ? [{ primaerEmail: email.toLowerCase() }] : []), ...(phone ? [{ primaerTelefon: phone }] : [])],
},
select: { id: true },
})
: null;
const lead =
!contact && (email || phone)
? await prisma.inboxLead.findFirst({
where: {
tenantId,
archivedAt: null,
OR: [...(email ? [{ email: email.toLowerCase() }] : []), ...(phone ? [{ telefon: phone }] : [])],
},
select: { id: true },
})
: null;
target = contact ? { type: "contact", id: contact.id } : lead ? { type: "lead", id: lead.id } : { type: "none", id: null };
if (target.id) {
await prisma.formSubmission.updateMany({
where: { id: submissionId, tenantId },
data: { linkedEntityType: target.type, linkedEntityId: target.id },
});
}
} else if (action.actionType === "create_lead") {
const lead = await prisma.inboxLead.create({
data: {
tenantId,
createdByUserId: typeof config.userId === "string" ? config.userId : null,
quelle: "form",
quelleDetail: { formId, submissionId },
sourceOrigin: `/forms/${formId}`,
vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]),
nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]),
email: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null,
telefon: stringValue(mapped, ["telefon", "phone"]),
message: stringValue(mapped, ["message", "nachricht", "details"]),
rohdaten: { formId, submissionId, values: mapped },
weiterverarbeitungFreigabe: "ja",
},
select: { id: true },
});
target = { type: "lead", id: lead.id };
} else if (action.actionType === "create_contact") {
const contact = await prisma.kontakt.create({
data: {
tenantId,
createdByUserId: typeof config.userId === "string" ? config.userId : "system",
kontaktTyp: "privatperson",
vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]),
nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]) ?? stringValue(mapped, ["name"]) ?? "Formular-Kontakt",
primaerEmail: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null,
primaerTelefon: stringValue(mapped, ["telefon", "phone"]),
},
select: { id: true },
});
target = { type: "contact", id: contact.id };
} else if (action.actionType === "create_project") {
const pipeline = await prisma.pipeline.findFirst({
where: { tenantId, archivedAt: null },
orderBy: { position: "asc" },
select: { id: true },
});
const stage = pipeline
? await prisma.pipelineStage.findFirst({
where: { tenantId, pipelineId: pipeline.id, archivedAt: null },
orderBy: { position: "asc" },
select: { id: true },
})
: null;
if (!pipeline || !stage) throw new Error("No project pipeline/stage available for form action");
const actorUserId = typeof config.userId === "string" ? config.userId : null;
const project = await prisma.$transaction(async (tx) => {
const created = await tx.projekt.create({
data: {
tenantId,
createdByUserId: actorUserId,
projektNummer: `FORM-${Date.now()}`,
projektname: stringValue(mapped, ["projektname", "projectName", "name"]) ?? "Projekt aus Formular",
beschreibung: stringValue(mapped, ["message", "description", "details"]),
quelle: "form",
pipelineId: pipeline.id,
stageId: stage.id,
},
select: { id: true },
});
await ensureAndSyncProjectChannelIfEnabledWithTx(tx, {
tenantId,
projectId: created.id,
actorUserId,
requestId: `form:${formId}:submission:${submissionId}:project:${created.id}:chat-created`,
});
return created;
});
target = { type: "project", id: project.id };
} else if (action.actionType === "create_task") {
const task = await prisma.task.create({
data: {
tenantId,
createdByUserId: typeof config.userId === "string" ? config.userId : "system",
title: stringValue(mapped, ["title", "taskTitle", "subject"]) ?? "Aufgabe aus Formular",
description: stringValue(mapped, ["message", "description", "details"]),
assignedToUserId:
typeof config.assignedToUserId === "string"
? config.assignedToUserId
: typeof config.userId === "string"
? config.userId
: "system",
source: "form_submission",
sourceRef: submissionId,
},
select: { id: true },
});
target = { type: "task", id: task.id };
} else if (action.actionType === "webhook") {
const webhookUrl = typeof config.webhookUrl === "string" ? config.webhookUrl : null;
if (!webhookUrl) throw new Error("Webhook action missing webhookUrl");
const webhookPayload = { submissionId, formId, tenantId, webhookUrl, values: mapped } as unknown as Prisma.InputJsonValue;
await publishOutboxEvent(prisma as never, {
tenantId,
type: "webhook.form-submission.deliver",
payload: webhookPayload,
dispatchKey: `form-action-webhook:${submissionId}:${action.id}`,
});
} else if (action.actionType === "send_email_to_team" || action.actionType === "send_email_to_submitter") {
const recipients =
action.actionType === "send_email_to_submitter"
? [stringValue(mapped, ["email", "email_address"])].filter((email): email is string => Boolean(email))
: Array.isArray(config.recipients)
? config.recipients.filter((email): email is string => typeof email === "string" && email.includes("@"))
: [];
if (recipients.length === 0) {
await prisma.formActionLog.update({
where: { id: log.id },
data: { status: "skipped", outputSnapshot: { reason: "no_recipients" }, finishedAt: new Date() },
});
continue;
}
const mailPayload = { submissionId, formId, tenantId, recipients, values: mapped } as unknown as Prisma.InputJsonValue;
await publishOutboxEvent(prisma as never, {
tenantId,
type: "mail.transactional.form-submission",
payload: mailPayload,
dispatchKey: `form-action-mail:${submissionId}:${action.id}`,
});
} else if (action.actionType === "trigger_automation") {
const automationData = { formId, submissionId, actionId: action.id, values: mapped } as unknown as QueueJsonValue;
await publishOutboxEvent(
prisma as never,
buildWorkflowTriggerOutboxEvent({
tenantId,
event: "form.submitted",
data: automationData,
userId: typeof config.userId === "string" ? config.userId : null,
sourceId: `form:${formId}:submission:${submissionId}:action:${action.id}`,
})
);
} else if (action.actionType === "start_double_opt_in") {
const result = await runStartDoubleOptInFormAction({
prisma: prisma as unknown as StartDoubleOptInPrisma,
publishOutboxEvent: (event) =>
publishOutboxEvent(prisma as never, {
tenantId: event.tenantId,
type: event.type,
payload: event.payload as Prisma.InputJsonValue,
dispatchKey: event.dispatchKey,
}),
tenantId,
formId,
submissionId,
actionId: action.id,
mapped,
config,
});
if (result.status === "skipped") {
await prisma.formActionLog.update({
where: { id: log.id },
data: { status: "skipped", outputSnapshot: result, finishedAt: new Date() },
});
continue;
}
target = { type: "marketing_consent", id: result.kontaktId };
} else {
await prisma.formActionLog.update({
where: { id: log.id },
data: { status: "skipped", outputSnapshot: { reason: "unsupported_action" }, finishedAt: new Date() },
});
continue;
}
await prisma.formActionLog.update({
where: { id: log.id },
data: {
status: "success",
targetType: target.type,
targetId: target.id,
outputSnapshot: { target },
finishedAt: new Date(),
},
});
} catch (error) {
await prisma.formActionLog.update({
where: { id: log.id },
data: {
status: "failed",
errorMessage: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500),
finishedAt: new Date(),
},
});
}
}
},
logger: workerLogger,
},
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as FormSubmissionRouteWorkerHandle;
},
});
} else {
formSubmissionRoute = { status: "disabled", reason: "no Redis URL configured" };
workerLogger.info({}, "form-submission-route worker skipped: no Redis URL configured.");
}
} catch (error) {
formSubmissionRoute = { status: "disabled", reason: "bootstrap failed" };
workerLogger.error({ error }, "Form-submission-route bootstrap failed; continuing without subscriber.");
}
}
let formSubmissionWebhookDelivery: FormSubmissionWebhookDeliveryBootstrapResult | undefined;
let formSubmissionWebhookQueue: BullMQQueue | undefined;
let transactionalMailOutbox: TransactionalMailOutboxBootstrapResult | undefined;
let portalStatusMail: PortalStatusMailBootstrapResult | undefined;
let emailSequences: EmailSequenceBootstrapResult | undefined;
if (groupResolution?.group === "mail") {
const transactionalMailOutboxRequired = isTransactionalMailOutboxRequired(process.env, groupResolution.group);
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const connection = parseRedisUrl(redisUrl);
const { getDbClient } = await import("@hivegrid/db/client.server");
const campaignCancellationPrisma = getDbClient() as unknown as CampaignScheduleCancellationPrisma;
formSubmissionWebhookQueue = new BullMQQueue(queueNames.email, { connection });
formSubmissionWebhookDelivery = bootstrapFormSubmissionWebhookDelivery(process.env, {
logger: workerLogger,
handlerDeps: {
enqueueWebhook: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options),
},
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as FormSubmissionWebhookDeliveryWorkerHandle;
},
});
try {
transactionalMailOutbox = bootstrapTransactionalMailOutbox(process.env, {
logger: workerLogger,
handlerDeps: {
enqueueMail: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options),
cancelCampaignSchedule: (input) => cancelCampaignScheduleWithPrisma(campaignCancellationPrisma, input),
isCampaignDeliveryAllowed: (input) => isCampaignDeliveryAllowedWithPrisma(campaignCancellationPrisma, input),
},
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as TransactionalMailOutboxWorkerHandle;
},
});
transactionalMailOutboxStatus = transactionalMailOutbox;
if (transactionalMailOutbox.status === "disabled") {
if (transactionalMailOutboxRequired) {
workerLogger.error({ reason: transactionalMailOutbox.reason }, "Transactional mail outbox disabled in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required transactional mail outbox disabled.");
}
workerLogger.warn({ reason: transactionalMailOutbox.reason }, "transactional mail outbox bridge disabled.");
}
} catch (error) {
transactionalMailOutboxStatus = { status: "disabled", reason: "bootstrap failed" };
if (transactionalMailOutboxRequired) {
workerLogger.error({ error }, "Transactional mail outbox bootstrap failed in required mode; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required transactional mail outbox failure.");
}
workerLogger.error({ error }, "Transactional mail outbox bootstrap failed; continuing without subscriber.");
}
try {
const { getDbClient } = await import("@hivegrid/db/client.server");
const store = createPrismaPortalStatusMailStore(getDbClient() as unknown as PortalStatusMailPrisma, {
portalBaseUrl: resolvePortalStatusMailBaseUrl(process.env),
});
portalStatusMail = bootstrapPortalStatusMail(process.env, {
logger: workerLogger,
handlerDeps: {
...store,
enqueueMail: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options),
logger: workerLogger,
},
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as PortalStatusMailWorkerHandle;
},
});
portalStatusMailStatus = portalStatusMail;
if (portalStatusMail.status === "disabled") {
workerLogger.error({ reason: portalStatusMail.reason }, "portal status mail worker disabled; queued deliveries will wait for recovery.");
}
} catch (error) {
portalStatusMail = { status: "disabled", reason: "bootstrap failed" };
portalStatusMailStatus = portalStatusMail;
workerLogger.error({ error }, "Portal status mail bootstrap failed; queued deliveries will wait for recovery.");
}
} else {
formSubmissionWebhookDelivery = { status: "disabled", reason: "no Redis URL configured" };
transactionalMailOutbox = { status: "disabled", reason: "no Redis URL configured" };
portalStatusMail = { status: "disabled", reason: "no Redis URL configured" };
portalStatusMailStatus = portalStatusMail;
transactionalMailOutboxStatus = transactionalMailOutbox;
if (transactionalMailOutboxRequired) {
workerLogger.error({}, "Transactional mail outbox required but Redis URL is not configured; exiting.");
process.exit(1);
throw new Error("Worker startup aborted after required transactional mail outbox missing Redis.");
}
workerLogger.info({}, "form-submission webhook bridge skipped: no Redis URL configured.");
workerLogger.info({}, "transactional mail outbox bridge skipped: no Redis URL configured.");
workerLogger.info({}, "portal status mail worker skipped: no Redis URL configured.");
}
const emailSequenceRedisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (!emailSequenceRedisUrl && process.env.MARKETING_SEQUENCES_ENABLED === "true") {
emailSequences = { status: "disabled", reason: "no Redis URL configured" };
workerLogger.error({}, "Email-sequence workers enabled but Redis URL is not configured.");
} else {
const [{ runSequenceTriggerJob }, { runEmailSequenceStepJob }] = await Promise.all([
import("./jobs/email-sequence-trigger.js"),
import("./jobs/email-sequence-scheduler.js"),
]);
// Production deps (Prisma store + BullMQ scheduleStep producer +
// template renderer + @hivegrid/mail Mailjet adapter +
// unsubscribe-url builder) are loaded lazily inside the
// processors so startup-time optional config issues do not
// crash the mail worker. Missing Redis/Mailjet config still
// fails closed at job time and is surfaced through worker logs.
emailSequences = bootstrapEmailSequenceWorkers(process.env, {
logger: workerLogger,
triggerProcessor: async (job) => {
const { createEmailSequenceTriggerDeps } = await import("./jobs/email-sequence-runtime.js");
const deps = await createEmailSequenceTriggerDeps({ logger: workerLogger });
if (!deps) return { matched: 0, started: 0, skipped: 0 };
return runSequenceTriggerJob(job.data, deps);
},
schedulerProcessor: async (job) => {
const { createEmailSequenceSchedulerDeps } = await import("./jobs/email-sequence-runtime.js");
const deps = await createEmailSequenceSchedulerDeps({ logger: workerLogger });
if (!deps) return { action: "cancelled" as const, completed: true };
return runEmailSequenceStepJob(job.data, deps);
},
createWorkerImpl: emailSequenceRedisUrl
? (queueName, processor, options) => {
const connection = parseRedisUrl(emailSequenceRedisUrl);
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as EmailSequenceWorkerHandle;
}
: undefined,
});
}
emailSequencesStatus = emailSequences;
} catch (error) {
emailSequences = { status: "disabled", reason: "bootstrap failed" };
emailSequencesStatus = emailSequences;
workerLogger.error({ error }, "Email-sequence bootstrap failed; continuing without sequence workers.");
}
}
let auditEventPersist: AuditEventPersistBootstrapResult | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient();
auditEventPersist = bootstrapAuditEventPersist(process.env, {
logger: workerLogger,
runInTransaction: async (fn) =>
// Serializable isolation guarantees concurrent dispatcher
// deliveries for the same tenant either serialize into a
// coherent chain or one of them retries on conflict.
(
prisma as unknown as {
$transaction: (runner: (tx: Parameters<typeof fn>[0]) => Promise<unknown>, opts?: { isolationLevel?: string }) => Promise<unknown>;
}
).$transaction(async (tx) => fn(tx as Parameters<typeof fn>[0]), { isolationLevel: "Serializable" }) as Promise<ReturnType<typeof fn>>,
createWorkerImpl: redisUrl
? (queueName, processor, options) => {
const connection = parseRedisUrl(redisUrl);
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as AuditEventPersistWorkerHandle;
}
: undefined,
});
} catch (error) {
workerLogger.error({ error }, "Audit-event-persist bootstrap failed; continuing without subscriber.");
}
}
let chatRealtimeWakeup: ChatRealtimeWakeupBootstrapResult | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
chatRealtimeWakeup = bootstrapChatRealtimeWakeup(process.env, {
logger: workerLogger,
baseStore: getSharedStore(),
createWorkerImpl: (queueName, processor, options) => {
const connection = parseRedisUrl(redisUrl);
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as ChatRealtimeWakeupWorkerHandle;
},
});
} else {
workerLogger.info({}, "chat realtime wakeup worker skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "Chat realtime wakeup bootstrap failed; continuing without subscriber.");
}
}
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
chatLinkPreviewStatus = bootstrapChatLinkPreview(process.env, {
logger: workerLogger,
baseStore: getSharedStore(),
createWorkerImpl: (queueName, processor, options) => {
const connection = parseRedisUrl(redisUrl);
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as ChatLinkPreviewWorkerHandle;
},
});
} else {
chatLinkPreviewStatus = { status: "disabled", reason: "no Redis URL configured" };
workerLogger.info({}, "chat link preview worker skipped: no Redis URL configured.");
}
} catch (error) {
chatLinkPreviewStatus = { status: "disabled", reason: "bootstrap failed" };
workerLogger.error({ error }, "Chat link preview bootstrap failed; continuing without subscriber.");
}
}
let pluginHookInvokeQueue: BullMQQueue | undefined;
if (groupResolution?.group === "general") {
const pluginHookFanoutRequired = isPluginHookFanoutRequired(process.env, groupResolution.group);
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient();
const connection = parseRedisUrl(redisUrl);
pluginHookInvokeQueue = new BullMQQueue(queueNames.pluginInvoke, { connection });
pluginHookFanoutStatus = bootstrapPluginHookFanout(process.env, {
logger: workerLogger,
prisma,
pluginInvokeQueue: pluginHookInvokeQueue as never,
idempotency: createIdempotencyStore({
store: getSharedStore(),
prefix: "plugin-hook-fanout",
ttlSeconds: 60 * 60 * 24,
}),
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as PluginHookFanoutWorkerHandle;
},
});
if (pluginHookFanoutStatus.status === "disabled" && pluginHookFanoutRequired) {
workerLogger.error({ reason: pluginHookFanoutStatus.reason }, "Plugin-hook fanout disabled in required mode; exiting.");
process.exit(1);
}
} else if (pluginHookFanoutRequired) {
workerLogger.error({}, "Plugin-hook fanout required but Redis URL is not configured; exiting.");
process.exit(1);
} else {
workerLogger.info({}, "plugin-hook fanout skipped: no Redis URL configured.");
}
} catch (error) {
if (pluginHookFanoutRequired) {
workerLogger.error({ error }, "Plugin-hook fanout bootstrap failed in required mode; exiting.");
process.exit(1);
} else {
workerLogger.error({ error }, "Plugin-hook fanout bootstrap failed; continuing without subscriber.");
}
}
}
let pluginCapabilityUse: PluginCapabilityUseBootstrapResult | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const { getDbClient } = await import("@hivegrid/db/client.server");
const prisma = getDbClient();
const connection = parseRedisUrl(redisUrl);
pluginCapabilityUse = bootstrapPluginCapabilityUse(process.env, {
logger: workerLogger,
store: createPrismaPluginCapabilityUseStore(prisma as unknown as PluginCapabilityUseStorePrisma),
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as PluginCapabilityUseWorkerHandle;
},
});
} else {
workerLogger.info({}, "plugin capability-use worker skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "Plugin capability-use bootstrap failed; continuing without subscriber.");
}
}
// Phase 03 #42 — Legacy-event outbox receiver. Subscribes the
// `outbox.notification` queue on the `general` group and writes
// billing.subscription.* events into the legacy Event table on
// behalf of the migrated Mollie webhook (apps/api/src/billing/
// mollie-webhook-route.ts). Disabled when the group is non-general
// or no Redis URL is configured.
let legacyEventReceiver: LegacyEventReceiverBootstrapResult | undefined;
let platformNotificationQueue: TypedQueue<typeof queueNames.notifications> | undefined;
if (groupResolution?.group === "general") {
try {
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
if (redisUrl) {
const { createLegacyEvent } = await import("@hivegrid/db/events.server");
const connection = parseRedisUrl(redisUrl);
platformNotificationQueue = createQueue(queueNames.notifications, { connection });
const queue = platformNotificationQueue;
legacyEventReceiver = bootstrapLegacyEventReceiver(process.env, {
logger: workerLogger,
store: {
create: async (data) => {
const row = await createLegacyEvent(data);
return { id: row.id };
},
},
platformNotification: {
enqueue: async (input, options) => {
await enqueueJob(queue, buildPlatformNotificationJob(input), options);
},
logger: workerLogger,
},
createWorkerImpl: (queueName, processor, options) => {
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
connection,
concurrency: options.concurrency,
});
return worker as unknown as LegacyEventReceiverWorkerHandle;
},
});
} else {
workerLogger.info({}, "legacy-event receiver skipped: no Redis URL configured.");
}
} catch (error) {
workerLogger.error({ error }, "Legacy-event receiver bootstrap failed; continuing without subscriber.");
}
}
const shutdownState: ShutdownState = {
adminServer,
healthServer,
redisHealthProbe,
outbox,
adminBookingOutbox,
workflowTrigger,
searchIndexer,
formSubmissionRoute,
formSubmissionWebhookDelivery,
formSubmissionWebhookQueue,
transactionalMailOutbox,
portalStatusMail,
portalDomainOrchestration,
productAcquisitionProviderSideEffect,
recurringInvoiceCollection,
emailSequences,
auditEventPersist,
chatRealtimeWakeup,
chatLinkPreview: chatLinkPreviewStatus,
legacyEventReceiver,
platformNotificationQueue,
pluginHookFanout: pluginHookFanoutStatus,
pluginCapabilityUse,
pluginHookInvokeQueue,
pluginAuditRetentionWorker,
auditRetentionWorker,
filePreviewQueue,
telephonyWebhookQueue,
tenantDeletionExportQueue,
telephonyWebhookWorker,
automationExecutorWorker,
};
process.on("SIGINT", () => {
void shutdown(runtime, shutdownState, "SIGINT");
});
process.on("SIGTERM", () => {
void shutdown(runtime, shutdownState, "SIGTERM");
});
}
} catch (error) {
workerLogger.error({ error }, "Worker startup failed; see worker health diagnostics for dependency details.");
process.exit(1);
}
function isRequiredTenantHardDeleteRuntime(env: NodeJS.ProcessEnv): boolean {
return (
env.WORKER_QUEUE_GROUP === "general" &&
(env.NODE_ENV === "production" || env.RLS_RUNTIME_MODE === "required" || env.SENTRY_ENVIRONMENT === "staging" || env.SENTRY_ENVIRONMENT === "production")
);
}
function shutdown(runtimeToClose: WorkerRuntimeStatus, state: ShutdownState, signal: NodeJS.Signals): Promise<void> {
state.signal ??= signal;
state.promise ??= closeRuntime(
runtimeToClose,
state.signal,
state.adminServer,
state.healthServer,
state.redisHealthProbe,
state.outbox,
state.adminBookingOutbox,
state.workflowTrigger,
state.searchIndexer,
state.formSubmissionRoute,
state.formSubmissionWebhookDelivery,
state.formSubmissionWebhookQueue,
state.transactionalMailOutbox,
state.portalStatusMail,
state.portalDomainOrchestration,
state.productAcquisitionProviderSideEffect,
state.recurringInvoiceCollection,
state.emailSequences,
state.auditEventPersist,
state.chatRealtimeWakeup,
state.chatLinkPreview,
state.legacyEventReceiver,
state.platformNotificationQueue,
state.pluginHookFanout,
state.pluginCapabilityUse,
state.pluginHookInvokeQueue,
state.pluginAuditRetentionWorker,
state.auditRetentionWorker,
state.filePreviewQueue,
state.telephonyWebhookQueue,
state.tenantDeletionExportQueue,
state.telephonyWebhookWorker,
state.automationExecutorWorker
);
return state.promise;
}
async function closeRuntime(
runtimeToClose: WorkerRuntimeStatus,
signal: NodeJS.Signals,
adminServer?: WorkerAdminServerHandle,
healthServer?: WorkerHealthServerHandle,
redisHealthProbe?: RedisHealthProbe,
outbox?: OutboxDispatcherBootstrapResult,
adminBookingOutbox?: AdminBookingOutboxBootstrapResult,
workflowTrigger?: WorkflowTriggerBootstrapResult,
searchIndexer?: SearchIndexerBootstrapResult,
formSubmissionRoute?: FormSubmissionRouteBootstrapResult,
formSubmissionWebhookDelivery?: FormSubmissionWebhookDeliveryBootstrapResult,
formSubmissionWebhookQueue?: BullMQQueue,
transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult,
portalStatusMail?: PortalStatusMailBootstrapResult,
portalDomainOrchestration?: PortalDomainOrchestrationBootstrapResult,
productAcquisitionProviderSideEffect?: ProductAcquisitionProviderSideEffectBootstrapResult,
recurringInvoiceCollection?: RecurringInvoiceCollectionBootstrapResult,
emailSequences?: EmailSequenceBootstrapResult,
auditEventPersist?: AuditEventPersistBootstrapResult,
chatRealtimeWakeup?: ChatRealtimeWakeupBootstrapResult,
chatLinkPreview?: ChatLinkPreviewBootstrapResult,
legacyEventReceiver?: LegacyEventReceiverBootstrapResult,
platformNotificationQueue?: TypedQueue<typeof queueNames.notifications>,
pluginHookFanout?: PluginHookFanoutBootstrapResult,
pluginCapabilityUse?: PluginCapabilityUseBootstrapResult,
pluginHookInvokeQueue?: BullMQQueue,
pluginAuditRetentionWorker?: BullMQWorker,
auditRetentionWorker?: BullMQWorker,
filePreviewQueue?: TypedQueue<typeof queueNames.filePreview>,
telephonyWebhookQueue?: BullMQQueue<TelephonyWebhookProcessJob>,
tenantDeletionExportQueue?: BullMQQueue,
telephonyWebhookWorker?: BullMQWorker,
automationExecutorWorker?: BullMQWorker
): Promise<void> {
try {
if (outbox?.handle) {
try {
await outbox.handle.close();
} catch (outboxError) {
workerLogger.error({ error: outboxError, signal }, "Outbox dispatcher shutdown failed.");
}
}
if (adminBookingOutbox?.status === "started") {
try {
await adminBookingOutbox.handle.close();
} catch (bookingError) {
workerLogger.error({ error: bookingError, signal }, "Admin booking provider worker shutdown failed.");
}
}
if (workflowTrigger?.status === "started") {
try {
await workflowTrigger.handle.close();
} catch (workflowTriggerError) {
workerLogger.error({ error: workflowTriggerError, signal }, "Workflow-trigger worker shutdown failed.");
}
}
if (searchIndexer?.status === "started") {
try {
await searchIndexer.handle.close();
} catch (searchError) {
workerLogger.error({ error: searchError, signal }, "Search-indexer shutdown failed.");
}
}
if (formSubmissionRoute?.status === "started") {
try {
await formSubmissionRoute.handle.close();
} catch (formSubmissionError) {
workerLogger.error({ error: formSubmissionError, signal }, "Form-submission-route worker shutdown failed.");
}
}
if (formSubmissionWebhookDelivery?.status === "started") {
try {
await formSubmissionWebhookDelivery.handle.close();
} catch (formSubmissionWebhookError) {
workerLogger.error({ error: formSubmissionWebhookError, signal }, "Form-submission webhook bridge shutdown failed.");
}
}
if (transactionalMailOutbox?.status === "started") {
try {
await transactionalMailOutbox.handle.close();
} catch (transactionalMailOutboxError) {
workerLogger.error({ error: transactionalMailOutboxError, signal }, "Transactional mail outbox bridge shutdown failed.");
}
}
if (portalStatusMail?.status === "started") {
try {
await portalStatusMail.handle.close();
} catch (portalStatusMailError) {
workerLogger.error({ error: portalStatusMailError, signal }, "Portal status mail worker shutdown failed.");
}
}
if (portalDomainOrchestration?.status === "started") {
try {
await portalDomainOrchestration.close();
} catch (portalDomainError) {
workerLogger.error({ error: portalDomainError, signal }, "Portal-domain orchestration shutdown failed.");
}
}
if (productAcquisitionProviderSideEffect?.status === "started") {
try {
await productAcquisitionProviderSideEffect.handle.close();
} catch (providerSideEffectError) {
workerLogger.error({ error: providerSideEffectError, signal }, "Product-acquisition provider side-effect worker shutdown failed.");
}
}
if (recurringInvoiceCollection?.status === "started") {
try {
await recurringInvoiceCollection.handle.close();
} catch (recurringInvoiceError) {
workerLogger.error({ error: recurringInvoiceError, signal }, "Recurring invoice collection worker shutdown failed.");
}
}
if (formSubmissionWebhookQueue) {
try {
await formSubmissionWebhookQueue.close();
} catch (formSubmissionWebhookQueueError) {
workerLogger.error({ error: formSubmissionWebhookQueueError, signal }, "Form-submission webhook queue shutdown failed.");
}
}
if (emailSequences?.status === "started") {
try {
await emailSequences.triggerHandle.close();
await emailSequences.schedulerHandle.close();
} catch (seqError) {
workerLogger.error({ error: seqError, signal }, "Email-sequence workers shutdown failed.");
}
}
if (auditEventPersist?.status === "started") {
try {
await auditEventPersist.handle.close();
} catch (auditError) {
workerLogger.error({ error: auditError, signal }, "Audit-event-persist shutdown failed.");
}
}
if (chatRealtimeWakeup?.status === "started") {
try {
await chatRealtimeWakeup.handle.close();
} catch (chatRealtimeError) {
workerLogger.error({ error: chatRealtimeError, signal }, "Chat realtime wakeup worker shutdown failed.");
}
}
if (chatLinkPreview?.status === "started") {
try {
await chatLinkPreview.handle.close();
} catch (chatLinkPreviewError) {
workerLogger.error({ error: chatLinkPreviewError, signal }, "Chat link preview worker shutdown failed.");
}
}
if (legacyEventReceiver?.status === "started") {
try {
await legacyEventReceiver.handle.close();
} catch (legacyError) {
workerLogger.error({ error: legacyError, signal }, "Legacy-event receiver shutdown failed.");
}
}
if (platformNotificationQueue) {
try {
await platformNotificationQueue.close();
} catch (platformNotificationQueueError) {
workerLogger.error({ error: platformNotificationQueueError, signal }, "Platform notification queue shutdown failed.");
}
}
if (pluginHookFanout?.status === "started") {
try {
await pluginHookFanout.handle.close();
} catch (pluginHookError) {
workerLogger.error({ error: pluginHookError, signal }, "Plugin-hook fanout shutdown failed.");
}
}
if (pluginCapabilityUse?.status === "started") {
try {
await pluginCapabilityUse.handle.close();
} catch (pluginCapabilityError) {
workerLogger.error({ error: pluginCapabilityError, signal }, "Plugin capability-use worker shutdown failed.");
}
}
if (pluginHookInvokeQueue) {
try {
await pluginHookInvokeQueue.close();
} catch (pluginHookQueueError) {
workerLogger.error({ error: pluginHookQueueError, signal }, "Plugin-hook invoke queue shutdown failed.");
}
}
if (pluginAuditRetentionWorker) {
try {
await pluginAuditRetentionWorker.close();
} catch (retentionError) {
workerLogger.error({ error: retentionError, signal }, "plugin-audit-retention worker shutdown failed.");
}
}
if (auditRetentionWorker) {
try {
await auditRetentionWorker.close();
} catch (retentionError) {
workerLogger.error({ error: retentionError, signal }, "audit-retention worker shutdown failed.");
}
}
if (filePreviewQueue) {
try {
await filePreviewQueue.close();
} catch (previewQueueError) {
workerLogger.error({ error: previewQueueError, signal }, "File-preview producer queue shutdown failed.");
}
}
if (telephonyWebhookQueue) {
try {
await telephonyWebhookQueue.close();
} catch (telephonyWebhookQueueError) {
workerLogger.error({ error: telephonyWebhookQueueError, signal }, "Telephony webhook producer queue shutdown failed.");
}
}
if (tenantDeletionExportQueue) {
try {
await tenantDeletionExportQueue.close();
} catch (tenantDeletionExportQueueError) {
workerLogger.error({ error: tenantDeletionExportQueueError, signal }, "Tenant deletion export queue shutdown failed.");
}
}
if (telephonyWebhookWorker) {
try {
await telephonyWebhookWorker.close();
} catch (telephonyWebhookError) {
workerLogger.error({ error: telephonyWebhookError, signal }, "Telephony webhook worker shutdown failed.");
}
}
if (automationExecutorWorker) {
try {
await automationExecutorWorker.close();
} catch (automationExecutorError) {
workerLogger.error({ error: automationExecutorError, signal }, "Automation forward-executor worker shutdown failed.");
}
}
await shutdownWorkerRuntime(runtimeToClose, { logger: workerLogger, reason: signal, reporter: workerErrorReporter });
if (adminServer) {
await adminServer.close();
}
if (healthServer) {
await healthServer.close();
}
if (redisHealthProbe) {
await redisHealthProbe.close();
}
try {
await closeSharedStore();
} catch (sharedStoreError) {
workerLogger.error({ error: sharedStoreError, signal }, "Worker shared-store shutdown failed.");
}
process.exit(0);
} catch (error) {
workerLogger.error({ error, signal }, "Worker shutdown failed.");
process.exit(1);
}
}