3070 lines
152 KiB
TypeScript
3070 lines
152 KiB
TypeScript
// IMPORTANT: `./otel.js` MUST stay the very first import in this file.
|
|
// The OpenTelemetry NodeSDK can only patch BullMQ, Prisma, Redis and
|
|
// friends when it loads before those modules are required for the first
|
|
// time. The module is a no-op when OTEL_EXPORTER_OTLP_ENDPOINT is unset.
|
|
import "./otel.js";
|
|
|
|
import { ensureAndSyncProjectChannelIfEnabledWithTx } from "@hivegrid/chat/project-sync.server";
|
|
import {
|
|
DEFAULT_WORKER_ADMIN_PORT,
|
|
isWorkerMetricsEnabled,
|
|
startWorkerAdminServer,
|
|
type WorkerAdminServerHandle,
|
|
type WorkerStatusReport,
|
|
} from "./admin-server.js";
|
|
import { createWorkerConnection } from "./connection.js";
|
|
import { bootstrapWorkerGroup, InvalidWorkerQueueGroupError } from "./group-runtime.js";
|
|
import { DEFAULT_WORKER_HEALTH_PORT, startWorkerHealthServer, type HealthProbe, type WorkerHealthServerHandle } from "./health-server.js";
|
|
import { bootstrapOutboxDispatcher, type OutboxDispatcherBootstrapResult } from "./outbox/bootstrap.js";
|
|
import { isOutboxDispatcherRequired } from "./outbox/dispatcher.js";
|
|
import { bootstrapSearchIndexer, type SearchIndexerBootstrapResult, type SearchIndexerWorkerHandle } from "./jobs/search-indexer-bootstrap.js";
|
|
import {
|
|
bootstrapFormSubmissionRoute,
|
|
type FormSubmissionRouteBootstrapResult,
|
|
type FormSubmissionRouteWorkerHandle,
|
|
} from "./jobs/form-submission-route-bootstrap.js";
|
|
import { runStartDoubleOptInFormAction, type StartDoubleOptInPrisma } from "./jobs/form-submission-actions.js";
|
|
import {
|
|
bootstrapFormSubmissionWebhookDelivery,
|
|
type FormSubmissionWebhookDeliveryBootstrapResult,
|
|
type FormSubmissionWebhookDeliveryWorkerHandle,
|
|
} from "./jobs/form-submission-webhook-delivery-bootstrap.js";
|
|
import {
|
|
bootstrapTransactionalMailOutbox,
|
|
isTransactionalMailOutboxRequired,
|
|
type TransactionalMailOutboxBootstrapResult,
|
|
type TransactionalMailOutboxWorkerHandle,
|
|
} from "./jobs/transactional-mail-outbox-bootstrap.js";
|
|
import {
|
|
cancelCampaignScheduleWithPrisma,
|
|
isCampaignDeliveryAllowedWithPrisma,
|
|
type CampaignScheduleCancellationPrisma,
|
|
} from "./jobs/campaign-schedule-cancellation.js";
|
|
import { bootstrapPortalStatusMail, type PortalStatusMailBootstrapResult, type PortalStatusMailWorkerHandle } from "./jobs/portal-status-mail-bootstrap.js";
|
|
import { bootstrapPortalDomainOrchestration, type PortalDomainOrchestrationBootstrapResult } from "./jobs/portal-domain-orchestration-bootstrap.js";
|
|
import {
|
|
bootstrapProductAcquisitionProviderSideEffect,
|
|
type ProductAcquisitionProviderSideEffectBootstrapResult,
|
|
} from "./jobs/product-acquisition-provider-side-effect-bootstrap.js";
|
|
import { bootstrapRecurringInvoiceCollection, type RecurringInvoiceCollectionBootstrapResult } from "./jobs/recurring-invoice-collection-bootstrap.js";
|
|
import {
|
|
claimProductProviderSideEffectOutboxEvent,
|
|
releaseProductProviderSideEffectOutboxClaim,
|
|
runProductProviderSideEffectWithReceipt,
|
|
} from "./jobs/product-acquisition-provider-side-effect-receipt.js";
|
|
import { assertProductAcquisitionRuntimeEnv, resolveProductAcquisitionProviderSideEffectMode } from "./jobs/product-acquisition-runtime-env.js";
|
|
import { createPrismaPortalStatusMailStore, resolvePortalStatusMailBaseUrl, type PortalStatusMailPrisma } from "./jobs/portal-status-mail-store.js";
|
|
import {
|
|
bootstrapWorkflowTrigger,
|
|
isWorkflowTriggerOutboxRequired,
|
|
type WorkflowTriggerBootstrapResult,
|
|
type WorkflowTriggerWorkerHandle,
|
|
} from "./jobs/workflow-trigger-bootstrap.js";
|
|
import type { QueueJsonValue, TelephonyWebhookProcessJob, TypedQueue } from "@hivegrid/queue";
|
|
import {
|
|
createWorkflowTriggerExecutorFromEnv,
|
|
parseWorkflowTriggerExecutorMode,
|
|
type WorkflowTriggerExecutorMode,
|
|
} from "./jobs/workflow-trigger-executor-factory.js";
|
|
import type { WorkflowTriggerNativePrisma } from "./jobs/workflow-trigger-native-executor.js";
|
|
import { createPrismaWorkflowTriggerStore, type WorkflowTriggerStorePrisma } from "./jobs/workflow-trigger-store.js";
|
|
import { runPortalAutomationTriggerJob } from "./jobs/portal-automation.js";
|
|
import { AUTOMATION_EXECUTOR_OUTBOX_QUEUE, createAutomationExecutorProcessor, createRunAutomationForwardTrigger } from "./jobs/automation-forward.js";
|
|
import { bootstrapEmailSequenceWorkers, type EmailSequenceBootstrapResult, type EmailSequenceWorkerHandle } from "./jobs/email-sequence-bootstrap.js";
|
|
import { bootstrapAuditEventPersist, type AuditEventPersistBootstrapResult, type AuditEventPersistWorkerHandle } from "./jobs/audit-event-persist-bootstrap.js";
|
|
import {
|
|
bootstrapChatRealtimeWakeup,
|
|
type ChatRealtimeWakeupBootstrapResult,
|
|
type ChatRealtimeWakeupWorkerHandle,
|
|
} from "./jobs/chat-realtime-wakeup-bootstrap.js";
|
|
import { bootstrapChatLinkPreview, type ChatLinkPreviewBootstrapResult, type ChatLinkPreviewWorkerHandle } from "./jobs/chat-link-preview-bootstrap.js";
|
|
import {
|
|
bootstrapLegacyEventReceiver,
|
|
type LegacyEventReceiverBootstrapResult,
|
|
type LegacyEventReceiverWorkerHandle,
|
|
} from "./jobs/legacy-event-receiver-bootstrap.js";
|
|
import {
|
|
bootstrapPluginHookFanout,
|
|
isPluginHookFanoutRequired,
|
|
type PluginHookFanoutBootstrapResult,
|
|
type PluginHookFanoutWorkerHandle,
|
|
} from "./jobs/plugin-hook-fanout-bootstrap.js";
|
|
import {
|
|
bootstrapPluginCapabilityUse,
|
|
type PluginCapabilityUseBootstrapResult,
|
|
type PluginCapabilityUseWorkerHandle,
|
|
} from "./jobs/plugin-capability-use-bootstrap.js";
|
|
import { createPrismaPluginCapabilityUseStore, type PluginCapabilityUseStorePrisma } from "./jobs/plugin-capability-use.js";
|
|
import { registerDsgvoGraceCron } from "./jobs/dsgvo-grace-cron.js";
|
|
import { registerOutboxDoneRetentionCron } from "./jobs/outbox-done-retention-cron.js";
|
|
import { registerPrivacyGdprExportRetentionCron } from "./jobs/privacy-gdpr-export-retention-cron.js";
|
|
import { registerPortalGdprExportRetentionCron } from "./jobs/portal-gdpr-export-retention-cron.js";
|
|
import { registerWorkflowExecutionStalledReconcilerCron } from "./jobs/workflow-execution-stalled-reconciler-cron.js";
|
|
import { registerAutomationSchedulerCron } from "./jobs/automation-scheduler-cron.js";
|
|
import { registerTaskArchiveRetentionCron } from "./jobs/task-archive-retention-cron.js";
|
|
import { registerLeadRetentionCron } from "./jobs/lead-retention-cron.js";
|
|
import { registerProjectAutoArchiveCron } from "./jobs/project-auto-archive-cron.js";
|
|
import { registerBillingDunningSweepCron } from "./jobs/billing-dunning-cron.js";
|
|
import {
|
|
registerIntegrationsHealthSweepCron,
|
|
registerIntegrationsTokenRefreshCron,
|
|
registerKwpProjectMaintenanceCron,
|
|
} from "./jobs/integrations-maintenance-cron.js";
|
|
import { runAllTenantKwpProjectMaintenance } from "./jobs/kwp-project-maintenance.js";
|
|
import {
|
|
registerCalendarInternalWriteSyncCron,
|
|
registerCalendarTokenRefreshCron,
|
|
registerCalendarWebhookDrainCron,
|
|
registerCalendarWebhookRenewalCron,
|
|
} from "./jobs/calendar-maintenance-cron.js";
|
|
import {
|
|
registerEmailInboxEventCleanupCron,
|
|
registerEmailInboxInitialSyncCron,
|
|
registerEmailInboxStalePollCron,
|
|
registerEmailInboxTokenRefreshCron,
|
|
registerEmailInboxWebhookDrainCron,
|
|
registerEmailInboxWebhookRenewalCron,
|
|
} from "./jobs/email-inbox-maintenance-cron.js";
|
|
import { registerTaskReminderSweepCron } from "./jobs/task-reminder-cron.js";
|
|
import { registerAdminSalesTaskReminderSweepCron } from "./jobs/admin-sales-task-reminder-cron.js";
|
|
import { registerAdminEmailDeliveryCron } from "./jobs/admin-email-delivery-cron.js";
|
|
import { registerAdminEmailEventRetentionCron } from "./jobs/admin-email-event-retention-cron.js";
|
|
import { registerAdminEmailSyncCron } from "./jobs/admin-email-sync-cron.js";
|
|
import { registerAdminEmailDomainCron } from "./jobs/admin-email-domain-cron.js";
|
|
import { registerAdminCallRecordingCrons } from "./jobs/admin-call-recording-cron.js";
|
|
import { registerAdminOutboundCallCron } from "./jobs/admin-outbound-call-cron.js";
|
|
import { registerAdminBookingReminderSweepCron } from "./jobs/admin-booking-reminder-cron.js";
|
|
import { registerAdminCalendarSyncCron } from "./jobs/admin-calendar-sync-cron.js";
|
|
import {
|
|
bootstrapAdminBookingOutbox,
|
|
type AdminBookingOutboxBootstrapResult,
|
|
type AdminBookingOutboxWorkerHandle,
|
|
} from "./jobs/admin-booking-outbox-bootstrap.js";
|
|
import { registerHrUploadReminderDispatchCron } from "./jobs/hr-upload-reminder-cron.js";
|
|
import { registerEmailDomainProbeCron } from "./jobs/email-domain-probe-cron.js";
|
|
import { registerTelephonyWebhookMaintenanceCron, runAllTenantTelephonyWebhookMaintenance } from "./jobs/telephony-webhook-maintenance.js";
|
|
import { registerProductAcquisitionReconcilerCron } from "./jobs/product-acquisition-reconciler-cron.js";
|
|
import {
|
|
PRODUCT_ACQUISITION_RECONCILE_JOB_ID,
|
|
asProductAcquisitionReconcileClient,
|
|
runAllTenantProductAcquisitionReconciliation,
|
|
} from "./jobs/product-acquisition-reconciler.js";
|
|
import {
|
|
PLUGIN_AUDIT_RETENTION_QUEUE,
|
|
registerPluginAuditRetentionCron,
|
|
resolvePluginAuditRetentionConfig,
|
|
runPluginAuditRetention,
|
|
} from "./jobs/plugin-audit-retention.js";
|
|
import { AUDIT_RETENTION_QUEUE, registerAuditRetentionCron, runAuditRetentionJob } from "./jobs/audit-retention.js";
|
|
import { createWorkflowExecutionStalledReconciler, type WorkflowExecutionStalledReconcilerPrisma } from "./jobs/workflow-execution-stalled-reconciler.js";
|
|
import type { HandlerDependencies } from "./handlers.js";
|
|
import { DelayedError as BullMQDelayedError, Queue as BullMQQueue, Worker as BullMQWorker } from "bullmq";
|
|
import {
|
|
TELEPHONY_WEBHOOK_QUEUE_NAME,
|
|
buildFilePreviewJob,
|
|
buildFileScanJob,
|
|
buildKwpProjectCatalogSyncJob,
|
|
buildPlatformNotificationJob,
|
|
buildTelephonyWebhookJob,
|
|
enqueueJob,
|
|
enqueueTelephonyWebhookJob,
|
|
parseRedisUrl,
|
|
queueNames,
|
|
recoverFailedTelephonyWebhookJob,
|
|
} from "@hivegrid/queue";
|
|
import { createIdempotencyStore } from "@hivegrid/shared-store";
|
|
import { resolvePlatformLeadNotificationRecipient } from "@hivegrid/db/platform-lead-notifications.server";
|
|
import type { SentryLikeForWorkerTracing } from "./runtime.js";
|
|
import { shutdownWorkerRuntime, startWorkerRuntime, type WorkerRuntimeStatus } from "./runtime.js";
|
|
import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js";
|
|
import { runWorkerPayloadWithRlsContext } from "./rls-context.js";
|
|
import { createQueue, createWorker } from "./queue-runtime.js";
|
|
import type { Prisma } from "@hivegrid/db/prisma-browser";
|
|
import { workerLogger } from "./logger.js";
|
|
import { getWorkerMetrics } from "./metrics.js";
|
|
import { workerErrorReporter } from "./sentry.js";
|
|
import { closeSharedStore, getSharedStore } from "./shared-store.js";
|
|
import { createWorkerStorageDeleteService } from "./storage.js";
|
|
import { buildKwpArchiveDiscoveryEnqueue, buildKwpArchiveFileEnqueue } from "./kwp-archive-queue.js";
|
|
|
|
const SCHEDULED_JOB_DEFAULT_OPTIONS = {
|
|
removeOnComplete: 100,
|
|
removeOnFail: 1_000,
|
|
} as const;
|
|
function createRlsAwareBullMqProcessor<Job extends { data: unknown }>(processor: (job: Job) => Promise<unknown>) {
|
|
return async (job: Job) => runWorkerPayloadWithRlsContext(job.data, () => processor(job));
|
|
}
|
|
|
|
type ShutdownState = {
|
|
promise?: Promise<void>;
|
|
signal?: NodeJS.Signals;
|
|
adminServer?: WorkerAdminServerHandle | undefined;
|
|
healthServer?: WorkerHealthServerHandle | undefined;
|
|
redisHealthProbe?: RedisHealthProbe | undefined;
|
|
outbox?: OutboxDispatcherBootstrapResult | undefined;
|
|
adminBookingOutbox?: AdminBookingOutboxBootstrapResult | undefined;
|
|
workflowTrigger?: WorkflowTriggerBootstrapResult | undefined;
|
|
searchIndexer?: SearchIndexerBootstrapResult | undefined;
|
|
formSubmissionRoute?: FormSubmissionRouteBootstrapResult | undefined;
|
|
formSubmissionWebhookDelivery?: FormSubmissionWebhookDeliveryBootstrapResult | undefined;
|
|
formSubmissionWebhookQueue?: BullMQQueue | undefined;
|
|
transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult | undefined;
|
|
portalStatusMail?: PortalStatusMailBootstrapResult | undefined;
|
|
portalDomainOrchestration?: PortalDomainOrchestrationBootstrapResult | undefined;
|
|
productAcquisitionProviderSideEffect?: ProductAcquisitionProviderSideEffectBootstrapResult | undefined;
|
|
recurringInvoiceCollection?: RecurringInvoiceCollectionBootstrapResult | undefined;
|
|
emailSequences?: EmailSequenceBootstrapResult | undefined;
|
|
auditEventPersist?: AuditEventPersistBootstrapResult | undefined;
|
|
chatRealtimeWakeup?: ChatRealtimeWakeupBootstrapResult | undefined;
|
|
chatLinkPreview?: ChatLinkPreviewBootstrapResult | undefined;
|
|
legacyEventReceiver?: LegacyEventReceiverBootstrapResult | undefined;
|
|
platformNotificationQueue?: TypedQueue<typeof queueNames.notifications> | undefined;
|
|
pluginHookFanout?: PluginHookFanoutBootstrapResult | undefined;
|
|
pluginCapabilityUse?: PluginCapabilityUseBootstrapResult | undefined;
|
|
pluginHookInvokeQueue?: BullMQQueue | undefined;
|
|
pluginAuditRetentionWorker?: BullMQWorker | undefined;
|
|
auditRetentionWorker?: BullMQWorker | undefined;
|
|
filePreviewQueue?: TypedQueue<typeof queueNames.filePreview> | undefined;
|
|
telephonyWebhookQueue?: BullMQQueue<TelephonyWebhookProcessJob> | undefined;
|
|
tenantDeletionExportQueue?: BullMQQueue | undefined;
|
|
telephonyWebhookWorker?: BullMQWorker | undefined;
|
|
automationExecutorWorker?: BullMQWorker | undefined;
|
|
};
|
|
|
|
type RedisHealthProbeResult = { ok: true } | { ok: false; error: string };
|
|
type RedisHealthProbe = (() => Promise<RedisHealthProbeResult>) & { close: () => Promise<void> };
|
|
|
|
function buildWorkerStatus(
|
|
runtime: WorkerRuntimeStatus,
|
|
options?: {
|
|
outboxDispatcher?: OutboxDispatcherBootstrapResult;
|
|
outboxDispatcherExpected?: boolean;
|
|
workflowTrigger?: WorkflowTriggerBootstrapResult;
|
|
workflowTriggerExpected?: boolean;
|
|
workflowTriggerExecutorMode?: WorkflowTriggerExecutorMode;
|
|
pluginHookFanout?: PluginHookFanoutBootstrapResult;
|
|
pluginHookFanoutExpected?: boolean;
|
|
transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult;
|
|
transactionalMailOutboxExpected?: boolean;
|
|
portalStatusMail?: PortalStatusMailBootstrapResult;
|
|
portalStatusMailExpected?: boolean;
|
|
emailSequences?: EmailSequenceBootstrapResult;
|
|
emailSequencesExpected?: boolean;
|
|
chatLinkPreview?: ChatLinkPreviewBootstrapResult;
|
|
chatLinkPreviewExpected?: boolean;
|
|
}
|
|
): WorkerStatusReport {
|
|
const runtimeCheck = runtime.status === "started" ? "ok" : "down";
|
|
const outboxDispatcherCheck = options?.outboxDispatcher?.status === "started" ? "ok" : options?.outboxDispatcherExpected ? "down" : "disabled";
|
|
const workflowTriggerCheck = options?.workflowTrigger?.status === "started" ? "ok" : options?.workflowTriggerExpected ? "down" : "disabled";
|
|
const pluginHookFanoutCheck = options?.pluginHookFanout?.status === "started" ? "ok" : options?.pluginHookFanoutExpected ? "down" : "disabled";
|
|
const transactionalMailOutboxCheck =
|
|
options?.transactionalMailOutbox?.status === "started" ? "ok" : options?.transactionalMailOutboxExpected ? "down" : "disabled";
|
|
const portalStatusMailCheck = options?.portalStatusMail?.status === "started" ? "ok" : options?.portalStatusMailExpected ? "down" : "disabled";
|
|
const emailSequencesCheck = options?.emailSequences?.status === "started" ? "ok" : options?.emailSequencesExpected ? "down" : "disabled";
|
|
const chatLinkPreviewCheck = options?.chatLinkPreview?.status === "started" ? "ok" : options?.chatLinkPreviewExpected ? "down" : "disabled";
|
|
const status =
|
|
runtimeCheck === "ok" &&
|
|
outboxDispatcherCheck !== "down" &&
|
|
workflowTriggerCheck !== "down" &&
|
|
pluginHookFanoutCheck !== "down" &&
|
|
transactionalMailOutboxCheck !== "down" &&
|
|
portalStatusMailCheck !== "down" &&
|
|
emailSequencesCheck !== "down" &&
|
|
chatLinkPreviewCheck !== "down"
|
|
? "ok"
|
|
: "down";
|
|
return {
|
|
status,
|
|
service: "worker",
|
|
checks: {
|
|
runtime: runtimeCheck,
|
|
outboxDispatcher: outboxDispatcherCheck,
|
|
workflowTrigger: workflowTriggerCheck,
|
|
pluginHookFanout: pluginHookFanoutCheck,
|
|
transactionalMailOutbox: transactionalMailOutboxCheck,
|
|
portalStatusMail: portalStatusMailCheck,
|
|
emailSequences: emailSequencesCheck,
|
|
chatLinkPreview: chatLinkPreviewCheck,
|
|
},
|
|
...(workflowTriggerCheck !== "disabled" && options?.workflowTriggerExecutorMode
|
|
? { workflowTriggerExecutorMode: options.workflowTriggerExecutorMode }
|
|
: {}),
|
|
version: process.env.APP_REVISION?.trim() || "unknown",
|
|
uptime_seconds: Math.round(process.uptime()),
|
|
region: process.env.APP_REGION?.trim() || process.env.FLY_REGION?.trim() || "unknown",
|
|
};
|
|
}
|
|
|
|
function createRedisHealthProbe(env: NodeJS.ProcessEnv): RedisHealthProbe {
|
|
const redisUrl = env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim();
|
|
let queue: import("bullmq").Queue | undefined;
|
|
let readyQueue: Promise<import("bullmq").Queue> | undefined;
|
|
|
|
const probe = async (): Promise<RedisHealthProbeResult> => {
|
|
if (!redisUrl) {
|
|
return { ok: false, error: "Redis URL not configured" };
|
|
}
|
|
try {
|
|
readyQueue ??= (async () => {
|
|
const { Queue } = await import("bullmq");
|
|
const nextQueue = new Queue("__hivegrid_worker_health__", { connection: parseRedisUrl(redisUrl) });
|
|
queue = nextQueue;
|
|
await nextQueue.waitUntilReady();
|
|
return nextQueue;
|
|
})();
|
|
await readyQueue;
|
|
return { ok: true };
|
|
} catch (error) {
|
|
const failedQueue = queue;
|
|
queue = undefined;
|
|
readyQueue = undefined;
|
|
await failedQueue?.close().catch(() => undefined);
|
|
return { ok: false, error: error instanceof Error ? error.name : "RedisProbeFailed" };
|
|
}
|
|
};
|
|
|
|
probe.close = async () => {
|
|
const queueToClose = queue;
|
|
queue = undefined;
|
|
readyQueue = undefined;
|
|
await queueToClose?.close();
|
|
};
|
|
|
|
return probe;
|
|
}
|
|
|
|
function createDatabaseHealthProbe(): () => Promise<{ ok: true } | { ok: false; error: string }> {
|
|
return async () => {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const db = getDbClient();
|
|
await db.$queryRaw`SELECT 1`;
|
|
return { ok: true };
|
|
} catch (error) {
|
|
return { ok: false, error: error instanceof Error ? error.name : "DatabaseProbeFailed" };
|
|
}
|
|
};
|
|
}
|
|
|
|
function createRuntimeReadinessProbes({
|
|
workerGroup,
|
|
outboxDispatcherExpected,
|
|
workflowTriggerExpected,
|
|
pluginHookFanoutExpected,
|
|
transactionalMailOutboxExpected,
|
|
portalStatusMailExpected,
|
|
emailSequencesExpected,
|
|
chatLinkPreviewExpected,
|
|
getOutboxDispatcherStatus,
|
|
getWorkflowTriggerStatus,
|
|
getPluginHookFanoutStatus,
|
|
getTransactionalMailOutboxStatus,
|
|
getPortalStatusMailStatus,
|
|
getEmailSequencesStatus,
|
|
getChatLinkPreviewStatus,
|
|
}: {
|
|
workerGroup?: string;
|
|
outboxDispatcherExpected: boolean;
|
|
workflowTriggerExpected: boolean;
|
|
pluginHookFanoutExpected: boolean;
|
|
transactionalMailOutboxExpected: boolean;
|
|
portalStatusMailExpected: boolean;
|
|
emailSequencesExpected: boolean;
|
|
chatLinkPreviewExpected: boolean;
|
|
getOutboxDispatcherStatus: () => OutboxDispatcherBootstrapResult | undefined;
|
|
getWorkflowTriggerStatus: () => WorkflowTriggerBootstrapResult | undefined;
|
|
getPluginHookFanoutStatus: () => PluginHookFanoutBootstrapResult | undefined;
|
|
getTransactionalMailOutboxStatus: () => TransactionalMailOutboxBootstrapResult | undefined;
|
|
getPortalStatusMailStatus: () => PortalStatusMailBootstrapResult | undefined;
|
|
getEmailSequencesStatus: () => EmailSequenceBootstrapResult | undefined;
|
|
getChatLinkPreviewStatus: () => ChatLinkPreviewBootstrapResult | undefined;
|
|
}): Record<string, HealthProbe> | undefined {
|
|
if (workerGroup !== "general" && workerGroup !== "mail") {
|
|
return undefined;
|
|
}
|
|
|
|
const probes: Record<string, HealthProbe> = {};
|
|
if (outboxDispatcherExpected) {
|
|
probes.outboxDispatcher = () =>
|
|
getOutboxDispatcherStatus()?.status === "started" ? { ok: true } : { ok: false, error: "outbox dispatcher is not started" };
|
|
}
|
|
if (workflowTriggerExpected) {
|
|
probes.workflowTrigger = () =>
|
|
getWorkflowTriggerStatus()?.status === "started" ? { ok: true } : { ok: false, error: "workflow trigger subscriber is not started" };
|
|
}
|
|
if (pluginHookFanoutExpected) {
|
|
probes.pluginHookFanout = () =>
|
|
getPluginHookFanoutStatus()?.status === "started" ? { ok: true } : { ok: false, error: "plugin hook fanout subscriber is not started" };
|
|
}
|
|
if (transactionalMailOutboxExpected) {
|
|
probes.transactionalMailOutbox = () =>
|
|
getTransactionalMailOutboxStatus()?.status === "started" ? { ok: true } : { ok: false, error: "transactional mail outbox subscriber is not started" };
|
|
}
|
|
if (portalStatusMailExpected) {
|
|
probes.portalStatusMail = () =>
|
|
getPortalStatusMailStatus()?.status === "started" ? { ok: true } : { ok: false, error: "portal status mail subscriber is not started" };
|
|
}
|
|
if (emailSequencesExpected) {
|
|
probes.emailSequences = () =>
|
|
getEmailSequencesStatus()?.status === "started" ? { ok: true } : { ok: false, error: "email-sequence workers are not started" };
|
|
}
|
|
if (chatLinkPreviewExpected) {
|
|
probes.chatLinkPreview = () =>
|
|
getChatLinkPreviewStatus()?.status === "started" ? { ok: true } : { ok: false, error: "chat link preview subscriber is not started" };
|
|
}
|
|
return Object.keys(probes).length > 0 ? probes : undefined;
|
|
}
|
|
|
|
function hasWorkflowTriggerOutboxRedis(env: NodeJS.ProcessEnv): boolean {
|
|
return Boolean(env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim());
|
|
}
|
|
|
|
function isChatLinkPreviewExpected(env: NodeJS.ProcessEnv, workerGroup: string | undefined): boolean {
|
|
return workerGroup === "general" && env.CHAT_LINK_PREVIEW_ENABLED !== "false";
|
|
}
|
|
|
|
function isWorkflowTriggerOutboxExpected(env: NodeJS.ProcessEnv): boolean {
|
|
if (isWorkflowTriggerOutboxRequired(env) || env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true") {
|
|
return true;
|
|
}
|
|
|
|
return env.WORKFLOW_TRIGGER_OUTBOX_ENABLED !== "false" && hasWorkflowTriggerOutboxRedis(env);
|
|
}
|
|
|
|
function isOutboxDispatcherExpected(env: NodeJS.ProcessEnv): boolean {
|
|
return env.OUTBOX_DISPATCHER_ENABLED === "true" || isOutboxDispatcherRequired(env) || isWorkflowTriggerOutboxExpected(env);
|
|
}
|
|
|
|
function isPluginHookFanoutExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean {
|
|
if (workerGroup !== "general") {
|
|
return false;
|
|
}
|
|
return env.PLUGIN_HOOK_FANOUT_ENABLED === "true" || isPluginHookFanoutRequired(env, workerGroup);
|
|
}
|
|
|
|
function isTransactionalMailOutboxExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean {
|
|
if (workerGroup !== "mail") {
|
|
return false;
|
|
}
|
|
return env.TRANSACTIONAL_MAIL_OUTBOX_ENABLED !== "false" || isTransactionalMailOutboxRequired(env, workerGroup);
|
|
}
|
|
|
|
function isPortalStatusMailExpected(workerGroup?: string): boolean {
|
|
return workerGroup === "mail";
|
|
}
|
|
|
|
function isEmailSequencesExpected(env: NodeJS.ProcessEnv, workerGroup?: string): boolean {
|
|
return workerGroup === "mail" && env.MARKETING_SEQUENCES_ENABLED === "true";
|
|
}
|
|
|
|
function isPluginInvokeRuntimeEnabled(env: NodeJS.ProcessEnv): boolean {
|
|
return env.NODE_ENV !== "production" || env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true";
|
|
}
|
|
|
|
function resolvePositiveInteger(value: string | undefined, fallback: number): number {
|
|
const parsed = Number(value);
|
|
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
}
|
|
|
|
/**
|
|
* Lazy-load @sentry/node for queue.process span tracing. When the SDK is
|
|
* not installed or SENTRY_DSN_WORKER is empty, the wrapper degrades to a
|
|
* pass-through processor automatically.
|
|
*/
|
|
async function loadSentryForWorker(): Promise<SentryLikeForWorkerTracing | undefined> {
|
|
if (!process.env.SENTRY_DSN_WORKER) return undefined;
|
|
try {
|
|
const moduleName = "@sentry/node";
|
|
const loaded = (await import(/* @vite-ignore */ moduleName)) as SentryLikeForWorkerTracing & {
|
|
default?: SentryLikeForWorkerTracing;
|
|
};
|
|
if (typeof loaded?.startSpan === "function") return loaded;
|
|
if (typeof loaded?.default?.startSpan === "function") return loaded.default;
|
|
return undefined;
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
try {
|
|
const sentryForTracing = await loadSentryForWorker();
|
|
const { metrics, registry } = getWorkerMetrics();
|
|
// Validate WORKER_QUEUE_GROUP before we start anything else. An invalid
|
|
// value must abort boot loudly — we don't want a half-started worker
|
|
// silently subscribing to the wrong queues. Disabled workers (eg local
|
|
// dev where WORKER_ENABLED=false) skip the group check.
|
|
// Only require WORKER_QUEUE_GROUP when the worker is actually enabled.
|
|
// Disabled workers (eg local dev where WORKER_ENABLED!=='true') skip
|
|
// group resolution so they can boot a no-op process for testing
|
|
// adjacent code paths.
|
|
let groupResolution: ReturnType<typeof bootstrapWorkerGroup> | undefined;
|
|
if (process.env.WORKER_ENABLED === "true") {
|
|
try {
|
|
groupResolution = bootstrapWorkerGroup(process.env);
|
|
} catch (error) {
|
|
if (error instanceof InvalidWorkerQueueGroupError) {
|
|
workerLogger.error({ error: { message: error.message } }, "Worker boot rejected: invalid WORKER_QUEUE_GROUP.");
|
|
process.exit(1);
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
const { assertRlsRuntimeReady } = await import("@hivegrid/db/rls-runtime-readiness.server");
|
|
await assertRlsRuntimeReady(undefined, {
|
|
...process.env,
|
|
DATABASE_RUNTIME_ROLE: process.env.DATABASE_RUNTIME_ROLE?.trim() || "hivegrid_worker",
|
|
});
|
|
assertProductAcquisitionRuntimeEnv(process.env);
|
|
}
|
|
|
|
// Incoming-call browser pushes are part of the webhook worker contract.
|
|
// Validate the provider before starting the queue runtime or readiness
|
|
// server so a broken VAPID configuration cannot produce a ready worker
|
|
// that silently drops ringing notifications.
|
|
let telephonyWebPushProvider: import("./jobs/push-notification.js").PushProviderSeam | undefined;
|
|
if (groupResolution?.group === "webhook") {
|
|
const { createWebPushProvider } = await import("./jobs/web-push-provider.js");
|
|
telephonyWebPushProvider = createWebPushProvider({ env: process.env });
|
|
}
|
|
|
|
// The legacy runtime entrypoint reads WORKER_CONCURRENCY from env. We
|
|
// funnel the resolved per-group concurrency through that same channel so
|
|
// the existing config validation + diagnostics path keeps working.
|
|
const runtimeEnv: NodeJS.ProcessEnv = groupResolution ? { ...process.env, WORKER_CONCURRENCY: String(groupResolution.concurrency) } : process.env;
|
|
|
|
// When a group is resolved we override the BullMQ worker factory so the
|
|
// ioredis connection is tagged `worker-<group>-<pid>` (visible in Redis
|
|
// CLIENT LIST for operator attribution) and so the per-group pool size
|
|
// is applied. The legacy path (no group) keeps the original factory.
|
|
const groupAwareCreateWorker = groupResolution
|
|
? <Name extends Parameters<typeof createWorker>[0]>(
|
|
queueName: Name,
|
|
processor: Parameters<typeof createWorker<Name>>[1],
|
|
options: Parameters<typeof createWorker<Name>>[2]
|
|
) => {
|
|
const augmented = createWorkerConnection(groupResolution.group, { env: process.env as NodeJS.ProcessEnv });
|
|
// BullMQ's WorkerOptions.connection union accepts plain connection
|
|
// options OR a Redis instance. We always pass plain options here
|
|
// (parseRedisUrl output), so a cast keeps the spread narrowed.
|
|
const mergedConnection = { ...(options.connection as Record<string, unknown>), ...augmented } as Parameters<typeof createWorker<Name>>[2]["connection"];
|
|
return createWorker(queueName, processor, {
|
|
...options,
|
|
connection: mergedConnection,
|
|
});
|
|
}
|
|
: undefined;
|
|
|
|
const extraHandlerDependencies: Partial<HandlerDependencies> = {};
|
|
extraHandlerDependencies.resolvePlatformLeadNotificationRecipient = resolvePlatformLeadNotificationRecipient;
|
|
if (groupResolution?.group === "general" || groupResolution?.group === "import") {
|
|
extraHandlerDependencies.tenantMaintenanceLeaseStore = getSharedStore();
|
|
}
|
|
let filePreviewQueue: TypedQueue<typeof queueNames.filePreview> | undefined;
|
|
let telephonyWebhookQueue: BullMQQueue<TelephonyWebhookProcessJob> | undefined;
|
|
let tenantDeletionExportQueue: BullMQQueue | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (redisUrl) {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
extraHandlerDependencies.runKwpProjectMaintenance = async () => {
|
|
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
|
|
try {
|
|
return await runAllTenantKwpProjectMaintenance({
|
|
prisma: getDbClient() as unknown as Parameters<typeof runAllTenantKwpProjectMaintenance>[0]["prisma"],
|
|
queue,
|
|
});
|
|
} finally {
|
|
await queue.close();
|
|
}
|
|
};
|
|
workerLogger.info({}, "KWP project maintenance scheduler service bootstrapped (general group).");
|
|
}
|
|
}
|
|
if (groupResolution?.group === "import") {
|
|
const [{ syncKwpProjectBinding }, { syncKwpProjectInbound }, { discoverKwpProjectArchiveBatch }, { syncKwpArchiveFile }, { syncKwpProjectCatalog }, { runKwpAllContractSentinels }] =
|
|
await Promise.all([
|
|
import("@hivegrid/integrations/kwp-project-sync.server"),
|
|
import("@hivegrid/integrations/kwp-inbound-sync.server"),
|
|
import("@hivegrid/integrations/kwp-archive-discovery.server"),
|
|
import("@hivegrid/integrations/kwp-archive-file.server"),
|
|
import("@hivegrid/integrations/kwp-project-catalog.server"),
|
|
import("@hivegrid/integrations/kwp-contract-sentinel.server"),
|
|
]);
|
|
extraHandlerDependencies.syncKwpProjectBinding = (payload) => syncKwpProjectBinding({ bindingId: payload.bindingId });
|
|
extraHandlerDependencies.syncKwpProjectInbound = (payload) => syncKwpProjectInbound({ bindingId: payload.bindingId });
|
|
extraHandlerDependencies.syncKwpArchiveDiscovery = async (payload) => {
|
|
const result = await discoverKwpProjectArchiveBatch({
|
|
bindingId: payload.bindingId,
|
|
...(payload.cursor ? { cursor: payload.cursor } : {}),
|
|
});
|
|
return {
|
|
...result,
|
|
cursor: result.cursor
|
|
? {
|
|
pendingFolderIds: [...result.cursor.pendingFolderIds],
|
|
visitedFolderIds: [...result.cursor.visitedFolderIds],
|
|
}
|
|
: null,
|
|
};
|
|
};
|
|
extraHandlerDependencies.syncKwpProjectCatalog = (payload) => syncKwpProjectCatalog({ connectionId: payload.dataConnectionId });
|
|
extraHandlerDependencies.enqueueKwpProjectCatalogContinuation = async (payload) => {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (!redisUrl) {
|
|
throw new Error("KWP project catalog continuation requires BULLMQ_REDIS_URL or REDIS_URL.");
|
|
}
|
|
const continuationBucket = `${payload.cadenceBucket}-${Date.now()}`;
|
|
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
|
|
try {
|
|
const descriptor = buildKwpProjectCatalogSyncJob({
|
|
tenantId: payload.tenantId,
|
|
dataConnectionId: payload.dataConnectionId,
|
|
cadenceBucket: continuationBucket,
|
|
});
|
|
await enqueueJob(queue, descriptor, {
|
|
idempotencyKey: `kwp-project-catalog:${payload.tenantId}:${payload.dataConnectionId}:${continuationBucket}`,
|
|
tenantId: payload.tenantId,
|
|
delay: 60_000,
|
|
attempts: 5,
|
|
backoff: { type: "exponential", delay: 30_000 },
|
|
});
|
|
} finally {
|
|
await queue.close();
|
|
}
|
|
};
|
|
extraHandlerDependencies.enqueueKwpArchiveDiscovery = async (payload, cursor) => {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (!redisUrl) {
|
|
throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL.");
|
|
}
|
|
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
|
|
try {
|
|
const { descriptor, options } = buildKwpArchiveDiscoveryEnqueue({ payload, cursor, now: new Date() });
|
|
await enqueueJob(queue, descriptor, options);
|
|
} finally {
|
|
await queue.close();
|
|
}
|
|
};
|
|
extraHandlerDependencies.enqueueKwpArchiveFileJobs = async (payload, files) => {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (!redisUrl) {
|
|
throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL.");
|
|
}
|
|
const queue = createQueue(queueNames.kwpImports, { connection: parseRedisUrl(redisUrl) });
|
|
try {
|
|
for (const archiveFile of files) {
|
|
const { descriptor, options } = buildKwpArchiveFileEnqueue(payload, archiveFile);
|
|
await enqueueJob(queue, descriptor, options);
|
|
}
|
|
} finally {
|
|
await queue.close();
|
|
}
|
|
return files.length;
|
|
};
|
|
extraHandlerDependencies.syncKwpArchiveFile = (payload) => {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (!redisUrl) {
|
|
throw new Error("KWP provider import services require BULLMQ_REDIS_URL or REDIS_URL.");
|
|
}
|
|
const storageBucket = process.env.STORAGE_BUCKET?.trim();
|
|
if (!storageBucket) {
|
|
throw new Error("KWP archive import requires STORAGE_BUCKET.");
|
|
}
|
|
return syncKwpArchiveFile({
|
|
bindingId: payload.bindingId,
|
|
archiveFile: payload.archiveFile,
|
|
storage: createWorkerStorageDeleteService(process.env),
|
|
storageBucket,
|
|
enqueueScan: async (input) => {
|
|
const queue = createQueue(queueNames.fileScan, { connection: parseRedisUrl(redisUrl) });
|
|
try {
|
|
const descriptor = buildFileScanJob({
|
|
bucket: input.bucket,
|
|
key: input.key,
|
|
tenantId: input.tenantId,
|
|
fileId: input.fileId,
|
|
contentType: input.contentType,
|
|
metadata: { source: "kwp" },
|
|
});
|
|
await enqueueJob(queue, descriptor, {
|
|
idempotencyKey: `kwp-archive-scan:${input.tenantId}:${input.fileId}:${payload.cadenceBucket}`,
|
|
tenantId: input.tenantId,
|
|
attempts: 5,
|
|
backoff: { type: "exponential", delay: 30_000 },
|
|
});
|
|
} finally {
|
|
await queue.close();
|
|
}
|
|
},
|
|
});
|
|
};
|
|
extraHandlerDependencies.runKwpContractSentinel = (payload) => runKwpAllContractSentinels({ connectionId: payload.dataConnectionId });
|
|
workerLogger.info({}, "KWP provider import services bootstrapped (import group).");
|
|
}
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (!redisUrl) {
|
|
throw new Error("Telephony webhook maintenance requires BULLMQ_REDIS_URL or REDIS_URL.");
|
|
}
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const queue = new BullMQQueue<TelephonyWebhookProcessJob>(TELEPHONY_WEBHOOK_QUEUE_NAME, {
|
|
connection: parseRedisUrl(redisUrl),
|
|
});
|
|
telephonyWebhookQueue = queue;
|
|
extraHandlerDependencies.runTelephonyWebhookMaintenance = () =>
|
|
runAllTenantTelephonyWebhookMaintenance(getDbClient(), async (input, source) => {
|
|
const descriptor = buildTelephonyWebhookJob(input);
|
|
if (source === "failed") {
|
|
await recoverFailedTelephonyWebhookJob(queue, descriptor);
|
|
} else {
|
|
await enqueueTelephonyWebhookJob(queue, descriptor);
|
|
}
|
|
});
|
|
workerLogger.info({ queue: TELEPHONY_WEBHOOK_QUEUE_NAME }, "telephony webhook maintenance service bootstrapped (general group).");
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "telephony webhook maintenance bootstrap failed; maintenance jobs will retry until fixed.");
|
|
}
|
|
}
|
|
if (groupResolution?.group === "general") {
|
|
const providerSideEffectMode = resolveProductAcquisitionProviderSideEffectMode(process.env);
|
|
if (providerSideEffectMode === "disabled") {
|
|
workerLogger.info({}, "product acquisition reconciler disabled with provider side effects.");
|
|
} else {
|
|
try {
|
|
const apiKey = process.env.MOLLIE_API_KEY?.trim();
|
|
const lagoApiUrl = process.env.LAGO_API_URL?.trim();
|
|
const lagoApiKey = process.env.LAGO_API_KEY?.trim();
|
|
if (!apiKey) throw new Error("Product acquisition reconciliation requires MOLLIE_API_KEY.");
|
|
if (!lagoApiUrl || !lagoApiKey) throw new Error("Product acquisition reconciliation requires Lago configuration.");
|
|
const [{ createLagoBillingGateway, createMolliePaymentGateway }, { applyMolliePaymentSync }, { getDbClient }] = await Promise.all([
|
|
import("@hivegrid/billing"),
|
|
import("@hivegrid/db/billing-sync.server"),
|
|
import("@hivegrid/db/client.server"),
|
|
]);
|
|
const prisma = asProductAcquisitionReconcileClient(getDbClient());
|
|
const mollie = createMolliePaymentGateway({
|
|
apiKey,
|
|
profileId: process.env.MOLLIE_PROFILE_ID?.trim() || null,
|
|
timeoutMs: resolvePositiveInteger(process.env.MOLLIE_TIMEOUT_MS, 10_000),
|
|
});
|
|
const lago = createLagoBillingGateway({
|
|
apiUrl: lagoApiUrl,
|
|
apiKey: lagoApiKey,
|
|
timeoutMs: resolvePositiveInteger(process.env.LAGO_TIMEOUT_MS, 10_000),
|
|
});
|
|
extraHandlerDependencies.runProductAcquisitionReconcile = ({ take } = {}) =>
|
|
runAllTenantProductAcquisitionReconciliation(
|
|
{
|
|
prisma,
|
|
mollie,
|
|
lago,
|
|
applyPaymentSync: (input) => applyMolliePaymentSync(input, getDbClient() as unknown as NonNullable<Parameters<typeof applyMolliePaymentSync>[1]>),
|
|
alert: async (alert) => {
|
|
const details = {
|
|
code: alert.code,
|
|
tenantId: alert.tenantId,
|
|
acquisitionId: alert.acquisitionId,
|
|
status: alert.status,
|
|
providerPaymentId: alert.providerPaymentId,
|
|
};
|
|
workerLogger.error(details, "billing product acquisition reconciliation requires operator attention");
|
|
await workerErrorReporter.report({
|
|
component: "worker",
|
|
operation: "billing.product-acquisition-reconcile.quarantined",
|
|
severity: "error",
|
|
...details,
|
|
});
|
|
},
|
|
},
|
|
{ batchSize: take }
|
|
);
|
|
workerLogger.info({}, "product acquisition reconciler bootstrapped (general group).");
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "product acquisition reconciler bootstrap failed; maintenance jobs will retry until fixed.");
|
|
}
|
|
}
|
|
}
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
extraHandlerDependencies.reconcileWorkflowExecutionStalls = createWorkflowExecutionStalledReconciler({
|
|
prisma: getDbClient() as unknown as WorkflowExecutionStalledReconcilerPrisma,
|
|
});
|
|
workerLogger.info({}, "workflow-execution-stalled-reconciler service bootstrapped (general group).");
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "workflow-execution-stalled-reconciler bootstrap failed; maintenance handler will use DB fallback.");
|
|
}
|
|
}
|
|
|
|
// AK-P4a §4 — file-preview generator service on the `general` group
|
|
// (the group that owns the hivegrid.file-preview queue). Composed with a
|
|
// Prisma client + STORAGE_* S3 seams; injected via extraHandlerDependencies.
|
|
// When STORAGE_* is unset the factory returns undefined and the handler
|
|
// dead-letters preview jobs.
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const { createWorkerFilePreviewService } = await import("./jobs/file-preview.js");
|
|
const previewService = createWorkerFilePreviewService(process.env as Parameters<typeof createWorkerFilePreviewService>[0], {
|
|
prisma: getDbClient() as unknown as Parameters<typeof createWorkerFilePreviewService>[1]["prisma"],
|
|
logger: workerLogger,
|
|
});
|
|
if (previewService) {
|
|
extraHandlerDependencies.filePreviewService = previewService;
|
|
workerLogger.info({}, "file-preview service bootstrapped (general group).");
|
|
} else {
|
|
workerLogger.info({}, "file-preview service skipped: STORAGE_* not configured.");
|
|
}
|
|
|
|
// AK-P4a §5 — scan-verdict write-back. The general group owns the
|
|
// hivegrid.file-scan queue, so wire the File.scanStatus writer here with
|
|
// the same Prisma seam the preview service uses.
|
|
const { createWorkerFileScanStatusWriter } = await import("./jobs/file-scan-status.js");
|
|
extraHandlerDependencies.fileScanStatusWriter = createWorkerFileScanStatusWriter({
|
|
prisma: getDbClient() as unknown as Parameters<typeof createWorkerFileScanStatusWriter>[0]["prisma"],
|
|
logger: workerLogger,
|
|
});
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (!redisUrl) {
|
|
throw new Error("Redis is required to enqueue previews after clean file scans.");
|
|
}
|
|
filePreviewQueue = createQueue(queueNames.filePreview, { connection: parseRedisUrl(redisUrl) });
|
|
extraHandlerDependencies.filePreviewEnqueuer = {
|
|
enqueueAfterCleanScan: async (input) => {
|
|
const descriptor = buildFilePreviewJob({
|
|
bucket: input.bucket,
|
|
fileId: input.fileId,
|
|
mimeType: input.contentType,
|
|
storageKey: input.storageKey,
|
|
tenantId: input.tenantId,
|
|
});
|
|
await enqueueJob(filePreviewQueue!, descriptor, {
|
|
jobId: `file-preview-after-scan-${input.fileId}`,
|
|
tenantId: input.tenantId,
|
|
});
|
|
},
|
|
};
|
|
workerLogger.info({}, "file-scan status writer bootstrapped (general group).");
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "file-preview bootstrap failed; preview jobs will dead-letter until fixed.");
|
|
}
|
|
}
|
|
|
|
// Phase 06 #21 / F-015 — wire the real plugin-invoke dependencies on
|
|
// the `plugin` worker group. Without this, `handlers.ts` throws when a
|
|
// plugin-invoke job arrives — which is the intentional fail-loud
|
|
// behaviour that F-015 introduced to replace the prior silent no-op
|
|
// stub. The composition lives in jobs/plugin-invoke-bootstrap.ts.
|
|
const pluginInvokeRuntimeEnabled = groupResolution?.group === "plugin" ? isPluginInvokeRuntimeEnabled(process.env) : true;
|
|
if (groupResolution?.group === "plugin" && pluginInvokeRuntimeEnabled) {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const { bootstrapPluginInvokeDeps } = await import("./jobs/plugin-invoke-bootstrap.js");
|
|
const { getSharedStore } = await import("./shared-store.js");
|
|
const prisma = getDbClient() as unknown as Parameters<typeof bootstrapPluginInvokeDeps>[0]["prisma"];
|
|
extraHandlerDependencies.pluginInvokeDeps = bootstrapPluginInvokeDeps({
|
|
env: process.env,
|
|
prisma,
|
|
store: getSharedStore(),
|
|
logger: workerLogger,
|
|
});
|
|
workerLogger.info({}, "plugin-invoke production deps bootstrapped (plugin group).");
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "plugin-invoke bootstrap failed in plugin worker group; exiting.");
|
|
throw error;
|
|
}
|
|
if (!extraHandlerDependencies.pluginInvokeDeps) {
|
|
workerLogger.error({}, "plugin-invoke bootstrap produced no dependencies in plugin worker group; exiting.");
|
|
process.exit(1);
|
|
throw new Error("plugin-invoke bootstrap produced no dependencies");
|
|
}
|
|
} else if (groupResolution?.group === "plugin") {
|
|
workerLogger.warn(
|
|
{ workerGroup: groupResolution.group, queueNames: groupResolution.queueNames },
|
|
"plugin-invoke runtime disabled before production signoff; starting plugin worker idle without queue consumers."
|
|
);
|
|
}
|
|
|
|
// The `import` group always owns the dedicated KWP queue. Generic imports and
|
|
// exports remain separately gated: full mode adds both, while the internal
|
|
// portal-GDPR mode adds only exports and wires only the two GDPR services.
|
|
const importQueueRuntimeMode = resolveImportQueueRuntimeMode(process.env);
|
|
const importQueueRuntimeEnabled = importQueueRuntimeMode === "full" || importQueueRuntimeMode === "portal-gdpr-export";
|
|
if (groupResolution?.group === "import" && importQueueRuntimeMode === "idle") {
|
|
workerLogger.warn(
|
|
{ workerGroup: groupResolution.group, queueNames: groupResolution.queueNames },
|
|
"Generic import/export runtime gated off; starting the import worker with the dedicated KWP queue only."
|
|
);
|
|
}
|
|
|
|
if (groupResolution?.group === "import" && importQueueRuntimeEnabled) {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
if (importQueueRuntimeMode === "full") {
|
|
const { createAdminSalesDataWorkerService } = await import("./jobs/admin-sales-data.js");
|
|
const adminSalesDataService = createAdminSalesDataWorkerService(process.env, {
|
|
prisma: getDbClient() as unknown as Parameters<typeof createAdminSalesDataWorkerService>[1]["prisma"],
|
|
});
|
|
if (adminSalesDataService) {
|
|
extraHandlerDependencies.adminSalesDataService = adminSalesDataService;
|
|
workerLogger.info({}, "admin sales import/export service bootstrapped (import group).");
|
|
}
|
|
}
|
|
const { createWorkerPrivacyGdprExportService } = await import("./jobs/privacy-gdpr-export-bootstrap.js");
|
|
const privacyGdprExportService = createWorkerPrivacyGdprExportService(process.env, {
|
|
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPrivacyGdprExportService>[1]["prisma"],
|
|
sharedStore: getSharedStore(),
|
|
});
|
|
if (privacyGdprExportService) {
|
|
extraHandlerDependencies.privacyGdprExportService = privacyGdprExportService;
|
|
workerLogger.info({}, "privacy GDPR export service bootstrapped (import group).");
|
|
// AK-P21 — best-effort "export ready" mail via the email queue. Both
|
|
// import runtime modes own the privacy GDPR export handler, so the
|
|
// mailer must not stay gated behind the `full` mode: in
|
|
// `portal-gdpr-export` mode the very same job finished silently.
|
|
const mailRedisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (mailRedisUrl) {
|
|
const { createGdprExportReadyMailer } = await import("./jobs/privacy-gdpr-export-bootstrap.js");
|
|
const { buildNotificationJob, buildRetryJobOptions } = await import("@hivegrid/queue");
|
|
// AGENTS.md DoD C15 — go through the M011 notification queue, not the
|
|
// raw transactional mail queue: the notification handler owns the
|
|
// preference check, the NotificationDelivery record and the provider
|
|
// call.
|
|
const notificationQueue = new BullMQQueue(queueNames.notifications, { connection: parseRedisUrl(mailRedisUrl) });
|
|
extraHandlerDependencies.sendGdprExportReadyMail = createGdprExportReadyMailer({
|
|
prisma: getDbClient() as unknown as Parameters<typeof createGdprExportReadyMailer>[0]["prisma"],
|
|
enqueueNotification: async (input) => {
|
|
const descriptor = buildNotificationJob(input);
|
|
await notificationQueue.add(descriptor.jobName, descriptor.payload, buildRetryJobOptions(descriptor));
|
|
},
|
|
...(process.env.APP_URL ? { appUrl: process.env.APP_URL } : {}),
|
|
logger: workerLogger,
|
|
});
|
|
workerLogger.info({}, "privacy GDPR export ready-mailer bootstrapped (import group).");
|
|
}
|
|
} else {
|
|
workerLogger.info({}, "privacy GDPR export service skipped: STORAGE_* not configured.");
|
|
}
|
|
|
|
const { createWorkerPortalGdprExportService } = await import("./jobs/portal-gdpr-export-bootstrap.js");
|
|
const portalGdprExportService = createWorkerPortalGdprExportService(process.env, {
|
|
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPortalGdprExportService>[1]["prisma"],
|
|
sharedStore: getSharedStore(),
|
|
logger: workerLogger,
|
|
});
|
|
if (portalGdprExportService) {
|
|
extraHandlerDependencies.portalGdprExportService = portalGdprExportService;
|
|
workerLogger.info({}, "portal GDPR JSON export service bootstrapped (import group).");
|
|
} else {
|
|
workerLogger.info({}, "portal GDPR JSON export service skipped: STORAGE_* not configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "privacy GDPR export bootstrap failed; GDPR export jobs will dead-letter until fixed.");
|
|
}
|
|
}
|
|
|
|
// AK-P21 — tenant hard-delete service (general group owns the maintenance
|
|
// queue). Local dev/test may explicitly skip external cleanup, while
|
|
// required staging/production runtimes fail closed before queue ownership.
|
|
if (groupResolution?.group === "general") {
|
|
const tenantHardDeleteRequired = isRequiredTenantHardDeleteRuntime(process.env);
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const { createWorkerPrivacyTenantDeletionService } = await import("./jobs/privacy-tenant-deletion-bootstrap.js");
|
|
if (tenantHardDeleteRequired && (!process.env.MEILI_URL?.trim() || !process.env.MEILI_API_KEY?.trim())) {
|
|
throw new Error("Required tenant hard-delete search cleanup configuration is missing.");
|
|
}
|
|
let searchClient;
|
|
if (process.env.MEILI_URL) {
|
|
const { createSearchClient } = await import("@hivegrid/search");
|
|
searchClient = createSearchClient({ url: process.env.MEILI_URL, apiKey: process.env.MEILI_API_KEY ?? undefined });
|
|
}
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (redisUrl) {
|
|
tenantDeletionExportQueue = new BullMQQueue(queueNames.exports, { connection: parseRedisUrl(redisUrl) });
|
|
}
|
|
extraHandlerDependencies.privacyTenantDeletionService = createWorkerPrivacyTenantDeletionService(process.env, {
|
|
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPrivacyTenantDeletionService>[1]["prisma"],
|
|
...(searchClient ? { searchClient } : {}),
|
|
...(tenantDeletionExportQueue ? { exportQueue: tenantDeletionExportQueue } : {}),
|
|
...(redisUrl ? { sharedStore: getSharedStore() } : {}),
|
|
logger: workerLogger,
|
|
});
|
|
workerLogger.info({}, "privacy tenant-deletion service bootstrapped (general group).");
|
|
} catch (error) {
|
|
if (tenantHardDeleteRequired) {
|
|
workerLogger.error(
|
|
{ errorName: error instanceof Error ? error.name : "UnknownError" },
|
|
"privacy tenant-deletion bootstrap failed in required mode; exiting."
|
|
);
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required tenant hard-delete bootstrap failure.");
|
|
}
|
|
workerLogger.error({ errorName: error instanceof Error ? error.name : "UnknownError" }, "privacy tenant-deletion bootstrap failed; continuing.");
|
|
}
|
|
}
|
|
|
|
// R-P2b — DSGVO grace-period cron service. Without this block the daily
|
|
// scan acks every tick as "not configured" and confirmed user deletion
|
|
// requests are never executed (found in the 2026-07-17 ticket audit).
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const { createWorkerDsgvoGraceCronService } = await import("./jobs/dsgvo-grace-bootstrap.js");
|
|
extraHandlerDependencies.dsgvoGraceCronService = createWorkerDsgvoGraceCronService({
|
|
prisma: getDbClient() as unknown as Parameters<typeof createWorkerDsgvoGraceCronService>[0]["prisma"],
|
|
logger: workerLogger,
|
|
});
|
|
workerLogger.info({}, "dsgvo grace-cron service bootstrapped (general group).");
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "dsgvo grace-cron bootstrap failed; grace scans will skip until fixed.");
|
|
}
|
|
}
|
|
|
|
// VA-P26b — PortalUser lifecycle is isolated from the internal User/TenantUser
|
|
// privacy path. The general group owns the bounded daily maintenance scan.
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const { createWorkerPortalLifecycleService } = await import("./jobs/portal-lifecycle-bootstrap.js");
|
|
extraHandlerDependencies.portalLifecycleService = createWorkerPortalLifecycleService(process.env, {
|
|
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPortalLifecycleService>[1]["prisma"],
|
|
logger: workerLogger,
|
|
});
|
|
workerLogger.info({}, "portal lifecycle service bootstrapped (general group).");
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "portal lifecycle bootstrap failed; maintenance jobs will retry until fixed.");
|
|
}
|
|
}
|
|
|
|
const runtimeQueueNames = groupResolution
|
|
? !pluginInvokeRuntimeEnabled
|
|
? []
|
|
: groupResolution.group === "import"
|
|
? getWorkerQueueNames(process.env)
|
|
: groupResolution.queueNames
|
|
: undefined;
|
|
const runtime = startWorkerRuntime({
|
|
logger: workerLogger,
|
|
reporter: workerErrorReporter,
|
|
sentry: sentryForTracing,
|
|
metrics,
|
|
env: runtimeEnv,
|
|
extraHandlerDependencies,
|
|
...(runtimeQueueNames ? { queueNames: runtimeQueueNames } : {}),
|
|
...(groupAwareCreateWorker ? { createWorkerImpl: groupAwareCreateWorker } : {}),
|
|
});
|
|
|
|
if (groupResolution) {
|
|
workerLogger.info(
|
|
{
|
|
workerGroup: groupResolution.group,
|
|
queueNames: runtime.queueNames,
|
|
concurrency: groupResolution.concurrency,
|
|
},
|
|
"Worker queue group resolved."
|
|
);
|
|
}
|
|
|
|
let outboxStatus: OutboxDispatcherBootstrapResult | undefined;
|
|
let workflowTriggerStatus: WorkflowTriggerBootstrapResult | undefined;
|
|
let pluginHookFanoutStatus: PluginHookFanoutBootstrapResult | undefined;
|
|
let transactionalMailOutboxStatus: TransactionalMailOutboxBootstrapResult | undefined;
|
|
let portalStatusMailStatus: PortalStatusMailBootstrapResult | undefined;
|
|
let emailSequencesStatus: EmailSequenceBootstrapResult | undefined;
|
|
let chatLinkPreviewStatus: ChatLinkPreviewBootstrapResult | undefined;
|
|
const outboxDispatcherExpected = groupResolution?.group === "general" && isOutboxDispatcherExpected(process.env);
|
|
const workflowTriggerExpected = groupResolution?.group === "general" && isWorkflowTriggerOutboxExpected(process.env);
|
|
const pluginHookFanoutExpected = isPluginHookFanoutExpected(process.env, groupResolution?.group);
|
|
const transactionalMailOutboxExpected = isTransactionalMailOutboxExpected(process.env, groupResolution?.group);
|
|
const portalStatusMailExpected = isPortalStatusMailExpected(groupResolution?.group);
|
|
const emailSequencesExpected = isEmailSequencesExpected(process.env, groupResolution?.group);
|
|
const chatLinkPreviewExpected = isChatLinkPreviewExpected(process.env, groupResolution?.group);
|
|
const adminServer = isWorkerMetricsEnabled(process.env)
|
|
? startWorkerAdminServer({
|
|
enabled: true,
|
|
token: process.env.METRICS_TOKEN,
|
|
registry,
|
|
port: Number.parseInt(process.env.WORKER_ADMIN_PORT ?? "", 10) || DEFAULT_WORKER_ADMIN_PORT,
|
|
host: process.env.WORKER_ADMIN_HOST,
|
|
logger: workerLogger,
|
|
buildStatus: () =>
|
|
buildWorkerStatus(runtime, {
|
|
outboxDispatcher: outboxStatus,
|
|
outboxDispatcherExpected,
|
|
workflowTrigger: workflowTriggerStatus,
|
|
workflowTriggerExpected,
|
|
workflowTriggerExecutorMode: parseWorkflowTriggerExecutorMode(process.env.WORKFLOW_TRIGGER_EXECUTOR_MODE),
|
|
pluginHookFanout: pluginHookFanoutStatus,
|
|
pluginHookFanoutExpected,
|
|
transactionalMailOutbox: transactionalMailOutboxStatus,
|
|
transactionalMailOutboxExpected,
|
|
portalStatusMail: portalStatusMailStatus,
|
|
portalStatusMailExpected,
|
|
emailSequences: emailSequencesStatus,
|
|
emailSequencesExpected,
|
|
chatLinkPreview: chatLinkPreviewStatus,
|
|
chatLinkPreviewExpected,
|
|
}),
|
|
})
|
|
: undefined;
|
|
|
|
// Health server: always on when a runtime started. Bound to a separate
|
|
// port from the admin server so liveness probes still work when
|
|
// METRICS_ENABLED is false (eg early-stage staging).
|
|
const redisHealthProbe = runtime.status === "started" ? createRedisHealthProbe(process.env) : undefined;
|
|
const healthServer =
|
|
runtime.status === "started"
|
|
? startWorkerHealthServer({
|
|
port: Number.parseInt(process.env.WORKER_HEALTH_PORT ?? "", 10) || DEFAULT_WORKER_HEALTH_PORT,
|
|
host: process.env.WORKER_HEALTH_HOST,
|
|
...(groupResolution ? { workerGroup: groupResolution.group } : {}),
|
|
redisProbe: redisHealthProbe!,
|
|
databaseProbe: createDatabaseHealthProbe(),
|
|
...(() => {
|
|
const readinessProbes = createRuntimeReadinessProbes({
|
|
workerGroup: groupResolution?.group,
|
|
outboxDispatcherExpected,
|
|
workflowTriggerExpected,
|
|
pluginHookFanoutExpected,
|
|
transactionalMailOutboxExpected,
|
|
portalStatusMailExpected,
|
|
emailSequencesExpected,
|
|
chatLinkPreviewExpected,
|
|
getOutboxDispatcherStatus: () => outboxStatus,
|
|
getWorkflowTriggerStatus: () => workflowTriggerStatus,
|
|
getPluginHookFanoutStatus: () => pluginHookFanoutStatus,
|
|
getTransactionalMailOutboxStatus: () => transactionalMailOutboxStatus,
|
|
getPortalStatusMailStatus: () => portalStatusMailStatus,
|
|
getEmailSequencesStatus: () => emailSequencesStatus,
|
|
getChatLinkPreviewStatus: () => chatLinkPreviewStatus,
|
|
});
|
|
return readinessProbes ? { readinessProbes } : {};
|
|
})(),
|
|
logger: workerLogger,
|
|
})
|
|
: undefined;
|
|
|
|
if (runtime.status === "disabled") {
|
|
if (telephonyWebhookQueue) {
|
|
await telephonyWebhookQueue.close();
|
|
}
|
|
if (tenantDeletionExportQueue) {
|
|
await tenantDeletionExportQueue.close();
|
|
}
|
|
if (adminServer) {
|
|
await adminServer.close();
|
|
}
|
|
if (healthServer) {
|
|
await healthServer.close();
|
|
}
|
|
process.exit(0);
|
|
} else {
|
|
let outbox: OutboxDispatcherBootstrapResult | undefined;
|
|
let adminBookingOutbox: AdminBookingOutboxBootstrapResult | undefined;
|
|
let portalDomainOrchestration: PortalDomainOrchestrationBootstrapResult | undefined;
|
|
let productAcquisitionProviderSideEffect: ProductAcquisitionProviderSideEffectBootstrapResult | undefined;
|
|
let recurringInvoiceCollection: RecurringInvoiceCollectionBootstrapResult | undefined;
|
|
let telephonyWebhookWorker: BullMQWorker | undefined;
|
|
let automationExecutorWorker: BullMQWorker | undefined;
|
|
if (groupResolution?.group === "webhook") {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (!redisUrl) {
|
|
throw new Error("Telephony webhook worker requires BULLMQ_REDIS_URL or REDIS_URL.");
|
|
}
|
|
const [
|
|
{ getDbClient },
|
|
{ listActiveDeviceTokens, revokeDeviceToken },
|
|
{ createTelephonyWebhookJobProcessor },
|
|
{ dispatchPushNotification },
|
|
{ withTenantRLS },
|
|
{ buildTelephonyPushHref },
|
|
] = await Promise.all([
|
|
import("@hivegrid/db/client.server"),
|
|
import("@hivegrid/db/device-tokens.server"),
|
|
import("./jobs/telephony-webhook-bootstrap.js"),
|
|
import("./jobs/push-notification.js"),
|
|
import("./db/tenantRls.js"),
|
|
import("./jobs/telephony-push-target.js"),
|
|
]);
|
|
const prisma = getDbClient();
|
|
const webPushProvider = telephonyWebPushProvider;
|
|
if (!webPushProvider) {
|
|
throw new Error("Telephony webhook worker started without a validated Web Push provider.");
|
|
}
|
|
const processor = createTelephonyWebhookJobProcessor(prisma, {
|
|
pushIncomingCall: async (input) => {
|
|
const tenantSlug = await withTenantRLS(prisma, input.tenantId, async (tx) => {
|
|
const tenant = await tx.tenant.findFirst({
|
|
where: { id: input.tenantId },
|
|
select: { slug: true },
|
|
});
|
|
if (!tenant) throw new Error("Telephony push tenant is unavailable.");
|
|
return tenant.slug;
|
|
});
|
|
|
|
await dispatchPushNotification({
|
|
payload: {
|
|
tenantId: input.tenantId,
|
|
userId: input.userId,
|
|
title: "HiveGrid",
|
|
body: input.externalNumber,
|
|
data: {
|
|
callId: input.callId,
|
|
href: buildTelephonyPushHref({
|
|
tenantSlug,
|
|
callId: input.callId,
|
|
target: input.target,
|
|
}),
|
|
messageKey: "telephony.notifications.incomingCall",
|
|
},
|
|
},
|
|
tokenLister: ({ userId, tenantId }) =>
|
|
withTenantRLS(prisma, input.tenantId, (tx) =>
|
|
listActiveDeviceTokens(
|
|
{
|
|
userId,
|
|
...(tenantId ? { tenantId } : {}),
|
|
},
|
|
tx as never
|
|
)
|
|
),
|
|
tokenRevoker: async (token) => {
|
|
await withTenantRLS(prisma, input.tenantId, (tx) =>
|
|
revokeDeviceToken(
|
|
{
|
|
token,
|
|
userId: input.userId,
|
|
tenantId: input.tenantId,
|
|
},
|
|
tx as never
|
|
)
|
|
);
|
|
},
|
|
providers: webPushProvider,
|
|
logger: workerLogger,
|
|
});
|
|
},
|
|
});
|
|
telephonyWebhookWorker = new BullMQWorker<TelephonyWebhookProcessJob>(TELEPHONY_WEBHOOK_QUEUE_NAME, createRlsAwareBullMqProcessor(processor), {
|
|
connection: parseRedisUrl(redisUrl),
|
|
concurrency: groupResolution.concurrency,
|
|
});
|
|
workerLogger.info({ queue: TELEPHONY_WEBHOOK_QUEUE_NAME, concurrency: groupResolution.concurrency }, "telephony webhook worker started");
|
|
}
|
|
// The outbox dispatcher is a cron-style polling job that lives on the
|
|
// `general` group. Running it from every worker group would either (a)
|
|
// multiply Redis/Postgres load by N replicas or (b) race for the same
|
|
// FOR UPDATE SKIP LOCKED rows. When the group is undefined (legacy
|
|
// WORKER_ENABLED=false path) we leave the dispatcher off too.
|
|
const shouldRunOutbox = groupResolution?.group === "general";
|
|
const outboxRequiredForGroup = shouldRunOutbox && (isOutboxDispatcherRequired(process.env) || isWorkflowTriggerOutboxRequired(process.env));
|
|
if (shouldRunOutbox) {
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient();
|
|
outbox = await bootstrapOutboxDispatcher(process.env, {
|
|
prisma,
|
|
logger: workerLogger,
|
|
reporter: workerErrorReporter,
|
|
});
|
|
outboxStatus = outbox;
|
|
if (outbox.status === "disabled" && outboxRequiredForGroup) {
|
|
workerLogger.error({ reason: "OUTBOX_DISPATCHER_ENABLED!=true" }, "Outbox dispatcher disabled in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required outbox dispatcher disabled.");
|
|
}
|
|
} catch (error) {
|
|
outboxStatus = { status: "disabled" };
|
|
if (outboxRequiredForGroup) {
|
|
workerLogger.error({ error }, "Outbox dispatcher bootstrap failed in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required outbox dispatcher failure.");
|
|
}
|
|
workerLogger.error({ error }, "Outbox dispatcher bootstrap failed; continuing without dispatcher.");
|
|
}
|
|
} else if (groupResolution) {
|
|
workerLogger.info({ workerGroup: groupResolution.group }, "Outbox dispatcher skipped on this worker group; it runs only on the general group.");
|
|
}
|
|
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
if (redisUrl) {
|
|
const [{ getDbClient }, { createAdminBookingOutboxDependencies }] = await Promise.all([
|
|
import("@hivegrid/db/client.server"),
|
|
import("@hivegrid/integrations/admin-booking-worker.server"),
|
|
]);
|
|
const connection = createWorkerConnection("general", {
|
|
env: process.env as NodeJS.ProcessEnv,
|
|
});
|
|
adminBookingOutbox = bootstrapAdminBookingOutbox(process.env, {
|
|
dependencies: createAdminBookingOutboxDependencies({
|
|
client: getDbClient(),
|
|
env: process.env,
|
|
}),
|
|
createWorker: (queueName, processor, options) =>
|
|
new BullMQWorker(queueName, processor, {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
}) as unknown as AdminBookingOutboxWorkerHandle,
|
|
logger: workerLogger,
|
|
});
|
|
} else {
|
|
adminBookingOutbox = {
|
|
status: "disabled",
|
|
reason: "no Redis URL configured",
|
|
};
|
|
if (process.env.ADMIN_BOOKING_OUTBOX_REQUIRED === "true" || process.env.RLS_RUNTIME_MODE === "required") {
|
|
throw new Error("Admin booking provider worker requires Redis in required mode.");
|
|
}
|
|
}
|
|
} catch (error) {
|
|
adminBookingOutbox = {
|
|
status: "disabled",
|
|
reason: "bootstrap failed",
|
|
};
|
|
workerLogger.error({ error }, "Admin booking provider worker bootstrap failed.");
|
|
if (process.env.ADMIN_BOOKING_OUTBOX_REQUIRED === "true" || process.env.RLS_RUNTIME_MODE === "required") {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
portalDomainOrchestration = await bootstrapPortalDomainOrchestration(process.env, {
|
|
logger: workerLogger,
|
|
reporter: workerErrorReporter,
|
|
});
|
|
if (portalDomainOrchestration.status === "disabled") {
|
|
if (process.env.PORTAL_DOMAIN_ORCHESTRATION_REQUIRED === "true") {
|
|
throw new Error(`Portal-domain orchestration is required but disabled: ${portalDomainOrchestration.reason}`);
|
|
}
|
|
workerLogger.info({ reason: portalDomainOrchestration.reason }, "portal-domain orchestration disabled.");
|
|
}
|
|
} catch (error) {
|
|
if (process.env.PORTAL_DOMAIN_ORCHESTRATION_REQUIRED === "true") {
|
|
workerLogger.error({ error }, "Portal-domain orchestration failed in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required portal-domain orchestration failure.");
|
|
}
|
|
portalDomainOrchestration = { status: "disabled", reason: "bootstrap failed" };
|
|
workerLogger.error({ error }, "Portal-domain orchestration bootstrap failed; continuing while custom-domain activation stays unavailable.");
|
|
}
|
|
}
|
|
|
|
if (groupResolution?.group === "general") {
|
|
const providerSideEffectRequired = resolveProductAcquisitionProviderSideEffectMode(process.env) === "required";
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient();
|
|
const connection = redisUrl ? parseRedisUrl(redisUrl) : null;
|
|
productAcquisitionProviderSideEffect = bootstrapProductAcquisitionProviderSideEffect(process.env, {
|
|
logger: workerLogger,
|
|
claimOutboxEvent: (envelope) => claimProductProviderSideEffectOutboxEvent(envelope, { store: prisma }),
|
|
releaseOutboxClaim: (envelope, claim) => releaseProductProviderSideEffectOutboxClaim(envelope, claim, { store: prisma }),
|
|
loadAcquisition: async ({ tenantId, acquisitionId }) =>
|
|
prisma.tenantProductAcquisition.findFirst({
|
|
where: { id: acquisitionId, tenantId },
|
|
select: {
|
|
id: true,
|
|
tenantId: true,
|
|
productKind: true,
|
|
status: true,
|
|
providerAttempt: true,
|
|
providerPaymentId: true,
|
|
invoiceExternalId: true,
|
|
lagoSubscriptionId: true,
|
|
lagoProviderSubscriptionId: true,
|
|
initialLagoInvoiceExternalId: true,
|
|
},
|
|
}),
|
|
createWorker: (queueName, processor, options) => {
|
|
if (!connection) throw new Error("Product-acquisition provider side-effect worker requires Redis.");
|
|
const worker = new BullMQWorker(
|
|
queueName,
|
|
async (job, token) =>
|
|
runWorkerPayloadWithRlsContext(job.data, () =>
|
|
runProductProviderSideEffectWithReceipt(job, processor, {
|
|
store: prisma,
|
|
logger: workerLogger,
|
|
report: (event) => workerErrorReporter.report(event as never),
|
|
deferReceiptFailure: async () => {
|
|
await job.moveToDelayed(Date.now() + 5_000, token);
|
|
throw new BullMQDelayedError();
|
|
},
|
|
})
|
|
),
|
|
{
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
}
|
|
);
|
|
return worker;
|
|
},
|
|
});
|
|
if (productAcquisitionProviderSideEffect.status === "disabled") {
|
|
workerLogger.info({ reason: productAcquisitionProviderSideEffect.reason }, "product-acquisition provider side-effect worker disabled");
|
|
}
|
|
} catch (error) {
|
|
if (providerSideEffectRequired) {
|
|
workerLogger.error({ error }, "Product-acquisition provider side-effect bootstrap failed in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required product-acquisition provider side-effect failure.");
|
|
}
|
|
productAcquisitionProviderSideEffect = { status: "disabled", reason: "bootstrap failed" };
|
|
workerLogger.error({ error }, "Product-acquisition provider side-effect bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
if (groupResolution?.group === "general") {
|
|
const providerSideEffectRequired = resolveProductAcquisitionProviderSideEffectMode(process.env) === "required";
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL?.trim() || process.env.REDIS_URL?.trim();
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient();
|
|
const connection = redisUrl ? parseRedisUrl(redisUrl) : null;
|
|
recurringInvoiceCollection = bootstrapRecurringInvoiceCollection(process.env, {
|
|
logger: workerLogger,
|
|
claimOutboxEvent: (envelope) => claimProductProviderSideEffectOutboxEvent(envelope, { store: prisma }),
|
|
releaseOutboxClaim: (envelope, claim) => releaseProductProviderSideEffectOutboxClaim(envelope, claim, { store: prisma }),
|
|
loadCollection: async ({ tenantId, collectionId, lagoSubscriptionId, billingProviderSubscriptionId }) => {
|
|
const [ledger, subscription] = await Promise.all([
|
|
prisma.checkoutSessionStatus.findFirst({
|
|
where: { id: collectionId, tenantId },
|
|
select: {
|
|
id: true,
|
|
tenantId: true,
|
|
pending: true,
|
|
status: true,
|
|
providerPaymentId: true,
|
|
invoiceExternalId: true,
|
|
lagoSubscriptionId: true,
|
|
scopeKey: true,
|
|
billingCadence: true,
|
|
amountCents: true,
|
|
currency: true,
|
|
},
|
|
}),
|
|
prisma.tenantSubscription.findUnique({
|
|
where: { tenantId },
|
|
select: {
|
|
status: true,
|
|
mollieCustomerId: true,
|
|
products: {
|
|
where: {
|
|
lagoSubscriptionId,
|
|
billingProviderSubscriptionId,
|
|
},
|
|
select: { id: true },
|
|
take: 1,
|
|
},
|
|
},
|
|
}),
|
|
]);
|
|
if (
|
|
!ledger ||
|
|
!subscription?.mollieCustomerId?.trim() ||
|
|
subscription.products.length !== 1 ||
|
|
(subscription.status !== "active" && subscription.status !== "trial")
|
|
) {
|
|
return null;
|
|
}
|
|
return {
|
|
...ledger,
|
|
billingProviderSubscriptionId: ledger.scopeKey,
|
|
mollieCustomerId: subscription.mollieCustomerId,
|
|
};
|
|
},
|
|
updateCollection: async ({ tenantId, collectionId, providerPaymentId, status, pending }) => {
|
|
const updated = await prisma.checkoutSessionStatus.updateMany({
|
|
where: {
|
|
id: collectionId,
|
|
tenantId,
|
|
pending: true,
|
|
providerPaymentId: null,
|
|
},
|
|
data: {
|
|
...(providerPaymentId ? { providerPaymentId } : {}),
|
|
status,
|
|
pending,
|
|
},
|
|
});
|
|
return updated.count === 1;
|
|
},
|
|
createWorker: (queueName, processor, options) => {
|
|
if (!connection) throw new Error("Recurring invoice collection worker requires Redis.");
|
|
const worker = new BullMQWorker(
|
|
queueName,
|
|
async (job, token) =>
|
|
runWorkerPayloadWithRlsContext(job.data, () =>
|
|
runProductProviderSideEffectWithReceipt(job, processor, {
|
|
store: prisma,
|
|
logger: workerLogger,
|
|
report: (event) => workerErrorReporter.report(event as never),
|
|
deferReceiptFailure: async () => {
|
|
await job.moveToDelayed(Date.now() + 5_000, token);
|
|
throw new BullMQDelayedError();
|
|
},
|
|
})
|
|
),
|
|
{
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
}
|
|
);
|
|
return worker;
|
|
},
|
|
});
|
|
if (recurringInvoiceCollection.status === "disabled") {
|
|
workerLogger.info({ reason: recurringInvoiceCollection.reason }, "recurring invoice collection worker disabled");
|
|
}
|
|
} catch (error) {
|
|
if (providerSideEffectRequired) {
|
|
workerLogger.error({ error }, "Recurring invoice collection bootstrap failed in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required recurring invoice collection failure.");
|
|
}
|
|
recurringInvoiceCollection = { status: "disabled", reason: "bootstrap failed" };
|
|
workerLogger.error({ error }, "Recurring invoice collection bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
let workflowTrigger: WorkflowTriggerBootstrapResult | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (process.env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "false") {
|
|
workflowTrigger = bootstrapWorkflowTrigger(process.env, { logger: workerLogger });
|
|
workflowTriggerStatus = workflowTrigger;
|
|
if (workflowTrigger.status === "disabled") {
|
|
workerLogger.info({ reason: workflowTrigger.reason }, "workflow-trigger worker disabled by configuration.");
|
|
}
|
|
} else if (redisUrl) {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient();
|
|
const executor = createWorkflowTriggerExecutorFromEnv(process.env, {
|
|
logger: workerLogger,
|
|
prisma: prisma as unknown as WorkflowTriggerNativePrisma,
|
|
});
|
|
const connection = createWorkerConnection("general", { env: process.env as NodeJS.ProcessEnv });
|
|
workflowTrigger = bootstrapWorkflowTrigger(process.env, {
|
|
logger: workerLogger,
|
|
store: createPrismaWorkflowTriggerStore(prisma as unknown as WorkflowTriggerStorePrisma, {
|
|
nativeCapabilityPreflight: parseWorkflowTriggerExecutorMode(process.env.WORKFLOW_TRIGGER_EXECUTOR_MODE) !== "http",
|
|
}),
|
|
...(executor ? { executor } : {}),
|
|
idempotency: createIdempotencyStore({
|
|
store: getSharedStore(),
|
|
prefix: "workflow-trigger",
|
|
ttlSeconds: 60 * 60 * 24,
|
|
}),
|
|
runPortalAutomation: (data) =>
|
|
runPortalAutomationTriggerJob(data, {
|
|
prisma,
|
|
logger: workerLogger,
|
|
portalBaseUrl: resolvePortalStatusMailBaseUrl(process.env),
|
|
}),
|
|
runAutomationForward: createRunAutomationForwardTrigger({ prisma, logger: workerLogger }),
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as WorkflowTriggerWorkerHandle;
|
|
},
|
|
});
|
|
if (workflowTrigger.status === "disabled") {
|
|
if (isWorkflowTriggerOutboxRequired(process.env)) {
|
|
workerLogger.error({ reason: workflowTrigger.reason }, "Workflow-trigger subscriber disabled in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required workflow-trigger subscriber disabled.");
|
|
}
|
|
workerLogger.warn({ reason: workflowTrigger.reason }, "workflow-trigger worker disabled.");
|
|
}
|
|
workflowTriggerStatus = workflowTrigger;
|
|
if (workflowTrigger.status === "started") {
|
|
// R-P12c: durable forward-executor consumer for
|
|
// automation.execution.* outbox events (retry/delay continuations
|
|
// arrive as scheduled outbox rows on the same queue).
|
|
const automationExecutorProcessor = createAutomationExecutorProcessor({ prisma, logger: workerLogger });
|
|
automationExecutorWorker = new BullMQWorker(
|
|
AUTOMATION_EXECUTOR_OUTBOX_QUEUE,
|
|
createRlsAwareBullMqProcessor(async (job: { data: unknown }) => automationExecutorProcessor(job.data)),
|
|
{ connection, concurrency: 5 }
|
|
);
|
|
workerLogger.info({ queue: AUTOMATION_EXECUTOR_OUTBOX_QUEUE }, "automation forward-executor worker started.");
|
|
}
|
|
} else {
|
|
if (isWorkflowTriggerOutboxRequired(process.env)) {
|
|
throw new Error("Workflow trigger outbox subscriber is required but disabled: no Redis URL configured.");
|
|
}
|
|
workflowTrigger = { status: "disabled", reason: "no Redis URL configured" };
|
|
workflowTriggerStatus = workflowTrigger;
|
|
workerLogger.info({}, "workflow-trigger worker skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
if (isWorkflowTriggerOutboxRequired(process.env)) {
|
|
workerLogger.error({ error }, "Workflow-trigger bootstrap failed in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required workflow-trigger failure.");
|
|
}
|
|
workerLogger.error({ error }, "Workflow-trigger bootstrap failed; continuing without subscriber.");
|
|
workflowTriggerStatus = { status: "disabled", reason: "bootstrap failed" };
|
|
}
|
|
} else if (groupResolution && isWorkflowTriggerOutboxRequired(process.env)) {
|
|
workerLogger.error({ workerGroup: groupResolution.group }, "Workflow-trigger subscriber required on a non-general worker group; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after workflow-trigger required on a non-general worker group.");
|
|
}
|
|
|
|
// Phase 10 #68 — DSGVO grace-period cron. Like the outbox dispatcher
|
|
// this is a single-instance scheduler and must therefore live on the
|
|
// `general` group only (one replica per cluster). We register the
|
|
// BullMQ job-scheduler entry; the actual processor is wired into the
|
|
// maintenance queue by the maintenance handler, which inspects
|
|
// job.name === DSGVO_GRACE_CRON_JOB_ID and calls runDsgvoGraceCron.
|
|
if (groupResolution?.group === "general") {
|
|
const exportRetentionRequired = process.env.EXPORT_RETENTION_QUEUE_ENABLED === "true";
|
|
const tenantHardDeleteRequired = isRequiredTenantHardDeleteRuntime(process.env);
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const dsgvoCronQueue = new BullMQQueue(queueNames.maintenance, {
|
|
connection,
|
|
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
|
|
});
|
|
try {
|
|
await registerDsgvoGraceCron(dsgvoCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "dsgvo-grace-cron scheduler registered (24h cadence).");
|
|
// AK-P21 — tenant hard-delete scan shares the maintenance queue
|
|
// and the single-scheduled general-group invariant.
|
|
const { registerPrivacyTenantDeletionCron } = await import("./jobs/privacy-tenant-deletion-bootstrap.js");
|
|
await registerPrivacyTenantDeletionCron(dsgvoCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "privacy-tenant-deletion scheduler registered (24h cadence).");
|
|
const { registerPortalLifecycleCron } = await import("./jobs/portal-lifecycle-bootstrap.js");
|
|
await registerPortalLifecycleCron(dsgvoCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "portal-lifecycle scheduler registered (24h cadence).");
|
|
if (exportRetentionRequired) {
|
|
await registerPrivacyGdprExportRetentionCron(dsgvoCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "privacy GDPR export-retention scheduler registered (15m cadence).");
|
|
await registerPortalGdprExportRetentionCron(dsgvoCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "portal GDPR export-retention scheduler registered (15m cadence).");
|
|
}
|
|
} finally {
|
|
await dsgvoCronQueue.close();
|
|
}
|
|
} else {
|
|
if (exportRetentionRequired) {
|
|
throw new Error("Required GDPR export-retention schedulers cannot start without Redis.");
|
|
}
|
|
workerLogger.info({}, "dsgvo-grace-cron scheduler skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
if (exportRetentionRequired || tenantHardDeleteRequired) {
|
|
workerLogger.error({}, "Required GDPR/tenant-deletion scheduler registration failed; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required GDPR/tenant-deletion scheduler failure.");
|
|
}
|
|
workerLogger.error({ error }, "dsgvo-grace-cron scheduler registration failed; continuing.");
|
|
}
|
|
}
|
|
|
|
// DONE outbox rows are useful for short replay/debug windows but
|
|
// should not grow forever. Keep retention single-scheduled on the
|
|
// general worker so every deployment has one bounded daily sweeper.
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const outboxRetentionQueue = new BullMQQueue(queueNames.maintenance, {
|
|
connection,
|
|
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
|
|
});
|
|
try {
|
|
await registerOutboxDoneRetentionCron(outboxRetentionQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "outbox-done-retention scheduler registered (24h cadence).");
|
|
} finally {
|
|
await outboxRetentionQueue.close();
|
|
}
|
|
} else {
|
|
workerLogger.info({}, "outbox-done-retention scheduler skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "outbox-done-retention scheduler registration failed; continuing.");
|
|
}
|
|
}
|
|
|
|
// Workflow executions are inserted as `running` before their outbox
|
|
// event is dispatched. If dispatch fails permanently, this singleton
|
|
// sweep repairs only untouched manual/API executions with a matching
|
|
// FAILED outbox row so they do not remain running forever.
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const workflowStalledReconcileQueue = new BullMQQueue(queueNames.maintenance, {
|
|
connection,
|
|
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
|
|
});
|
|
try {
|
|
await registerWorkflowExecutionStalledReconcilerCron(workflowStalledReconcileQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "workflow-execution-stalled-reconcile scheduler registered (5m cadence).");
|
|
} finally {
|
|
await workflowStalledReconcileQueue.close();
|
|
}
|
|
} else {
|
|
workerLogger.info({}, "workflow-execution-stalled-reconcile scheduler skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "workflow-execution-stalled-reconcile scheduler registration failed; continuing.");
|
|
}
|
|
}
|
|
|
|
// P4 Tasks (AK-P4b) — 30-day auto-archive (DoD 5) + hourly reminder sweep
|
|
// (DoD 4). Single-instance schedulers on the `general` group (same
|
|
// constraint as the other maintenance crons: must not multiply across
|
|
// replicas). Both are no-ops without Redis configured.
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const taskCronQueue = new BullMQQueue(queueNames.maintenance, {
|
|
connection,
|
|
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
|
|
});
|
|
try {
|
|
await registerTaskArchiveRetentionCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "task-archive-retention scheduler registered (24h cadence).");
|
|
await registerLeadRetentionCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "lead-retention-sweep scheduler registered (24h cadence).");
|
|
await registerProjectAutoArchiveCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "project-auto-archive scheduler registered (24h cadence).");
|
|
await registerBillingDunningSweepCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "billing-dunning-sweep scheduler registered (24h cadence).");
|
|
await registerTaskReminderSweepCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "task-reminder-sweep scheduler registered (1h cadence).");
|
|
await registerAutomationSchedulerCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "automation-scheduler-sweep scheduler registered (1m cadence).");
|
|
await registerAdminSalesTaskReminderSweepCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-sales-task-reminder-sweep scheduler registered (15m cadence).");
|
|
await registerAdminBookingReminderSweepCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-booking-reminder-sweep scheduler registered (1m cadence).");
|
|
await registerAdminCalendarSyncCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-calendar-sync scheduler registered (5m cadence).");
|
|
await registerAdminEmailDeliveryCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-delivery scheduler registered (1m cadence).");
|
|
await registerAdminEmailEventRetentionCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-event-retention scheduler registered (24h cadence).");
|
|
await registerAdminEmailSyncCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-sync scheduler registered (2m cadence).");
|
|
await registerAdminEmailDomainCron(taskCronQueue, process.env);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-email-domain-check scheduler registered (15m cadence).");
|
|
await registerAdminCallRecordingCrons(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-call-recording schedulers registered (1m process, 1h retention).");
|
|
await registerAdminOutboundCallCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "admin-outbound-call scheduler registered (1m cadence).");
|
|
await registerIntegrationsTokenRefreshCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "integrations-token-refresh scheduler registered (1m cadence).");
|
|
await registerIntegrationsHealthSweepCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "integrations-health-sweep scheduler registered (15m cadence).");
|
|
await registerKwpProjectMaintenanceCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "kwp-project-maintenance scheduler registered (15m cadence).");
|
|
await registerCalendarTokenRefreshCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "calendar-token-refresh scheduler registered (5m cadence).");
|
|
await registerCalendarWebhookRenewalCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-renewal scheduler registered (6h cadence).");
|
|
await registerCalendarWebhookDrainCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-drain scheduler registered (1m cadence).");
|
|
await registerCalendarInternalWriteSyncCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "calendar-internal-write-sync scheduler registered (5m cadence).");
|
|
await registerEmailInboxTokenRefreshCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-token-refresh scheduler registered (5m cadence).");
|
|
await registerEmailInboxWebhookRenewalCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-renewal scheduler registered (6h cadence).");
|
|
await registerEmailInboxWebhookDrainCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-drain scheduler registered (1m cadence).");
|
|
await registerEmailInboxInitialSyncCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-initial-sync scheduler registered (2m cadence).");
|
|
await registerEmailInboxStalePollCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-stale-poll scheduler registered (30m cadence).");
|
|
await registerEmailInboxEventCleanupCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-event-cleanup scheduler registered (24h cadence).");
|
|
await registerHrUploadReminderDispatchCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "hr-upload-reminder-dispatch scheduler registered (24h cadence).");
|
|
await registerEmailDomainProbeCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "email-domain-probe scheduler registered (5m cadence).");
|
|
await registerTelephonyWebhookMaintenanceCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "telephony-webhook-maintenance scheduler registered (5m cadence).");
|
|
if (resolveProductAcquisitionProviderSideEffectMode(process.env) === "disabled") {
|
|
await taskCronQueue.removeJobScheduler(PRODUCT_ACQUISITION_RECONCILE_JOB_ID);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "product-acquisition-reconcile scheduler removed while provider side effects are disabled.");
|
|
} else {
|
|
await registerProductAcquisitionReconcilerCron(taskCronQueue);
|
|
workerLogger.info({ queue: queueNames.maintenance }, "product-acquisition-reconcile scheduler registered (5m cadence).");
|
|
}
|
|
} finally {
|
|
await taskCronQueue.close();
|
|
}
|
|
} else {
|
|
workerLogger.info({}, "task maintenance schedulers skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "task maintenance scheduler registration failed; continuing.");
|
|
}
|
|
}
|
|
|
|
// Phase 06 #23 — PluginCapabilityUse audit-retention cron. Nightly
|
|
// at 03:00 UTC (cron pattern overridable via PLUGIN_AUDIT_RETENTION_CRON).
|
|
// Single-instance scheduler + worker on the `general` group, on a
|
|
// dedicated queue (PLUGIN_AUDIT_RETENTION_QUEUE) so we don't have to
|
|
// extend the maintenance CleanupJob discriminator. The worker is
|
|
// attached to the SAME BullMQ Queue instance so the scheduler-fired
|
|
// job lands here and gets processed.
|
|
let pluginAuditRetentionWorker: import("bullmq").Worker | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const pluginAuditCronQueue = new BullMQQueue(PLUGIN_AUDIT_RETENTION_QUEUE, {
|
|
connection,
|
|
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
|
|
});
|
|
try {
|
|
const { pattern } = await registerPluginAuditRetentionCron(pluginAuditCronQueue, process.env);
|
|
workerLogger.info({ queue: PLUGIN_AUDIT_RETENTION_QUEUE, pattern }, "plugin-audit-retention scheduler registered.");
|
|
} finally {
|
|
await pluginAuditCronQueue.close();
|
|
}
|
|
|
|
const retentionConfig = resolvePluginAuditRetentionConfig(process.env);
|
|
pluginAuditRetentionWorker = new BullMQWorker(
|
|
PLUGIN_AUDIT_RETENTION_QUEUE,
|
|
async () => {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient() as unknown as {
|
|
pluginCapabilityUse: {
|
|
deleteMany: (args: { where: { id: { in: string[] } } }) => Promise<{ count: number }>;
|
|
findMany: (args: {
|
|
where: { timestamp: { lt: Date } };
|
|
select: { id: true };
|
|
take: number;
|
|
orderBy: { timestamp: "asc" };
|
|
}) => Promise<Array<{ id: string }>>;
|
|
count: () => Promise<number>;
|
|
};
|
|
};
|
|
return runPluginAuditRetention({
|
|
config: retentionConfig,
|
|
logger: workerLogger,
|
|
deleteBatch: async (cutoff, batchSize) => {
|
|
// Postgres does not support LIMIT on DELETE directly; pick
|
|
// a bounded id-set first, then deleteMany by id.
|
|
const candidates = await prisma.pluginCapabilityUse.findMany({
|
|
where: { timestamp: { lt: cutoff } },
|
|
select: { id: true },
|
|
take: batchSize,
|
|
orderBy: { timestamp: "asc" },
|
|
});
|
|
if (candidates.length === 0) return 0;
|
|
const ids = candidates.map((row) => row.id);
|
|
const { count } = await prisma.pluginCapabilityUse.deleteMany({
|
|
where: { id: { in: ids } },
|
|
});
|
|
return count;
|
|
},
|
|
// F-020 — surviving-rows count fed to the size-warning
|
|
// step. Soft-fails inside runPluginAuditRetention if the
|
|
// count() call throws, so a flaky DB doesn't poison the
|
|
// retention result.
|
|
countRows: () => prisma.pluginCapabilityUse.count(),
|
|
});
|
|
},
|
|
{ connection: parseRedisUrl(redisUrl), concurrency: 1 }
|
|
);
|
|
workerLogger.info({ queue: PLUGIN_AUDIT_RETENTION_QUEUE, retentionDays: retentionConfig.retentionDays }, "plugin-audit-retention worker started.");
|
|
} else {
|
|
workerLogger.info({}, "plugin-audit-retention scheduler skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "plugin-audit-retention scheduler registration failed; continuing.");
|
|
}
|
|
}
|
|
|
|
// AK-P3 AP-2 — AuditEvent retention cron. Nightly at 03:30 UTC
|
|
// (AUDIT_RETENTION_CRON overridable). Single-instance scheduler + worker on
|
|
// the `general` group, dedicated queue (AUDIT_RETENTION_QUEUE). Mirrors the
|
|
// plugin-audit-retention pattern. The policy is fixed at ten years; every
|
|
// bounded delete batch persists an authenticated chain checkpoint first.
|
|
let auditRetentionWorker: import("bullmq").Worker | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const auditRetentionQueue = new BullMQQueue(AUDIT_RETENTION_QUEUE, {
|
|
connection,
|
|
defaultJobOptions: SCHEDULED_JOB_DEFAULT_OPTIONS,
|
|
});
|
|
try {
|
|
const { pattern } = await registerAuditRetentionCron(auditRetentionQueue, process.env);
|
|
workerLogger.info({ queue: AUDIT_RETENTION_QUEUE, pattern }, "audit-retention scheduler registered.");
|
|
} finally {
|
|
await auditRetentionQueue.close();
|
|
}
|
|
|
|
auditRetentionWorker = new BullMQWorker(
|
|
AUDIT_RETENTION_QUEUE,
|
|
async () => {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const { createAuditEvent } = await import("@hivegrid/db/audit-events.server");
|
|
const { pruneAuditChainBatch } = await import("@hivegrid/db/audit-chain-checkpoint.server");
|
|
const prisma = getDbClient() as unknown as {
|
|
auditEvent: {
|
|
groupBy: (args: Record<string, unknown>) => Promise<Array<{ tenantId: string | null }>>;
|
|
};
|
|
};
|
|
return runAuditRetentionJob({
|
|
logger: workerLogger,
|
|
// One aggregate query discovers all expired chains, including
|
|
// the platform chain (tenantId=null); there is no per-tenant
|
|
// regulatory-scope query fan-out.
|
|
chainLister: async (threshold: Date): Promise<ReadonlyArray<string | null>> => {
|
|
const rows = await prisma.auditEvent.groupBy({
|
|
by: ["tenantId"],
|
|
where: { createdAt: { lt: threshold } },
|
|
});
|
|
return rows.map((row) => row.tenantId);
|
|
},
|
|
batchPruner: (tenantId, threshold, batchSize) => pruneAuditChainBatch(tenantId, threshold, { batchSize }),
|
|
// The cleanup is itself audited as a system action (traceable).
|
|
auditWriter: async ({ tenantId, deletedCount, threshold, retentionYears }) => {
|
|
await createAuditEvent({
|
|
tenantId,
|
|
actorType: "system",
|
|
action: "audit.retention.pruned",
|
|
targetType: "AuditEvent",
|
|
sourceModule: "audit",
|
|
dataSensitivity: "internal",
|
|
metadata: { deletedCount, threshold: threshold.toISOString(), retentionYears },
|
|
});
|
|
},
|
|
});
|
|
},
|
|
{ connection: parseRedisUrl(redisUrl), concurrency: 1 }
|
|
);
|
|
workerLogger.info({ queue: AUDIT_RETENTION_QUEUE }, "audit-retention worker started.");
|
|
} else {
|
|
workerLogger.info({}, "audit-retention scheduler skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "audit-retention scheduler registration failed; continuing.");
|
|
}
|
|
}
|
|
|
|
let searchIndexer: SearchIndexerBootstrapResult | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const { createSearchIndexerBulkPagers } = await import("./jobs/search-indexer-bulk-pagers.js");
|
|
const prisma = getDbClient();
|
|
searchIndexer = await bootstrapSearchIndexer(process.env, {
|
|
logger: workerLogger,
|
|
bulkPagers: createSearchIndexerBulkPagers({ prisma }),
|
|
createWorkerImpl: redisUrl
|
|
? (queueName, processor, options) => {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as SearchIndexerWorkerHandle;
|
|
}
|
|
: undefined,
|
|
});
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "Search-indexer bootstrap failed; stopping general worker.");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
let formSubmissionRoute: FormSubmissionRouteBootstrapResult | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient() as unknown as {
|
|
formSubmission: {
|
|
findUnique: (args: { where: { id: string }; select: { status: true; tenantId: true } }) => Promise<{ status: string; tenantId: string } | null>;
|
|
updateMany: (args: { where: { id: string; tenantId: string }; data: Record<string, unknown> }) => Promise<{ count: number }>;
|
|
};
|
|
formAction: {
|
|
findMany: (args: Record<string, unknown>) => Promise<
|
|
Array<{
|
|
id: string;
|
|
actionType: string;
|
|
name: string;
|
|
fieldMapping: unknown;
|
|
actionConfig: unknown;
|
|
}>
|
|
>;
|
|
};
|
|
formActionLog: {
|
|
create: (args: Record<string, unknown>) => Promise<{ id: string; attempt: number }>;
|
|
update: (args: Record<string, unknown>) => Promise<unknown>;
|
|
};
|
|
inboxLead: {
|
|
findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null>;
|
|
create: (args: Record<string, unknown>) => Promise<{ id: string }>;
|
|
};
|
|
kontakt: {
|
|
findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null>;
|
|
create: (args: Record<string, unknown>) => Promise<{ id: string }>;
|
|
};
|
|
kontaktMarketingConsent: {
|
|
upsert: (args: Record<string, unknown>) => Promise<unknown>;
|
|
};
|
|
pipeline: { findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null> };
|
|
pipelineStage: { findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null> };
|
|
projekt: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
|
|
task: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
|
|
$transaction: <Result>(
|
|
callback: (
|
|
tx: Parameters<typeof ensureAndSyncProjectChannelIfEnabledWithTx>[0] & {
|
|
projekt: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
|
|
}
|
|
) => Promise<Result>
|
|
) => Promise<Result>;
|
|
};
|
|
const [{ publishOutboxEvent }, { planFormSubmissionFollowUps }, { buildWorkflowTriggerOutboxEvent }, { applyFieldMapping }] = await Promise.all([
|
|
import("@hivegrid/db/outbox"),
|
|
import("./jobs/form-submission-followups.js"),
|
|
import("@hivegrid/queue"),
|
|
import("@hivegrid/form-widget"),
|
|
]);
|
|
const connection = parseRedisUrl(redisUrl);
|
|
function stringValue(values: Record<string, unknown>, keys: readonly string[]): string | null {
|
|
for (const key of keys) {
|
|
const value = values[key];
|
|
if (typeof value === "string" && value.trim()) return value.trim();
|
|
}
|
|
return null;
|
|
}
|
|
function objectValue(value: unknown): Record<string, unknown> {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
|
}
|
|
formSubmissionRoute = bootstrapFormSubmissionRoute(process.env, {
|
|
logger: workerLogger,
|
|
handlerDeps: {
|
|
store: {
|
|
load: (submissionId) =>
|
|
prisma.formSubmission.findUnique({
|
|
where: { id: submissionId },
|
|
select: { status: true, tenantId: true },
|
|
}),
|
|
markRouted: async (input) => {
|
|
const { count } = await prisma.formSubmission.updateMany({
|
|
where: { id: input.submissionId, tenantId: input.tenantId },
|
|
data: { status: input.status, result: input.result },
|
|
});
|
|
if (count !== 1) {
|
|
throw new Error("form-submission-route could not update the tenant-scoped submission row");
|
|
}
|
|
},
|
|
},
|
|
emitFollowUps: async (followUp) => {
|
|
const events = planFormSubmissionFollowUps(followUp);
|
|
for (const event of events) {
|
|
await publishOutboxEvent(prisma as never, {
|
|
tenantId: event.tenantId,
|
|
type: event.type,
|
|
payload: event.payload as Prisma.InputJsonValue,
|
|
dispatchKey: event.dispatchKey,
|
|
});
|
|
}
|
|
},
|
|
actionRunner: async ({ submissionId, formId, tenantId, values }) => {
|
|
const actions = await prisma.formAction.findMany({
|
|
where: { formId, tenantId, enabled: true },
|
|
orderBy: { position: "asc" },
|
|
select: { id: true, actionType: true, name: true, fieldMapping: true, actionConfig: true },
|
|
});
|
|
for (const action of actions) {
|
|
const mapped = applyFieldMapping({
|
|
payload: values,
|
|
fieldMapping: objectValue(action.fieldMapping) as Record<string, string>,
|
|
});
|
|
const idempotencyKey = `form-action:${submissionId}:${action.id}`;
|
|
const log = await prisma.formActionLog.create({
|
|
data: {
|
|
formActionId: action.id,
|
|
submissionId,
|
|
formId,
|
|
tenantId,
|
|
actionType: action.actionType,
|
|
status: "pending",
|
|
inputSnapshot: { values: mapped },
|
|
idempotencyKey,
|
|
},
|
|
select: { id: true, attempt: true },
|
|
});
|
|
try {
|
|
const config = objectValue(action.actionConfig);
|
|
let target: { type: string; id: string | null } = { type: action.actionType, id: null };
|
|
if (action.actionType === "auto_match") {
|
|
const email = stringValue(mapped, ["email", "primaerEmail", "email_address"]);
|
|
const phone = stringValue(mapped, ["phone", "telefon", "primaerTelefon"]);
|
|
const contact =
|
|
email || phone
|
|
? await prisma.kontakt.findFirst({
|
|
where: {
|
|
tenantId,
|
|
archivedAt: null,
|
|
OR: [...(email ? [{ primaerEmail: email.toLowerCase() }] : []), ...(phone ? [{ primaerTelefon: phone }] : [])],
|
|
},
|
|
select: { id: true },
|
|
})
|
|
: null;
|
|
const lead =
|
|
!contact && (email || phone)
|
|
? await prisma.inboxLead.findFirst({
|
|
where: {
|
|
tenantId,
|
|
archivedAt: null,
|
|
OR: [...(email ? [{ email: email.toLowerCase() }] : []), ...(phone ? [{ telefon: phone }] : [])],
|
|
},
|
|
select: { id: true },
|
|
})
|
|
: null;
|
|
target = contact ? { type: "contact", id: contact.id } : lead ? { type: "lead", id: lead.id } : { type: "none", id: null };
|
|
if (target.id) {
|
|
await prisma.formSubmission.updateMany({
|
|
where: { id: submissionId, tenantId },
|
|
data: { linkedEntityType: target.type, linkedEntityId: target.id },
|
|
});
|
|
}
|
|
} else if (action.actionType === "create_lead") {
|
|
const lead = await prisma.inboxLead.create({
|
|
data: {
|
|
tenantId,
|
|
createdByUserId: typeof config.userId === "string" ? config.userId : null,
|
|
quelle: "form",
|
|
quelleDetail: { formId, submissionId },
|
|
sourceOrigin: `/forms/${formId}`,
|
|
vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]),
|
|
nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]),
|
|
email: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null,
|
|
telefon: stringValue(mapped, ["telefon", "phone"]),
|
|
message: stringValue(mapped, ["message", "nachricht", "details"]),
|
|
rohdaten: { formId, submissionId, values: mapped },
|
|
weiterverarbeitungFreigabe: "ja",
|
|
},
|
|
select: { id: true },
|
|
});
|
|
target = { type: "lead", id: lead.id };
|
|
} else if (action.actionType === "create_contact") {
|
|
const contact = await prisma.kontakt.create({
|
|
data: {
|
|
tenantId,
|
|
createdByUserId: typeof config.userId === "string" ? config.userId : "system",
|
|
kontaktTyp: "privatperson",
|
|
vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]),
|
|
nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]) ?? stringValue(mapped, ["name"]) ?? "Formular-Kontakt",
|
|
primaerEmail: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null,
|
|
primaerTelefon: stringValue(mapped, ["telefon", "phone"]),
|
|
},
|
|
select: { id: true },
|
|
});
|
|
target = { type: "contact", id: contact.id };
|
|
} else if (action.actionType === "create_project") {
|
|
const pipeline = await prisma.pipeline.findFirst({
|
|
where: { tenantId, archivedAt: null },
|
|
orderBy: { position: "asc" },
|
|
select: { id: true },
|
|
});
|
|
const stage = pipeline
|
|
? await prisma.pipelineStage.findFirst({
|
|
where: { tenantId, pipelineId: pipeline.id, archivedAt: null },
|
|
orderBy: { position: "asc" },
|
|
select: { id: true },
|
|
})
|
|
: null;
|
|
if (!pipeline || !stage) throw new Error("No project pipeline/stage available for form action");
|
|
const actorUserId = typeof config.userId === "string" ? config.userId : null;
|
|
const project = await prisma.$transaction(async (tx) => {
|
|
const created = await tx.projekt.create({
|
|
data: {
|
|
tenantId,
|
|
createdByUserId: actorUserId,
|
|
projektNummer: `FORM-${Date.now()}`,
|
|
projektname: stringValue(mapped, ["projektname", "projectName", "name"]) ?? "Projekt aus Formular",
|
|
beschreibung: stringValue(mapped, ["message", "description", "details"]),
|
|
quelle: "form",
|
|
pipelineId: pipeline.id,
|
|
stageId: stage.id,
|
|
},
|
|
select: { id: true },
|
|
});
|
|
await ensureAndSyncProjectChannelIfEnabledWithTx(tx, {
|
|
tenantId,
|
|
projectId: created.id,
|
|
actorUserId,
|
|
requestId: `form:${formId}:submission:${submissionId}:project:${created.id}:chat-created`,
|
|
});
|
|
return created;
|
|
});
|
|
target = { type: "project", id: project.id };
|
|
} else if (action.actionType === "create_task") {
|
|
const task = await prisma.task.create({
|
|
data: {
|
|
tenantId,
|
|
createdByUserId: typeof config.userId === "string" ? config.userId : "system",
|
|
title: stringValue(mapped, ["title", "taskTitle", "subject"]) ?? "Aufgabe aus Formular",
|
|
description: stringValue(mapped, ["message", "description", "details"]),
|
|
assignedToUserId:
|
|
typeof config.assignedToUserId === "string"
|
|
? config.assignedToUserId
|
|
: typeof config.userId === "string"
|
|
? config.userId
|
|
: "system",
|
|
source: "form_submission",
|
|
sourceRef: submissionId,
|
|
},
|
|
select: { id: true },
|
|
});
|
|
target = { type: "task", id: task.id };
|
|
} else if (action.actionType === "webhook") {
|
|
const webhookUrl = typeof config.webhookUrl === "string" ? config.webhookUrl : null;
|
|
if (!webhookUrl) throw new Error("Webhook action missing webhookUrl");
|
|
const webhookPayload = { submissionId, formId, tenantId, webhookUrl, values: mapped } as unknown as Prisma.InputJsonValue;
|
|
await publishOutboxEvent(prisma as never, {
|
|
tenantId,
|
|
type: "webhook.form-submission.deliver",
|
|
payload: webhookPayload,
|
|
dispatchKey: `form-action-webhook:${submissionId}:${action.id}`,
|
|
});
|
|
} else if (action.actionType === "send_email_to_team" || action.actionType === "send_email_to_submitter") {
|
|
const recipients =
|
|
action.actionType === "send_email_to_submitter"
|
|
? [stringValue(mapped, ["email", "email_address"])].filter((email): email is string => Boolean(email))
|
|
: Array.isArray(config.recipients)
|
|
? config.recipients.filter((email): email is string => typeof email === "string" && email.includes("@"))
|
|
: [];
|
|
if (recipients.length === 0) {
|
|
await prisma.formActionLog.update({
|
|
where: { id: log.id },
|
|
data: { status: "skipped", outputSnapshot: { reason: "no_recipients" }, finishedAt: new Date() },
|
|
});
|
|
continue;
|
|
}
|
|
const mailPayload = { submissionId, formId, tenantId, recipients, values: mapped } as unknown as Prisma.InputJsonValue;
|
|
await publishOutboxEvent(prisma as never, {
|
|
tenantId,
|
|
type: "mail.transactional.form-submission",
|
|
payload: mailPayload,
|
|
dispatchKey: `form-action-mail:${submissionId}:${action.id}`,
|
|
});
|
|
} else if (action.actionType === "trigger_automation") {
|
|
const automationData = { formId, submissionId, actionId: action.id, values: mapped } as unknown as QueueJsonValue;
|
|
await publishOutboxEvent(
|
|
prisma as never,
|
|
buildWorkflowTriggerOutboxEvent({
|
|
tenantId,
|
|
event: "form.submitted",
|
|
data: automationData,
|
|
userId: typeof config.userId === "string" ? config.userId : null,
|
|
sourceId: `form:${formId}:submission:${submissionId}:action:${action.id}`,
|
|
})
|
|
);
|
|
} else if (action.actionType === "start_double_opt_in") {
|
|
const result = await runStartDoubleOptInFormAction({
|
|
prisma: prisma as unknown as StartDoubleOptInPrisma,
|
|
publishOutboxEvent: (event) =>
|
|
publishOutboxEvent(prisma as never, {
|
|
tenantId: event.tenantId,
|
|
type: event.type,
|
|
payload: event.payload as Prisma.InputJsonValue,
|
|
dispatchKey: event.dispatchKey,
|
|
}),
|
|
tenantId,
|
|
formId,
|
|
submissionId,
|
|
actionId: action.id,
|
|
mapped,
|
|
config,
|
|
});
|
|
if (result.status === "skipped") {
|
|
await prisma.formActionLog.update({
|
|
where: { id: log.id },
|
|
data: { status: "skipped", outputSnapshot: result, finishedAt: new Date() },
|
|
});
|
|
continue;
|
|
}
|
|
target = { type: "marketing_consent", id: result.kontaktId };
|
|
} else {
|
|
await prisma.formActionLog.update({
|
|
where: { id: log.id },
|
|
data: { status: "skipped", outputSnapshot: { reason: "unsupported_action" }, finishedAt: new Date() },
|
|
});
|
|
continue;
|
|
}
|
|
await prisma.formActionLog.update({
|
|
where: { id: log.id },
|
|
data: {
|
|
status: "success",
|
|
targetType: target.type,
|
|
targetId: target.id,
|
|
outputSnapshot: { target },
|
|
finishedAt: new Date(),
|
|
},
|
|
});
|
|
} catch (error) {
|
|
await prisma.formActionLog.update({
|
|
where: { id: log.id },
|
|
data: {
|
|
status: "failed",
|
|
errorMessage: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500),
|
|
finishedAt: new Date(),
|
|
},
|
|
});
|
|
}
|
|
}
|
|
},
|
|
logger: workerLogger,
|
|
},
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as FormSubmissionRouteWorkerHandle;
|
|
},
|
|
});
|
|
} else {
|
|
formSubmissionRoute = { status: "disabled", reason: "no Redis URL configured" };
|
|
workerLogger.info({}, "form-submission-route worker skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
formSubmissionRoute = { status: "disabled", reason: "bootstrap failed" };
|
|
workerLogger.error({ error }, "Form-submission-route bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
let formSubmissionWebhookDelivery: FormSubmissionWebhookDeliveryBootstrapResult | undefined;
|
|
let formSubmissionWebhookQueue: BullMQQueue | undefined;
|
|
let transactionalMailOutbox: TransactionalMailOutboxBootstrapResult | undefined;
|
|
let portalStatusMail: PortalStatusMailBootstrapResult | undefined;
|
|
let emailSequences: EmailSequenceBootstrapResult | undefined;
|
|
if (groupResolution?.group === "mail") {
|
|
const transactionalMailOutboxRequired = isTransactionalMailOutboxRequired(process.env, groupResolution.group);
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const campaignCancellationPrisma = getDbClient() as unknown as CampaignScheduleCancellationPrisma;
|
|
formSubmissionWebhookQueue = new BullMQQueue(queueNames.email, { connection });
|
|
formSubmissionWebhookDelivery = bootstrapFormSubmissionWebhookDelivery(process.env, {
|
|
logger: workerLogger,
|
|
handlerDeps: {
|
|
enqueueWebhook: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options),
|
|
},
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as FormSubmissionWebhookDeliveryWorkerHandle;
|
|
},
|
|
});
|
|
try {
|
|
transactionalMailOutbox = bootstrapTransactionalMailOutbox(process.env, {
|
|
logger: workerLogger,
|
|
handlerDeps: {
|
|
enqueueMail: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options),
|
|
cancelCampaignSchedule: (input) => cancelCampaignScheduleWithPrisma(campaignCancellationPrisma, input),
|
|
isCampaignDeliveryAllowed: (input) => isCampaignDeliveryAllowedWithPrisma(campaignCancellationPrisma, input),
|
|
},
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as TransactionalMailOutboxWorkerHandle;
|
|
},
|
|
});
|
|
transactionalMailOutboxStatus = transactionalMailOutbox;
|
|
if (transactionalMailOutbox.status === "disabled") {
|
|
if (transactionalMailOutboxRequired) {
|
|
workerLogger.error({ reason: transactionalMailOutbox.reason }, "Transactional mail outbox disabled in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required transactional mail outbox disabled.");
|
|
}
|
|
workerLogger.warn({ reason: transactionalMailOutbox.reason }, "transactional mail outbox bridge disabled.");
|
|
}
|
|
} catch (error) {
|
|
transactionalMailOutboxStatus = { status: "disabled", reason: "bootstrap failed" };
|
|
if (transactionalMailOutboxRequired) {
|
|
workerLogger.error({ error }, "Transactional mail outbox bootstrap failed in required mode; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required transactional mail outbox failure.");
|
|
}
|
|
workerLogger.error({ error }, "Transactional mail outbox bootstrap failed; continuing without subscriber.");
|
|
}
|
|
try {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const store = createPrismaPortalStatusMailStore(getDbClient() as unknown as PortalStatusMailPrisma, {
|
|
portalBaseUrl: resolvePortalStatusMailBaseUrl(process.env),
|
|
});
|
|
portalStatusMail = bootstrapPortalStatusMail(process.env, {
|
|
logger: workerLogger,
|
|
handlerDeps: {
|
|
...store,
|
|
enqueueMail: (descriptor, options) => enqueueJob(formSubmissionWebhookQueue as never, descriptor as never, options),
|
|
logger: workerLogger,
|
|
},
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as PortalStatusMailWorkerHandle;
|
|
},
|
|
});
|
|
portalStatusMailStatus = portalStatusMail;
|
|
if (portalStatusMail.status === "disabled") {
|
|
workerLogger.error({ reason: portalStatusMail.reason }, "portal status mail worker disabled; queued deliveries will wait for recovery.");
|
|
}
|
|
} catch (error) {
|
|
portalStatusMail = { status: "disabled", reason: "bootstrap failed" };
|
|
portalStatusMailStatus = portalStatusMail;
|
|
workerLogger.error({ error }, "Portal status mail bootstrap failed; queued deliveries will wait for recovery.");
|
|
}
|
|
} else {
|
|
formSubmissionWebhookDelivery = { status: "disabled", reason: "no Redis URL configured" };
|
|
transactionalMailOutbox = { status: "disabled", reason: "no Redis URL configured" };
|
|
portalStatusMail = { status: "disabled", reason: "no Redis URL configured" };
|
|
portalStatusMailStatus = portalStatusMail;
|
|
transactionalMailOutboxStatus = transactionalMailOutbox;
|
|
if (transactionalMailOutboxRequired) {
|
|
workerLogger.error({}, "Transactional mail outbox required but Redis URL is not configured; exiting.");
|
|
process.exit(1);
|
|
throw new Error("Worker startup aborted after required transactional mail outbox missing Redis.");
|
|
}
|
|
workerLogger.info({}, "form-submission webhook bridge skipped: no Redis URL configured.");
|
|
workerLogger.info({}, "transactional mail outbox bridge skipped: no Redis URL configured.");
|
|
workerLogger.info({}, "portal status mail worker skipped: no Redis URL configured.");
|
|
}
|
|
const emailSequenceRedisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (!emailSequenceRedisUrl && process.env.MARKETING_SEQUENCES_ENABLED === "true") {
|
|
emailSequences = { status: "disabled", reason: "no Redis URL configured" };
|
|
workerLogger.error({}, "Email-sequence workers enabled but Redis URL is not configured.");
|
|
} else {
|
|
const [{ runSequenceTriggerJob }, { runEmailSequenceStepJob }] = await Promise.all([
|
|
import("./jobs/email-sequence-trigger.js"),
|
|
import("./jobs/email-sequence-scheduler.js"),
|
|
]);
|
|
// Production deps (Prisma store + BullMQ scheduleStep producer +
|
|
// template renderer + @hivegrid/mail Mailjet adapter +
|
|
// unsubscribe-url builder) are loaded lazily inside the
|
|
// processors so startup-time optional config issues do not
|
|
// crash the mail worker. Missing Redis/Mailjet config still
|
|
// fails closed at job time and is surfaced through worker logs.
|
|
emailSequences = bootstrapEmailSequenceWorkers(process.env, {
|
|
logger: workerLogger,
|
|
triggerProcessor: async (job) => {
|
|
const { createEmailSequenceTriggerDeps } = await import("./jobs/email-sequence-runtime.js");
|
|
const deps = await createEmailSequenceTriggerDeps({ logger: workerLogger });
|
|
if (!deps) return { matched: 0, started: 0, skipped: 0 };
|
|
return runSequenceTriggerJob(job.data, deps);
|
|
},
|
|
schedulerProcessor: async (job) => {
|
|
const { createEmailSequenceSchedulerDeps } = await import("./jobs/email-sequence-runtime.js");
|
|
const deps = await createEmailSequenceSchedulerDeps({ logger: workerLogger });
|
|
if (!deps) return { action: "cancelled" as const, completed: true };
|
|
return runEmailSequenceStepJob(job.data, deps);
|
|
},
|
|
createWorkerImpl: emailSequenceRedisUrl
|
|
? (queueName, processor, options) => {
|
|
const connection = parseRedisUrl(emailSequenceRedisUrl);
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as EmailSequenceWorkerHandle;
|
|
}
|
|
: undefined,
|
|
});
|
|
}
|
|
emailSequencesStatus = emailSequences;
|
|
} catch (error) {
|
|
emailSequences = { status: "disabled", reason: "bootstrap failed" };
|
|
emailSequencesStatus = emailSequences;
|
|
workerLogger.error({ error }, "Email-sequence bootstrap failed; continuing without sequence workers.");
|
|
}
|
|
}
|
|
|
|
let auditEventPersist: AuditEventPersistBootstrapResult | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient();
|
|
auditEventPersist = bootstrapAuditEventPersist(process.env, {
|
|
logger: workerLogger,
|
|
runInTransaction: async (fn) =>
|
|
// Serializable isolation guarantees concurrent dispatcher
|
|
// deliveries for the same tenant either serialize into a
|
|
// coherent chain or one of them retries on conflict.
|
|
(
|
|
prisma as unknown as {
|
|
$transaction: (runner: (tx: Parameters<typeof fn>[0]) => Promise<unknown>, opts?: { isolationLevel?: string }) => Promise<unknown>;
|
|
}
|
|
).$transaction(async (tx) => fn(tx as Parameters<typeof fn>[0]), { isolationLevel: "Serializable" }) as Promise<ReturnType<typeof fn>>,
|
|
createWorkerImpl: redisUrl
|
|
? (queueName, processor, options) => {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as AuditEventPersistWorkerHandle;
|
|
}
|
|
: undefined,
|
|
});
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "Audit-event-persist bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
let chatRealtimeWakeup: ChatRealtimeWakeupBootstrapResult | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
chatRealtimeWakeup = bootstrapChatRealtimeWakeup(process.env, {
|
|
logger: workerLogger,
|
|
baseStore: getSharedStore(),
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as ChatRealtimeWakeupWorkerHandle;
|
|
},
|
|
});
|
|
} else {
|
|
workerLogger.info({}, "chat realtime wakeup worker skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "Chat realtime wakeup bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
chatLinkPreviewStatus = bootstrapChatLinkPreview(process.env, {
|
|
logger: workerLogger,
|
|
baseStore: getSharedStore(),
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const connection = parseRedisUrl(redisUrl);
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as ChatLinkPreviewWorkerHandle;
|
|
},
|
|
});
|
|
} else {
|
|
chatLinkPreviewStatus = { status: "disabled", reason: "no Redis URL configured" };
|
|
workerLogger.info({}, "chat link preview worker skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
chatLinkPreviewStatus = { status: "disabled", reason: "bootstrap failed" };
|
|
workerLogger.error({ error }, "Chat link preview bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
let pluginHookInvokeQueue: BullMQQueue | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
const pluginHookFanoutRequired = isPluginHookFanoutRequired(process.env, groupResolution.group);
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient();
|
|
const connection = parseRedisUrl(redisUrl);
|
|
pluginHookInvokeQueue = new BullMQQueue(queueNames.pluginInvoke, { connection });
|
|
pluginHookFanoutStatus = bootstrapPluginHookFanout(process.env, {
|
|
logger: workerLogger,
|
|
prisma,
|
|
pluginInvokeQueue: pluginHookInvokeQueue as never,
|
|
idempotency: createIdempotencyStore({
|
|
store: getSharedStore(),
|
|
prefix: "plugin-hook-fanout",
|
|
ttlSeconds: 60 * 60 * 24,
|
|
}),
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as PluginHookFanoutWorkerHandle;
|
|
},
|
|
});
|
|
if (pluginHookFanoutStatus.status === "disabled" && pluginHookFanoutRequired) {
|
|
workerLogger.error({ reason: pluginHookFanoutStatus.reason }, "Plugin-hook fanout disabled in required mode; exiting.");
|
|
process.exit(1);
|
|
}
|
|
} else if (pluginHookFanoutRequired) {
|
|
workerLogger.error({}, "Plugin-hook fanout required but Redis URL is not configured; exiting.");
|
|
process.exit(1);
|
|
} else {
|
|
workerLogger.info({}, "plugin-hook fanout skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
if (pluginHookFanoutRequired) {
|
|
workerLogger.error({ error }, "Plugin-hook fanout bootstrap failed in required mode; exiting.");
|
|
process.exit(1);
|
|
} else {
|
|
workerLogger.error({ error }, "Plugin-hook fanout bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
}
|
|
|
|
let pluginCapabilityUse: PluginCapabilityUseBootstrapResult | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
const prisma = getDbClient();
|
|
const connection = parseRedisUrl(redisUrl);
|
|
pluginCapabilityUse = bootstrapPluginCapabilityUse(process.env, {
|
|
logger: workerLogger,
|
|
store: createPrismaPluginCapabilityUseStore(prisma as unknown as PluginCapabilityUseStorePrisma),
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as PluginCapabilityUseWorkerHandle;
|
|
},
|
|
});
|
|
} else {
|
|
workerLogger.info({}, "plugin capability-use worker skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "Plugin capability-use bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
// Phase 03 #42 — Legacy-event outbox receiver. Subscribes the
|
|
// `outbox.notification` queue on the `general` group and writes
|
|
// billing.subscription.* events into the legacy Event table on
|
|
// behalf of the migrated Mollie webhook (apps/api/src/billing/
|
|
// mollie-webhook-route.ts). Disabled when the group is non-general
|
|
// or no Redis URL is configured.
|
|
let legacyEventReceiver: LegacyEventReceiverBootstrapResult | undefined;
|
|
let platformNotificationQueue: TypedQueue<typeof queueNames.notifications> | undefined;
|
|
if (groupResolution?.group === "general") {
|
|
try {
|
|
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
|
if (redisUrl) {
|
|
const { createLegacyEvent } = await import("@hivegrid/db/events.server");
|
|
const connection = parseRedisUrl(redisUrl);
|
|
platformNotificationQueue = createQueue(queueNames.notifications, { connection });
|
|
const queue = platformNotificationQueue;
|
|
legacyEventReceiver = bootstrapLegacyEventReceiver(process.env, {
|
|
logger: workerLogger,
|
|
store: {
|
|
create: async (data) => {
|
|
const row = await createLegacyEvent(data);
|
|
return { id: row.id };
|
|
},
|
|
},
|
|
platformNotification: {
|
|
enqueue: async (input, options) => {
|
|
await enqueueJob(queue, buildPlatformNotificationJob(input), options);
|
|
},
|
|
logger: workerLogger,
|
|
},
|
|
createWorkerImpl: (queueName, processor, options) => {
|
|
const worker = new BullMQWorker(queueName, createRlsAwareBullMqProcessor(processor), {
|
|
connection,
|
|
concurrency: options.concurrency,
|
|
});
|
|
return worker as unknown as LegacyEventReceiverWorkerHandle;
|
|
},
|
|
});
|
|
} else {
|
|
workerLogger.info({}, "legacy-event receiver skipped: no Redis URL configured.");
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "Legacy-event receiver bootstrap failed; continuing without subscriber.");
|
|
}
|
|
}
|
|
|
|
const shutdownState: ShutdownState = {
|
|
adminServer,
|
|
healthServer,
|
|
redisHealthProbe,
|
|
outbox,
|
|
adminBookingOutbox,
|
|
workflowTrigger,
|
|
searchIndexer,
|
|
formSubmissionRoute,
|
|
formSubmissionWebhookDelivery,
|
|
formSubmissionWebhookQueue,
|
|
transactionalMailOutbox,
|
|
portalStatusMail,
|
|
portalDomainOrchestration,
|
|
productAcquisitionProviderSideEffect,
|
|
recurringInvoiceCollection,
|
|
emailSequences,
|
|
auditEventPersist,
|
|
chatRealtimeWakeup,
|
|
chatLinkPreview: chatLinkPreviewStatus,
|
|
legacyEventReceiver,
|
|
platformNotificationQueue,
|
|
pluginHookFanout: pluginHookFanoutStatus,
|
|
pluginCapabilityUse,
|
|
pluginHookInvokeQueue,
|
|
pluginAuditRetentionWorker,
|
|
auditRetentionWorker,
|
|
filePreviewQueue,
|
|
telephonyWebhookQueue,
|
|
tenantDeletionExportQueue,
|
|
telephonyWebhookWorker,
|
|
automationExecutorWorker,
|
|
};
|
|
|
|
process.on("SIGINT", () => {
|
|
void shutdown(runtime, shutdownState, "SIGINT");
|
|
});
|
|
|
|
process.on("SIGTERM", () => {
|
|
void shutdown(runtime, shutdownState, "SIGTERM");
|
|
});
|
|
}
|
|
} catch (error) {
|
|
workerLogger.error({ error }, "Worker startup failed; see worker health diagnostics for dependency details.");
|
|
process.exit(1);
|
|
}
|
|
|
|
function isRequiredTenantHardDeleteRuntime(env: NodeJS.ProcessEnv): boolean {
|
|
return (
|
|
env.WORKER_QUEUE_GROUP === "general" &&
|
|
(env.NODE_ENV === "production" || env.RLS_RUNTIME_MODE === "required" || env.SENTRY_ENVIRONMENT === "staging" || env.SENTRY_ENVIRONMENT === "production")
|
|
);
|
|
}
|
|
|
|
function shutdown(runtimeToClose: WorkerRuntimeStatus, state: ShutdownState, signal: NodeJS.Signals): Promise<void> {
|
|
state.signal ??= signal;
|
|
state.promise ??= closeRuntime(
|
|
runtimeToClose,
|
|
state.signal,
|
|
state.adminServer,
|
|
state.healthServer,
|
|
state.redisHealthProbe,
|
|
state.outbox,
|
|
state.adminBookingOutbox,
|
|
state.workflowTrigger,
|
|
state.searchIndexer,
|
|
state.formSubmissionRoute,
|
|
state.formSubmissionWebhookDelivery,
|
|
state.formSubmissionWebhookQueue,
|
|
state.transactionalMailOutbox,
|
|
state.portalStatusMail,
|
|
state.portalDomainOrchestration,
|
|
state.productAcquisitionProviderSideEffect,
|
|
state.recurringInvoiceCollection,
|
|
state.emailSequences,
|
|
state.auditEventPersist,
|
|
state.chatRealtimeWakeup,
|
|
state.chatLinkPreview,
|
|
state.legacyEventReceiver,
|
|
state.platformNotificationQueue,
|
|
state.pluginHookFanout,
|
|
state.pluginCapabilityUse,
|
|
state.pluginHookInvokeQueue,
|
|
state.pluginAuditRetentionWorker,
|
|
state.auditRetentionWorker,
|
|
state.filePreviewQueue,
|
|
state.telephonyWebhookQueue,
|
|
state.tenantDeletionExportQueue,
|
|
state.telephonyWebhookWorker,
|
|
state.automationExecutorWorker
|
|
);
|
|
return state.promise;
|
|
}
|
|
|
|
async function closeRuntime(
|
|
runtimeToClose: WorkerRuntimeStatus,
|
|
signal: NodeJS.Signals,
|
|
adminServer?: WorkerAdminServerHandle,
|
|
healthServer?: WorkerHealthServerHandle,
|
|
redisHealthProbe?: RedisHealthProbe,
|
|
outbox?: OutboxDispatcherBootstrapResult,
|
|
adminBookingOutbox?: AdminBookingOutboxBootstrapResult,
|
|
workflowTrigger?: WorkflowTriggerBootstrapResult,
|
|
searchIndexer?: SearchIndexerBootstrapResult,
|
|
formSubmissionRoute?: FormSubmissionRouteBootstrapResult,
|
|
formSubmissionWebhookDelivery?: FormSubmissionWebhookDeliveryBootstrapResult,
|
|
formSubmissionWebhookQueue?: BullMQQueue,
|
|
transactionalMailOutbox?: TransactionalMailOutboxBootstrapResult,
|
|
portalStatusMail?: PortalStatusMailBootstrapResult,
|
|
portalDomainOrchestration?: PortalDomainOrchestrationBootstrapResult,
|
|
productAcquisitionProviderSideEffect?: ProductAcquisitionProviderSideEffectBootstrapResult,
|
|
recurringInvoiceCollection?: RecurringInvoiceCollectionBootstrapResult,
|
|
emailSequences?: EmailSequenceBootstrapResult,
|
|
auditEventPersist?: AuditEventPersistBootstrapResult,
|
|
chatRealtimeWakeup?: ChatRealtimeWakeupBootstrapResult,
|
|
chatLinkPreview?: ChatLinkPreviewBootstrapResult,
|
|
legacyEventReceiver?: LegacyEventReceiverBootstrapResult,
|
|
platformNotificationQueue?: TypedQueue<typeof queueNames.notifications>,
|
|
pluginHookFanout?: PluginHookFanoutBootstrapResult,
|
|
pluginCapabilityUse?: PluginCapabilityUseBootstrapResult,
|
|
pluginHookInvokeQueue?: BullMQQueue,
|
|
pluginAuditRetentionWorker?: BullMQWorker,
|
|
auditRetentionWorker?: BullMQWorker,
|
|
filePreviewQueue?: TypedQueue<typeof queueNames.filePreview>,
|
|
telephonyWebhookQueue?: BullMQQueue<TelephonyWebhookProcessJob>,
|
|
tenantDeletionExportQueue?: BullMQQueue,
|
|
telephonyWebhookWorker?: BullMQWorker,
|
|
automationExecutorWorker?: BullMQWorker
|
|
): Promise<void> {
|
|
try {
|
|
if (outbox?.handle) {
|
|
try {
|
|
await outbox.handle.close();
|
|
} catch (outboxError) {
|
|
workerLogger.error({ error: outboxError, signal }, "Outbox dispatcher shutdown failed.");
|
|
}
|
|
}
|
|
if (adminBookingOutbox?.status === "started") {
|
|
try {
|
|
await adminBookingOutbox.handle.close();
|
|
} catch (bookingError) {
|
|
workerLogger.error({ error: bookingError, signal }, "Admin booking provider worker shutdown failed.");
|
|
}
|
|
}
|
|
if (workflowTrigger?.status === "started") {
|
|
try {
|
|
await workflowTrigger.handle.close();
|
|
} catch (workflowTriggerError) {
|
|
workerLogger.error({ error: workflowTriggerError, signal }, "Workflow-trigger worker shutdown failed.");
|
|
}
|
|
}
|
|
if (searchIndexer?.status === "started") {
|
|
try {
|
|
await searchIndexer.handle.close();
|
|
} catch (searchError) {
|
|
workerLogger.error({ error: searchError, signal }, "Search-indexer shutdown failed.");
|
|
}
|
|
}
|
|
if (formSubmissionRoute?.status === "started") {
|
|
try {
|
|
await formSubmissionRoute.handle.close();
|
|
} catch (formSubmissionError) {
|
|
workerLogger.error({ error: formSubmissionError, signal }, "Form-submission-route worker shutdown failed.");
|
|
}
|
|
}
|
|
if (formSubmissionWebhookDelivery?.status === "started") {
|
|
try {
|
|
await formSubmissionWebhookDelivery.handle.close();
|
|
} catch (formSubmissionWebhookError) {
|
|
workerLogger.error({ error: formSubmissionWebhookError, signal }, "Form-submission webhook bridge shutdown failed.");
|
|
}
|
|
}
|
|
if (transactionalMailOutbox?.status === "started") {
|
|
try {
|
|
await transactionalMailOutbox.handle.close();
|
|
} catch (transactionalMailOutboxError) {
|
|
workerLogger.error({ error: transactionalMailOutboxError, signal }, "Transactional mail outbox bridge shutdown failed.");
|
|
}
|
|
}
|
|
if (portalStatusMail?.status === "started") {
|
|
try {
|
|
await portalStatusMail.handle.close();
|
|
} catch (portalStatusMailError) {
|
|
workerLogger.error({ error: portalStatusMailError, signal }, "Portal status mail worker shutdown failed.");
|
|
}
|
|
}
|
|
if (portalDomainOrchestration?.status === "started") {
|
|
try {
|
|
await portalDomainOrchestration.close();
|
|
} catch (portalDomainError) {
|
|
workerLogger.error({ error: portalDomainError, signal }, "Portal-domain orchestration shutdown failed.");
|
|
}
|
|
}
|
|
if (productAcquisitionProviderSideEffect?.status === "started") {
|
|
try {
|
|
await productAcquisitionProviderSideEffect.handle.close();
|
|
} catch (providerSideEffectError) {
|
|
workerLogger.error({ error: providerSideEffectError, signal }, "Product-acquisition provider side-effect worker shutdown failed.");
|
|
}
|
|
}
|
|
if (recurringInvoiceCollection?.status === "started") {
|
|
try {
|
|
await recurringInvoiceCollection.handle.close();
|
|
} catch (recurringInvoiceError) {
|
|
workerLogger.error({ error: recurringInvoiceError, signal }, "Recurring invoice collection worker shutdown failed.");
|
|
}
|
|
}
|
|
if (formSubmissionWebhookQueue) {
|
|
try {
|
|
await formSubmissionWebhookQueue.close();
|
|
} catch (formSubmissionWebhookQueueError) {
|
|
workerLogger.error({ error: formSubmissionWebhookQueueError, signal }, "Form-submission webhook queue shutdown failed.");
|
|
}
|
|
}
|
|
if (emailSequences?.status === "started") {
|
|
try {
|
|
await emailSequences.triggerHandle.close();
|
|
await emailSequences.schedulerHandle.close();
|
|
} catch (seqError) {
|
|
workerLogger.error({ error: seqError, signal }, "Email-sequence workers shutdown failed.");
|
|
}
|
|
}
|
|
if (auditEventPersist?.status === "started") {
|
|
try {
|
|
await auditEventPersist.handle.close();
|
|
} catch (auditError) {
|
|
workerLogger.error({ error: auditError, signal }, "Audit-event-persist shutdown failed.");
|
|
}
|
|
}
|
|
if (chatRealtimeWakeup?.status === "started") {
|
|
try {
|
|
await chatRealtimeWakeup.handle.close();
|
|
} catch (chatRealtimeError) {
|
|
workerLogger.error({ error: chatRealtimeError, signal }, "Chat realtime wakeup worker shutdown failed.");
|
|
}
|
|
}
|
|
if (chatLinkPreview?.status === "started") {
|
|
try {
|
|
await chatLinkPreview.handle.close();
|
|
} catch (chatLinkPreviewError) {
|
|
workerLogger.error({ error: chatLinkPreviewError, signal }, "Chat link preview worker shutdown failed.");
|
|
}
|
|
}
|
|
if (legacyEventReceiver?.status === "started") {
|
|
try {
|
|
await legacyEventReceiver.handle.close();
|
|
} catch (legacyError) {
|
|
workerLogger.error({ error: legacyError, signal }, "Legacy-event receiver shutdown failed.");
|
|
}
|
|
}
|
|
if (platformNotificationQueue) {
|
|
try {
|
|
await platformNotificationQueue.close();
|
|
} catch (platformNotificationQueueError) {
|
|
workerLogger.error({ error: platformNotificationQueueError, signal }, "Platform notification queue shutdown failed.");
|
|
}
|
|
}
|
|
if (pluginHookFanout?.status === "started") {
|
|
try {
|
|
await pluginHookFanout.handle.close();
|
|
} catch (pluginHookError) {
|
|
workerLogger.error({ error: pluginHookError, signal }, "Plugin-hook fanout shutdown failed.");
|
|
}
|
|
}
|
|
if (pluginCapabilityUse?.status === "started") {
|
|
try {
|
|
await pluginCapabilityUse.handle.close();
|
|
} catch (pluginCapabilityError) {
|
|
workerLogger.error({ error: pluginCapabilityError, signal }, "Plugin capability-use worker shutdown failed.");
|
|
}
|
|
}
|
|
if (pluginHookInvokeQueue) {
|
|
try {
|
|
await pluginHookInvokeQueue.close();
|
|
} catch (pluginHookQueueError) {
|
|
workerLogger.error({ error: pluginHookQueueError, signal }, "Plugin-hook invoke queue shutdown failed.");
|
|
}
|
|
}
|
|
if (pluginAuditRetentionWorker) {
|
|
try {
|
|
await pluginAuditRetentionWorker.close();
|
|
} catch (retentionError) {
|
|
workerLogger.error({ error: retentionError, signal }, "plugin-audit-retention worker shutdown failed.");
|
|
}
|
|
}
|
|
if (auditRetentionWorker) {
|
|
try {
|
|
await auditRetentionWorker.close();
|
|
} catch (retentionError) {
|
|
workerLogger.error({ error: retentionError, signal }, "audit-retention worker shutdown failed.");
|
|
}
|
|
}
|
|
if (filePreviewQueue) {
|
|
try {
|
|
await filePreviewQueue.close();
|
|
} catch (previewQueueError) {
|
|
workerLogger.error({ error: previewQueueError, signal }, "File-preview producer queue shutdown failed.");
|
|
}
|
|
}
|
|
if (telephonyWebhookQueue) {
|
|
try {
|
|
await telephonyWebhookQueue.close();
|
|
} catch (telephonyWebhookQueueError) {
|
|
workerLogger.error({ error: telephonyWebhookQueueError, signal }, "Telephony webhook producer queue shutdown failed.");
|
|
}
|
|
}
|
|
if (tenantDeletionExportQueue) {
|
|
try {
|
|
await tenantDeletionExportQueue.close();
|
|
} catch (tenantDeletionExportQueueError) {
|
|
workerLogger.error({ error: tenantDeletionExportQueueError, signal }, "Tenant deletion export queue shutdown failed.");
|
|
}
|
|
}
|
|
if (telephonyWebhookWorker) {
|
|
try {
|
|
await telephonyWebhookWorker.close();
|
|
} catch (telephonyWebhookError) {
|
|
workerLogger.error({ error: telephonyWebhookError, signal }, "Telephony webhook worker shutdown failed.");
|
|
}
|
|
}
|
|
if (automationExecutorWorker) {
|
|
try {
|
|
await automationExecutorWorker.close();
|
|
} catch (automationExecutorError) {
|
|
workerLogger.error({ error: automationExecutorError, signal }, "Automation forward-executor worker shutdown failed.");
|
|
}
|
|
}
|
|
await shutdownWorkerRuntime(runtimeToClose, { logger: workerLogger, reason: signal, reporter: workerErrorReporter });
|
|
if (adminServer) {
|
|
await adminServer.close();
|
|
}
|
|
if (healthServer) {
|
|
await healthServer.close();
|
|
}
|
|
if (redisHealthProbe) {
|
|
await redisHealthProbe.close();
|
|
}
|
|
try {
|
|
await closeSharedStore();
|
|
} catch (sharedStoreError) {
|
|
workerLogger.error({ error: sharedStoreError, signal }, "Worker shared-store shutdown failed.");
|
|
}
|
|
process.exit(0);
|
|
} catch (error) {
|
|
workerLogger.error({ error, signal }, "Worker shutdown failed.");
|
|
process.exit(1);
|
|
}
|
|
}
|