import { updateLegacyEventWebhookAttempt as updateLegacyEventWebhookAttemptInDb, type LegacyEventWebhookAttemptUpdate, } from "@hivegrid/db/event-webhook-attempts.server"; import { listExpiredCompletedExportJobs as listExpiredCompletedExportJobsInDb, listSoftDeletedExportJobsForHardDelete as listSoftDeletedExportJobsForHardDeleteInDb, markExportJobExpired as markExportJobExpiredInDb, markExportJobFailed as markExportJobFailedInDb, markExportJobHardDeleteFailed as markExportJobHardDeleteFailedInDb, markExportJobHardDeleteSucceeded as markExportJobHardDeleteSucceededInDb, markExportJobPaused as markExportJobPausedInDb, markExportJobResumed as markExportJobResumedInDb, markExportJobRunning as markExportJobRunningInDb, markExportJobSucceeded as markExportJobSucceededInDb, type ExportJobExpiredInput, type ExportJobFailedInput, type ExportJobHardDeleteFailedInput, type ExportJobHardDeleteSucceededInput, type ExportJobPausedInput, type ExportJobResumedInput, type ExportJobRetentionListInput, type ExportJobRetentionRecord, type ExportJobRunningInput, type ExportJobSucceededInput, } from "@hivegrid/db/export-jobs.server"; import { listExpiredGdprExportArtifacts as listExpiredGdprExportArtifactsInDb, markGdprExportArtifactDeleted as markGdprExportArtifactDeletedInDb, markGdprExportArtifactExpired as markGdprExportArtifactExpiredInDb, markGdprExportFailed as markGdprExportFailedInDb, markGdprExportProcessing as markGdprExportProcessingInDb, markGdprExportReady as markGdprExportReadyInDb, type GdprExportRetentionRecord, type GdprExportArtifactRetentionInput, type MarkGdprExportFailedInput, type MarkGdprExportProcessingInput, type MarkGdprExportReadyInput, type MarkGdprExportReadyResult, } from "@hivegrid/db/privacy-gdpr.server"; import { markImportJobFailed as markImportJobFailedInDb, markImportJobRunning as markImportJobRunningInDb, markImportJobSucceeded as markImportJobSucceededInDb, type ImportJobFailedInput, type ImportJobRunningInput, type ImportJobSucceededInput, } from "@hivegrid/db/import-jobs.server"; import { runAdminSalesTaskReminderSweep as runAdminSalesTaskReminderSweepInDb } from "@hivegrid/db/admin-sales-tasks.server"; import { runAdminBookingReminderSweep as runAdminBookingReminderSweepInDb } from "@hivegrid/db/admin-booking-reminders.server"; import { syncAdminCalendarBusySnapshots } from "@hivegrid/integrations/admin-calendar-sync-worker.server"; import { redactExpiredAdminEmailEventRawData as redactExpiredAdminEmailEventRawDataInDb } from "@hivegrid/db/admin-email-events.server"; import { createAdminPlatformNotificationDelivery as createAdminPlatformNotificationDeliveryInDb, createNotificationDelivery as createNotificationDeliveryInDb, createPlatformNotificationDelivery as createPlatformNotificationDeliveryInDb, getUserNotificationPreferences as getUserNotificationPreferencesInDb, markAdminPlatformNotificationDeliveryStatus as markAdminPlatformNotificationDeliveryStatusInDb, markNotificationDeliveryStatus as markNotificationDeliveryStatusInDb, markPlatformNotificationDeliveryStatus as markPlatformNotificationDeliveryStatusInDb, redactNotificationDiagnosticText, type CreateAdminPlatformNotificationDeliveryInput, type CreateNotificationDeliveryInput, type CreatePlatformNotificationDeliveryInput, type MarkAdminPlatformNotificationDeliveryStatusInput, type MarkNotificationDeliveryStatusInput, type MarkPlatformNotificationDeliveryStatusInput, } from "@hivegrid/db/notifications.server"; import { resolvePlatformLeadNotificationRecipient as resolvePlatformLeadNotificationRecipientInDb } from "@hivegrid/db/platform-lead-notifications.server"; import { deleteDoneOutboxEvents as deleteDoneOutboxEventsInDb, type DeleteDoneOutboxEventsOptions, type DeleteDoneOutboxEventsResult, } from "@hivegrid/db/outbox-retention"; import { type EnvRecord } from "@hivegrid/config"; import type { ValidationReport } from "@hivegrid/imports"; import { createMailjetProviderFromEnv, type MailMetricsHook, type MailProvider } from "@hivegrid/mail"; import { publishOutboxEvent, type OutboxPublisherTransaction } from "@hivegrid/db/outbox"; import { runAutomationSchedulerForTenantWithTx } from "@hivegrid/db/automation-scheduler.server"; import { buildWorkflowTriggerOutboxEvent, queueNames, type ExportJob, type AdminSalesContactsExportJob, type AdminSalesContactsImportJob, type FilePreviewJob, type FileImportJob, type ImportJob, type KwpArchiveDiscoveryJob, type KwpArchiveFileSyncJob, type KwpContractSentinelJob, type KwpProjectCatalogSyncJob, type KwpProjectSyncJob, type PluginInvokeJob, type PlatformNotificationJob, type QueueName, type QueuePayload, type TypedProcessor, } from "@hivegrid/queue"; import { assertGdprExportArtifactKey, authorizeStorageObjectAccess, type DeleteObjectInput, type DeleteObjectResult, type StorageScanDiagnostic, type StorageScanResult, } from "@hivegrid/storage"; import { isCriticalNotificationCategory, mapWithConcurrency, parsePortalStatusMailDeliveryMetadata, type NotificationClient } from "@hivegrid/foundation"; import { OutboundHttpUrlValidationError, fetchOutboundHttpWithRedirects, type OutboundHttpAddressResolver } from "@hivegrid/foundation/outbound-http"; import { runWithRlsContext } from "@hivegrid/db/rls-context.server"; import { notifyWithTenantRlsContext } from "./notify-rls.js"; import type { SharedStore } from "@hivegrid/shared-store"; import { UnrecoverableError } from "bullmq"; import { hasValidWorkspaceOwnerInvitationMailContract, isRedactedWorkspaceOwnerInvitationMailPayload, isTerminalWorkspaceOwnerInvitationMailPayload, isWorkspaceOwnerInvitationMailIntent, readWorkspaceOwnerInvitationRecoveryBinding, redactTerminalWorkspaceOwnerInvitationMailPayload, redactWorkspaceOwnerInvitationError, } from "./security/workspace-owner-invitation-secret.js"; import { abandonWorkspaceOwnerInvitationDelivery, acceptWorkspaceOwnerInvitationDelivery, claimWorkspaceOwnerInvitationDelivery, scheduleWorkspaceOwnerInvitationTerminalRecovery, terminalizeWorkspaceOwnerInvitationDelivery, type WorkspaceOwnerInvitationDeliveryClaimResult, } from "./security/workspace-owner-invitation-terminalization.js"; import { hasValidPortalSetupInvitationMailContract, isMaterializedPortalSetupInvitationMailPayload, isPortalSetupMailIntent, isPortalSetupInvitationProviderAccepted, isRedactedPortalSetupInvitationMailPayload, isTerminalPortalSetupInvitationMailPayload, readPortalSetupInvitationRecoveryBinding, redactPortalSetupInvitationError, redactTerminalPortalSetupInvitationMailPayload, } from "./security/portal-setup-invitation-secret.js"; import { acceptPortalSetupInvitationDelivery, claimPortalSetupInvitationDelivery, schedulePortalSetupInvitationTerminalRecovery, terminalizePortalSetupInvitationDelivery, type PortalSetupInvitationDeliveryClaimResult, } from "./security/portal-setup-invitation-terminalization.js"; import { drainCalendarWebhookEventsForTenant, refreshCalendarTokensForTenant, renewCalendarWebhooksForTenant, syncInternalCalendarEventsForTenant, } from "@hivegrid/integrations/calendar-worker-tasks.server"; import { cleanupEmailWebhookEventsForTenant, drainEmailWebhookEventsForTenant, pollStaleEmailMailboxesForTenant, refreshEmailTokensForTenant, renewEmailWebhooksForTenant, runEmailInitialSyncForTenant, } from "@hivegrid/integrations/email-worker-tasks.server"; import { checkAdminSendingDomains, deliverDueAdminEmails, syncClaimedAdminMailboxes, } from "@hivegrid/integrations"; import { createWorkerAdminEmailAttachmentProcessor } from "./admin-email-attachments.js"; import { runAdminCallRecordingProcessing, runAdminCallRecordingRetention, } from "./admin-call-recordings.js"; import { runAdminOutboundCalls } from "./admin-outbound-calls.js"; import { createEmailAutoMatchHook } from "@hivegrid/integrations/email-match-hook.server"; import { ensureKwpAccessTokenForConnection } from "@hivegrid/integrations/kwp-resource-client.server"; import { cleanupExpiredKwpPendingAttemptsForTenant, executeKwpHealthProbe, listKwpHealthConnectionIdsForTenant, persistKwpHealthProbe, prepareKwpHealthProbeForConnection, refreshExpiringTokensForTenant, sweepConnectionHealthForTenant, } from "@hivegrid/integrations/worker-tasks.server"; import { autoArchiveProjectsForTenantWithTx } from "@hivegrid/db/project-auto-archive.server"; import { evaluateBillingDunning } from "@hivegrid/db/billing-domain.server"; import { createAuditEvent } from "@hivegrid/db/audit-events.server"; import { archiveProjectChannelIfEnabledWithTx } from "@hivegrid/chat/project-sync.server"; import { withTenantRLS } from "./db/tenantRls.js"; import { runActiveTenantSweep, throwOnActiveTenantSweepFailures, type ActiveTenantReader } from "./jobs/active-tenant-sweep.js"; import { withDistributedJobLease } from "./jobs/distributed-job-lease.js"; import { getSharedStore } from "./shared-store.js"; import { runDsgvoGraceCron, type DsgvoGraceCandidate } from "./jobs/dsgvo-grace-cron.js"; import { reminderChannel, reminderMessageKey, reminderScanBounds, reminderSourceKey, selectReminderWindows } from "./jobs/task-reminder-cron.js"; import { audienceMatches, nextReminderDue, HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS } from "./jobs/hr-upload-reminder-cron.js"; import { createDnsDomainResolver, decideDomainProbeStatus, type DomainDnsResolver } from "./jobs/email-domain-probe-cron.js"; import { handlePluginInvokeJob, type PluginInvokeDeps } from "./jobs/plugin-invoke.js"; import { reconcileWorkflowExecutionStalls, type WorkflowExecutionStalledReconcileInput, type WorkflowExecutionStalledReconcileResult, type WorkflowExecutionStalledReconcilerPrisma, } from "./jobs/workflow-execution-stalled-reconciler.js"; import { isPrivacyGdprExportType, PrivacyGdprExportSizeLimitExceededError, resolvePrivacyGdprExportSubject, type PrivacyGdprExportRunResult, } from "./jobs/privacy-gdpr-export.js"; import { isPortalGdprExportType, PortalGdprExportSizeLimitExceededError, type PortalGdprExportRunResult } from "./jobs/portal-gdpr-export.js"; import { PrivacyGdprExportPutOutcomeUnknownError } from "./jobs/privacy-gdpr-export-bootstrap.js"; import { TenantExportQuiescedError, type TenantExportBarrierCheckpoint } from "./jobs/tenant-export-quiescence.js"; import { runPortalGdprExportRetentionSweep, type PortalGdprExportRetentionPrisma, type PortalGdprExportRetentionResult, } from "./jobs/portal-gdpr-export-retention.js"; import { notifyPortalStatusMailPermanentFailure } from "./jobs/portal-status-mail-store.js"; import type { PortalStatusMailFailureInput } from "./jobs/portal-status-mail.js"; const DEFAULT_WORKER_FETCH_TIMEOUT_MS = 10_000; const WEBHOOK_RESPONSE_BODY_LIMIT_BYTES = 4_096; const WEBHOOK_REDIRECT_LIMIT = 5; const EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY = 5; const EXPORT_ARTIFACT_RETENTION_DELETE_TIMEOUT_MS = 60_000; export type HandlerResult = { handled: true; action: string; details?: Record; }; export type WebhookDeliveryMetricsHook = (input: { status: "success" | "retry" | "dlq" | "rejected" }) => void; export type HandlerDependencies = { mailProvider?: MailProvider; /** Notifies the responsible internal TenantUser after final portal-mail delivery failure. */ notifyPortalStatusMailFailure?: (input: PortalStatusMailFailureInput) => Promise; /** Invalidates an undeliverable OWNER capability on the final mail attempt. */ terminalizeOwnerInvitation?: (input: { tenantId: string; payload: unknown }) => Promise; /** Claims the one permitted OWNER provider attempt in the originating DB outbox row. */ claimOwnerInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise; /** Marks the claimed OWNER provider attempt as accepted. */ acceptOwnerInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise; /** Invalidates a claimed OWNER delivery whose provider outcome is ambiguous. */ abandonOwnerInvitationDelivery?: (input: { tenantId: string; payload: unknown; lastError: string }) => Promise; /** Durably rearms the originating DB outbox row when invalidation itself fails. */ scheduleOwnerInvitationRecovery?: (input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => Promise; /** Claims the one permitted customer-portal setup provider attempt. */ claimPortalSetupInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise; /** Marks a claimed customer-portal setup delivery as provider-accepted. */ acceptPortalSetupInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise; /** Revokes the setup capability and terminalizes its exact outbox row. */ terminalizePortalSetupInvitation?: (input: { tenantId: string; payload: unknown; lastError?: string }) => Promise; /** Durably rearms setup-capability invalidation when the database is transiently unavailable. */ schedulePortalSetupInvitationRecovery?: (input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => Promise; /** * Optional metrics hook forwarded to the lazily-constructed Mailjet * provider. When the worker entry wires this to the shared Prometheus * registry, every send increments mail_sent_total{provider, status}. */ mailMetricsHook?: MailMetricsHook; /** * Optional metrics hook fired for each outbound webhook attempt. Status: * - success : 2xx response * - retry : retryable failure (5xx, transient network) below the * attempts cap, BullMQ will requeue * - dlq : terminal failure (final attempt or non-retryable status) * - rejected : URL validation failure (SSRF guard, etc.) */ webhookDeliveryMetricsHook?: WebhookDeliveryMetricsHook; webhookAddressResolver?: OutboundHttpAddressResolver; fetch?: typeof fetch; env?: EnvRecord; /** Test/deployment seam for replica-wide maintenance overlap protection. */ tenantMaintenanceLeaseStore?: SharedStore; /** Schedules due KWP project/contract work onto the dedicated KWP queue. */ runKwpProjectMaintenance?: () => Promise<{ inspectedTenants: number; projectJobs: number; catalogJobs: number; contractJobs: number; failures: number; }>; /** Import-group provider work. Operations run under resource-scoped Redis leases. */ syncKwpProjectBinding?: (payload: KwpProjectSyncJob) => Promise<{ runId: string; state: "synced" | "unchanged" | "missing" | "quarantined"; projectId: string; stageChanged: boolean; }>; syncKwpProjectCatalog?: (payload: KwpProjectCatalogSyncJob) => Promise<{ discoveredCount: number; hydratedCount: number; processed: number; hasMore: boolean; status: "pending" | "syncing" | "ready" | "partial" | "paused"; }>; enqueueKwpProjectCatalogContinuation?: (payload: KwpProjectCatalogSyncJob) => Promise; syncKwpProjectInbound?: (payload: KwpProjectSyncJob) => Promise<{ runId: string; projectId: string; appointments: number; tasks: number; materialPositions: number; failedAreas: readonly ("appointments" | "tasks" | "material")[]; changed: boolean; }>; syncKwpArchiveDiscovery?: (payload: KwpArchiveDiscoveryJob) => Promise<{ archiveFiles: readonly KwpArchiveFileSyncJob["archiveFile"][]; processedFolders: number; cursor: KwpArchiveDiscoveryJob["cursor"] | null; }>; enqueueKwpArchiveDiscovery?: ( payload: KwpProjectSyncJob | KwpArchiveDiscoveryJob, cursor?: KwpArchiveDiscoveryJob["cursor"], ) => Promise; enqueueKwpArchiveFileJobs?: ( payload: KwpArchiveDiscoveryJob, files: readonly KwpArchiveFileSyncJob["archiveFile"][], ) => Promise; syncKwpArchiveFile?: (payload: KwpArchiveFileSyncJob) => Promise<{ runId: string; fileId: string; projectId: string; storageKey: string; state: "created" | "replayed"; }>; runKwpContractSentinel?: (payload: KwpContractSentinelJob) => Promise<{ status: "initialized" | "unchanged" | "paused"; fingerprint: string; }>; /** Test seam for bounded cross-tenant maintenance enumeration. */ activeTenantReader?: ActiveTenantReader; /** General-group service for bounded telephony webhook retention and event recovery. */ runTelephonyWebhookMaintenance?: () => Promise<{ deletedCompleted: number; deletedFailed: number; recoveredPending: number; recoveredFailed: number; inspectedTenants: number; }>; /** General-group service for bounded, tenant-RLS Mollie acquisition recovery. */ runProductAcquisitionReconcile?: (options?: { take?: number }) => Promise<{ inspected: number; expired: number; recoveredProviderPayments: number; refreshed: number; appliedTerminal: number; quarantined: number; raced: number; inspectedTenants: number; }>; updateLegacyEventWebhookAttempt?: (attempt: { id: string; tenantId: string | null }, data: LegacyEventWebhookAttemptUpdate) => Promise; scanService?: { scanObject(target: WorkerStorageScanTarget): Promise; }; /** * AK-P4a §5 — writes the scan verdict (clean|infected) back onto * File.scanStatus so the fail-closed media proxy (AD-23) can serve the * object. Composed in the `general` group bootstrap (jobs/file-scan-status.ts) * with a Prisma seam. Jobs without a fileId are legacy scan-only imports; * jobs with a fileId fail/retry when persistence is not configured. */ fileScanStatusWriter?: { recordVerdict(input: { fileId: string; storageKey: string; tenantId?: string; verdict: "clean" | "infected" }): Promise; }; /** * A clean AV verdict is the only producer path for an upload's initial * preview. Enqueue failure rejects the scan job, so BullMQ retries the * scan/write/enqueue sequence; the preview job id is deterministic. */ filePreviewEnqueuer?: { enqueueAfterCleanScan(input: { bucket: string; contentType: string; fileId: string; storageKey: string; tenantId: string }): Promise; }; /** * AK-P4a §4 — file-preview generator. Composed in the `general` worker * group bootstrap (jobs/file-preview.ts) with Prisma + storage seams. * The handler stays thin: it delegates to this service, which loads the * object, renders the preview (HEIC->JPG, image->WebP, PDF->PNG) and * writes previewStatus + preview/thumbnail keys back to the File row. * When unset, the handler throws so the job dead-letters loudly rather * than silently leaving previewStatus stuck on `generating`. */ filePreviewService?: { generatePreview(payload: FilePreviewJob): Promise; }; exportService?: { runExport(payload: ExportJob): Promise; }; privacyGdprExportService?: { withTenantBarrier(payload: ExportJob, writer: (checkpoint: TenantExportBarrierCheckpoint) => Promise): Promise; resolveReady(payload: ExportJob): Promise<{ artifactBucket: string; artifactKey: string; rowCount: null; sizeBytes: number; expiresAt: Date; } | null>; resolveReclaimableProcessing(payload: ExportJob): Promise<{ processingStartedAt: Date; attempt: { attemptId: string; artifactBucket: string; artifactKey: string }; } | null>; createAttempt(payload: ExportJob): { attemptId: string; artifactBucket: string; artifactKey: string }; runExport(payload: ExportJob, attempt: { attemptId: string; artifactBucket: string; artifactKey: string }): Promise; deleteArtifact(attempt: { attemptId: string; artifactBucket: string; artifactKey: string }): Promise; terminalizeReadyAfterQuiescence(input: { payload: ExportJob; result: PrivacyGdprExportRunResult }): Promise; }; /** VA-P26b — separate PortalUser JSON export; never uses a TenantUser. */ portalGdprExportService?: { runExport(payload: ExportJob): Promise; }; /** Dedicated 15-minute PortalUser export retention sweep; never part of the daily portal lifecycle. */ portalGdprExportRetentionService?: { runSweep(input: { now: Date; take?: number }): Promise; }; importService?: { runImport(payload: ImportJob): Promise; }; adminSalesDataService?: { runImport(payload: AdminSalesContactsImportJob): Promise<{ readonly totalRows: number; readonly acceptedRows: number; readonly rejectedRows: number; }>; runExport(payload: AdminSalesContactsExportJob): Promise<{ readonly rowCount: number; readonly artifactBucket: string; readonly artifactKey: string; readonly expiresAt: Date; }>; }; storageDeleteService?: { deleteObject(input: DeleteObjectInput): Promise; }; listSoftDeletedExportJobsForHardDelete?: (data: ExportJobRetentionListInput) => Promise; listExpiredCompletedExportJobs?: (data: ExportJobRetentionListInput) => Promise; markExportJobExpired?: (data: ExportJobExpiredInput) => Promise; markExportJobHardDeleteSucceeded?: (data: ExportJobHardDeleteSucceededInput) => Promise; markExportJobHardDeleteFailed?: (data: ExportJobHardDeleteFailedInput) => Promise; markExportJobPaused?: (data: ExportJobPausedInput) => Promise; markExportJobResumed?: (data: ExportJobResumedInput) => Promise; markExportJobRunning?: (data: ExportJobRunningInput) => Promise; markExportJobSucceeded?: (data: ExportJobSucceededInput) => Promise; markExportJobFailed?: (data: ExportJobFailedInput) => Promise; markGdprExportProcessing?: (data: MarkGdprExportProcessingInput) => Promise; markGdprExportReady?: (data: MarkGdprExportReadyInput) => Promise; markGdprExportFailed?: (data: MarkGdprExportFailedInput) => Promise; listExpiredGdprExportArtifacts?: (data: { tenantId: string; now: Date; take?: number }) => Promise; markGdprExportArtifactExpired?: (data: GdprExportArtifactRetentionInput) => Promise; markGdprExportArtifactDeleted?: (data: Pick) => Promise; markImportJobRunning?: (data: ImportJobRunningInput) => Promise; markImportJobSucceeded?: (data: ImportJobSucceededInput) => Promise; markImportJobFailed?: (data: ImportJobFailedInput) => Promise; getUserNotificationPreferences?: (filters: { tenantId: string; userId: string; channel?: string | null; category?: string | null }) => Promise; createNotificationDelivery?: (data: CreateNotificationDeliveryInput) => Promise; markNotificationDeliveryStatus?: (data: MarkNotificationDeliveryStatusInput) => Promise; createPlatformNotificationDelivery?: (data: CreatePlatformNotificationDeliveryInput) => Promise; createAdminPlatformNotificationDelivery?: ( data: CreateAdminPlatformNotificationDeliveryInput ) => Promise; markPlatformNotificationDeliveryStatus?: (data: MarkPlatformNotificationDeliveryStatusInput) => Promise; markAdminPlatformNotificationDeliveryStatus?: ( data: MarkAdminPlatformNotificationDeliveryStatusInput ) => Promise; runAdminSalesTaskReminderSweep?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly scanned: number; readonly dispatched: number }>; runAdminBookingReminderSweep?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly scanned: number; readonly dispatched: number; readonly failed: number; }>; runAdminCalendarSync?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly inspected: number; readonly synced: number; readonly failed: readonly string[]; }>; runAdminEmailDelivery?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly inspected: number; readonly claimed: number; readonly sent: number; readonly failed: readonly string[]; readonly blocked: number; readonly deferred: number; readonly recoveredUnknown: number; readonly noReplyReminders: { readonly scanned: number; readonly created: number; }; }>; redactExpiredAdminEmailEventRawData?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly inspected: number; readonly redacted: number; readonly deletedLinks: number; }>; runAdminEmailSync?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly inspected: number; readonly synced: number; readonly upserted: number; readonly replayed: number; readonly failed: readonly string[]; }>; runAdminEmailDomainCheck?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly inspected: number; readonly verified: number; readonly degraded: number; readonly failed: readonly string[]; }>; runAdminCallRecordingProcessing?: (input: { readonly take?: number; }) => Promise<{ readonly claimed: number; readonly processed: number }>; runAdminCallRecordingRetention?: (input: { readonly now: Date; readonly take?: number; }) => Promise<{ readonly claimed: number; readonly deleted: number; readonly failed: number; }>; runAdminOutboundCalls?: (input: { readonly take?: number; }) => Promise<{ readonly claimed: number; readonly accepted: number; readonly failed: number; }>; resolvePlatformLeadNotificationRecipient?: (data: { userId: string; leadId: string }) => Promise<{ eligible?: boolean; email: string | null }>; /** * Phase 10 — DSGVO grace-period cron service. The maintenance handler * looks up overdue User rows and dispatches cascade-deletions through * this slot. When unset, the handler falls back to a Prisma-backed * implementation that lists candidates and logs them without running * the destructive cascade — the cascade-runtime wiring lands in a * follow-up. The split keeps the cron observable from day-one without * blocking on the full cascade implementation. */ dsgvoGraceCronService?: { listOverdueUsers: (now: Date) => Promise; enqueueDeletion: (input: { userId: string; tenantId: string | null; requestId: string; actorType: "system"; actorId: string; mode: "pseudonymize"; reason: string; }) => Promise<{ jobId?: string }>; }; /** * AK-P21 — best-effort "export ready" mail to the requesting user. Wired by * the import-group bootstrap next to privacyGdprExportService; failures are * logged and never fail the export job. */ sendGdprExportReadyMail?: (input: { tenantId: string; exportJobId: string; requestedByUserId: string; scope: string; expiresAt: Date }) => Promise; /** * AK-P21 — tenant hard-delete cron service. The maintenance handler runs * one scan over due TenantDeletionRequest rows (grace period elapsed) and * executes the orchestrated hard delete (billing cancel, storage prefix, * search documents, DEK crypto-shredding, tenant rows). When unset the * handler acks the cron tick with an explicit skip log. */ privacyTenantDeletionService?: { runScan(now?: Date): Promise<{ scanned: number; completed: number; failed: number; failures: Array<{ requestId: string; tenantId: string; error: string }>; }>; }; /** VA-P26b — bounded customer-portal deletion, retention and access scan. */ portalLifecycleService?: { runScan(now?: Date): Promise<{ inspectedTenants: number; deletionScanned: number; deletionCompleted: number; exportArtifactsExpired: number; inactivityWarnings: number; inactivityDisabled: number; projectCompletionMails: number; projectAccessesClosed: number; failed: number; }>; }; deleteDoneOutboxEvents?: (data: DeleteDoneOutboxEventsOptions) => Promise; reconcileWorkflowExecutionStalls?: (data: WorkflowExecutionStalledReconcileInput) => Promise; /** * Phase 06 #21 / F-015 — production dependencies for the plugin-invoke * handler. The `plugin` worker group bootstrap (`apps/worker/src/jobs/ * plugin-invoke-bootstrap.ts`) composes the real Prisma / filesystem / * SharedStore / sandbox seams and injects them here. When the slot is * unset (e.g. a misconfigured `general` worker receives a plugin-invoke * job because someone changed group routing) the registry entry throws * so the job dead-letters loudly rather than being silently acked. */ pluginInvokeDeps?: PluginInvokeDeps; /** * Optional structured logger used by handlers that need to emit * non-error progress (currently only the dsgvo-grace-cron branch). A * pino-shaped surface keeps this compatible with the worker bootstrap * logger without dragging the pino type-graph into this module. */ logger?: { info: (fields: Record, msg: string) => void; error: (fields: Record, msg: string) => void; }; }; type WorkerExportRunResult = { artifactBucket: string; artifactKey: string; rowCount: number; expiresAt: Date; }; type WorkerImportRunResult = { rowCount: number; acceptedRows: number; rejectedRows: number; duplicateCount: number; validationReport?: Pick; errorReportBucket?: string; errorReportKey?: string; }; type WorkerStorageScanTarget = { bucket: string; key: string; tenantId?: string; uploadId?: string; contentType?: string; metadata?: Record; }; export type FilePreviewServiceResult = { /** Terminal preview state written to the File row by the service. */ previewStatus: "ready" | "failed" | "not_supported"; previewStorageKey?: string; thumbnailStorageKey?: string; /** Optional diagnostic reason (e.g. why generation failed). */ reason?: string; }; class WebhookDeliveryError extends Error {} class WebhookUrlValidationError extends Error {} class NotificationHandlerError extends Error { readonly action = "notification.rejected"; } class UnsupportedWorkerJobError extends Error { readonly action = "worker.unsupported"; readonly details: Record; constructor(message: string, details: Record) { super(message); this.name = "UnsupportedWorkerJobError"; this.details = details; } } class PluginInvokePayloadError extends Error { readonly action = "plugin.invoke.invalid_payload"; readonly details: Record; constructor(message: string, details: Record) { super(message); this.name = "PluginInvokePayloadError"; this.details = details; } } function getNotificationDeliveryId(record: unknown): string | undefined { if (!record || typeof record !== "object" || typeof (record as { id?: unknown }).id !== "string") { return undefined; } const id = (record as { id: string }).id.trim(); return /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/u.test(id) ? id : undefined; } function getNotificationDeliveryStatus(record: unknown): string | undefined { return record && typeof record === "object" && typeof (record as { status?: unknown }).status === "string" ? (record as { status: string }).status : undefined; } async function createSkippedNotificationDelivery( createNotificationDelivery: NonNullable, payload: QueuePayload, payloadMetadata: Record, reason: string, sourceKey?: string ) { return createNotificationDelivery({ tenantId: payload.tenantId ?? "", userId: payload.userId, channel: payload.channel, category: payload.category, messageKey: payload.messageKey, sourceKey, status: "SKIPPED", attemptsMade: 0, lastError: reason, payloadMetadata: { ...payloadMetadata, reason }, }); } function isNotificationPreferenceDisabled(preferences: unknown[]): boolean { return preferences.some((preference) => preference && typeof preference === "object" && (preference as { enabled?: unknown }).enabled === false); } function getNotificationErrorMessage(error: unknown): string { return redactNotificationDiagnosticText(error instanceof Error ? error.message : "Notification email delivery failed."); } function getPreferenceChannel(channel: string): "email" | "in-app" { if (channel === "email" || channel === "in-app") { return channel; } throw new NotificationHandlerError(`Unsupported notification channel: ${channel}`); } async function getNotificationPreferencesForPayload( payload: QueuePayload, getUserNotificationPreferences: NonNullable ): Promise { if (!payload.tenantId) { return []; } // Critical categories are delivered regardless of preferences — the same // rule foundation's isNotificationEnabled() applies to the in-process path. // Without this, a stale "critical" preference row could silence a locked // category (e.g. a GDPR export or deletion notice) on the queue path only. if (isCriticalNotificationCategory(payload.category)) { return []; } const channel = getPreferenceChannel(payload.channel); const category = payload.category ?? "default"; const preferences = await getUserNotificationPreferences({ tenantId: payload.tenantId, userId: payload.userId, channel, category, }); if (preferences.length > 0 || !payload.category || payload.category === "default") { return preferences; } return getUserNotificationPreferences({ tenantId: payload.tenantId, userId: payload.userId, channel, category: "default", }); } export class ScanHandlerError extends Error { readonly action = "scan.error"; readonly details: Record; constructor(message: string, details: Record) { super(message); this.name = "ScanHandlerError"; this.details = details; } } export class PreviewHandlerError extends Error { readonly action = "preview.error"; readonly details: Record; constructor(message: string, details: Record) { super(message); this.name = "PreviewHandlerError"; this.details = details; } } export class ExportHandlerError extends Error { readonly action = "export.error"; readonly details: Record; constructor(message: string, details: Record) { super(message); this.name = "ExportHandlerError"; this.details = details; } } class UnrecoverableExportHandlerError extends UnrecoverableError { readonly action = "export.error"; readonly details: Record; constructor(message: string, details: Record) { super(message); this.name = "UnrecoverableExportHandlerError"; this.details = details; } } export class ImportHandlerError extends Error { readonly action = "import.error"; readonly details: Record; constructor(message: string, details: Record) { super(message); this.name = "ImportHandlerError"; this.details = details; } } type HandlerExecutionContext = { attemptsMade?: number; attemptsLimit?: number; jobId?: string; updatePayload?: (payload: QueuePayload) => Promise; }; type Handler = ( payload: QueuePayload, dependencies: HandlerDependencies, context?: HandlerExecutionContext ) => Promise; type HandlerRegistry = { [Name in QueueName]: Handler; }; function getAttemptNumber(context?: { attemptsMade?: number }): number { return Math.max(1, (context?.attemptsMade ?? 0) + 1); } function getAttemptsLimit(context?: { attemptsLimit?: number }): number { return Math.max(1, context?.attemptsLimit ?? 1); } function isRetryableWebhookStatus(status: number): boolean { return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500; } function getPayloadNotificationDeliveryId(payload: QueuePayload): string | undefined { return typeof payload.metadata?.deliveryId === "string" ? payload.metadata.deliveryId : undefined; } function getNotificationDeliverySourceKey( payload: QueuePayload, context: HandlerExecutionContext | undefined ): string | undefined { const metadata = payload.metadata as Record | undefined; if (typeof metadata?.idempotencyKey === "string" && metadata.idempotencyKey.trim()) { return metadata.idempotencyKey.trim(); } if (context?.jobId?.trim()) { return `job:${context.jobId.trim()}`; } if (typeof payload.metadata?.requestId === "string" && payload.metadata.requestId.trim()) { return `request:${payload.metadata.requestId.trim()}`; } return undefined; } async function persistPayloadNotificationDeliveryId( payload: QueuePayload, context: HandlerExecutionContext | undefined, deliveryId: string | undefined ): Promise { if (!deliveryId || getPayloadNotificationDeliveryId(payload) || !context?.updatePayload) { return; } await context.updatePayload({ ...payload, metadata: { ...payload.metadata, deliveryId, }, }); } async function runPlatformNotificationHandler( payload: PlatformNotificationJob, deps: HandlerDependencies, context: HandlerExecutionContext | undefined ): Promise { if (payload.tenantId !== null) { throw new NotificationHandlerError("Platform notification jobs require tenantId=null."); } const userId = payload.userId?.trim() || null; const adminUserId = payload.adminUserId?.trim() || null; if (Boolean(userId) === Boolean(adminUserId)) { throw new NotificationHandlerError( "Platform notification jobs require exactly one recipient identity." ); } const recipientId = userId ?? adminUserId!; if (!payload.sourceKey?.trim()) { throw new NotificationHandlerError("Platform notification jobs require sourceKey."); } getPreferenceChannel(payload.channel); const createUserDelivery = deps.createPlatformNotificationDelivery ?? createPlatformNotificationDeliveryInDb; const createAdminDelivery = deps.createAdminPlatformNotificationDelivery ?? createAdminPlatformNotificationDeliveryInDb; const markUserDelivery = deps.markPlatformNotificationDeliveryStatus ?? markPlatformNotificationDeliveryStatusInDb; const markAdminDelivery = deps.markAdminPlatformNotificationDeliveryStatus ?? markAdminPlatformNotificationDeliveryStatusInDb; const payloadDeliveryId = getPayloadNotificationDeliveryId(payload); const delivery = adminUserId ? await createAdminDelivery({ adminUserId, channel: payload.channel === "email" ? "EMAIL" : "IN_APP", category: payload.category, messageKey: payload.messageKey, sourceKey: payload.sourceKey, payloadMetadata: payload.metadata, }) : await createUserDelivery({ userId: userId!, channel: payload.channel === "email" ? "EMAIL" : "IN_APP", category: payload.category, messageKey: payload.messageKey, sourceKey: payload.sourceKey, payloadMetadata: payload.metadata, }); const markDelivery = ( input: Omit ) => adminUserId ? markAdminDelivery({ ...input, adminUserId }) : markUserDelivery({ ...input, userId: userId! }); const persistedDeliveryId = getNotificationDeliveryId(delivery); if (payloadDeliveryId && persistedDeliveryId && payloadDeliveryId !== persistedDeliveryId) { throw new NotificationHandlerError("Platform notification delivery intent does not match the queued delivery id."); } const deliveryId = persistedDeliveryId ?? payloadDeliveryId; await persistPayloadNotificationDeliveryId(payload, context, deliveryId); if (!deliveryId) { throw new NotificationHandlerError("Platform notification delivery intent did not return an id."); } const existingStatus = getNotificationDeliveryStatus(delivery); if (existingStatus === "SENT" || existingStatus === "READ") { return { handled: true, action: "notification.platform.deduplicated", details: { deliveryId, userId: recipientId, channel: payload.channel, messageKey: payload.messageKey }, }; } const attemptNumber = getAttemptNumber(context); const taskId = typeof payload.metadata?.taskId === "string" ? payload.metadata.taskId.trim() : ""; const leadId = typeof payload.metadata?.leadId === "string" ? payload.metadata.leadId.trim() : ""; const emailMessageId = typeof payload.metadata?.emailMessageId === "string" ? payload.metadata.emailMessageId.trim() : ""; if (!leadId && !taskId && !emailMessageId) { throw new NotificationHandlerError("Platform notifications require a supported subject."); } if ((taskId || emailMessageId || adminUserId) && payload.channel !== "in-app") { throw new NotificationHandlerError( "Admin platform notifications only support the in-app channel." ); } let recipient: { eligible?: boolean; email: string | null } = { eligible: true, email: null, }; if (leadId && userId) { const resolvePlatformLeadNotificationRecipient = deps.resolvePlatformLeadNotificationRecipient ?? resolvePlatformLeadNotificationRecipientInDb; recipient = await resolvePlatformLeadNotificationRecipient({ userId, leadId, }); const recipientEligible = recipient.eligible ?? Boolean(recipient.email); if ( !recipientEligible || (payload.channel === "email" && !recipient.email) ) { await markDelivery({ id: deliveryId, status: "SKIPPED", attemptsMade: attemptNumber, lastError: "Platform lead notification recipient is no longer eligible.", }); return { handled: true, action: payload.channel === "email" ? "notification.platform.email.skipped" : "notification.platform.in-app.skipped", details: { deliveryId, userId: recipientId, channel: payload.channel, messageKey: payload.messageKey, reason: "recipient_ineligible", }, }; } } if (payload.channel === "in-app") { await markDelivery({ id: deliveryId, status: "SENT", attemptsMade: attemptNumber, sentAt: new Date(), }); return { handled: true, action: "notification.platform.in-app.sent", details: { deliveryId, userId: recipientId, channel: payload.channel, messageKey: payload.messageKey }, }; } const recipientEmail = recipient.email; if (!recipientEmail) { throw new NotificationHandlerError("Platform lead email recipient resolution returned no deliverable address."); } const isTerminalFailure = attemptNumber >= getAttemptsLimit(context); let result: Awaited>; try { const provider = deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined); result = await provider.send({ to: [recipientEmail], subject: payload.subject, text: payload.text, html: payload.html, templateId: payload.templateId, idempotencyKey: `notification-delivery:${deliveryId}`, personalization: payload.templateData ? [{ email: recipientEmail, data: payload.templateData }] : undefined, }); } catch (error) { const diagnostic = getNotificationErrorMessage(error); try { await markDelivery({ id: deliveryId, status: isTerminalFailure ? "FAILED" : "PENDING", attemptsMade: attemptNumber, lastError: diagnostic, }); } catch { // The redacted provider failure below remains BullMQ's retry reason. } throw new NotificationHandlerError(diagnostic); } try { await markDelivery({ id: deliveryId, status: "SENT", attemptsMade: attemptNumber, providerMessageId: result.messageId ?? null, sentAt: new Date(), }); } catch (error) { throw new NotificationHandlerError(`Platform notification delivery status persistence failed: ${getNotificationErrorMessage(error)}`); } return { handled: true, action: "notification.platform.email.sent", details: { provider: result.provider, messageId: result.messageId, userId: recipientId, messageKey: payload.messageKey, deliveryId }, }; } export function buildHandlerRegistry(dependencies: HandlerDependencies = {}): HandlerRegistry { const deps = dependencies; return { [queueNames.email]: async (payload, _dependencies, context) => { if (payload.kind === "mail.transactional") { const portalSetupIntent = isPortalSetupMailIntent(payload); if ( portalSetupIntent && (!hasValidPortalSetupInvitationMailContract(payload) || (!isMaterializedPortalSetupInvitationMailPayload(payload) && !isRedactedPortalSetupInvitationMailPayload(payload))) ) { if (isTerminalMailAttempt(context) && context?.updatePayload) { await context.updatePayload(redactTerminalPortalSetupInvitationMailPayload(payload) as QueuePayload); } throw redactPortalSetupInvitationError(new Error("Portal setup invitation mail contract is invalid."), payload); } if (portalSetupIntent && isRedactedPortalSetupInvitationMailPayload(payload)) { if (isTerminalPortalSetupInvitationMailPayload(payload)) { await recoverTerminalPortalSetupMail(payload, deps); } else if (!isPortalSetupInvitationProviderAccepted(payload)) { await terminalizeFailedPortalSetupMail(payload, deps, context, "Portal setup invitation delivery outcome is ambiguous."); throw new UnrecoverableError("Portal setup invitation delivery outcome is ambiguous; capability rotation is required."); } return { handled: true, action: "mail.sent", details: { deduplicated: true }, }; } if (isRedactedWorkspaceOwnerInvitationMailPayload(payload)) { if (isTerminalWorkspaceOwnerInvitationMailPayload(payload)) { await recoverTerminalOwnerMail(payload, deps); } return { handled: true, action: "mail.sent", details: { deduplicated: true }, }; } const ownerIntent = isWorkspaceOwnerInvitationMailIntent(payload); let ownerClaim: WorkspaceOwnerInvitationDeliveryClaimResult | undefined; let portalSetupClaim: PortalSetupInvitationDeliveryClaimResult | undefined; let result: Awaited>; try { if (ownerIntent && !hasValidWorkspaceOwnerInvitationMailContract(payload)) { throw new Error("Workspace-owner invitation mail contract is invalid."); } if (portalSetupIntent && !isMaterializedPortalSetupInvitationMailPayload(payload)) { throw new Error("Portal setup invitation mail must be materialized only in the provider queue."); } const provider = deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined); const recipients = payload.to.map(addressToEmail); const providerInput = { from: payload.from, to: payload.to, cc: payload.cc, bcc: payload.bcc, replyTo: payload.replyTo, subject: payload.subject, text: payload.text, html: payload.html, templateId: payload.templateId, attachments: payload.attachments, tags: payload.tags, precedenceBulk: payload.precedenceBulk, sendAt: payload.sendAt, inReplyTo: payload.inReplyTo, references: payload.references, settings: portalSetupIntent ? { ...payload.settings, trackClicks: false, trackOpens: false, } : payload.settings, headers: payload.headers, listUnsubscribe: payload.listUnsubscribe, ...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}), personalization: buildMailPersonalization(recipients, payload.templateData), }; if (ownerIntent) { const claim = deps.claimOwnerInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => claimWorkspaceOwnerInvitationDelivery(input)); ownerClaim = await claim({ tenantId: payload.tenantId ?? "", payload }); if (ownerClaim.state === "not_ready") { throw new Error("Workspace-owner invitation delivery outbox is not ready."); } if (ownerClaim.state === "uncertain") { throw new Error("Workspace-owner invitation delivery outcome is ambiguous; capability rotation is required."); } if (ownerClaim.state === "provider_accepted") { if (!context?.updatePayload) { throw new Error("Workspace-owner invitation mail job cannot persist its safe delivery marker."); } await context.updatePayload(ownerClaim.payload as QueuePayload); return { handled: true, action: "mail.sent", details: { deduplicated: true }, }; } } if (portalSetupIntent) { const claim = deps.claimPortalSetupInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => claimPortalSetupInvitationDelivery(input)); portalSetupClaim = await claim({ tenantId: payload.tenantId ?? "", payload }); if (portalSetupClaim.state === "not_ready") { throw new Error("Portal setup invitation delivery outbox is not ready."); } if (portalSetupClaim.state === "uncertain") { throw new Error("Portal setup invitation delivery outcome is ambiguous; capability rotation is required."); } if (portalSetupClaim.state === "inactive") { await terminalizeFailedPortalSetupMail(payload, deps, context, "Portal setup invitation capability is no longer active."); return { handled: true, action: "mail.skipped", details: { reason: "portal_setup_capability_inactive" }, }; } if (portalSetupClaim.state === "provider_accepted") { if (!context?.updatePayload) { throw new Error("Portal setup invitation mail job cannot persist its safe delivery marker."); } await context.updatePayload(portalSetupClaim.payload as QueuePayload); return { handled: true, action: "mail.sent", details: { deduplicated: true }, }; } if (!context?.updatePayload) { throw new Error("Portal setup invitation mail job cannot persist its pre-provider delivery marker."); } await context.updatePayload(portalSetupClaim.payload as QueuePayload); } result = await provider.send(providerInput); if (ownerIntent) { const accept = deps.acceptOwnerInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => acceptWorkspaceOwnerInvitationDelivery(input)); const acceptedPayload = await accept({ tenantId: payload.tenantId ?? "", payload }); ownerClaim = { state: "provider_accepted", payload: acceptedPayload }; if (!context?.updatePayload) { throw new Error("Workspace-owner invitation mail job cannot persist its safe delivery marker."); } await context.updatePayload(acceptedPayload as QueuePayload); } if (portalSetupIntent) { const accept = deps.acceptPortalSetupInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => acceptPortalSetupInvitationDelivery(input)); const acceptedPayload = await accept({ tenantId: payload.tenantId ?? "", payload }); portalSetupClaim = { state: "provider_accepted", payload: acceptedPayload }; if (!context?.updatePayload) { throw new Error("Portal setup invitation mail job cannot persist its safe delivery marker."); } await context.updatePayload(acceptedPayload as QueuePayload); } } catch (error) { if (portalSetupIntent) { const safeError = redactPortalSetupInvitationError(error, payload); if (portalSetupClaim?.state === "provider_accepted") throw safeError; if (portalSetupClaim?.state === "claimed" || portalSetupClaim?.state === "uncertain") { try { await terminalizeFailedPortalSetupMail( payload, deps, context, "Portal setup invitation delivery outcome was ambiguous; capability rotation is required." ); } catch { throw new Error("Portal setup invitation ambiguous delivery cleanup failed; retry is required."); } throw new UnrecoverableError(safeError.message); } if (isTerminalMailAttempt(context)) { await terminalizeFailedPortalSetupMail(payload, deps, context, safeError.message); } throw safeError; } if (ownerIntent) { const safeError = redactWorkspaceOwnerInvitationError(error, payload); if (ownerClaim?.state === "claimed" || ownerClaim?.state === "uncertain") { const abandon = deps.abandonOwnerInvitationDelivery ?? ((input: { tenantId: string; payload: unknown; lastError: string }) => abandonWorkspaceOwnerInvitationDelivery(input)); try { await abandon({ tenantId: payload.tenantId ?? "", payload, lastError: "Workspace-owner invitation delivery outcome was ambiguous; capability rotation is required.", }); if (context?.updatePayload) { await context.updatePayload(redactTerminalWorkspaceOwnerInvitationMailPayload(payload) as QueuePayload); } } catch { if (isTerminalMailAttempt(context)) { await terminalizeFailedOwnerMail(payload, deps, context); } throw new Error("Workspace-owner invitation ambiguous delivery cleanup failed; retry is required."); } throw new UnrecoverableError(safeError.message); } if (isTerminalMailAttempt(context)) { await terminalizeFailedOwnerMail(payload, deps, context); } throw safeError; } const portalStatusMetadata = parsePortalStatusMailDeliveryMetadata(payload.metadata?.portalStatusMail); if (portalStatusMetadata && payload.tenantId && isTerminalMailAttempt(context)) { const notifyFailure = deps.notifyPortalStatusMailFailure ?? notifyPortalStatusMailPermanentFailure; try { await notifyFailure({ tenantId: payload.tenantId, projectId: portalStatusMetadata.projectId, expectedStageId: portalStatusMetadata.expectedStageId, stageChangeId: portalStatusMetadata.stageChangeId, internalRecipientUserId: portalStatusMetadata.internalRecipientUserId, }); } catch { // The provider error remains the retry/DLQ reason. The notifier // is idempotent and can be replayed by an operator if DB was down. } } throw error; } return { handled: true, action: "mail.sent", details: { provider: result.provider, messageId: result.messageId }, }; } // Only an injected (test) fetch is passed through; in production we leave // it undefined so the outbound guard uses its own undici fetch with the // DNS-rebinding IP pin engaged (see fetchWebhookWithRedirects). const fetchImpl = deps.fetch; const updateLegacyEventWebhookAttempt = deps.updateLegacyEventWebhookAttempt ?? updateLegacyEventWebhookAttemptInDb; const legacyAttemptId = typeof payload.metadata?.legacyAttemptId === "string" ? payload.metadata.legacyAttemptId : undefined; const legacyAttemptTenantId = typeof payload.metadata?.legacyAttemptTenantId === "string" ? payload.metadata.legacyAttemptTenantId : null; const attemptNumber = getAttemptNumber(context); const isTerminalFailure = attemptNumber >= getAttemptsLimit(context); const emitWebhookMetric = (status: "success" | "retry" | "dlq" | "rejected") => { if (!deps.webhookDeliveryMetricsHook) return; try { deps.webhookDeliveryMetricsHook({ status }); } catch { // never block delivery } }; try { const response = await fetchWebhookWithRedirects( fetchImpl, payload.url, { method: payload.method ?? "POST", headers: { "content-type": "application/json", ...payload.headers, }, body: payload.body === undefined ? undefined : JSON.stringify(payload.body), }, deps.webhookAddressResolver ); const responseBody = await readLimitedResponseText(response, WEBHOOK_RESPONSE_BODY_LIMIT_BYTES); const responseDiagnostics = serializeWebhookResponseDiagnostics(response, responseBody); if (!response.ok) { const message = `Webhook delivery failed with ${response.status}.`; const retryableFailure = isRetryableWebhookStatus(response.status); const shouldFinishAttempt = !retryableFailure || isTerminalFailure; if (legacyAttemptId) { await updateLegacyEventWebhookAttempt( { id: legacyAttemptId, tenantId: legacyAttemptTenantId }, { ...(shouldFinishAttempt ? { finishedAt: new Date(), success: false } : {}), status: response.status, message, body: responseDiagnostics, } ); } if (!retryableFailure) { emitWebhookMetric("dlq"); return { handled: true, action: "webhook.failed", details: { status: response.status, eventName: payload.eventName, retryable: false }, }; } emitWebhookMetric(isTerminalFailure ? "dlq" : "retry"); throw new WebhookDeliveryError(message); } if (legacyAttemptId) { await updateLegacyEventWebhookAttempt( { id: legacyAttemptId, tenantId: legacyAttemptTenantId }, { finishedAt: new Date(), success: true, status: response.status, message: response.statusText, body: responseDiagnostics, } ); } emitWebhookMetric("success"); return { handled: true, action: "webhook.delivered", details: { status: response.status, eventName: payload.eventName }, }; } catch (error) { if (error instanceof WebhookUrlValidationError) { const message = redactSecretLikeText(error.message); if (legacyAttemptId) { await updateLegacyEventWebhookAttempt( { id: legacyAttemptId, tenantId: legacyAttemptTenantId }, { finishedAt: new Date(), success: false, message, } ); } emitWebhookMetric("rejected"); return { handled: true, action: "webhook.rejected", details: { eventName: payload.eventName, reason: message }, }; } if (error instanceof WebhookDeliveryError) { // Already counted at throw-site (retry / dlq). } else if (legacyAttemptId) { await updateLegacyEventWebhookAttempt( { id: legacyAttemptId, tenantId: legacyAttemptTenantId }, { ...(isTerminalFailure ? { finishedAt: new Date(), success: false } : {}), message: redactSecretLikeText(error instanceof Error ? error.message : String(error)), } ); emitWebhookMetric(isTerminalFailure ? "dlq" : "retry"); } else if (!legacyAttemptId) { emitWebhookMetric(isTerminalFailure ? "dlq" : "retry"); } throw error; } }, [queueNames.notifications]: async (payload, _dependencies, context) => { if (payload.kind === "notification.platform.send") { return runPlatformNotificationHandler(payload, deps, context); } const getUserNotificationPreferences = deps.getUserNotificationPreferences ?? getUserNotificationPreferencesInDb; const createNotificationDelivery = deps.createNotificationDelivery ?? createNotificationDeliveryInDb; const markNotificationDeliveryStatus = deps.markNotificationDeliveryStatus ?? markNotificationDeliveryStatusInDb; const payloadMetadata = { ...(typeof payload.metadata?.requestId === "string" ? { requestId: payload.metadata.requestId } : {}), ...(typeof payload.metadata?.source === "string" ? { source: payload.metadata.source } : {}), ...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}), }; const sourceKey = getNotificationDeliverySourceKey(payload, context); getPreferenceChannel(payload.channel); if (!payload.tenantId) { throw new NotificationHandlerError("Notification jobs require tenantId."); } if (payload.channel !== "email") { const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences); if (isNotificationPreferenceDisabled(preferences)) { const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey); return { handled: true, action: "notification.in-app.skipped", details: { userId: payload.userId, channel: payload.channel, messageKey: payload.messageKey, deliveryId: getNotificationDeliveryId(delivery), reason: "preference_disabled", }, }; } const delivery = await createNotificationDelivery({ tenantId: payload.tenantId ?? null, userId: payload.userId, channel: payload.channel, category: payload.category, messageKey: payload.messageKey, sourceKey, status: "SENT", attemptsMade: 1, payloadMetadata, sentAt: new Date(), }); return { handled: true, action: "notification.accepted", details: { userId: payload.userId, channel: payload.channel, messageKey: payload.messageKey, deliveryId: getNotificationDeliveryId(delivery), }, }; } if (!payload.email) { throw new NotificationHandlerError("Email notifications require email recipient."); } const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences); if (isNotificationPreferenceDisabled(preferences)) { const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey); return { handled: true, action: "notification.email.skipped", details: { userId: payload.userId, channel: payload.channel, messageKey: payload.messageKey, deliveryId: getNotificationDeliveryId(delivery), reason: "preference_disabled", }, }; } let deliveryId = getPayloadNotificationDeliveryId(payload); if (!deliveryId) { const delivery = await createNotificationDelivery({ tenantId: payload.tenantId, userId: payload.userId, channel: "email", category: payload.category, messageKey: payload.messageKey, sourceKey, status: "PENDING", attemptsMade: 0, payloadMetadata, }); deliveryId = getNotificationDeliveryId(delivery); await persistPayloadNotificationDeliveryId(payload, context, deliveryId); } const attemptNumber = getAttemptNumber(context); const isTerminalFailure = attemptNumber >= getAttemptsLimit(context); let result: Awaited>; try { const provider = deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined); result = await provider.send({ to: [payload.email], subject: payload.subject, text: payload.text, html: payload.html, templateId: payload.templateId, idempotencyKey: `notification-delivery:${deliveryId}`, personalization: payload.templateData ? [{ email: payload.email, data: payload.templateData }] : undefined, }); } catch (error) { if (deliveryId) { try { await markNotificationDeliveryStatus({ id: deliveryId, tenantId: payload.tenantId, status: isTerminalFailure ? "FAILED" : "PENDING", attemptsMade: attemptNumber, lastError: getNotificationErrorMessage(error), }); } catch { // Preserve the provider failure as the retry reason. } } throw error; } if (deliveryId) { try { await markNotificationDeliveryStatus({ id: deliveryId, tenantId: payload.tenantId, status: "SENT", attemptsMade: attemptNumber, providerMessageId: result.messageId ?? null, sentAt: new Date(), }); } catch (error) { return { handled: true, action: "notification.email.sent.diagnostic_failed", details: { provider: result.provider, messageId: result.messageId, userId: payload.userId, messageKey: payload.messageKey, deliveryId, diagnosticError: getNotificationErrorMessage(error), }, }; } } return { handled: true, action: "notification.email.sent", details: { provider: result.provider, messageId: result.messageId, userId: payload.userId, messageKey: payload.messageKey }, }; }, [queueNames.maintenance]: async (payload) => { if (payload.task === "privacy-gdpr-export-retention") { return runPrivacyGdprExportRetentionHandler(payload, deps); } if (payload.task === "portal-gdpr-export-retention") { return runPortalGdprExportRetentionHandler(payload, deps); } if (payload.task === "expired-export-artifacts") { return runExportArtifactRetentionHandler(payload, deps); } if (payload.task === "dsgvo-grace-period-scan") { return runDsgvoGracePeriodScanHandler(payload, deps); } if (payload.task === "privacy-tenant-deletion-scan") { return runPrivacyTenantDeletionScanHandler(payload, deps); } if (payload.task === "portal-lifecycle-scan") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runPortalLifecycleScanHandler(payload, deps)); } if (payload.task === "outbox-done-retention") { return runOutboxDoneRetentionHandler(payload, deps); } if (payload.task === "workflow-execution-stalled-reconcile") { return runWorkflowExecutionStalledReconcileHandler(payload, deps); } if (payload.task === "task-archive-retention") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runTaskArchiveRetentionHandler(payload, deps)); } if (payload.task === "project-auto-archive") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runProjectAutoArchiveHandler(payload, deps)); } if (payload.task === "billing-dunning-sweep") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runBillingDunningSweepHandler(payload, deps)); } if (payload.task === "lead-retention-sweep") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runLeadRetentionHandler(payload, deps)); } if (payload.task === "task-reminder-sweep") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runTaskReminderSweepHandler(payload, deps)); } if (payload.task === "automation-scheduler-sweep") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAutomationSchedulerSweepHandler(payload, deps)); } if (payload.task === "admin-sales-task-reminder-sweep") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminSalesTaskReminderSweepHandler(payload, deps) ); } if (payload.task === "admin-booking-reminder-sweep") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminBookingReminderSweepHandler(payload, deps) ); } if (payload.task === "admin-calendar-sync") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminCalendarSyncHandler(payload, deps) ); } if (payload.task === "admin-email-delivery") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminEmailDeliveryHandler(payload, deps) ); } if (payload.task === "admin-email-event-retention") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminEmailEventRetentionHandler(payload, deps) ); } if (payload.task === "admin-email-sync") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminEmailSyncHandler(payload, deps) ); } if (payload.task === "admin-email-domain-check") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminEmailDomainCheckHandler(payload, deps) ); } if (payload.task === "admin-call-recording-process") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminCallRecordingProcessHandler(payload, deps) ); } if (payload.task === "admin-call-recording-retention") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminCallRecordingRetentionHandler(payload, deps) ); } if (payload.task === "admin-outbound-call-process") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runAdminOutboundCallHandler(payload, deps) ); } if (payload.task === "integrations-token-refresh") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runIntegrationsTokenRefreshHandler(payload, deps)); } if (payload.task === "integrations-health-sweep") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runIntegrationsHealthSweepHandler(payload, deps)); } if (payload.task === "kwp-project-maintenance") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runKwpProjectMaintenanceHandler(payload, deps)); } if (payload.task === "calendar-token-refresh") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarTokenRefreshHandler(payload, deps)); } if (payload.task === "calendar-webhook-renewal") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarWebhookRenewalHandler(payload, deps)); } if (payload.task === "calendar-webhook-drain") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarWebhookDrainHandler(payload, deps)); } if (payload.task === "calendar-internal-write-sync") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarInternalWriteSyncHandler(payload, deps)); } if (payload.task === "email-inbox-token-refresh") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxTokenRefreshHandler(payload, deps)); } if (payload.task === "email-inbox-webhook-renewal") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxWebhookRenewalHandler(payload, deps)); } if (payload.task === "email-inbox-webhook-drain") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxWebhookDrainHandler(payload, deps)); } if (payload.task === "email-inbox-initial-sync") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxInitialSyncHandler(payload, deps)); } if (payload.task === "email-inbox-stale-poll") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxStalePollHandler(payload, deps)); } if (payload.task === "email-inbox-event-cleanup") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxEventCleanupHandler(payload, deps)); } if (payload.task === "hr-upload-reminder-dispatch") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runHrUploadReminderDispatchHandler(payload, deps)); } if (payload.task === "email-domain-probe") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailDomainProbeHandler(payload, deps)); } if (payload.task === "telephony-webhook-maintenance") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runTelephonyWebhookMaintenanceHandler(payload, deps)); } if (payload.task === "product-acquisition-reconcile") { return runLeasedTenantMaintenanceHandler(payload, deps, () => runProductAcquisitionReconcileHandler(payload, deps)); } throw new UnsupportedWorkerJobError("Unsupported maintenance task.", { queueName: queueNames.maintenance, task: payload.task, olderThanHours: payload.olderThanHours, olderThanMinutes: payload.olderThanMinutes, }); }, [queueNames.browserAutomation]: async (payload) => { throw new UnsupportedWorkerJobError("Browser automation worker job is not configured.", { queueName: queueNames.browserAutomation, kind: payload.kind, sessionId: payload.sessionId, }); }, [queueNames.fileScan]: async (payload) => { if (payload.kind !== "scan.file") { throw new ScanHandlerError("Unsupported file scan payload kind.", { verdict: "error", code: "invalid_payload", message: "Unsupported file scan payload kind.", }); } if (!deps.scanService) { throw new ScanHandlerError("File scan service is not configured.", { verdict: "error", bucket: payload.bucket, key: payload.key, uploadId: payload.uploadId, code: "scan_service_missing", message: "File scan service is not configured.", }); } let result: StorageScanResult; try { result = await deps.scanService.scanObject({ bucket: payload.bucket, key: payload.key, tenantId: payload.tenantId, uploadId: payload.uploadId, contentType: payload.contentType, metadata: payload.metadata, }); } catch (error) { throw new ScanHandlerError("File scan failed.", buildRejectedScanDetails(payload, error)); } const details = buildScanResultDetails(result, payload.uploadId); if (result.verdict === "clean" || result.verdict === "infected") { // AK-P4a §5 — persist the verdict so the fail-closed media proxy can // serve clean files (and keep refusing infected ones). The write also // binds fileId to the exact scanned storage key. Legacy scans without // a File row carry no fileId and remain scan-only. if (payload.fileId) { if (!payload.tenantId || !deps.fileScanStatusWriter) { throw new ScanHandlerError("File scan verdict persistence is not configured.", { ...details, code: "scan_verdict_persistence_missing", message: "File scan verdict persistence is not configured.", }); } const persisted = await deps.fileScanStatusWriter.recordVerdict({ fileId: payload.fileId, storageKey: payload.key, tenantId: payload.tenantId, verdict: result.verdict, }); if (result.verdict === "clean" && persisted) { if (!deps.filePreviewEnqueuer) { throw new ScanHandlerError("Clean file preview enqueue is not configured.", { ...details, code: "preview_enqueue_missing", message: "Clean file preview enqueue is not configured.", }); } await deps.filePreviewEnqueuer.enqueueAfterCleanScan({ bucket: payload.bucket, contentType: payload.contentType ?? "application/octet-stream", fileId: payload.fileId, storageKey: payload.key, tenantId: payload.tenantId, }); } } return { handled: true, action: `scan.${result.verdict}`, details }; } throw new ScanHandlerError("File scan failed.", details); }, [queueNames.filePreview]: async (payload) => { if (payload.kind !== "preview.file") { throw new PreviewHandlerError("Unsupported file preview payload kind.", { code: "invalid_payload", message: "Unsupported file preview payload kind.", }); } if (!deps.filePreviewService) { throw new PreviewHandlerError("File preview service is not configured.", { fileId: payload.fileId, tenantId: payload.tenantId, code: "preview_service_missing", message: "File preview service is not configured.", }); } let result: FilePreviewServiceResult; try { result = await deps.filePreviewService.generatePreview(payload); } catch (error) { throw new PreviewHandlerError("File preview generation failed.", { fileId: payload.fileId, tenantId: payload.tenantId, code: "preview_failed", message: redactSecretLikeText(error instanceof Error ? error.message : String(error)), }); } return { handled: true, action: `preview.${result.previewStatus}`, details: { fileId: payload.fileId, tenantId: payload.tenantId, previewStatus: result.previewStatus, previewStorageKey: result.previewStorageKey, thumbnailStorageKey: result.thumbnailStorageKey, ...(result.reason ? { reason: redactSecretLikeText(result.reason) } : {}), }, }; }, [queueNames.exports]: async (payload, _dependencies, context) => runExportHandler(payload, deps, context), [queueNames.imports]: async (payload, _dependencies, context) => runImportHandler(payload, deps, context), [queueNames.kwpImports]: async (payload) => runKwpImportHandler(payload, deps), // Phase 06 #21 / F-015 — real plugin-invoke dispatch. The handler is // dependency-injected via `deps.pluginInvokeDeps` (composed in the // `plugin` worker group bootstrap). When the slot is missing the // handler throws so BullMQ dead-letters the job instead of silently // acknowledging it — silent ack was the F-015 Critical finding. [queueNames.pluginInvoke]: async (payload, _dependencies, context) => { void context; assertPluginInvokePayload(payload, deps); if (!deps.pluginInvokeDeps) { const message = "plugin-invoke job received but pluginInvokeDeps is not configured on this worker. " + "Set WORKER_QUEUE_GROUP=plugin and ensure the plugin bootstrap injected pluginInvokeDeps."; deps.logger?.error?.({ queue: queueNames.pluginInvoke, pluginId: payload.pluginId, hookName: payload.hookName }, message); throw new Error(message); } const outcome = await handlePluginInvokeJob( { tenantId: payload.tenantId, pluginId: payload.pluginId, version: payload.version, hookName: payload.hookName, input: payload.input, metadata: payload.metadata, }, deps.pluginInvokeDeps ); if (outcome.ok) { return { handled: true, action: "plugin.invoke.completed", details: { pluginId: payload.pluginId, hookName: payload.hookName, fuelUsed: outcome.fuelUsed, durationMs: outcome.durationMs, }, }; } return { handled: true, action: "plugin.invoke.failed", details: { pluginId: payload.pluginId, hookName: payload.hookName, reason: outcome.reason, message: outcome.message, durationMs: outcome.durationMs, }, }; }, }; } async function terminalizeFailedOwnerMail( payload: QueuePayload, dependencies: HandlerDependencies, context: HandlerExecutionContext | undefined ): Promise { const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(payload); const redactionTask = redacted === payload ? Promise.resolve() : context?.updatePayload ? context.updatePayload(redacted as QueuePayload) : Promise.reject(new Error("Terminal workspace-owner invitation mail job cannot persist its rotation marker.")); let terminalizationTask: Promise = Promise.resolve(); if (hasValidWorkspaceOwnerInvitationMailContract(payload)) { const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : ""; const terminalize = dependencies.terminalizeOwnerInvitation ?? ((input: { tenantId: string; payload: unknown }) => terminalizeWorkspaceOwnerInvitationDelivery(input)); terminalizationTask = terminalize({ tenantId, payload }); } const results = await Promise.allSettled([redactionTask, terminalizationTask]); if (results[1]?.status === "rejected" && isTerminalWorkspaceOwnerInvitationMailPayload(redacted)) { const binding = readWorkspaceOwnerInvitationRecoveryBinding(redacted); const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : ""; if (binding && binding.tenantId === tenantId) { const schedule = dependencies.scheduleOwnerInvitationRecovery ?? ((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => scheduleWorkspaceOwnerInvitationTerminalRecovery(input)); try { await schedule({ tenantId, outboxEventId: binding.outboxEventId, redactedPayload: redacted, lastError: "Workspace-owner invitation database invalidation requires retry.", }); } catch { results.push({ status: "rejected", reason: new Error("Workspace-owner invitation recovery scheduling failed.") }); } } else { results.push({ status: "rejected", reason: new Error("Workspace-owner invitation recovery outbox id is missing.") }); } } const failures = results.filter((result): result is PromiseRejectedResult => result.status === "rejected"); if (failures.length > 0) { throw new Error(`Terminal workspace-owner invitation mail cleanup failed (${failures.length}/${results.length} operations).`); } } async function recoverTerminalOwnerMail(payload: QueuePayload, dependencies: HandlerDependencies): Promise { const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : ""; const terminalize = dependencies.terminalizeOwnerInvitation ?? ((input: { tenantId: string; payload: unknown }) => terminalizeWorkspaceOwnerInvitationDelivery(input)); try { await terminalize({ tenantId, payload }); return; } catch { const binding = readWorkspaceOwnerInvitationRecoveryBinding(payload); if (!binding || binding.tenantId !== tenantId) { throw new Error("Terminal workspace-owner invitation mail recovery binding is missing."); } const schedule = dependencies.scheduleOwnerInvitationRecovery ?? ((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => scheduleWorkspaceOwnerInvitationTerminalRecovery(input)); try { await schedule({ tenantId, outboxEventId: binding.outboxEventId, redactedPayload: payload, lastError: "Workspace-owner invitation database invalidation requires retry.", }); return; } catch { throw new Error("Terminal workspace-owner invitation mail recovery failed."); } } } async function terminalizeFailedPortalSetupMail( payload: QueuePayload, dependencies: HandlerDependencies, context: HandlerExecutionContext | undefined, lastError: string ): Promise { const redacted = redactTerminalPortalSetupInvitationMailPayload(payload); const redactionTask = redacted === payload ? Promise.resolve() : context?.updatePayload ? context.updatePayload(redacted as QueuePayload) : Promise.reject(new Error("Terminal portal setup invitation mail job cannot persist its rotation marker.")); let terminalizationTask: Promise = Promise.resolve(); if (hasValidPortalSetupInvitationMailContract(payload)) { const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : ""; const terminalize = dependencies.terminalizePortalSetupInvitation ?? ((input: { tenantId: string; payload: unknown; lastError?: string }) => terminalizePortalSetupInvitationDelivery(input)); terminalizationTask = terminalize({ tenantId, payload, lastError }); } const results = await Promise.allSettled([redactionTask, terminalizationTask]); if (results[1]?.status === "rejected" && isTerminalPortalSetupInvitationMailPayload(redacted)) { const binding = readPortalSetupInvitationRecoveryBinding(redacted); const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : ""; if (binding && binding.tenantId === tenantId) { const schedule = dependencies.schedulePortalSetupInvitationRecovery ?? ((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => schedulePortalSetupInvitationTerminalRecovery(input)); try { await schedule({ tenantId, outboxEventId: binding.outboxEventId, redactedPayload: redacted, lastError: "Portal setup invitation database invalidation requires retry.", }); } catch { results.push({ status: "rejected", reason: new Error("Portal setup invitation recovery scheduling failed.") }); } } else { results.push({ status: "rejected", reason: new Error("Portal setup invitation recovery outbox id is missing.") }); } } const failures = results.filter((result): result is PromiseRejectedResult => result.status === "rejected"); if (failures.length > 0) { throw new Error(`Terminal portal setup invitation mail cleanup failed (${failures.length}/${results.length} operations).`); } } async function recoverTerminalPortalSetupMail(payload: QueuePayload, dependencies: HandlerDependencies): Promise { const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : ""; const terminalize = dependencies.terminalizePortalSetupInvitation ?? ((input: { tenantId: string; payload: unknown; lastError?: string }) => terminalizePortalSetupInvitationDelivery(input)); try { await terminalize({ tenantId, payload, lastError: "Portal setup invitation recovery requires capability rotation.", }); return; } catch { const binding = readPortalSetupInvitationRecoveryBinding(payload); if (!binding || binding.tenantId !== tenantId) { throw new Error("Terminal portal setup invitation mail recovery binding is missing."); } const schedule = dependencies.schedulePortalSetupInvitationRecovery ?? ((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => schedulePortalSetupInvitationTerminalRecovery(input)); try { await schedule({ tenantId, outboxEventId: binding.outboxEventId, redactedPayload: payload, lastError: "Portal setup invitation database invalidation requires retry.", }); return; } catch { throw new Error("Terminal portal setup invitation mail recovery failed."); } } } function isTerminalMailAttempt(context: HandlerExecutionContext | undefined): boolean { return typeof context?.attemptsLimit === "number" && context.attemptsLimit > 0 && Math.max(0, context.attemptsMade ?? 0) + 1 >= context.attemptsLimit; } function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } function isNonEmptyString(value: unknown): value is string { return typeof value === "string" && value.trim().length > 0; } function assertPluginInvokePayload(payload: unknown, deps: HandlerDependencies): asserts payload is PluginInvokeJob { const record = isRecord(payload) ? payload : {}; const invalidFields: string[] = []; if (record.kind !== "plugin.invoke") { invalidFields.push("kind"); } for (const field of ["tenantId", "pluginId", "version", "hookName"] as const) { if (!isNonEmptyString(record[field])) { invalidFields.push(field); } } if (invalidFields.length === 0) { return; } const details = { queueName: queueNames.pluginInvoke, code: "invalid_payload", invalidFields, }; deps.logger?.error?.( { ...details, payload: summarizePluginInvokePayloadForLog(record), }, "Invalid plugin-invoke queue payload." ); throw new PluginInvokePayloadError("Invalid plugin-invoke queue payload.", details); } function summarizePluginInvokePayloadForLog(record: Record): Record { return { kind: typeof record.kind === "string" ? record.kind : typeof record.kind, tenantId: typeof record.tenantId === "string" ? record.tenantId : typeof record.tenantId, pluginId: typeof record.pluginId === "string" ? record.pluginId : typeof record.pluginId, version: typeof record.version === "string" ? record.version : typeof record.version, hookName: typeof record.hookName === "string" ? record.hookName : typeof record.hookName, }; } function addressToEmail(address: string | { email: string }) { return typeof address === "string" ? address : address.email; } function buildMailPersonalization(to: readonly string[], templateData: Record | undefined) { if (!templateData) { return undefined; } const hasPerRecipientData = to.some((email) => isRecord(templateData[email])); return to.map((email) => ({ email, data: hasPerRecipientData && isRecord(templateData[email]) ? templateData[email] : templateData, })); } type PrivacyGdprExportRetentionState = { processed: number; deleted: number; failed: number; jobs: Record[]; }; async function runPortalGdprExportRetentionHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const now = new Date(); const service = deps.portalGdprExportRetentionService; let result: PortalGdprExportRetentionResult; if (service) { result = await service.runSweep({ now, ...(payload.take !== undefined ? { take: payload.take } : {}) }); } else { const configuredBucket = deps.env?.STORAGE_BUCKET?.trim(); if (!configuredBucket || !deps.storageDeleteService) { throw new Error("Portal GDPR export retention storage deletion is not fully configured."); } const prisma = (await getDefaultDbClient()) as unknown as PortalGdprExportRetentionPrisma; result = await runPortalGdprExportRetentionSweep( { prisma, configuredBucket, deleteObject: (input) => deps.storageDeleteService!.deleteObject(input), }, { now, ...(payload.take !== undefined ? { take: payload.take } : {}) } ); } const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined; return { handled: true, action: "portal.gdpr-export-retention.completed", details: { task: payload.task, requestId, ...result }, }; } async function runPrivacyGdprExportRetentionHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const now = new Date(); const activeTenantReader = deps.activeTenantReader ?? (await getDefaultDbClient()); const sweep = await runActiveTenantSweep(activeTenantReader, { initialState: { processed: 0, deleted: 0, failed: 0, jobs: [] } as PrivacyGdprExportRetentionState, // Legal retention cleanup must continue for disabled/frozen tenants. includeInactive: true, // Each tenant already performs up to five bounded object deletions. concurrency: 1, worker: (tenantId) => runWithRlsContext({ tenantId }, () => runPrivacyGdprExportRetentionForTenant({ tenantId, now, take: payload.take, deps })), merge: (state, result) => ({ processed: state.processed + result.processed, deleted: state.deleted + result.deleted, failed: state.failed + result.failed, jobs: [...state.jobs, ...result.jobs], }), }); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined; return { handled: true, action: "privacy.gdpr-export-retention.completed", details: { task: payload.task, requestId, ...sweep.state }, }; } async function runPrivacyGdprExportRetentionForTenant(input: { tenantId: string; now: Date; take: number | undefined; deps: HandlerDependencies; }): Promise { const listExpired = input.deps.listExpiredGdprExportArtifacts ?? listExpiredGdprExportArtifactsInDb; const markExpired = input.deps.markGdprExportArtifactExpired ?? markGdprExportArtifactExpiredInDb; const markDeleted = input.deps.markGdprExportArtifactDeleted ?? markGdprExportArtifactDeletedInDb; const artifacts = await listExpired({ tenantId: input.tenantId, now: input.now, ...(input.take !== undefined ? { take: input.take } : {}) }); const mismatched = artifacts.find((artifact) => artifact.tenantId !== input.tenantId); if (mismatched) throw new Error(`GDPR export retention tenant scope mismatch for job ${mismatched.id}.`); const configuredBucket = input.deps.env?.STORAGE_BUCKET?.trim(); if (artifacts.length > 0 && (!input.deps.storageDeleteService || !configuredBucket)) { throw new Error("GDPR export artifact storage deletion is not fully configured."); } const jobs: Record[] = new Array(artifacts.length); await mapWithConcurrency(artifacts, EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY, async (artifact, index) => { try { if (!artifact.resultBucket || artifact.resultBucket !== configuredBucket) { throw new Error("GDPR export artifact bucket does not match the configured private bucket."); } if (!artifact.resultKey) throw new Error("GDPR export retention candidate has no persisted attempt key."); const artifactKey = assertGdprExportArtifactKey({ tenantId: input.tenantId, exportJobId: artifact.id, key: artifact.resultKey, scope: "privacy", }); // Claim the exact attempt before deletion. A concurrent retry either // wins first (claim fails, no delete) or observes `expired` and cannot // overwrite this attempt while its idempotent DeleteObject runs. await markExpired({ ...artifact, resultBucket: artifact.resultBucket, resultKey: artifact.resultKey, now: input.now, }); await deleteRetentionObject(input.deps.storageDeleteService!, { bucket: configuredBucket!, key: artifactKey }); await markDeleted({ id: artifact.id, tenantId: input.tenantId, resultBucket: artifact.resultBucket, resultKey: artifact.resultKey }); jobs[index] = { gdprExportJobId: artifact.id, status: "deleted" }; } catch (error) { jobs[index] = { gdprExportJobId: artifact.id, status: "failed", message: redactSecretLikeText(error instanceof Error ? error.message : String(error)), }; } }); const deleted = jobs.filter((job) => job?.status === "deleted").length; const failed = jobs.filter((job) => job?.status === "failed").length; if (failed > 0) { throw new AggregateError( jobs.filter((job) => job?.status === "failed").map((job) => new Error(`${String(job.gdprExportJobId)}: ${String(job.message)}`)), `GDPR export artifact retention failed for ${failed} object(s).` ); } return { processed: artifacts.length, deleted, failed, jobs, }; } async function runExportArtifactRetentionHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const now = new Date(); const retentionCutoff = getRetentionCutoff(now, payload.olderThanHours); const take = payload.take ?? undefined; const activeTenantReader = deps.activeTenantReader ?? (await getDefaultDbClient()); const sweep = await runActiveTenantSweep(activeTenantReader, { initialState: { expired: 0, processed: 0, deleted: 0, failed: 0, jobs: [] } as ExportArtifactRetentionState, // Each tenant already deletes up to five storage objects concurrently. // Serial tenant scopes preserve that process-wide ceiling. concurrency: 1, worker: (tenantId) => runWithRlsContext({ tenantId }, () => runExportArtifactRetentionForTenant({ tenantId, now, retentionCutoff, take, deps })), merge: (state, result) => ({ expired: state.expired + result.expired, processed: state.processed + result.processed, deleted: state.deleted + result.deleted, failed: state.failed + result.failed, jobs: [...state.jobs, ...result.jobs], }), }); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined; return { handled: true, action: "export-artifact-retention.completed", details: { task: payload.task, requestId, ...sweep.state, }, }; } type ExportArtifactRetentionState = { expired: number; processed: number; deleted: number; failed: number; jobs: Record[]; }; async function runExportArtifactRetentionForTenant(input: { tenantId: string; now: Date; retentionCutoff: Date; take: number | undefined; deps: HandlerDependencies; }): Promise { const { tenantId, now, retentionCutoff, take, deps } = input; const listExpiredCompleted = deps.listExpiredCompletedExportJobs ?? listExpiredCompletedExportJobsInDb; const markExpired = deps.markExportJobExpired ?? markExportJobExpiredInDb; const listPending = deps.listSoftDeletedExportJobsForHardDelete ?? listSoftDeletedExportJobsForHardDeleteInDb; const markDeleted = deps.markExportJobHardDeleteSucceeded ?? markExportJobHardDeleteSucceededInDb; const markFailed = deps.markExportJobHardDeleteFailed ?? markExportJobHardDeleteFailedInDb; const expiredCompletedJobs = await listExpiredCompleted({ now, take }); assertExportJobsBelongToTenant(expiredCompletedJobs, tenantId); await Promise.all(expiredCompletedJobs.map((job) => markExpired({ id: job.id, tenantId: job.tenantId, deletedAt: now }))); const jobs = await listPending({ now: retentionCutoff, take }); assertExportJobsBelongToTenant(jobs, tenantId); const details: Record[] = new Array(jobs.length); if (jobs.length > 0 && !deps.storageDeleteService) { throw new Error("Export artifact storage delete service is not configured."); } await mapWithConcurrency(jobs, EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY, async (job, index) => { try { await deleteRetentionObject(deps.storageDeleteService!, { bucket: job.artifactBucket, key: job.artifactKey }); await markDeleted({ id: job.id, tenantId: job.tenantId }); details[index] = { exportJobId: job.id, status: "deleted" }; } catch (error) { const message = redactSecretLikeText(error instanceof Error ? error.message : String(error)); await markFailed({ id: job.id, tenantId: job.tenantId, errorMessage: message }); details[index] = { exportJobId: job.id, status: "failed", message }; } }); const deleted = details.filter((job) => job?.status === "deleted").length; const failed = details.filter((job) => job?.status === "failed").length; return { expired: expiredCompletedJobs.length, processed: jobs.length, deleted, failed, jobs: details }; } function assertExportJobsBelongToTenant(jobs: readonly ExportJobRetentionRecord[], tenantId: string): void { const mismatched = jobs.find((job) => job.tenantId !== tenantId); if (mismatched) { throw new Error(`Export retention tenant scope mismatch for job ${mismatched.id}.`); } } function getRetentionCutoff(now: Date, olderThanHours: number | undefined) { if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) { return now; } return new Date(now.getTime() - olderThanHours * 60 * 60 * 1000); } async function deleteRetentionObject(service: NonNullable, input: { bucket: string; key: string }): Promise { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(new Error("Export artifact deletion timed out.")), EXPORT_ARTIFACT_RETENTION_DELETE_TIMEOUT_MS); try { await service.deleteObject({ ...input, signal: controller.signal }); } finally { clearTimeout(timeout); } } async function runOutboxDoneRetentionHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const now = new Date(); const olderThanHours = resolveOutboxDoneRetentionHours(payload.olderThanHours); const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000); const deleteDoneOutboxEvents = deps.deleteDoneOutboxEvents ?? (async (options) => deleteDoneOutboxEventsInDb(await getDefaultDbClient(), options)); const result = await deleteDoneOutboxEvents({ cutoff, take: payload.take }); const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined; return { handled: true, action: "outbox.done-retention.completed", details: { task: payload.task, requestId, olderThanHours, cutoff: result.cutoff.toISOString(), take: result.take, deleted: result.deleted, }, }; } function resolveOutboxDoneRetentionHours(olderThanHours: number | undefined): number { if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) { return 720; } return Math.floor(olderThanHours); } async function runWorkflowExecutionStalledReconcileHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const runReconcile = deps.reconcileWorkflowExecutionStalls ?? (async (options) => reconcileWorkflowExecutionStalls((await getDefaultDbClient()) as unknown as WorkflowExecutionStalledReconcilerPrisma, options)); const options = { olderThanMinutes: payload.olderThanMinutes, take: payload.take, }; const activeTenantReader = deps.activeTenantReader ?? (await getDefaultDbClient()); const tenantSweep = await runActiveTenantSweep(activeTenantReader, { initialState: emptyWorkflowExecutionStalledTotals(), worker: (tenantId) => runWithRlsContext({ tenantId }, () => runReconcile(options)), merge: mergeWorkflowExecutionStalledTotals, }); // Tenantless platform executions are intentionally visible to the unbound // worker policy and must be reconciled separately from tenant scopes. const platformResult = await runReconcile(options); throwOnActiveTenantSweepFailures(payload.task, tenantSweep.failures); const totals = mergeWorkflowExecutionStalledTotals(tenantSweep.state, platformResult); const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined; return { handled: true, action: "workflow.execution-stalled-reconcile.completed", details: { task: payload.task, requestId, olderThanMinutes: payload.olderThanMinutes, cutoff: platformResult.cutoff.toISOString(), take: platformResult.take, ...totals, }, }; } type WorkflowExecutionStalledTotals = Pick< WorkflowExecutionStalledReconcileResult, "inspected" | "failedOutboxEvents" | "repaired" | "skippedWithoutFailedOutbox" | "updateMisses" >; function emptyWorkflowExecutionStalledTotals(): WorkflowExecutionStalledTotals { return { inspected: 0, failedOutboxEvents: 0, repaired: 0, skippedWithoutFailedOutbox: 0, updateMisses: 0 }; } function mergeWorkflowExecutionStalledTotals( totals: WorkflowExecutionStalledTotals, result: WorkflowExecutionStalledReconcileResult ): WorkflowExecutionStalledTotals { return { inspected: totals.inspected + result.inspected, failedOutboxEvents: totals.failedOutboxEvents + result.failedOutboxEvents, repaired: totals.repaired + result.repaired, skippedWithoutFailedOutbox: totals.skippedWithoutFailedOutbox + result.skippedWithoutFailedOutbox, updateMisses: totals.updateMisses + result.updateMisses, }; } async function runTelephonyWebhookMaintenanceHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { if (!deps.runTelephonyWebhookMaintenance) { throw new UnsupportedWorkerJobError("Telephony webhook maintenance is not configured.", { queueName: queueNames.maintenance, task: payload.task, }); } const result = await deps.runTelephonyWebhookMaintenance(); (deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "telephony.webhook-maintenance.completed"); return { handled: true, action: "telephony.webhook-maintenance.completed", details: { task: payload.task, ...result }, }; } async function runProductAcquisitionReconcileHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { if (!deps.runProductAcquisitionReconcile) { throw new UnsupportedWorkerJobError("Product acquisition reconciliation is not configured.", { queueName: queueNames.maintenance, task: payload.task, }); } const result = await deps.runProductAcquisitionReconcile({ take: payload.take }); (deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "billing.product-acquisition-reconcile.completed"); return { handled: true, action: "billing.product-acquisition-reconcile.completed", details: { task: payload.task, ...result }, }; } async function runPortalLifecycleScanHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { if (!deps.portalLifecycleService) { throw new UnsupportedWorkerJobError("Portal lifecycle service is not configured.", { queueName: queueNames.maintenance, task: payload.task, }); } const result = await deps.portalLifecycleService.runScan(); (deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "portal.lifecycle.scan-completed"); return { handled: true, action: "portal.lifecycle.scanned", details: { task: payload.task, ...result }, }; } async function runKwpProjectMaintenanceHandler( payload: QueuePayload, deps: HandlerDependencies, ): Promise { if (!deps.runKwpProjectMaintenance) { throw new UnsupportedWorkerJobError("KWP project maintenance service is not configured.", { queueName: queueNames.maintenance, task: payload.task, }); } const result = await deps.runKwpProjectMaintenance(); (deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "kwp.project-maintenance.completed"); if (result.failures > 0) { throw new Error(`KWP project maintenance failed for ${result.failures} tenant(s).`); } return { handled: true, action: "kwp.project-maintenance.completed", details: { task: payload.task, ...result }, }; } async function runLeasedTenantMaintenanceHandler( payload: QueuePayload, deps: HandlerDependencies, worker: () => Promise ): Promise { const lease = await withDistributedJobLease(deps.tenantMaintenanceLeaseStore ?? getSharedStore(), payload.task, worker); if (lease.acquired) return lease.value; (deps.logger ?? noopHandlerLogger).info({ task: payload.task, requestId: payload.metadata?.requestId ?? null }, "maintenance.tenant-sweep.overlap-skipped"); return { handled: true, action: "maintenance.tenant-sweep.overlap-skipped", details: { task: payload.task, reason: "active-lease" }, }; } async function getDefaultDbClient() { const { getDbClient } = await import("@hivegrid/db/client.server"); return getDbClient(); } /** * P4 Tasks (AK-P4b) §7 — 30-day auto-archive (DoD 5). Archives completed tasks * whose `completedAt` is older than the retention window by stamping * `archivedAt`. Idempotent: already-archived rows are excluded by the filter. * * RLS-ready (AK-P4b "Teil A"): a cross-tenant sweep cannot run under the * restricted app role once Task RLS is active (an unscoped query returns zero * rows). So we enumerate active tenants from the global Tenant registry (not * RLS-scoped) and run the scoped archive per tenant inside withTenantRLS(), which * sets `app.tenant_id` so the policy binds. Functional no-op until the policy is * activated (prisma/rls/task-isolation.sql, P5 cutover — at which point * hivegrid_app needs SELECT on "Tenant" or a privileged maintenance connection; * see that file's Teil-B notes). */ async function runTaskArchiveRetentionHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const olderThanHours = typeof payload.olderThanHours === "number" && payload.olderThanHours > 0 ? Math.floor(payload.olderThanHours) : 720; const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000); const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { archived: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => tx.task.updateMany({ where: { tenantId, status: "done", completedAt: { lte: cutoff }, archivedAt: null }, data: { archivedAt: now }, }) ), merge: (state, result) => ({ archived: state.archived + result.count }), }); const { archived } = sweep.state; logger.info( { task: payload.task, cutoff: cutoff.toISOString(), tenants: sweep.inspected, tenantFailures: sweep.failures.length, archived }, "tasks.archive-retention.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "tasks.archive-retention.completed", details: { task: payload.task, olderThanHours, cutoff: cutoff.toISOString(), tenants: sweep.inspected, archived }, }; } /** * R-P9a DoD 4 — project auto-archive. Archives projects that sat in a terminal * order pipeline stage (AuftragPipelineStageConfig.autoArchiveAfterDays) past * the configured window, and archives the project chat channel in the same * transaction (AK-P15b). Same per-tenant RLS sweep pattern as the other * maintenance crons: enumerate tenants globally, run the scoped sweep inside * withTenantRLS() so the FORCE-RLS policies bind (an unscoped query would * silently return zero rows). */ async function runProjectAutoArchiveHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 100; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { archived: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => autoArchiveProjectsForTenantWithTx(tx as unknown as Parameters[0], { tenantId, now, limit: take, archiveChannel: (channelInput) => archiveProjectChannelIfEnabledWithTx(tx as unknown as Parameters[0], channelInput), }) ), merge: (state, result) => ({ archived: state.archived + result.archived }), }); const { archived } = sweep.state; logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, archived }, "projects.auto-archive.completed"); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "projects.auto-archive.completed", details: { task: payload.task, tenants: sweep.inspected, archived, take }, }; } async function runLeadRetentionHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { archived: 0 }, worker: async (tenantId) => { const dueIds = await withTenantRLS(db, tenantId, (tx) => tx.inboxLead.findMany({ where: { tenantId, archivedAt: null, autoDeleteAt: { lte: now } }, select: { id: true }, orderBy: { autoDeleteAt: "asc" }, take, }) ); if (dueIds.length === 0) return { count: 0 }; return withTenantRLS(db, tenantId, (tx) => tx.inboxLead.updateMany({ where: { tenantId, id: { in: dueIds.map((row) => row.id) }, archivedAt: null }, data: { archivedAt: now }, }) ); }, merge: (state, result) => ({ archived: state.archived + result.count }), }); const { archived } = sweep.state; logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, archived }, "leads.retention.completed"); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "leads.retention.completed", details: { task: payload.task, tenants: sweep.inspected, archived, take }, }; } /** * R-P5 §P2d — refresh OAuth tokens nearing expiry across all tenants. Enumerates * active tenants from the global registry (not RLS-scoped) and runs the scoped * refresh per tenant inside withTenantRLS() (so the policy binds at cutover). * KWP is deliberately excluded here: its one-time refresh token rotates only * immediately before an actual resource/health window. */ async function runIntegrationsTokenRefreshHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const sweepStartedAt = new Date(); const genericWindowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000; const db = await getDefaultDbClient(); const cleanupSweep = await runActiveTenantSweep(db, { initialState: { restored: 0, purged: 0, failed: 0 }, includeInactive: true, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => cleanupExpiredKwpPendingAttemptsForTenant( tx as unknown as Parameters[0], tenantId, { now: sweepStartedAt }, ) ), merge: (state, result) => ({ restored: state.restored + result.restored, purged: state.purged + result.purged, failed: state.failed + result.failed, }), }); const sweep = await runActiveTenantSweep(db, { initialState: { refreshed: 0, failed: 0, retryScheduled: 0 }, worker: async (tenantId) => { const legacyResult = await withTenantRLS(db, tenantId, (tx) => refreshExpiringTokensForTenant( tx as unknown as Parameters[0], tenantId, { now: sweepStartedAt, windowMs: genericWindowMs }, ) ); return { ...legacyResult, retryScheduled: 0 }; }, merge: (state, result) => ({ refreshed: state.refreshed + result.refreshed, failed: state.failed + result.failed, retryScheduled: state.retryScheduled + result.retryScheduled, }), }); const { refreshed, failed, retryScheduled } = sweep.state; const { restored: pendingRestored, purged: pendingPurged, failed: pendingCleanupFailed } = cleanupSweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length + cleanupSweep.failures.length, refreshed, failed, retryScheduled, pendingRestored, pendingPurged, pendingCleanupFailed, }, "integrations.token-refresh.completed" ); throwOnActiveTenantSweepFailures(payload.task, [...cleanupSweep.failures, ...sweep.failures]); return { handled: true, action: "integrations.token-refresh.completed", details: { task: payload.task, tenants: sweep.inspected, refreshed, failed, retryScheduled, pendingRestored, pendingPurged, pendingCleanupFailed, }, }; } /** * R-P5 §P2d — probe every active connection's health across all tenants, persist * the status, and notify owners on degraded/broken. Same per-tenant RLS sweep * pattern; per-connection logic shared via worker-tasks.server. */ async function runIntegrationsHealthSweepHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const sweepStartedAt = new Date(); const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { checked: 0, unhealthy: 0 }, worker: async (tenantId) => { const legacyResult = await withTenantRLS(db, tenantId, (tx) => sweepConnectionHealthForTenant(tx as unknown as Parameters[0], tenantId, { now: sweepStartedAt }) ); const connectionIds = await withTenantRLS(db, tenantId, (tx) => listKwpHealthConnectionIdsForTenant( tx as unknown as Parameters[0], tenantId, { now: new Date() }, ) ); let result = legacyResult; for (const connectionId of connectionIds) { // Health is a real KWP work window. Refresh on demand once here; the // same durable claim also serializes simultaneous import/web calls. await runWithRlsContext({ tenantId }, () => ensureKwpAccessTokenForConnection({ connectionId, allowPaused: true }), ); const probeNow = new Date(); const prepared = await withTenantRLS(db, tenantId, (tx) => prepareKwpHealthProbeForConnection( tx as unknown as Parameters[0], tenantId, connectionId, { now: probeNow, kwpHiveGridTenantId: deps.env?.KWP_HIVEGRID_TENANT_ID }, ) ); if (prepared.state === "skipped") continue; const providerResult = await executeKwpHealthProbe(prepared.snapshot); const item = await withTenantRLS(db, tenantId, (tx) => persistKwpHealthProbe( tx as unknown as Parameters[0], prepared.snapshot, providerResult, { checkedAt: new Date() }, ) ); result = { checked: result.checked + item.checked, unhealthy: result.unhealthy + item.unhealthy }; } return result; }, merge: (state, result) => ({ checked: state.checked + result.checked, unhealthy: state.unhealthy + result.unhealthy }), }); const { checked, unhealthy } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, checked, unhealthy }, "integrations.health-sweep.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "integrations.health-sweep.completed", details: { task: payload.task, tenants: sweep.inspected, checked, unhealthy } }; } async function runCalendarTokenRefreshHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { refreshed: 0, failed: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => refreshCalendarTokensForTenant(tx as never, tenantId, { now, windowMs })), merge: (state, result) => ({ refreshed: state.refreshed + result.refreshed, failed: state.failed + result.failed }), }); const { refreshed, failed } = sweep.state; logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, refreshed, failed }, "calendar.token-refresh.completed"); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "calendar.token-refresh.completed", details: { task: payload.task, tenants: sweep.inspected, refreshed, failed } }; } async function runCalendarWebhookRenewalHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000; const webhookUrl = process.env.GOOGLE_CALENDAR_WEBHOOK_URL; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { checked: 0, renewed: 0, failed: 0, skipped: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => renewCalendarWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, webhookUrl })), merge: (state, result) => ({ checked: state.checked + result.checked, renewed: state.renewed + result.renewed, failed: state.failed + result.failed, skipped: state.skipped + result.skipped, }), }); const { checked, renewed, failed, skipped } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, checked, renewed, failed, skipped }, "calendar.webhook-renewal.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "calendar.webhook-renewal.completed", details: { task: payload.task, tenants: sweep.inspected, checked, renewed, failed, skipped }, }; } async function runCalendarWebhookDrainHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { claimed: 0, completed: 0, failed: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => drainCalendarWebhookEventsForTenant(tx as never, tenantId, { now, take })), merge: (state, result) => ({ claimed: state.claimed + result.claimed, completed: state.completed + result.completed, failed: state.failed + result.failed, }), }); const { claimed, completed, failed } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, claimed, completed, failed }, "calendar.webhook-drain.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "calendar.webhook-drain.completed", details: { task: payload.task, tenants: sweep.inspected, claimed, completed, failed } }; } async function runCalendarInternalWriteSyncHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { scanned: 0, created: 0, updated: 0, deleted: 0, conflicts: 0, failed: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => syncInternalCalendarEventsForTenant(tx as never, tenantId, { now, take })), merge: (state, result) => ({ scanned: state.scanned + result.scanned, created: state.created + result.created, updated: state.updated + result.updated, deleted: state.deleted + result.deleted, conflicts: state.conflicts + result.conflicts, failed: state.failed + result.failed, }), }); const { scanned, created, updated, deleted, conflicts, failed } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, scanned, created, updated, deleted, conflicts, failed }, "calendar.internal-write-sync.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "calendar.internal-write-sync.completed", details: { task: payload.task, tenants: sweep.inspected, scanned, created, updated, deleted, conflicts, failed }, }; } async function runEmailInboxTokenRefreshHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { refreshed: 0, failed: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => refreshEmailTokensForTenant(tx as never, tenantId, { now, windowMs })), merge: (state, result) => ({ refreshed: state.refreshed + result.refreshed, failed: state.failed + result.failed }), }); const { refreshed, failed } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, refreshed, failed }, "email-inbox.token-refresh.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "email-inbox.token-refresh.completed", details: { task: payload.task, tenants: sweep.inspected, refreshed, failed } }; } async function runEmailInboxWebhookRenewalHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000; const graphWebhookUrl = process.env.MICROSOFT_MAIL_WEBHOOK_URL; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { checked: 0, renewed: 0, failed: 0, skipped: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => renewEmailWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, graphWebhookUrl })), merge: (state, result) => ({ checked: state.checked + result.checked, renewed: state.renewed + result.renewed, failed: state.failed + result.failed, skipped: state.skipped + result.skipped, }), }); const { checked, renewed, failed, skipped } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, checked, renewed, failed, skipped }, "email-inbox.webhook-renewal.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "email-inbox.webhook-renewal.completed", details: { task: payload.task, tenants: sweep.inspected, checked, renewed, failed, skipped }, }; } async function runEmailInboxWebhookDrainHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { claimed: 0, completed: 0, failed: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => drainEmailWebhookEventsForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook })), merge: (state, result) => ({ claimed: state.claimed + result.claimed, completed: state.completed + result.completed, failed: state.failed + result.failed, }), }); const { claimed, completed, failed } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, claimed, completed, failed }, "email-inbox.webhook-drain.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "email-inbox.webhook-drain.completed", details: { task: payload.task, tenants: sweep.inspected, claimed, completed, failed }, }; } async function runEmailInboxInitialSyncHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 10; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { folders: 0, fetched: 0, completed: 0, failed: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => runEmailInitialSyncForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook })), merge: (state, result) => ({ folders: state.folders + result.folders, fetched: state.fetched + result.fetched, completed: state.completed + result.completed, failed: state.failed + result.failed, }), }); const { folders, fetched, completed, failed } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, folders, fetched, completed, failed }, "email-inbox.initial-sync.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "email-inbox.initial-sync.completed", details: { task: payload.task, tenants: sweep.inspected, folders, fetched, completed, failed }, }; } async function runEmailInboxStalePollHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { polled: 0, failed: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => pollStaleEmailMailboxesForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook })), merge: (state, result) => ({ polled: state.polled + result.polled, failed: state.failed + result.failed }), }); const { polled, failed } = sweep.state; logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, polled, failed }, "email-inbox.stale-poll.completed"); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "email-inbox.stale-poll.completed", details: { task: payload.task, tenants: sweep.inspected, polled, failed } }; } async function runEmailInboxEventCleanupHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { deleted: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => cleanupEmailWebhookEventsForTenant(tx as never, tenantId, { now })), merge: (state, result) => ({ deleted: state.deleted + result.deleted }), }); const { deleted } = sweep.state; logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, deleted }, "email-inbox.event-cleanup.completed"); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "email-inbox.event-cleanup.completed", details: { task: payload.task, tenants: sweep.inspected, deleted } }; } // VA-P23 · auto-match + activity logging seam of the sync engine (M09 Block E/H). const emailAutoMatchHook = createEmailAutoMatchHook(); // R-P12c · newly mirrored inbound mail feeds the generic automation trigger // bus. Published on the same tenant transaction as the message upsert so the // event commits atomically with the mirror row; failures must not break sync, // so the publish shares the hook's best-effort contract via its own catch. const emailInboxMessageHook: typeof emailAutoMatchHook = async (input) => { await emailAutoMatchHook(input); if (!input.created || input.message.direction !== "inbound") { return; } try { await publishOutboxEvent( input.client as unknown as OutboxPublisherTransaction, buildWorkflowTriggerOutboxEvent({ tenantId: input.tenantId, event: "email.inbound_received", data: { messageId: input.messageId, mailboxId: input.mailboxId, fromAddress: input.message.from.address, subject: input.message.subject ?? null, }, userId: null, sourceId: `email-inbound:${input.mailboxId}:${input.messageId}`, }) ); } catch { // Best-effort like the auto-match hook itself: the mirror row is already // persisted and mail sync must never break on a missing trigger event. // A lost event only means this one inbound mail fires no automation. } }; /** * AK-P20 DoD 2 — billing dunning sweep. Daily cron that drives the past_due * dunning sequence: one notice at day 1 / 3 / 7 after `pastDueSince`, then a * freeze once every mark is exhausted. Decisions come from the pure * evaluateBillingDunning() (@hivegrid/db/billing-domain.server); the sweep is * driven per tenant from the global Tenant registry, the TenantSubscription * read/writes run inside withTenantRLS() (FORCE RLS), and the notice is sent * through notifyWithTenantRlsContext() — NotificationDelivery is FORCE RLS, so * an unbound write is rejected by the policy (R-P1b precedent). notify()'s * sourceKey upsert dedups per attempt, so re-running a tick never double-sends. * The freeze writes an AuditEvent + `billing_state_changed` activity in the * same shape as the BILL-05 provider sync. */ async function runBillingDunningSweepHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { notified: 0, frozen: 0 }, worker: async (tenantId) => { const subscription = await withTenantRLS(db, tenantId, (tx) => tx.tenantSubscription.findUnique({ where: { tenantId }, select: { status: true, pastDueSince: true, dunningAttemptCount: true }, }) ); if (!subscription) return { notified: 0, frozen: 0 }; const decision = evaluateBillingDunning(subscription, now); if (decision.kind === "send_dunning") { // notify() runs outside the tenant read transaction (slow I/O) but must // bind the tenant context itself: NotificationDelivery is FORCE RLS. await notifyWithTenantRlsContext( { tenantId, channel: "EMAIL", category: "billing", messageKey: `billing.notify.dunning.attempt${decision.attempt}`, sourceKey: `billing-dunning:${tenantId}:attempt:${decision.attempt}`, payloadMetadata: { attempt: decision.attempt, thresholdDays: decision.thresholdDays }, }, db as unknown as NotificationClient ); await withTenantRLS(db, tenantId, (tx) => tx.tenantSubscription.update({ where: { tenantId }, data: { dunningAttemptCount: { increment: 1 }, lastDunningSentAt: now }, }) ); return { notified: 1, frozen: 0 }; } if (decision.kind === "freeze") { await withTenantRLS(db, tenantId, (tx) => tx.tenantSubscription.update({ where: { tenantId }, data: { status: "frozen", frozenAt: now }, }) ); await runWithRlsContext({ tenantId }, async () => { await createAuditEvent({ tenantId, actorType: "system", action: "BILLING_DUNNING_FROZEN", targetType: "TenantSubscription", targetId: tenantId, sourceModule: "billing", outcome: "success", dataSensitivity: "internal", metadata: { thresholdDays: decision.thresholdDays }, }); await db.activity.create({ data: { tenantId, actorUserId: null, actorKind: "system", activityType: "billing_state_changed", subjectType: "tenant", subjectId: tenantId, summaryText: "activity.runtime.billing.statusChanged", metadata: { reason: "dunning_exhausted", thresholdDays: decision.thresholdDays }, }, }); }); return { notified: 0, frozen: 1 }; } return { notified: 0, frozen: 0 }; }, merge: (state, result) => ({ notified: state.notified + result.notified, frozen: state.frozen + result.frozen }), }); const { notified, frozen } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, notified, frozen }, "billing.dunning-sweep.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "billing.dunning-sweep.completed", details: { task: payload.task, tenants: sweep.inspected, notified, frozen } }; } /** * P4 Tasks (AK-P4b) §7 — reminder sweep (DoD 4). For each open task with a * deadline in scan range, sends reminders to the assignee (or assigned group's * members) + watchers at the 24h-before / due / 24h-after windows. The * after-window escalates to EMAIL. notify()'s sourceKey upsert dedups, so the * hourly sweep never double-sends. * * RLS-ready (AK-P4b "Teil A"): like the archive sweep, the cross-tenant scan is * driven per tenant from the global Tenant registry, and the Task/TaskWatcher * reads run inside withTenantRLS() so the policy binds once activated. notify() * runs OUTSIDE the read transaction (slow I/O) but binds its own tenant context * via notifyWithTenantRlsContext() — NotificationDelivery is FORCE RLS, so an * unbound write is rejected by the policy. `take` now bounds tasks per tenant * rather than globally. */ async function runTaskReminderSweepHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const bounds = reminderScanBounds(now); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000; const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { sent: 0, scanned: 0 }, worker: async (tenantId) => { let sent = 0; const tasks = await withTenantRLS(db, tenantId, (tx) => tx.task.findMany({ where: { tenantId, deadline: { not: null, gte: bounds.deadlineGte, lte: bounds.deadlineLte }, status: { in: ["open", "in_progress", "blocked"] }, archivedAt: null, }, select: { id: true, tenantId: true, deadline: true, assignedToUserId: true, assignedToGroupId: true }, take, }) ); for (const task of tasks) { if (!task.deadline) continue; const windows = selectReminderWindows(task.deadline, now); if (windows.length === 0) continue; // Recipients: the assigned user (or every member of the assigned group) // plus all watchers, de-duplicated. TaskWatcher is RLS-scoped, so the reads // run inside the tenant transaction; group membership is read alongside. const recipientIds = new Set(); if (task.assignedToUserId) recipientIds.add(task.assignedToUserId); const { groupMembers, watchers } = await withTenantRLS(db, tenantId, async (tx) => { const groupMembers = task.assignedToGroupId ? await tx.groupUser.findMany({ where: { groupId: task.assignedToGroupId }, select: { userId: true } }) : []; const watcherRows = await tx.taskWatcher.findMany({ where: { tenantId, taskId: task.id }, select: { userId: true } }); return { groupMembers, watchers: watcherRows }; }); for (const member of groupMembers) recipientIds.add(member.userId); for (const watcher of watchers) recipientIds.add(watcher.userId); for (const window of windows) { for (const userId of recipientIds) { try { // notify() runs outside the tenant read transaction (slow I/O) but must // bind the tenant context itself: NotificationDelivery is FORCE RLS. await notifyWithTenantRlsContext( { tenantId: task.tenantId, userId, channel: reminderChannel(window), category: "tasks.reminder", messageKey: reminderMessageKey(window), sourceKey: reminderSourceKey(task.id, window), payloadMetadata: { taskId: task.id, deadline: task.deadline.toISOString() }, }, db as unknown as NotificationClient ); sent += 1; } catch (err) { logger.error( { task: payload.task, taskId: task.id, userId, window, err: redactSecretLikeText(err instanceof Error ? err.message : String(err)) }, "tasks.reminder.notify-failed" ); } } } } return { scanned: tasks.length, sent }; }, merge: (state, result) => ({ scanned: state.scanned + result.scanned, sent: state.sent + result.sent }), }); const { scanned, sent } = sweep.state; logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, scanned, sent }, "tasks.reminder-sweep.completed"); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "tasks.reminder-sweep.completed", details: { task: payload.task, tenants: sweep.inspected, scanned, sent } }; } async function runAutomationSchedulerSweepHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { inspectedRules: 0, scheduledEvents: 0, skippedInvalidRules: 0, truncatedRules: 0 }, worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => runAutomationSchedulerForTenantWithTx(tx as never, { tenantId, now })), merge: (state, result) => ({ inspectedRules: state.inspectedRules + result.inspectedRules, scheduledEvents: state.scheduledEvents + result.scheduledEvents, skippedInvalidRules: state.skippedInvalidRules + result.skippedInvalidRules, truncatedRules: state.truncatedRules + result.truncatedRules, }), }); logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, ...sweep.state }, "automation.scheduler-sweep.completed"); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "automation.scheduler-sweep.completed", details: { task: payload.task, tenants: sweep.inspected, ...sweep.state }, }; } async function runAdminSalesTaskReminderSweepHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const runSweep = deps.runAdminSalesTaskReminderSweep ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); return runAdminSalesTaskReminderSweepInDb(db, input); }); const result = await runSweep({ now, ...(take ? { take } : {}) }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, scanned: result.scanned, dispatched: result.dispatched, }, "admin-sales.task-reminder-sweep.completed" ); return { handled: true, action: "admin-sales.task-reminder-sweep.completed", details: result, }; } async function runAdminBookingReminderSweepHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const runSweep = deps.runAdminBookingReminderSweep ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); return runAdminBookingReminderSweepInDb(db, input); }); const result = await runSweep({ now, ...(take ? { take } : {}) }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, scanned: result.scanned, dispatched: result.dispatched, failed: result.failed, }, "admin-booking.reminder-sweep.completed" ); return { handled: true, action: "admin-booking.reminder-sweep.completed", details: result, }; } async function runAdminEmailDeliveryHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const runDelivery = deps.runAdminEmailDelivery ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); const env = deps.env ?? process.env; return deliverDueAdminEmails(db as never, { ...input, tracking: env.ADMIN_APP_HOST && env.MARKETING_TOKEN_SECRET ? { host: env.ADMIN_APP_HOST, secret: env.MARKETING_TOKEN_SECRET, } : null, }); }); const result = await runDelivery({ now, ...(take ? { take } : {}) }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, inspected: result.inspected, claimed: result.claimed, sent: result.sent, failed: result.failed.length, blocked: result.blocked, deferred: result.deferred, recoveredUnknown: result.recoveredUnknown, noReplyReminderScanned: result.noReplyReminders.scanned, noReplyReminderCreated: result.noReplyReminders.created, }, "admin-sales.email-delivery.completed" ); return { handled: true, action: "admin-sales.email-delivery.completed", details: { inspected: result.inspected, claimed: result.claimed, sent: result.sent, failed: result.failed.length, blocked: result.blocked, deferred: result.deferred, recoveredUnknown: result.recoveredUnknown, noReplyReminderScanned: result.noReplyReminders.scanned, noReplyReminderCreated: result.noReplyReminders.created, }, }; } async function runAdminEmailEventRetentionHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const redact = deps.redactExpiredAdminEmailEventRawData ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); return redactExpiredAdminEmailEventRawDataInDb(db, input); }); const result = await redact({ now: new Date(), ...(take ? { take } : {}), }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, ...result }, "admin-sales.email-event-retention.completed" ); return { handled: true, action: "admin-sales.email-event-retention.completed", details: result, }; } async function runAdminCalendarSyncHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const run = deps.runAdminCalendarSync ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); return syncAdminCalendarBusySnapshots(db as never, { ...input, env: deps.env ?? process.env, }); }); const result = await run({ now: new Date(), ...(take ? { take } : {}), }); const details = { inspected: result.inspected, synced: result.synced, failed: result.failed.length, }; (deps.logger ?? noopHandlerLogger).info( { task: payload.task, ...details }, "admin-sales.calendar-sync.completed" ); return { handled: true, action: "admin-sales.calendar-sync.completed", details, }; } async function runAdminEmailSyncHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const now = new Date(); const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const runSync = deps.runAdminEmailSync ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); return syncClaimedAdminMailboxes(db as never, { ...input, attachmentProcessor: createWorkerAdminEmailAttachmentProcessor( (deps.env ?? process.env) as never ), }); }); const result = await runSync({ now, ...(take ? { take } : {}) }); const details = { inspected: result.inspected, synced: result.synced, upserted: result.upserted, replayed: result.replayed, failed: result.failed.length, }; (deps.logger ?? noopHandlerLogger).info( { task: payload.task, ...details }, "admin-sales.email-sync.completed" ); return { handled: true, action: "admin-sales.email-sync.completed", details, }; } async function runAdminEmailDomainCheckHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const run = deps.runAdminEmailDomainCheck ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); return checkAdminSendingDomains(db as never, input); }); const result = await run({ now: new Date(), ...(take ? { take } : {}), }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, inspected: result.inspected, verified: result.verified, degraded: result.degraded, failed: result.failed.length, }, "admin-sales.email-domain-check.completed" ); return { handled: true, action: "admin-sales.email-domain-check.completed", details: { inspected: result.inspected, verified: result.verified, degraded: result.degraded, failed: result.failed.length, }, }; } async function runAdminCallRecordingProcessHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const run = deps.runAdminCallRecordingProcessing ?? (async (input: { readonly take?: number }) => { const db = await getDefaultDbClient(); return runAdminCallRecordingProcessing({ db, env: deps.env ?? process.env, ...input, }); }); const result = await run({ ...(take ? { take } : {}) }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, ...result }, "admin-sales.call-recording-process.completed" ); return { handled: true, action: "admin-sales.call-recording-process.completed", details: result, }; } async function runAdminCallRecordingRetentionHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const run = deps.runAdminCallRecordingRetention ?? (async (input: { readonly now: Date; readonly take?: number }) => { const db = await getDefaultDbClient(); return runAdminCallRecordingRetention({ db, env: deps.env ?? process.env, ...input, }); }); const result = await run({ now: new Date(), ...(take ? { take } : {}), }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, ...result }, "admin-sales.call-recording-retention.completed" ); return { handled: true, action: "admin-sales.call-recording-retention.completed", details: result, }; } async function runAdminOutboundCallHandler( payload: QueuePayload, deps: HandlerDependencies ): Promise { const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : undefined; const run = deps.runAdminOutboundCalls ?? (async (input: { readonly take?: number }) => { const db = await getDefaultDbClient(); return runAdminOutboundCalls({ db, ...input }); }); const result = await run({ ...(take ? { take } : {}) }); (deps.logger ?? noopHandlerLogger).info( { task: payload.task, ...result }, "admin-sales.outbound-call-process.completed" ); return { handled: true, action: "admin-sales.outbound-call-process.completed", details: result, }; } /** * P17 HR-Hub (AK-P17b) — upload-reminder dispatch. For every active reminder * past its `dueAt`, fan out per-employee dispatch rows (idempotent via the unique * reminder+employee+dueAt), notify each employee, advance the reminder by its * cadence (or complete a one-off), then escalate overdue mandatory dispatches to * the employee's manager. Per-tenant from the global Tenant registry; all HR * reads/writes run inside withTenantRLS() so the policy binds once activated. * notify() runs outside the tenant transaction (slow I/O) with its own bound * tenant context (notifyWithTenantRlsContext — FORCE RLS), deduped by sourceKey. */ async function runHrUploadReminderDispatchHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const db = await getDefaultDbClient(); const sweep = await runActiveTenantSweep(db, { initialState: { dispatched: 0, escalated: 0 }, worker: async (tenantId) => { let dispatched = 0; let escalated = 0; const due = await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminder.findMany({ where: { tenantId, status: "active", archivedAt: null, dueAt: { not: null, lte: now } } }) ); const profiles = due.length > 0 ? await withTenantRLS(db, tenantId, (tx) => tx.hrEmployeeProfile.findMany({ where: { tenantId }, select: { userId: true, team: true, locationId: true } }) ) : []; for (const reminder of due) { const dueAt = reminder.dueAt ?? now; const members = profiles.filter((profile) => audienceMatches(reminder.audienceKind, reminder.audienceRef, profile)); for (const member of members) { const created = await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminderDispatch.createMany({ data: [{ tenantId, reminderId: reminder.id, employeeUserId: member.userId, dueAt, status: "sent", sentAt: now }], skipDuplicates: true, }) ); if (created.count > 0) { dispatched += created.count; await notifyWithTenantRlsContext( { tenantId, userId: member.userId, channel: "IN_APP", category: "hr", messageKey: "hr.notify.reminder.due", sourceKey: `hr-reminder:${reminder.id}:${dueAt.getTime()}:${member.userId}`, payloadMetadata: { reminderId: reminder.id }, }, db as unknown as NotificationClient ).catch((err) => logger.error( { task: payload.task, reminderId: reminder.id, userId: member.userId, err: redactSecretLikeText(err instanceof Error ? err.message : String(err)), }, "hr.upload-reminder.notify-failed" ) ); } } const next = nextReminderDue({ cadenceKind: reminder.cadenceKind, intervalMonths: reminder.intervalMonths, from: dueAt }); await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminder.update({ where: { id: reminder.id }, data: next ? { dueAt: next } : { dueAt: null, status: "completed" } }) ); } // Escalate still-unacknowledged mandatory dispatches past the grace window. const cutoff = new Date(now.getTime() - HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS); const stale = await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminderDispatch.findMany({ where: { tenantId, status: "sent", dueAt: { lt: cutoff }, reminder: { escalateToManager: true, mandatory: true } }, }) ); const staleEmployeeUserIds = [...new Set(stale.map((dispatch) => dispatch.employeeUserId))]; const escalationProfiles = staleEmployeeUserIds.length > 0 ? await withTenantRLS(db, tenantId, (tx) => tx.hrEmployeeProfile.findMany({ where: { tenantId, userId: { in: staleEmployeeUserIds } }, select: { userId: true, managerUserId: true }, }) ) : []; const managerUserIdsByEmployee = new Map(escalationProfiles.map((profile) => [profile.userId, profile.managerUserId])); for (const dispatch of stale) { await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminderDispatch.update({ where: { id: dispatch.id }, data: { status: "escalated", escalatedAt: now } }) ); escalated += 1; const managerUserId = managerUserIdsByEmployee.get(dispatch.employeeUserId); if (managerUserId) { await notifyWithTenantRlsContext( { tenantId, userId: managerUserId, channel: "IN_APP", category: "hr", messageKey: "hr.notify.reminder.escalated", sourceKey: `hr-reminder-escalate:${dispatch.id}`, payloadMetadata: { dispatchId: dispatch.id }, }, db as unknown as NotificationClient ).catch((err) => logger.error( { task: payload.task, dispatchId: dispatch.id, err: redactSecretLikeText(err instanceof Error ? err.message : String(err)) }, "hr.upload-reminder.escalate-notify-failed" ) ); } } return { dispatched, escalated }; }, merge: (state, result) => ({ dispatched: state.dispatched + result.dispatched, escalated: state.escalated + result.escalated }), }); const { dispatched, escalated } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, dispatched, escalated }, "hr.upload-reminder-dispatch.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "hr.upload-reminder-dispatch.completed", details: { task: payload.task, tenants: sweep.inspected, dispatched, escalated } }; } /** * AK-P14a · P14 Email — sending-domain DNS auto-probe (5-minute cadence). For * each active tenant, resolves DKIM/SPF/DMARC for every not-yet-settled sending * domain and advances `verificationStatus` via the pure decideDomainProbeStatus() * helper. Reads/writes run inside withTenantRLS() so the policy binds once the * P5 cutover activates RLS; the domain UPDATE is auto-audited by the M001 Prisma * middleware. A tenant-level notification fires only on the verified/failed * transition (not on every probe). The DNS resolver is the node:dns binding; the * decision logic is unit-tested without DNS/DB. */ async function runEmailDomainProbeHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const now = new Date(); const db = await getDefaultDbClient(); const resolver: DomainDnsResolver = createDnsDomainResolver(); const sweep = await runActiveTenantSweep(db, { initialState: { probed: 0, verified: 0, failed: 0 }, worker: async (tenantId) => { let verified = 0; let failed = 0; const domains = await withTenantRLS(db, tenantId, (tx) => tx.tenantSendingDomain.findMany({ where: { tenantId, archivedAt: null, verificationStatus: { in: ["pending", "verifying"] } } }) ); for (const domain of domains) { const dns = await resolver({ domain: domain.domain, dkimSelector: domain.dkimSelector }); const firstProbeAt = domain.firstProbeAt ?? now; const decision = decideDomainProbeStatus({ dns, firstProbeAt, now }); const changed = decision.status !== domain.verificationStatus; await withTenantRLS(db, tenantId, (tx) => tx.tenantSendingDomain.update({ where: { id: domain.id }, data: { verificationStatus: decision.status, lastError: decision.lastError, lastProbedAt: now, firstProbeAt }, }) ); if (changed && (decision.status === "verified" || decision.status === "failed")) { if (decision.status === "verified") verified += 1; else failed += 1; await notifyWithTenantRlsContext( { tenantId, channel: "IN_APP", category: "email", messageKey: decision.status === "verified" ? "email.domain.notify.verified" : "email.domain.notify.failed", sourceKey: `email-domain-${decision.status}:${domain.id}`, payloadMetadata: { domainId: domain.id, domain: domain.domain }, }, db as unknown as NotificationClient ).catch((err) => logger.error( { task: payload.task, domainId: domain.id, err: redactSecretLikeText(err instanceof Error ? err.message : String(err)) }, "email.domain-probe.notify-failed", ) ); } } return { probed: domains.length, verified, failed }; }, merge: (state, result) => ({ probed: state.probed + result.probed, verified: state.verified + result.verified, failed: state.failed + result.failed, }), }); const { probed, verified, failed } = sweep.state; logger.info( { task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, probed, verified, failed }, "email.domain-probe.completed" ); throwOnActiveTenantSweepFailures(payload.task, sweep.failures); return { handled: true, action: "email.domain-probe.completed", details: { task: payload.task, tenants: sweep.inspected, probed, verified, failed } }; } /** * Phase 10 #68 — maintenance-handler adapter for the DSGVO grace-period * scan. Wraps `runDsgvoGraceCron` (the pure unit-tested core) with the * production HandlerDependencies surface. * * Why a separate adapter: `runDsgvoGraceCron` is intentionally * host-agnostic (no BullMQ, no Prisma). The maintenance queue receives * a CleanupJob discriminator and this adapter is what binds the cron * to the worker's logger + Prisma-backed candidate-reader + cascade * dispatcher (via `deps.dsgvoGraceCronService`). * * When `deps.dsgvoGraceCronService` is not configured we still need to * acknowledge the job (otherwise BullMQ retries forever and we are * back to the UnsupportedWorkerJobError loop that motivated this fix). * In that case we log a single warning and return `scanned=0`. The * follow-up phase wires the real Prisma reader + cascade dispatcher. */ async function runDsgvoGracePeriodScanHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const service = deps.dsgvoGraceCronService; if (!service) { // Deliberately not throwing: we want the cron to ack so BullMQ does // not retry; the missing service is a deployment-config gap, not a // payload-shape bug. Emit an explicit log for operators. logger.info({ task: payload.task, requestId: payload.metadata?.requestId ?? null }, "dsgvo-grace-cron.skipped: dsgvoGraceCronService not configured"); return { handled: true, action: "dsgvo.grace-period.skipped", details: { task: payload.task, reason: "dsgvoGraceCronService not configured", scanned: 0, dispatched: 0, failed: 0 }, }; } const result = await runDsgvoGraceCron({ listOverdueUsers: (now: Date): Promise => service.listOverdueUsers(now), enqueueDeletion: (input) => service.enqueueDeletion(input), logger, }); return { handled: true, action: "dsgvo.grace-period.scanned", details: { task: payload.task, scanned: result.scanned, dispatched: result.dispatched, failed: result.failed, // `failures` contains userIds — keep them so audit can correlate // but redact in case any future error message embeds secrets. failures: result.failures.map((f) => ({ userId: f.userId, error: redactSecretLikeText(f.error) })), }, }; } /** * AK-P21 — maintenance-handler adapter for the tenant hard-delete scan. * Mirrors the dsgvo-grace-cron adapter: when the service is not wired the * cron tick is acked with an explicit skip (a config gap must not turn into * an endless BullMQ retry loop). */ async function runPrivacyTenantDeletionScanHandler(payload: QueuePayload, deps: HandlerDependencies): Promise { const logger = deps.logger ?? noopHandlerLogger; const service = deps.privacyTenantDeletionService; if (!service) { logger.info( { task: payload.task, requestId: payload.metadata?.requestId ?? null }, "privacy-tenant-deletion.skipped: privacyTenantDeletionService not configured" ); return { handled: true, action: "privacy.tenant-deletion.skipped", details: { task: payload.task, reason: "privacyTenantDeletionService not configured", scanned: 0, completed: 0, failed: 0 }, }; } const result = await service.runScan(); return { handled: true, action: "privacy.tenant-deletion.scanned", details: { task: payload.task, scanned: result.scanned, completed: result.completed, failed: result.failed, failures: result.failures.map((failure) => ({ requestId: failure.requestId, tenantId: failure.tenantId, error: redactSecretLikeText(failure.error), })), }, }; } const noopHandlerLogger = { info: () => undefined, error: () => undefined, } satisfies NonNullable; async function runExportHandler( payload: QueuePayload, deps: HandlerDependencies, context?: HandlerExecutionContext ): Promise { payload = normalizePortalGdprExportQueuePayload(payload); if (payload.kind !== "export.run") { throw new ExportHandlerError("Unsupported export payload kind.", { code: "invalid_payload", message: "Unsupported export payload kind.", }); } if (isPortalGdprExportType(payload.exportType)) { return runPortalGdprExportHandler(payload, deps, context); } if (isPrivacyGdprExportType(payload.exportType)) { return runPrivacyGdprExportHandler(payload, deps, context); } const markPaused = deps.markExportJobPaused ?? markExportJobPausedInDb; const markResumed = deps.markExportJobResumed ?? markExportJobResumedInDb; const markRunning = deps.markExportJobRunning ?? markExportJobRunningInDb; const markSucceeded = deps.markExportJobSucceeded ?? markExportJobSucceededInDb; const markFailed = deps.markExportJobFailed ?? markExportJobFailedInDb; const attemptsMade = readAttemptsMade(payload.metadata, context); const complianceGate = payload.complianceGate; if (complianceGate?.state === "paused") { const pauseReason = redactSecretLikeText(complianceGate.reason ?? "Export paused by compliance gate."); await markPaused({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade, reason: pauseReason }); return { handled: true, action: "export.paused", details: buildExportDetails(payload, { attemptsMade, pauseReason, pauseRequestId: complianceGate.requestId, status: "paused", }), }; } if (complianceGate?.state === "active" || complianceGate?.resumedAt) { await markResumed({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade }); } await markRunning({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade }); if (!deps.exportService) { const message = "Export service is not configured."; await markFailed({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade, errorName: "ExportHandlerError", errorMessage: message, }); throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "export_service_missing" })); } try { const result = await deps.exportService.runExport(payload); await markSucceeded({ id: payload.exportJobId, tenantId: payload.tenantId, artifactBucket: result.artifactBucket, artifactKey: result.artifactKey, rowCount: result.rowCount, attemptsMade, expiresAt: result.expiresAt, }); return { handled: true, action: "export.completed", details: buildExportDetails(payload, { artifactBucket: result.artifactBucket, artifactKey: result.artifactKey, attemptsMade, expiresAt: result.expiresAt.toISOString(), rowCount: result.rowCount, status: "completed", }), }; } catch (error) { const errorName = error instanceof Error ? error.name : "Error"; const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error)); if (isTerminalJobAttempt(attemptsMade, context)) { await markFailed({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade, errorName, errorMessage, }); } throw new ExportHandlerError("Export job failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage })); } } /** * Portal export requests originate in a SECURITY DEFINER transaction and are * dispatched as an OutboxEvent envelope. The exports queue otherwise accepts * direct ExportJob payloads, so normalize only this exact event type and * reject tenant/actor mismatches before any storage or Prisma access. */ export function normalizePortalGdprExportQueuePayload(payload: unknown): ExportJob { const record = isRecord(payload) ? payload : {}; if (record.kind === "export.run") return record as ExportJob; if (record.eventType !== "portal.gdpr_export.requested") { throw new ExportHandlerError("Unsupported export payload kind.", { code: "invalid_payload", eventType: typeof record.eventType === "string" ? record.eventType : null, }); } const outboxPayload = isRecord(record.payload) ? record.payload : {}; const tenantId = isNonEmptyString(record.tenantId) ? record.tenantId.trim() : ""; const payloadTenantId = isNonEmptyString(outboxPayload.tenantId) ? outboxPayload.tenantId.trim() : ""; const portalUserId = isNonEmptyString(outboxPayload.portalUserId) ? outboxPayload.portalUserId.trim() : ""; const exportJobId = isNonEmptyString(outboxPayload.jobId) ? outboxPayload.jobId.trim() : ""; if (!tenantId || tenantId !== payloadTenantId || !portalUserId || !exportJobId) { throw new ExportHandlerError("Portal GDPR export outbox envelope is invalid.", { code: "portal_gdpr_export_invalid_envelope", tenantMatch: Boolean(tenantId && tenantId === payloadTenantId), hasPortalUserId: Boolean(portalUserId), hasExportJobId: Boolean(exportJobId), }); } const outboxEventId = isNonEmptyString(record.id) ? record.id.trim() : isNonEmptyString(record.outboxEventId) ? record.outboxEventId.trim() : undefined; return { kind: "export.run", exportJobId, tenantId, requestedByUserId: portalUserId, exportType: "gdpr_portal_user_self", format: "json", ...(outboxEventId ? { requestId: outboxEventId } : {}), metadata: { actorKind: "portal_user", sourceEventType: "portal.gdpr_export.requested", ...(outboxEventId ? { outboxEventId } : {}), }, }; } async function runPortalGdprExportHandler( payload: ExportJob, deps: HandlerDependencies, context?: HandlerExecutionContext ): Promise { const attemptsMade = readAttemptsMade(payload.metadata, context); if (!deps.portalGdprExportService) { throw new ExportHandlerError( "Portal GDPR export service is not configured.", buildExportDetails(payload, { status: "failed", attemptsMade, code: "portal_gdpr_export_service_missing" }) ); } try { const result = await deps.portalGdprExportService.runExport(payload); return { handled: true, action: "portal-gdpr-export.completed", details: buildExportDetails(payload, { artifactBucket: result.artifactBucket, artifactKey: result.artifactKey, attemptsMade, expiresAt: result.expiresAt.toISOString(), rowCount: result.rowCount, sizeBytes: result.sizeBytes, status: "ready", }), }; } catch (error) { const errorName = error instanceof Error ? error.name : "Error"; const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error)); const details = buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }); if (error instanceof PortalGdprExportSizeLimitExceededError) { throw new UnrecoverableExportHandlerError("Portal GDPR export job failed.", details); } throw new ExportHandlerError("Portal GDPR export job failed.", details); } } async function runPrivacyGdprExportHandler( payload: ExportJob, deps: HandlerDependencies, context?: HandlerExecutionContext ): Promise { const attemptsMade = readAttemptsMade(payload.metadata, context); const subject = resolvePrivacyGdprExportSubject(payload); const markProcessing = deps.markGdprExportProcessing ?? markGdprExportProcessingInDb; const markReady = deps.markGdprExportReady ?? markGdprExportReadyInDb; const markFailed = deps.markGdprExportFailed ?? markGdprExportFailedInDb; const service = deps.privacyGdprExportService; if (!service) { await markProcessing({ id: payload.exportJobId, ...subject }); const message = "Privacy GDPR export service is not configured."; await markFailed({ id: payload.exportJobId, tenantId: payload.tenantId, errorMessage: message, }); throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "privacy_gdpr_export_service_missing" })); } return service.withTenantBarrier(payload, async (checkpoint) => { let existingReady: Awaited["resolveReady"]>>; let reclaimableProcessing: Awaited["resolveReclaimableProcessing"]>> = null; try { existingReady = await service.resolveReady(payload); if (!existingReady && (context?.attemptsMade ?? 0) > 0) { reclaimableProcessing = await service.resolveReclaimableProcessing(payload); } } catch (error) { const errorName = error instanceof Error ? error.name : "Error"; const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error)); throw new ExportHandlerError( "Privacy GDPR export ready-state reconciliation failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }) ); } if (existingReady) { await checkpoint(); return completePrivacyGdprExport(payload, existingReady, attemptsMade, deps); } const processingStartedAt = new Date(Math.max(Date.now(), (reclaimableProcessing?.processingStartedAt.getTime() ?? Number.NEGATIVE_INFINITY) + 1)); const attempt = reclaimableProcessing?.attempt ?? service.createAttempt(payload); await markProcessing({ id: payload.exportJobId, ...subject, resultBucket: attempt.artifactBucket, resultKey: attempt.artifactKey, now: processingStartedAt, ...(reclaimableProcessing ? { previousProcessingStartedAt: reclaimableProcessing.processingStartedAt } : {}), }); let artifactCleanupAllowed = true; let failureTransitionAllowed = true; try { await checkpoint(); const result = await service.runExport(payload, attempt); await checkpoint(); let readyTransition: MarkGdprExportReadyResult; try { await checkpoint(); readyTransition = await markReady({ id: payload.exportJobId, tenantId: payload.tenantId, resultBucket: result.artifactBucket, resultKey: result.artifactKey, resultSizeBytes: result.sizeBytes, resultExpiresAt: result.expiresAt, processingStartedAt, }); } catch (error) { // The UPDATE may have committed even when its response was lost. Never // delete or downgrade this attempt until a retry reconciles the ready row. artifactCleanupAllowed = false; failureTransitionAllowed = false; throw error; } if (readyTransition === "lost_owner") { // A retry may have adopted this exact attempt after an ambiguous PUT. // Once ownership is lost, deleting the shared attempt key is unsafe. artifactCleanupAllowed = false; failureTransitionAllowed = false; throw new Error("Privacy GDPR export completion lost its processing claim."); } // Once Ready committed, the artifact is no longer an unowned attempt and // must never pass through the generic cleanup path. A deletion that began // in the narrow post-CAS race terminalizes this exact Ready row while // retaining its bucket/key for the hard-delete final sweep. artifactCleanupAllowed = false; failureTransitionAllowed = false; try { await checkpoint(); } catch (error) { if (error instanceof TenantExportQuiescedError) { await service.terminalizeReadyAfterQuiescence({ payload, result }); } throw error; } return completePrivacyGdprExport(payload, result, attemptsMade, deps); } catch (error) { const errorName = error instanceof Error ? error.name : "Error"; const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error)); const isSizeLimitExceeded = error instanceof PrivacyGdprExportSizeLimitExceededError; if (error instanceof PrivacyGdprExportPutOutcomeUnknownError) { // The object may already exist. Preserve both the exact owner token and // DB pointer so a BullMQ retry reclaims and overwrites the same key. artifactCleanupAllowed = false; failureTransitionAllowed = false; } let cleanupSucceeded = false; if (artifactCleanupAllowed) { try { await service.deleteArtifact(attempt); cleanupSucceeded = true; } catch (cleanupError) { (deps.logger ?? noopHandlerLogger).error( { exportJobId: payload.exportJobId, error: redactSecretLikeText(cleanupError instanceof Error ? cleanupError.message : String(cleanupError)), }, "privacy-gdpr-export.attempt-cleanup-failed" ); } } if (failureTransitionAllowed && cleanupSucceeded) { try { await markFailed({ id: payload.exportJobId, tenantId: payload.tenantId, errorMessage, processingStartedAt, resultBucket: attempt.artifactBucket, resultKey: attempt.artifactKey, clearResultLocation: cleanupSucceeded, }); } catch (markError) { (deps.logger ?? noopHandlerLogger).error( { exportJobId: payload.exportJobId, error: redactSecretLikeText(markError instanceof Error ? markError.message : String(markError)), }, "privacy-gdpr-export.failure-owner-cas-missed" ); } } const details = buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }); if (isSizeLimitExceeded && cleanupSucceeded) { throw new UnrecoverableExportHandlerError("Privacy GDPR export job failed.", details); } throw new ExportHandlerError("Privacy GDPR export job failed.", details); } }); } async function completePrivacyGdprExport( payload: ExportJob, result: { artifactBucket: string; artifactKey: string; rowCount: number | null; sizeBytes: number; expiresAt: Date }, attemptsMade: number, deps: HandlerDependencies ): Promise { if (deps.sendGdprExportReadyMail) { try { await deps.sendGdprExportReadyMail({ tenantId: payload.tenantId, exportJobId: payload.exportJobId, requestedByUserId: payload.requestedByUserId, scope: payload.exportType, expiresAt: result.expiresAt, }); } catch (error) { (deps.logger ?? noopHandlerLogger).error( { exportJobId: payload.exportJobId, error: redactSecretLikeText(error instanceof Error ? error.message : String(error)) }, "privacy-gdpr-export.ready-mail-failed" ); } } return { handled: true, action: "privacy-gdpr-export.completed", details: buildExportDetails(payload, { artifactBucket: result.artifactBucket, artifactKey: result.artifactKey, attemptsMade, expiresAt: result.expiresAt.toISOString(), rowCount: result.rowCount, sizeBytes: result.sizeBytes, status: "ready", }), }; } async function runImportHandler( payload: QueuePayload, deps: HandlerDependencies, context?: HandlerExecutionContext ): Promise { // Jobs enqueued before the dedicated KWP queue rollout can still be present // on hivegrid.imports. Keep them replayable without permitting new typed // producers to target the generic queue. const legacyPayload = payload as QueuePayload | QueuePayload; if ( payload.kind === "admin-sales.contacts-import" || payload.kind === "admin-sales.contacts-export" ) { if (!deps.adminSalesDataService) { throw new ImportHandlerError( "Admin sales data worker service is not configured.", { code: "admin_sales_data_service_missing", kind: payload.kind, } ); } if (payload.kind === "admin-sales.contacts-import") { const result = await deps.adminSalesDataService.runImport(payload); return { handled: true, action: "admin-sales.contacts-import.completed", details: { importJobId: payload.importJobId, requestedByAdminUserId: payload.requestedByAdminUserId, ...result, }, }; } const result = await deps.adminSalesDataService.runExport(payload); return { handled: true, action: "admin-sales.contacts-export.completed", details: { exportJobId: payload.exportJobId, requestedByAdminUserId: payload.requestedByAdminUserId, rowCount: result.rowCount, artifactBucket: result.artifactBucket, artifactKey: result.artifactKey, expiresAt: result.expiresAt.toISOString(), }, }; } if ( legacyPayload.kind === "kwp.project-sync" || legacyPayload.kind === "kwp.project-catalog-sync" || legacyPayload.kind === "kwp.contract-sentinel" || legacyPayload.kind === "kwp.archive-discovery" || legacyPayload.kind === "kwp.archive-file-sync" ) { return runKwpImportHandler(legacyPayload, deps); } if (payload.kind !== "import.run") { throw new ImportHandlerError("Unsupported import payload kind.", { code: "invalid_payload", message: "Unsupported import payload kind.", }); } const markRunning = deps.markImportJobRunning ?? markImportJobRunningInDb; const markSucceeded = deps.markImportJobSucceeded ?? markImportJobSucceededInDb; const markFailed = deps.markImportJobFailed ?? markImportJobFailedInDb; const attemptsMade = readAttemptsMade(payload.metadata, context); await markRunning({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade }); const accessDecision = authorizeStorageObjectAccess({ tenantId: payload.tenantId, key: payload.sourceKey, operation: "download", }); if (!accessDecision.allowed) { const message = "Import source storage access denied."; await markFailed({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade, errorName: "ImportHandlerError", errorMessage: `${message} reason=${accessDecision.reason}`, }); throw new ImportHandlerError( message, buildImportDetails(payload, { status: "failed", attemptsMade, code: "source_access_denied", reason: accessDecision.reason, message }) ); } if (!deps.importService) { const message = "Import service is not configured."; await markFailed({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade, errorName: "ImportHandlerError", errorMessage: message, }); throw new ImportHandlerError(message, buildImportDetails(payload, { status: "failed", attemptsMade, code: "import_service_missing" })); } if (!deps.scanService) { const details = buildImportScanFailureDetails(payload, attemptsMade, { verdict: "error", code: "scan_service_missing", message: "Import scan service is not configured.", }); await markFailed({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade, errorName: "ImportHandlerError", errorMessage: buildImportScanFailureMessage(details), }); throw new ImportHandlerError("Import source scan failed.", details); } let scanResult: StorageScanResult; try { scanResult = await deps.scanService.scanObject({ bucket: payload.sourceBucket, key: accessDecision.key, tenantId: payload.tenantId, uploadId: payload.importJobId, contentType: payload.contentType, metadata: { ...payload.metadata, requestId: payload.requestId, importJobId: payload.importJobId, importType: payload.importType, format: payload.format, }, }); } catch (error) { const details = buildImportScanFailureDetails(payload, attemptsMade, { verdict: "error", code: "scanner_rejected", message: redactSecretLikeText(error instanceof Error ? error.message : String(error)), }); if (isTerminalJobAttempt(attemptsMade, context)) { await markFailed({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade, errorName: "ImportHandlerError", errorMessage: buildImportScanFailureMessage(details), }); } throw new ImportHandlerError("Import source scan failed.", details); } if (scanResult.verdict !== "clean") { const details = buildImportScanFailureDetails(payload, attemptsMade, buildImportScanResultDetails(scanResult)); await markFailed({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade, errorName: "ImportHandlerError", errorMessage: buildImportScanFailureMessage(details), }); throw new ImportHandlerError("Import source scan failed.", details); } try { const result = await deps.importService.runImport(payload); await markSucceeded({ id: payload.importJobId, tenantId: payload.tenantId, rowCount: result.rowCount, acceptedRows: result.acceptedRows, rejectedRows: result.rejectedRows, attemptsMade, }); return { handled: true, action: "import.completed", details: buildImportDetails(payload, { status: "completed", attemptsMade, rowCount: result.rowCount, acceptedRows: result.acceptedRows, rejectedRows: result.rejectedRows, duplicateCount: result.duplicateCount, errorReportBucket: result.errorReportBucket, errorReportKey: result.errorReportKey ?? result.validationReport?.errorReport?.errorReportKey, }), }; } catch (error) { const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error)); if (isTerminalJobAttempt(attemptsMade, context)) { await markFailed({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade, errorName: error instanceof Error ? error.name : "Error", errorMessage, }); } throw new ImportHandlerError("Import job failed.", buildImportDetails(payload, { status: "failed", attemptsMade, message: errorMessage })); } } async function runKwpImportHandler( payload: QueuePayload, deps: HandlerDependencies, ): Promise { const leaseKey = payload.kind === "kwp.project-sync" ? `kwp-project:${payload.tenantId}:${payload.bindingId}` : payload.kind === "kwp.project-catalog-sync" ? `kwp-catalog:${payload.tenantId}:${payload.dataConnectionId}` : payload.kind === "kwp.archive-discovery" ? `kwp-archive-discovery:${payload.tenantId}:${payload.bindingId}:${payload.cadenceBucket}:${payload.batchNumber}` : payload.kind === "kwp.archive-file-sync" ? `kwp-archive:${payload.tenantId}:${payload.dataConnectionId}:${payload.archiveFile.externalId}` : `kwp-contract:${payload.tenantId}:${payload.dataConnectionId}`; let operation: (() => Promise>) | null; if (payload.kind === "kwp.project-sync") { operation = deps.syncKwpProjectBinding ? async () => { const project = await deps.syncKwpProjectBinding!(payload); if (!deps.syncKwpProjectInbound || project.state === "missing" || project.state === "quarantined") { return { ...project }; } const inbound = await deps.syncKwpProjectInbound(payload); if (deps.enqueueKwpArchiveDiscovery) await deps.enqueueKwpArchiveDiscovery(payload); return { ...project, inboundRunId: inbound.runId, inboundChanged: inbound.changed, appointments: inbound.appointments, tasks: inbound.tasks, materialPositions: inbound.materialPositions, failedAreas: inbound.failedAreas, archiveDiscoveryScheduled: Boolean(deps.enqueueKwpArchiveDiscovery), }; } : null; } else if (payload.kind === "kwp.project-catalog-sync") { operation = deps.syncKwpProjectCatalog ? async () => { const result = await deps.syncKwpProjectCatalog!(payload); if (result.hasMore && deps.enqueueKwpProjectCatalogContinuation) { await deps.enqueueKwpProjectCatalogContinuation(payload); } return { ...result }; } : null; } else if (payload.kind === "kwp.archive-discovery") { operation = deps.syncKwpArchiveDiscovery ? async () => { const result = await deps.syncKwpArchiveDiscovery!(payload); const archiveJobs = deps.enqueueKwpArchiveFileJobs ? await deps.enqueueKwpArchiveFileJobs(payload, result.archiveFiles) : 0; if (result.cursor && deps.enqueueKwpArchiveDiscovery) { await deps.enqueueKwpArchiveDiscovery(payload, result.cursor); } return { processedFolders: result.processedFolders, archiveJobs, hasMore: result.cursor !== null, }; } : null; } else if (payload.kind === "kwp.archive-file-sync") { operation = deps.syncKwpArchiveFile ? async () => ({ ...(await deps.syncKwpArchiveFile!(payload)) }) : null; } else { operation = deps.runKwpContractSentinel ? async () => ({ ...(await deps.runKwpContractSentinel!(payload)) }) : null; } if (!operation) { throw new UnsupportedWorkerJobError("KWP import worker service is not configured.", { queueName: queueNames.kwpImports, kind: payload.kind, tenantId: payload.tenantId, }); } let lease; try { lease = await withDistributedJobLease( deps.tenantMaintenanceLeaseStore ?? getSharedStore(), leaseKey, operation, { leaseSeconds: 300 }, ); } catch (error) { if (isUnrecoverableKwpImportError(error)) { throw new UnrecoverableError("KWP-Import kann mit diesem Datensatz oder Vertrag nicht fortgesetzt werden."); } throw error; } if (!lease.acquired) { return { handled: true, action: "kwp.import.overlap-skipped", details: { kind: payload.kind, tenantId: payload.tenantId, reason: "active-lease" }, }; } return { handled: true, action: payload.kind === "kwp.project-sync" ? "kwp.project-sync.completed" : payload.kind === "kwp.project-catalog-sync" ? "kwp.project-catalog-sync.completed" : payload.kind === "kwp.archive-discovery" ? "kwp.archive-discovery.completed" : payload.kind === "kwp.archive-file-sync" ? "kwp.archive-file-sync.completed" : "kwp.contract-sentinel.completed", details: { kind: payload.kind, tenantId: payload.tenantId, dataConnectionId: payload.dataConnectionId, cadenceBucket: payload.cadenceBucket, ...lease.value, }, }; } function isUnrecoverableKwpImportError(error: unknown): boolean { if (error instanceof Error && error.name === "KwpInboundLimitError") return true; if (!error || typeof error !== "object" || !("code" in error)) return false; const code = (error as { code?: unknown }).code; return code === "auth_scope_missing" || code === "capability_paused" || code === "contract_drift" || code === "invalid_credentials" || code === "resource_not_found"; } function readAttemptsMade(metadata: Record | undefined, context?: { attemptsMade?: number }): number { if (context?.attemptsMade !== undefined) { return Math.max(1, context.attemptsMade + 1); } return typeof metadata?.attemptsMade === "number" && Number.isFinite(metadata.attemptsMade) ? metadata.attemptsMade : 0; } function isTerminalJobAttempt(attemptsMade: number, context?: { attemptsLimit?: number }): boolean { if (context?.attemptsLimit === undefined) { return true; } return attemptsMade >= Math.max(1, context.attemptsLimit); } function buildExportDetails(payload: ExportJob, details: Record): Record { return { exportJobId: payload.exportJobId, tenantId: payload.tenantId, exportType: payload.exportType, format: payload.format, requestId: payload.requestId, ...details, }; } function buildImportDetails(payload: FileImportJob, details: Record): Record { return { importJobId: payload.importJobId, tenantId: payload.tenantId, importType: payload.importType, format: payload.format, requestId: payload.requestId, ...details, }; } function buildScanResultDetails(result: StorageScanResult, uploadId: string | undefined): Record { const base = { verdict: result.verdict, bucket: result.bucket, key: result.key, uploadId, }; if (result.verdict === "clean") { return { ...base, engine: result.engine, }; } if (result.verdict === "infected") { return { ...base, engine: result.engine, signature: result.signature, quarantineKey: result.quarantineKey, }; } return { ...base, ...buildRedactedErrorDetails(result.error), }; } function buildImportScanResultDetails(result: StorageScanResult): Record { if (result.verdict === "clean") { return { verdict: result.verdict, engine: result.engine, }; } if (result.verdict === "infected") { return { verdict: result.verdict, code: "source_infected", engine: result.engine, signature: result.signature, quarantineKey: result.quarantineKey, message: "Import source object failed malware scan.", }; } return { verdict: result.verdict, ...buildRedactedErrorDetails(result.error), }; } function buildImportScanFailureDetails(payload: FileImportJob, attemptsMade: number, scanDetails: Record): Record { return buildImportDetails(payload, { status: "failed", attemptsMade, scan: scanDetails, }); } function buildImportScanFailureMessage(details: Record): string { const scan = details.scan && typeof details.scan === "object" ? (details.scan as Record) : {}; const parts = ["Import source scan failed.", `verdict=${String(scan.verdict ?? "error")}`, `code=${String(scan.code ?? "scan_failed")}`]; const requestId = typeof details.requestId === "string" ? details.requestId : undefined; if (requestId) { parts.push(`requestId=${requestId}`); } const message = typeof scan.message === "string" ? scan.message : undefined; if (message) { parts.push(`message=${redactSecretLikeText(message)}`); } return parts.join(" "); } function buildRejectedScanDetails(payload: QueuePayload, error: unknown): Record { return { verdict: "error", bucket: payload.bucket, key: payload.key, uploadId: payload.uploadId, code: "scanner_rejected", message: redactSecretLikeText(error instanceof Error ? error.message : String(error)), }; } function buildRedactedErrorDetails(error: StorageScanDiagnostic): Record { return { code: error.code, message: redactSecretLikeText(error.message), upstream: error.upstream ? { name: error.upstream.name, message: error.upstream.message ? redactSecretLikeText(error.upstream.message) : undefined, status: error.upstream.status, requestId: error.upstream.requestId, } : undefined, }; } function redactSecretLikeText(message: string): string { return message .replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]") .replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]") .replace( /(authorization|access[_-]?key(?:id)?|api[-_]?key|client[-_]?secret|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]", ) .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]") .slice(0, 1_000); } function serializeWebhookResponseDiagnostics(response: Response, body: string) { return JSON.stringify({ status: response.status, statusText: response.statusText, contentType: response.headers.get("content-type"), bodyBytes: new TextEncoder().encode(body).byteLength, }); } async function fetchWebhookWithRedirects( fetchImpl: typeof fetch | undefined, input: string, init: RequestInit, addressResolver?: OutboundHttpAddressResolver ): Promise { try { return await fetchOutboundHttpWithRedirects(input, init, { // In production fetchImpl is undefined, so the guard uses its pinned // undici fetch; a test-injected fetchImpl disables pinning. ...(fetchImpl ? { fetchImpl } : {}), ...(addressResolver ? { addressResolver } : {}), timeoutMs: DEFAULT_WORKER_FETCH_TIMEOUT_MS, redirectLimit: WEBHOOK_REDIRECT_LIMIT, }); } catch (error) { if (error instanceof OutboundHttpUrlValidationError) { throw new WebhookUrlValidationError(toWebhookUrlValidationMessage(error.message)); } throw error; } } function toWebhookUrlValidationMessage(message: string): string { return message.replace(/^Outbound HTTP endpoint/u, "Webhook endpoint").replace(/^Outbound HTTP redirect/u, "Webhook redirect"); } async function readLimitedResponseText(response: Response, limitBytes: number): Promise { if (!response.body) { return ""; } const reader = response.body.getReader(); const decoder = new TextDecoder(); const chunks: string[] = []; let bytes = 0; while (bytes < limitBytes) { const result = await reader.read(); if (result.done) { chunks.push(decoder.decode()); return chunks.join(""); } const chunk = result.value; const remaining = limitBytes - bytes; const slice = chunk.byteLength > remaining ? chunk.slice(0, remaining) : chunk; chunks.push(decoder.decode(slice, { stream: chunk.byteLength <= remaining })); bytes += slice.byteLength; if (chunk.byteLength > remaining) { await reader.cancel(); break; } } chunks.push(decoder.decode()); return chunks.join(""); } export type CreateRegisteredProcessorOptions = { dependencies?: HandlerDependencies; registry?: HandlerRegistry; }; export function createRegisteredProcessor(queueName: Name, options: CreateRegisteredProcessorOptions = {}): TypedProcessor { const dependencies = options.dependencies ?? {}; const registry = options.registry ?? buildHandlerRegistry(dependencies); return async (job) => { return registry[queueName](job.data, dependencies, { attemptsMade: job.attemptsMade, attemptsLimit: job.opts?.attempts, jobId: job.id === undefined ? undefined : String(job.id), updatePayload: job.updateData ? async (payload) => { await job.updateData?.(payload); } : undefined, }); }; }