5034 lines
205 KiB
TypeScript
5034 lines
205 KiB
TypeScript
import {
|
|
updateLegacyEventWebhookAttempt as updateLegacyEventWebhookAttemptInDb,
|
|
type LegacyEventWebhookAttemptUpdate,
|
|
} from "@hivegrid/db/event-webhook-attempts.server";
|
|
import {
|
|
listExpiredCompletedExportJobs as listExpiredCompletedExportJobsInDb,
|
|
listSoftDeletedExportJobsForHardDelete as listSoftDeletedExportJobsForHardDeleteInDb,
|
|
markExportJobExpired as markExportJobExpiredInDb,
|
|
markExportJobFailed as markExportJobFailedInDb,
|
|
markExportJobHardDeleteFailed as markExportJobHardDeleteFailedInDb,
|
|
markExportJobHardDeleteSucceeded as markExportJobHardDeleteSucceededInDb,
|
|
markExportJobPaused as markExportJobPausedInDb,
|
|
markExportJobResumed as markExportJobResumedInDb,
|
|
markExportJobRunning as markExportJobRunningInDb,
|
|
markExportJobSucceeded as markExportJobSucceededInDb,
|
|
type ExportJobExpiredInput,
|
|
type ExportJobFailedInput,
|
|
type ExportJobHardDeleteFailedInput,
|
|
type ExportJobHardDeleteSucceededInput,
|
|
type ExportJobPausedInput,
|
|
type ExportJobResumedInput,
|
|
type ExportJobRetentionListInput,
|
|
type ExportJobRetentionRecord,
|
|
type ExportJobRunningInput,
|
|
type ExportJobSucceededInput,
|
|
} from "@hivegrid/db/export-jobs.server";
|
|
import {
|
|
listExpiredGdprExportArtifacts as listExpiredGdprExportArtifactsInDb,
|
|
markGdprExportArtifactDeleted as markGdprExportArtifactDeletedInDb,
|
|
markGdprExportArtifactExpired as markGdprExportArtifactExpiredInDb,
|
|
markGdprExportFailed as markGdprExportFailedInDb,
|
|
markGdprExportProcessing as markGdprExportProcessingInDb,
|
|
markGdprExportReady as markGdprExportReadyInDb,
|
|
type GdprExportRetentionRecord,
|
|
type GdprExportArtifactRetentionInput,
|
|
type MarkGdprExportFailedInput,
|
|
type MarkGdprExportProcessingInput,
|
|
type MarkGdprExportReadyInput,
|
|
type MarkGdprExportReadyResult,
|
|
} from "@hivegrid/db/privacy-gdpr.server";
|
|
import {
|
|
markImportJobFailed as markImportJobFailedInDb,
|
|
markImportJobRunning as markImportJobRunningInDb,
|
|
markImportJobSucceeded as markImportJobSucceededInDb,
|
|
type ImportJobFailedInput,
|
|
type ImportJobRunningInput,
|
|
type ImportJobSucceededInput,
|
|
} from "@hivegrid/db/import-jobs.server";
|
|
import { runAdminSalesTaskReminderSweep as runAdminSalesTaskReminderSweepInDb } from "@hivegrid/db/admin-sales-tasks.server";
|
|
import { runAdminBookingReminderSweep as runAdminBookingReminderSweepInDb } from "@hivegrid/db/admin-booking-reminders.server";
|
|
import { syncAdminCalendarBusySnapshots } from "@hivegrid/integrations/admin-calendar-sync-worker.server";
|
|
import { redactExpiredAdminEmailEventRawData as redactExpiredAdminEmailEventRawDataInDb } from "@hivegrid/db/admin-email-events.server";
|
|
import {
|
|
createAdminPlatformNotificationDelivery as createAdminPlatformNotificationDeliveryInDb,
|
|
createNotificationDelivery as createNotificationDeliveryInDb,
|
|
createPlatformNotificationDelivery as createPlatformNotificationDeliveryInDb,
|
|
getUserNotificationPreferences as getUserNotificationPreferencesInDb,
|
|
markAdminPlatformNotificationDeliveryStatus as markAdminPlatformNotificationDeliveryStatusInDb,
|
|
markNotificationDeliveryStatus as markNotificationDeliveryStatusInDb,
|
|
markPlatformNotificationDeliveryStatus as markPlatformNotificationDeliveryStatusInDb,
|
|
redactNotificationDiagnosticText,
|
|
type CreateAdminPlatformNotificationDeliveryInput,
|
|
type CreateNotificationDeliveryInput,
|
|
type CreatePlatformNotificationDeliveryInput,
|
|
type MarkAdminPlatformNotificationDeliveryStatusInput,
|
|
type MarkNotificationDeliveryStatusInput,
|
|
type MarkPlatformNotificationDeliveryStatusInput,
|
|
} from "@hivegrid/db/notifications.server";
|
|
import { resolvePlatformLeadNotificationRecipient as resolvePlatformLeadNotificationRecipientInDb } from "@hivegrid/db/platform-lead-notifications.server";
|
|
import {
|
|
deleteDoneOutboxEvents as deleteDoneOutboxEventsInDb,
|
|
type DeleteDoneOutboxEventsOptions,
|
|
type DeleteDoneOutboxEventsResult,
|
|
} from "@hivegrid/db/outbox-retention";
|
|
import { type EnvRecord } from "@hivegrid/config";
|
|
import type { ValidationReport } from "@hivegrid/imports";
|
|
import { createMailjetProviderFromEnv, type MailMetricsHook, type MailProvider } from "@hivegrid/mail";
|
|
import { publishOutboxEvent, type OutboxPublisherTransaction } from "@hivegrid/db/outbox";
|
|
import { runAutomationSchedulerForTenantWithTx } from "@hivegrid/db/automation-scheduler.server";
|
|
import {
|
|
buildWorkflowTriggerOutboxEvent,
|
|
queueNames,
|
|
type ExportJob,
|
|
type AdminSalesContactsExportJob,
|
|
type AdminSalesContactsImportJob,
|
|
type FilePreviewJob,
|
|
type FileImportJob,
|
|
type ImportJob,
|
|
type KwpArchiveDiscoveryJob,
|
|
type KwpArchiveFileSyncJob,
|
|
type KwpContractSentinelJob,
|
|
type KwpProjectCatalogSyncJob,
|
|
type KwpProjectSyncJob,
|
|
type PluginInvokeJob,
|
|
type PlatformNotificationJob,
|
|
type QueueName,
|
|
type QueuePayload,
|
|
type TypedProcessor,
|
|
} from "@hivegrid/queue";
|
|
import {
|
|
assertGdprExportArtifactKey,
|
|
authorizeStorageObjectAccess,
|
|
type DeleteObjectInput,
|
|
type DeleteObjectResult,
|
|
type StorageScanDiagnostic,
|
|
type StorageScanResult,
|
|
} from "@hivegrid/storage";
|
|
import { isCriticalNotificationCategory, mapWithConcurrency, parsePortalStatusMailDeliveryMetadata, type NotificationClient } from "@hivegrid/foundation";
|
|
import { OutboundHttpUrlValidationError, fetchOutboundHttpWithRedirects, type OutboundHttpAddressResolver } from "@hivegrid/foundation/outbound-http";
|
|
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
|
|
import { notifyWithTenantRlsContext } from "./notify-rls.js";
|
|
import type { SharedStore } from "@hivegrid/shared-store";
|
|
import { UnrecoverableError } from "bullmq";
|
|
import {
|
|
hasValidWorkspaceOwnerInvitationMailContract,
|
|
isRedactedWorkspaceOwnerInvitationMailPayload,
|
|
isTerminalWorkspaceOwnerInvitationMailPayload,
|
|
isWorkspaceOwnerInvitationMailIntent,
|
|
readWorkspaceOwnerInvitationRecoveryBinding,
|
|
redactTerminalWorkspaceOwnerInvitationMailPayload,
|
|
redactWorkspaceOwnerInvitationError,
|
|
} from "./security/workspace-owner-invitation-secret.js";
|
|
import {
|
|
abandonWorkspaceOwnerInvitationDelivery,
|
|
acceptWorkspaceOwnerInvitationDelivery,
|
|
claimWorkspaceOwnerInvitationDelivery,
|
|
scheduleWorkspaceOwnerInvitationTerminalRecovery,
|
|
terminalizeWorkspaceOwnerInvitationDelivery,
|
|
type WorkspaceOwnerInvitationDeliveryClaimResult,
|
|
} from "./security/workspace-owner-invitation-terminalization.js";
|
|
import {
|
|
hasValidPortalSetupInvitationMailContract,
|
|
isMaterializedPortalSetupInvitationMailPayload,
|
|
isPortalSetupMailIntent,
|
|
isPortalSetupInvitationProviderAccepted,
|
|
isRedactedPortalSetupInvitationMailPayload,
|
|
isTerminalPortalSetupInvitationMailPayload,
|
|
readPortalSetupInvitationRecoveryBinding,
|
|
redactPortalSetupInvitationError,
|
|
redactTerminalPortalSetupInvitationMailPayload,
|
|
} from "./security/portal-setup-invitation-secret.js";
|
|
import {
|
|
acceptPortalSetupInvitationDelivery,
|
|
claimPortalSetupInvitationDelivery,
|
|
schedulePortalSetupInvitationTerminalRecovery,
|
|
terminalizePortalSetupInvitationDelivery,
|
|
type PortalSetupInvitationDeliveryClaimResult,
|
|
} from "./security/portal-setup-invitation-terminalization.js";
|
|
|
|
import {
|
|
drainCalendarWebhookEventsForTenant,
|
|
refreshCalendarTokensForTenant,
|
|
renewCalendarWebhooksForTenant,
|
|
syncInternalCalendarEventsForTenant,
|
|
} from "@hivegrid/integrations/calendar-worker-tasks.server";
|
|
import {
|
|
cleanupEmailWebhookEventsForTenant,
|
|
drainEmailWebhookEventsForTenant,
|
|
pollStaleEmailMailboxesForTenant,
|
|
refreshEmailTokensForTenant,
|
|
renewEmailWebhooksForTenant,
|
|
runEmailInitialSyncForTenant,
|
|
} from "@hivegrid/integrations/email-worker-tasks.server";
|
|
import {
|
|
checkAdminSendingDomains,
|
|
deliverDueAdminEmails,
|
|
syncClaimedAdminMailboxes,
|
|
} from "@hivegrid/integrations";
|
|
import { createWorkerAdminEmailAttachmentProcessor } from "./admin-email-attachments.js";
|
|
import {
|
|
runAdminCallRecordingProcessing,
|
|
runAdminCallRecordingRetention,
|
|
} from "./admin-call-recordings.js";
|
|
import { runAdminOutboundCalls } from "./admin-outbound-calls.js";
|
|
import { createEmailAutoMatchHook } from "@hivegrid/integrations/email-match-hook.server";
|
|
import { ensureKwpAccessTokenForConnection } from "@hivegrid/integrations/kwp-resource-client.server";
|
|
import {
|
|
cleanupExpiredKwpPendingAttemptsForTenant,
|
|
executeKwpHealthProbe,
|
|
listKwpHealthConnectionIdsForTenant,
|
|
persistKwpHealthProbe,
|
|
prepareKwpHealthProbeForConnection,
|
|
refreshExpiringTokensForTenant,
|
|
sweepConnectionHealthForTenant,
|
|
} from "@hivegrid/integrations/worker-tasks.server";
|
|
import { autoArchiveProjectsForTenantWithTx } from "@hivegrid/db/project-auto-archive.server";
|
|
import { evaluateBillingDunning } from "@hivegrid/db/billing-domain.server";
|
|
import { createAuditEvent } from "@hivegrid/db/audit-events.server";
|
|
import { archiveProjectChannelIfEnabledWithTx } from "@hivegrid/chat/project-sync.server";
|
|
import { withTenantRLS } from "./db/tenantRls.js";
|
|
import { runActiveTenantSweep, throwOnActiveTenantSweepFailures, type ActiveTenantReader } from "./jobs/active-tenant-sweep.js";
|
|
import { withDistributedJobLease } from "./jobs/distributed-job-lease.js";
|
|
import { getSharedStore } from "./shared-store.js";
|
|
import { runDsgvoGraceCron, type DsgvoGraceCandidate } from "./jobs/dsgvo-grace-cron.js";
|
|
import { reminderChannel, reminderMessageKey, reminderScanBounds, reminderSourceKey, selectReminderWindows } from "./jobs/task-reminder-cron.js";
|
|
import { audienceMatches, nextReminderDue, HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS } from "./jobs/hr-upload-reminder-cron.js";
|
|
import { createDnsDomainResolver, decideDomainProbeStatus, type DomainDnsResolver } from "./jobs/email-domain-probe-cron.js";
|
|
import { handlePluginInvokeJob, type PluginInvokeDeps } from "./jobs/plugin-invoke.js";
|
|
import {
|
|
reconcileWorkflowExecutionStalls,
|
|
type WorkflowExecutionStalledReconcileInput,
|
|
type WorkflowExecutionStalledReconcileResult,
|
|
type WorkflowExecutionStalledReconcilerPrisma,
|
|
} from "./jobs/workflow-execution-stalled-reconciler.js";
|
|
import {
|
|
isPrivacyGdprExportType,
|
|
PrivacyGdprExportSizeLimitExceededError,
|
|
resolvePrivacyGdprExportSubject,
|
|
type PrivacyGdprExportRunResult,
|
|
} from "./jobs/privacy-gdpr-export.js";
|
|
import { isPortalGdprExportType, PortalGdprExportSizeLimitExceededError, type PortalGdprExportRunResult } from "./jobs/portal-gdpr-export.js";
|
|
import { PrivacyGdprExportPutOutcomeUnknownError } from "./jobs/privacy-gdpr-export-bootstrap.js";
|
|
import { TenantExportQuiescedError, type TenantExportBarrierCheckpoint } from "./jobs/tenant-export-quiescence.js";
|
|
import {
|
|
runPortalGdprExportRetentionSweep,
|
|
type PortalGdprExportRetentionPrisma,
|
|
type PortalGdprExportRetentionResult,
|
|
} from "./jobs/portal-gdpr-export-retention.js";
|
|
import { notifyPortalStatusMailPermanentFailure } from "./jobs/portal-status-mail-store.js";
|
|
import type { PortalStatusMailFailureInput } from "./jobs/portal-status-mail.js";
|
|
|
|
const DEFAULT_WORKER_FETCH_TIMEOUT_MS = 10_000;
|
|
const WEBHOOK_RESPONSE_BODY_LIMIT_BYTES = 4_096;
|
|
const WEBHOOK_REDIRECT_LIMIT = 5;
|
|
const EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY = 5;
|
|
const EXPORT_ARTIFACT_RETENTION_DELETE_TIMEOUT_MS = 60_000;
|
|
|
|
export type HandlerResult = {
|
|
handled: true;
|
|
action: string;
|
|
details?: Record<string, unknown>;
|
|
};
|
|
|
|
export type WebhookDeliveryMetricsHook = (input: { status: "success" | "retry" | "dlq" | "rejected" }) => void;
|
|
|
|
export type HandlerDependencies = {
|
|
mailProvider?: MailProvider;
|
|
/** Notifies the responsible internal TenantUser after final portal-mail delivery failure. */
|
|
notifyPortalStatusMailFailure?: (input: PortalStatusMailFailureInput) => Promise<unknown>;
|
|
/** Invalidates an undeliverable OWNER capability on the final mail attempt. */
|
|
terminalizeOwnerInvitation?: (input: { tenantId: string; payload: unknown }) => Promise<unknown>;
|
|
/** Claims the one permitted OWNER provider attempt in the originating DB outbox row. */
|
|
claimOwnerInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise<WorkspaceOwnerInvitationDeliveryClaimResult>;
|
|
/** Marks the claimed OWNER provider attempt as accepted. */
|
|
acceptOwnerInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise<unknown>;
|
|
/** Invalidates a claimed OWNER delivery whose provider outcome is ambiguous. */
|
|
abandonOwnerInvitationDelivery?: (input: { tenantId: string; payload: unknown; lastError: string }) => Promise<unknown>;
|
|
/** Durably rearms the originating DB outbox row when invalidation itself fails. */
|
|
scheduleOwnerInvitationRecovery?: (input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => Promise<unknown>;
|
|
/** Claims the one permitted customer-portal setup provider attempt. */
|
|
claimPortalSetupInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise<PortalSetupInvitationDeliveryClaimResult>;
|
|
/** Marks a claimed customer-portal setup delivery as provider-accepted. */
|
|
acceptPortalSetupInvitationDelivery?: (input: { tenantId: string; payload: unknown }) => Promise<unknown>;
|
|
/** Revokes the setup capability and terminalizes its exact outbox row. */
|
|
terminalizePortalSetupInvitation?: (input: { tenantId: string; payload: unknown; lastError?: string }) => Promise<unknown>;
|
|
/** Durably rearms setup-capability invalidation when the database is transiently unavailable. */
|
|
schedulePortalSetupInvitationRecovery?: (input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) => Promise<unknown>;
|
|
/**
|
|
* Optional metrics hook forwarded to the lazily-constructed Mailjet
|
|
* provider. When the worker entry wires this to the shared Prometheus
|
|
* registry, every send increments mail_sent_total{provider, status}.
|
|
*/
|
|
mailMetricsHook?: MailMetricsHook;
|
|
/**
|
|
* Optional metrics hook fired for each outbound webhook attempt. Status:
|
|
* - success : 2xx response
|
|
* - retry : retryable failure (5xx, transient network) below the
|
|
* attempts cap, BullMQ will requeue
|
|
* - dlq : terminal failure (final attempt or non-retryable status)
|
|
* - rejected : URL validation failure (SSRF guard, etc.)
|
|
*/
|
|
webhookDeliveryMetricsHook?: WebhookDeliveryMetricsHook;
|
|
webhookAddressResolver?: OutboundHttpAddressResolver;
|
|
fetch?: typeof fetch;
|
|
env?: EnvRecord;
|
|
/** Test/deployment seam for replica-wide maintenance overlap protection. */
|
|
tenantMaintenanceLeaseStore?: SharedStore;
|
|
/** Schedules due KWP project/contract work onto the dedicated KWP queue. */
|
|
runKwpProjectMaintenance?: () => Promise<{
|
|
inspectedTenants: number;
|
|
projectJobs: number;
|
|
catalogJobs: number;
|
|
contractJobs: number;
|
|
failures: number;
|
|
}>;
|
|
/** Import-group provider work. Operations run under resource-scoped Redis leases. */
|
|
syncKwpProjectBinding?: (payload: KwpProjectSyncJob) => Promise<{
|
|
runId: string;
|
|
state: "synced" | "unchanged" | "missing" | "quarantined";
|
|
projectId: string;
|
|
stageChanged: boolean;
|
|
}>;
|
|
syncKwpProjectCatalog?: (payload: KwpProjectCatalogSyncJob) => Promise<{
|
|
discoveredCount: number;
|
|
hydratedCount: number;
|
|
processed: number;
|
|
hasMore: boolean;
|
|
status: "pending" | "syncing" | "ready" | "partial" | "paused";
|
|
}>;
|
|
enqueueKwpProjectCatalogContinuation?: (payload: KwpProjectCatalogSyncJob) => Promise<void>;
|
|
syncKwpProjectInbound?: (payload: KwpProjectSyncJob) => Promise<{
|
|
runId: string;
|
|
projectId: string;
|
|
appointments: number;
|
|
tasks: number;
|
|
materialPositions: number;
|
|
failedAreas: readonly ("appointments" | "tasks" | "material")[];
|
|
changed: boolean;
|
|
}>;
|
|
syncKwpArchiveDiscovery?: (payload: KwpArchiveDiscoveryJob) => Promise<{
|
|
archiveFiles: readonly KwpArchiveFileSyncJob["archiveFile"][];
|
|
processedFolders: number;
|
|
cursor: KwpArchiveDiscoveryJob["cursor"] | null;
|
|
}>;
|
|
enqueueKwpArchiveDiscovery?: (
|
|
payload: KwpProjectSyncJob | KwpArchiveDiscoveryJob,
|
|
cursor?: KwpArchiveDiscoveryJob["cursor"],
|
|
) => Promise<void>;
|
|
enqueueKwpArchiveFileJobs?: (
|
|
payload: KwpArchiveDiscoveryJob,
|
|
files: readonly KwpArchiveFileSyncJob["archiveFile"][],
|
|
) => Promise<number>;
|
|
syncKwpArchiveFile?: (payload: KwpArchiveFileSyncJob) => Promise<{
|
|
runId: string;
|
|
fileId: string;
|
|
projectId: string;
|
|
storageKey: string;
|
|
state: "created" | "replayed";
|
|
}>;
|
|
runKwpContractSentinel?: (payload: KwpContractSentinelJob) => Promise<{
|
|
status: "initialized" | "unchanged" | "paused";
|
|
fingerprint: string;
|
|
}>;
|
|
/** Test seam for bounded cross-tenant maintenance enumeration. */
|
|
activeTenantReader?: ActiveTenantReader;
|
|
/** General-group service for bounded telephony webhook retention and event recovery. */
|
|
runTelephonyWebhookMaintenance?: () => Promise<{
|
|
deletedCompleted: number;
|
|
deletedFailed: number;
|
|
recoveredPending: number;
|
|
recoveredFailed: number;
|
|
inspectedTenants: number;
|
|
}>;
|
|
/** General-group service for bounded, tenant-RLS Mollie acquisition recovery. */
|
|
runProductAcquisitionReconcile?: (options?: { take?: number }) => Promise<{
|
|
inspected: number;
|
|
expired: number;
|
|
recoveredProviderPayments: number;
|
|
refreshed: number;
|
|
appliedTerminal: number;
|
|
quarantined: number;
|
|
raced: number;
|
|
inspectedTenants: number;
|
|
}>;
|
|
updateLegacyEventWebhookAttempt?: (attempt: { id: string; tenantId: string | null }, data: LegacyEventWebhookAttemptUpdate) => Promise<void>;
|
|
scanService?: {
|
|
scanObject(target: WorkerStorageScanTarget): Promise<StorageScanResult>;
|
|
};
|
|
/**
|
|
* AK-P4a §5 — writes the scan verdict (clean|infected) back onto
|
|
* File.scanStatus so the fail-closed media proxy (AD-23) can serve the
|
|
* object. Composed in the `general` group bootstrap (jobs/file-scan-status.ts)
|
|
* with a Prisma seam. Jobs without a fileId are legacy scan-only imports;
|
|
* jobs with a fileId fail/retry when persistence is not configured.
|
|
*/
|
|
fileScanStatusWriter?: {
|
|
recordVerdict(input: { fileId: string; storageKey: string; tenantId?: string; verdict: "clean" | "infected" }): Promise<boolean>;
|
|
};
|
|
/**
|
|
* A clean AV verdict is the only producer path for an upload's initial
|
|
* preview. Enqueue failure rejects the scan job, so BullMQ retries the
|
|
* scan/write/enqueue sequence; the preview job id is deterministic.
|
|
*/
|
|
filePreviewEnqueuer?: {
|
|
enqueueAfterCleanScan(input: { bucket: string; contentType: string; fileId: string; storageKey: string; tenantId: string }): Promise<void>;
|
|
};
|
|
/**
|
|
* AK-P4a §4 — file-preview generator. Composed in the `general` worker
|
|
* group bootstrap (jobs/file-preview.ts) with Prisma + storage seams.
|
|
* The handler stays thin: it delegates to this service, which loads the
|
|
* object, renders the preview (HEIC->JPG, image->WebP, PDF->PNG) and
|
|
* writes previewStatus + preview/thumbnail keys back to the File row.
|
|
* When unset, the handler throws so the job dead-letters loudly rather
|
|
* than silently leaving previewStatus stuck on `generating`.
|
|
*/
|
|
filePreviewService?: {
|
|
generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult>;
|
|
};
|
|
exportService?: {
|
|
runExport(payload: ExportJob): Promise<WorkerExportRunResult>;
|
|
};
|
|
privacyGdprExportService?: {
|
|
withTenantBarrier<Value>(payload: ExportJob, writer: (checkpoint: TenantExportBarrierCheckpoint) => Promise<Value>): Promise<Value>;
|
|
resolveReady(payload: ExportJob): Promise<{
|
|
artifactBucket: string;
|
|
artifactKey: string;
|
|
rowCount: null;
|
|
sizeBytes: number;
|
|
expiresAt: Date;
|
|
} | null>;
|
|
resolveReclaimableProcessing(payload: ExportJob): Promise<{
|
|
processingStartedAt: Date;
|
|
attempt: { attemptId: string; artifactBucket: string; artifactKey: string };
|
|
} | null>;
|
|
createAttempt(payload: ExportJob): { attemptId: string; artifactBucket: string; artifactKey: string };
|
|
runExport(payload: ExportJob, attempt: { attemptId: string; artifactBucket: string; artifactKey: string }): Promise<PrivacyGdprExportRunResult>;
|
|
deleteArtifact(attempt: { attemptId: string; artifactBucket: string; artifactKey: string }): Promise<void>;
|
|
terminalizeReadyAfterQuiescence(input: { payload: ExportJob; result: PrivacyGdprExportRunResult }): Promise<void>;
|
|
};
|
|
/** VA-P26b — separate PortalUser JSON export; never uses a TenantUser. */
|
|
portalGdprExportService?: {
|
|
runExport(payload: ExportJob): Promise<PortalGdprExportRunResult>;
|
|
};
|
|
/** Dedicated 15-minute PortalUser export retention sweep; never part of the daily portal lifecycle. */
|
|
portalGdprExportRetentionService?: {
|
|
runSweep(input: { now: Date; take?: number }): Promise<PortalGdprExportRetentionResult>;
|
|
};
|
|
importService?: {
|
|
runImport(payload: ImportJob): Promise<WorkerImportRunResult>;
|
|
};
|
|
adminSalesDataService?: {
|
|
runImport(payload: AdminSalesContactsImportJob): Promise<{
|
|
readonly totalRows: number;
|
|
readonly acceptedRows: number;
|
|
readonly rejectedRows: number;
|
|
}>;
|
|
runExport(payload: AdminSalesContactsExportJob): Promise<{
|
|
readonly rowCount: number;
|
|
readonly artifactBucket: string;
|
|
readonly artifactKey: string;
|
|
readonly expiresAt: Date;
|
|
}>;
|
|
};
|
|
storageDeleteService?: {
|
|
deleteObject(input: DeleteObjectInput): Promise<DeleteObjectResult>;
|
|
};
|
|
listSoftDeletedExportJobsForHardDelete?: (data: ExportJobRetentionListInput) => Promise<ExportJobRetentionRecord[]>;
|
|
listExpiredCompletedExportJobs?: (data: ExportJobRetentionListInput) => Promise<ExportJobRetentionRecord[]>;
|
|
markExportJobExpired?: (data: ExportJobExpiredInput) => Promise<unknown>;
|
|
markExportJobHardDeleteSucceeded?: (data: ExportJobHardDeleteSucceededInput) => Promise<unknown>;
|
|
markExportJobHardDeleteFailed?: (data: ExportJobHardDeleteFailedInput) => Promise<unknown>;
|
|
markExportJobPaused?: (data: ExportJobPausedInput) => Promise<unknown>;
|
|
markExportJobResumed?: (data: ExportJobResumedInput) => Promise<unknown>;
|
|
markExportJobRunning?: (data: ExportJobRunningInput) => Promise<unknown>;
|
|
markExportJobSucceeded?: (data: ExportJobSucceededInput) => Promise<unknown>;
|
|
markExportJobFailed?: (data: ExportJobFailedInput) => Promise<unknown>;
|
|
markGdprExportProcessing?: (data: MarkGdprExportProcessingInput) => Promise<unknown>;
|
|
markGdprExportReady?: (data: MarkGdprExportReadyInput) => Promise<MarkGdprExportReadyResult>;
|
|
markGdprExportFailed?: (data: MarkGdprExportFailedInput) => Promise<unknown>;
|
|
listExpiredGdprExportArtifacts?: (data: { tenantId: string; now: Date; take?: number }) => Promise<GdprExportRetentionRecord[]>;
|
|
markGdprExportArtifactExpired?: (data: GdprExportArtifactRetentionInput) => Promise<unknown>;
|
|
markGdprExportArtifactDeleted?: (data: Pick<GdprExportArtifactRetentionInput, "id" | "tenantId" | "resultBucket" | "resultKey">) => Promise<unknown>;
|
|
markImportJobRunning?: (data: ImportJobRunningInput) => Promise<unknown>;
|
|
markImportJobSucceeded?: (data: ImportJobSucceededInput) => Promise<unknown>;
|
|
markImportJobFailed?: (data: ImportJobFailedInput) => Promise<unknown>;
|
|
getUserNotificationPreferences?: (filters: { tenantId: string; userId: string; channel?: string | null; category?: string | null }) => Promise<unknown[]>;
|
|
createNotificationDelivery?: (data: CreateNotificationDeliveryInput) => Promise<unknown>;
|
|
markNotificationDeliveryStatus?: (data: MarkNotificationDeliveryStatusInput) => Promise<unknown>;
|
|
createPlatformNotificationDelivery?: (data: CreatePlatformNotificationDeliveryInput) => Promise<unknown>;
|
|
createAdminPlatformNotificationDelivery?: (
|
|
data: CreateAdminPlatformNotificationDeliveryInput
|
|
) => Promise<unknown>;
|
|
markPlatformNotificationDeliveryStatus?: (data: MarkPlatformNotificationDeliveryStatusInput) => Promise<unknown>;
|
|
markAdminPlatformNotificationDeliveryStatus?: (
|
|
data: MarkAdminPlatformNotificationDeliveryStatusInput
|
|
) => Promise<unknown>;
|
|
runAdminSalesTaskReminderSweep?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{ readonly scanned: number; readonly dispatched: number }>;
|
|
runAdminBookingReminderSweep?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly scanned: number;
|
|
readonly dispatched: number;
|
|
readonly failed: number;
|
|
}>;
|
|
runAdminCalendarSync?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly inspected: number;
|
|
readonly synced: number;
|
|
readonly failed: readonly string[];
|
|
}>;
|
|
runAdminEmailDelivery?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly inspected: number;
|
|
readonly claimed: number;
|
|
readonly sent: number;
|
|
readonly failed: readonly string[];
|
|
readonly blocked: number;
|
|
readonly deferred: number;
|
|
readonly recoveredUnknown: number;
|
|
readonly noReplyReminders: {
|
|
readonly scanned: number;
|
|
readonly created: number;
|
|
};
|
|
}>;
|
|
redactExpiredAdminEmailEventRawData?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly inspected: number;
|
|
readonly redacted: number;
|
|
readonly deletedLinks: number;
|
|
}>;
|
|
runAdminEmailSync?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly inspected: number;
|
|
readonly synced: number;
|
|
readonly upserted: number;
|
|
readonly replayed: number;
|
|
readonly failed: readonly string[];
|
|
}>;
|
|
runAdminEmailDomainCheck?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly inspected: number;
|
|
readonly verified: number;
|
|
readonly degraded: number;
|
|
readonly failed: readonly string[];
|
|
}>;
|
|
runAdminCallRecordingProcessing?: (input: {
|
|
readonly take?: number;
|
|
}) => Promise<{ readonly claimed: number; readonly processed: number }>;
|
|
runAdminCallRecordingRetention?: (input: {
|
|
readonly now: Date;
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly claimed: number;
|
|
readonly deleted: number;
|
|
readonly failed: number;
|
|
}>;
|
|
runAdminOutboundCalls?: (input: {
|
|
readonly take?: number;
|
|
}) => Promise<{
|
|
readonly claimed: number;
|
|
readonly accepted: number;
|
|
readonly failed: number;
|
|
}>;
|
|
resolvePlatformLeadNotificationRecipient?: (data: { userId: string; leadId: string }) => Promise<{ eligible?: boolean; email: string | null }>;
|
|
/**
|
|
* Phase 10 — DSGVO grace-period cron service. The maintenance handler
|
|
* looks up overdue User rows and dispatches cascade-deletions through
|
|
* this slot. When unset, the handler falls back to a Prisma-backed
|
|
* implementation that lists candidates and logs them without running
|
|
* the destructive cascade — the cascade-runtime wiring lands in a
|
|
* follow-up. The split keeps the cron observable from day-one without
|
|
* blocking on the full cascade implementation.
|
|
*/
|
|
dsgvoGraceCronService?: {
|
|
listOverdueUsers: (now: Date) => Promise<DsgvoGraceCandidate[]>;
|
|
enqueueDeletion: (input: {
|
|
userId: string;
|
|
tenantId: string | null;
|
|
requestId: string;
|
|
actorType: "system";
|
|
actorId: string;
|
|
mode: "pseudonymize";
|
|
reason: string;
|
|
}) => Promise<{ jobId?: string }>;
|
|
};
|
|
/**
|
|
* AK-P21 — best-effort "export ready" mail to the requesting user. Wired by
|
|
* the import-group bootstrap next to privacyGdprExportService; failures are
|
|
* logged and never fail the export job.
|
|
*/
|
|
sendGdprExportReadyMail?: (input: { tenantId: string; exportJobId: string; requestedByUserId: string; scope: string; expiresAt: Date }) => Promise<void>;
|
|
/**
|
|
* AK-P21 — tenant hard-delete cron service. The maintenance handler runs
|
|
* one scan over due TenantDeletionRequest rows (grace period elapsed) and
|
|
* executes the orchestrated hard delete (billing cancel, storage prefix,
|
|
* search documents, DEK crypto-shredding, tenant rows). When unset the
|
|
* handler acks the cron tick with an explicit skip log.
|
|
*/
|
|
privacyTenantDeletionService?: {
|
|
runScan(now?: Date): Promise<{
|
|
scanned: number;
|
|
completed: number;
|
|
failed: number;
|
|
failures: Array<{ requestId: string; tenantId: string; error: string }>;
|
|
}>;
|
|
};
|
|
/** VA-P26b — bounded customer-portal deletion, retention and access scan. */
|
|
portalLifecycleService?: {
|
|
runScan(now?: Date): Promise<{
|
|
inspectedTenants: number;
|
|
deletionScanned: number;
|
|
deletionCompleted: number;
|
|
exportArtifactsExpired: number;
|
|
inactivityWarnings: number;
|
|
inactivityDisabled: number;
|
|
projectCompletionMails: number;
|
|
projectAccessesClosed: number;
|
|
failed: number;
|
|
}>;
|
|
};
|
|
deleteDoneOutboxEvents?: (data: DeleteDoneOutboxEventsOptions) => Promise<DeleteDoneOutboxEventsResult>;
|
|
reconcileWorkflowExecutionStalls?: (data: WorkflowExecutionStalledReconcileInput) => Promise<WorkflowExecutionStalledReconcileResult>;
|
|
/**
|
|
* Phase 06 #21 / F-015 — production dependencies for the plugin-invoke
|
|
* handler. The `plugin` worker group bootstrap (`apps/worker/src/jobs/
|
|
* plugin-invoke-bootstrap.ts`) composes the real Prisma / filesystem /
|
|
* SharedStore / sandbox seams and injects them here. When the slot is
|
|
* unset (e.g. a misconfigured `general` worker receives a plugin-invoke
|
|
* job because someone changed group routing) the registry entry throws
|
|
* so the job dead-letters loudly rather than being silently acked.
|
|
*/
|
|
pluginInvokeDeps?: PluginInvokeDeps;
|
|
/**
|
|
* Optional structured logger used by handlers that need to emit
|
|
* non-error progress (currently only the dsgvo-grace-cron branch). A
|
|
* pino-shaped surface keeps this compatible with the worker bootstrap
|
|
* logger without dragging the pino type-graph into this module.
|
|
*/
|
|
logger?: {
|
|
info: (fields: Record<string, unknown>, msg: string) => void;
|
|
error: (fields: Record<string, unknown>, msg: string) => void;
|
|
};
|
|
};
|
|
|
|
type WorkerExportRunResult = {
|
|
artifactBucket: string;
|
|
artifactKey: string;
|
|
rowCount: number;
|
|
expiresAt: Date;
|
|
};
|
|
|
|
type WorkerImportRunResult = {
|
|
rowCount: number;
|
|
acceptedRows: number;
|
|
rejectedRows: number;
|
|
duplicateCount: number;
|
|
validationReport?: Pick<ValidationReport, "summary" | "errorReport">;
|
|
errorReportBucket?: string;
|
|
errorReportKey?: string;
|
|
};
|
|
|
|
type WorkerStorageScanTarget = {
|
|
bucket: string;
|
|
key: string;
|
|
tenantId?: string;
|
|
uploadId?: string;
|
|
contentType?: string;
|
|
metadata?: Record<string, unknown>;
|
|
};
|
|
|
|
export type FilePreviewServiceResult = {
|
|
/** Terminal preview state written to the File row by the service. */
|
|
previewStatus: "ready" | "failed" | "not_supported";
|
|
previewStorageKey?: string;
|
|
thumbnailStorageKey?: string;
|
|
/** Optional diagnostic reason (e.g. why generation failed). */
|
|
reason?: string;
|
|
};
|
|
|
|
class WebhookDeliveryError extends Error {}
|
|
class WebhookUrlValidationError extends Error {}
|
|
|
|
class NotificationHandlerError extends Error {
|
|
readonly action = "notification.rejected";
|
|
}
|
|
|
|
class UnsupportedWorkerJobError extends Error {
|
|
readonly action = "worker.unsupported";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "UnsupportedWorkerJobError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
class PluginInvokePayloadError extends Error {
|
|
readonly action = "plugin.invoke.invalid_payload";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "PluginInvokePayloadError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
function getNotificationDeliveryId(record: unknown): string | undefined {
|
|
if (!record || typeof record !== "object" || typeof (record as { id?: unknown }).id !== "string") {
|
|
return undefined;
|
|
}
|
|
const id = (record as { id: string }).id.trim();
|
|
return /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/u.test(id) ? id : undefined;
|
|
}
|
|
|
|
function getNotificationDeliveryStatus(record: unknown): string | undefined {
|
|
return record && typeof record === "object" && typeof (record as { status?: unknown }).status === "string"
|
|
? (record as { status: string }).status
|
|
: undefined;
|
|
}
|
|
|
|
async function createSkippedNotificationDelivery(
|
|
createNotificationDelivery: NonNullable<HandlerDependencies["createNotificationDelivery"]>,
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
payloadMetadata: Record<string, string>,
|
|
reason: string,
|
|
sourceKey?: string
|
|
) {
|
|
return createNotificationDelivery({
|
|
tenantId: payload.tenantId ?? "",
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey,
|
|
status: "SKIPPED",
|
|
attemptsMade: 0,
|
|
lastError: reason,
|
|
payloadMetadata: { ...payloadMetadata, reason },
|
|
});
|
|
}
|
|
|
|
function isNotificationPreferenceDisabled(preferences: unknown[]): boolean {
|
|
return preferences.some((preference) => preference && typeof preference === "object" && (preference as { enabled?: unknown }).enabled === false);
|
|
}
|
|
|
|
function getNotificationErrorMessage(error: unknown): string {
|
|
return redactNotificationDiagnosticText(error instanceof Error ? error.message : "Notification email delivery failed.");
|
|
}
|
|
|
|
function getPreferenceChannel(channel: string): "email" | "in-app" {
|
|
if (channel === "email" || channel === "in-app") {
|
|
return channel;
|
|
}
|
|
|
|
throw new NotificationHandlerError(`Unsupported notification channel: ${channel}`);
|
|
}
|
|
|
|
async function getNotificationPreferencesForPayload(
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
getUserNotificationPreferences: NonNullable<HandlerDependencies["getUserNotificationPreferences"]>
|
|
): Promise<unknown[]> {
|
|
if (!payload.tenantId) {
|
|
return [];
|
|
}
|
|
|
|
// Critical categories are delivered regardless of preferences — the same
|
|
// rule foundation's isNotificationEnabled() applies to the in-process path.
|
|
// Without this, a stale "critical" preference row could silence a locked
|
|
// category (e.g. a GDPR export or deletion notice) on the queue path only.
|
|
if (isCriticalNotificationCategory(payload.category)) {
|
|
return [];
|
|
}
|
|
|
|
const channel = getPreferenceChannel(payload.channel);
|
|
const category = payload.category ?? "default";
|
|
const preferences = await getUserNotificationPreferences({
|
|
tenantId: payload.tenantId,
|
|
userId: payload.userId,
|
|
channel,
|
|
category,
|
|
});
|
|
if (preferences.length > 0 || !payload.category || payload.category === "default") {
|
|
return preferences;
|
|
}
|
|
|
|
return getUserNotificationPreferences({
|
|
tenantId: payload.tenantId,
|
|
userId: payload.userId,
|
|
channel,
|
|
category: "default",
|
|
});
|
|
}
|
|
|
|
export class ScanHandlerError extends Error {
|
|
readonly action = "scan.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "ScanHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
export class PreviewHandlerError extends Error {
|
|
readonly action = "preview.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "PreviewHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
export class ExportHandlerError extends Error {
|
|
readonly action = "export.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "ExportHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
class UnrecoverableExportHandlerError extends UnrecoverableError {
|
|
readonly action = "export.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "UnrecoverableExportHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
export class ImportHandlerError extends Error {
|
|
readonly action = "import.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "ImportHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
type HandlerExecutionContext<Name extends QueueName = QueueName> = {
|
|
attemptsMade?: number;
|
|
attemptsLimit?: number;
|
|
jobId?: string;
|
|
updatePayload?: (payload: QueuePayload<Name>) => Promise<void>;
|
|
};
|
|
type Handler<Name extends QueueName> = (
|
|
payload: QueuePayload<Name>,
|
|
dependencies: HandlerDependencies,
|
|
context?: HandlerExecutionContext<Name>
|
|
) => Promise<HandlerResult>;
|
|
type HandlerRegistry = {
|
|
[Name in QueueName]: Handler<Name>;
|
|
};
|
|
|
|
function getAttemptNumber(context?: { attemptsMade?: number }): number {
|
|
return Math.max(1, (context?.attemptsMade ?? 0) + 1);
|
|
}
|
|
|
|
function getAttemptsLimit(context?: { attemptsLimit?: number }): number {
|
|
return Math.max(1, context?.attemptsLimit ?? 1);
|
|
}
|
|
|
|
function isRetryableWebhookStatus(status: number): boolean {
|
|
return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500;
|
|
}
|
|
|
|
function getPayloadNotificationDeliveryId(payload: QueuePayload<typeof queueNames.notifications>): string | undefined {
|
|
return typeof payload.metadata?.deliveryId === "string" ? payload.metadata.deliveryId : undefined;
|
|
}
|
|
|
|
function getNotificationDeliverySourceKey(
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
context: HandlerExecutionContext<typeof queueNames.notifications> | undefined
|
|
): string | undefined {
|
|
const metadata = payload.metadata as Record<string, unknown> | undefined;
|
|
if (typeof metadata?.idempotencyKey === "string" && metadata.idempotencyKey.trim()) {
|
|
return metadata.idempotencyKey.trim();
|
|
}
|
|
if (context?.jobId?.trim()) {
|
|
return `job:${context.jobId.trim()}`;
|
|
}
|
|
if (typeof payload.metadata?.requestId === "string" && payload.metadata.requestId.trim()) {
|
|
return `request:${payload.metadata.requestId.trim()}`;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
async function persistPayloadNotificationDeliveryId(
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
context: HandlerExecutionContext<typeof queueNames.notifications> | undefined,
|
|
deliveryId: string | undefined
|
|
): Promise<void> {
|
|
if (!deliveryId || getPayloadNotificationDeliveryId(payload) || !context?.updatePayload) {
|
|
return;
|
|
}
|
|
|
|
await context.updatePayload({
|
|
...payload,
|
|
metadata: {
|
|
...payload.metadata,
|
|
deliveryId,
|
|
},
|
|
});
|
|
}
|
|
|
|
async function runPlatformNotificationHandler(
|
|
payload: PlatformNotificationJob,
|
|
deps: HandlerDependencies,
|
|
context: HandlerExecutionContext<typeof queueNames.notifications> | undefined
|
|
): Promise<HandlerResult> {
|
|
if (payload.tenantId !== null) {
|
|
throw new NotificationHandlerError("Platform notification jobs require tenantId=null.");
|
|
}
|
|
const userId = payload.userId?.trim() || null;
|
|
const adminUserId = payload.adminUserId?.trim() || null;
|
|
if (Boolean(userId) === Boolean(adminUserId)) {
|
|
throw new NotificationHandlerError(
|
|
"Platform notification jobs require exactly one recipient identity."
|
|
);
|
|
}
|
|
const recipientId = userId ?? adminUserId!;
|
|
if (!payload.sourceKey?.trim()) {
|
|
throw new NotificationHandlerError("Platform notification jobs require sourceKey.");
|
|
}
|
|
getPreferenceChannel(payload.channel);
|
|
|
|
const createUserDelivery =
|
|
deps.createPlatformNotificationDelivery ??
|
|
createPlatformNotificationDeliveryInDb;
|
|
const createAdminDelivery =
|
|
deps.createAdminPlatformNotificationDelivery ??
|
|
createAdminPlatformNotificationDeliveryInDb;
|
|
const markUserDelivery =
|
|
deps.markPlatformNotificationDeliveryStatus ??
|
|
markPlatformNotificationDeliveryStatusInDb;
|
|
const markAdminDelivery =
|
|
deps.markAdminPlatformNotificationDeliveryStatus ??
|
|
markAdminPlatformNotificationDeliveryStatusInDb;
|
|
const payloadDeliveryId = getPayloadNotificationDeliveryId(payload);
|
|
const delivery = adminUserId
|
|
? await createAdminDelivery({
|
|
adminUserId,
|
|
channel: payload.channel === "email" ? "EMAIL" : "IN_APP",
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey: payload.sourceKey,
|
|
payloadMetadata: payload.metadata,
|
|
})
|
|
: await createUserDelivery({
|
|
userId: userId!,
|
|
channel: payload.channel === "email" ? "EMAIL" : "IN_APP",
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey: payload.sourceKey,
|
|
payloadMetadata: payload.metadata,
|
|
});
|
|
const markDelivery = (
|
|
input: Omit<MarkPlatformNotificationDeliveryStatusInput, "userId">
|
|
) =>
|
|
adminUserId
|
|
? markAdminDelivery({ ...input, adminUserId })
|
|
: markUserDelivery({ ...input, userId: userId! });
|
|
const persistedDeliveryId = getNotificationDeliveryId(delivery);
|
|
|
|
if (payloadDeliveryId && persistedDeliveryId && payloadDeliveryId !== persistedDeliveryId) {
|
|
throw new NotificationHandlerError("Platform notification delivery intent does not match the queued delivery id.");
|
|
}
|
|
|
|
const deliveryId = persistedDeliveryId ?? payloadDeliveryId;
|
|
await persistPayloadNotificationDeliveryId(payload, context, deliveryId);
|
|
|
|
if (!deliveryId) {
|
|
throw new NotificationHandlerError("Platform notification delivery intent did not return an id.");
|
|
}
|
|
|
|
const existingStatus = getNotificationDeliveryStatus(delivery);
|
|
if (existingStatus === "SENT" || existingStatus === "READ") {
|
|
return {
|
|
handled: true,
|
|
action: "notification.platform.deduplicated",
|
|
details: { deliveryId, userId: recipientId, channel: payload.channel, messageKey: payload.messageKey },
|
|
};
|
|
}
|
|
|
|
const attemptNumber = getAttemptNumber(context);
|
|
|
|
const taskId = typeof payload.metadata?.taskId === "string" ? payload.metadata.taskId.trim() : "";
|
|
const leadId = typeof payload.metadata?.leadId === "string" ? payload.metadata.leadId.trim() : "";
|
|
const emailMessageId =
|
|
typeof payload.metadata?.emailMessageId === "string"
|
|
? payload.metadata.emailMessageId.trim()
|
|
: "";
|
|
if (!leadId && !taskId && !emailMessageId) {
|
|
throw new NotificationHandlerError("Platform notifications require a supported subject.");
|
|
}
|
|
if ((taskId || emailMessageId || adminUserId) && payload.channel !== "in-app") {
|
|
throw new NotificationHandlerError(
|
|
"Admin platform notifications only support the in-app channel."
|
|
);
|
|
}
|
|
|
|
let recipient: { eligible?: boolean; email: string | null } = {
|
|
eligible: true,
|
|
email: null,
|
|
};
|
|
if (leadId && userId) {
|
|
const resolvePlatformLeadNotificationRecipient =
|
|
deps.resolvePlatformLeadNotificationRecipient ??
|
|
resolvePlatformLeadNotificationRecipientInDb;
|
|
recipient = await resolvePlatformLeadNotificationRecipient({
|
|
userId,
|
|
leadId,
|
|
});
|
|
const recipientEligible =
|
|
recipient.eligible ?? Boolean(recipient.email);
|
|
if (
|
|
!recipientEligible ||
|
|
(payload.channel === "email" && !recipient.email)
|
|
) {
|
|
await markDelivery({
|
|
id: deliveryId,
|
|
status: "SKIPPED",
|
|
attemptsMade: attemptNumber,
|
|
lastError:
|
|
"Platform lead notification recipient is no longer eligible.",
|
|
});
|
|
return {
|
|
handled: true,
|
|
action:
|
|
payload.channel === "email"
|
|
? "notification.platform.email.skipped"
|
|
: "notification.platform.in-app.skipped",
|
|
details: {
|
|
deliveryId,
|
|
userId: recipientId,
|
|
channel: payload.channel,
|
|
messageKey: payload.messageKey,
|
|
reason: "recipient_ineligible",
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
if (payload.channel === "in-app") {
|
|
await markDelivery({
|
|
id: deliveryId,
|
|
status: "SENT",
|
|
attemptsMade: attemptNumber,
|
|
sentAt: new Date(),
|
|
});
|
|
return {
|
|
handled: true,
|
|
action: "notification.platform.in-app.sent",
|
|
details: { deliveryId, userId: recipientId, channel: payload.channel, messageKey: payload.messageKey },
|
|
};
|
|
}
|
|
|
|
const recipientEmail = recipient.email;
|
|
if (!recipientEmail) {
|
|
throw new NotificationHandlerError("Platform lead email recipient resolution returned no deliverable address.");
|
|
}
|
|
|
|
const isTerminalFailure = attemptNumber >= getAttemptsLimit(context);
|
|
let result: Awaited<ReturnType<MailProvider["send"]>>;
|
|
|
|
try {
|
|
const provider = deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined);
|
|
result = await provider.send({
|
|
to: [recipientEmail],
|
|
subject: payload.subject,
|
|
text: payload.text,
|
|
html: payload.html,
|
|
templateId: payload.templateId,
|
|
idempotencyKey: `notification-delivery:${deliveryId}`,
|
|
personalization: payload.templateData ? [{ email: recipientEmail, data: payload.templateData }] : undefined,
|
|
});
|
|
} catch (error) {
|
|
const diagnostic = getNotificationErrorMessage(error);
|
|
try {
|
|
await markDelivery({
|
|
id: deliveryId,
|
|
status: isTerminalFailure ? "FAILED" : "PENDING",
|
|
attemptsMade: attemptNumber,
|
|
lastError: diagnostic,
|
|
});
|
|
} catch {
|
|
// The redacted provider failure below remains BullMQ's retry reason.
|
|
}
|
|
throw new NotificationHandlerError(diagnostic);
|
|
}
|
|
|
|
try {
|
|
await markDelivery({
|
|
id: deliveryId,
|
|
status: "SENT",
|
|
attemptsMade: attemptNumber,
|
|
providerMessageId: result.messageId ?? null,
|
|
sentAt: new Date(),
|
|
});
|
|
} catch (error) {
|
|
throw new NotificationHandlerError(`Platform notification delivery status persistence failed: ${getNotificationErrorMessage(error)}`);
|
|
}
|
|
|
|
return {
|
|
handled: true,
|
|
action: "notification.platform.email.sent",
|
|
details: { provider: result.provider, messageId: result.messageId, userId: recipientId, messageKey: payload.messageKey, deliveryId },
|
|
};
|
|
}
|
|
|
|
export function buildHandlerRegistry(dependencies: HandlerDependencies = {}): HandlerRegistry {
|
|
const deps = dependencies;
|
|
|
|
return {
|
|
[queueNames.email]: async (payload, _dependencies, context) => {
|
|
if (payload.kind === "mail.transactional") {
|
|
const portalSetupIntent = isPortalSetupMailIntent(payload);
|
|
if (
|
|
portalSetupIntent &&
|
|
(!hasValidPortalSetupInvitationMailContract(payload) ||
|
|
(!isMaterializedPortalSetupInvitationMailPayload(payload) && !isRedactedPortalSetupInvitationMailPayload(payload)))
|
|
) {
|
|
if (isTerminalMailAttempt(context) && context?.updatePayload) {
|
|
await context.updatePayload(redactTerminalPortalSetupInvitationMailPayload(payload) as QueuePayload<typeof queueNames.email>);
|
|
}
|
|
throw redactPortalSetupInvitationError(new Error("Portal setup invitation mail contract is invalid."), payload);
|
|
}
|
|
if (portalSetupIntent && isRedactedPortalSetupInvitationMailPayload(payload)) {
|
|
if (isTerminalPortalSetupInvitationMailPayload(payload)) {
|
|
await recoverTerminalPortalSetupMail(payload, deps);
|
|
} else if (!isPortalSetupInvitationProviderAccepted(payload)) {
|
|
await terminalizeFailedPortalSetupMail(payload, deps, context, "Portal setup invitation delivery outcome is ambiguous.");
|
|
throw new UnrecoverableError("Portal setup invitation delivery outcome is ambiguous; capability rotation is required.");
|
|
}
|
|
return {
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { deduplicated: true },
|
|
};
|
|
}
|
|
if (isRedactedWorkspaceOwnerInvitationMailPayload(payload)) {
|
|
if (isTerminalWorkspaceOwnerInvitationMailPayload(payload)) {
|
|
await recoverTerminalOwnerMail(payload, deps);
|
|
}
|
|
return {
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { deduplicated: true },
|
|
};
|
|
}
|
|
const ownerIntent = isWorkspaceOwnerInvitationMailIntent(payload);
|
|
let ownerClaim: WorkspaceOwnerInvitationDeliveryClaimResult | undefined;
|
|
let portalSetupClaim: PortalSetupInvitationDeliveryClaimResult | undefined;
|
|
let result: Awaited<ReturnType<MailProvider["send"]>>;
|
|
try {
|
|
if (ownerIntent && !hasValidWorkspaceOwnerInvitationMailContract(payload)) {
|
|
throw new Error("Workspace-owner invitation mail contract is invalid.");
|
|
}
|
|
if (portalSetupIntent && !isMaterializedPortalSetupInvitationMailPayload(payload)) {
|
|
throw new Error("Portal setup invitation mail must be materialized only in the provider queue.");
|
|
}
|
|
const provider =
|
|
deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined);
|
|
const recipients = payload.to.map(addressToEmail);
|
|
const providerInput = {
|
|
from: payload.from,
|
|
to: payload.to,
|
|
cc: payload.cc,
|
|
bcc: payload.bcc,
|
|
replyTo: payload.replyTo,
|
|
subject: payload.subject,
|
|
text: payload.text,
|
|
html: payload.html,
|
|
templateId: payload.templateId,
|
|
attachments: payload.attachments,
|
|
tags: payload.tags,
|
|
precedenceBulk: payload.precedenceBulk,
|
|
sendAt: payload.sendAt,
|
|
inReplyTo: payload.inReplyTo,
|
|
references: payload.references,
|
|
settings: portalSetupIntent
|
|
? {
|
|
...payload.settings,
|
|
trackClicks: false,
|
|
trackOpens: false,
|
|
}
|
|
: payload.settings,
|
|
headers: payload.headers,
|
|
listUnsubscribe: payload.listUnsubscribe,
|
|
...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}),
|
|
personalization: buildMailPersonalization(recipients, payload.templateData),
|
|
};
|
|
|
|
if (ownerIntent) {
|
|
const claim =
|
|
deps.claimOwnerInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => claimWorkspaceOwnerInvitationDelivery(input));
|
|
ownerClaim = await claim({ tenantId: payload.tenantId ?? "", payload });
|
|
if (ownerClaim.state === "not_ready") {
|
|
throw new Error("Workspace-owner invitation delivery outbox is not ready.");
|
|
}
|
|
if (ownerClaim.state === "uncertain") {
|
|
throw new Error("Workspace-owner invitation delivery outcome is ambiguous; capability rotation is required.");
|
|
}
|
|
if (ownerClaim.state === "provider_accepted") {
|
|
if (!context?.updatePayload) {
|
|
throw new Error("Workspace-owner invitation mail job cannot persist its safe delivery marker.");
|
|
}
|
|
await context.updatePayload(ownerClaim.payload as QueuePayload<typeof queueNames.email>);
|
|
return {
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { deduplicated: true },
|
|
};
|
|
}
|
|
}
|
|
|
|
if (portalSetupIntent) {
|
|
const claim =
|
|
deps.claimPortalSetupInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => claimPortalSetupInvitationDelivery(input));
|
|
portalSetupClaim = await claim({ tenantId: payload.tenantId ?? "", payload });
|
|
if (portalSetupClaim.state === "not_ready") {
|
|
throw new Error("Portal setup invitation delivery outbox is not ready.");
|
|
}
|
|
if (portalSetupClaim.state === "uncertain") {
|
|
throw new Error("Portal setup invitation delivery outcome is ambiguous; capability rotation is required.");
|
|
}
|
|
if (portalSetupClaim.state === "inactive") {
|
|
await terminalizeFailedPortalSetupMail(payload, deps, context, "Portal setup invitation capability is no longer active.");
|
|
return {
|
|
handled: true,
|
|
action: "mail.skipped",
|
|
details: { reason: "portal_setup_capability_inactive" },
|
|
};
|
|
}
|
|
if (portalSetupClaim.state === "provider_accepted") {
|
|
if (!context?.updatePayload) {
|
|
throw new Error("Portal setup invitation mail job cannot persist its safe delivery marker.");
|
|
}
|
|
await context.updatePayload(portalSetupClaim.payload as QueuePayload<typeof queueNames.email>);
|
|
return {
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { deduplicated: true },
|
|
};
|
|
}
|
|
if (!context?.updatePayload) {
|
|
throw new Error("Portal setup invitation mail job cannot persist its pre-provider delivery marker.");
|
|
}
|
|
await context.updatePayload(portalSetupClaim.payload as QueuePayload<typeof queueNames.email>);
|
|
}
|
|
|
|
result = await provider.send(providerInput);
|
|
|
|
if (ownerIntent) {
|
|
const accept =
|
|
deps.acceptOwnerInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => acceptWorkspaceOwnerInvitationDelivery(input));
|
|
const acceptedPayload = await accept({ tenantId: payload.tenantId ?? "", payload });
|
|
ownerClaim = { state: "provider_accepted", payload: acceptedPayload };
|
|
if (!context?.updatePayload) {
|
|
throw new Error("Workspace-owner invitation mail job cannot persist its safe delivery marker.");
|
|
}
|
|
await context.updatePayload(acceptedPayload as QueuePayload<typeof queueNames.email>);
|
|
}
|
|
if (portalSetupIntent) {
|
|
const accept =
|
|
deps.acceptPortalSetupInvitationDelivery ?? ((input: { tenantId: string; payload: unknown }) => acceptPortalSetupInvitationDelivery(input));
|
|
const acceptedPayload = await accept({ tenantId: payload.tenantId ?? "", payload });
|
|
portalSetupClaim = { state: "provider_accepted", payload: acceptedPayload };
|
|
if (!context?.updatePayload) {
|
|
throw new Error("Portal setup invitation mail job cannot persist its safe delivery marker.");
|
|
}
|
|
await context.updatePayload(acceptedPayload as QueuePayload<typeof queueNames.email>);
|
|
}
|
|
} catch (error) {
|
|
if (portalSetupIntent) {
|
|
const safeError = redactPortalSetupInvitationError(error, payload);
|
|
if (portalSetupClaim?.state === "provider_accepted") throw safeError;
|
|
if (portalSetupClaim?.state === "claimed" || portalSetupClaim?.state === "uncertain") {
|
|
try {
|
|
await terminalizeFailedPortalSetupMail(
|
|
payload,
|
|
deps,
|
|
context,
|
|
"Portal setup invitation delivery outcome was ambiguous; capability rotation is required."
|
|
);
|
|
} catch {
|
|
throw new Error("Portal setup invitation ambiguous delivery cleanup failed; retry is required.");
|
|
}
|
|
throw new UnrecoverableError(safeError.message);
|
|
}
|
|
if (isTerminalMailAttempt(context)) {
|
|
await terminalizeFailedPortalSetupMail(payload, deps, context, safeError.message);
|
|
}
|
|
throw safeError;
|
|
}
|
|
if (ownerIntent) {
|
|
const safeError = redactWorkspaceOwnerInvitationError(error, payload);
|
|
if (ownerClaim?.state === "claimed" || ownerClaim?.state === "uncertain") {
|
|
const abandon =
|
|
deps.abandonOwnerInvitationDelivery ??
|
|
((input: { tenantId: string; payload: unknown; lastError: string }) => abandonWorkspaceOwnerInvitationDelivery(input));
|
|
try {
|
|
await abandon({
|
|
tenantId: payload.tenantId ?? "",
|
|
payload,
|
|
lastError: "Workspace-owner invitation delivery outcome was ambiguous; capability rotation is required.",
|
|
});
|
|
if (context?.updatePayload) {
|
|
await context.updatePayload(redactTerminalWorkspaceOwnerInvitationMailPayload(payload) as QueuePayload<typeof queueNames.email>);
|
|
}
|
|
} catch {
|
|
if (isTerminalMailAttempt(context)) {
|
|
await terminalizeFailedOwnerMail(payload, deps, context);
|
|
}
|
|
throw new Error("Workspace-owner invitation ambiguous delivery cleanup failed; retry is required.");
|
|
}
|
|
throw new UnrecoverableError(safeError.message);
|
|
}
|
|
if (isTerminalMailAttempt(context)) {
|
|
await terminalizeFailedOwnerMail(payload, deps, context);
|
|
}
|
|
throw safeError;
|
|
}
|
|
const portalStatusMetadata = parsePortalStatusMailDeliveryMetadata(payload.metadata?.portalStatusMail);
|
|
if (portalStatusMetadata && payload.tenantId && isTerminalMailAttempt(context)) {
|
|
const notifyFailure = deps.notifyPortalStatusMailFailure ?? notifyPortalStatusMailPermanentFailure;
|
|
try {
|
|
await notifyFailure({
|
|
tenantId: payload.tenantId,
|
|
projectId: portalStatusMetadata.projectId,
|
|
expectedStageId: portalStatusMetadata.expectedStageId,
|
|
stageChangeId: portalStatusMetadata.stageChangeId,
|
|
internalRecipientUserId: portalStatusMetadata.internalRecipientUserId,
|
|
});
|
|
} catch {
|
|
// The provider error remains the retry/DLQ reason. The notifier
|
|
// is idempotent and can be replayed by an operator if DB was down.
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
return {
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { provider: result.provider, messageId: result.messageId },
|
|
};
|
|
}
|
|
|
|
// Only an injected (test) fetch is passed through; in production we leave
|
|
// it undefined so the outbound guard uses its own undici fetch with the
|
|
// DNS-rebinding IP pin engaged (see fetchWebhookWithRedirects).
|
|
const fetchImpl = deps.fetch;
|
|
const updateLegacyEventWebhookAttempt = deps.updateLegacyEventWebhookAttempt ?? updateLegacyEventWebhookAttemptInDb;
|
|
const legacyAttemptId = typeof payload.metadata?.legacyAttemptId === "string" ? payload.metadata.legacyAttemptId : undefined;
|
|
const legacyAttemptTenantId = typeof payload.metadata?.legacyAttemptTenantId === "string" ? payload.metadata.legacyAttemptTenantId : null;
|
|
const attemptNumber = getAttemptNumber(context);
|
|
const isTerminalFailure = attemptNumber >= getAttemptsLimit(context);
|
|
const emitWebhookMetric = (status: "success" | "retry" | "dlq" | "rejected") => {
|
|
if (!deps.webhookDeliveryMetricsHook) return;
|
|
try {
|
|
deps.webhookDeliveryMetricsHook({ status });
|
|
} catch {
|
|
// never block delivery
|
|
}
|
|
};
|
|
|
|
try {
|
|
const response = await fetchWebhookWithRedirects(
|
|
fetchImpl,
|
|
payload.url,
|
|
{
|
|
method: payload.method ?? "POST",
|
|
headers: {
|
|
"content-type": "application/json",
|
|
...payload.headers,
|
|
},
|
|
body: payload.body === undefined ? undefined : JSON.stringify(payload.body),
|
|
},
|
|
deps.webhookAddressResolver
|
|
);
|
|
const responseBody = await readLimitedResponseText(response, WEBHOOK_RESPONSE_BODY_LIMIT_BYTES);
|
|
const responseDiagnostics = serializeWebhookResponseDiagnostics(response, responseBody);
|
|
|
|
if (!response.ok) {
|
|
const message = `Webhook delivery failed with ${response.status}.`;
|
|
const retryableFailure = isRetryableWebhookStatus(response.status);
|
|
const shouldFinishAttempt = !retryableFailure || isTerminalFailure;
|
|
if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
...(shouldFinishAttempt ? { finishedAt: new Date(), success: false } : {}),
|
|
status: response.status,
|
|
message,
|
|
body: responseDiagnostics,
|
|
}
|
|
);
|
|
}
|
|
if (!retryableFailure) {
|
|
emitWebhookMetric("dlq");
|
|
return {
|
|
handled: true,
|
|
action: "webhook.failed",
|
|
details: { status: response.status, eventName: payload.eventName, retryable: false },
|
|
};
|
|
}
|
|
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
|
|
throw new WebhookDeliveryError(message);
|
|
}
|
|
|
|
if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
finishedAt: new Date(),
|
|
success: true,
|
|
status: response.status,
|
|
message: response.statusText,
|
|
body: responseDiagnostics,
|
|
}
|
|
);
|
|
}
|
|
|
|
emitWebhookMetric("success");
|
|
return {
|
|
handled: true,
|
|
action: "webhook.delivered",
|
|
details: { status: response.status, eventName: payload.eventName },
|
|
};
|
|
} catch (error) {
|
|
if (error instanceof WebhookUrlValidationError) {
|
|
const message = redactSecretLikeText(error.message);
|
|
if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
finishedAt: new Date(),
|
|
success: false,
|
|
message,
|
|
}
|
|
);
|
|
}
|
|
emitWebhookMetric("rejected");
|
|
return {
|
|
handled: true,
|
|
action: "webhook.rejected",
|
|
details: { eventName: payload.eventName, reason: message },
|
|
};
|
|
}
|
|
|
|
if (error instanceof WebhookDeliveryError) {
|
|
// Already counted at throw-site (retry / dlq).
|
|
} else if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
...(isTerminalFailure ? { finishedAt: new Date(), success: false } : {}),
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
}
|
|
);
|
|
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
|
|
} else if (!legacyAttemptId) {
|
|
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
|
|
}
|
|
throw error;
|
|
}
|
|
},
|
|
[queueNames.notifications]: async (payload, _dependencies, context) => {
|
|
if (payload.kind === "notification.platform.send") {
|
|
return runPlatformNotificationHandler(payload, deps, context);
|
|
}
|
|
|
|
const getUserNotificationPreferences = deps.getUserNotificationPreferences ?? getUserNotificationPreferencesInDb;
|
|
const createNotificationDelivery = deps.createNotificationDelivery ?? createNotificationDeliveryInDb;
|
|
const markNotificationDeliveryStatus = deps.markNotificationDeliveryStatus ?? markNotificationDeliveryStatusInDb;
|
|
const payloadMetadata = {
|
|
...(typeof payload.metadata?.requestId === "string" ? { requestId: payload.metadata.requestId } : {}),
|
|
...(typeof payload.metadata?.source === "string" ? { source: payload.metadata.source } : {}),
|
|
...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}),
|
|
};
|
|
const sourceKey = getNotificationDeliverySourceKey(payload, context);
|
|
|
|
getPreferenceChannel(payload.channel);
|
|
if (!payload.tenantId) {
|
|
throw new NotificationHandlerError("Notification jobs require tenantId.");
|
|
}
|
|
|
|
if (payload.channel !== "email") {
|
|
const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences);
|
|
|
|
if (isNotificationPreferenceDisabled(preferences)) {
|
|
const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey);
|
|
return {
|
|
handled: true,
|
|
action: "notification.in-app.skipped",
|
|
details: {
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
messageKey: payload.messageKey,
|
|
deliveryId: getNotificationDeliveryId(delivery),
|
|
reason: "preference_disabled",
|
|
},
|
|
};
|
|
}
|
|
|
|
const delivery = await createNotificationDelivery({
|
|
tenantId: payload.tenantId ?? null,
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey,
|
|
status: "SENT",
|
|
attemptsMade: 1,
|
|
payloadMetadata,
|
|
sentAt: new Date(),
|
|
});
|
|
return {
|
|
handled: true,
|
|
action: "notification.accepted",
|
|
details: {
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
messageKey: payload.messageKey,
|
|
deliveryId: getNotificationDeliveryId(delivery),
|
|
},
|
|
};
|
|
}
|
|
|
|
if (!payload.email) {
|
|
throw new NotificationHandlerError("Email notifications require email recipient.");
|
|
}
|
|
|
|
const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences);
|
|
|
|
if (isNotificationPreferenceDisabled(preferences)) {
|
|
const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey);
|
|
return {
|
|
handled: true,
|
|
action: "notification.email.skipped",
|
|
details: {
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
messageKey: payload.messageKey,
|
|
deliveryId: getNotificationDeliveryId(delivery),
|
|
reason: "preference_disabled",
|
|
},
|
|
};
|
|
}
|
|
let deliveryId = getPayloadNotificationDeliveryId(payload);
|
|
if (!deliveryId) {
|
|
const delivery = await createNotificationDelivery({
|
|
tenantId: payload.tenantId,
|
|
userId: payload.userId,
|
|
channel: "email",
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey,
|
|
status: "PENDING",
|
|
attemptsMade: 0,
|
|
payloadMetadata,
|
|
});
|
|
deliveryId = getNotificationDeliveryId(delivery);
|
|
await persistPayloadNotificationDeliveryId(payload, context, deliveryId);
|
|
}
|
|
|
|
const attemptNumber = getAttemptNumber(context);
|
|
const isTerminalFailure = attemptNumber >= getAttemptsLimit(context);
|
|
|
|
let result: Awaited<ReturnType<MailProvider["send"]>>;
|
|
try {
|
|
const provider =
|
|
deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined);
|
|
result = await provider.send({
|
|
to: [payload.email],
|
|
subject: payload.subject,
|
|
text: payload.text,
|
|
html: payload.html,
|
|
templateId: payload.templateId,
|
|
idempotencyKey: `notification-delivery:${deliveryId}`,
|
|
personalization: payload.templateData ? [{ email: payload.email, data: payload.templateData }] : undefined,
|
|
});
|
|
} catch (error) {
|
|
if (deliveryId) {
|
|
try {
|
|
await markNotificationDeliveryStatus({
|
|
id: deliveryId,
|
|
tenantId: payload.tenantId,
|
|
status: isTerminalFailure ? "FAILED" : "PENDING",
|
|
attemptsMade: attemptNumber,
|
|
lastError: getNotificationErrorMessage(error),
|
|
});
|
|
} catch {
|
|
// Preserve the provider failure as the retry reason.
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
if (deliveryId) {
|
|
try {
|
|
await markNotificationDeliveryStatus({
|
|
id: deliveryId,
|
|
tenantId: payload.tenantId,
|
|
status: "SENT",
|
|
attemptsMade: attemptNumber,
|
|
providerMessageId: result.messageId ?? null,
|
|
sentAt: new Date(),
|
|
});
|
|
} catch (error) {
|
|
return {
|
|
handled: true,
|
|
action: "notification.email.sent.diagnostic_failed",
|
|
details: {
|
|
provider: result.provider,
|
|
messageId: result.messageId,
|
|
userId: payload.userId,
|
|
messageKey: payload.messageKey,
|
|
deliveryId,
|
|
diagnosticError: getNotificationErrorMessage(error),
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
return {
|
|
handled: true,
|
|
action: "notification.email.sent",
|
|
details: { provider: result.provider, messageId: result.messageId, userId: payload.userId, messageKey: payload.messageKey },
|
|
};
|
|
},
|
|
[queueNames.maintenance]: async (payload) => {
|
|
if (payload.task === "privacy-gdpr-export-retention") {
|
|
return runPrivacyGdprExportRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "portal-gdpr-export-retention") {
|
|
return runPortalGdprExportRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "expired-export-artifacts") {
|
|
return runExportArtifactRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "dsgvo-grace-period-scan") {
|
|
return runDsgvoGracePeriodScanHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "privacy-tenant-deletion-scan") {
|
|
return runPrivacyTenantDeletionScanHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "portal-lifecycle-scan") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runPortalLifecycleScanHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "outbox-done-retention") {
|
|
return runOutboxDoneRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "workflow-execution-stalled-reconcile") {
|
|
return runWorkflowExecutionStalledReconcileHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "task-archive-retention") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runTaskArchiveRetentionHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "project-auto-archive") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runProjectAutoArchiveHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "billing-dunning-sweep") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runBillingDunningSweepHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "lead-retention-sweep") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runLeadRetentionHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "task-reminder-sweep") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runTaskReminderSweepHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "automation-scheduler-sweep") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runAutomationSchedulerSweepHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "admin-sales-task-reminder-sweep") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminSalesTaskReminderSweepHandler(payload, deps)
|
|
);
|
|
}
|
|
if (payload.task === "admin-booking-reminder-sweep") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminBookingReminderSweepHandler(payload, deps)
|
|
);
|
|
}
|
|
if (payload.task === "admin-calendar-sync") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminCalendarSyncHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "admin-email-delivery") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminEmailDeliveryHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "admin-email-event-retention") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminEmailEventRetentionHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "admin-email-sync") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminEmailSyncHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "admin-email-domain-check") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminEmailDomainCheckHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "admin-call-recording-process") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminCallRecordingProcessHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "admin-call-recording-retention") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminCallRecordingRetentionHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "admin-outbound-call-process") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () =>
|
|
runAdminOutboundCallHandler(payload, deps)
|
|
);
|
|
}
|
|
|
|
if (payload.task === "integrations-token-refresh") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runIntegrationsTokenRefreshHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "integrations-health-sweep") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runIntegrationsHealthSweepHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "kwp-project-maintenance") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runKwpProjectMaintenanceHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "calendar-token-refresh") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarTokenRefreshHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "calendar-webhook-renewal") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarWebhookRenewalHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "calendar-webhook-drain") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarWebhookDrainHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "calendar-internal-write-sync") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runCalendarInternalWriteSyncHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "email-inbox-token-refresh") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxTokenRefreshHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "email-inbox-webhook-renewal") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxWebhookRenewalHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "email-inbox-webhook-drain") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxWebhookDrainHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "email-inbox-initial-sync") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxInitialSyncHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "email-inbox-stale-poll") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxStalePollHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "email-inbox-event-cleanup") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailInboxEventCleanupHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "hr-upload-reminder-dispatch") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runHrUploadReminderDispatchHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "email-domain-probe") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runEmailDomainProbeHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "telephony-webhook-maintenance") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runTelephonyWebhookMaintenanceHandler(payload, deps));
|
|
}
|
|
|
|
if (payload.task === "product-acquisition-reconcile") {
|
|
return runLeasedTenantMaintenanceHandler(payload, deps, () => runProductAcquisitionReconcileHandler(payload, deps));
|
|
}
|
|
|
|
throw new UnsupportedWorkerJobError("Unsupported maintenance task.", {
|
|
queueName: queueNames.maintenance,
|
|
task: payload.task,
|
|
olderThanHours: payload.olderThanHours,
|
|
olderThanMinutes: payload.olderThanMinutes,
|
|
});
|
|
},
|
|
[queueNames.browserAutomation]: async (payload) => {
|
|
throw new UnsupportedWorkerJobError("Browser automation worker job is not configured.", {
|
|
queueName: queueNames.browserAutomation,
|
|
kind: payload.kind,
|
|
sessionId: payload.sessionId,
|
|
});
|
|
},
|
|
[queueNames.fileScan]: async (payload) => {
|
|
if (payload.kind !== "scan.file") {
|
|
throw new ScanHandlerError("Unsupported file scan payload kind.", {
|
|
verdict: "error",
|
|
code: "invalid_payload",
|
|
message: "Unsupported file scan payload kind.",
|
|
});
|
|
}
|
|
|
|
if (!deps.scanService) {
|
|
throw new ScanHandlerError("File scan service is not configured.", {
|
|
verdict: "error",
|
|
bucket: payload.bucket,
|
|
key: payload.key,
|
|
uploadId: payload.uploadId,
|
|
code: "scan_service_missing",
|
|
message: "File scan service is not configured.",
|
|
});
|
|
}
|
|
|
|
let result: StorageScanResult;
|
|
try {
|
|
result = await deps.scanService.scanObject({
|
|
bucket: payload.bucket,
|
|
key: payload.key,
|
|
tenantId: payload.tenantId,
|
|
uploadId: payload.uploadId,
|
|
contentType: payload.contentType,
|
|
metadata: payload.metadata,
|
|
});
|
|
} catch (error) {
|
|
throw new ScanHandlerError("File scan failed.", buildRejectedScanDetails(payload, error));
|
|
}
|
|
|
|
const details = buildScanResultDetails(result, payload.uploadId);
|
|
|
|
if (result.verdict === "clean" || result.verdict === "infected") {
|
|
// AK-P4a §5 — persist the verdict so the fail-closed media proxy can
|
|
// serve clean files (and keep refusing infected ones). The write also
|
|
// binds fileId to the exact scanned storage key. Legacy scans without
|
|
// a File row carry no fileId and remain scan-only.
|
|
if (payload.fileId) {
|
|
if (!payload.tenantId || !deps.fileScanStatusWriter) {
|
|
throw new ScanHandlerError("File scan verdict persistence is not configured.", {
|
|
...details,
|
|
code: "scan_verdict_persistence_missing",
|
|
message: "File scan verdict persistence is not configured.",
|
|
});
|
|
}
|
|
const persisted = await deps.fileScanStatusWriter.recordVerdict({
|
|
fileId: payload.fileId,
|
|
storageKey: payload.key,
|
|
tenantId: payload.tenantId,
|
|
verdict: result.verdict,
|
|
});
|
|
if (result.verdict === "clean" && persisted) {
|
|
if (!deps.filePreviewEnqueuer) {
|
|
throw new ScanHandlerError("Clean file preview enqueue is not configured.", {
|
|
...details,
|
|
code: "preview_enqueue_missing",
|
|
message: "Clean file preview enqueue is not configured.",
|
|
});
|
|
}
|
|
await deps.filePreviewEnqueuer.enqueueAfterCleanScan({
|
|
bucket: payload.bucket,
|
|
contentType: payload.contentType ?? "application/octet-stream",
|
|
fileId: payload.fileId,
|
|
storageKey: payload.key,
|
|
tenantId: payload.tenantId,
|
|
});
|
|
}
|
|
}
|
|
return { handled: true, action: `scan.${result.verdict}`, details };
|
|
}
|
|
|
|
throw new ScanHandlerError("File scan failed.", details);
|
|
},
|
|
[queueNames.filePreview]: async (payload) => {
|
|
if (payload.kind !== "preview.file") {
|
|
throw new PreviewHandlerError("Unsupported file preview payload kind.", {
|
|
code: "invalid_payload",
|
|
message: "Unsupported file preview payload kind.",
|
|
});
|
|
}
|
|
|
|
if (!deps.filePreviewService) {
|
|
throw new PreviewHandlerError("File preview service is not configured.", {
|
|
fileId: payload.fileId,
|
|
tenantId: payload.tenantId,
|
|
code: "preview_service_missing",
|
|
message: "File preview service is not configured.",
|
|
});
|
|
}
|
|
|
|
let result: FilePreviewServiceResult;
|
|
try {
|
|
result = await deps.filePreviewService.generatePreview(payload);
|
|
} catch (error) {
|
|
throw new PreviewHandlerError("File preview generation failed.", {
|
|
fileId: payload.fileId,
|
|
tenantId: payload.tenantId,
|
|
code: "preview_failed",
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
});
|
|
}
|
|
|
|
return {
|
|
handled: true,
|
|
action: `preview.${result.previewStatus}`,
|
|
details: {
|
|
fileId: payload.fileId,
|
|
tenantId: payload.tenantId,
|
|
previewStatus: result.previewStatus,
|
|
previewStorageKey: result.previewStorageKey,
|
|
thumbnailStorageKey: result.thumbnailStorageKey,
|
|
...(result.reason ? { reason: redactSecretLikeText(result.reason) } : {}),
|
|
},
|
|
};
|
|
},
|
|
[queueNames.exports]: async (payload, _dependencies, context) => runExportHandler(payload, deps, context),
|
|
[queueNames.imports]: async (payload, _dependencies, context) => runImportHandler(payload, deps, context),
|
|
[queueNames.kwpImports]: async (payload) => runKwpImportHandler(payload, deps),
|
|
// Phase 06 #21 / F-015 — real plugin-invoke dispatch. The handler is
|
|
// dependency-injected via `deps.pluginInvokeDeps` (composed in the
|
|
// `plugin` worker group bootstrap). When the slot is missing the
|
|
// handler throws so BullMQ dead-letters the job instead of silently
|
|
// acknowledging it — silent ack was the F-015 Critical finding.
|
|
[queueNames.pluginInvoke]: async (payload, _dependencies, context) => {
|
|
void context;
|
|
assertPluginInvokePayload(payload, deps);
|
|
if (!deps.pluginInvokeDeps) {
|
|
const message =
|
|
"plugin-invoke job received but pluginInvokeDeps is not configured on this worker. " +
|
|
"Set WORKER_QUEUE_GROUP=plugin and ensure the plugin bootstrap injected pluginInvokeDeps.";
|
|
deps.logger?.error?.({ queue: queueNames.pluginInvoke, pluginId: payload.pluginId, hookName: payload.hookName }, message);
|
|
throw new Error(message);
|
|
}
|
|
const outcome = await handlePluginInvokeJob(
|
|
{
|
|
tenantId: payload.tenantId,
|
|
pluginId: payload.pluginId,
|
|
version: payload.version,
|
|
hookName: payload.hookName,
|
|
input: payload.input,
|
|
metadata: payload.metadata,
|
|
},
|
|
deps.pluginInvokeDeps
|
|
);
|
|
if (outcome.ok) {
|
|
return {
|
|
handled: true,
|
|
action: "plugin.invoke.completed",
|
|
details: {
|
|
pluginId: payload.pluginId,
|
|
hookName: payload.hookName,
|
|
fuelUsed: outcome.fuelUsed,
|
|
durationMs: outcome.durationMs,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
handled: true,
|
|
action: "plugin.invoke.failed",
|
|
details: {
|
|
pluginId: payload.pluginId,
|
|
hookName: payload.hookName,
|
|
reason: outcome.reason,
|
|
message: outcome.message,
|
|
durationMs: outcome.durationMs,
|
|
},
|
|
};
|
|
},
|
|
};
|
|
}
|
|
|
|
async function terminalizeFailedOwnerMail(
|
|
payload: QueuePayload<typeof queueNames.email>,
|
|
dependencies: HandlerDependencies,
|
|
context: HandlerExecutionContext<typeof queueNames.email> | undefined
|
|
): Promise<void> {
|
|
const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(payload);
|
|
const redactionTask =
|
|
redacted === payload
|
|
? Promise.resolve()
|
|
: context?.updatePayload
|
|
? context.updatePayload(redacted as QueuePayload<typeof queueNames.email>)
|
|
: Promise.reject(new Error("Terminal workspace-owner invitation mail job cannot persist its rotation marker."));
|
|
let terminalizationTask: Promise<unknown> = Promise.resolve();
|
|
if (hasValidWorkspaceOwnerInvitationMailContract(payload)) {
|
|
const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : "";
|
|
const terminalize =
|
|
dependencies.terminalizeOwnerInvitation ?? ((input: { tenantId: string; payload: unknown }) => terminalizeWorkspaceOwnerInvitationDelivery(input));
|
|
terminalizationTask = terminalize({ tenantId, payload });
|
|
}
|
|
const results = await Promise.allSettled([redactionTask, terminalizationTask]);
|
|
if (results[1]?.status === "rejected" && isTerminalWorkspaceOwnerInvitationMailPayload(redacted)) {
|
|
const binding = readWorkspaceOwnerInvitationRecoveryBinding(redacted);
|
|
const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : "";
|
|
if (binding && binding.tenantId === tenantId) {
|
|
const schedule =
|
|
dependencies.scheduleOwnerInvitationRecovery ??
|
|
((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) =>
|
|
scheduleWorkspaceOwnerInvitationTerminalRecovery(input));
|
|
try {
|
|
await schedule({
|
|
tenantId,
|
|
outboxEventId: binding.outboxEventId,
|
|
redactedPayload: redacted,
|
|
lastError: "Workspace-owner invitation database invalidation requires retry.",
|
|
});
|
|
} catch {
|
|
results.push({ status: "rejected", reason: new Error("Workspace-owner invitation recovery scheduling failed.") });
|
|
}
|
|
} else {
|
|
results.push({ status: "rejected", reason: new Error("Workspace-owner invitation recovery outbox id is missing.") });
|
|
}
|
|
}
|
|
const failures = results.filter((result): result is PromiseRejectedResult => result.status === "rejected");
|
|
if (failures.length > 0) {
|
|
throw new Error(`Terminal workspace-owner invitation mail cleanup failed (${failures.length}/${results.length} operations).`);
|
|
}
|
|
}
|
|
|
|
async function recoverTerminalOwnerMail(payload: QueuePayload<typeof queueNames.email>, dependencies: HandlerDependencies): Promise<void> {
|
|
const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : "";
|
|
const terminalize =
|
|
dependencies.terminalizeOwnerInvitation ?? ((input: { tenantId: string; payload: unknown }) => terminalizeWorkspaceOwnerInvitationDelivery(input));
|
|
try {
|
|
await terminalize({ tenantId, payload });
|
|
return;
|
|
} catch {
|
|
const binding = readWorkspaceOwnerInvitationRecoveryBinding(payload);
|
|
if (!binding || binding.tenantId !== tenantId) {
|
|
throw new Error("Terminal workspace-owner invitation mail recovery binding is missing.");
|
|
}
|
|
const schedule =
|
|
dependencies.scheduleOwnerInvitationRecovery ??
|
|
((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) =>
|
|
scheduleWorkspaceOwnerInvitationTerminalRecovery(input));
|
|
try {
|
|
await schedule({
|
|
tenantId,
|
|
outboxEventId: binding.outboxEventId,
|
|
redactedPayload: payload,
|
|
lastError: "Workspace-owner invitation database invalidation requires retry.",
|
|
});
|
|
return;
|
|
} catch {
|
|
throw new Error("Terminal workspace-owner invitation mail recovery failed.");
|
|
}
|
|
}
|
|
}
|
|
|
|
async function terminalizeFailedPortalSetupMail(
|
|
payload: QueuePayload<typeof queueNames.email>,
|
|
dependencies: HandlerDependencies,
|
|
context: HandlerExecutionContext<typeof queueNames.email> | undefined,
|
|
lastError: string
|
|
): Promise<void> {
|
|
const redacted = redactTerminalPortalSetupInvitationMailPayload(payload);
|
|
const redactionTask =
|
|
redacted === payload
|
|
? Promise.resolve()
|
|
: context?.updatePayload
|
|
? context.updatePayload(redacted as QueuePayload<typeof queueNames.email>)
|
|
: Promise.reject(new Error("Terminal portal setup invitation mail job cannot persist its rotation marker."));
|
|
let terminalizationTask: Promise<unknown> = Promise.resolve();
|
|
if (hasValidPortalSetupInvitationMailContract(payload)) {
|
|
const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : "";
|
|
const terminalize =
|
|
dependencies.terminalizePortalSetupInvitation ??
|
|
((input: { tenantId: string; payload: unknown; lastError?: string }) => terminalizePortalSetupInvitationDelivery(input));
|
|
terminalizationTask = terminalize({ tenantId, payload, lastError });
|
|
}
|
|
const results = await Promise.allSettled([redactionTask, terminalizationTask]);
|
|
if (results[1]?.status === "rejected" && isTerminalPortalSetupInvitationMailPayload(redacted)) {
|
|
const binding = readPortalSetupInvitationRecoveryBinding(redacted);
|
|
const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : "";
|
|
if (binding && binding.tenantId === tenantId) {
|
|
const schedule =
|
|
dependencies.schedulePortalSetupInvitationRecovery ??
|
|
((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) =>
|
|
schedulePortalSetupInvitationTerminalRecovery(input));
|
|
try {
|
|
await schedule({
|
|
tenantId,
|
|
outboxEventId: binding.outboxEventId,
|
|
redactedPayload: redacted,
|
|
lastError: "Portal setup invitation database invalidation requires retry.",
|
|
});
|
|
} catch {
|
|
results.push({ status: "rejected", reason: new Error("Portal setup invitation recovery scheduling failed.") });
|
|
}
|
|
} else {
|
|
results.push({ status: "rejected", reason: new Error("Portal setup invitation recovery outbox id is missing.") });
|
|
}
|
|
}
|
|
const failures = results.filter((result): result is PromiseRejectedResult => result.status === "rejected");
|
|
if (failures.length > 0) {
|
|
throw new Error(`Terminal portal setup invitation mail cleanup failed (${failures.length}/${results.length} operations).`);
|
|
}
|
|
}
|
|
|
|
async function recoverTerminalPortalSetupMail(payload: QueuePayload<typeof queueNames.email>, dependencies: HandlerDependencies): Promise<void> {
|
|
const tenantId = typeof payload.tenantId === "string" ? payload.tenantId : "";
|
|
const terminalize =
|
|
dependencies.terminalizePortalSetupInvitation ??
|
|
((input: { tenantId: string; payload: unknown; lastError?: string }) => terminalizePortalSetupInvitationDelivery(input));
|
|
try {
|
|
await terminalize({
|
|
tenantId,
|
|
payload,
|
|
lastError: "Portal setup invitation recovery requires capability rotation.",
|
|
});
|
|
return;
|
|
} catch {
|
|
const binding = readPortalSetupInvitationRecoveryBinding(payload);
|
|
if (!binding || binding.tenantId !== tenantId) {
|
|
throw new Error("Terminal portal setup invitation mail recovery binding is missing.");
|
|
}
|
|
const schedule =
|
|
dependencies.schedulePortalSetupInvitationRecovery ??
|
|
((input: { tenantId: string; outboxEventId: string; redactedPayload: unknown; lastError: string }) =>
|
|
schedulePortalSetupInvitationTerminalRecovery(input));
|
|
try {
|
|
await schedule({
|
|
tenantId,
|
|
outboxEventId: binding.outboxEventId,
|
|
redactedPayload: payload,
|
|
lastError: "Portal setup invitation database invalidation requires retry.",
|
|
});
|
|
return;
|
|
} catch {
|
|
throw new Error("Terminal portal setup invitation mail recovery failed.");
|
|
}
|
|
}
|
|
}
|
|
|
|
function isTerminalMailAttempt(context: HandlerExecutionContext<typeof queueNames.email> | undefined): boolean {
|
|
return typeof context?.attemptsLimit === "number" && context.attemptsLimit > 0 && Math.max(0, context.attemptsMade ?? 0) + 1 >= context.attemptsLimit;
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function isNonEmptyString(value: unknown): value is string {
|
|
return typeof value === "string" && value.trim().length > 0;
|
|
}
|
|
|
|
function assertPluginInvokePayload(payload: unknown, deps: HandlerDependencies): asserts payload is PluginInvokeJob {
|
|
const record = isRecord(payload) ? payload : {};
|
|
const invalidFields: string[] = [];
|
|
if (record.kind !== "plugin.invoke") {
|
|
invalidFields.push("kind");
|
|
}
|
|
for (const field of ["tenantId", "pluginId", "version", "hookName"] as const) {
|
|
if (!isNonEmptyString(record[field])) {
|
|
invalidFields.push(field);
|
|
}
|
|
}
|
|
|
|
if (invalidFields.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const details = {
|
|
queueName: queueNames.pluginInvoke,
|
|
code: "invalid_payload",
|
|
invalidFields,
|
|
};
|
|
deps.logger?.error?.(
|
|
{
|
|
...details,
|
|
payload: summarizePluginInvokePayloadForLog(record),
|
|
},
|
|
"Invalid plugin-invoke queue payload."
|
|
);
|
|
throw new PluginInvokePayloadError("Invalid plugin-invoke queue payload.", details);
|
|
}
|
|
|
|
function summarizePluginInvokePayloadForLog(record: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
kind: typeof record.kind === "string" ? record.kind : typeof record.kind,
|
|
tenantId: typeof record.tenantId === "string" ? record.tenantId : typeof record.tenantId,
|
|
pluginId: typeof record.pluginId === "string" ? record.pluginId : typeof record.pluginId,
|
|
version: typeof record.version === "string" ? record.version : typeof record.version,
|
|
hookName: typeof record.hookName === "string" ? record.hookName : typeof record.hookName,
|
|
};
|
|
}
|
|
|
|
function addressToEmail(address: string | { email: string }) {
|
|
return typeof address === "string" ? address : address.email;
|
|
}
|
|
|
|
function buildMailPersonalization(to: readonly string[], templateData: Record<string, unknown> | undefined) {
|
|
if (!templateData) {
|
|
return undefined;
|
|
}
|
|
|
|
const hasPerRecipientData = to.some((email) => isRecord(templateData[email]));
|
|
return to.map((email) => ({
|
|
email,
|
|
data: hasPerRecipientData && isRecord(templateData[email]) ? templateData[email] : templateData,
|
|
}));
|
|
}
|
|
|
|
type PrivacyGdprExportRetentionState = {
|
|
processed: number;
|
|
deleted: number;
|
|
failed: number;
|
|
jobs: Record<string, unknown>[];
|
|
};
|
|
|
|
async function runPortalGdprExportRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const service = deps.portalGdprExportRetentionService;
|
|
let result: PortalGdprExportRetentionResult;
|
|
if (service) {
|
|
result = await service.runSweep({ now, ...(payload.take !== undefined ? { take: payload.take } : {}) });
|
|
} else {
|
|
const configuredBucket = deps.env?.STORAGE_BUCKET?.trim();
|
|
if (!configuredBucket || !deps.storageDeleteService) {
|
|
throw new Error("Portal GDPR export retention storage deletion is not fully configured.");
|
|
}
|
|
const prisma = (await getDefaultDbClient()) as unknown as PortalGdprExportRetentionPrisma;
|
|
result = await runPortalGdprExportRetentionSweep(
|
|
{
|
|
prisma,
|
|
configuredBucket,
|
|
deleteObject: (input) => deps.storageDeleteService!.deleteObject(input),
|
|
},
|
|
{ now, ...(payload.take !== undefined ? { take: payload.take } : {}) }
|
|
);
|
|
}
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
return {
|
|
handled: true,
|
|
action: "portal.gdpr-export-retention.completed",
|
|
details: { task: payload.task, requestId, ...result },
|
|
};
|
|
}
|
|
|
|
async function runPrivacyGdprExportRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const activeTenantReader = deps.activeTenantReader ?? (await getDefaultDbClient());
|
|
const sweep = await runActiveTenantSweep(activeTenantReader, {
|
|
initialState: { processed: 0, deleted: 0, failed: 0, jobs: [] } as PrivacyGdprExportRetentionState,
|
|
// Legal retention cleanup must continue for disabled/frozen tenants.
|
|
includeInactive: true,
|
|
// Each tenant already performs up to five bounded object deletions.
|
|
concurrency: 1,
|
|
worker: (tenantId) => runWithRlsContext({ tenantId }, () => runPrivacyGdprExportRetentionForTenant({ tenantId, now, take: payload.take, deps })),
|
|
merge: (state, result) => ({
|
|
processed: state.processed + result.processed,
|
|
deleted: state.deleted + result.deleted,
|
|
failed: state.failed + result.failed,
|
|
jobs: [...state.jobs, ...result.jobs],
|
|
}),
|
|
});
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
return {
|
|
handled: true,
|
|
action: "privacy.gdpr-export-retention.completed",
|
|
details: { task: payload.task, requestId, ...sweep.state },
|
|
};
|
|
}
|
|
|
|
async function runPrivacyGdprExportRetentionForTenant(input: {
|
|
tenantId: string;
|
|
now: Date;
|
|
take: number | undefined;
|
|
deps: HandlerDependencies;
|
|
}): Promise<PrivacyGdprExportRetentionState> {
|
|
const listExpired = input.deps.listExpiredGdprExportArtifacts ?? listExpiredGdprExportArtifactsInDb;
|
|
const markExpired = input.deps.markGdprExportArtifactExpired ?? markGdprExportArtifactExpiredInDb;
|
|
const markDeleted = input.deps.markGdprExportArtifactDeleted ?? markGdprExportArtifactDeletedInDb;
|
|
const artifacts = await listExpired({ tenantId: input.tenantId, now: input.now, ...(input.take !== undefined ? { take: input.take } : {}) });
|
|
const mismatched = artifacts.find((artifact) => artifact.tenantId !== input.tenantId);
|
|
if (mismatched) throw new Error(`GDPR export retention tenant scope mismatch for job ${mismatched.id}.`);
|
|
const configuredBucket = input.deps.env?.STORAGE_BUCKET?.trim();
|
|
if (artifacts.length > 0 && (!input.deps.storageDeleteService || !configuredBucket)) {
|
|
throw new Error("GDPR export artifact storage deletion is not fully configured.");
|
|
}
|
|
|
|
const jobs: Record<string, unknown>[] = new Array(artifacts.length);
|
|
await mapWithConcurrency(artifacts, EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY, async (artifact, index) => {
|
|
try {
|
|
if (!artifact.resultBucket || artifact.resultBucket !== configuredBucket) {
|
|
throw new Error("GDPR export artifact bucket does not match the configured private bucket.");
|
|
}
|
|
if (!artifact.resultKey) throw new Error("GDPR export retention candidate has no persisted attempt key.");
|
|
const artifactKey = assertGdprExportArtifactKey({
|
|
tenantId: input.tenantId,
|
|
exportJobId: artifact.id,
|
|
key: artifact.resultKey,
|
|
scope: "privacy",
|
|
});
|
|
// Claim the exact attempt before deletion. A concurrent retry either
|
|
// wins first (claim fails, no delete) or observes `expired` and cannot
|
|
// overwrite this attempt while its idempotent DeleteObject runs.
|
|
await markExpired({
|
|
...artifact,
|
|
resultBucket: artifact.resultBucket,
|
|
resultKey: artifact.resultKey,
|
|
now: input.now,
|
|
});
|
|
await deleteRetentionObject(input.deps.storageDeleteService!, { bucket: configuredBucket!, key: artifactKey });
|
|
await markDeleted({ id: artifact.id, tenantId: input.tenantId, resultBucket: artifact.resultBucket, resultKey: artifact.resultKey });
|
|
jobs[index] = { gdprExportJobId: artifact.id, status: "deleted" };
|
|
} catch (error) {
|
|
jobs[index] = {
|
|
gdprExportJobId: artifact.id,
|
|
status: "failed",
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
};
|
|
}
|
|
});
|
|
|
|
const deleted = jobs.filter((job) => job?.status === "deleted").length;
|
|
const failed = jobs.filter((job) => job?.status === "failed").length;
|
|
if (failed > 0) {
|
|
throw new AggregateError(
|
|
jobs.filter((job) => job?.status === "failed").map((job) => new Error(`${String(job.gdprExportJobId)}: ${String(job.message)}`)),
|
|
`GDPR export artifact retention failed for ${failed} object(s).`
|
|
);
|
|
}
|
|
|
|
return {
|
|
processed: artifacts.length,
|
|
deleted,
|
|
failed,
|
|
jobs,
|
|
};
|
|
}
|
|
|
|
async function runExportArtifactRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const retentionCutoff = getRetentionCutoff(now, payload.olderThanHours);
|
|
const take = payload.take ?? undefined;
|
|
const activeTenantReader = deps.activeTenantReader ?? (await getDefaultDbClient());
|
|
const sweep = await runActiveTenantSweep(activeTenantReader, {
|
|
initialState: { expired: 0, processed: 0, deleted: 0, failed: 0, jobs: [] } as ExportArtifactRetentionState,
|
|
// Each tenant already deletes up to five storage objects concurrently.
|
|
// Serial tenant scopes preserve that process-wide ceiling.
|
|
concurrency: 1,
|
|
worker: (tenantId) => runWithRlsContext({ tenantId }, () => runExportArtifactRetentionForTenant({ tenantId, now, retentionCutoff, take, deps })),
|
|
merge: (state, result) => ({
|
|
expired: state.expired + result.expired,
|
|
processed: state.processed + result.processed,
|
|
deleted: state.deleted + result.deleted,
|
|
failed: state.failed + result.failed,
|
|
jobs: [...state.jobs, ...result.jobs],
|
|
}),
|
|
});
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
|
|
return {
|
|
handled: true,
|
|
action: "export-artifact-retention.completed",
|
|
details: {
|
|
task: payload.task,
|
|
requestId,
|
|
...sweep.state,
|
|
},
|
|
};
|
|
}
|
|
|
|
type ExportArtifactRetentionState = {
|
|
expired: number;
|
|
processed: number;
|
|
deleted: number;
|
|
failed: number;
|
|
jobs: Record<string, unknown>[];
|
|
};
|
|
|
|
async function runExportArtifactRetentionForTenant(input: {
|
|
tenantId: string;
|
|
now: Date;
|
|
retentionCutoff: Date;
|
|
take: number | undefined;
|
|
deps: HandlerDependencies;
|
|
}): Promise<ExportArtifactRetentionState> {
|
|
const { tenantId, now, retentionCutoff, take, deps } = input;
|
|
const listExpiredCompleted = deps.listExpiredCompletedExportJobs ?? listExpiredCompletedExportJobsInDb;
|
|
const markExpired = deps.markExportJobExpired ?? markExportJobExpiredInDb;
|
|
const listPending = deps.listSoftDeletedExportJobsForHardDelete ?? listSoftDeletedExportJobsForHardDeleteInDb;
|
|
const markDeleted = deps.markExportJobHardDeleteSucceeded ?? markExportJobHardDeleteSucceededInDb;
|
|
const markFailed = deps.markExportJobHardDeleteFailed ?? markExportJobHardDeleteFailedInDb;
|
|
const expiredCompletedJobs = await listExpiredCompleted({ now, take });
|
|
assertExportJobsBelongToTenant(expiredCompletedJobs, tenantId);
|
|
|
|
await Promise.all(expiredCompletedJobs.map((job) => markExpired({ id: job.id, tenantId: job.tenantId, deletedAt: now })));
|
|
|
|
const jobs = await listPending({ now: retentionCutoff, take });
|
|
assertExportJobsBelongToTenant(jobs, tenantId);
|
|
const details: Record<string, unknown>[] = new Array(jobs.length);
|
|
|
|
if (jobs.length > 0 && !deps.storageDeleteService) {
|
|
throw new Error("Export artifact storage delete service is not configured.");
|
|
}
|
|
|
|
await mapWithConcurrency(jobs, EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY, async (job, index) => {
|
|
try {
|
|
await deleteRetentionObject(deps.storageDeleteService!, { bucket: job.artifactBucket, key: job.artifactKey });
|
|
await markDeleted({ id: job.id, tenantId: job.tenantId });
|
|
details[index] = { exportJobId: job.id, status: "deleted" };
|
|
} catch (error) {
|
|
const message = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
await markFailed({ id: job.id, tenantId: job.tenantId, errorMessage: message });
|
|
details[index] = { exportJobId: job.id, status: "failed", message };
|
|
}
|
|
});
|
|
|
|
const deleted = details.filter((job) => job?.status === "deleted").length;
|
|
const failed = details.filter((job) => job?.status === "failed").length;
|
|
|
|
return { expired: expiredCompletedJobs.length, processed: jobs.length, deleted, failed, jobs: details };
|
|
}
|
|
|
|
function assertExportJobsBelongToTenant(jobs: readonly ExportJobRetentionRecord[], tenantId: string): void {
|
|
const mismatched = jobs.find((job) => job.tenantId !== tenantId);
|
|
if (mismatched) {
|
|
throw new Error(`Export retention tenant scope mismatch for job ${mismatched.id}.`);
|
|
}
|
|
}
|
|
|
|
function getRetentionCutoff(now: Date, olderThanHours: number | undefined) {
|
|
if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) {
|
|
return now;
|
|
}
|
|
|
|
return new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
|
|
}
|
|
|
|
async function deleteRetentionObject(service: NonNullable<HandlerDependencies["storageDeleteService"]>, input: { bucket: string; key: string }): Promise<void> {
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(new Error("Export artifact deletion timed out.")), EXPORT_ARTIFACT_RETENTION_DELETE_TIMEOUT_MS);
|
|
try {
|
|
await service.deleteObject({ ...input, signal: controller.signal });
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
async function runOutboxDoneRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const olderThanHours = resolveOutboxDoneRetentionHours(payload.olderThanHours);
|
|
const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
|
|
const deleteDoneOutboxEvents = deps.deleteDoneOutboxEvents ?? (async (options) => deleteDoneOutboxEventsInDb(await getDefaultDbClient(), options));
|
|
const result = await deleteDoneOutboxEvents({ cutoff, take: payload.take });
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
|
|
return {
|
|
handled: true,
|
|
action: "outbox.done-retention.completed",
|
|
details: {
|
|
task: payload.task,
|
|
requestId,
|
|
olderThanHours,
|
|
cutoff: result.cutoff.toISOString(),
|
|
take: result.take,
|
|
deleted: result.deleted,
|
|
},
|
|
};
|
|
}
|
|
|
|
function resolveOutboxDoneRetentionHours(olderThanHours: number | undefined): number {
|
|
if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) {
|
|
return 720;
|
|
}
|
|
return Math.floor(olderThanHours);
|
|
}
|
|
|
|
async function runWorkflowExecutionStalledReconcileHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const runReconcile =
|
|
deps.reconcileWorkflowExecutionStalls ??
|
|
(async (options) => reconcileWorkflowExecutionStalls((await getDefaultDbClient()) as unknown as WorkflowExecutionStalledReconcilerPrisma, options));
|
|
const options = {
|
|
olderThanMinutes: payload.olderThanMinutes,
|
|
take: payload.take,
|
|
};
|
|
const activeTenantReader = deps.activeTenantReader ?? (await getDefaultDbClient());
|
|
const tenantSweep = await runActiveTenantSweep(activeTenantReader, {
|
|
initialState: emptyWorkflowExecutionStalledTotals(),
|
|
worker: (tenantId) => runWithRlsContext({ tenantId }, () => runReconcile(options)),
|
|
merge: mergeWorkflowExecutionStalledTotals,
|
|
});
|
|
|
|
// Tenantless platform executions are intentionally visible to the unbound
|
|
// worker policy and must be reconciled separately from tenant scopes.
|
|
const platformResult = await runReconcile(options);
|
|
throwOnActiveTenantSweepFailures(payload.task, tenantSweep.failures);
|
|
const totals = mergeWorkflowExecutionStalledTotals(tenantSweep.state, platformResult);
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
|
|
return {
|
|
handled: true,
|
|
action: "workflow.execution-stalled-reconcile.completed",
|
|
details: {
|
|
task: payload.task,
|
|
requestId,
|
|
olderThanMinutes: payload.olderThanMinutes,
|
|
cutoff: platformResult.cutoff.toISOString(),
|
|
take: platformResult.take,
|
|
...totals,
|
|
},
|
|
};
|
|
}
|
|
|
|
type WorkflowExecutionStalledTotals = Pick<
|
|
WorkflowExecutionStalledReconcileResult,
|
|
"inspected" | "failedOutboxEvents" | "repaired" | "skippedWithoutFailedOutbox" | "updateMisses"
|
|
>;
|
|
|
|
function emptyWorkflowExecutionStalledTotals(): WorkflowExecutionStalledTotals {
|
|
return { inspected: 0, failedOutboxEvents: 0, repaired: 0, skippedWithoutFailedOutbox: 0, updateMisses: 0 };
|
|
}
|
|
|
|
function mergeWorkflowExecutionStalledTotals(
|
|
totals: WorkflowExecutionStalledTotals,
|
|
result: WorkflowExecutionStalledReconcileResult
|
|
): WorkflowExecutionStalledTotals {
|
|
return {
|
|
inspected: totals.inspected + result.inspected,
|
|
failedOutboxEvents: totals.failedOutboxEvents + result.failedOutboxEvents,
|
|
repaired: totals.repaired + result.repaired,
|
|
skippedWithoutFailedOutbox: totals.skippedWithoutFailedOutbox + result.skippedWithoutFailedOutbox,
|
|
updateMisses: totals.updateMisses + result.updateMisses,
|
|
};
|
|
}
|
|
|
|
async function runTelephonyWebhookMaintenanceHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
if (!deps.runTelephonyWebhookMaintenance) {
|
|
throw new UnsupportedWorkerJobError("Telephony webhook maintenance is not configured.", {
|
|
queueName: queueNames.maintenance,
|
|
task: payload.task,
|
|
});
|
|
}
|
|
|
|
const result = await deps.runTelephonyWebhookMaintenance();
|
|
(deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "telephony.webhook-maintenance.completed");
|
|
return {
|
|
handled: true,
|
|
action: "telephony.webhook-maintenance.completed",
|
|
details: { task: payload.task, ...result },
|
|
};
|
|
}
|
|
|
|
async function runProductAcquisitionReconcileHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
if (!deps.runProductAcquisitionReconcile) {
|
|
throw new UnsupportedWorkerJobError("Product acquisition reconciliation is not configured.", {
|
|
queueName: queueNames.maintenance,
|
|
task: payload.task,
|
|
});
|
|
}
|
|
|
|
const result = await deps.runProductAcquisitionReconcile({ take: payload.take });
|
|
(deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "billing.product-acquisition-reconcile.completed");
|
|
return {
|
|
handled: true,
|
|
action: "billing.product-acquisition-reconcile.completed",
|
|
details: { task: payload.task, ...result },
|
|
};
|
|
}
|
|
|
|
async function runPortalLifecycleScanHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
if (!deps.portalLifecycleService) {
|
|
throw new UnsupportedWorkerJobError("Portal lifecycle service is not configured.", {
|
|
queueName: queueNames.maintenance,
|
|
task: payload.task,
|
|
});
|
|
}
|
|
const result = await deps.portalLifecycleService.runScan();
|
|
(deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "portal.lifecycle.scan-completed");
|
|
return {
|
|
handled: true,
|
|
action: "portal.lifecycle.scanned",
|
|
details: { task: payload.task, ...result },
|
|
};
|
|
}
|
|
|
|
async function runKwpProjectMaintenanceHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies,
|
|
): Promise<HandlerResult> {
|
|
if (!deps.runKwpProjectMaintenance) {
|
|
throw new UnsupportedWorkerJobError("KWP project maintenance service is not configured.", {
|
|
queueName: queueNames.maintenance,
|
|
task: payload.task,
|
|
});
|
|
}
|
|
const result = await deps.runKwpProjectMaintenance();
|
|
(deps.logger ?? noopHandlerLogger).info({ task: payload.task, ...result }, "kwp.project-maintenance.completed");
|
|
if (result.failures > 0) {
|
|
throw new Error(`KWP project maintenance failed for ${result.failures} tenant(s).`);
|
|
}
|
|
return {
|
|
handled: true,
|
|
action: "kwp.project-maintenance.completed",
|
|
details: { task: payload.task, ...result },
|
|
};
|
|
}
|
|
|
|
async function runLeasedTenantMaintenanceHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies,
|
|
worker: () => Promise<HandlerResult>
|
|
): Promise<HandlerResult> {
|
|
const lease = await withDistributedJobLease(deps.tenantMaintenanceLeaseStore ?? getSharedStore(), payload.task, worker);
|
|
if (lease.acquired) return lease.value;
|
|
|
|
(deps.logger ?? noopHandlerLogger).info({ task: payload.task, requestId: payload.metadata?.requestId ?? null }, "maintenance.tenant-sweep.overlap-skipped");
|
|
return {
|
|
handled: true,
|
|
action: "maintenance.tenant-sweep.overlap-skipped",
|
|
details: { task: payload.task, reason: "active-lease" },
|
|
};
|
|
}
|
|
|
|
async function getDefaultDbClient() {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
return getDbClient();
|
|
}
|
|
|
|
/**
|
|
* P4 Tasks (AK-P4b) §7 — 30-day auto-archive (DoD 5). Archives completed tasks
|
|
* whose `completedAt` is older than the retention window by stamping
|
|
* `archivedAt`. Idempotent: already-archived rows are excluded by the filter.
|
|
*
|
|
* RLS-ready (AK-P4b "Teil A"): a cross-tenant sweep cannot run under the
|
|
* restricted app role once Task RLS is active (an unscoped query returns zero
|
|
* rows). So we enumerate active tenants from the global Tenant registry (not
|
|
* RLS-scoped) and run the scoped archive per tenant inside withTenantRLS(), which
|
|
* sets `app.tenant_id` so the policy binds. Functional no-op until the policy is
|
|
* activated (prisma/rls/task-isolation.sql, P5 cutover — at which point
|
|
* hivegrid_app needs SELECT on "Tenant" or a privileged maintenance connection;
|
|
* see that file's Teil-B notes).
|
|
*/
|
|
async function runTaskArchiveRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const olderThanHours = typeof payload.olderThanHours === "number" && payload.olderThanHours > 0 ? Math.floor(payload.olderThanHours) : 720;
|
|
const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { archived: 0 },
|
|
worker: (tenantId) =>
|
|
withTenantRLS(db, tenantId, (tx) =>
|
|
tx.task.updateMany({
|
|
where: { tenantId, status: "done", completedAt: { lte: cutoff }, archivedAt: null },
|
|
data: { archivedAt: now },
|
|
})
|
|
),
|
|
merge: (state, result) => ({ archived: state.archived + result.count }),
|
|
});
|
|
const { archived } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, cutoff: cutoff.toISOString(), tenants: sweep.inspected, tenantFailures: sweep.failures.length, archived },
|
|
"tasks.archive-retention.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "tasks.archive-retention.completed",
|
|
details: { task: payload.task, olderThanHours, cutoff: cutoff.toISOString(), tenants: sweep.inspected, archived },
|
|
};
|
|
}
|
|
|
|
/**
|
|
* R-P9a DoD 4 — project auto-archive. Archives projects that sat in a terminal
|
|
* order pipeline stage (AuftragPipelineStageConfig.autoArchiveAfterDays) past
|
|
* the configured window, and archives the project chat channel in the same
|
|
* transaction (AK-P15b). Same per-tenant RLS sweep pattern as the other
|
|
* maintenance crons: enumerate tenants globally, run the scoped sweep inside
|
|
* withTenantRLS() so the FORCE-RLS policies bind (an unscoped query would
|
|
* silently return zero rows).
|
|
*/
|
|
async function runProjectAutoArchiveHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 100;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { archived: 0 },
|
|
worker: (tenantId) =>
|
|
withTenantRLS(db, tenantId, (tx) =>
|
|
autoArchiveProjectsForTenantWithTx(tx as unknown as Parameters<typeof autoArchiveProjectsForTenantWithTx>[0], {
|
|
tenantId,
|
|
now,
|
|
limit: take,
|
|
archiveChannel: (channelInput) =>
|
|
archiveProjectChannelIfEnabledWithTx(tx as unknown as Parameters<typeof archiveProjectChannelIfEnabledWithTx>[0], channelInput),
|
|
})
|
|
),
|
|
merge: (state, result) => ({ archived: state.archived + result.archived }),
|
|
});
|
|
const { archived } = sweep.state;
|
|
logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, archived }, "projects.auto-archive.completed");
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "projects.auto-archive.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, archived, take },
|
|
};
|
|
}
|
|
|
|
async function runLeadRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { archived: 0 },
|
|
worker: async (tenantId) => {
|
|
const dueIds = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.inboxLead.findMany({
|
|
where: { tenantId, archivedAt: null, autoDeleteAt: { lte: now } },
|
|
select: { id: true },
|
|
orderBy: { autoDeleteAt: "asc" },
|
|
take,
|
|
})
|
|
);
|
|
if (dueIds.length === 0) return { count: 0 };
|
|
return withTenantRLS(db, tenantId, (tx) =>
|
|
tx.inboxLead.updateMany({
|
|
where: { tenantId, id: { in: dueIds.map((row) => row.id) }, archivedAt: null },
|
|
data: { archivedAt: now },
|
|
})
|
|
);
|
|
},
|
|
merge: (state, result) => ({ archived: state.archived + result.count }),
|
|
});
|
|
const { archived } = sweep.state;
|
|
logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, archived }, "leads.retention.completed");
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "leads.retention.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, archived, take },
|
|
};
|
|
}
|
|
|
|
/**
|
|
* R-P5 §P2d — refresh OAuth tokens nearing expiry across all tenants. Enumerates
|
|
* active tenants from the global registry (not RLS-scoped) and runs the scoped
|
|
* refresh per tenant inside withTenantRLS() (so the policy binds at cutover).
|
|
* KWP is deliberately excluded here: its one-time refresh token rotates only
|
|
* immediately before an actual resource/health window.
|
|
*/
|
|
async function runIntegrationsTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const sweepStartedAt = new Date();
|
|
const genericWindowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
|
const db = await getDefaultDbClient();
|
|
const cleanupSweep = await runActiveTenantSweep(db, {
|
|
initialState: { restored: 0, purged: 0, failed: 0 },
|
|
includeInactive: true,
|
|
worker: (tenantId) =>
|
|
withTenantRLS(db, tenantId, (tx) =>
|
|
cleanupExpiredKwpPendingAttemptsForTenant(
|
|
tx as unknown as Parameters<typeof cleanupExpiredKwpPendingAttemptsForTenant>[0],
|
|
tenantId,
|
|
{ now: sweepStartedAt },
|
|
)
|
|
),
|
|
merge: (state, result) => ({
|
|
restored: state.restored + result.restored,
|
|
purged: state.purged + result.purged,
|
|
failed: state.failed + result.failed,
|
|
}),
|
|
});
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { refreshed: 0, failed: 0, retryScheduled: 0 },
|
|
worker: async (tenantId) => {
|
|
const legacyResult = await withTenantRLS(db, tenantId, (tx) =>
|
|
refreshExpiringTokensForTenant(
|
|
tx as unknown as Parameters<typeof refreshExpiringTokensForTenant>[0],
|
|
tenantId,
|
|
{ now: sweepStartedAt, windowMs: genericWindowMs },
|
|
)
|
|
);
|
|
return { ...legacyResult, retryScheduled: 0 };
|
|
},
|
|
merge: (state, result) => ({
|
|
refreshed: state.refreshed + result.refreshed,
|
|
failed: state.failed + result.failed,
|
|
retryScheduled: state.retryScheduled + result.retryScheduled,
|
|
}),
|
|
});
|
|
const { refreshed, failed, retryScheduled } = sweep.state;
|
|
const { restored: pendingRestored, purged: pendingPurged, failed: pendingCleanupFailed } = cleanupSweep.state;
|
|
logger.info(
|
|
{
|
|
task: payload.task,
|
|
tenants: sweep.inspected,
|
|
tenantFailures: sweep.failures.length + cleanupSweep.failures.length,
|
|
refreshed,
|
|
failed,
|
|
retryScheduled,
|
|
pendingRestored,
|
|
pendingPurged,
|
|
pendingCleanupFailed,
|
|
},
|
|
"integrations.token-refresh.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, [...cleanupSweep.failures, ...sweep.failures]);
|
|
return {
|
|
handled: true,
|
|
action: "integrations.token-refresh.completed",
|
|
details: {
|
|
task: payload.task,
|
|
tenants: sweep.inspected,
|
|
refreshed,
|
|
failed,
|
|
retryScheduled,
|
|
pendingRestored,
|
|
pendingPurged,
|
|
pendingCleanupFailed,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* R-P5 §P2d — probe every active connection's health across all tenants, persist
|
|
* the status, and notify owners on degraded/broken. Same per-tenant RLS sweep
|
|
* pattern; per-connection logic shared via worker-tasks.server.
|
|
*/
|
|
async function runIntegrationsHealthSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const sweepStartedAt = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { checked: 0, unhealthy: 0 },
|
|
worker: async (tenantId) => {
|
|
const legacyResult = await withTenantRLS(db, tenantId, (tx) =>
|
|
sweepConnectionHealthForTenant(tx as unknown as Parameters<typeof sweepConnectionHealthForTenant>[0], tenantId, { now: sweepStartedAt })
|
|
);
|
|
const connectionIds = await withTenantRLS(db, tenantId, (tx) =>
|
|
listKwpHealthConnectionIdsForTenant(
|
|
tx as unknown as Parameters<typeof listKwpHealthConnectionIdsForTenant>[0],
|
|
tenantId,
|
|
{ now: new Date() },
|
|
)
|
|
);
|
|
let result = legacyResult;
|
|
for (const connectionId of connectionIds) {
|
|
// Health is a real KWP work window. Refresh on demand once here; the
|
|
// same durable claim also serializes simultaneous import/web calls.
|
|
await runWithRlsContext({ tenantId }, () =>
|
|
ensureKwpAccessTokenForConnection({ connectionId, allowPaused: true }),
|
|
);
|
|
const probeNow = new Date();
|
|
const prepared = await withTenantRLS(db, tenantId, (tx) =>
|
|
prepareKwpHealthProbeForConnection(
|
|
tx as unknown as Parameters<typeof prepareKwpHealthProbeForConnection>[0],
|
|
tenantId,
|
|
connectionId,
|
|
{ now: probeNow, kwpHiveGridTenantId: deps.env?.KWP_HIVEGRID_TENANT_ID },
|
|
)
|
|
);
|
|
if (prepared.state === "skipped") continue;
|
|
const providerResult = await executeKwpHealthProbe(prepared.snapshot);
|
|
const item = await withTenantRLS(db, tenantId, (tx) =>
|
|
persistKwpHealthProbe(
|
|
tx as unknown as Parameters<typeof persistKwpHealthProbe>[0],
|
|
prepared.snapshot,
|
|
providerResult,
|
|
{ checkedAt: new Date() },
|
|
)
|
|
);
|
|
result = { checked: result.checked + item.checked, unhealthy: result.unhealthy + item.unhealthy };
|
|
}
|
|
return result;
|
|
},
|
|
merge: (state, result) => ({ checked: state.checked + result.checked, unhealthy: state.unhealthy + result.unhealthy }),
|
|
});
|
|
const { checked, unhealthy } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, checked, unhealthy },
|
|
"integrations.health-sweep.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "integrations.health-sweep.completed", details: { task: payload.task, tenants: sweep.inspected, checked, unhealthy } };
|
|
}
|
|
|
|
async function runCalendarTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { refreshed: 0, failed: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => refreshCalendarTokensForTenant(tx as never, tenantId, { now, windowMs })),
|
|
merge: (state, result) => ({ refreshed: state.refreshed + result.refreshed, failed: state.failed + result.failed }),
|
|
});
|
|
const { refreshed, failed } = sweep.state;
|
|
logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, refreshed, failed }, "calendar.token-refresh.completed");
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "calendar.token-refresh.completed", details: { task: payload.task, tenants: sweep.inspected, refreshed, failed } };
|
|
}
|
|
|
|
async function runCalendarWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
|
|
const webhookUrl = process.env.GOOGLE_CALENDAR_WEBHOOK_URL;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { checked: 0, renewed: 0, failed: 0, skipped: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => renewCalendarWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, webhookUrl })),
|
|
merge: (state, result) => ({
|
|
checked: state.checked + result.checked,
|
|
renewed: state.renewed + result.renewed,
|
|
failed: state.failed + result.failed,
|
|
skipped: state.skipped + result.skipped,
|
|
}),
|
|
});
|
|
const { checked, renewed, failed, skipped } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, checked, renewed, failed, skipped },
|
|
"calendar.webhook-renewal.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "calendar.webhook-renewal.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, checked, renewed, failed, skipped },
|
|
};
|
|
}
|
|
|
|
async function runCalendarWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { claimed: 0, completed: 0, failed: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => drainCalendarWebhookEventsForTenant(tx as never, tenantId, { now, take })),
|
|
merge: (state, result) => ({
|
|
claimed: state.claimed + result.claimed,
|
|
completed: state.completed + result.completed,
|
|
failed: state.failed + result.failed,
|
|
}),
|
|
});
|
|
const { claimed, completed, failed } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, claimed, completed, failed },
|
|
"calendar.webhook-drain.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "calendar.webhook-drain.completed", details: { task: payload.task, tenants: sweep.inspected, claimed, completed, failed } };
|
|
}
|
|
|
|
async function runCalendarInternalWriteSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { scanned: 0, created: 0, updated: 0, deleted: 0, conflicts: 0, failed: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => syncInternalCalendarEventsForTenant(tx as never, tenantId, { now, take })),
|
|
merge: (state, result) => ({
|
|
scanned: state.scanned + result.scanned,
|
|
created: state.created + result.created,
|
|
updated: state.updated + result.updated,
|
|
deleted: state.deleted + result.deleted,
|
|
conflicts: state.conflicts + result.conflicts,
|
|
failed: state.failed + result.failed,
|
|
}),
|
|
});
|
|
const { scanned, created, updated, deleted, conflicts, failed } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, scanned, created, updated, deleted, conflicts, failed },
|
|
"calendar.internal-write-sync.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "calendar.internal-write-sync.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, scanned, created, updated, deleted, conflicts, failed },
|
|
};
|
|
}
|
|
|
|
async function runEmailInboxTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { refreshed: 0, failed: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => refreshEmailTokensForTenant(tx as never, tenantId, { now, windowMs })),
|
|
merge: (state, result) => ({ refreshed: state.refreshed + result.refreshed, failed: state.failed + result.failed }),
|
|
});
|
|
const { refreshed, failed } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, refreshed, failed },
|
|
"email-inbox.token-refresh.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "email-inbox.token-refresh.completed", details: { task: payload.task, tenants: sweep.inspected, refreshed, failed } };
|
|
}
|
|
|
|
async function runEmailInboxWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
|
|
const graphWebhookUrl = process.env.MICROSOFT_MAIL_WEBHOOK_URL;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { checked: 0, renewed: 0, failed: 0, skipped: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => renewEmailWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, graphWebhookUrl })),
|
|
merge: (state, result) => ({
|
|
checked: state.checked + result.checked,
|
|
renewed: state.renewed + result.renewed,
|
|
failed: state.failed + result.failed,
|
|
skipped: state.skipped + result.skipped,
|
|
}),
|
|
});
|
|
const { checked, renewed, failed, skipped } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, checked, renewed, failed, skipped },
|
|
"email-inbox.webhook-renewal.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "email-inbox.webhook-renewal.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, checked, renewed, failed, skipped },
|
|
};
|
|
}
|
|
|
|
async function runEmailInboxWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { claimed: 0, completed: 0, failed: 0 },
|
|
worker: (tenantId) =>
|
|
withTenantRLS(db, tenantId, (tx) => drainEmailWebhookEventsForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook })),
|
|
merge: (state, result) => ({
|
|
claimed: state.claimed + result.claimed,
|
|
completed: state.completed + result.completed,
|
|
failed: state.failed + result.failed,
|
|
}),
|
|
});
|
|
const { claimed, completed, failed } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, claimed, completed, failed },
|
|
"email-inbox.webhook-drain.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "email-inbox.webhook-drain.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, claimed, completed, failed },
|
|
};
|
|
}
|
|
|
|
async function runEmailInboxInitialSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 10;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { folders: 0, fetched: 0, completed: 0, failed: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => runEmailInitialSyncForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook })),
|
|
merge: (state, result) => ({
|
|
folders: state.folders + result.folders,
|
|
fetched: state.fetched + result.fetched,
|
|
completed: state.completed + result.completed,
|
|
failed: state.failed + result.failed,
|
|
}),
|
|
});
|
|
const { folders, fetched, completed, failed } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, folders, fetched, completed, failed },
|
|
"email-inbox.initial-sync.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "email-inbox.initial-sync.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, folders, fetched, completed, failed },
|
|
};
|
|
}
|
|
|
|
async function runEmailInboxStalePollHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { polled: 0, failed: 0 },
|
|
worker: (tenantId) =>
|
|
withTenantRLS(db, tenantId, (tx) => pollStaleEmailMailboxesForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook })),
|
|
merge: (state, result) => ({ polled: state.polled + result.polled, failed: state.failed + result.failed }),
|
|
});
|
|
const { polled, failed } = sweep.state;
|
|
logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, polled, failed }, "email-inbox.stale-poll.completed");
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "email-inbox.stale-poll.completed", details: { task: payload.task, tenants: sweep.inspected, polled, failed } };
|
|
}
|
|
|
|
async function runEmailInboxEventCleanupHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { deleted: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => cleanupEmailWebhookEventsForTenant(tx as never, tenantId, { now })),
|
|
merge: (state, result) => ({ deleted: state.deleted + result.deleted }),
|
|
});
|
|
const { deleted } = sweep.state;
|
|
logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, deleted }, "email-inbox.event-cleanup.completed");
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "email-inbox.event-cleanup.completed", details: { task: payload.task, tenants: sweep.inspected, deleted } };
|
|
}
|
|
|
|
// VA-P23 · auto-match + activity logging seam of the sync engine (M09 Block E/H).
|
|
const emailAutoMatchHook = createEmailAutoMatchHook();
|
|
|
|
// R-P12c · newly mirrored inbound mail feeds the generic automation trigger
|
|
// bus. Published on the same tenant transaction as the message upsert so the
|
|
// event commits atomically with the mirror row; failures must not break sync,
|
|
// so the publish shares the hook's best-effort contract via its own catch.
|
|
const emailInboxMessageHook: typeof emailAutoMatchHook = async (input) => {
|
|
await emailAutoMatchHook(input);
|
|
if (!input.created || input.message.direction !== "inbound") {
|
|
return;
|
|
}
|
|
try {
|
|
await publishOutboxEvent(
|
|
input.client as unknown as OutboxPublisherTransaction,
|
|
buildWorkflowTriggerOutboxEvent({
|
|
tenantId: input.tenantId,
|
|
event: "email.inbound_received",
|
|
data: {
|
|
messageId: input.messageId,
|
|
mailboxId: input.mailboxId,
|
|
fromAddress: input.message.from.address,
|
|
subject: input.message.subject ?? null,
|
|
},
|
|
userId: null,
|
|
sourceId: `email-inbound:${input.mailboxId}:${input.messageId}`,
|
|
})
|
|
);
|
|
} catch {
|
|
// Best-effort like the auto-match hook itself: the mirror row is already
|
|
// persisted and mail sync must never break on a missing trigger event.
|
|
// A lost event only means this one inbound mail fires no automation.
|
|
}
|
|
};
|
|
|
|
/**
|
|
* AK-P20 DoD 2 — billing dunning sweep. Daily cron that drives the past_due
|
|
* dunning sequence: one notice at day 1 / 3 / 7 after `pastDueSince`, then a
|
|
* freeze once every mark is exhausted. Decisions come from the pure
|
|
* evaluateBillingDunning() (@hivegrid/db/billing-domain.server); the sweep is
|
|
* driven per tenant from the global Tenant registry, the TenantSubscription
|
|
* read/writes run inside withTenantRLS() (FORCE RLS), and the notice is sent
|
|
* through notifyWithTenantRlsContext() — NotificationDelivery is FORCE RLS, so
|
|
* an unbound write is rejected by the policy (R-P1b precedent). notify()'s
|
|
* sourceKey upsert dedups per attempt, so re-running a tick never double-sends.
|
|
* The freeze writes an AuditEvent + `billing_state_changed` activity in the
|
|
* same shape as the BILL-05 provider sync.
|
|
*/
|
|
async function runBillingDunningSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { notified: 0, frozen: 0 },
|
|
worker: async (tenantId) => {
|
|
const subscription = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.tenantSubscription.findUnique({
|
|
where: { tenantId },
|
|
select: { status: true, pastDueSince: true, dunningAttemptCount: true },
|
|
})
|
|
);
|
|
if (!subscription) return { notified: 0, frozen: 0 };
|
|
|
|
const decision = evaluateBillingDunning(subscription, now);
|
|
if (decision.kind === "send_dunning") {
|
|
// notify() runs outside the tenant read transaction (slow I/O) but must
|
|
// bind the tenant context itself: NotificationDelivery is FORCE RLS.
|
|
await notifyWithTenantRlsContext(
|
|
{
|
|
tenantId,
|
|
channel: "EMAIL",
|
|
category: "billing",
|
|
messageKey: `billing.notify.dunning.attempt${decision.attempt}`,
|
|
sourceKey: `billing-dunning:${tenantId}:attempt:${decision.attempt}`,
|
|
payloadMetadata: { attempt: decision.attempt, thresholdDays: decision.thresholdDays },
|
|
},
|
|
db as unknown as NotificationClient
|
|
);
|
|
await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.tenantSubscription.update({
|
|
where: { tenantId },
|
|
data: { dunningAttemptCount: { increment: 1 }, lastDunningSentAt: now },
|
|
})
|
|
);
|
|
return { notified: 1, frozen: 0 };
|
|
}
|
|
|
|
if (decision.kind === "freeze") {
|
|
await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.tenantSubscription.update({
|
|
where: { tenantId },
|
|
data: { status: "frozen", frozenAt: now },
|
|
})
|
|
);
|
|
await runWithRlsContext({ tenantId }, async () => {
|
|
await createAuditEvent({
|
|
tenantId,
|
|
actorType: "system",
|
|
action: "BILLING_DUNNING_FROZEN",
|
|
targetType: "TenantSubscription",
|
|
targetId: tenantId,
|
|
sourceModule: "billing",
|
|
outcome: "success",
|
|
dataSensitivity: "internal",
|
|
metadata: { thresholdDays: decision.thresholdDays },
|
|
});
|
|
await db.activity.create({
|
|
data: {
|
|
tenantId,
|
|
actorUserId: null,
|
|
actorKind: "system",
|
|
activityType: "billing_state_changed",
|
|
subjectType: "tenant",
|
|
subjectId: tenantId,
|
|
summaryText: "activity.runtime.billing.statusChanged",
|
|
metadata: { reason: "dunning_exhausted", thresholdDays: decision.thresholdDays },
|
|
},
|
|
});
|
|
});
|
|
return { notified: 0, frozen: 1 };
|
|
}
|
|
|
|
return { notified: 0, frozen: 0 };
|
|
},
|
|
merge: (state, result) => ({ notified: state.notified + result.notified, frozen: state.frozen + result.frozen }),
|
|
});
|
|
|
|
const { notified, frozen } = sweep.state;
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, notified, frozen },
|
|
"billing.dunning-sweep.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "billing.dunning-sweep.completed", details: { task: payload.task, tenants: sweep.inspected, notified, frozen } };
|
|
}
|
|
|
|
/**
|
|
* P4 Tasks (AK-P4b) §7 — reminder sweep (DoD 4). For each open task with a
|
|
* deadline in scan range, sends reminders to the assignee (or assigned group's
|
|
* members) + watchers at the 24h-before / due / 24h-after windows. The
|
|
* after-window escalates to EMAIL. notify()'s sourceKey upsert dedups, so the
|
|
* hourly sweep never double-sends.
|
|
*
|
|
* RLS-ready (AK-P4b "Teil A"): like the archive sweep, the cross-tenant scan is
|
|
* driven per tenant from the global Tenant registry, and the Task/TaskWatcher
|
|
* reads run inside withTenantRLS() so the policy binds once activated. notify()
|
|
* runs OUTSIDE the read transaction (slow I/O) but binds its own tenant context
|
|
* via notifyWithTenantRlsContext() — NotificationDelivery is FORCE RLS, so an
|
|
* unbound write is rejected by the policy. `take` now bounds tasks per tenant
|
|
* rather than globally.
|
|
*/
|
|
async function runTaskReminderSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const bounds = reminderScanBounds(now);
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000;
|
|
const db = await getDefaultDbClient();
|
|
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { sent: 0, scanned: 0 },
|
|
worker: async (tenantId) => {
|
|
let sent = 0;
|
|
const tasks = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.task.findMany({
|
|
where: {
|
|
tenantId,
|
|
deadline: { not: null, gte: bounds.deadlineGte, lte: bounds.deadlineLte },
|
|
status: { in: ["open", "in_progress", "blocked"] },
|
|
archivedAt: null,
|
|
},
|
|
select: { id: true, tenantId: true, deadline: true, assignedToUserId: true, assignedToGroupId: true },
|
|
take,
|
|
})
|
|
);
|
|
|
|
for (const task of tasks) {
|
|
if (!task.deadline) continue;
|
|
const windows = selectReminderWindows(task.deadline, now);
|
|
if (windows.length === 0) continue;
|
|
|
|
// Recipients: the assigned user (or every member of the assigned group)
|
|
// plus all watchers, de-duplicated. TaskWatcher is RLS-scoped, so the reads
|
|
// run inside the tenant transaction; group membership is read alongside.
|
|
const recipientIds = new Set<string>();
|
|
if (task.assignedToUserId) recipientIds.add(task.assignedToUserId);
|
|
const { groupMembers, watchers } = await withTenantRLS(db, tenantId, async (tx) => {
|
|
const groupMembers = task.assignedToGroupId
|
|
? await tx.groupUser.findMany({ where: { groupId: task.assignedToGroupId }, select: { userId: true } })
|
|
: [];
|
|
const watcherRows = await tx.taskWatcher.findMany({ where: { tenantId, taskId: task.id }, select: { userId: true } });
|
|
return { groupMembers, watchers: watcherRows };
|
|
});
|
|
for (const member of groupMembers) recipientIds.add(member.userId);
|
|
for (const watcher of watchers) recipientIds.add(watcher.userId);
|
|
|
|
for (const window of windows) {
|
|
for (const userId of recipientIds) {
|
|
try {
|
|
// notify() runs outside the tenant read transaction (slow I/O) but must
|
|
// bind the tenant context itself: NotificationDelivery is FORCE RLS.
|
|
await notifyWithTenantRlsContext(
|
|
{
|
|
tenantId: task.tenantId,
|
|
userId,
|
|
channel: reminderChannel(window),
|
|
category: "tasks.reminder",
|
|
messageKey: reminderMessageKey(window),
|
|
sourceKey: reminderSourceKey(task.id, window),
|
|
payloadMetadata: { taskId: task.id, deadline: task.deadline.toISOString() },
|
|
},
|
|
db as unknown as NotificationClient
|
|
);
|
|
sent += 1;
|
|
} catch (err) {
|
|
logger.error(
|
|
{ task: payload.task, taskId: task.id, userId, window, err: redactSecretLikeText(err instanceof Error ? err.message : String(err)) },
|
|
"tasks.reminder.notify-failed"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return { scanned: tasks.length, sent };
|
|
},
|
|
merge: (state, result) => ({ scanned: state.scanned + result.scanned, sent: state.sent + result.sent }),
|
|
});
|
|
const { scanned, sent } = sweep.state;
|
|
|
|
logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, scanned, sent }, "tasks.reminder-sweep.completed");
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "tasks.reminder-sweep.completed", details: { task: payload.task, tenants: sweep.inspected, scanned, sent } };
|
|
}
|
|
|
|
async function runAutomationSchedulerSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { inspectedRules: 0, scheduledEvents: 0, skippedInvalidRules: 0, truncatedRules: 0 },
|
|
worker: (tenantId) => withTenantRLS(db, tenantId, (tx) => runAutomationSchedulerForTenantWithTx(tx as never, { tenantId, now })),
|
|
merge: (state, result) => ({
|
|
inspectedRules: state.inspectedRules + result.inspectedRules,
|
|
scheduledEvents: state.scheduledEvents + result.scheduledEvents,
|
|
skippedInvalidRules: state.skippedInvalidRules + result.skippedInvalidRules,
|
|
truncatedRules: state.truncatedRules + result.truncatedRules,
|
|
}),
|
|
});
|
|
logger.info({ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, ...sweep.state }, "automation.scheduler-sweep.completed");
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return {
|
|
handled: true,
|
|
action: "automation.scheduler-sweep.completed",
|
|
details: { task: payload.task, tenants: sweep.inspected, ...sweep.state },
|
|
};
|
|
}
|
|
|
|
async function runAdminSalesTaskReminderSweepHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const runSweep =
|
|
deps.runAdminSalesTaskReminderSweep ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return runAdminSalesTaskReminderSweepInDb(db, input);
|
|
});
|
|
const result = await runSweep({ now, ...(take ? { take } : {}) });
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{
|
|
task: payload.task,
|
|
scanned: result.scanned,
|
|
dispatched: result.dispatched,
|
|
},
|
|
"admin-sales.task-reminder-sweep.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.task-reminder-sweep.completed",
|
|
details: result,
|
|
};
|
|
}
|
|
|
|
async function runAdminBookingReminderSweepHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const runSweep =
|
|
deps.runAdminBookingReminderSweep ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return runAdminBookingReminderSweepInDb(db, input);
|
|
});
|
|
const result = await runSweep({ now, ...(take ? { take } : {}) });
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{
|
|
task: payload.task,
|
|
scanned: result.scanned,
|
|
dispatched: result.dispatched,
|
|
failed: result.failed,
|
|
},
|
|
"admin-booking.reminder-sweep.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-booking.reminder-sweep.completed",
|
|
details: result,
|
|
};
|
|
}
|
|
|
|
async function runAdminEmailDeliveryHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const runDelivery =
|
|
deps.runAdminEmailDelivery ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
const env = deps.env ?? process.env;
|
|
return deliverDueAdminEmails(db as never, {
|
|
...input,
|
|
tracking:
|
|
env.ADMIN_APP_HOST && env.MARKETING_TOKEN_SECRET
|
|
? {
|
|
host: env.ADMIN_APP_HOST,
|
|
secret: env.MARKETING_TOKEN_SECRET,
|
|
}
|
|
: null,
|
|
});
|
|
});
|
|
const result = await runDelivery({ now, ...(take ? { take } : {}) });
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{
|
|
task: payload.task,
|
|
inspected: result.inspected,
|
|
claimed: result.claimed,
|
|
sent: result.sent,
|
|
failed: result.failed.length,
|
|
blocked: result.blocked,
|
|
deferred: result.deferred,
|
|
recoveredUnknown: result.recoveredUnknown,
|
|
noReplyReminderScanned: result.noReplyReminders.scanned,
|
|
noReplyReminderCreated: result.noReplyReminders.created,
|
|
},
|
|
"admin-sales.email-delivery.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.email-delivery.completed",
|
|
details: {
|
|
inspected: result.inspected,
|
|
claimed: result.claimed,
|
|
sent: result.sent,
|
|
failed: result.failed.length,
|
|
blocked: result.blocked,
|
|
deferred: result.deferred,
|
|
recoveredUnknown: result.recoveredUnknown,
|
|
noReplyReminderScanned: result.noReplyReminders.scanned,
|
|
noReplyReminderCreated: result.noReplyReminders.created,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runAdminEmailEventRetentionHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const redact =
|
|
deps.redactExpiredAdminEmailEventRawData ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return redactExpiredAdminEmailEventRawDataInDb(db, input);
|
|
});
|
|
const result = await redact({
|
|
now: new Date(),
|
|
...(take ? { take } : {}),
|
|
});
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{ task: payload.task, ...result },
|
|
"admin-sales.email-event-retention.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.email-event-retention.completed",
|
|
details: result,
|
|
};
|
|
}
|
|
|
|
async function runAdminCalendarSyncHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const run =
|
|
deps.runAdminCalendarSync ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return syncAdminCalendarBusySnapshots(db as never, {
|
|
...input,
|
|
env: deps.env ?? process.env,
|
|
});
|
|
});
|
|
const result = await run({
|
|
now: new Date(),
|
|
...(take ? { take } : {}),
|
|
});
|
|
const details = {
|
|
inspected: result.inspected,
|
|
synced: result.synced,
|
|
failed: result.failed.length,
|
|
};
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{ task: payload.task, ...details },
|
|
"admin-sales.calendar-sync.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.calendar-sync.completed",
|
|
details,
|
|
};
|
|
}
|
|
|
|
async function runAdminEmailSyncHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const runSync =
|
|
deps.runAdminEmailSync ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return syncClaimedAdminMailboxes(db as never, {
|
|
...input,
|
|
attachmentProcessor:
|
|
createWorkerAdminEmailAttachmentProcessor(
|
|
(deps.env ?? process.env) as never
|
|
),
|
|
});
|
|
});
|
|
const result = await runSync({ now, ...(take ? { take } : {}) });
|
|
const details = {
|
|
inspected: result.inspected,
|
|
synced: result.synced,
|
|
upserted: result.upserted,
|
|
replayed: result.replayed,
|
|
failed: result.failed.length,
|
|
};
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{ task: payload.task, ...details },
|
|
"admin-sales.email-sync.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.email-sync.completed",
|
|
details,
|
|
};
|
|
}
|
|
|
|
async function runAdminEmailDomainCheckHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const run =
|
|
deps.runAdminEmailDomainCheck ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return checkAdminSendingDomains(db as never, input);
|
|
});
|
|
const result = await run({
|
|
now: new Date(),
|
|
...(take ? { take } : {}),
|
|
});
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{
|
|
task: payload.task,
|
|
inspected: result.inspected,
|
|
verified: result.verified,
|
|
degraded: result.degraded,
|
|
failed: result.failed.length,
|
|
},
|
|
"admin-sales.email-domain-check.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.email-domain-check.completed",
|
|
details: {
|
|
inspected: result.inspected,
|
|
verified: result.verified,
|
|
degraded: result.degraded,
|
|
failed: result.failed.length,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runAdminCallRecordingProcessHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const run =
|
|
deps.runAdminCallRecordingProcessing ??
|
|
(async (input: { readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return runAdminCallRecordingProcessing({
|
|
db,
|
|
env: deps.env ?? process.env,
|
|
...input,
|
|
});
|
|
});
|
|
const result = await run({ ...(take ? { take } : {}) });
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{ task: payload.task, ...result },
|
|
"admin-sales.call-recording-process.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.call-recording-process.completed",
|
|
details: result,
|
|
};
|
|
}
|
|
|
|
async function runAdminCallRecordingRetentionHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const run =
|
|
deps.runAdminCallRecordingRetention ??
|
|
(async (input: { readonly now: Date; readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return runAdminCallRecordingRetention({
|
|
db,
|
|
env: deps.env ?? process.env,
|
|
...input,
|
|
});
|
|
});
|
|
const result = await run({
|
|
now: new Date(),
|
|
...(take ? { take } : {}),
|
|
});
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{ task: payload.task, ...result },
|
|
"admin-sales.call-recording-retention.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.call-recording-retention.completed",
|
|
details: result,
|
|
};
|
|
}
|
|
|
|
async function runAdminOutboundCallHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const take =
|
|
typeof payload.take === "number" && payload.take > 0
|
|
? Math.floor(payload.take)
|
|
: undefined;
|
|
const run =
|
|
deps.runAdminOutboundCalls ??
|
|
(async (input: { readonly take?: number }) => {
|
|
const db = await getDefaultDbClient();
|
|
return runAdminOutboundCalls({ db, ...input });
|
|
});
|
|
const result = await run({ ...(take ? { take } : {}) });
|
|
(deps.logger ?? noopHandlerLogger).info(
|
|
{ task: payload.task, ...result },
|
|
"admin-sales.outbound-call-process.completed"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.outbound-call-process.completed",
|
|
details: result,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* P17 HR-Hub (AK-P17b) — upload-reminder dispatch. For every active reminder
|
|
* past its `dueAt`, fan out per-employee dispatch rows (idempotent via the unique
|
|
* reminder+employee+dueAt), notify each employee, advance the reminder by its
|
|
* cadence (or complete a one-off), then escalate overdue mandatory dispatches to
|
|
* the employee's manager. Per-tenant from the global Tenant registry; all HR
|
|
* reads/writes run inside withTenantRLS() so the policy binds once activated.
|
|
* notify() runs outside the tenant transaction (slow I/O) with its own bound
|
|
* tenant context (notifyWithTenantRlsContext — FORCE RLS), deduped by sourceKey.
|
|
*/
|
|
async function runHrUploadReminderDispatchHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { dispatched: 0, escalated: 0 },
|
|
worker: async (tenantId) => {
|
|
let dispatched = 0;
|
|
let escalated = 0;
|
|
const due = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminder.findMany({ where: { tenantId, status: "active", archivedAt: null, dueAt: { not: null, lte: now } } })
|
|
);
|
|
const profiles =
|
|
due.length > 0
|
|
? await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.hrEmployeeProfile.findMany({ where: { tenantId }, select: { userId: true, team: true, locationId: true } })
|
|
)
|
|
: [];
|
|
|
|
for (const reminder of due) {
|
|
const dueAt = reminder.dueAt ?? now;
|
|
const members = profiles.filter((profile) => audienceMatches(reminder.audienceKind, reminder.audienceRef, profile));
|
|
|
|
for (const member of members) {
|
|
const created = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminderDispatch.createMany({
|
|
data: [{ tenantId, reminderId: reminder.id, employeeUserId: member.userId, dueAt, status: "sent", sentAt: now }],
|
|
skipDuplicates: true,
|
|
})
|
|
);
|
|
if (created.count > 0) {
|
|
dispatched += created.count;
|
|
await notifyWithTenantRlsContext(
|
|
{
|
|
tenantId,
|
|
userId: member.userId,
|
|
channel: "IN_APP",
|
|
category: "hr",
|
|
messageKey: "hr.notify.reminder.due",
|
|
sourceKey: `hr-reminder:${reminder.id}:${dueAt.getTime()}:${member.userId}`,
|
|
payloadMetadata: { reminderId: reminder.id },
|
|
},
|
|
db as unknown as NotificationClient
|
|
).catch((err) =>
|
|
logger.error(
|
|
{
|
|
task: payload.task,
|
|
reminderId: reminder.id,
|
|
userId: member.userId,
|
|
err: redactSecretLikeText(err instanceof Error ? err.message : String(err)),
|
|
},
|
|
"hr.upload-reminder.notify-failed"
|
|
)
|
|
);
|
|
}
|
|
}
|
|
|
|
const next = nextReminderDue({ cadenceKind: reminder.cadenceKind, intervalMonths: reminder.intervalMonths, from: dueAt });
|
|
await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminder.update({ where: { id: reminder.id }, data: next ? { dueAt: next } : { dueAt: null, status: "completed" } })
|
|
);
|
|
}
|
|
|
|
// Escalate still-unacknowledged mandatory dispatches past the grace window.
|
|
const cutoff = new Date(now.getTime() - HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS);
|
|
const stale = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminderDispatch.findMany({
|
|
where: { tenantId, status: "sent", dueAt: { lt: cutoff }, reminder: { escalateToManager: true, mandatory: true } },
|
|
})
|
|
);
|
|
const staleEmployeeUserIds = [...new Set(stale.map((dispatch) => dispatch.employeeUserId))];
|
|
const escalationProfiles =
|
|
staleEmployeeUserIds.length > 0
|
|
? await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.hrEmployeeProfile.findMany({
|
|
where: { tenantId, userId: { in: staleEmployeeUserIds } },
|
|
select: { userId: true, managerUserId: true },
|
|
})
|
|
)
|
|
: [];
|
|
const managerUserIdsByEmployee = new Map(escalationProfiles.map((profile) => [profile.userId, profile.managerUserId]));
|
|
for (const dispatch of stale) {
|
|
await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminderDispatch.update({ where: { id: dispatch.id }, data: { status: "escalated", escalatedAt: now } })
|
|
);
|
|
escalated += 1;
|
|
const managerUserId = managerUserIdsByEmployee.get(dispatch.employeeUserId);
|
|
if (managerUserId) {
|
|
await notifyWithTenantRlsContext(
|
|
{
|
|
tenantId,
|
|
userId: managerUserId,
|
|
channel: "IN_APP",
|
|
category: "hr",
|
|
messageKey: "hr.notify.reminder.escalated",
|
|
sourceKey: `hr-reminder-escalate:${dispatch.id}`,
|
|
payloadMetadata: { dispatchId: dispatch.id },
|
|
},
|
|
db as unknown as NotificationClient
|
|
).catch((err) =>
|
|
logger.error(
|
|
{ task: payload.task, dispatchId: dispatch.id, err: redactSecretLikeText(err instanceof Error ? err.message : String(err)) },
|
|
"hr.upload-reminder.escalate-notify-failed"
|
|
)
|
|
);
|
|
}
|
|
}
|
|
return { dispatched, escalated };
|
|
},
|
|
merge: (state, result) => ({ dispatched: state.dispatched + result.dispatched, escalated: state.escalated + result.escalated }),
|
|
});
|
|
const { dispatched, escalated } = sweep.state;
|
|
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, dispatched, escalated },
|
|
"hr.upload-reminder-dispatch.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "hr.upload-reminder-dispatch.completed", details: { task: payload.task, tenants: sweep.inspected, dispatched, escalated } };
|
|
}
|
|
|
|
/**
|
|
* AK-P14a · P14 Email — sending-domain DNS auto-probe (5-minute cadence). For
|
|
* each active tenant, resolves DKIM/SPF/DMARC for every not-yet-settled sending
|
|
* domain and advances `verificationStatus` via the pure decideDomainProbeStatus()
|
|
* helper. Reads/writes run inside withTenantRLS() so the policy binds once the
|
|
* P5 cutover activates RLS; the domain UPDATE is auto-audited by the M001 Prisma
|
|
* middleware. A tenant-level notification fires only on the verified/failed
|
|
* transition (not on every probe). The DNS resolver is the node:dns binding; the
|
|
* decision logic is unit-tested without DNS/DB.
|
|
*/
|
|
async function runEmailDomainProbeHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const resolver: DomainDnsResolver = createDnsDomainResolver();
|
|
const sweep = await runActiveTenantSweep(db, {
|
|
initialState: { probed: 0, verified: 0, failed: 0 },
|
|
worker: async (tenantId) => {
|
|
let verified = 0;
|
|
let failed = 0;
|
|
const domains = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.tenantSendingDomain.findMany({ where: { tenantId, archivedAt: null, verificationStatus: { in: ["pending", "verifying"] } } })
|
|
);
|
|
|
|
for (const domain of domains) {
|
|
const dns = await resolver({ domain: domain.domain, dkimSelector: domain.dkimSelector });
|
|
const firstProbeAt = domain.firstProbeAt ?? now;
|
|
const decision = decideDomainProbeStatus({ dns, firstProbeAt, now });
|
|
const changed = decision.status !== domain.verificationStatus;
|
|
|
|
await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.tenantSendingDomain.update({
|
|
where: { id: domain.id },
|
|
data: { verificationStatus: decision.status, lastError: decision.lastError, lastProbedAt: now, firstProbeAt },
|
|
})
|
|
);
|
|
|
|
if (changed && (decision.status === "verified" || decision.status === "failed")) {
|
|
if (decision.status === "verified") verified += 1;
|
|
else failed += 1;
|
|
await notifyWithTenantRlsContext(
|
|
{
|
|
tenantId,
|
|
channel: "IN_APP",
|
|
category: "email",
|
|
messageKey: decision.status === "verified" ? "email.domain.notify.verified" : "email.domain.notify.failed",
|
|
sourceKey: `email-domain-${decision.status}:${domain.id}`,
|
|
payloadMetadata: { domainId: domain.id, domain: domain.domain },
|
|
},
|
|
db as unknown as NotificationClient
|
|
).catch((err) =>
|
|
logger.error(
|
|
{ task: payload.task, domainId: domain.id, err: redactSecretLikeText(err instanceof Error ? err.message : String(err)) },
|
|
"email.domain-probe.notify-failed",
|
|
)
|
|
);
|
|
}
|
|
}
|
|
return { probed: domains.length, verified, failed };
|
|
},
|
|
merge: (state, result) => ({
|
|
probed: state.probed + result.probed,
|
|
verified: state.verified + result.verified,
|
|
failed: state.failed + result.failed,
|
|
}),
|
|
});
|
|
const { probed, verified, failed } = sweep.state;
|
|
|
|
logger.info(
|
|
{ task: payload.task, tenants: sweep.inspected, tenantFailures: sweep.failures.length, probed, verified, failed },
|
|
"email.domain-probe.completed"
|
|
);
|
|
throwOnActiveTenantSweepFailures(payload.task, sweep.failures);
|
|
return { handled: true, action: "email.domain-probe.completed", details: { task: payload.task, tenants: sweep.inspected, probed, verified, failed } };
|
|
}
|
|
|
|
/**
|
|
* Phase 10 #68 — maintenance-handler adapter for the DSGVO grace-period
|
|
* scan. Wraps `runDsgvoGraceCron` (the pure unit-tested core) with the
|
|
* production HandlerDependencies surface.
|
|
*
|
|
* Why a separate adapter: `runDsgvoGraceCron` is intentionally
|
|
* host-agnostic (no BullMQ, no Prisma). The maintenance queue receives
|
|
* a CleanupJob discriminator and this adapter is what binds the cron
|
|
* to the worker's logger + Prisma-backed candidate-reader + cascade
|
|
* dispatcher (via `deps.dsgvoGraceCronService`).
|
|
*
|
|
* When `deps.dsgvoGraceCronService` is not configured we still need to
|
|
* acknowledge the job (otherwise BullMQ retries forever and we are
|
|
* back to the UnsupportedWorkerJobError loop that motivated this fix).
|
|
* In that case we log a single warning and return `scanned=0`. The
|
|
* follow-up phase wires the real Prisma reader + cascade dispatcher.
|
|
*/
|
|
async function runDsgvoGracePeriodScanHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const service = deps.dsgvoGraceCronService;
|
|
|
|
if (!service) {
|
|
// Deliberately not throwing: we want the cron to ack so BullMQ does
|
|
// not retry; the missing service is a deployment-config gap, not a
|
|
// payload-shape bug. Emit an explicit log for operators.
|
|
logger.info({ task: payload.task, requestId: payload.metadata?.requestId ?? null }, "dsgvo-grace-cron.skipped: dsgvoGraceCronService not configured");
|
|
return {
|
|
handled: true,
|
|
action: "dsgvo.grace-period.skipped",
|
|
details: { task: payload.task, reason: "dsgvoGraceCronService not configured", scanned: 0, dispatched: 0, failed: 0 },
|
|
};
|
|
}
|
|
|
|
const result = await runDsgvoGraceCron({
|
|
listOverdueUsers: (now: Date): Promise<DsgvoGraceCandidate[]> => service.listOverdueUsers(now),
|
|
enqueueDeletion: (input) => service.enqueueDeletion(input),
|
|
logger,
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "dsgvo.grace-period.scanned",
|
|
details: {
|
|
task: payload.task,
|
|
scanned: result.scanned,
|
|
dispatched: result.dispatched,
|
|
failed: result.failed,
|
|
// `failures` contains userIds — keep them so audit can correlate
|
|
// but redact in case any future error message embeds secrets.
|
|
failures: result.failures.map((f) => ({ userId: f.userId, error: redactSecretLikeText(f.error) })),
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* AK-P21 — maintenance-handler adapter for the tenant hard-delete scan.
|
|
* Mirrors the dsgvo-grace-cron adapter: when the service is not wired the
|
|
* cron tick is acked with an explicit skip (a config gap must not turn into
|
|
* an endless BullMQ retry loop).
|
|
*/
|
|
async function runPrivacyTenantDeletionScanHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const service = deps.privacyTenantDeletionService;
|
|
|
|
if (!service) {
|
|
logger.info(
|
|
{ task: payload.task, requestId: payload.metadata?.requestId ?? null },
|
|
"privacy-tenant-deletion.skipped: privacyTenantDeletionService not configured"
|
|
);
|
|
return {
|
|
handled: true,
|
|
action: "privacy.tenant-deletion.skipped",
|
|
details: { task: payload.task, reason: "privacyTenantDeletionService not configured", scanned: 0, completed: 0, failed: 0 },
|
|
};
|
|
}
|
|
|
|
const result = await service.runScan();
|
|
return {
|
|
handled: true,
|
|
action: "privacy.tenant-deletion.scanned",
|
|
details: {
|
|
task: payload.task,
|
|
scanned: result.scanned,
|
|
completed: result.completed,
|
|
failed: result.failed,
|
|
failures: result.failures.map((failure) => ({
|
|
requestId: failure.requestId,
|
|
tenantId: failure.tenantId,
|
|
error: redactSecretLikeText(failure.error),
|
|
})),
|
|
},
|
|
};
|
|
}
|
|
|
|
const noopHandlerLogger = {
|
|
info: () => undefined,
|
|
error: () => undefined,
|
|
} satisfies NonNullable<HandlerDependencies["logger"]>;
|
|
|
|
async function runExportHandler(
|
|
payload: QueuePayload<typeof queueNames.exports>,
|
|
deps: HandlerDependencies,
|
|
context?: HandlerExecutionContext<typeof queueNames.exports>
|
|
): Promise<HandlerResult> {
|
|
payload = normalizePortalGdprExportQueuePayload(payload);
|
|
if (payload.kind !== "export.run") {
|
|
throw new ExportHandlerError("Unsupported export payload kind.", {
|
|
code: "invalid_payload",
|
|
message: "Unsupported export payload kind.",
|
|
});
|
|
}
|
|
|
|
if (isPortalGdprExportType(payload.exportType)) {
|
|
return runPortalGdprExportHandler(payload, deps, context);
|
|
}
|
|
|
|
if (isPrivacyGdprExportType(payload.exportType)) {
|
|
return runPrivacyGdprExportHandler(payload, deps, context);
|
|
}
|
|
|
|
const markPaused = deps.markExportJobPaused ?? markExportJobPausedInDb;
|
|
const markResumed = deps.markExportJobResumed ?? markExportJobResumedInDb;
|
|
const markRunning = deps.markExportJobRunning ?? markExportJobRunningInDb;
|
|
const markSucceeded = deps.markExportJobSucceeded ?? markExportJobSucceededInDb;
|
|
const markFailed = deps.markExportJobFailed ?? markExportJobFailedInDb;
|
|
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
|
const complianceGate = payload.complianceGate;
|
|
|
|
if (complianceGate?.state === "paused") {
|
|
const pauseReason = redactSecretLikeText(complianceGate.reason ?? "Export paused by compliance gate.");
|
|
await markPaused({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade, reason: pauseReason });
|
|
|
|
return {
|
|
handled: true,
|
|
action: "export.paused",
|
|
details: buildExportDetails(payload, {
|
|
attemptsMade,
|
|
pauseReason,
|
|
pauseRequestId: complianceGate.requestId,
|
|
status: "paused",
|
|
}),
|
|
};
|
|
}
|
|
|
|
if (complianceGate?.state === "active" || complianceGate?.resumedAt) {
|
|
await markResumed({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade });
|
|
}
|
|
|
|
await markRunning({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade });
|
|
|
|
if (!deps.exportService) {
|
|
const message = "Export service is not configured.";
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ExportHandlerError",
|
|
errorMessage: message,
|
|
});
|
|
throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "export_service_missing" }));
|
|
}
|
|
|
|
try {
|
|
const result = await deps.exportService.runExport(payload);
|
|
await markSucceeded({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
rowCount: result.rowCount,
|
|
attemptsMade,
|
|
expiresAt: result.expiresAt,
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "export.completed",
|
|
details: buildExportDetails(payload, {
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
attemptsMade,
|
|
expiresAt: result.expiresAt.toISOString(),
|
|
rowCount: result.rowCount,
|
|
status: "completed",
|
|
}),
|
|
};
|
|
} catch (error) {
|
|
const errorName = error instanceof Error ? error.name : "Error";
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
if (isTerminalJobAttempt(attemptsMade, context)) {
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName,
|
|
errorMessage,
|
|
});
|
|
}
|
|
throw new ExportHandlerError("Export job failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Portal export requests originate in a SECURITY DEFINER transaction and are
|
|
* dispatched as an OutboxEvent envelope. The exports queue otherwise accepts
|
|
* direct ExportJob payloads, so normalize only this exact event type and
|
|
* reject tenant/actor mismatches before any storage or Prisma access.
|
|
*/
|
|
export function normalizePortalGdprExportQueuePayload(payload: unknown): ExportJob {
|
|
const record = isRecord(payload) ? payload : {};
|
|
if (record.kind === "export.run") return record as ExportJob;
|
|
|
|
if (record.eventType !== "portal.gdpr_export.requested") {
|
|
throw new ExportHandlerError("Unsupported export payload kind.", {
|
|
code: "invalid_payload",
|
|
eventType: typeof record.eventType === "string" ? record.eventType : null,
|
|
});
|
|
}
|
|
|
|
const outboxPayload = isRecord(record.payload) ? record.payload : {};
|
|
const tenantId = isNonEmptyString(record.tenantId) ? record.tenantId.trim() : "";
|
|
const payloadTenantId = isNonEmptyString(outboxPayload.tenantId) ? outboxPayload.tenantId.trim() : "";
|
|
const portalUserId = isNonEmptyString(outboxPayload.portalUserId) ? outboxPayload.portalUserId.trim() : "";
|
|
const exportJobId = isNonEmptyString(outboxPayload.jobId) ? outboxPayload.jobId.trim() : "";
|
|
if (!tenantId || tenantId !== payloadTenantId || !portalUserId || !exportJobId) {
|
|
throw new ExportHandlerError("Portal GDPR export outbox envelope is invalid.", {
|
|
code: "portal_gdpr_export_invalid_envelope",
|
|
tenantMatch: Boolean(tenantId && tenantId === payloadTenantId),
|
|
hasPortalUserId: Boolean(portalUserId),
|
|
hasExportJobId: Boolean(exportJobId),
|
|
});
|
|
}
|
|
|
|
const outboxEventId = isNonEmptyString(record.id) ? record.id.trim() : isNonEmptyString(record.outboxEventId) ? record.outboxEventId.trim() : undefined;
|
|
return {
|
|
kind: "export.run",
|
|
exportJobId,
|
|
tenantId,
|
|
requestedByUserId: portalUserId,
|
|
exportType: "gdpr_portal_user_self",
|
|
format: "json",
|
|
...(outboxEventId ? { requestId: outboxEventId } : {}),
|
|
metadata: {
|
|
actorKind: "portal_user",
|
|
sourceEventType: "portal.gdpr_export.requested",
|
|
...(outboxEventId ? { outboxEventId } : {}),
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runPortalGdprExportHandler(
|
|
payload: ExportJob,
|
|
deps: HandlerDependencies,
|
|
context?: HandlerExecutionContext<typeof queueNames.exports>
|
|
): Promise<HandlerResult> {
|
|
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
|
if (!deps.portalGdprExportService) {
|
|
throw new ExportHandlerError(
|
|
"Portal GDPR export service is not configured.",
|
|
buildExportDetails(payload, { status: "failed", attemptsMade, code: "portal_gdpr_export_service_missing" })
|
|
);
|
|
}
|
|
|
|
try {
|
|
const result = await deps.portalGdprExportService.runExport(payload);
|
|
return {
|
|
handled: true,
|
|
action: "portal-gdpr-export.completed",
|
|
details: buildExportDetails(payload, {
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
attemptsMade,
|
|
expiresAt: result.expiresAt.toISOString(),
|
|
rowCount: result.rowCount,
|
|
sizeBytes: result.sizeBytes,
|
|
status: "ready",
|
|
}),
|
|
};
|
|
} catch (error) {
|
|
const errorName = error instanceof Error ? error.name : "Error";
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
const details = buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage });
|
|
if (error instanceof PortalGdprExportSizeLimitExceededError) {
|
|
throw new UnrecoverableExportHandlerError("Portal GDPR export job failed.", details);
|
|
}
|
|
throw new ExportHandlerError("Portal GDPR export job failed.", details);
|
|
}
|
|
}
|
|
|
|
async function runPrivacyGdprExportHandler(
|
|
payload: ExportJob,
|
|
deps: HandlerDependencies,
|
|
context?: HandlerExecutionContext<typeof queueNames.exports>
|
|
): Promise<HandlerResult> {
|
|
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
|
const subject = resolvePrivacyGdprExportSubject(payload);
|
|
const markProcessing = deps.markGdprExportProcessing ?? markGdprExportProcessingInDb;
|
|
const markReady = deps.markGdprExportReady ?? markGdprExportReadyInDb;
|
|
const markFailed = deps.markGdprExportFailed ?? markGdprExportFailedInDb;
|
|
|
|
const service = deps.privacyGdprExportService;
|
|
if (!service) {
|
|
await markProcessing({ id: payload.exportJobId, ...subject });
|
|
const message = "Privacy GDPR export service is not configured.";
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
errorMessage: message,
|
|
});
|
|
throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "privacy_gdpr_export_service_missing" }));
|
|
}
|
|
|
|
return service.withTenantBarrier(payload, async (checkpoint) => {
|
|
let existingReady: Awaited<ReturnType<NonNullable<HandlerDependencies["privacyGdprExportService"]>["resolveReady"]>>;
|
|
let reclaimableProcessing: Awaited<ReturnType<NonNullable<HandlerDependencies["privacyGdprExportService"]>["resolveReclaimableProcessing"]>> = null;
|
|
try {
|
|
existingReady = await service.resolveReady(payload);
|
|
if (!existingReady && (context?.attemptsMade ?? 0) > 0) {
|
|
reclaimableProcessing = await service.resolveReclaimableProcessing(payload);
|
|
}
|
|
} catch (error) {
|
|
const errorName = error instanceof Error ? error.name : "Error";
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
throw new ExportHandlerError(
|
|
"Privacy GDPR export ready-state reconciliation failed.",
|
|
buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage })
|
|
);
|
|
}
|
|
if (existingReady) {
|
|
await checkpoint();
|
|
return completePrivacyGdprExport(payload, existingReady, attemptsMade, deps);
|
|
}
|
|
|
|
const processingStartedAt = new Date(Math.max(Date.now(), (reclaimableProcessing?.processingStartedAt.getTime() ?? Number.NEGATIVE_INFINITY) + 1));
|
|
const attempt = reclaimableProcessing?.attempt ?? service.createAttempt(payload);
|
|
await markProcessing({
|
|
id: payload.exportJobId,
|
|
...subject,
|
|
resultBucket: attempt.artifactBucket,
|
|
resultKey: attempt.artifactKey,
|
|
now: processingStartedAt,
|
|
...(reclaimableProcessing ? { previousProcessingStartedAt: reclaimableProcessing.processingStartedAt } : {}),
|
|
});
|
|
|
|
let artifactCleanupAllowed = true;
|
|
let failureTransitionAllowed = true;
|
|
try {
|
|
await checkpoint();
|
|
const result = await service.runExport(payload, attempt);
|
|
await checkpoint();
|
|
let readyTransition: MarkGdprExportReadyResult;
|
|
try {
|
|
await checkpoint();
|
|
readyTransition = await markReady({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
resultBucket: result.artifactBucket,
|
|
resultKey: result.artifactKey,
|
|
resultSizeBytes: result.sizeBytes,
|
|
resultExpiresAt: result.expiresAt,
|
|
processingStartedAt,
|
|
});
|
|
} catch (error) {
|
|
// The UPDATE may have committed even when its response was lost. Never
|
|
// delete or downgrade this attempt until a retry reconciles the ready row.
|
|
artifactCleanupAllowed = false;
|
|
failureTransitionAllowed = false;
|
|
throw error;
|
|
}
|
|
if (readyTransition === "lost_owner") {
|
|
// A retry may have adopted this exact attempt after an ambiguous PUT.
|
|
// Once ownership is lost, deleting the shared attempt key is unsafe.
|
|
artifactCleanupAllowed = false;
|
|
failureTransitionAllowed = false;
|
|
throw new Error("Privacy GDPR export completion lost its processing claim.");
|
|
}
|
|
|
|
// Once Ready committed, the artifact is no longer an unowned attempt and
|
|
// must never pass through the generic cleanup path. A deletion that began
|
|
// in the narrow post-CAS race terminalizes this exact Ready row while
|
|
// retaining its bucket/key for the hard-delete final sweep.
|
|
artifactCleanupAllowed = false;
|
|
failureTransitionAllowed = false;
|
|
try {
|
|
await checkpoint();
|
|
} catch (error) {
|
|
if (error instanceof TenantExportQuiescedError) {
|
|
await service.terminalizeReadyAfterQuiescence({ payload, result });
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
return completePrivacyGdprExport(payload, result, attemptsMade, deps);
|
|
} catch (error) {
|
|
const errorName = error instanceof Error ? error.name : "Error";
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
const isSizeLimitExceeded = error instanceof PrivacyGdprExportSizeLimitExceededError;
|
|
if (error instanceof PrivacyGdprExportPutOutcomeUnknownError) {
|
|
// The object may already exist. Preserve both the exact owner token and
|
|
// DB pointer so a BullMQ retry reclaims and overwrites the same key.
|
|
artifactCleanupAllowed = false;
|
|
failureTransitionAllowed = false;
|
|
}
|
|
let cleanupSucceeded = false;
|
|
if (artifactCleanupAllowed) {
|
|
try {
|
|
await service.deleteArtifact(attempt);
|
|
cleanupSucceeded = true;
|
|
} catch (cleanupError) {
|
|
(deps.logger ?? noopHandlerLogger).error(
|
|
{
|
|
exportJobId: payload.exportJobId,
|
|
error: redactSecretLikeText(cleanupError instanceof Error ? cleanupError.message : String(cleanupError)),
|
|
},
|
|
"privacy-gdpr-export.attempt-cleanup-failed"
|
|
);
|
|
}
|
|
}
|
|
if (failureTransitionAllowed && cleanupSucceeded) {
|
|
try {
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
errorMessage,
|
|
processingStartedAt,
|
|
resultBucket: attempt.artifactBucket,
|
|
resultKey: attempt.artifactKey,
|
|
clearResultLocation: cleanupSucceeded,
|
|
});
|
|
} catch (markError) {
|
|
(deps.logger ?? noopHandlerLogger).error(
|
|
{
|
|
exportJobId: payload.exportJobId,
|
|
error: redactSecretLikeText(markError instanceof Error ? markError.message : String(markError)),
|
|
},
|
|
"privacy-gdpr-export.failure-owner-cas-missed"
|
|
);
|
|
}
|
|
}
|
|
const details = buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage });
|
|
if (isSizeLimitExceeded && cleanupSucceeded) {
|
|
throw new UnrecoverableExportHandlerError("Privacy GDPR export job failed.", details);
|
|
}
|
|
throw new ExportHandlerError("Privacy GDPR export job failed.", details);
|
|
}
|
|
});
|
|
}
|
|
|
|
async function completePrivacyGdprExport(
|
|
payload: ExportJob,
|
|
result: { artifactBucket: string; artifactKey: string; rowCount: number | null; sizeBytes: number; expiresAt: Date },
|
|
attemptsMade: number,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
if (deps.sendGdprExportReadyMail) {
|
|
try {
|
|
await deps.sendGdprExportReadyMail({
|
|
tenantId: payload.tenantId,
|
|
exportJobId: payload.exportJobId,
|
|
requestedByUserId: payload.requestedByUserId,
|
|
scope: payload.exportType,
|
|
expiresAt: result.expiresAt,
|
|
});
|
|
} catch (error) {
|
|
(deps.logger ?? noopHandlerLogger).error(
|
|
{ exportJobId: payload.exportJobId, error: redactSecretLikeText(error instanceof Error ? error.message : String(error)) },
|
|
"privacy-gdpr-export.ready-mail-failed"
|
|
);
|
|
}
|
|
}
|
|
|
|
return {
|
|
handled: true,
|
|
action: "privacy-gdpr-export.completed",
|
|
details: buildExportDetails(payload, {
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
attemptsMade,
|
|
expiresAt: result.expiresAt.toISOString(),
|
|
rowCount: result.rowCount,
|
|
sizeBytes: result.sizeBytes,
|
|
status: "ready",
|
|
}),
|
|
};
|
|
}
|
|
|
|
async function runImportHandler(
|
|
payload: QueuePayload<typeof queueNames.imports>,
|
|
deps: HandlerDependencies,
|
|
context?: HandlerExecutionContext<typeof queueNames.imports>
|
|
): Promise<HandlerResult> {
|
|
// Jobs enqueued before the dedicated KWP queue rollout can still be present
|
|
// on hivegrid.imports. Keep them replayable without permitting new typed
|
|
// producers to target the generic queue.
|
|
const legacyPayload = payload as QueuePayload<typeof queueNames.imports> | QueuePayload<typeof queueNames.kwpImports>;
|
|
if (
|
|
payload.kind === "admin-sales.contacts-import" ||
|
|
payload.kind === "admin-sales.contacts-export"
|
|
) {
|
|
if (!deps.adminSalesDataService) {
|
|
throw new ImportHandlerError(
|
|
"Admin sales data worker service is not configured.",
|
|
{
|
|
code: "admin_sales_data_service_missing",
|
|
kind: payload.kind,
|
|
}
|
|
);
|
|
}
|
|
if (payload.kind === "admin-sales.contacts-import") {
|
|
const result = await deps.adminSalesDataService.runImport(payload);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.contacts-import.completed",
|
|
details: {
|
|
importJobId: payload.importJobId,
|
|
requestedByAdminUserId: payload.requestedByAdminUserId,
|
|
...result,
|
|
},
|
|
};
|
|
}
|
|
const result = await deps.adminSalesDataService.runExport(payload);
|
|
return {
|
|
handled: true,
|
|
action: "admin-sales.contacts-export.completed",
|
|
details: {
|
|
exportJobId: payload.exportJobId,
|
|
requestedByAdminUserId: payload.requestedByAdminUserId,
|
|
rowCount: result.rowCount,
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
expiresAt: result.expiresAt.toISOString(),
|
|
},
|
|
};
|
|
}
|
|
if (
|
|
legacyPayload.kind === "kwp.project-sync"
|
|
|| legacyPayload.kind === "kwp.project-catalog-sync"
|
|
|| legacyPayload.kind === "kwp.contract-sentinel"
|
|
|| legacyPayload.kind === "kwp.archive-discovery"
|
|
|| legacyPayload.kind === "kwp.archive-file-sync"
|
|
) {
|
|
return runKwpImportHandler(legacyPayload, deps);
|
|
}
|
|
if (payload.kind !== "import.run") {
|
|
throw new ImportHandlerError("Unsupported import payload kind.", {
|
|
code: "invalid_payload",
|
|
message: "Unsupported import payload kind.",
|
|
});
|
|
}
|
|
|
|
const markRunning = deps.markImportJobRunning ?? markImportJobRunningInDb;
|
|
const markSucceeded = deps.markImportJobSucceeded ?? markImportJobSucceededInDb;
|
|
const markFailed = deps.markImportJobFailed ?? markImportJobFailedInDb;
|
|
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
|
await markRunning({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade });
|
|
|
|
const accessDecision = authorizeStorageObjectAccess({
|
|
tenantId: payload.tenantId,
|
|
key: payload.sourceKey,
|
|
operation: "download",
|
|
});
|
|
|
|
if (!accessDecision.allowed) {
|
|
const message = "Import source storage access denied.";
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: `${message} reason=${accessDecision.reason}`,
|
|
});
|
|
throw new ImportHandlerError(
|
|
message,
|
|
buildImportDetails(payload, { status: "failed", attemptsMade, code: "source_access_denied", reason: accessDecision.reason, message })
|
|
);
|
|
}
|
|
|
|
if (!deps.importService) {
|
|
const message = "Import service is not configured.";
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: message,
|
|
});
|
|
throw new ImportHandlerError(message, buildImportDetails(payload, { status: "failed", attemptsMade, code: "import_service_missing" }));
|
|
}
|
|
|
|
if (!deps.scanService) {
|
|
const details = buildImportScanFailureDetails(payload, attemptsMade, {
|
|
verdict: "error",
|
|
code: "scan_service_missing",
|
|
message: "Import scan service is not configured.",
|
|
});
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: buildImportScanFailureMessage(details),
|
|
});
|
|
throw new ImportHandlerError("Import source scan failed.", details);
|
|
}
|
|
|
|
let scanResult: StorageScanResult;
|
|
try {
|
|
scanResult = await deps.scanService.scanObject({
|
|
bucket: payload.sourceBucket,
|
|
key: accessDecision.key,
|
|
tenantId: payload.tenantId,
|
|
uploadId: payload.importJobId,
|
|
contentType: payload.contentType,
|
|
metadata: {
|
|
...payload.metadata,
|
|
requestId: payload.requestId,
|
|
importJobId: payload.importJobId,
|
|
importType: payload.importType,
|
|
format: payload.format,
|
|
},
|
|
});
|
|
} catch (error) {
|
|
const details = buildImportScanFailureDetails(payload, attemptsMade, {
|
|
verdict: "error",
|
|
code: "scanner_rejected",
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
});
|
|
if (isTerminalJobAttempt(attemptsMade, context)) {
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: buildImportScanFailureMessage(details),
|
|
});
|
|
}
|
|
throw new ImportHandlerError("Import source scan failed.", details);
|
|
}
|
|
|
|
if (scanResult.verdict !== "clean") {
|
|
const details = buildImportScanFailureDetails(payload, attemptsMade, buildImportScanResultDetails(scanResult));
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: buildImportScanFailureMessage(details),
|
|
});
|
|
throw new ImportHandlerError("Import source scan failed.", details);
|
|
}
|
|
|
|
try {
|
|
const result = await deps.importService.runImport(payload);
|
|
await markSucceeded({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
rowCount: result.rowCount,
|
|
acceptedRows: result.acceptedRows,
|
|
rejectedRows: result.rejectedRows,
|
|
attemptsMade,
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "import.completed",
|
|
details: buildImportDetails(payload, {
|
|
status: "completed",
|
|
attemptsMade,
|
|
rowCount: result.rowCount,
|
|
acceptedRows: result.acceptedRows,
|
|
rejectedRows: result.rejectedRows,
|
|
duplicateCount: result.duplicateCount,
|
|
errorReportBucket: result.errorReportBucket,
|
|
errorReportKey: result.errorReportKey ?? result.validationReport?.errorReport?.errorReportKey,
|
|
}),
|
|
};
|
|
} catch (error) {
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
if (isTerminalJobAttempt(attemptsMade, context)) {
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: error instanceof Error ? error.name : "Error",
|
|
errorMessage,
|
|
});
|
|
}
|
|
throw new ImportHandlerError("Import job failed.", buildImportDetails(payload, { status: "failed", attemptsMade, message: errorMessage }));
|
|
}
|
|
}
|
|
|
|
async function runKwpImportHandler(
|
|
payload: QueuePayload<typeof queueNames.kwpImports>,
|
|
deps: HandlerDependencies,
|
|
): Promise<HandlerResult> {
|
|
const leaseKey = payload.kind === "kwp.project-sync"
|
|
? `kwp-project:${payload.tenantId}:${payload.bindingId}`
|
|
: payload.kind === "kwp.project-catalog-sync"
|
|
? `kwp-catalog:${payload.tenantId}:${payload.dataConnectionId}`
|
|
: payload.kind === "kwp.archive-discovery"
|
|
? `kwp-archive-discovery:${payload.tenantId}:${payload.bindingId}:${payload.cadenceBucket}:${payload.batchNumber}`
|
|
: payload.kind === "kwp.archive-file-sync"
|
|
? `kwp-archive:${payload.tenantId}:${payload.dataConnectionId}:${payload.archiveFile.externalId}`
|
|
: `kwp-contract:${payload.tenantId}:${payload.dataConnectionId}`;
|
|
let operation: (() => Promise<Record<string, unknown>>) | null;
|
|
if (payload.kind === "kwp.project-sync") {
|
|
operation = deps.syncKwpProjectBinding
|
|
? async () => {
|
|
const project = await deps.syncKwpProjectBinding!(payload);
|
|
if (!deps.syncKwpProjectInbound || project.state === "missing" || project.state === "quarantined") {
|
|
return { ...project };
|
|
}
|
|
const inbound = await deps.syncKwpProjectInbound(payload);
|
|
if (deps.enqueueKwpArchiveDiscovery) await deps.enqueueKwpArchiveDiscovery(payload);
|
|
return {
|
|
...project,
|
|
inboundRunId: inbound.runId,
|
|
inboundChanged: inbound.changed,
|
|
appointments: inbound.appointments,
|
|
tasks: inbound.tasks,
|
|
materialPositions: inbound.materialPositions,
|
|
failedAreas: inbound.failedAreas,
|
|
archiveDiscoveryScheduled: Boolean(deps.enqueueKwpArchiveDiscovery),
|
|
};
|
|
}
|
|
: null;
|
|
} else if (payload.kind === "kwp.project-catalog-sync") {
|
|
operation = deps.syncKwpProjectCatalog
|
|
? async () => {
|
|
const result = await deps.syncKwpProjectCatalog!(payload);
|
|
if (result.hasMore && deps.enqueueKwpProjectCatalogContinuation) {
|
|
await deps.enqueueKwpProjectCatalogContinuation(payload);
|
|
}
|
|
return { ...result };
|
|
}
|
|
: null;
|
|
} else if (payload.kind === "kwp.archive-discovery") {
|
|
operation = deps.syncKwpArchiveDiscovery
|
|
? async () => {
|
|
const result = await deps.syncKwpArchiveDiscovery!(payload);
|
|
const archiveJobs = deps.enqueueKwpArchiveFileJobs
|
|
? await deps.enqueueKwpArchiveFileJobs(payload, result.archiveFiles)
|
|
: 0;
|
|
if (result.cursor && deps.enqueueKwpArchiveDiscovery) {
|
|
await deps.enqueueKwpArchiveDiscovery(payload, result.cursor);
|
|
}
|
|
return {
|
|
processedFolders: result.processedFolders,
|
|
archiveJobs,
|
|
hasMore: result.cursor !== null,
|
|
};
|
|
}
|
|
: null;
|
|
} else if (payload.kind === "kwp.archive-file-sync") {
|
|
operation = deps.syncKwpArchiveFile
|
|
? async () => ({ ...(await deps.syncKwpArchiveFile!(payload)) })
|
|
: null;
|
|
} else {
|
|
operation = deps.runKwpContractSentinel
|
|
? async () => ({ ...(await deps.runKwpContractSentinel!(payload)) })
|
|
: null;
|
|
}
|
|
if (!operation) {
|
|
throw new UnsupportedWorkerJobError("KWP import worker service is not configured.", {
|
|
queueName: queueNames.kwpImports,
|
|
kind: payload.kind,
|
|
tenantId: payload.tenantId,
|
|
});
|
|
}
|
|
let lease;
|
|
try {
|
|
lease = await withDistributedJobLease(
|
|
deps.tenantMaintenanceLeaseStore ?? getSharedStore(),
|
|
leaseKey,
|
|
operation,
|
|
{ leaseSeconds: 300 },
|
|
);
|
|
} catch (error) {
|
|
if (isUnrecoverableKwpImportError(error)) {
|
|
throw new UnrecoverableError("KWP-Import kann mit diesem Datensatz oder Vertrag nicht fortgesetzt werden.");
|
|
}
|
|
throw error;
|
|
}
|
|
if (!lease.acquired) {
|
|
return {
|
|
handled: true,
|
|
action: "kwp.import.overlap-skipped",
|
|
details: { kind: payload.kind, tenantId: payload.tenantId, reason: "active-lease" },
|
|
};
|
|
}
|
|
return {
|
|
handled: true,
|
|
action: payload.kind === "kwp.project-sync"
|
|
? "kwp.project-sync.completed"
|
|
: payload.kind === "kwp.project-catalog-sync"
|
|
? "kwp.project-catalog-sync.completed"
|
|
: payload.kind === "kwp.archive-discovery"
|
|
? "kwp.archive-discovery.completed"
|
|
: payload.kind === "kwp.archive-file-sync"
|
|
? "kwp.archive-file-sync.completed"
|
|
: "kwp.contract-sentinel.completed",
|
|
details: {
|
|
kind: payload.kind,
|
|
tenantId: payload.tenantId,
|
|
dataConnectionId: payload.dataConnectionId,
|
|
cadenceBucket: payload.cadenceBucket,
|
|
...lease.value,
|
|
},
|
|
};
|
|
}
|
|
|
|
function isUnrecoverableKwpImportError(error: unknown): boolean {
|
|
if (error instanceof Error && error.name === "KwpInboundLimitError") return true;
|
|
if (!error || typeof error !== "object" || !("code" in error)) return false;
|
|
const code = (error as { code?: unknown }).code;
|
|
return code === "auth_scope_missing"
|
|
|| code === "capability_paused"
|
|
|| code === "contract_drift"
|
|
|| code === "invalid_credentials"
|
|
|| code === "resource_not_found";
|
|
}
|
|
|
|
function readAttemptsMade(metadata: Record<string, unknown> | undefined, context?: { attemptsMade?: number }): number {
|
|
if (context?.attemptsMade !== undefined) {
|
|
return Math.max(1, context.attemptsMade + 1);
|
|
}
|
|
return typeof metadata?.attemptsMade === "number" && Number.isFinite(metadata.attemptsMade) ? metadata.attemptsMade : 0;
|
|
}
|
|
|
|
function isTerminalJobAttempt(attemptsMade: number, context?: { attemptsLimit?: number }): boolean {
|
|
if (context?.attemptsLimit === undefined) {
|
|
return true;
|
|
}
|
|
return attemptsMade >= Math.max(1, context.attemptsLimit);
|
|
}
|
|
|
|
function buildExportDetails(payload: ExportJob, details: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
exportJobId: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
exportType: payload.exportType,
|
|
format: payload.format,
|
|
requestId: payload.requestId,
|
|
...details,
|
|
};
|
|
}
|
|
|
|
function buildImportDetails(payload: FileImportJob, details: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
importJobId: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
importType: payload.importType,
|
|
format: payload.format,
|
|
requestId: payload.requestId,
|
|
...details,
|
|
};
|
|
}
|
|
|
|
function buildScanResultDetails(result: StorageScanResult, uploadId: string | undefined): Record<string, unknown> {
|
|
const base = {
|
|
verdict: result.verdict,
|
|
bucket: result.bucket,
|
|
key: result.key,
|
|
uploadId,
|
|
};
|
|
|
|
if (result.verdict === "clean") {
|
|
return {
|
|
...base,
|
|
engine: result.engine,
|
|
};
|
|
}
|
|
|
|
if (result.verdict === "infected") {
|
|
return {
|
|
...base,
|
|
engine: result.engine,
|
|
signature: result.signature,
|
|
quarantineKey: result.quarantineKey,
|
|
};
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
...buildRedactedErrorDetails(result.error),
|
|
};
|
|
}
|
|
|
|
function buildImportScanResultDetails(result: StorageScanResult): Record<string, unknown> {
|
|
if (result.verdict === "clean") {
|
|
return {
|
|
verdict: result.verdict,
|
|
engine: result.engine,
|
|
};
|
|
}
|
|
|
|
if (result.verdict === "infected") {
|
|
return {
|
|
verdict: result.verdict,
|
|
code: "source_infected",
|
|
engine: result.engine,
|
|
signature: result.signature,
|
|
quarantineKey: result.quarantineKey,
|
|
message: "Import source object failed malware scan.",
|
|
};
|
|
}
|
|
|
|
return {
|
|
verdict: result.verdict,
|
|
...buildRedactedErrorDetails(result.error),
|
|
};
|
|
}
|
|
|
|
function buildImportScanFailureDetails(payload: FileImportJob, attemptsMade: number, scanDetails: Record<string, unknown>): Record<string, unknown> {
|
|
return buildImportDetails(payload, {
|
|
status: "failed",
|
|
attemptsMade,
|
|
scan: scanDetails,
|
|
});
|
|
}
|
|
|
|
function buildImportScanFailureMessage(details: Record<string, unknown>): string {
|
|
const scan = details.scan && typeof details.scan === "object" ? (details.scan as Record<string, unknown>) : {};
|
|
const parts = ["Import source scan failed.", `verdict=${String(scan.verdict ?? "error")}`, `code=${String(scan.code ?? "scan_failed")}`];
|
|
const requestId = typeof details.requestId === "string" ? details.requestId : undefined;
|
|
if (requestId) {
|
|
parts.push(`requestId=${requestId}`);
|
|
}
|
|
const message = typeof scan.message === "string" ? scan.message : undefined;
|
|
if (message) {
|
|
parts.push(`message=${redactSecretLikeText(message)}`);
|
|
}
|
|
return parts.join(" ");
|
|
}
|
|
|
|
function buildRejectedScanDetails(payload: QueuePayload<typeof queueNames.fileScan>, error: unknown): Record<string, unknown> {
|
|
return {
|
|
verdict: "error",
|
|
bucket: payload.bucket,
|
|
key: payload.key,
|
|
uploadId: payload.uploadId,
|
|
code: "scanner_rejected",
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
};
|
|
}
|
|
|
|
function buildRedactedErrorDetails(error: StorageScanDiagnostic): Record<string, unknown> {
|
|
return {
|
|
code: error.code,
|
|
message: redactSecretLikeText(error.message),
|
|
upstream: error.upstream
|
|
? {
|
|
name: error.upstream.name,
|
|
message: error.upstream.message ? redactSecretLikeText(error.upstream.message) : undefined,
|
|
status: error.upstream.status,
|
|
requestId: error.upstream.requestId,
|
|
}
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
function redactSecretLikeText(message: string): string {
|
|
return message
|
|
.replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]")
|
|
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
|
.replace(
|
|
/(authorization|access[_-]?key(?:id)?|api[-_]?key|client[-_]?secret|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi,
|
|
"$1$2[redacted]",
|
|
)
|
|
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]")
|
|
.slice(0, 1_000);
|
|
}
|
|
|
|
function serializeWebhookResponseDiagnostics(response: Response, body: string) {
|
|
return JSON.stringify({
|
|
status: response.status,
|
|
statusText: response.statusText,
|
|
contentType: response.headers.get("content-type"),
|
|
bodyBytes: new TextEncoder().encode(body).byteLength,
|
|
});
|
|
}
|
|
|
|
async function fetchWebhookWithRedirects(
|
|
fetchImpl: typeof fetch | undefined,
|
|
input: string,
|
|
init: RequestInit,
|
|
addressResolver?: OutboundHttpAddressResolver
|
|
): Promise<Response> {
|
|
try {
|
|
return await fetchOutboundHttpWithRedirects(input, init, {
|
|
// In production fetchImpl is undefined, so the guard uses its pinned
|
|
// undici fetch; a test-injected fetchImpl disables pinning.
|
|
...(fetchImpl ? { fetchImpl } : {}),
|
|
...(addressResolver ? { addressResolver } : {}),
|
|
timeoutMs: DEFAULT_WORKER_FETCH_TIMEOUT_MS,
|
|
redirectLimit: WEBHOOK_REDIRECT_LIMIT,
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof OutboundHttpUrlValidationError) {
|
|
throw new WebhookUrlValidationError(toWebhookUrlValidationMessage(error.message));
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function toWebhookUrlValidationMessage(message: string): string {
|
|
return message.replace(/^Outbound HTTP endpoint/u, "Webhook endpoint").replace(/^Outbound HTTP redirect/u, "Webhook redirect");
|
|
}
|
|
|
|
async function readLimitedResponseText(response: Response, limitBytes: number): Promise<string> {
|
|
if (!response.body) {
|
|
return "";
|
|
}
|
|
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
const chunks: string[] = [];
|
|
let bytes = 0;
|
|
|
|
while (bytes < limitBytes) {
|
|
const result = await reader.read();
|
|
if (result.done) {
|
|
chunks.push(decoder.decode());
|
|
return chunks.join("");
|
|
}
|
|
|
|
const chunk = result.value;
|
|
const remaining = limitBytes - bytes;
|
|
const slice = chunk.byteLength > remaining ? chunk.slice(0, remaining) : chunk;
|
|
chunks.push(decoder.decode(slice, { stream: chunk.byteLength <= remaining }));
|
|
bytes += slice.byteLength;
|
|
|
|
if (chunk.byteLength > remaining) {
|
|
await reader.cancel();
|
|
break;
|
|
}
|
|
}
|
|
|
|
chunks.push(decoder.decode());
|
|
return chunks.join("");
|
|
}
|
|
|
|
export type CreateRegisteredProcessorOptions = {
|
|
dependencies?: HandlerDependencies;
|
|
registry?: HandlerRegistry;
|
|
};
|
|
|
|
export function createRegisteredProcessor<Name extends QueueName>(queueName: Name, options: CreateRegisteredProcessorOptions = {}): TypedProcessor<Name> {
|
|
const dependencies = options.dependencies ?? {};
|
|
const registry = options.registry ?? buildHandlerRegistry(dependencies);
|
|
|
|
return async (job) => {
|
|
return registry[queueName](job.data, dependencies, {
|
|
attemptsMade: job.attemptsMade,
|
|
attemptsLimit: job.opts?.attempts,
|
|
jobId: job.id === undefined ? undefined : String(job.id),
|
|
updatePayload: job.updateData
|
|
? async (payload) => {
|
|
await job.updateData?.(payload);
|
|
}
|
|
: undefined,
|
|
});
|
|
};
|
|
}
|