import { createHash } from "node:crypto"; import { describe, expect, test, vi } from "vitest"; import { buildBrowserSessionCleanupJob, buildBrowserSessionRunJob, buildBrowserUsageFlushJob, buildCleanupJob, buildExportJob, buildFileScanJob, buildImportJob, buildAdminSalesContactsImportJob, buildAdminSalesContactsExportJob, buildKwpArchiveDiscoveryJob, buildKwpArchiveFileSyncJob, buildKwpContractSentinelJob, buildKwpProjectCatalogSyncJob, buildKwpProjectSyncJob, buildNotificationJob, buildPlatformNotificationJob, buildTransactionalMailJob, queueNames, type WebhookDeliveryJobDescriptor, } from "@hivegrid/queue"; import { buildHandlerRegistry, type HandlerDependencies } from "./handlers.js"; import { PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, PrivacyGdprExportSizeLimitExceededError } from "./jobs/privacy-gdpr-export.js"; import { PrivacyGdprExportPutOutcomeUnknownError } from "./jobs/privacy-gdpr-export-bootstrap.js"; import { TenantExportQuiescedError } from "./jobs/tenant-export-quiescence.js"; import type { CreateNotificationDeliveryInput } from "@hivegrid/db/notifications.server"; import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server"; import { hashWorkspaceOwnerInvitationMailContent } from "@hivegrid/db/workspace-owner-invite-mail-contract.server"; import { bindEncryptedPortalSetupMailToOutbox, hashPortalSetupMailContent, materializePortalSetupMailContent, PORTAL_SETUP_MAIL_SOURCE, PORTAL_SETUP_TOKEN_PLACEHOLDER, } from "@hivegrid/db/portal-setup-mail-contract.server"; import { MailProviderError, type MailProvider } from "@hivegrid/mail"; import type { StorageScanResult } from "@hivegrid/storage"; import { createSharedStore, MemoryAdapter } from "@hivegrid/shared-store"; import { UnrecoverableError } from "bullmq"; import { markWorkspaceOwnerInvitationDeliveryStarted, markWorkspaceOwnerInvitationProviderAccepted, redactTerminalWorkspaceOwnerInvitationMailPayload, redactWorkspaceOwnerInvitationMailPayload, } from "./security/workspace-owner-invitation-secret.js"; import { markPortalSetupInvitationDeliveryStarted, markPortalSetupInvitationProviderAccepted, redactTerminalPortalSetupInvitationMailPayload, } from "./security/portal-setup-invitation-secret.js"; type TestScanService = NonNullable; type TestExportService = NonNullable; type TestPrivacyGdprExportService = NonNullable; type TestPortalGdprExportService = NonNullable; type TestPortalGdprExportRetentionService = NonNullable; type TestImportService = NonNullable; type TestStorageDeleteService = NonNullable; type TestGdprRetentionJob = Awaited>>[number]; type TestRetentionListInput = Parameters>[0]; type TestRetentionJob = Awaited>>[number]; const privacyAttempt = { attemptId: "attempt_12345678", artifactBucket: "exports", artifactKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_job_1/attempt_12345678.zip", }; function createPrivacyGdprService( runExport: TestPrivacyGdprExportService["runExport"], overrides: Partial = {} ): TestPrivacyGdprExportService { return { withTenantBarrier: async (_payload, writer) => writer(async () => undefined), resolveReady: vi.fn(async () => null), resolveReclaimableProcessing: vi.fn(async () => null), createAttempt: vi.fn(() => privacyAttempt), runExport, deleteArtifact: vi.fn(async () => undefined), terminalizeReadyAfterQuiescence: vi.fn(async () => undefined), ...overrides, }; } const publicWebhookAddressResolver = vi.fn(async () => ["93.184.216.34"]); const scanResult = (result: Result): Result => result; const retentionJob = (overrides: Partial & Pick): TestRetentionJob => ({ tenantId: "tenant_1", status: "FAILED", rowCount: 1, expiresAt: new Date("2026-05-08T00:00:00.000Z"), deletedAt: new Date("2026-05-09T00:00:00.000Z"), requestId: `req_${overrides.id}`, ...overrides, }); function createActiveTenantReader(ids: string[] = ["tenant_1"]): NonNullable { return { tenant: { findMany: vi.fn(async (args: { take: number; cursor?: { id: string }; skip?: 1 }) => { const start = args.cursor ? ids.indexOf(args.cursor.id) + (args.skip ?? 0) : 0; return ids.slice(start, start + args.take).map((id) => ({ id })); }), }, }; } function createOwnerDeliveryDeps() { return { claimOwnerInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) => ({ state: "claimed" as const, payload: markWorkspaceOwnerInvitationDeliveryStarted(payload), })), acceptOwnerInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) => markWorkspaceOwnerInvitationProviderAccepted(payload)), abandonOwnerInvitationDelivery: vi.fn(async () => ({ state: "invalidated" })), }; } function createPortalSetupMailFixture(token = "p".repeat(43), tokenId = "portal_token_1") { const setupTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Kundenportal-Zugang einrichten"; const text = `Zugang einrichten: https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}`; const html = `Zugang einrichten`; const encrypted = { tenantId: "tenant_1", to: ["customer@example.test"], subject, text, html, metadata: { source: PORTAL_SETUP_MAIL_SOURCE, invitationSource: "portal_admin_invite", portalId: "portal_1", portalUserId: "portal_user_1", portalTokenId: tokenId, setupTokenHash, expiresAt: "2026-07-25T12:00:00.000Z", mailContentHash: hashPortalSetupMailContent({ recipient: "customer@example.test", subject, text, html, }), encryptedCapability: "v1.encrypted-capability", }, }; const bound = bindEncryptedPortalSetupMailToOutbox(encrypted, { outboxEventId: `outbox_${tokenId}`, dispatchKey: `portal-setup:${tokenId}`, }); const materialized = materializePortalSetupMailContent(bound, token); const descriptor = buildTransactionalMailJob(materialized as Parameters[0]); return { descriptor, token }; } function createPortalSetupDeliveryDeps() { return { claimPortalSetupInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) => ({ state: "claimed" as const, payload: markPortalSetupInvitationDeliveryStarted(payload), })), acceptPortalSetupInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) => markPortalSetupInvitationProviderAccepted(payload) ), terminalizePortalSetupInvitation: vi.fn(async () => ({ state: "invalidated" })), }; } describe("worker handlers", () => { test("sends queue-provider transactional email jobs through configured mail provider", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_1" })), }; const registry = buildHandlerRegistry({ mailProvider }); const descriptor = buildTransactionalMailJob({ from: { email: "sender@example.com", name: "Sender" }, to: ["user@example.com", { email: "admin@example.com", name: "Admin" }], cc: [{ email: "cc@example.com", name: "CC" }], bcc: ["bcc@example.com"], replyTo: "reply@example.com", subject: "Welcome to HiveGrid", text: "Hello from HiveGrid", html: "

Hello from HiveGrid

", templateId: "tmpl_welcome", attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }], tags: ["welcome"], precedenceBulk: true, sendAt: 1_778_580_000, inReplyTo: "message_1", references: ["message_0"], settings: { trackClicks: true, trackOpens: false, trackContent: true }, headers: [{ name: "X-Test", value: "1" }], listUnsubscribe: "", templateData: { firstName: "Ada", tenantName: "Example Co" }, tenantId: "tenant_1", userId: "user_1", metadata: { source: "queue-provider" }, }); expect(descriptor).toMatchObject({ queueName: queueNames.email, jobName: "send-transactional-email", payload: { kind: "mail.transactional" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "mail.sent", details: { provider: "mailjet", messageId: "msg_1" }, }); expect(mailProvider.send).toHaveBeenCalledTimes(1); expect(mailProvider.send).toHaveBeenCalledWith({ from: { email: "sender@example.com", name: "Sender" }, to: ["user@example.com", { email: "admin@example.com", name: "Admin" }], cc: [{ email: "cc@example.com", name: "CC" }], bcc: ["bcc@example.com"], replyTo: "reply@example.com", subject: "Welcome to HiveGrid", text: "Hello from HiveGrid", html: "

Hello from HiveGrid

", templateId: "tmpl_welcome", attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }], tags: ["welcome"], precedenceBulk: true, sendAt: 1_778_580_000, inReplyTo: "message_1", references: ["message_0"], settings: { trackClicks: true, trackOpens: false, trackContent: true }, headers: [{ name: "X-Test", value: "1" }], listUnsubscribe: "", personalization: [ { email: "user@example.com", data: { firstName: "Ada", tenantName: "Example Co" } }, { email: "admin@example.com", data: { firstName: "Ada", tenantName: "Example Co" } }, ], }); }); test("claims and acknowledges a portal setup capability around exactly one provider call", async () => { const { descriptor, token } = createPortalSetupMailFixture(); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_portal_setup" })), }; const updatePayload = vi.fn(async (_payload: unknown) => undefined); const delivery = createPortalSetupDeliveryDeps(); const registry = buildHandlerRegistry({ mailProvider, ...delivery }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload })).resolves.toEqual({ handled: true, action: "mail.sent", details: { provider: "mailjet", messageId: "msg_portal_setup" }, }); expect(delivery.claimPortalSetupInvitationDelivery).toHaveBeenCalledOnce(); expect(mailProvider.send).toHaveBeenCalledOnce(); expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ settings: { trackClicks: false, trackOpens: false } })); expect(delivery.acceptPortalSetupInvitationDelivery).toHaveBeenCalledOnce(); expect(updatePayload).toHaveBeenCalledTimes(2); expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token); expect(updatePayload.mock.calls.at(-1)?.[0]).toMatchObject({ metadata: expect.objectContaining({ portalSetupDeliveryStatus: "provider_accepted", secretBodyRedacted: true }), }); }); test("rotates an uncertain portal setup capability without a second provider call", async () => { const { descriptor, token } = createPortalSetupMailFixture("u".repeat(43), "portal_token_uncertain"); const startedPayload = markPortalSetupInvitationDeliveryStarted(descriptor.payload); const updatePayload = vi.fn(async (_payload: unknown) => undefined); const terminalizePortalSetupInvitation = vi.fn(async () => ({ state: "invalidated" })); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const registry = buildHandlerRegistry({ mailProvider, claimPortalSetupInvitationDelivery: vi.fn(async () => ({ state: "uncertain" as const, payload: startedPayload })), terminalizePortalSetupInvitation, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload })).rejects.toBeInstanceOf(UnrecoverableError); expect(mailProvider.send).not.toHaveBeenCalled(); expect(terminalizePortalSetupInvitation).toHaveBeenCalledOnce(); expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token); }); test("deduplicates a provider-accepted portal setup retry after BullMQ persistence failed", async () => { const { descriptor } = createPortalSetupMailFixture("a".repeat(43), "portal_token_accepted"); const acceptedPayload = markPortalSetupInvitationProviderAccepted(descriptor.payload); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_once" })), }; const claimPortalSetupInvitationDelivery = vi .fn() .mockResolvedValueOnce({ state: "claimed", payload: markPortalSetupInvitationDeliveryStarted(descriptor.payload) }) .mockResolvedValueOnce({ state: "provider_accepted", payload: acceptedPayload }); const acceptPortalSetupInvitationDelivery = vi.fn(async () => acceptedPayload); const registry = buildHandlerRegistry({ mailProvider, claimPortalSetupInvitationDelivery, acceptPortalSetupInvitationDelivery, }); const firstUpdatePayload = vi.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error("redis failed")); await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: firstUpdatePayload })).rejects.toThrow("redis failed"); await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: vi.fn(async () => undefined) })).resolves.toMatchObject({ details: { deduplicated: true }, }); expect(mailProvider.send).toHaveBeenCalledOnce(); expect(acceptPortalSetupInvitationDelivery).toHaveBeenCalledOnce(); }); test("redacts and revokes a portal setup capability after an ambiguous provider failure", async () => { const { descriptor, token } = createPortalSetupMailFixture("f".repeat(43), "portal_token_failure"); const providerError = new Error(`provider failed for /setup/${token}`); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(providerError)) }; const updatePayload = vi.fn(async (_payload: unknown) => undefined); const delivery = createPortalSetupDeliveryDeps(); const registry = buildHandlerRegistry({ mailProvider, ...delivery }); let thrown: unknown; try { await registry[descriptor.queueName](descriptor.payload, {}, { updatePayload }); } catch (error) { thrown = error; } expect(thrown).toBeInstanceOf(UnrecoverableError); expect((thrown as Error).message).not.toContain(token); expect(delivery.terminalizePortalSetupInvitation).toHaveBeenCalledOnce(); expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token); expect(updatePayload.mock.calls.at(-1)?.[0]).toEqual(redactTerminalPortalSetupInvitationMailPayload(descriptor.payload)); }); test("persists a token-free OWNER invitation marker before the mail job can complete", async () => { const token = "o".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_owner_1" })), }; const updatePayload = vi.fn(async (_payload: unknown) => undefined); const ownerDelivery = createOwnerDeliveryDeps(); const registry = buildHandlerRegistry({ mailProvider, ...ownerDelivery }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_1", invitationTokenHash, issuanceKeyHash: "b".repeat(64), mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload })).resolves.toEqual({ handled: true, action: "mail.sent", details: { provider: "mailjet", messageId: "msg_owner_1" }, }); expect(mailProvider.send).toHaveBeenCalledOnce(); expect(JSON.stringify(vi.mocked(mailProvider.send).mock.calls[0]?.[0])).toContain(token); expect(updatePayload).toHaveBeenCalledOnce(); expect(JSON.stringify(updatePayload.mock.calls[0]?.[0])).not.toContain(token); expect(updatePayload.mock.calls[0]?.[0]).toMatchObject({ kind: "mail.transactional", metadata: expect.objectContaining({ source: "workspace-owner-invitation", secretBodyRedacted: true }), }); }); test("deduplicates an OWNER retry when provider delivery succeeded but BullMQ redaction failed", async () => { const token = "u".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_owner_2" })), }; const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_2", invitationTokenHash, issuanceKeyHash: "b".repeat(64), mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); const acceptedPayload = markWorkspaceOwnerInvitationProviderAccepted(descriptor.payload); const claimOwnerInvitationDelivery = vi .fn() .mockResolvedValueOnce({ state: "claimed", payload: markWorkspaceOwnerInvitationDeliveryStarted(descriptor.payload) }) .mockResolvedValueOnce({ state: "provider_accepted", payload: acceptedPayload }); const acceptOwnerInvitationDelivery = vi.fn(async () => acceptedPayload); const registry = buildHandlerRegistry({ mailProvider, claimOwnerInvitationDelivery, acceptOwnerInvitationDelivery }); await expect( registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: vi.fn(async () => Promise.reject(new Error("redis write failed"))) }) ).rejects.toThrow("redis write failed"); const retryUpdatePayload = vi.fn(async () => undefined); await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: retryUpdatePayload })).resolves.toMatchObject({ details: { deduplicated: true }, }); expect(mailProvider.send).toHaveBeenCalledOnce(); expect(claimOwnerInvitationDelivery).toHaveBeenCalledTimes(2); expect(acceptOwnerInvitationDelivery).toHaveBeenCalledOnce(); expect(JSON.stringify(retryUpdatePayload.mock.calls)).not.toContain(token); }); test("fails closed before calling the provider for a corrupted OWNER mail contract", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const registry = buildHandlerRegistry({ mailProvider }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject: "Workspace invitation", html: `Accept`, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_corrupt", invitationTokenHash: "broken", issuanceKeyHash: "b".repeat(64), }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow(/contract is invalid/u); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("routes OWNER provider construction failures through terminal fail-closed cleanup", async () => { const token = "c".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const terminalizeOwnerInvitation = vi.fn(async () => ({ state: "invalidated" })); const updatePayload = vi.fn(async (_payload: unknown) => undefined); const registry = buildHandlerRegistry({ env: {}, terminalizeOwnerInvitation }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_provider_setup", invitationTokenHash, issuanceKeyHash: "b".repeat(64), outboxEventId: "outbox_provider_setup", dispatchKey: "mail:workspace-owner-invite:invite_provider_setup", mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload })).rejects.toThrow( /MAILJET_API_KEY/u ); expect(terminalizeOwnerInvitation).toHaveBeenCalledWith({ tenantId: "tenant_1", payload: descriptor.payload }); expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token); }); test("redacts and invalidates a permanently failed OWNER invitation before BullMQ dead-letters it", async () => { const token = "p".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const providerFailure = new Error(`provider permanently unavailable html=${html} token=${token}`); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(providerFailure)), }; const updatePayload = vi.fn(async (_payload: unknown) => undefined); const ownerDelivery = createOwnerDeliveryDeps(); const registry = buildHandlerRegistry({ mailProvider, ...ownerDelivery }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_terminal", invitationTokenHash, issuanceKeyHash: "b".repeat(64), mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); let thrown: unknown; try { await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload }); } catch (error) { thrown = error; } expect(thrown).toBeInstanceOf(Error); expect((thrown as Error).message).toContain("provider permanently unavailable"); expect((thrown as Error).message).not.toContain(token); expect((thrown as Error).message).not.toContain(html); expect(ownerDelivery.abandonOwnerInvitationDelivery).toHaveBeenCalledWith({ tenantId: "tenant_1", payload: descriptor.payload, lastError: expect.stringContaining("rotation is required"), }); expect(updatePayload).toHaveBeenCalledOnce(); expect(JSON.stringify(updatePayload.mock.calls[0]?.[0])).not.toContain(token); expect(updatePayload.mock.calls[0]?.[0]).toMatchObject({ metadata: expect.objectContaining({ secretBodyRedacted: true, ownerInvitationDeliveryStatus: "rotation_required" }), }); }); test("rearms the durable outbox marker when final Bull invalidation transiently fails", async () => { const token = "d".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const terminalizeOwnerInvitation = vi.fn(async () => Promise.reject(new Error("database unavailable"))); const scheduleOwnerInvitationRecovery = vi.fn(async () => undefined); const updatePayload = vi.fn(async (_payload: unknown) => undefined); const registry = buildHandlerRegistry({ mailProvider: { id: "mailjet", send: vi.fn(async () => Promise.reject(new Error("provider failed"))) }, terminalizeOwnerInvitation, scheduleOwnerInvitationRecovery, }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_recovery", invitationTokenHash, issuanceKeyHash: "b".repeat(64), outboxEventId: "outbox_recovery", dispatchKey: "mail:workspace-owner-invite:invite_recovery", mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload })).rejects.toThrow( /cleanup failed/u ); expect(scheduleOwnerInvitationRecovery).toHaveBeenCalledWith(expect.objectContaining({ tenantId: "tenant_1", outboxEventId: "outbox_recovery" })); expect(JSON.stringify(scheduleOwnerInvitationRecovery.mock.calls)).not.toContain(token); }); test("manually retrying a terminal-redacted Bull job retries invalidation instead of deduplicating it", async () => { const token = "e".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const terminalizeOwnerInvitation = vi.fn(async () => ({ state: "invalidated" })); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn() }; const registry = buildHandlerRegistry({ mailProvider, terminalizeOwnerInvitation }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_manual_retry", invitationTokenHash, issuanceKeyHash: "b".repeat(64), outboxEventId: "outbox_manual_retry", dispatchKey: "mail:workspace-owner-invite:invite_manual_retry", mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload; await expect(registry[descriptor.queueName](redacted, {})).resolves.toMatchObject({ details: { deduplicated: true } }); expect(terminalizeOwnerInvitation).toHaveBeenCalledWith({ tenantId: "tenant_1", payload: redacted }); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("retries durable recovery for a terminal-redacted Bull job after both final DB operations failed", async () => { const token = "f".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_recovery_retry", invitationTokenHash, issuanceKeyHash: "b".repeat(64), outboxEventId: "outbox_recovery_retry", dispatchKey: "mail:workspace-owner-invite:invite_recovery_retry", mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload; const scheduleOwnerInvitationRecovery = vi.fn(async () => undefined); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn() }; const registry = buildHandlerRegistry({ mailProvider, terminalizeOwnerInvitation: vi.fn(async () => Promise.reject(new Error("database still unavailable"))), scheduleOwnerInvitationRecovery, }); await expect(registry[descriptor.queueName](redacted, {})).resolves.toMatchObject({ details: { deduplicated: true } }); expect(scheduleOwnerInvitationRecovery).toHaveBeenCalledWith( expect.objectContaining({ tenantId: "tenant_1", outboxEventId: "outbox_recovery_retry", redactedPayload: redacted }) ); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("does not deduplicate a terminal-redacted Bull job while invalidation and scheduling still fail", async () => { const token = "g".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_recovery_blocked", invitationTokenHash, issuanceKeyHash: "b".repeat(64), outboxEventId: "outbox_recovery_blocked", dispatchKey: "mail:workspace-owner-invite:invite_recovery_blocked", mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload; const mailProvider: MailProvider = { id: "mailjet", send: vi.fn() }; const registry = buildHandlerRegistry({ mailProvider, terminalizeOwnerInvitation: vi.fn(async () => Promise.reject(new Error("database unavailable"))), scheduleOwnerInvitationRecovery: vi.fn(async () => Promise.reject(new Error("database unavailable"))), }); await expect(registry[descriptor.queueName](redacted, {})).rejects.toThrow(/recovery failed/u); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("never retries an OWNER provider attempt after its delivery claim became ambiguous", async () => { const token = "r".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(new Error(`provider temporarily unavailable html=${html} token=${token}`))), }; const updatePayload = vi.fn(async (_payload: unknown) => undefined); const ownerDelivery = createOwnerDeliveryDeps(); const registry = buildHandlerRegistry({ mailProvider, ...ownerDelivery }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_retry", invitationTokenHash, issuanceKeyHash: "b".repeat(64), mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); let thrown: unknown; try { await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3, updatePayload }); } catch (error) { thrown = error; } expect(thrown).toBeInstanceOf(Error); expect((thrown as Error).message).toContain("temporarily unavailable"); expect((thrown as Error).message).not.toContain(token); expect((thrown as Error).message).not.toContain(html); expect(ownerDelivery.abandonOwnerInvitationDelivery).toHaveBeenCalledOnce(); expect(updatePayload).toHaveBeenCalledOnce(); expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token); }); test("durably schedules terminal OWNER recovery when ambiguous-delivery invalidation keeps failing", async () => { const token = "q".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(new Error("provider outcome unavailable"))), }; const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_abandon_recovery", invitationTokenHash, issuanceKeyHash: "b".repeat(64), outboxEventId: "outbox_abandon_recovery", dispatchKey: "mail:workspace-owner-invite:invite_abandon_recovery", mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); const claimOwnerInvitationDelivery = vi .fn() .mockResolvedValueOnce({ state: "claimed", payload: markWorkspaceOwnerInvitationDeliveryStarted(descriptor.payload) }) .mockResolvedValueOnce({ state: "uncertain", payload: markWorkspaceOwnerInvitationDeliveryStarted(descriptor.payload) }); const abandonOwnerInvitationDelivery = vi.fn(async () => Promise.reject(new Error("database invalidation unavailable"))); const terminalizeOwnerInvitation = vi.fn(async () => Promise.reject(new Error("database terminalization unavailable"))); const scheduleOwnerInvitationRecovery = vi.fn(async () => undefined); const updatePayload = vi.fn(async (_payload: unknown) => undefined); const registry = buildHandlerRegistry({ mailProvider, claimOwnerInvitationDelivery, abandonOwnerInvitationDelivery, terminalizeOwnerInvitation, scheduleOwnerInvitationRecovery, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 2, updatePayload })).rejects.toThrow( /ambiguous delivery cleanup failed/u ); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 2, updatePayload })).rejects.toThrow( /cleanup failed/u ); expect(mailProvider.send).toHaveBeenCalledOnce(); expect(claimOwnerInvitationDelivery).toHaveBeenCalledTimes(2); expect(abandonOwnerInvitationDelivery).toHaveBeenCalledTimes(2); expect(terminalizeOwnerInvitation).toHaveBeenCalledOnce(); expect(scheduleOwnerInvitationRecovery).toHaveBeenCalledWith( expect.objectContaining({ tenantId: "tenant_1", outboxEventId: "outbox_abandon_recovery", redactedPayload: expect.objectContaining({ metadata: expect.objectContaining({ ownerInvitationDeliveryStatus: "rotation_required", secretBodyRedacted: true, }), }), }) ); expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token); expect(JSON.stringify(scheduleOwnerInvitationRecovery.mock.calls)).not.toContain(token); }); test("keeps ordinary MEMBER transactional-mail failures outside OWNER recovery", async () => { const providerFailure = new Error("member mail provider unavailable"); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(providerFailure)), }; const claimOwnerInvitationDelivery = vi.fn(); const abandonOwnerInvitationDelivery = vi.fn(); const terminalizeOwnerInvitation = vi.fn(); const scheduleOwnerInvitationRecovery = vi.fn(); const registry = buildHandlerRegistry({ mailProvider, claimOwnerInvitationDelivery, abandonOwnerInvitationDelivery, terminalizeOwnerInvitation, scheduleOwnerInvitationRecovery, }); const descriptor = buildTransactionalMailJob({ to: ["member@example.test"], subject: "Workspace invitation", html: "

Ordinary member invitation

", tenantId: "tenant_1", metadata: { source: "member-invitation" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload: vi.fn() })).rejects.toBe( providerFailure ); expect(mailProvider.send).toHaveBeenCalledOnce(); expect(claimOwnerInvitationDelivery).not.toHaveBeenCalled(); expect(abandonOwnerInvitationDelivery).not.toHaveBeenCalled(); expect(terminalizeOwnerInvitation).not.toHaveBeenCalled(); expect(scheduleOwnerInvitationRecovery).not.toHaveBeenCalled(); }); test("completes a redacted OWNER invitation retry without sending the mail twice", async () => { const token = "n".repeat(43); const invitationTokenHash = createHash("sha256").update(token).digest("hex"); const subject = "Workspace invitation"; const html = `Accept`; const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "unexpected" })), }; const registry = buildHandlerRegistry({ mailProvider }); const descriptor = buildTransactionalMailJob({ to: ["owner@example.test"], subject, html, tenantId: "tenant_1", metadata: { source: "workspace-owner-invitation", invitationId: "invite_3", invitationTokenHash, issuanceKeyHash: "b".repeat(64), mailContentHash: hashWorkspaceOwnerInvitationMailContent({ recipient: "owner@example.test", subject, html, rawToken: token, invitationTokenHash, }), }, }); const redactedPayload = redactWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload; await expect(registry[descriptor.queueName](redactedPayload, {})).resolves.toEqual({ handled: true, action: "mail.sent", details: { deduplicated: true }, }); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("preserves per-recipient transactional email template data", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_personalized" })), }; const registry = buildHandlerRegistry({ mailProvider }); const descriptor = buildTransactionalMailJob({ to: ["user@example.com", "admin@example.com"], subject: "Welcome to HiveGrid", templateId: "tmpl_welcome", templateData: { "user@example.com": { firstName: "Ada" }, "admin@example.com": { firstName: "Grace" }, }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ handled: true, action: "mail.sent", }); expect(mailProvider.send).toHaveBeenCalledWith({ to: ["user@example.com", "admin@example.com"], subject: "Welcome to HiveGrid", text: undefined, html: undefined, templateId: "tmpl_welcome", personalization: [ { email: "user@example.com", data: { firstName: "Ada" } }, { email: "admin@example.com", data: { firstName: "Grace" } }, ], }); }); test("creates Mailjet provider from explicit worker env", async () => { const fetchMock = vi.fn( async () => new Response(JSON.stringify({ Messages: [{ Status: "success", To: [{ Email: "user@example.com", MessageUUID: "msg_env" }] }] }), { status: 200, headers: { "content-type": "application/json" }, }) ); const registry = buildHandlerRegistry({ env: { MAILJET_API_KEY: "fixture-api-key", MAILJET_SECRET_KEY: "fixture-secret-key", MAILJET_FROM_EMAIL: "sender@example.com", MAILJET_FROM_NAME: "Fixture Sender", }, }); const originalFetch = globalThis.fetch; vi.stubGlobal("fetch", fetchMock); try { await expect( registry[queueNames.email]( { kind: "mail.transactional", to: ["user@example.com"], subject: "Hello", text: "Body", }, {} ) ).resolves.toEqual({ handled: true, action: "mail.sent", details: { provider: "mailjet", messageId: "msg_env" }, }); } finally { vi.stubGlobal("fetch", originalFetch); } expect(fetchMock).toHaveBeenCalledWith( "https://api.mailjet.com/v3.1/send", expect.objectContaining({ headers: expect.objectContaining({ Authorization: `Basic ${Buffer.from("fixture-api-key:fixture-secret-key", "utf8").toString("base64")}` }), body: expect.stringContaining("Fixture Sender"), }) ); }); test("reports missing Mailjet fixture env with key names only", async () => { const registry = buildHandlerRegistry({ env: { MAILJET_API_KEY: "super-secret-api-key" } }); await expect( registry[queueNames.email]( { kind: "mail.transactional", to: ["user@example.com"], subject: "Hello", text: "Body", }, {} ) ).rejects.toMatchObject({ message: "MAILJET_SECRET_KEY is required for Mailjet email delivery." }); await expect( registry[queueNames.email]( { kind: "mail.transactional", to: ["user@example.com"], subject: "Hello", text: "Body", }, {} ) ).rejects.not.toThrow("super-secret-api-key"); }); test("propagates provider rejection diagnostics without exposing payload secrets", async () => { const providerError = new MailProviderError("Mailjet rejected message", { provider: "mailjet", statusCode: 422, messageId: "msg_rejected", responseBody: { message: "Invalid recipient" }, }); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => { throw providerError; }), }; const registry = buildHandlerRegistry({ mailProvider }); const descriptor = buildTransactionalMailJob({ to: ["user@example.com"], subject: "Hello", text: "Body", templateData: { otp: "123456", apiKey: "payload-secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ message: "Mailjet rejected message", provider: "mailjet", statusCode: 422, messageId: "msg_rejected", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("payload-secret"); expect(mailProvider.send).toHaveBeenCalledTimes(2); }); test("notifies the responsible TenantUser after permanent portal status-mail delivery failure", async () => { const providerError = new MailProviderError("Mailjet unavailable", { provider: "mailjet", statusCode: 503 }); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(providerError)) }; const notifyPortalStatusMailFailure = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ mailProvider, notifyPortalStatusMailFailure }); const descriptor = buildTransactionalMailJob({ tenantId: "tenant_1", to: ["customer@example.test"], subject: "Status", text: "Body", metadata: { idempotencyKey: "portal-status:change_1:7f8b67d82471956e2fb6e2e39d4d9ae9", recipientKind: "portal_user", portalStatusMail: { version: 1, stageChangeId: "change_1", projectId: "project_1", expectedStageId: "stage_new", internalRecipientUserId: "tenant_user_1", }, }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).rejects.toThrow("Mailjet unavailable"); expect(notifyPortalStatusMailFailure).not.toHaveBeenCalled(); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("Mailjet unavailable"); expect(notifyPortalStatusMailFailure).toHaveBeenCalledWith({ tenantId: "tenant_1", projectId: "project_1", expectedStageId: "stage_new", stageChangeId: "change_1", internalRecipientUserId: "tenant_user_1", }); expect(JSON.stringify(notifyPortalStatusMailFailure.mock.calls)).not.toContain("portalUserId"); }); test("delivers queued webhook descriptors and updates bounded legacy attempt diagnostics", async () => { const fetchMock = vi.fn( async () => new Response("accepted Authorization: Bearer response-secret", { status: 202, statusText: "Accepted", headers: { "content-type": "text/plain" } }) ); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); const descriptor = { queueName: queueNames.email, jobName: "deliver-webhook", payload: { kind: "webhook.deliver", url: "https://example.com/webhook", method: "POST", eventName: "legacy.event_webhook_attempt", body: { id: "row_1" }, metadata: { legacyAttemptId: "attempt_1", legacyAttemptTenantId: "tenant_1" }, }, } satisfies WebhookDeliveryJobDescriptor; await expect(registry[queueNames.email](descriptor.payload, {})).resolves.toMatchObject({ handled: true, action: "webhook.delivered", }); expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ method: "POST", redirect: "manual" })); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_1", tenantId: "tenant_1" }, expect.objectContaining({ finishedAt: expect.any(Date), success: true, status: 202, message: "Accepted", body: expect.stringContaining('"status":202'), }) ); const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>; const body = updateCalls[0]?.[1]?.body ?? ""; expect(body).toContain('"bodyBytes":'); expect(body).not.toContain("response-secret"); expect(body).not.toContain("Authorization"); }); test("rejects unsafe queued webhook URLs as non-retryable before fetching", async () => { const fetchMock = vi.fn(async () => new Response(null, { status: 204 })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); const blockedUrls = [ "ftp://example.com/webhook", "https://localhost/webhook", "https://service.internal/webhook", "https://service/webhook", "http://127.0.0.1/webhook", "http://10.1.2.3/webhook", "http://169.254.169.254/latest/meta-data", "http://[::1]/webhook", "http://[fd00::1]/webhook", ]; for (const url of blockedUrls) { await expect( registry[queueNames.email]( { kind: "webhook.deliver", url, eventName: "entity.created", metadata: { legacyAttemptId: `attempt_${blockedUrls.indexOf(url)}`, legacyAttemptTenantId: "tenant_blocked" }, }, {} ) ).resolves.toMatchObject({ handled: true, action: "webhook.rejected", }); } expect(fetchMock).not.toHaveBeenCalled(); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledTimes(blockedUrls.length); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( expect.objectContaining({ tenantId: "tenant_blocked" }), expect.objectContaining({ finishedAt: expect.any(Date), success: false, message: expect.stringMatching(/Webhook endpoint/), }) ); }); test("rejects public webhook hostnames that resolve to private addresses before fetching", async () => { const fetchMock = vi.fn(async () => new Response(null, { status: 204 })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const privateAddressResolver = vi.fn(async () => ["169.254.169.254"]); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: privateAddressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://public.example.test/latest/meta-data", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_dns_blocked", legacyAttemptTenantId: "tenant_dns" }, }, {} ) ).resolves.toMatchObject({ handled: true, action: "webhook.rejected", }); expect(privateAddressResolver).toHaveBeenCalledWith("public.example.test"); expect(fetchMock).not.toHaveBeenCalled(); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_dns_blocked", tenantId: "tenant_dns" }, expect.objectContaining({ finishedAt: expect.any(Date), success: false, message: "Webhook endpoint resolved to a blocked address.", }) ); }); test("follows queued webhook redirects only after validating the target", async () => { const fetchMock = vi .fn() .mockResolvedValueOnce(new Response(null, { status: 307, headers: { location: "https://hooks.example.com/next" } })) .mockResolvedValueOnce(new Response("accepted", { status: 202, statusText: "Accepted" })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://example.com/webhook", method: "POST", headers: { authorization: "Bearer customer-secret", "x-hivegrid-signature": "sha256=customer-signature", "x-delivery-id": "delivery-1", }, eventName: "entity.created", metadata: { legacyAttemptId: "attempt_redirect", legacyAttemptTenantId: "tenant_redirect" }, }, {} ) ).resolves.toMatchObject({ handled: true, action: "webhook.delivered", }); expect(fetchMock).toHaveBeenNthCalledWith(1, "https://example.com/webhook", expect.objectContaining({ method: "POST", redirect: "manual" })); expect(fetchMock).toHaveBeenNthCalledWith(2, "https://hooks.example.com/next", expect.objectContaining({ method: "POST", redirect: "manual" })); const redirectedHeaders = new Headers(fetchMock.mock.calls[1]?.[1]?.headers); expect(redirectedHeaders.get("authorization")).toBeNull(); expect(redirectedHeaders.get("x-hivegrid-signature")).toBeNull(); expect(redirectedHeaders.get("x-delivery-id")).toBe("delivery-1"); expect(redirectedHeaders.get("content-type")).toBe("application/json"); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_redirect", tenantId: "tenant_redirect" }, expect.objectContaining({ success: true, status: 202 }) ); }); test("rejects queued webhook redirect hostnames that resolve to private addresses before following them", async () => { const fetchMock = vi.fn(async () => new Response(null, { status: 302, headers: { location: "https://metadata.example.test/latest" } })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const addressResolver = vi.fn(async (hostname: string) => (hostname === "metadata.example.test" ? ["10.0.0.10"] : ["93.184.216.34"])); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: addressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_redirect_dns_blocked", legacyAttemptTenantId: "tenant_redirect" }, }, {}, { attemptsMade: 0, attemptsLimit: 3 } ) ).resolves.toMatchObject({ handled: true, action: "webhook.rejected", }); expect(fetchMock).toHaveBeenCalledTimes(1); expect(addressResolver).toHaveBeenCalledWith("example.com"); expect(addressResolver).toHaveBeenCalledWith("metadata.example.test"); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_redirect_dns_blocked", tenantId: "tenant_redirect" }, expect.objectContaining({ finishedAt: expect.any(Date), success: false, message: "Webhook endpoint resolved to a blocked address.", }) ); }); test("rejects queued webhook redirects to unsafe targets without fetching them", async () => { const fetchMock = vi.fn(async () => new Response(null, { status: 302, headers: { location: "http://127.0.0.1/webhook" } })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_redirect_blocked", legacyAttemptTenantId: "tenant_redirect" }, }, {}, { attemptsMade: 0, attemptsLimit: 3 } ) ).resolves.toMatchObject({ handled: true, action: "webhook.rejected", }); expect(fetchMock).toHaveBeenCalledTimes(1); expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ redirect: "manual" })); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_redirect_blocked", tenantId: "tenant_redirect" }, expect.objectContaining({ finishedAt: expect.any(Date), success: false, message: "Webhook endpoint host is not allowed.", }) ); }); test("updates failed legacy attempt diagnostics and rejects non-2xx webhook responses", async () => { const fetchMock = vi.fn(async () => new Response("upstream rejected token=upstream-secret body=payload", { status: 503, statusText: "Unavailable" })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_2", legacyAttemptTenantId: "tenant_2" }, }, {} ) ).rejects.toThrow("Webhook delivery failed with 503."); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledTimes(1); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_2", tenantId: "tenant_2" }, expect.objectContaining({ finishedAt: expect.any(Date), success: false, status: 503, message: "Webhook delivery failed with 503.", body: expect.stringContaining('"status":503'), }) ); const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>; const body = updateCalls[0]?.[1]?.body ?? ""; expect(body).toContain('"bodyBytes":'); expect(body).not.toContain("upstream-secret"); expect(body).not.toContain("payload"); }); test("keeps queued webhook attempt diagnostics open until retries are exhausted", async () => { const fetchMock = vi.fn(async () => new Response("temporarily unavailable", { status: 503, statusText: "Unavailable" })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); const payload = { kind: "webhook.deliver" as const, url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_retry", legacyAttemptTenantId: "tenant_retry" }, }; await expect(registry[queueNames.email](payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow("Webhook delivery failed with 503."); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_retry", tenantId: "tenant_retry" }, expect.not.objectContaining({ finishedAt: expect.any(Date), success: false, }) ); expect(updateLegacyEventWebhookAttempt).toHaveBeenLastCalledWith( { id: "attempt_retry", tenantId: "tenant_retry" }, expect.objectContaining({ status: 503, message: "Webhook delivery failed with 503." }) ); await expect(registry[queueNames.email](payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("Webhook delivery failed with 503."); expect(updateLegacyEventWebhookAttempt).toHaveBeenLastCalledWith( { id: "attempt_retry", tenantId: "tenant_retry" }, expect.objectContaining({ finishedAt: expect.any(Date), success: false, status: 503, }) ); }); test("does not retry permanent webhook client failures", async () => { const fetchMock = vi.fn(async () => new Response("missing", { status: 404, statusText: "Not Found" })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); const payload = { kind: "webhook.deliver" as const, url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_permanent", legacyAttemptTenantId: "tenant_permanent" }, }; await expect(registry[queueNames.email](payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({ handled: true, action: "webhook.failed", details: { status: 404, eventName: "entity.created", retryable: false }, }); expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith( { id: "attempt_permanent", tenantId: "tenant_permanent" }, expect.objectContaining({ finishedAt: expect.any(Date), success: false, status: 404, message: "Webhook delivery failed with 404.", }) ); }); test("caps webhook response diagnostics before persisting legacy attempt bodies", async () => { const oversizedBody = `${"a".repeat(4_096)}token=tail-secret`; const fetchMock = vi.fn(async () => new Response(oversizedBody, { status: 502, statusText: "Bad Gateway" })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_capped", legacyAttemptTenantId: "tenant_capped" }, }, {} ) ).rejects.toThrow("Webhook delivery failed with 502."); expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ signal: expect.any(AbortSignal) })); const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>; const body = updateCalls[0]?.[1]?.body ?? ""; expect(body).toContain('"bodyBytes":4096'); expect(body).not.toContain("tail-secret"); }); test("updates redacted failed legacy attempt diagnostics when webhook fetch throws", async () => { const fetchError = new Error("network unavailable token=fetch-secret Authorization: Bearer bearer-secret"); const fetchMock = vi.fn(async () => { throw fetchError; }); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: "attempt_3", legacyAttemptTenantId: "tenant_3" }, }, {} ) ).rejects.toThrow(fetchError); const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { message?: string }]>; expect(updateCalls[0]?.[0]).toEqual({ id: "attempt_3", tenantId: "tenant_3" }); expect(updateCalls[0]?.[1]).toEqual( expect.objectContaining({ finishedAt: expect.any(Date), success: false, message: expect.stringContaining("network unavailable token=[redacted] Authorization: [redacted]"), }) ); expect(updateCalls[0]?.[1]?.message).not.toContain("fetch-secret"); expect(updateCalls[0]?.[1]?.message).not.toContain("bearer-secret"); }); test("does not update legacy attempts for generic webhook jobs without attempt metadata", async () => { const fetchMock = vi.fn(async () => new Response(null, { status: 204 })); const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver }); await expect( registry[queueNames.email]( { kind: "webhook.deliver", url: "https://example.com/webhook", eventName: "entity.created", metadata: { legacyAttemptId: 123 }, }, {} ) ).resolves.toMatchObject({ handled: true, action: "webhook.delivered", }); expect(updateLegacyEventWebhookAttempt).not.toHaveBeenCalled(); }); test("sends opted-in email notification jobs and marks delivery diagnostics sent", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_notification_1" })), }; const getUserNotificationPreferences = vi.fn(async () => []); const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_1" })); const markNotificationDeliveryStatus = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery, markNotificationDeliveryStatus }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing", email: "user@example.com", messageKey: "billing.invoice-ready", subject: "Invoice ready", text: "Your invoice is ready", templateData: { secretToken: "not-diagnostic" }, metadata: { requestId: "req_123", source: "billing", apiKey: "secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ handled: true, action: "notification.email.sent", details: { provider: "mailjet", messageId: "msg_notification_1" }, }); expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing" }); expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "default" }); expect(createNotificationDelivery).toHaveBeenCalledWith( expect.objectContaining({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing", messageKey: "billing.invoice-ready", sourceKey: "request:req_123", status: "PENDING", attemptsMade: 0, payloadMetadata: { requestId: "req_123", source: "billing" }, }) ); expect(mailProvider.send).toHaveBeenCalledWith({ to: ["user@example.com"], subject: "Invoice ready", text: "Your invoice is ready", html: undefined, templateId: undefined, idempotencyKey: "notification-delivery:delivery_1", personalization: [{ email: "user@example.com", data: { secretToken: "not-diagnostic" } }], }); expect(markNotificationDeliveryStatus).toHaveBeenCalledWith( expect.objectContaining({ id: "delivery_1", tenantId: "tenant_123", status: "SENT", attemptsMade: 1, providerMessageId: "msg_notification_1", sentAt: expect.any(Date), }) ); }); test("uses worker job id as a stable notification delivery source key when payload persistence fails", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_retry_success" })), }; const deliveries = new Map(); const createNotificationDelivery = vi.fn(async (input: CreateNotificationDeliveryInput) => { const key = input.sourceKey ?? "none"; const existing = deliveries.get(key); if (existing) { return existing; } const delivery = { id: `delivery_${deliveries.size + 1}` }; deliveries.set(key, delivery); return delivery; }); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences: vi.fn(async () => []), createNotificationDelivery, markNotificationDeliveryStatus: vi.fn(async () => undefined), }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", }); await expect( registry[descriptor.queueName]( descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3, jobId: "job_123", updatePayload: vi.fn(async () => { throw new Error("job data write failed"); }), } ) ).rejects.toThrow("job data write failed"); await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3, jobId: "job_123" }); expect(createNotificationDelivery).toHaveBeenCalledTimes(2); expect(createNotificationDelivery).toHaveBeenCalledWith(expect.objectContaining({ sourceKey: "job:job_123" })); expect(mailProvider.send).toHaveBeenCalledTimes(1); expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:delivery_1" })); }); test("persists delivery id in notification job metadata for retry-safe diagnostics", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_notification_1" })), }; const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_retry" })); const markNotificationDeliveryStatus = vi.fn(async () => undefined); const updatePayload = vi.fn(async (_payload: unknown) => undefined); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences: vi.fn(async () => []), createNotificationDelivery, markNotificationDeliveryStatus, }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", }); await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3, updatePayload }); expect(updatePayload).toHaveBeenCalledWith({ ...descriptor.payload, metadata: { deliveryId: "delivery_retry" }, }); expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(expect.objectContaining({ id: "delivery_retry", status: "SENT", attemptsMade: 1 })); }); test("reuses notification delivery id on email retry without creating duplicate diagnostics", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_retry_success" })), }; const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_new" })); const markNotificationDeliveryStatus = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences: vi.fn(async () => []), createNotificationDelivery, markNotificationDeliveryStatus, }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", metadata: { deliveryId: "delivery_existing" }, }); await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 }); expect(createNotificationDelivery).not.toHaveBeenCalled(); expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:delivery_existing" })); expect(markNotificationDeliveryStatus).toHaveBeenCalledWith( expect.objectContaining({ id: "delivery_existing", tenantId: "tenant_123", status: "SENT", attemptsMade: 2, providerMessageId: "msg_retry_success" }) ); }); test("keeps email delivery pending until the terminal notification retry fails", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => { throw new Error("provider token=secret failed"); }), }; const markNotificationDeliveryStatus = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences: vi.fn(async () => []), createNotificationDelivery: vi.fn(async () => ({ id: "delivery_existing" })), markNotificationDeliveryStatus, }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", metadata: { deliveryId: "delivery_existing" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).rejects.toThrow("provider"); expect(markNotificationDeliveryStatus).toHaveBeenLastCalledWith({ id: "delivery_existing", tenantId: "tenant_123", status: "PENDING", attemptsMade: 2, lastError: "provider token=[redacted] failed", }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("provider"); expect(markNotificationDeliveryStatus).toHaveBeenLastCalledWith({ id: "delivery_existing", tenantId: "tenant_123", status: "FAILED", attemptsMade: 3, lastError: "provider token=[redacted] failed", }); }); test("does not retry an already-sent email when sent status persistence fails", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_sent_once" })), }; const markNotificationDeliveryStatus = vi.fn(async () => { throw new Error("database token=secret unavailable"); }); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences: vi.fn(async () => []), createNotificationDelivery: vi.fn(async () => ({ id: "delivery_sent_once" })), markNotificationDeliveryStatus, }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", metadata: { deliveryId: "delivery_sent_once" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({ handled: true, action: "notification.email.sent.diagnostic_failed", details: { provider: "mailjet", messageId: "msg_sent_once", userId: "user_123", messageKey: "billing.invoice-ready", deliveryId: "delivery_sent_once", diagnosticError: "database token=[redacted] unavailable", }, }); expect(mailProvider.send).toHaveBeenCalledTimes(1); }); test("skips opted-out email notification jobs without calling mail provider", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_skip" })); const markNotificationDeliveryStatus = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences: vi.fn(async () => [{ enabled: false }]), createNotificationDelivery, markNotificationDeliveryStatus, }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "notification.email.skipped", details: { userId: "user_123", channel: "email", messageKey: "billing.invoice-ready", deliveryId: "delivery_skip", reason: "preference_disabled", }, }); expect(mailProvider.send).not.toHaveBeenCalled(); expect(createNotificationDelivery).toHaveBeenCalledWith( expect.objectContaining({ tenantId: "tenant_123", userId: "user_123", channel: "email", messageKey: "billing.invoice-ready", status: "SKIPPED", lastError: "preference_disabled", payloadMetadata: { reason: "preference_disabled" }, }) ); expect(markNotificationDeliveryStatus).not.toHaveBeenCalled(); }); test("delivers a critical-category email even when a preference row disables it", async () => { // AK-P21: GDPR export/deletion notices ride the locked "critical" category. // foundation's isNotificationEnabled() never suppresses it, and the queue // path must not either — otherwise a stale row silences a legal notice. const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const getUserNotificationPreferences = vi.fn(async () => [{ enabled: false }]); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery: vi.fn(async () => ({ id: "delivery_critical" })), markNotificationDeliveryStatus: vi.fn(async () => undefined), }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "critical", email: "user@example.com", messageKey: "settings.privacy.notify.exportReady", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ handled: true, action: "notification.email.sent", }); expect(mailProvider.send).toHaveBeenCalledTimes(1); // The lookup is skipped outright, so no stale row can reach the gate. expect(getUserNotificationPreferences).not.toHaveBeenCalled(); }); test("lets explicit category notification preferences override disabled defaults", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_category_enabled" })), }; const getUserNotificationPreferences = vi.fn().mockResolvedValueOnce([{ enabled: true, category: "billing" }]); const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_category_enabled" })); const markNotificationDeliveryStatus = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery, markNotificationDeliveryStatus }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing", email: "user@example.com", messageKey: "billing.invoice-ready", subject: "Invoice ready", text: "Your invoice is ready", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ handled: true, action: "notification.email.sent", details: { provider: "mailjet", messageId: "msg_category_enabled" }, }); expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing" }); expect(getUserNotificationPreferences).toHaveBeenCalledTimes(1); expect(mailProvider.send).toHaveBeenCalledTimes(1); expect(markNotificationDeliveryStatus).toHaveBeenCalledWith( expect.objectContaining({ id: "delivery_category_enabled", tenantId: "tenant_123", status: "SENT" }) ); }); test("records skipped diagnostics for opted-out in-app notification jobs", async () => { const getUserNotificationPreferences = vi.fn(async () => [{ enabled: false }]); const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_in_app_skip" })); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "in-app", category: "billing", messageKey: "billing.invoice-ready", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "notification.in-app.skipped", details: { userId: "user_123", channel: "in-app", messageKey: "billing.invoice-ready", deliveryId: "delivery_in_app_skip", reason: "preference_disabled", }, }); expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "in-app", category: "billing", }); expect(getUserNotificationPreferences).toHaveBeenCalledTimes(1); expect(createNotificationDelivery).toHaveBeenCalledWith( expect.objectContaining({ tenantId: "tenant_123", userId: "user_123", channel: "in-app", category: "billing", messageKey: "billing.invoice-ready", status: "SKIPPED", lastError: "preference_disabled", payloadMetadata: { reason: "preference_disabled" }, }) ); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("marks email notification diagnostics failed when provider rejects", async () => { const providerError = new Error("SMTP token=secret-token rejected for user@example.com with Authorization: Bearer abc123 and payload-secret"); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => { throw providerError; }), }; const markNotificationDeliveryStatus = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences: vi.fn(async () => [{ enabled: true }]), createNotificationDelivery: vi.fn(async () => ({ id: "delivery_failed" })), markNotificationDeliveryStatus, }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", templateData: { apiKey: "payload-secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toBe(providerError); expect(markNotificationDeliveryStatus).toHaveBeenCalledWith({ id: "delivery_failed", tenantId: "tenant_123", status: "FAILED", attemptsMade: 1, lastError: "SMTP token=[redacted] rejected for [redacted-email] with Authorization: Bearer [redacted] and payload-[redacted]", }); }); test("marks email notification diagnostics failed when provider setup rejects", async () => { const markNotificationDeliveryStatus = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ env: {}, getUserNotificationPreferences: vi.fn(async () => [{ enabled: true }]), createNotificationDelivery: vi.fn(async () => ({ id: "delivery_setup_failed" })), markNotificationDeliveryStatus, }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "billing.invoice-ready", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("MAILJET_API_KEY is required"); expect(markNotificationDeliveryStatus).toHaveBeenCalledWith({ id: "delivery_setup_failed", tenantId: "tenant_123", status: "FAILED", attemptsMade: 1, lastError: "MAILJET_API_KEY is required for Mailjet email delivery.", }); }); test("rejects email notification jobs missing tenant or email before diagnostics", async () => { const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_missing" })); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const registry = buildHandlerRegistry({ mailProvider, createNotificationDelivery }); await expect( registry[queueNames.notifications]( { kind: "notification.send", userId: "user_123", channel: "email", email: "user@example.com", messageKey: "account.updated", } as Parameters<(typeof registry)[typeof queueNames.notifications]>[0], {} ) ).rejects.toThrow("Notification jobs require tenantId."); await expect( registry[queueNames.notifications]( buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", messageKey: "account.updated" }).payload, {} ) ).rejects.toThrow("Email notifications require email recipient."); expect(createNotificationDelivery).not.toHaveBeenCalled(); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("rejects unsupported notification channels before diagnostics", async () => { const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_sms" })); const registry = buildHandlerRegistry({ createNotificationDelivery }); await expect( registry[queueNames.notifications]( { kind: "notification.send", tenantId: "tenant_123", userId: "user_123", channel: "sms", messageKey: "account.updated", } as unknown as Parameters<(typeof registry)[typeof queueNames.notifications]>[0], {} ) ).rejects.toThrow("Unsupported notification channel: sms"); expect(createNotificationDelivery).not.toHaveBeenCalled(); }); test("accepts descriptor-derived non-email notification jobs through registry", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_in_app" })); const registry = buildHandlerRegistry({ mailProvider, createNotificationDelivery, getUserNotificationPreferences: vi.fn(async () => []) }); const descriptor = buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "in-app", messageKey: "account.updated", templateData: { ignoredBody: "not surfaced" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "notification.accepted", details: { userId: "user_123", channel: "in-app", messageKey: "account.updated", deliveryId: "delivery_in_app", }, }); expect(createNotificationDelivery).toHaveBeenCalledWith( expect.objectContaining({ userId: "user_123", channel: "in-app", messageKey: "account.updated", status: "SENT", attemptsMade: 1, sentAt: expect.any(Date), }) ); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("processes tenantless platform in-app delivery through the notification worker", async () => { const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "PENDING" })); const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 })); const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ eligible: true, email: null })); const getUserNotificationPreferences = vi.fn(async () => []); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const registry = buildHandlerRegistry({ createPlatformNotificationDelivery, markPlatformNotificationDeliveryStatus, getUserNotificationPreferences, mailProvider, resolvePlatformLeadNotificationRecipient, }); const descriptor = buildPlatformNotificationJob({ userId: "user_haris", sourceKey: "platform-lead:lead_1:in-app", channel: "in-app", messageKey: "platform.leads.captured", category: "leads", metadata: { leadId: "lead_1", sourceId: "hivegrid-website", deeplink: "/leads/lead_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({ handled: true, action: "notification.platform.in-app.sent", details: { deliveryId: "platform_delivery_1", userId: "user_haris", channel: "in-app", messageKey: "platform.leads.captured", }, }); expect(createPlatformNotificationDelivery).toHaveBeenCalledWith({ userId: "user_haris", channel: "IN_APP", category: "leads", messageKey: "platform.leads.captured", sourceKey: "platform-lead:lead_1:in-app", payloadMetadata: { leadId: "lead_1", sourceId: "hivegrid-website", deeplink: "/leads/lead_1" }, }); expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({ id: "platform_delivery_1", userId: "user_haris", status: "SENT", attemptsMade: 1, sentAt: expect.any(Date), }); expect(getUserNotificationPreferences).not.toHaveBeenCalled(); expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledWith({ userId: "user_haris", leadId: "lead_1" }); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("delivers task reminders without coupling them to the lead recipient capability", async () => { const createAdminPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_task_delivery_1", status: "PENDING", })); const markAdminPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1, })); const resolvePlatformLeadNotificationRecipient = vi.fn(); const registry = buildHandlerRegistry({ createAdminPlatformNotificationDelivery, markAdminPlatformNotificationDeliveryStatus, resolvePlatformLeadNotificationRecipient, }); const descriptor = buildPlatformNotificationJob({ adminUserId: "admin_1", sourceKey: "admin-sales:task-reminder:task_1:2026-07-25", channel: "in-app", messageKey: "admin.sales.notifications.taskreminder", category: "sales", metadata: { taskId: "task_1", deeplink: "/sales/tasks/today", }, }); await expect( registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3, }) ).resolves.toMatchObject({ action: "notification.platform.in-app.sent", details: { deliveryId: "platform_task_delivery_1", userId: "admin_1", }, }); expect(resolvePlatformLeadNotificationRecipient).not.toHaveBeenCalled(); expect(createAdminPlatformNotificationDelivery).toHaveBeenCalledWith( expect.objectContaining({ adminUserId: "admin_1", messageKey: "admin.sales.notifications.taskreminder", }) ); expect(markAdminPlatformNotificationDeliveryStatus).toHaveBeenCalledWith( expect.objectContaining({ id: "platform_task_delivery_1", adminUserId: "admin_1", status: "SENT", }) ); }); test("skips platform in-app delivery when current lead access was revoked", async () => { const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 })); const registry = buildHandlerRegistry({ createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_revoked", status: "PENDING" })), markPlatformNotificationDeliveryStatus, resolvePlatformLeadNotificationRecipient: vi.fn(async () => ({ eligible: false, email: null })), }); const descriptor = buildPlatformNotificationJob({ userId: "user_former_operator", sourceKey: "platform-lead:lead_1:in-app", channel: "in-app", messageKey: "platform.leads.captured", metadata: { leadId: "lead_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({ handled: true, action: "notification.platform.in-app.skipped", details: { deliveryId: "platform_delivery_revoked", userId: "user_former_operator", channel: "in-app", messageKey: "platform.leads.captured", reason: "recipient_ineligible", }, }); expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({ id: "platform_delivery_revoked", userId: "user_former_operator", status: "SKIPPED", attemptsMade: 1, lastError: "Platform lead notification recipient is no longer eligible.", }); }); test("sends platform email with provider idempotency and PII-minimized diagnostics", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_platform_1" })), }; const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "PENDING" })); const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 })); const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ email: "current-haris@example.test" })); const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery, markPlatformNotificationDeliveryStatus, resolvePlatformLeadNotificationRecipient, }); const descriptor = buildPlatformNotificationJob({ userId: "user_haris", sourceKey: "platform-lead:lead_1:email", channel: "email", email: "haris@example.test", subject: "Neuer Lead in HiveGrid", text: "Ein neuer Lead ist eingegangen.", messageKey: "platform.leads.captured", metadata: { leadId: "lead_1", sourceId: "hivegrid-website" }, }); const result = await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 }); expect(result).toEqual({ handled: true, action: "notification.platform.email.sent", details: { provider: "mailjet", messageId: "msg_platform_1", userId: "user_haris", messageKey: "platform.leads.captured", deliveryId: "platform_delivery_1", }, }); expect(mailProvider.send).toHaveBeenCalledWith( expect.objectContaining({ to: ["current-haris@example.test"], idempotencyKey: "notification-delivery:platform_delivery_1", }) ); expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledWith({ userId: "user_haris", leadId: "lead_1" }); expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({ id: "platform_delivery_1", userId: "user_haris", status: "SENT", attemptsMade: 1, providerMessageId: "msg_platform_1", sentAt: expect.any(Date), }); expect(JSON.stringify(result)).not.toContain("haris@example.test"); }); test("retries a platform email when SENT persistence fails after provider acceptance", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_platform_retry" })), }; const markPlatformNotificationDeliveryStatus = vi.fn(async () => { throw new Error("database token=secret unavailable"); }); const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_retry", status: "PENDING" })), markPlatformNotificationDeliveryStatus, resolvePlatformLeadNotificationRecipient: vi.fn(async () => ({ email: "haris@example.test" })), }); const descriptor = buildPlatformNotificationJob({ userId: "user_haris", sourceKey: "platform-lead:lead_retry:email", channel: "email", email: "haris@example.test", messageKey: "platform.leads.captured", metadata: { leadId: "lead_retry" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow( "Platform notification delivery status persistence failed: database token=[redacted] unavailable" ); expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:platform_delivery_retry" })); }); test("keeps platform email pending until terminal retry and redacts the failure", async () => { const providerError = new Error("SMTP token=secret-token rejected for haris@example.test"); const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => { throw providerError; }), }; const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "PENDING" })); const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 })); const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ email: "haris@example.test" })); const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery, markPlatformNotificationDeliveryStatus, resolvePlatformLeadNotificationRecipient, }); const descriptor = buildPlatformNotificationJob({ userId: "user_haris", sourceKey: "platform-lead:lead_1:email", channel: "email", email: "haris@example.test", messageKey: "platform.leads.captured", metadata: { leadId: "lead_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow( "SMTP token=[redacted] rejected for [redacted-email]" ); expect(markPlatformNotificationDeliveryStatus).toHaveBeenLastCalledWith({ id: "platform_delivery_1", userId: "user_haris", status: "PENDING", attemptsMade: 1, lastError: "SMTP token=[redacted] rejected for [redacted-email]", }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow( "SMTP token=[redacted] rejected for [redacted-email]" ); expect(markPlatformNotificationDeliveryStatus).toHaveBeenLastCalledWith({ id: "platform_delivery_1", userId: "user_haris", status: "FAILED", attemptsMade: 3, lastError: "SMTP token=[redacted] rejected for [redacted-email]", }); expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledTimes(2); }); test("skips platform lead email when the operator was demoted, deactivated, or lost record access", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 })); const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ email: null })); const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_ineligible", status: "PENDING" })), markPlatformNotificationDeliveryStatus, resolvePlatformLeadNotificationRecipient, }); const descriptor = buildPlatformNotificationJob({ userId: "user_former_operator", sourceKey: "platform-lead:lead_1:email", channel: "email", email: "stale-address@example.test", messageKey: "platform.leads.captured", metadata: { leadId: "lead_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).resolves.toEqual({ handled: true, action: "notification.platform.email.skipped", details: { deliveryId: "platform_delivery_ineligible", userId: "user_former_operator", channel: "email", messageKey: "platform.leads.captured", reason: "recipient_ineligible", }, }); expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledWith({ userId: "user_former_operator", leadId: "lead_1" }); expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({ id: "platform_delivery_ineligible", userId: "user_former_operator", status: "SKIPPED", attemptsMade: 2, lastError: "Platform lead notification recipient is no longer eligible.", }); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("retries without sending when the recipient revalidation capability is unavailable", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 })); const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_retry_resolution", status: "PENDING" })), markPlatformNotificationDeliveryStatus, resolvePlatformLeadNotificationRecipient: vi.fn(async () => { throw new Error("recipient capability unavailable"); }), }); const descriptor = buildPlatformNotificationJob({ userId: "user_haris", sourceKey: "platform-lead:lead_1:email", channel: "email", email: "stale-address@example.test", messageKey: "platform.leads.captured", metadata: { leadId: "lead_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow( "recipient capability unavailable" ); expect(mailProvider.send).not.toHaveBeenCalled(); expect(markPlatformNotificationDeliveryStatus).not.toHaveBeenCalled(); }); test("deduplicates an already-sent platform delivery without calling provider", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "SENT" })); const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 })); const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery, markPlatformNotificationDeliveryStatus }); const descriptor = buildPlatformNotificationJob({ userId: "user_haris", sourceKey: "platform-lead:lead_1:email", channel: "email", email: "haris@example.test", messageKey: "platform.leads.captured", metadata: { deliveryId: "platform_delivery_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "notification.platform.deduplicated", details: { deliveryId: "platform_delivery_1" }, }); expect(mailProvider.send).not.toHaveBeenCalled(); expect(markPlatformNotificationDeliveryStatus).not.toHaveBeenCalled(); }); test("rejects a queued delivery id that does not match the durable logical intent", async () => { const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })), }; const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_actual", status: "PENDING" })); const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery }); const descriptor = buildPlatformNotificationJob({ userId: "user_haris", sourceKey: "platform-lead:lead_1:email", channel: "email", email: "haris@example.test", messageKey: "platform.leads.captured", metadata: { deliveryId: "platform_delivery_queued" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("does not match the queued delivery id"); expect(mailProvider.send).not.toHaveBeenCalled(); }); test("rejects malformed platform notification scope before delivery", async () => { const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1" })); const registry = buildHandlerRegistry({ createPlatformNotificationDelivery }); await expect( registry[queueNames.notifications]( { kind: "notification.platform.send", tenantId: "tenant_escape", userId: "user_haris", sourceKey: "platform-lead:lead_1:in-app", channel: "in-app", messageKey: "platform.leads.captured", } as never, {} ) ).rejects.toThrow("Platform notification jobs require tenantId=null"); expect(createPlatformNotificationDelivery).not.toHaveBeenCalled(); }); test("rejects unsupported cleanup jobs through registry", async () => { const registry = buildHandlerRegistry(); const descriptor = buildCleanupJob({ task: "expired-sessions", olderThanHours: 24, requestedBy: "system-fixture", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "worker.unsupported", message: "Unsupported maintenance task.", details: { queueName: queueNames.maintenance, task: "expired-sessions", olderThanHours: 24, }, }); }); test("runs outbox DONE retention cleanup through maintenance handler", async () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-05-28T08:00:00.000Z")); const deleteDoneOutboxEvents = vi.fn(async (input: { cutoff: Date; take?: number }) => ({ deleted: 17, cutoff: input.cutoff, take: input.take ?? 1000, })); const registry = buildHandlerRegistry({ deleteDoneOutboxEvents }); const descriptor = buildCleanupJob({ task: "outbox-done-retention", olderThanHours: 48, take: 500, metadata: { requestId: "req_outbox_retention_1" }, }); try { await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "outbox.done-retention.completed", details: { task: "outbox-done-retention", requestId: "req_outbox_retention_1", olderThanHours: 48, cutoff: "2026-05-26T08:00:00.000Z", take: 500, deleted: 17, }, }); } finally { vi.useRealTimers(); } expect(deleteDoneOutboxEvents).toHaveBeenCalledWith({ cutoff: new Date("2026-05-26T08:00:00.000Z"), take: 500, }); }); test("runs workflow stalled execution reconciliation through maintenance handler", async () => { const observedScopes: Array = []; const reconcileWorkflowExecutionStalls = vi.fn(async () => { const tenantId = getRlsExecutionContext()?.tenantId ?? null; observedScopes.push(tenantId); return { cutoff: new Date("2026-05-29T02:30:00.000Z"), take: 100, inspected: 1, failedOutboxEvents: tenantId === "tenant_b" ? 0 : 1, repaired: tenantId === null ? 0 : 1, skippedWithoutFailedOutbox: tenantId === null ? 1 : 0, updateMisses: 0, }; }); const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(["tenant_a", "tenant_b"]), reconcileWorkflowExecutionStalls, }); const descriptor = buildCleanupJob({ task: "workflow-execution-stalled-reconcile", olderThanMinutes: 30, take: 100, metadata: { requestId: "req_workflow_stalled_reconcile_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "workflow.execution-stalled-reconcile.completed", details: { task: "workflow-execution-stalled-reconcile", requestId: "req_workflow_stalled_reconcile_1", olderThanMinutes: 30, cutoff: "2026-05-29T02:30:00.000Z", take: 100, inspected: 3, failedOutboxEvents: 2, repaired: 2, skippedWithoutFailedOutbox: 1, updateMisses: 0, }, }); expect(reconcileWorkflowExecutionStalls).toHaveBeenCalledTimes(3); expect(reconcileWorkflowExecutionStalls).toHaveBeenNthCalledWith(1, { olderThanMinutes: 30, take: 100, }); expect(reconcileWorkflowExecutionStalls).toHaveBeenNthCalledWith(2, { olderThanMinutes: 30, take: 100, }); expect(reconcileWorkflowExecutionStalls).toHaveBeenNthCalledWith(3, { olderThanMinutes: 30, take: 100, }); expect(observedScopes).toEqual(["tenant_a", "tenant_b", null]); }); test("acknowledges an overlapping tenant maintenance sweep without touching the database", async () => { const store = createSharedStore({ namespace: "worker-handler-lease-test", adapter: new MemoryAdapter() }); await store.set("job-lease:task-archive-retention", "other-replica", { ttlSeconds: 120 }); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, logger }); const descriptor = buildCleanupJob({ task: "task-archive-retention" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "maintenance.tenant-sweep.overlap-skipped", details: { task: "task-archive-retention", reason: "active-lease" }, }); expect(logger.info).toHaveBeenCalledWith({ task: "task-archive-retention", requestId: null }, "maintenance.tenant-sweep.overlap-skipped"); }); test("dispatches billing-dunning-sweep as a leased tenant maintenance sweep", async () => { // AK-P20 DoD 2 — proves the maintenance registry routes the dunning sweep // into the leased sweep path (an unknown task would reject with worker.unsupported). const store = createSharedStore({ namespace: "worker-handler-lease-test", adapter: new MemoryAdapter() }); await store.set("job-lease:billing-dunning-sweep", "other-replica", { ttlSeconds: 120 }); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, logger }); const descriptor = buildCleanupJob({ task: "billing-dunning-sweep" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "maintenance.tenant-sweep.overlap-skipped", details: { task: "billing-dunning-sweep", reason: "active-lease" }, }); expect(logger.info).toHaveBeenCalledWith({ task: "billing-dunning-sweep", requestId: null }, "maintenance.tenant-sweep.overlap-skipped"); }); test("dispatches project-auto-archive as a leased tenant maintenance sweep", async () => { // R-P9a DoD 4 — proves the maintenance registry routes the new task into the // leased sweep path (an unknown task would reject with worker.unsupported). const store = createSharedStore({ namespace: "worker-handler-lease-test", adapter: new MemoryAdapter() }); await store.set("job-lease:project-auto-archive", "other-replica", { ttlSeconds: 120 }); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, logger }); const descriptor = buildCleanupJob({ task: "project-auto-archive" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "maintenance.tenant-sweep.overlap-skipped", details: { task: "project-auto-archive", reason: "active-lease" }, }); expect(logger.info).toHaveBeenCalledWith({ task: "project-auto-archive", requestId: null }, "maintenance.tenant-sweep.overlap-skipped"); }); test("runs the admin sales task reminder sweep through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-sales-task-reminder-handler-test", adapter: new MemoryAdapter(), }); const runAdminSalesTaskReminderSweep = vi.fn(async () => ({ scanned: 3, dispatched: 2, })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runAdminSalesTaskReminderSweep, logger, }); const descriptor = buildCleanupJob({ task: "admin-sales-task-reminder-sweep", take: 75, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.task-reminder-sweep.completed", details: { scanned: 3, dispatched: 2 }, }); expect(runAdminSalesTaskReminderSweep).toHaveBeenCalledWith({ now: expect.any(Date), take: 75, }); expect(logger.info).toHaveBeenCalledWith( { task: "admin-sales-task-reminder-sweep", scanned: 3, dispatched: 2, }, "admin-sales.task-reminder-sweep.completed" ); }); test("queues due admin booking reminders through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-booking-reminder-handler-test", adapter: new MemoryAdapter(), }); const runAdminBookingReminderSweep = vi.fn(async () => ({ scanned: 4, dispatched: 3, failed: 1, })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runAdminBookingReminderSweep, logger, }); const descriptor = buildCleanupJob({ task: "admin-booking-reminder-sweep", take: 80, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-booking.reminder-sweep.completed", details: { scanned: 4, dispatched: 3, failed: 1 }, }); expect(runAdminBookingReminderSweep).toHaveBeenCalledWith({ now: expect.any(Date), take: 80, }); }); test("runs admin email delivery through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-email-delivery-handler-test", adapter: new MemoryAdapter(), }); const runAdminEmailDelivery = vi.fn(async () => ({ inspected: 4, claimed: 2, sent: 1, failed: ["scheduled-2"], blocked: 1, deferred: 1, recoveredUnknown: 0, noReplyReminders: { scanned: 3, created: 2 }, })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runAdminEmailDelivery, logger, }); const descriptor = buildCleanupJob({ task: "admin-email-delivery", take: 25, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.email-delivery.completed", details: { inspected: 4, claimed: 2, sent: 1, failed: 1, blocked: 1, deferred: 1, recoveredUnknown: 0, noReplyReminderScanned: 3, noReplyReminderCreated: 2, }, }); expect(runAdminEmailDelivery).toHaveBeenCalledWith({ now: expect.any(Date), take: 25, }); }); test("runs admin mailbox sync through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-email-sync-handler-test", adapter: new MemoryAdapter(), }); const runAdminEmailSync = vi.fn(async () => ({ inspected: 3, synced: 2, upserted: 4, replayed: 1, failed: ["cursor-3"], })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runAdminEmailSync, logger, }); const descriptor = buildCleanupJob({ task: "admin-email-sync", take: 20, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.email-sync.completed", details: { inspected: 3, synced: 2, upserted: 4, replayed: 1, failed: 1, }, }); expect(runAdminEmailSync).toHaveBeenCalledWith({ now: expect.any(Date), take: 20, }); }); test("checks admin sending domains through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-email-domain-handler-test", adapter: new MemoryAdapter(), }); const runAdminEmailDomainCheck = vi.fn(async () => ({ inspected: 3, verified: 2, degraded: 1, failed: [], })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runAdminEmailDomainCheck, logger, }); const descriptor = buildCleanupJob({ task: "admin-email-domain-check", take: 20, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.email-domain-check.completed", details: { inspected: 3, verified: 2, degraded: 1, failed: 0, }, }); expect(runAdminEmailDomainCheck).toHaveBeenCalledWith({ now: expect.any(Date), take: 20, }); }); test("processes encrypted admin call recordings through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-call-recording-process-handler-test", adapter: new MemoryAdapter(), }); const runAdminCallRecordingProcessing = vi.fn(async () => ({ claimed: 2, processed: 2, })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runAdminCallRecordingProcessing, logger: { info: vi.fn(), error: vi.fn() }, }); const descriptor = buildCleanupJob({ task: "admin-call-recording-process", take: 10, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.call-recording-process.completed", details: { claimed: 2, processed: 2 }, }); expect(runAdminCallRecordingProcessing).toHaveBeenCalledWith({ take: 10, }); }); test("expires admin call recordings through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-call-recording-retention-handler-test", adapter: new MemoryAdapter(), }); const runAdminCallRecordingRetention = vi.fn(async () => ({ claimed: 3, deleted: 2, failed: 1, })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runAdminCallRecordingRetention, logger: { info: vi.fn(), error: vi.fn() }, }); const descriptor = buildCleanupJob({ task: "admin-call-recording-retention", take: 50, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.call-recording-retention.completed", details: { claimed: 3, deleted: 2, failed: 1 }, }); expect(runAdminCallRecordingRetention).toHaveBeenCalledWith({ now: expect.any(Date), take: 50, }); }); test("redacts expired admin email event raw data through the leased worker path", async () => { const store = createSharedStore({ namespace: "admin-email-event-retention-handler-test", adapter: new MemoryAdapter(), }); const redactExpiredAdminEmailEventRawData = vi.fn(async () => ({ inspected: 8, redacted: 7, deletedLinks: 3, })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, redactExpiredAdminEmailEventRawData, logger, }); const descriptor = buildCleanupJob({ task: "admin-email-event-retention", take: 50, }); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.email-event-retention.completed", details: { inspected: 8, redacted: 7, deletedLinks: 3 }, }); expect(redactExpiredAdminEmailEventRawData).toHaveBeenCalledWith({ now: expect.any(Date), take: 50, }); }); test("runs telephony webhook maintenance through the injected general-group service", async () => { const runTelephonyWebhookMaintenance = vi.fn(async () => ({ deletedCompleted: 3, deletedFailed: 1, recoveredPending: 2, recoveredFailed: 1, inspectedTenants: 4, })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ runTelephonyWebhookMaintenance, logger }); const descriptor = buildCleanupJob({ task: "telephony-webhook-maintenance" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "telephony.webhook-maintenance.completed", details: { task: "telephony-webhook-maintenance", deletedCompleted: 3, deletedFailed: 1, recoveredPending: 2, recoveredFailed: 1, inspectedTenants: 4, }, }); expect(runTelephonyWebhookMaintenance).toHaveBeenCalledTimes(1); expect(logger.info).toHaveBeenCalledWith( { task: "telephony-webhook-maintenance", deletedCompleted: 3, deletedFailed: 1, recoveredPending: 2, recoveredFailed: 1, inspectedTenants: 4, }, "telephony.webhook-maintenance.completed" ); }); test("skips telephony webhook maintenance while another replica holds its lease", async () => { const store = createSharedStore({ namespace: "telephony-maintenance-handler-lease-test", adapter: new MemoryAdapter() }); await store.set("job-lease:telephony-webhook-maintenance", "other-replica", { ttlSeconds: 120 }); const runTelephonyWebhookMaintenance = vi.fn(async () => ({ deletedCompleted: 0, deletedFailed: 0, recoveredPending: 0, recoveredFailed: 0, inspectedTenants: 0, })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runTelephonyWebhookMaintenance }); const descriptor = buildCleanupJob({ task: "telephony-webhook-maintenance" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "maintenance.tenant-sweep.overlap-skipped", details: { task: "telephony-webhook-maintenance", reason: "active-lease" }, }); expect(runTelephonyWebhookMaintenance).not.toHaveBeenCalled(); }); test("runs product acquisition reconciliation through the leased general-group service", async () => { const runProductAcquisitionReconcile = vi.fn(async () => ({ inspected: 5, expired: 1, recoveredProviderPayments: 1, refreshed: 1, appliedTerminal: 2, quarantined: 0, raced: 0, inspectedTenants: 3, })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ runProductAcquisitionReconcile, logger }); const descriptor = buildCleanupJob({ task: "product-acquisition-reconcile", take: 25 }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "billing.product-acquisition-reconcile.completed", details: { task: "product-acquisition-reconcile", inspected: 5, expired: 1, recoveredProviderPayments: 1, refreshed: 1, appliedTerminal: 2, quarantined: 0, raced: 0, inspectedTenants: 3, }, }); expect(runProductAcquisitionReconcile).toHaveBeenCalledWith({ take: 25 }); expect(logger.info).toHaveBeenCalledWith( expect.objectContaining({ task: "product-acquisition-reconcile", inspectedTenants: 3, appliedTerminal: 2 }), "billing.product-acquisition-reconcile.completed" ); }); test("skips product acquisition reconciliation while another replica holds its lease", async () => { const store = createSharedStore({ namespace: "product-acquisition-reconcile-handler-lease-test", adapter: new MemoryAdapter() }); await store.set("job-lease:product-acquisition-reconcile", "other-replica", { ttlSeconds: 120 }); const runProductAcquisitionReconcile = vi.fn(async () => ({ inspected: 0, expired: 0, recoveredProviderPayments: 0, refreshed: 0, appliedTerminal: 0, quarantined: 0, raced: 0, inspectedTenants: 0, })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runProductAcquisitionReconcile }); const descriptor = buildCleanupJob({ task: "product-acquisition-reconcile" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "maintenance.tenant-sweep.overlap-skipped", details: { task: "product-acquisition-reconcile", reason: "active-lease" }, }); expect(runProductAcquisitionReconcile).not.toHaveBeenCalled(); }); test("runs the portal lifecycle scan through the leased general-group service", async () => { const runScan = vi.fn(async () => ({ inspectedTenants: 2, deletionScanned: 1, deletionCompleted: 1, exportArtifactsExpired: 2, inactivityWarnings: 3, inactivityDisabled: 1, projectCompletionMails: 4, projectAccessesClosed: 2, failed: 0, })); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ portalLifecycleService: { runScan }, logger }); const descriptor = buildCleanupJob({ task: "portal-lifecycle-scan" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "portal.lifecycle.scanned", details: { task: "portal-lifecycle-scan", inspectedTenants: 2, deletionScanned: 1, deletionCompleted: 1, exportArtifactsExpired: 2, inactivityWarnings: 3, inactivityDisabled: 1, projectCompletionMails: 4, projectAccessesClosed: 2, failed: 0, }, }); expect(runScan).toHaveBeenCalledTimes(1); expect(logger.info).toHaveBeenCalledWith( expect.objectContaining({ task: "portal-lifecycle-scan", inspectedTenants: 2 }), "portal.lifecycle.scan-completed" ); }); test("dispatches dsgvo-grace-period-scan through runDsgvoGraceCron with the configured service", async () => { const listOverdueUsers = vi .fn() .mockResolvedValue([{ userId: "u-overdue-1", tenantId: "tenant_a", scheduledDeletionAt: new Date("2026-05-01T00:00:00Z"), deletionRequestId: "req-1" }]); const enqueueDeletion = vi.fn().mockResolvedValue({ jobId: "dsgvo-cascade-1" }); const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ dsgvoGraceCronService: { listOverdueUsers, enqueueDeletion }, logger, }); const descriptor = buildCleanupJob({ task: "dsgvo-grace-period-scan" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "dsgvo.grace-period.scanned", details: { task: "dsgvo-grace-period-scan", scanned: 1, dispatched: 1, failed: 0, failures: [], }, }); expect(listOverdueUsers).toHaveBeenCalledTimes(1); expect(enqueueDeletion).toHaveBeenCalledTimes(1); expect(enqueueDeletion).toHaveBeenCalledWith( expect.objectContaining({ userId: "u-overdue-1", tenantId: "tenant_a", requestId: "req-1", actorType: "system", mode: "pseudonymize", reason: "dsgvo-grace-period-elapsed", }) ); }); test("acks dsgvo-grace-period-scan with skipped action when service dep is missing (no UnsupportedWorkerJobError loop)", async () => { const logger = { info: vi.fn(), error: vi.fn() }; const registry = buildHandlerRegistry({ logger }); const descriptor = buildCleanupJob({ task: "dsgvo-grace-period-scan", metadata: { requestId: "req_grace_1" } }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "dsgvo.grace-period.skipped", details: { task: "dsgvo-grace-period-scan", reason: "dsgvoGraceCronService not configured", scanned: 0, dispatched: 0, failed: 0, }, }); expect(logger.info).toHaveBeenCalledWith( expect.objectContaining({ task: "dsgvo-grace-period-scan", requestId: "req_grace_1" }), expect.stringContaining("dsgvo-grace-cron.skipped") ); }); test("hard-deletes expired export artifacts and reports scalar counts", async () => { const jobs = [ retentionJob({ id: "export_job_1", artifactBucket: "exports", artifactKey: "tenant_1/export-1.csv" }), retentionJob({ id: "export_job_2", artifactBucket: "exports", artifactKey: "tenant_1/export-2.csv" }), ]; const observedScopes: Array = []; const listExpiredCompletedExportJobs = vi.fn(async () => { observedScopes.push(getRlsExecutionContext()?.tenantId); return []; }); const markExportJobExpired = vi.fn(async () => undefined); const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => { observedScopes.push(getRlsExecutionContext()?.tenantId); return jobs; }); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async (input) => { return { bucket: input.bucket ?? "exports", key: input.key }; }), }; const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined); const markExportJobHardDeleteFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), listExpiredCompletedExportJobs, markExportJobExpired, listSoftDeletedExportJobsForHardDelete, storageDeleteService, markExportJobHardDeleteSucceeded, markExportJobHardDeleteFailed, }); const descriptor = buildCleanupJob({ task: "expired-export-artifacts", olderThanHours: 2, metadata: { requestId: "req_retention_1", token: "payload-secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "export-artifact-retention.completed", details: { task: "expired-export-artifacts", requestId: "req_retention_1", expired: 0, processed: 2, deleted: 2, failed: 0, jobs: [ { exportJobId: "export_job_1", status: "deleted" }, { exportJobId: "export_job_2", status: "deleted" }, ], }, }); expect(listExpiredCompletedExportJobs).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined }); expect(markExportJobExpired).not.toHaveBeenCalled(); expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined }); const hardDeleteListInput = (listSoftDeletedExportJobsForHardDelete.mock.calls as unknown as [TestRetentionListInput][])[0]?.[0]; expect(hardDeleteListInput?.now.getTime()).toBeLessThan(Date.now()); expect(storageDeleteService.deleteObject).toHaveBeenNthCalledWith(1, { bucket: "exports", key: "tenant_1/export-1.csv", signal: expect.any(AbortSignal), }); expect(storageDeleteService.deleteObject).toHaveBeenNthCalledWith(2, { bucket: "exports", key: "tenant_1/export-2.csv", signal: expect.any(AbortSignal), }); expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1" }); expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_2", tenantId: "tenant_1" }); expect(markExportJobHardDeleteFailed).not.toHaveBeenCalled(); expect(observedScopes).toEqual(["tenant_1", "tenant_1"]); }); test("routes the dedicated portal GDPR retention task with its bounded take and request id", async () => { const runSweep: TestPortalGdprExportRetentionService["runSweep"] = vi.fn(async () => ({ inspectedTenants: 2, processed: 4, claimed: 3, deleted: 2, raced: 1, })); const registry = buildHandlerRegistry({ portalGdprExportRetentionService: { runSweep }, }); const descriptor = buildCleanupJob({ task: "portal-gdpr-export-retention", take: 25, metadata: { requestId: "req_portal_gdpr_retention_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "portal.gdpr-export-retention.completed", details: { task: "portal-gdpr-export-retention", requestId: "req_portal_gdpr_retention_1", inspectedTenants: 2, processed: 4, claimed: 3, deleted: 2, raced: 1, }, }); expect(runSweep).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 }); }); test("rejects the maintenance job when portal GDPR artifact deletion fails so BullMQ can retry", async () => { const deleteFailure = new Error("portal GDPR retention deletion failed"); const runSweep: TestPortalGdprExportRetentionService["runSweep"] = vi.fn(async () => { throw deleteFailure; }); const registry = buildHandlerRegistry({ portalGdprExportRetentionService: { runSweep }, }); const descriptor = buildCleanupJob({ task: "portal-gdpr-export-retention" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toBe(deleteFailure); }); test("owner-claims and hard-deletes persisted privacy GDPR attempts for active and inactive tenants", async () => { const tenantFindMany = vi.fn(async () => [{ id: "tenant_1" }]); const activeTenantReader = { tenant: { findMany: tenantFindMany } }; const artifact: TestGdprRetentionJob = { id: "gdpr_export_1", tenantId: "tenant_1", status: "failed", createdAt: new Date("2026-07-01T00:00:00.000Z"), resultBucket: "exports", resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_1/attempt_1.zip", resultExpiresAt: null, processingStartedAt: null, }; const observedContexts: Array = []; const sequence: string[] = []; const listExpiredGdprExportArtifacts = vi.fn(async () => { observedContexts.push(getRlsExecutionContext()?.tenantId); return [artifact]; }); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async (input) => { sequence.push("storage-delete"); return { bucket: input.bucket ?? "exports", key: input.key }; }), }; const markGdprExportArtifactExpired = vi.fn(async () => { observedContexts.push(getRlsExecutionContext()?.tenantId); sequence.push("db-claim"); }); const markGdprExportArtifactDeleted = vi.fn(async () => { observedContexts.push(getRlsExecutionContext()?.tenantId); sequence.push("db-finalize"); }); const registry = buildHandlerRegistry({ activeTenantReader, env: { STORAGE_BUCKET: "exports" }, listExpiredGdprExportArtifacts, markGdprExportArtifactDeleted, markGdprExportArtifactExpired, storageDeleteService, }); const descriptor = buildCleanupJob({ task: "privacy-gdpr-export-retention", take: 25, metadata: { requestId: "req_gdpr_retention_1" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "privacy.gdpr-export-retention.completed", details: { task: "privacy-gdpr-export-retention", requestId: "req_gdpr_retention_1", processed: 1, deleted: 1, failed: 0, jobs: [{ gdprExportJobId: "gdpr_export_1", status: "deleted" }], }, }); expect(tenantFindMany).toHaveBeenCalledWith(expect.objectContaining({ where: {} })); expect(listExpiredGdprExportArtifacts).toHaveBeenCalledWith({ tenantId: "tenant_1", now: expect.any(Date), take: 25 }); expect(storageDeleteService.deleteObject).toHaveBeenCalledWith({ bucket: "exports", key: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_1/attempt_1.zip", signal: expect.any(AbortSignal), }); expect(markGdprExportArtifactExpired).toHaveBeenCalledWith({ ...artifact, now: expect.any(Date) }); expect(markGdprExportArtifactDeleted).toHaveBeenCalledWith({ id: "gdpr_export_1", tenantId: "tenant_1", resultBucket: "exports", resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_1/attempt_1.zip", }); expect(sequence).toEqual(["db-claim", "storage-delete", "db-finalize"]); expect(observedContexts).toEqual(["tenant_1", "tenant_1", "tenant_1"]); }); test("fails the maintenance job so failed privacy GDPR deletes retry on the next scheduler tick", async () => { const listExpiredGdprExportArtifacts = vi.fn( async (): Promise => [ { id: "gdpr_export_bad", tenantId: "tenant_1", status: "ready", createdAt: new Date("2026-07-01T00:00:00.000Z"), resultBucket: "exports", resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_bad/attempt_bad.zip", resultExpiresAt: new Date("2026-07-05T00:00:00.000Z"), processingStartedAt: null, }, ] ); const markGdprExportArtifactExpired = vi.fn(async () => undefined); const markGdprExportArtifactDeleted = vi.fn(async () => undefined); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async () => { throw new Error("delete failed token=storage-secret authorization=Bearer-secret"); }), }; const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), env: { STORAGE_BUCKET: "exports" }, listExpiredGdprExportArtifacts, markGdprExportArtifactDeleted, markGdprExportArtifactExpired, storageDeleteService, }); const descriptor = buildCleanupJob({ task: "privacy-gdpr-export-retention" }); let failure: unknown; try { await registry[descriptor.queueName](descriptor.payload, {}); } catch (error) { failure = error; } expect(failure).toBeInstanceOf(AggregateError); expect(String(failure)).not.toContain("storage-secret"); expect(String(failure)).not.toContain("Bearer-secret"); expect(markGdprExportArtifactExpired).toHaveBeenCalledOnce(); expect(markGdprExportArtifactDeleted).not.toHaveBeenCalled(); }); test("continues a mixed privacy GDPR retention batch after one delete fails", async () => { const artifacts: TestGdprRetentionJob[] = [ { id: "gdpr_export_bad", tenantId: "tenant_1", status: "ready", createdAt: new Date("2026-07-01T00:00:00.000Z"), resultBucket: "exports", resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_bad/attempt_bad.zip", resultExpiresAt: new Date("2026-07-05T00:00:00.000Z"), processingStartedAt: null, }, { id: "gdpr_export_good", tenantId: "tenant_1", status: "ready", createdAt: new Date("2026-07-01T00:00:00.000Z"), resultBucket: "exports", resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_good/attempt_good.zip", resultExpiresAt: new Date("2026-07-05T00:00:00.000Z"), processingStartedAt: null, }, ]; const listExpiredGdprExportArtifacts = vi.fn(async () => artifacts); const markGdprExportArtifactExpired = vi.fn(async () => undefined); const markGdprExportArtifactDeleted = vi.fn(async () => undefined); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async ({ bucket, key }) => { if (key.includes("gdpr_export_bad")) throw new Error("delete failed token=storage-secret"); return { bucket, key }; }), }; const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), env: { STORAGE_BUCKET: "exports" }, listExpiredGdprExportArtifacts, markGdprExportArtifactDeleted, markGdprExportArtifactExpired, storageDeleteService, }); const descriptor = buildCleanupJob({ task: "privacy-gdpr-export-retention" }); let failure: unknown; try { await registry[descriptor.queueName](descriptor.payload, {}); } catch (error) { failure = error; } expect(failure).toBeInstanceOf(AggregateError); expect(String(failure)).not.toContain("storage-secret"); expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(2); expect(markGdprExportArtifactExpired).toHaveBeenCalledTimes(2); expect(markGdprExportArtifactDeleted).toHaveBeenCalledOnce(); expect(markGdprExportArtifactDeleted).toHaveBeenCalledWith({ id: "gdpr_export_good", tenantId: "tenant_1", resultBucket: "exports", resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_good/attempt_good.zip", }); }); test("caps export artifact hard-delete concurrency while preserving reported job order", async () => { const jobs = Array.from({ length: 6 }, (_, index) => retentionJob({ id: `export_job_${index + 1}`, artifactBucket: "exports", artifactKey: `tenant_1/export-${index + 1}.csv`, }) ); let activeDeletes = 0; let maxActiveDeletes = 0; let releaseFirstBatch: (() => void) | undefined; const firstBatchWait = new Promise((resolve) => { releaseFirstBatch = resolve; }); const listExpiredCompletedExportJobs = vi.fn(async () => []); const markExportJobExpired = vi.fn(async () => undefined); const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async (input) => { activeDeletes += 1; maxActiveDeletes = Math.max(maxActiveDeletes, activeDeletes); if (input.key !== "tenant_1/export-6.csv") { await firstBatchWait; } activeDeletes -= 1; return { bucket: input.bucket ?? "exports", key: input.key }; }), }; const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), listExpiredCompletedExportJobs, markExportJobExpired, listSoftDeletedExportJobsForHardDelete, storageDeleteService, markExportJobHardDeleteSucceeded, }); const descriptor = buildCleanupJob({ task: "expired-export-artifacts" }); const run = registry[descriptor.queueName](descriptor.payload, {}); await vi.waitFor(() => expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(5)); expect(maxActiveDeletes).toBe(5); expect(markExportJobHardDeleteSucceeded).not.toHaveBeenCalled(); releaseFirstBatch?.(); await expect(run).resolves.toEqual({ handled: true, action: "export-artifact-retention.completed", details: { task: "expired-export-artifacts", requestId: undefined, expired: 0, processed: 6, deleted: 6, failed: 0, jobs: jobs.map((job) => ({ exportJobId: job.id, status: "deleted" })), }, }); expect(maxActiveDeletes).toBe(5); expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(6); }); test("continues export artifact retention after storage failures with redacted diagnostics", async () => { const jobs = [ retentionJob({ id: "export_job_ok", artifactBucket: "exports", artifactKey: "tenant_1/ok.csv" }), retentionJob({ id: "export_job_bad", artifactBucket: "exports", artifactKey: "tenant_1/bad.csv" }), ]; const listExpiredCompletedExportJobs = vi.fn(async () => []); const markExportJobExpired = vi.fn(async () => undefined); const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async (input) => { if (input.key === "tenant_1/bad.csv") { throw new Error("delete failed token=storage-secret body=payload-bytes authorization=Bearer-secret"); } return { bucket: input.bucket ?? "exports", key: input.key }; }), }; const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined); const markExportJobHardDeleteFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), listExpiredCompletedExportJobs, markExportJobExpired, listSoftDeletedExportJobsForHardDelete, storageDeleteService, markExportJobHardDeleteSucceeded, markExportJobHardDeleteFailed, }); const descriptor = buildCleanupJob({ task: "expired-export-artifacts", metadata: { requestId: "req_retention_partial" } }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "export-artifact-retention.completed", details: { task: "expired-export-artifacts", requestId: "req_retention_partial", expired: 0, processed: 2, deleted: 1, failed: 1, jobs: [ { exportJobId: "export_job_ok", status: "deleted" }, { exportJobId: "export_job_bad", status: "failed", message: "delete failed token=[redacted] body=[redacted] authorization=[redacted]" }, ], }, }); expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_ok", tenantId: "tenant_1" }); expect(markExportJobHardDeleteFailed).toHaveBeenCalledWith({ id: "export_job_bad", tenantId: "tenant_1", errorMessage: "delete failed token=[redacted] body=[redacted] authorization=[redacted]", }); const resultText = JSON.stringify((await registry[descriptor.queueName](descriptor.payload, {})).details); expect(resultText).not.toContain("storage-secret"); expect(resultText).not.toContain("payload-bytes"); expect(resultText).not.toContain("Bearer-secret"); }); test("fails export artifact retention when delete dependency is missing", async () => { const jobs = [retentionJob({ id: "export_job_missing_delete", artifactBucket: "exports", artifactKey: "tenant_1/missing.csv" })]; const listExpiredCompletedExportJobs = vi.fn(async () => []); const markExportJobExpired = vi.fn(async () => undefined); const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs); const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined); const markExportJobHardDeleteFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), listExpiredCompletedExportJobs, markExportJobExpired, listSoftDeletedExportJobsForHardDelete, markExportJobHardDeleteSucceeded, markExportJobHardDeleteFailed, }); const descriptor = buildCleanupJob({ task: "expired-export-artifacts" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("expired-export-artifacts failed for 1 tenant(s)"); expect(markExportJobHardDeleteFailed).not.toHaveBeenCalled(); expect(markExportJobHardDeleteSucceeded).not.toHaveBeenCalled(); }); test("marks expired completed exports before hard-delete retention scan", async () => { const expiredJobs = [retentionJob({ id: "export_job_expired", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired.csv" })]; const listExpiredCompletedExportJobs = vi.fn(async () => expiredJobs); const markExportJobExpired = vi.fn(async () => undefined); const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => []); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async (input) => ({ bucket: input.bucket ?? "exports", key: input.key })), }; const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), listExpiredCompletedExportJobs, markExportJobExpired, listSoftDeletedExportJobsForHardDelete, storageDeleteService, }); const descriptor = buildCleanupJob({ task: "expired-export-artifacts", take: 25 }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "export-artifact-retention.completed", details: { expired: 1, processed: 0, deleted: 0, failed: 0, }, }); expect(listExpiredCompletedExportJobs).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 }); expect(markExportJobExpired).toHaveBeenCalledWith({ id: "export_job_expired", tenantId: "tenant_1", deletedAt: expect.any(Date) }); expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 }); expect(storageDeleteService.deleteObject).not.toHaveBeenCalled(); }); test("marks independent expired completed exports in parallel before hard-delete retention scan", async () => { const expiredJobs = [ retentionJob({ id: "export_job_expired_1", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired-1.csv" }), retentionJob({ id: "export_job_expired_2", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired-2.csv" }), ]; const listExpiredCompletedExportJobs = vi.fn(async () => expiredJobs); let firstMarkCanFinish: (() => void) | undefined; const firstMarkWait = new Promise((resolve) => { firstMarkCanFinish = resolve; }); const markExportJobExpired = vi.fn(async ({ id }) => { if (id === "export_job_expired_1") { await firstMarkWait; } }); const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => []); const storageDeleteService: TestStorageDeleteService = { deleteObject: vi.fn(async (input) => ({ bucket: input.bucket ?? "exports", key: input.key })), }; const registry = buildHandlerRegistry({ activeTenantReader: createActiveTenantReader(), listExpiredCompletedExportJobs, markExportJobExpired, listSoftDeletedExportJobsForHardDelete, storageDeleteService, }); const descriptor = buildCleanupJob({ task: "expired-export-artifacts" }); const run = registry[descriptor.queueName](descriptor.payload, {}); await vi.waitFor(() => expect(markExportJobExpired).toHaveBeenCalledTimes(2)); expect(listSoftDeletedExportJobsForHardDelete).not.toHaveBeenCalled(); firstMarkCanFinish?.(); await expect(run).resolves.toMatchObject({ action: "export-artifact-retention.completed", details: { expired: 2, processed: 0 }, }); expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined }); }); test("rejects descriptor-derived browser automation jobs through registry without global fetch", async () => { const fetchSpy = vi.spyOn(globalThis, "fetch"); const registry = buildHandlerRegistry(); const descriptors = [ buildBrowserSessionRunJob({ sessionId: "session_123", tenantId: "tenant_123", userId: "user_123", allowedOrigins: ["https://example.com"], task: { instruction: "fixture browser session task not surfaced" }, }), buildBrowserSessionCleanupJob({ sessionId: "session_123", tenantId: "tenant_123", metadata: { reason: "fixture cleanup reason not surfaced" }, }), buildBrowserUsageFlushJob({ sessionId: "session_123", tenantId: "tenant_123", metadata: { token: "fixture token not surfaced" }, }), ]; try { await Promise.all( descriptors.map((descriptor) => expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "worker.unsupported", message: "Browser automation worker job is not configured.", details: { queueName: queueNames.browserAutomation, kind: descriptor.payload.kind, sessionId: "session_123" }, }) ) ); expect(fetchSpy).not.toHaveBeenCalled(); } finally { fetchSpy.mockRestore(); } }); test("pauses gated export jobs without running the export service", async () => { const exportService: TestExportService = { runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt: new Date() })), }; const markExportJobPaused = vi.fn(async () => undefined); const markExportJobRunning = vi.fn(async () => undefined); const markExportJobSucceeded = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobPaused, markExportJobRunning, markExportJobSucceeded }); const descriptor = buildExportJob({ exportJobId: "export_job_paused", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "csv", requestId: "req_export_paused", complianceGate: { state: "paused", reason: "workspace-retention-pause", requestId: "req_pause_1", }, metadata: { attemptsMade: 3 }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "export.paused", details: { exportJobId: "export_job_paused", tenantId: "tenant_1", exportType: "members", format: "csv", requestId: "req_export_paused", attemptsMade: 3, pauseReason: "workspace-retention-pause", pauseRequestId: "req_pause_1", status: "paused", }, }); expect(markExportJobPaused).toHaveBeenCalledWith({ id: "export_job_paused", tenantId: "tenant_1", attemptsMade: 3, reason: "workspace-retention-pause" }); expect(exportService.runExport).not.toHaveBeenCalled(); expect(markExportJobRunning).not.toHaveBeenCalled(); expect(markExportJobSucceeded).not.toHaveBeenCalled(); }); test("redacts secret-like pause reasons in persisted and returned diagnostics", async () => { const exportService: TestExportService = { runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt: new Date() })), }; const markExportJobPaused = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobPaused }); const descriptor = buildExportJob({ exportJobId: "export_job_secret_pause", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "json", requestId: "req_export_secret_pause", complianceGate: { state: "paused", reason: "pause token=service-secret body=payload-bytes authorization=Bearer-secret", requestId: "req_pause_secret", }, metadata: { attemptsMade: 1 }, filters: { apiKey: "payload-secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "export.paused", details: { pauseReason: "pause token=[redacted] body=[redacted] authorization=[redacted]", }, }); expect(markExportJobPaused).toHaveBeenCalledWith({ id: "export_job_secret_pause", tenantId: "tenant_1", attemptsMade: 1, reason: "pause token=[redacted] body=[redacted] authorization=[redacted]", }); expect(exportService.runExport).not.toHaveBeenCalled(); expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("service-secret"); expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("payload-secret"); expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("Bearer-secret"); }); test("resumes active gated export jobs before running them", async () => { const expiresAt = new Date("2026-05-09T00:00:00.000Z"); const events: string[] = []; const exportService: TestExportService = { runExport: vi.fn(async () => { events.push("export"); return { artifactBucket: "exports", artifactKey: "tenant_1/resumed.csv", rowCount: 7, expiresAt }; }), }; const markExportJobResumed = vi.fn(async () => { events.push("resume"); }); const markExportJobRunning = vi.fn(async () => { events.push("running"); }); const markExportJobSucceeded = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobResumed, markExportJobRunning, markExportJobSucceeded }); const descriptor = buildExportJob({ exportJobId: "export_job_resumed", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "csv", requestId: "req_export_resumed", complianceGate: { state: "active", resumedAt: "2026-05-08T00:00:00.000Z", requestId: "req_resume_1", }, metadata: { attemptsMade: 2 }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "export.completed", details: { exportJobId: "export_job_resumed", status: "completed", attemptsMade: 2 }, }); expect(markExportJobResumed).toHaveBeenCalledWith({ id: "export_job_resumed", tenantId: "tenant_1", attemptsMade: 2 }); expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_resumed", tenantId: "tenant_1", attemptsMade: 2 }); expect(exportService.runExport).toHaveBeenCalledWith(descriptor.payload); expect(events).toEqual(["resume", "running", "export"]); }); test("runs export jobs through injected service and persists completed artifact diagnostics", async () => { const expiresAt = new Date("2026-05-09T00:00:00.000Z"); const exportService: TestExportService = { runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt })), }; const markExportJobRunning = vi.fn(async () => undefined); const markExportJobSucceeded = vi.fn(async () => undefined); const markExportJobFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobSucceeded, markExportJobFailed }); const descriptor = buildExportJob({ exportJobId: "export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "csv", requestId: "req_export_success", metadata: { attemptsMade: 2, token: "payload-secret" }, filters: { authorization: "Bearer payload-secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "export.completed", details: { exportJobId: "export_job_1", tenantId: "tenant_1", exportType: "members", format: "csv", requestId: "req_export_success", artifactBucket: "exports", artifactKey: "tenant_1/export.csv", attemptsMade: 2, expiresAt: "2026-05-09T00:00:00.000Z", rowCount: 42, status: "completed", }, }); expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1", attemptsMade: 2 }); expect(exportService.runExport).toHaveBeenCalledWith(descriptor.payload); expect(markExportJobSucceeded).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1", artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, attemptsMade: 2, expiresAt, }); expect(markExportJobFailed).not.toHaveBeenCalled(); expect(JSON.stringify(markExportJobRunning.mock.calls)).not.toContain("payload-secret"); expect(JSON.stringify(markExportJobSucceeded.mock.calls)).not.toContain("payload-secret"); }); test("routes privacy GDPR export jobs through the GDPR service and status table", async () => { const expiresAt = new Date("2026-05-09T00:00:00.000Z"); const exportService: TestExportService = { runExport: vi.fn(async () => ({ artifactBucket: "unused", artifactKey: "unused", rowCount: 0, expiresAt })), }; const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => ({ artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4096, expiresAt, })) ); const markGdprExportProcessing = vi.fn>(async () => undefined); const markGdprExportReady = vi.fn(async () => "committed" as const); const markGdprExportFailed = vi.fn(async () => undefined); const markExportJobRunning = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed, markExportJobRunning, }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", requestId: "req_gdpr_export_success", metadata: { attemptsMade: 1, token: "payload-secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "privacy-gdpr-export.completed", details: { exportJobId: "gdpr_export_job_1", tenantId: "tenant_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", requestId: "req_gdpr_export_success", artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, attemptsMade: 1, expiresAt: "2026-05-09T00:00:00.000Z", rowCount: 7, sizeBytes: 4096, status: "ready", }, }); expect(markGdprExportProcessing).toHaveBeenCalledWith( expect.objectContaining({ id: "gdpr_export_job_1", tenantId: "tenant_1", resultBucket: "exports", resultKey: privacyAttempt.artifactKey, now: expect.any(Date), }) ); expect(privacyGdprExportService.runExport).toHaveBeenCalledWith(descriptor.payload, privacyAttempt); const processingStartedAt = markGdprExportProcessing.mock.calls[0]?.[0]?.now; expect(markGdprExportReady).toHaveBeenCalledWith({ id: "gdpr_export_job_1", tenantId: "tenant_1", resultBucket: "exports", resultKey: privacyAttempt.artifactKey, resultSizeBytes: 4096, resultExpiresAt: expiresAt, processingStartedAt, }); expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled(); expect(markGdprExportFailed).not.toHaveBeenCalled(); expect(exportService.runExport).not.toHaveBeenCalled(); expect(markExportJobRunning).not.toHaveBeenCalled(); }); test("does not run a privacy export whose queue subject loses the persisted-job claim", async () => { const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => { throw new Error("must not read or store export data"); }) ); const markGdprExportProcessing = vi.fn>(async (input) => { if ( input.requestedByUserId !== "user_1" || input.scope !== "user_self" || input.targetUserId !== "user_1" ) { throw new Error("Privacy GDPR export subject does not match the persisted job."); } }); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_subject_bound", tenantId: "tenant_1", requestedByUserId: "user_stale", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", metadata: { targetUserId: "user_stale" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow( "Privacy GDPR export subject does not match the persisted job." ); expect(markGdprExportProcessing).toHaveBeenCalledWith( expect.objectContaining({ requestedByUserId: "user_stale", scope: "user_self", targetUserId: "user_stale" }) ); expect(privacyGdprExportService.runExport).not.toHaveBeenCalled(); }); test("quiesces a privacy GDPR export after PUT and before the Ready transition", async () => { const expiresAt = new Date("2026-07-23T10:00:00.000Z"); const result = { artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt, }; let checkpointCount = 0; const checkpoint = vi.fn(async () => { checkpointCount += 1; if (checkpointCount === 2) throw new TenantExportQuiescedError(); }); const withTenantBarrier: TestPrivacyGdprExportService["withTenantBarrier"] = async (_payload, writer) => writer(checkpoint); const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => result), { withTenantBarrier } ); const markGdprExportProcessing = vi.fn>(async () => undefined); const markGdprExportReady = vi.fn(async () => "committed" as const); const markGdprExportFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_quiesced_after_put", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error", details: expect.objectContaining({ errorName: "TenantExportQuiescedError" }), }); const processingStartedAt = markGdprExportProcessing.mock.calls[0]?.[0]?.now; expect(checkpoint).toHaveBeenCalledTimes(2); expect(privacyGdprExportService.runExport).toHaveBeenCalledWith(descriptor.payload, privacyAttempt); expect(markGdprExportReady).not.toHaveBeenCalled(); expect(privacyGdprExportService.deleteArtifact).toHaveBeenCalledWith(privacyAttempt); expect(markGdprExportFailed).toHaveBeenCalledWith({ id: "gdpr_export_job_quiesced_after_put", tenantId: "tenant_1", errorMessage: "Tenant exports are blocked because hard deletion has started.", processingStartedAt, resultBucket: privacyAttempt.artifactBucket, resultKey: privacyAttempt.artifactKey, clearResultLocation: true, }); expect(privacyGdprExportService.terminalizeReadyAfterQuiescence).not.toHaveBeenCalled(); }); test("terminalizes the exact Ready artifact when quiescence starts after the Ready CAS", async () => { const expiresAt = new Date("2026-07-23T10:00:00.000Z"); const result = { artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt, }; let checkpointCount = 0; const checkpoint = vi.fn(async () => { checkpointCount += 1; if (checkpointCount === 4) throw new TenantExportQuiescedError(); }); const withTenantBarrier: TestPrivacyGdprExportService["withTenantBarrier"] = async (_payload, writer) => writer(checkpoint); const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => result), { withTenantBarrier } ); const markGdprExportProcessing = vi.fn>(async () => undefined); const markGdprExportReady = vi.fn(async () => "committed" as const); const markGdprExportFailed = vi.fn(async () => undefined); const sendGdprExportReadyMail = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed, sendGdprExportReadyMail, }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_quiesced_after_ready", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error", details: expect.objectContaining({ errorName: "TenantExportQuiescedError" }), }); expect(checkpoint).toHaveBeenCalledTimes(4); expect(markGdprExportReady).toHaveBeenCalledOnce(); expect(privacyGdprExportService.terminalizeReadyAfterQuiescence).toHaveBeenCalledWith({ payload: descriptor.payload, result }); expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled(); expect(markGdprExportFailed).not.toHaveBeenCalled(); expect(sendGdprExportReadyMail).not.toHaveBeenCalled(); }); test("short-circuits a retried privacy export when the ready row already owns a valid artifact", async () => { const expiresAt = new Date("2026-07-23T10:00:00.000Z"); const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => { throw new Error("must not upload again"); }) ); vi.mocked(privacyGdprExportService.resolveReady).mockResolvedValue({ artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: null, sizeBytes: 4_096, expiresAt, }); const markGdprExportProcessing = vi.fn(async (_input: Parameters>[0]) => undefined); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "privacy-gdpr-export.completed", details: { artifactKey: privacyAttempt.artifactKey, rowCount: null, status: "ready" }, }); expect(privacyGdprExportService.createAttempt).not.toHaveBeenCalled(); expect(privacyGdprExportService.runExport).not.toHaveBeenCalled(); expect(markGdprExportProcessing).not.toHaveBeenCalled(); }); test("does not delete or downgrade an artifact when the ready commit outcome is ambiguous, then reconciles on retry", async () => { const expiresAt = new Date("2026-07-23T10:00:00.000Z"); const result = { artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt, }; const privacyGdprExportService = createPrivacyGdprService(vi.fn(async () => result)); const markGdprExportReady = vi.fn(async () => { throw new Error("database response lost"); }); const markGdprExportProcessing = vi.fn(async (_input: Parameters>[0]) => undefined); const markGdprExportFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error" }); expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled(); expect(markGdprExportFailed).not.toHaveBeenCalled(); vi.mocked(privacyGdprExportService.resolveReady).mockResolvedValue({ ...result, rowCount: null }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "privacy-gdpr-export.completed", details: { artifactKey: privacyAttempt.artifactKey, rowCount: null, status: "ready" }, }); expect(privacyGdprExportService.createAttempt).toHaveBeenCalledTimes(1); expect(privacyGdprExportService.runExport).toHaveBeenCalledTimes(1); expect(markGdprExportReady).toHaveBeenCalledTimes(1); }); test("does not delete an adopted attempt when the privacy ready transition explicitly loses ownership", async () => { const expiresAt = new Date("2026-07-23T10:00:00.000Z"); const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => ({ artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt })) ); const markGdprExportReady = vi.fn(async () => "lost_owner" as const); const markGdprExportProcessing = vi.fn(async (_input: Parameters>[0]) => undefined); const markGdprExportFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error" }); expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled(); expect(markGdprExportFailed).not.toHaveBeenCalled(); }); test("retries an ambiguously committed privacy PUT with the exact persisted attempt", async () => { const expiresAt = new Date("2026-07-23T13:00:00.000Z"); const committedKeys: string[] = []; const privacyGdprExportService = createPrivacyGdprService( vi.fn(async (_payload, attempt) => { committedKeys.push(attempt.artifactKey); if (committedKeys.length === 1) throw new PrivacyGdprExportPutOutcomeUnknownError(); return { artifactBucket: attempt.artifactBucket, artifactKey: attempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt }; }) ); const markGdprExportProcessing = vi.fn>(async () => undefined); const markGdprExportReady = vi.fn(async () => "committed" as const); const markGdprExportFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({ action: "export.error", }); const firstClaim = markGdprExportProcessing.mock.calls[0]?.[0]?.now; expect(firstClaim).toBeInstanceOf(Date); expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled(); expect(markGdprExportFailed).not.toHaveBeenCalled(); vi.mocked(privacyGdprExportService.resolveReclaimableProcessing).mockResolvedValue({ processingStartedAt: firstClaim!, attempt: privacyAttempt, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).resolves.toMatchObject({ action: "privacy-gdpr-export.completed", details: { artifactKey: privacyAttempt.artifactKey, status: "ready" }, }); expect(committedKeys).toEqual([privacyAttempt.artifactKey, privacyAttempt.artifactKey]); expect(privacyGdprExportService.createAttempt).toHaveBeenCalledOnce(); expect(markGdprExportProcessing).toHaveBeenNthCalledWith( 2, expect.objectContaining({ previousProcessingStartedAt: firstClaim, resultBucket: privacyAttempt.artifactBucket, resultKey: privacyAttempt.artifactKey, }) ); expect(markGdprExportReady).toHaveBeenCalledOnce(); expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled(); expect(privacyGdprExportService.terminalizeReadyAfterQuiescence).not.toHaveBeenCalled(); expect(markGdprExportFailed).not.toHaveBeenCalled(); }); test("reclaims a stale privacy processing claim on a BullMQ retry with an exact compare-and-swap token", async () => { const previousProcessingStartedAt = new Date("2026-07-23T09:59:00.000Z"); const expiresAt = new Date("2026-07-23T13:00:00.000Z"); const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => ({ artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt })) ); vi.mocked(privacyGdprExportService.resolveReclaimableProcessing).mockResolvedValue({ processingStartedAt: previousProcessingStartedAt, attempt: privacyAttempt, }); const markGdprExportProcessing = vi.fn(async (_input: Parameters>[0]) => undefined); const markGdprExportReady = vi.fn(async () => "committed" as const); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1 })).resolves.toMatchObject({ action: "privacy-gdpr-export.completed", }); expect(markGdprExportProcessing).toHaveBeenCalledWith({ id: "gdpr_export_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", scope: "user_self", targetUserId: "user_1", resultBucket: "exports", resultKey: privacyAttempt.artifactKey, now: expect.any(Date), previousProcessingStartedAt, }); expect(markGdprExportReady).toHaveBeenCalledWith(expect.objectContaining({ processingStartedAt: expect.any(Date) })); }); test("normalizes a PortalUser GDPR outbox envelope and routes only the safe export contract", async () => { const expiresAt = new Date("2026-07-17T10:00:00.000Z"); const portalGdprExportService: TestPortalGdprExportService = { runExport: vi.fn(async () => ({ artifactBucket: "private-exports", artifactKey: "gdpr-exports/portal/tenants/tenant_1/export_1/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json", rowCount: 9, sizeBytes: 4096, expiresAt, })), }; const registry = buildHandlerRegistry({ portalGdprExportService }); const outboxEnvelope = { id: "outbox_1", tenantId: "tenant_1", eventType: "portal.gdpr_export.requested", payload: { tenantId: "tenant_1", portalUserId: "portal_user_1", jobId: "export_1", password: "must-not-forward" }, }; await expect(registry[queueNames.exports](outboxEnvelope as never, {})).resolves.toEqual({ handled: true, action: "portal-gdpr-export.completed", details: { exportJobId: "export_1", tenantId: "tenant_1", exportType: "gdpr_portal_user_self", format: "json", requestId: "outbox_1", artifactBucket: "private-exports", artifactKey: "gdpr-exports/portal/tenants/tenant_1/export_1/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json", attemptsMade: 0, expiresAt: "2026-07-17T10:00:00.000Z", rowCount: 9, sizeBytes: 4096, status: "ready", }, }); expect(portalGdprExportService.runExport).toHaveBeenCalledWith({ kind: "export.run", exportJobId: "export_1", tenantId: "tenant_1", requestedByUserId: "portal_user_1", exportType: "gdpr_portal_user_self", format: "json", requestId: "outbox_1", metadata: { actorKind: "portal_user", sourceEventType: "portal.gdpr_export.requested", outboxEventId: "outbox_1" }, }); expect(JSON.stringify(vi.mocked(portalGdprExportService.runExport).mock.calls)).not.toContain("must-not-forward"); }); test("rejects a cross-tenant portal GDPR outbox envelope before the export service", async () => { const portalGdprExportService: TestPortalGdprExportService = { runExport: vi.fn(async () => { throw new Error("must not run"); }), }; const registry = buildHandlerRegistry({ portalGdprExportService }); const outboxEnvelope = { id: "outbox_cross_tenant", tenantId: "tenant_a", eventType: "portal.gdpr_export.requested", payload: { tenantId: "tenant_b", portalUserId: "portal_user_b", jobId: "export_b" }, }; await expect(registry[queueNames.exports](outboxEnvelope as never, {})).rejects.toMatchObject({ action: "export.error", details: { code: "portal_gdpr_export_invalid_envelope", tenantMatch: false }, }); expect(portalGdprExportService.runExport).not.toHaveBeenCalled(); }); test("fails privacy GDPR export jobs when the dedicated service is missing", async () => { const markGdprExportProcessing = vi.fn(async () => undefined); const markGdprExportFailed = vi.fn(async () => undefined); const markExportJobRunning = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ markGdprExportProcessing, markGdprExportFailed, markExportJobRunning }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_missing_service", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", requestId: "req_gdpr_export_missing_service", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error", message: "Privacy GDPR export service is not configured.", details: { exportJobId: "gdpr_export_job_missing_service", tenantId: "tenant_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", requestId: "req_gdpr_export_missing_service", attemptsMade: 0, status: "failed", code: "privacy_gdpr_export_service_missing", }, }); expect(markGdprExportProcessing).toHaveBeenCalledWith({ id: "gdpr_export_job_missing_service", tenantId: "tenant_1", requestedByUserId: "user_1", scope: "user_self", targetUserId: "user_1", }); expect(markGdprExportFailed).toHaveBeenCalledWith({ id: "gdpr_export_job_missing_service", tenantId: "tenant_1", errorMessage: "Privacy GDPR export service is not configured.", }); expect(markExportJobRunning).not.toHaveBeenCalled(); }); test("persists the clear privacy GDPR size-limit diagnostic instead of an OOM-shaped failure", async () => { const privacyGdprExportService = createPrivacyGdprService( vi.fn(async () => { throw new PrivacyGdprExportSizeLimitExceededError(2_048, 1_024); }) ); const markGdprExportProcessing = vi.fn(async () => undefined); const markGdprExportFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportFailed }); const descriptor = buildExportJob({ exportJobId: "gdpr_export_job_too_large", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, format: "json", }); const run = registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 2 }); await expect(run).rejects.toBeInstanceOf(UnrecoverableError); await expect(run).rejects.toMatchObject({ action: "export.error", details: expect.objectContaining({ errorName: "PrivacyGdprExportSizeLimitExceededError", message: expect.stringContaining("stopped before a partial archive was stored"), }), }); expect(markGdprExportFailed).toHaveBeenCalledWith( expect.objectContaining({ id: "gdpr_export_job_too_large", tenantId: "tenant_1", errorMessage: expect.stringContaining("streaming GDPR export path"), processingStartedAt: expect.any(Date), clearResultLocation: true, }) ); expect(privacyGdprExportService.deleteArtifact).toHaveBeenCalledWith(privacyAttempt); }); test("uses BullMQ attempt metadata for export job persistence when available", async () => { const expiresAt = new Date("2026-05-09T00:00:00.000Z"); const exportService: TestExportService = { runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt })), }; const markExportJobRunning = vi.fn(async () => undefined); const markExportJobSucceeded = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobSucceeded }); const descriptor = buildExportJob({ exportJobId: "export_job_bullmq_attempt", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "csv", requestId: "req_export_bullmq_attempt", metadata: { attemptsMade: 1 }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 4 })).resolves.toMatchObject({ action: "export.completed", details: { attemptsMade: 5 }, }); expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_bullmq_attempt", tenantId: "tenant_1", attemptsMade: 5 }); expect(markExportJobSucceeded).toHaveBeenCalledWith(expect.objectContaining({ id: "export_job_bullmq_attempt", tenantId: "tenant_1", attemptsMade: 5 })); }); test("fails export jobs when service is missing", async () => { const markExportJobRunning = vi.fn(async () => undefined); const markExportJobSucceeded = vi.fn(async () => undefined); const markExportJobFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ markExportJobRunning, markExportJobSucceeded, markExportJobFailed }); const descriptor = buildExportJob({ exportJobId: "export_job_noop", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "json", requestId: "req_export_noop", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error", message: "Export service is not configured.", details: { exportJobId: "export_job_noop", tenantId: "tenant_1", exportType: "members", format: "json", requestId: "req_export_noop", attemptsMade: 0, status: "failed", code: "export_service_missing", }, }); expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_noop", tenantId: "tenant_1", attemptsMade: 0 }); expect(markExportJobFailed).toHaveBeenCalledWith({ id: "export_job_noop", tenantId: "tenant_1", attemptsMade: 0, errorName: "ExportHandlerError", errorMessage: "Export service is not configured.", }); expect(markExportJobSucceeded).not.toHaveBeenCalled(); }); test("marks export jobs failed with redacted scalar diagnostics when service rejects", async () => { const exportService: TestExportService = { runExport: vi.fn(async () => { throw new Error("export failed token=service-secret authorization=Bearer-secret body=payload-bytes"); }), }; const markExportJobRunning = vi.fn(async () => undefined); const markExportJobFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobFailed }); const descriptor = buildExportJob({ exportJobId: "export_job_failed", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "csv", metadata: { attemptsMade: 1, token: "payload-secret" }, filters: { apiKey: "payload-secret" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error", message: "Export job failed.", details: expect.objectContaining({ status: "failed", message: "export failed token=[redacted] authorization=[redacted] body=[redacted]", }), }); expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_failed", tenantId: "tenant_1", attemptsMade: 1 }); expect(markExportJobFailed).toHaveBeenCalledWith({ id: "export_job_failed", tenantId: "tenant_1", attemptsMade: 1, errorName: "Error", errorMessage: "export failed token=[redacted] authorization=[redacted] body=[redacted]", }); expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("payload-secret"); expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("service-secret"); expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("Bearer-secret"); }); test("keeps export jobs running until terminal retry failure", async () => { const exportService: TestExportService = { runExport: vi.fn(async () => { throw new Error("temporary export outage"); }), }; const markExportJobRunning = vi.fn(async () => undefined); const markExportJobFailed = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobFailed }); const descriptor = buildExportJob({ exportJobId: "export_job_retrying", tenantId: "tenant_1", requestedByUserId: "user_1", exportType: "members", format: "csv", }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({ action: "export.error", message: "Export job failed.", }); expect(markExportJobFailed).not.toHaveBeenCalled(); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toMatchObject({ action: "export.error", message: "Export job failed.", }); expect(markExportJobFailed).toHaveBeenCalledWith({ id: "export_job_retrying", tenantId: "tenant_1", attemptsMade: 3, errorName: "Error", errorMessage: "temporary export outage", }); }); test("rejects malformed export payload kinds", async () => { const exportService: TestExportService = { runExport: vi.fn(async () => ({ artifactBucket: "unused", artifactKey: "unused", rowCount: 0, expiresAt: new Date() })), }; const markExportJobRunning = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ exportService, markExportJobRunning }); await expect(registry[queueNames.exports]({ kind: "export.other", exportJobId: "bad", tenantId: "tenant_1" } as never, {})).rejects.toMatchObject({ action: "export.error", message: "Unsupported export payload kind.", details: { code: "invalid_payload" }, }); expect(markExportJobRunning).not.toHaveBeenCalled(); expect(exportService.runExport).not.toHaveBeenCalled(); }); test("runs tenant-less admin sales import and export jobs through their dedicated service", async () => { const adminSalesDataService = { runImport: vi.fn(async () => ({ totalRows: 3, acceptedRows: 2, rejectedRows: 1, })), runExport: vi.fn(async () => ({ rowCount: 2, artifactBucket: "hivegrid", artifactKey: "admin-sales/exports/export_1.csv", expiresAt: new Date("2026-08-01T00:00:00.000Z"), })), }; const registry = buildHandlerRegistry({ adminSalesDataService }); const importJob = buildAdminSalesContactsImportJob({ importJobId: "import_1", requestedByAdminUserId: "admin_1", }); const exportJob = buildAdminSalesContactsExportJob({ exportJobId: "export_1", requestedByAdminUserId: "admin_1", }); await expect( registry[importJob.queueName](importJob.payload, {}) ).resolves.toEqual({ handled: true, action: "admin-sales.contacts-import.completed", details: { importJobId: "import_1", requestedByAdminUserId: "admin_1", totalRows: 3, acceptedRows: 2, rejectedRows: 1, }, }); await expect( registry[exportJob.queueName](exportJob.payload, {}) ).resolves.toMatchObject({ handled: true, action: "admin-sales.contacts-export.completed", details: { exportJobId: "export_1", rowCount: 2, expiresAt: "2026-08-01T00:00:00.000Z", }, }); }); test("fails admin sales data jobs when their dedicated service is missing", async () => { const descriptor = buildAdminSalesContactsImportJob({ importJobId: "import_missing", requestedByAdminUserId: "admin_1", }); const registry = buildHandlerRegistry({}); await expect( registry[descriptor.queueName](descriptor.payload, {}) ).rejects.toMatchObject({ action: "import.error", message: "Admin sales data worker service is not configured.", details: { code: "admin_sales_data_service_missing", kind: "admin-sales.contacts-import", }, }); }); test("fails import jobs when service is missing", async () => { const markImportJobRunning = vi.fn(async () => undefined); const markImportJobSucceeded = vi.fn(async () => undefined); const markImportJobFailed = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_1", tenantId: "tenant_1", requestedByUserId: "user_1", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_1/members.csv", requestId: "req_import_1", metadata: { attemptsMade: 2 }, }); const registry = buildHandlerRegistry({ markImportJobRunning, markImportJobSucceeded, markImportJobFailed }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "import.error", message: "Import service is not configured.", details: { importJobId: "import_job_1", tenantId: "tenant_1", importType: "members", format: "csv", requestId: "req_import_1", status: "failed", attemptsMade: 2, code: "import_service_missing", }, }); expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_1", tenantId: "tenant_1", attemptsMade: 2 }); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_1", tenantId: "tenant_1", attemptsMade: 2, errorName: "ImportHandlerError", errorMessage: "Import service is not configured.", }); expect(markImportJobSucceeded).not.toHaveBeenCalled(); }); test("rejects import jobs whose source key is outside the tenant prefix before scanning", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })), }; const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" }) ), }; const markImportJobRunning = vi.fn(async () => undefined); const markImportJobFailed = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_cross_tenant", tenantId: "tenant_1", requestedByUserId: "user_1", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_2/members.csv", requestId: "req_import_cross_tenant", }); const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobFailed }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "import.error", message: "Import source storage access denied.", details: { importJobId: "import_job_cross_tenant", tenantId: "tenant_1", requestId: "req_import_cross_tenant", status: "failed", code: "source_access_denied", reason: "tenant_mismatch", }, }); expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_cross_tenant", tenantId: "tenant_1", attemptsMade: 0 }); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_cross_tenant", tenantId: "tenant_1", attemptsMade: 0, errorName: "ImportHandlerError", errorMessage: "Import source storage access denied. reason=tenant_mismatch", }); expect(scanService.scanObject).not.toHaveBeenCalled(); expect(importService.runImport).not.toHaveBeenCalled(); }); test("runs import jobs through the injected service and persists duplicate-aware counters", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 4, acceptedRows: 2, rejectedRows: 2, duplicateCount: 1, validationReport: { summary: { format: "csv" as const, totalRows: 4, acceptedRows: 2, rejectedRows: 2, diagnostics: [], duplicateGroups: 1, duplicateRows: 2, }, errorReport: { diagnostics: [], rejectedRows: 2, errorReportKey: "reports/import_job_2.json" }, }, errorReportBucket: "diagnostics", })), }; const markImportJobRunning = vi.fn(async () => undefined); const markImportJobSucceeded = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_2", tenantId: "tenant_2", requestedByUserId: "user_2", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_2/members.csv", requestId: "req_import_2", }); const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" })), }; const events: string[] = []; vi.mocked(scanService.scanObject).mockImplementation(async () => { events.push("scan"); return scanResult({ verdict: "clean", bucket: "imports", key: "tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" }); }); vi.mocked(importService.runImport).mockImplementation(async () => { events.push("import"); return { rowCount: 4, acceptedRows: 2, rejectedRows: 2, duplicateCount: 1, validationReport: { summary: { format: "csv" as const, totalRows: 4, acceptedRows: 2, rejectedRows: 2, diagnostics: [], duplicateGroups: 1, duplicateRows: 2, }, errorReport: { diagnostics: [], rejectedRows: 2, errorReportKey: "reports/import_job_2.json" }, }, errorReportBucket: "diagnostics", }; }); const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobSucceeded }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "import.completed", details: { importJobId: "import_job_2", tenantId: "tenant_2", importType: "members", format: "csv", requestId: "req_import_2", status: "completed", attemptsMade: 0, rowCount: 4, acceptedRows: 2, rejectedRows: 2, duplicateCount: 1, errorReportBucket: "diagnostics", errorReportKey: "reports/import_job_2.json", }, }); expect(events).toEqual(["scan", "import"]); expect(scanService.scanObject).toHaveBeenCalledWith({ bucket: "imports", key: "tenants/tenant_2/members.csv", tenantId: "tenant_2", uploadId: "import_job_2", contentType: undefined, metadata: { requestId: "req_import_2", importJobId: "import_job_2", importType: "members", format: "csv", }, }); expect(importService.runImport).toHaveBeenCalledWith(descriptor.payload); expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_2", tenantId: "tenant_2", attemptsMade: 0 }); expect(markImportJobSucceeded).toHaveBeenCalledWith({ id: "import_job_2", tenantId: "tenant_2", rowCount: 4, acceptedRows: 2, rejectedRows: 2, attemptsMade: 0, }); }); test("uses BullMQ attempt metadata for import job persistence when available", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 10, acceptedRows: 9, rejectedRows: 1, duplicateCount: 0 })), }; const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_2/imports/import_job_bullmq_attempt.csv", scannedAt: new Date() }) ), }; const markImportJobRunning = vi.fn(async () => undefined); const markImportJobSucceeded = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobSucceeded }); const descriptor = buildImportJob({ importJobId: "import_job_bullmq_attempt", tenantId: "tenant_2", requestedByUserId: "user_2", importType: "members", format: "csv", requestId: "req_import_bullmq_attempt", sourceBucket: "imports", sourceKey: "tenants/tenant_2/imports/import_job_bullmq_attempt.csv", metadata: { attemptsMade: 1 }, }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 3 })).resolves.toMatchObject({ action: "import.completed", details: { attemptsMade: 4 }, }); expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_bullmq_attempt", tenantId: "tenant_2", attemptsMade: 4 }); expect(markImportJobSucceeded).toHaveBeenCalledWith(expect.objectContaining({ id: "import_job_bullmq_attempt", tenantId: "tenant_2", attemptsMade: 4 })); }); test("persists failed import diagnostics and throws redacted handler errors", async () => { const importService: TestImportService = { runImport: vi.fn(async () => { throw new Error("import failed token=secret-token body=row-secret password=hunter2"); }), }; const markImportJobFailed = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_3", tenantId: "tenant_3", requestedByUserId: "user_3", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_3/members.csv", requestId: "req_import_3", metadata: { attemptsMade: 1 }, }); const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenant_3/members.csv", scannedAt: new Date(), engine: "clamav" })), }; const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "import.error", message: "Import job failed.", details: { importJobId: "import_job_3", tenantId: "tenant_3", requestId: "req_import_3", status: "failed", attemptsMade: 1, message: "import failed token=[redacted] body=[redacted] password=[redacted]", }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("secret-token"); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_3", tenantId: "tenant_3", attemptsMade: 1, errorName: "Error", errorMessage: "import failed token=[redacted] body=[redacted] password=[redacted]", }); }); test("keeps import jobs running until terminal retry failure", async () => { const importService: TestImportService = { runImport: vi.fn(async () => { throw new Error("temporary import outage"); }), }; const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_3/members.csv", scannedAt: new Date() })), }; const markImportJobFailed = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_retrying", tenantId: "tenant_3", requestedByUserId: "user_3", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_3/members.csv", }); const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed }); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({ action: "import.error", message: "Import job failed.", }); expect(markImportJobFailed).not.toHaveBeenCalled(); await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toMatchObject({ action: "import.error", message: "Import job failed.", }); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_retrying", tenantId: "tenant_3", attemptsMade: 3, errorName: "Error", errorMessage: "temporary import outage", }); }); test("rejects import jobs with configured import service when scan service is missing", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })), }; const markImportJobFailed = vi.fn(async () => undefined); const markImportJobSucceeded = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_missing_scan", tenantId: "tenant_scan", requestedByUserId: "user_scan", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_scan/members.csv", requestId: "req_missing_scan", metadata: { attemptsMade: 3 }, }); const registry = buildHandlerRegistry({ importService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed, markImportJobSucceeded, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "import.error", message: "Import source scan failed.", details: { importJobId: "import_job_missing_scan", tenantId: "tenant_scan", requestId: "req_missing_scan", status: "failed", attemptsMade: 3, scan: { verdict: "error", code: "scan_service_missing", message: "Import scan service is not configured.", }, }, }); expect(importService.runImport).not.toHaveBeenCalled(); expect(markImportJobSucceeded).not.toHaveBeenCalled(); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_missing_scan", tenantId: "tenant_scan", attemptsMade: 3, errorName: "ImportHandlerError", errorMessage: "Import source scan failed. verdict=error code=scan_service_missing requestId=req_missing_scan message=Import scan service is not configured.", }); }); test("rejects infected import source scans without running the import", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })), }; const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "infected", bucket: "imports", key: "tenant_bad/members.csv", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav", signature: "Eicar-Test-Signature", quarantineKey: "quarantine/tenant_bad/members.csv", }) ), }; const markImportJobFailed = vi.fn(async () => undefined); const markImportJobSucceeded = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_infected", tenantId: "tenant_bad", requestedByUserId: "user_bad", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_bad/members.csv", requestId: "req_infected_scan", }); const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed, markImportJobSucceeded, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "import.error", message: "Import source scan failed.", details: { importJobId: "import_job_infected", tenantId: "tenant_bad", requestId: "req_infected_scan", status: "failed", attemptsMade: 0, scan: { verdict: "infected", code: "source_infected", engine: "clamav", signature: "Eicar-Test-Signature", quarantineKey: "quarantine/tenant_bad/members.csv", message: "Import source object failed malware scan.", }, }, }); expect(importService.runImport).not.toHaveBeenCalled(); expect(markImportJobSucceeded).not.toHaveBeenCalled(); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_infected", tenantId: "tenant_bad", attemptsMade: 0, errorName: "ImportHandlerError", errorMessage: "Import source scan failed. verdict=infected code=source_infected requestId=req_infected_scan message=Import source object failed malware scan.", }); }); test("rejects scanner errors for import sources with redacted diagnostics", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })), }; const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "error", bucket: "imports", key: "tenant_error/members.csv", scannedAt: new Date("2026-05-08T00:00:00.000Z"), error: { code: "scanner_failed", message: "scanner failed token=top-secret body=file-bytes authorization=Bearer-secret", upstream: { name: "ScannerError", message: "upstream password=hunter2 body=row-secret", status: 503, requestId: "scan_req_error", }, }, }) ), }; const markImportJobFailed = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_scan_error", tenantId: "tenant_error", requestedByUserId: "user_error", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_error/members.csv", requestId: "req_scan_error", metadata: { attemptsMade: 5 }, }); const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed, markImportJobSucceeded: vi.fn(async () => undefined), }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "import.error", details: { importJobId: "import_job_scan_error", tenantId: "tenant_error", requestId: "req_scan_error", status: "failed", attemptsMade: 5, scan: { verdict: "error", code: "scanner_failed", message: "scanner failed token=[redacted] body=[redacted] authorization=[redacted]", upstream: { name: "ScannerError", message: "upstream password=[redacted] body=[redacted]", status: 503, requestId: "scan_req_error", }, }, }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("top-secret"); expect(importService.runImport).not.toHaveBeenCalled(); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_scan_error", tenantId: "tenant_error", attemptsMade: 5, errorName: "ImportHandlerError", errorMessage: "Import source scan failed. verdict=error code=scanner_failed requestId=req_scan_error message=scanner failed token=[redacted] body=[redacted] authorization=[redacted]", }); expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("top-secret"); expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("Bearer-secret"); expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("hunter2"); }); test("rejects scanner promise rejections for import sources with redacted diagnostics", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })), }; const scanService: TestScanService = { scanObject: vi.fn(async () => { throw new Error("scan rejected token=reject-secret body=file-bytes password=hunter2"); }), }; const markImportJobFailed = vi.fn(async () => undefined); const descriptor = buildImportJob({ importJobId: "import_job_scan_rejected", tenantId: "tenant_rejected", requestedByUserId: "user_rejected", importType: "members", format: "csv", sourceBucket: "imports", sourceKey: "tenants/tenant_rejected/members.csv", requestId: "req_scan_rejected", }); const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed, markImportJobSucceeded: vi.fn(async () => undefined), }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "import.error", details: { importJobId: "import_job_scan_rejected", tenantId: "tenant_rejected", requestId: "req_scan_rejected", status: "failed", attemptsMade: 0, scan: { verdict: "error", code: "scanner_rejected", message: "scan rejected token=[redacted] body=[redacted] password=[redacted]", }, }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("reject-secret"); expect(importService.runImport).not.toHaveBeenCalled(); expect(markImportJobFailed).toHaveBeenCalledWith({ id: "import_job_scan_rejected", tenantId: "tenant_rejected", attemptsMade: 0, errorName: "ImportHandlerError", errorMessage: "Import source scan failed. verdict=error code=scanner_rejected requestId=req_scan_rejected message=scan rejected token=[redacted] body=[redacted] password=[redacted]", }); }); test("rejects malformed import payload kinds without leaking source secrets", async () => { const importService: TestImportService = { runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })), }; const markImportJobRunning = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ importService, markImportJobRunning }); await expect( registry[queueNames.imports]({ kind: "import.other", importJobId: "bad", tenantId: "tenant_1", sourceKey: "secret/body.csv" } as never, {}) ).rejects.toMatchObject({ action: "import.error", message: "Unsupported import payload kind.", details: { code: "invalid_payload" }, }); await expect( registry[queueNames.imports]({ kind: "import.other", importJobId: "bad", tenantId: "tenant_1", sourceKey: "secret/body.csv" } as never, {}) ).rejects.not.toThrow("secret/body.csv"); expect(markImportJobRunning).not.toHaveBeenCalled(); expect(importService.runImport).not.toHaveBeenCalled(); }); test("runs clean file scan jobs through the injected scan service", async () => { const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav", }) ), }; const registry = buildHandlerRegistry({ scanService }); const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/file.pdf", tenantId: "tenant_1", uploadId: "upload_1", contentType: "application/pdf", metadata: { requestId: "req_scan_clean" }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "scan.clean", details: { verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", uploadId: "upload_1", engine: "clamav", }, }); expect(scanService.scanObject).toHaveBeenCalledWith({ bucket: "uploads", key: "tenant_1/file.pdf", tenantId: "tenant_1", uploadId: "upload_1", contentType: "application/pdf", metadata: { requestId: "req_scan_clean" }, }); }); test("returns infected scan action with signature and quarantine key", async () => { const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "infected", bucket: "uploads", key: "tenant_1/infected.exe", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav", signature: "Eicar-Test-Signature", quarantineKey: "quarantine/tenant_1/infected.exe", }) ), }; const registry = buildHandlerRegistry({ scanService }); const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/infected.exe", uploadId: "upload_bad" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "scan.infected", details: { verdict: "infected", bucket: "uploads", key: "tenant_1/infected.exe", uploadId: "upload_bad", engine: "clamav", signature: "Eicar-Test-Signature", quarantineKey: "quarantine/tenant_1/infected.exe", }, }); }); test("writes the verdict back onto File.scanStatus when fileId + writer are present", async () => { const cleanScan: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav" }) ), }; const infectedScan: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "infected", bucket: "uploads", key: "tenant_1/bad.exe", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav", signature: "Eicar-Test-Signature", }) ), }; const fileScanStatusWriter = { recordVerdict: vi.fn(async () => true) }; const filePreviewEnqueuer = { enqueueAfterCleanScan: vi.fn(async () => undefined) }; const cleanRegistry = buildHandlerRegistry({ scanService: cleanScan, fileScanStatusWriter, filePreviewEnqueuer }); const cleanJob = buildFileScanJob({ bucket: "uploads", key: "tenant_1/file.pdf", tenantId: "tenant_1", fileId: "file_1", contentType: "application/pdf", }); await cleanRegistry[cleanJob.queueName](cleanJob.payload, {}); expect(fileScanStatusWriter.recordVerdict).toHaveBeenCalledWith({ fileId: "file_1", storageKey: "tenant_1/file.pdf", tenantId: "tenant_1", verdict: "clean", }); expect(filePreviewEnqueuer.enqueueAfterCleanScan).toHaveBeenCalledWith({ bucket: "uploads", contentType: "application/pdf", fileId: "file_1", storageKey: "tenant_1/file.pdf", tenantId: "tenant_1", }); const infectedRegistry = buildHandlerRegistry({ scanService: infectedScan, fileScanStatusWriter, filePreviewEnqueuer }); const infectedJob = buildFileScanJob({ bucket: "uploads", key: "tenant_1/bad.exe", tenantId: "tenant_1", fileId: "file_2" }); await infectedRegistry[infectedJob.queueName](infectedJob.payload, {}); expect(fileScanStatusWriter.recordVerdict).toHaveBeenCalledWith({ fileId: "file_2", storageKey: "tenant_1/bad.exe", tenantId: "tenant_1", verdict: "infected", }); expect(filePreviewEnqueuer.enqueueAfterCleanScan).toHaveBeenCalledTimes(1); }); test("skips the verdict write-back when the job carries no fileId", async () => { const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav" }) ), }; const fileScanStatusWriter = { recordVerdict: vi.fn(async () => true) }; const registry = buildHandlerRegistry({ scanService, fileScanStatusWriter }); const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/file.pdf", tenantId: "tenant_1" }); await registry[descriptor.queueName](descriptor.payload, {}); expect(fileScanStatusWriter.recordVerdict).not.toHaveBeenCalled(); }); test("rejects storage error scan verdicts with redacted diagnostics", async () => { const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "error", bucket: "uploads", key: "tenant_1/error.bin", scannedAt: new Date("2026-05-08T00:00:00.000Z"), error: { code: "scanner_failed", message: "Storage scanner failed while scanning object.", upstream: { name: "ScannerError", message: "scanner failed token=top-secret body=file-bytes credential=aws-secret", status: 503, requestId: "scan_req_1", }, }, }) ), }; const registry = buildHandlerRegistry({ scanService }); const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/error.bin", uploadId: "upload_error" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "scan.error", details: { verdict: "error", bucket: "uploads", key: "tenant_1/error.bin", uploadId: "upload_error", code: "scanner_failed", message: "Storage scanner failed while scanning object.", upstream: { name: "ScannerError", message: "scanner failed token=[redacted] body=[redacted] credential=[redacted]", status: 503, requestId: "scan_req_1", }, }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("top-secret"); }); test("rejects scanner service rejections without leaking secret-bearing messages", async () => { const scanService: TestScanService = { scanObject: vi.fn(async () => { throw new Error("scanner rejected authorization=Bearer-secret body=file-bytes password=hunter2"); }), }; const registry = buildHandlerRegistry({ scanService }); const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/rejected.bin", uploadId: "upload_rejected" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "scan.error", details: { verdict: "error", bucket: "uploads", key: "tenant_1/rejected.bin", uploadId: "upload_rejected", code: "scanner_rejected", message: "scanner rejected authorization=[redacted] body=[redacted] password=[redacted]", }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("Bearer-secret"); }); test("rejects file scan jobs when scan service is missing", async () => { const registry = buildHandlerRegistry(); const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/missing-service.bin", uploadId: "upload_missing" }); await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "scan.error", message: "File scan service is not configured.", details: { verdict: "error", bucket: "uploads", key: "tenant_1/missing-service.bin", uploadId: "upload_missing", code: "scan_service_missing", }, }); }); test("rejects malformed file scan payload kinds", async () => { const scanService: TestScanService = { scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "unused", scannedAt: new Date() })), }; const registry = buildHandlerRegistry({ scanService }); await expect(registry[queueNames.fileScan]({ kind: "scan.other", bucket: "uploads", key: "bad" } as never, {})).rejects.toMatchObject({ action: "scan.error", message: "Unsupported file scan payload kind.", details: { verdict: "error", code: "invalid_payload" }, }); expect(scanService.scanObject).not.toHaveBeenCalled(); }); test("runs KWP project sync on the dedicated KWP queue under a binding-scoped lease", async () => { const store = createSharedStore({ namespace: "kwp-project-handler-lease-test", adapter: new MemoryAdapter() }); const syncKwpProjectBinding = vi.fn(async () => ({ runId: "run_1", state: "synced" as const, projectId: "project_1", stageChanged: true, })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectBinding }); const descriptor = buildKwpProjectSyncJob({ tenantId: "tenant_1", bindingId: "binding_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "kwp.project-sync.completed", details: { kind: "kwp.project-sync", tenantId: "tenant_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00", runId: "run_1", state: "synced", projectId: "project_1", stageChanged: true, }, }); expect(syncKwpProjectBinding).toHaveBeenCalledOnce(); expect(await store.get("job-lease:kwp-project:tenant_1:binding_1")).toBeNull(); }); test("replays a pre-rollout KWP project job left on the generic import queue", async () => { const store = createSharedStore({ namespace: "kwp-project-handler-legacy-queue-test", adapter: new MemoryAdapter() }); const syncKwpProjectBinding = vi.fn(async () => ({ runId: "run_legacy", state: "unchanged" as const, projectId: "project_1", stageChanged: false, })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectBinding }); const descriptor = buildKwpProjectSyncJob({ tenantId: "tenant_1", bindingId: "binding_1", dataConnectionId: "connection_1", cadenceBucket: "legacy", }); await expect(registry[queueNames.imports](descriptor.payload as never, {})).resolves.toMatchObject({ action: "kwp.project-sync.completed", details: { runId: "run_legacy" }, }); expect(syncKwpProjectBinding).toHaveBeenCalledOnce(); }); test("acknowledges overlapping KWP project replays without a second provider call", async () => { const store = createSharedStore({ namespace: "kwp-project-handler-overlap-test", adapter: new MemoryAdapter() }); await store.set("job-lease:kwp-project:tenant_1:binding_1", "other-replica", { ttlSeconds: 120 }); const syncKwpProjectBinding = vi.fn(); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectBinding }); const descriptor = buildKwpProjectSyncJob({ tenantId: "tenant_1", bindingId: "binding_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({ handled: true, action: "kwp.import.overlap-skipped", details: { kind: "kwp.project-sync", tenantId: "tenant_1", reason: "active-lease" }, }); expect(syncKwpProjectBinding).not.toHaveBeenCalled(); }); test("runs a bounded KWP project-catalog batch and schedules its continuation", async () => { const store = createSharedStore({ namespace: "kwp-catalog-handler-test", adapter: new MemoryAdapter() }); const syncKwpProjectCatalog = vi.fn(async () => ({ discoveredCount: 50_000, hydratedCount: 20, processed: 20, hasMore: true, status: "syncing" as const, })); const enqueueKwpProjectCatalogContinuation = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectCatalog, enqueueKwpProjectCatalogContinuation, }); const descriptor = buildKwpProjectCatalogSyncJob({ tenantId: "tenant_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ handled: true, action: "kwp.project-catalog-sync.completed", details: { kind: "kwp.project-catalog-sync", discoveredCount: 50_000, hydratedCount: 20, processed: 20, hasMore: true, }, }); expect(syncKwpProjectCatalog).toHaveBeenCalledOnce(); expect(enqueueKwpProjectCatalogContinuation).toHaveBeenCalledWith(descriptor.payload); expect(await store.get("job-lease:kwp-catalog:tenant_1:connection_1")).toBeNull(); }); test("runs core project inbound reconciliation and schedules archive discovery independently", async () => { const store = createSharedStore({ namespace: "kwp-project-inbound-handler-test", adapter: new MemoryAdapter() }); const syncKwpProjectBinding = vi.fn(async () => ({ runId: "project-run", state: "unchanged" as const, projectId: "project_1", stageChanged: false, })); const syncKwpProjectInbound = vi.fn(async () => ({ runId: "inbound-run", projectId: "project_1", appointments: 2, tasks: 3, materialPositions: 4, failedAreas: [], changed: true, })); const enqueueKwpArchiveDiscovery = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectBinding, syncKwpProjectInbound, enqueueKwpArchiveDiscovery, }); const descriptor = buildKwpProjectSyncJob({ tenantId: "tenant_1", bindingId: "binding_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "kwp.project-sync.completed", details: { inboundRunId: "inbound-run", inboundChanged: true, appointments: 2, tasks: 3, materialPositions: 4, failedAreas: [], archiveDiscoveryScheduled: true, }, }); expect(syncKwpProjectBinding).toHaveBeenCalledBefore(syncKwpProjectInbound); expect(enqueueKwpArchiveDiscovery).toHaveBeenCalledWith(descriptor.payload); }); test("continues bounded archive discovery and fans files out without rerunning core inbound", async () => { const store = createSharedStore({ namespace: "kwp-archive-discovery-handler-test", adapter: new MemoryAdapter() }); const archiveFile = { externalId: "1:2:3", headerId: 1, revisionId: 3, revision: 1, folderId: 0, displayName: "plan.pdf", documentType: "PLAN", providerFileType: "PDF", providerSizeBytes: 123, lastModifiedAt: null, }; const cursor = { pendingFolderIds: [6], visitedFolderIds: [0, 1, 2, 3, 4, 5] }; const syncKwpArchiveDiscovery = vi.fn(async () => ({ archiveFiles: [archiveFile], processedFolders: 5, cursor, })); const enqueueKwpArchiveFileJobs = vi.fn(async () => 1); const enqueueKwpArchiveDiscovery = vi.fn(async () => undefined); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpArchiveDiscovery, enqueueKwpArchiveFileJobs, enqueueKwpArchiveDiscovery, }); const descriptor = buildKwpArchiveDiscoveryJob({ tenantId: "tenant_1", bindingId: "binding_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00:00.000Z", batchNumber: 0, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "kwp.archive-discovery.completed", details: { processedFolders: 5, archiveJobs: 1, hasMore: true }, }); expect(enqueueKwpArchiveFileJobs).toHaveBeenCalledWith(descriptor.payload, [archiveFile]); expect(enqueueKwpArchiveDiscovery).toHaveBeenCalledWith(descriptor.payload, cursor); }); test("stops retrying deterministic KWP failures but keeps provider outages retryable", async () => { const deterministic = Object.assign(new Error("schema changed"), { code: "contract_drift" }); const providerOutage = Object.assign(new Error("upstream unavailable"), { code: "provider_outage" }); const descriptor = buildKwpArchiveDiscoveryJob({ tenantId: "tenant_1", bindingId: "binding_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00:00.000Z", batchNumber: 0, }); const deterministicRegistry = buildHandlerRegistry({ syncKwpArchiveDiscovery: vi.fn(async () => { throw deterministic; }) }); await expect(deterministicRegistry[descriptor.queueName](descriptor.payload, {})).rejects.toBeInstanceOf(UnrecoverableError); const retryableRegistry = buildHandlerRegistry({ syncKwpArchiveDiscovery: vi.fn(async () => { throw providerOutage; }) }); await expect(retryableRegistry[descriptor.queueName](descriptor.payload, {})).rejects.toBe(providerOutage); }); test("runs one KWP archive file under an external-file lease", async () => { const store = createSharedStore({ namespace: "kwp-archive-handler-test", adapter: new MemoryAdapter() }); const syncKwpArchiveFile = vi.fn(async () => ({ runId: "archive-run", fileId: "file_1", projectId: "project_1", storageKey: "tenants/tenant_1/kwp-archive/file.pdf", state: "created" as const, })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpArchiveFile }); const descriptor = buildKwpArchiveFileSyncJob({ tenantId: "tenant_1", bindingId: "binding_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23T12:00", archiveFile: { externalId: "1:2:3", headerId: 1, revisionId: 3, revision: 1, folderId: 0, displayName: "plan.pdf", documentType: null, providerFileType: "PDF", providerSizeBytes: 123, lastModifiedAt: null, }, }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ action: "kwp.archive-file-sync.completed", details: { kind: "kwp.archive-file-sync", fileId: "file_1", state: "created", }, }); expect(syncKwpArchiveFile).toHaveBeenCalledOnce(); expect(await store.get("job-lease:kwp-archive:tenant_1:connection_1:1:2:3")).toBeNull(); }); test("runs the KWP contract sentinel independently of project capability state", async () => { const store = createSharedStore({ namespace: "kwp-contract-handler-lease-test", adapter: new MemoryAdapter() }); const runKwpContractSentinel = vi.fn(async () => ({ status: "unchanged" as const, fingerprint: "f".repeat(64), })); const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runKwpContractSentinel }); const descriptor = buildKwpContractSentinelJob({ tenantId: "tenant_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23", }); await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({ handled: true, action: "kwp.contract-sentinel.completed", details: { kind: "kwp.contract-sentinel", tenantId: "tenant_1", dataConnectionId: "connection_1", cadenceBucket: "2026-07-23", status: "unchanged", }, }); expect(runKwpContractSentinel).toHaveBeenCalledOnce(); }); // Phase 06 #21 / F-015 — plugin-invoke dispatch. describe("plugin-invoke handler (F-015)", () => { const samplePayload = { kind: "plugin.invoke" as const, tenantId: "tenant_abc", pluginId: "com.acme.demo", version: "1.0.0", hookName: "tenant.created", input: { tenantId: "tenant_abc", slug: "acme", ownerEmail: "owner@example.com", createdAt: "2026-05-28T10:00:00.000Z" }, metadata: { logicalHookName: "tenant.created" }, }; test("rejects malformed queue payloads before invoking plugin deps", async () => { const errorLogs: Array<{ fields: Record; msg: string }> = []; const pluginInvokeDeps = { loadPlugin: vi.fn(), loadCapabilityGrants: vi.fn(), loadBinary: vi.fn(), verifySignature: vi.fn(), consumeRateLimit: vi.fn(), invokeInSandbox: vi.fn(), recordAudit: vi.fn(), trustedPublicKeysHex: [], now: () => 1_000_000, }; const registry = buildHandlerRegistry({ pluginInvokeDeps: pluginInvokeDeps as never, logger: { info: () => undefined, error: (fields, msg) => errorLogs.push({ fields, msg: msg ?? "" }), }, }); await expect( registry[queueNames.pluginInvoke]( { kind: "plugin.invoke", tenantId: "tenant_abc", pluginId: "com.acme.demo", version: 1, input: { secret: "not logged" }, } as never, {} as never ) ).rejects.toMatchObject({ action: "plugin.invoke.invalid_payload", message: "Invalid plugin-invoke queue payload.", details: { queueName: queueNames.pluginInvoke, code: "invalid_payload", invalidFields: ["version", "hookName"], }, }); expect(pluginInvokeDeps.loadPlugin).not.toHaveBeenCalled(); expect(pluginInvokeDeps.invokeInSandbox).not.toHaveBeenCalled(); expect(errorLogs).toHaveLength(1); expect(errorLogs[0]).toMatchObject({ msg: "Invalid plugin-invoke queue payload.", fields: { queueName: queueNames.pluginInvoke, code: "invalid_payload", invalidFields: ["version", "hookName"], payload: { kind: "plugin.invoke", tenantId: "tenant_abc", pluginId: "com.acme.demo", version: "number", hookName: "undefined", }, }, }); expect(JSON.stringify(errorLogs[0])).not.toContain("not logged"); }); test("throws when pluginInvokeDeps is not configured (fail-loud, no silent ack)", async () => { const errorLogs: Array<{ fields: Record; msg: string }> = []; const registry = buildHandlerRegistry({ logger: { info: () => undefined, error: (fields, msg) => errorLogs.push({ fields, msg: msg ?? "" }), }, }); await expect(registry[queueNames.pluginInvoke](samplePayload, {} as never)).rejects.toThrow(/pluginInvokeDeps is not configured/); expect(errorLogs).toHaveLength(1); expect(errorLogs[0]!.fields).toMatchObject({ queue: queueNames.pluginInvoke, pluginId: samplePayload.pluginId, hookName: samplePayload.hookName, }); }); test("dispatches to handlePluginInvokeJob and maps success outcome", async () => { const pluginInvokeDeps = { loadPlugin: vi.fn(async () => ({ tenantId: samplePayload.tenantId, pluginId: samplePayload.pluginId, version: samplePayload.version, manifest: { capabilities: [], hooks: [{ name: "tenant.created", export: "tenant.created" }], wasmSha256: "x", signedBy: "y" } as never, wasmSha256: "x", signaturePublicKey: "y", enabled: true, })), loadCapabilityGrants: vi.fn(async () => [{ capabilityName: "hook:tenant.created", scopeJson: null }]), loadBinary: vi.fn(async () => new Uint8Array([0])), verifySignature: vi.fn(() => ({ ok: true as const })), consumeRateLimit: vi.fn(async () => ({ allowed: true, remaining: 99 })), invokeInSandbox: vi.fn(async () => ({ ok: true as const, output: "ok", fuelUsed: 42 })), recordAudit: vi.fn(async () => undefined), trustedPublicKeysHex: [], now: () => 1_000_000, }; const registry = buildHandlerRegistry({ pluginInvokeDeps: pluginInvokeDeps as never }); const result = await registry[queueNames.pluginInvoke](samplePayload, {} as never); expect(result).toMatchObject({ handled: true, action: "plugin.invoke.completed", details: { pluginId: samplePayload.pluginId, hookName: samplePayload.hookName, fuelUsed: 42 }, }); expect(pluginInvokeDeps.invokeInSandbox).toHaveBeenCalledTimes(1); expect(pluginInvokeDeps.invokeInSandbox).toHaveBeenCalledWith( expect.objectContaining({ capabilityGrants: [{ capabilityName: "hook:tenant.created", scopeJson: null }] }) ); expect(pluginInvokeDeps.recordAudit).toHaveBeenCalledTimes(1); }); test("maps signature_invalid outcome to plugin.invoke.failed", async () => { const pluginInvokeDeps = { loadPlugin: vi.fn(async () => ({ tenantId: samplePayload.tenantId, pluginId: samplePayload.pluginId, version: samplePayload.version, manifest: { capabilities: [], hooks: [{ name: "tenant.created", export: "tenant.created" }], wasmSha256: "x", signedBy: "y" } as never, wasmSha256: "x", signaturePublicKey: "y", enabled: true, })), loadCapabilityGrants: vi.fn(async () => [{ capabilityName: "hook:tenant.created", scopeJson: null }]), loadBinary: vi.fn(async () => new Uint8Array([0])), verifySignature: vi.fn(() => ({ ok: false as const, reason: "binary_hash_mismatch" as const })), consumeRateLimit: vi.fn(async () => ({ allowed: true, remaining: 99 })), invokeInSandbox: vi.fn(), recordAudit: vi.fn(async () => undefined), trustedPublicKeysHex: [], now: () => 1_000_000, }; const registry = buildHandlerRegistry({ pluginInvokeDeps: pluginInvokeDeps as never }); const result = await registry[queueNames.pluginInvoke](samplePayload, {} as never); expect(result).toMatchObject({ handled: true, action: "plugin.invoke.failed", details: { reason: "signature_invalid", message: "binary_hash_mismatch" }, }); expect(pluginInvokeDeps.invokeInSandbox).not.toHaveBeenCalled(); }); }); });