Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/worker/src/handlers.test.ts
T

5960 lines
242 KiB
TypeScript

import { createHash } from "node:crypto";
import { describe, expect, test, vi } from "vitest";
import {
buildBrowserSessionCleanupJob,
buildBrowserSessionRunJob,
buildBrowserUsageFlushJob,
buildCleanupJob,
buildExportJob,
buildFileScanJob,
buildImportJob,
buildAdminSalesContactsImportJob,
buildAdminSalesContactsExportJob,
buildKwpArchiveDiscoveryJob,
buildKwpArchiveFileSyncJob,
buildKwpContractSentinelJob,
buildKwpProjectCatalogSyncJob,
buildKwpProjectSyncJob,
buildNotificationJob,
buildPlatformNotificationJob,
buildTransactionalMailJob,
queueNames,
type WebhookDeliveryJobDescriptor,
} from "@hivegrid/queue";
import { buildHandlerRegistry, type HandlerDependencies } from "./handlers.js";
import { PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, PrivacyGdprExportSizeLimitExceededError } from "./jobs/privacy-gdpr-export.js";
import { PrivacyGdprExportPutOutcomeUnknownError } from "./jobs/privacy-gdpr-export-bootstrap.js";
import { TenantExportQuiescedError } from "./jobs/tenant-export-quiescence.js";
import type { CreateNotificationDeliveryInput } from "@hivegrid/db/notifications.server";
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
import { hashWorkspaceOwnerInvitationMailContent } from "@hivegrid/db/workspace-owner-invite-mail-contract.server";
import {
bindEncryptedPortalSetupMailToOutbox,
hashPortalSetupMailContent,
materializePortalSetupMailContent,
PORTAL_SETUP_MAIL_SOURCE,
PORTAL_SETUP_TOKEN_PLACEHOLDER,
} from "@hivegrid/db/portal-setup-mail-contract.server";
import { MailProviderError, type MailProvider } from "@hivegrid/mail";
import type { StorageScanResult } from "@hivegrid/storage";
import { createSharedStore, MemoryAdapter } from "@hivegrid/shared-store";
import { UnrecoverableError } from "bullmq";
import {
markWorkspaceOwnerInvitationDeliveryStarted,
markWorkspaceOwnerInvitationProviderAccepted,
redactTerminalWorkspaceOwnerInvitationMailPayload,
redactWorkspaceOwnerInvitationMailPayload,
} from "./security/workspace-owner-invitation-secret.js";
import {
markPortalSetupInvitationDeliveryStarted,
markPortalSetupInvitationProviderAccepted,
redactTerminalPortalSetupInvitationMailPayload,
} from "./security/portal-setup-invitation-secret.js";
type TestScanService = NonNullable<HandlerDependencies["scanService"]>;
type TestExportService = NonNullable<HandlerDependencies["exportService"]>;
type TestPrivacyGdprExportService = NonNullable<HandlerDependencies["privacyGdprExportService"]>;
type TestPortalGdprExportService = NonNullable<HandlerDependencies["portalGdprExportService"]>;
type TestPortalGdprExportRetentionService = NonNullable<HandlerDependencies["portalGdprExportRetentionService"]>;
type TestImportService = NonNullable<HandlerDependencies["importService"]>;
type TestStorageDeleteService = NonNullable<HandlerDependencies["storageDeleteService"]>;
type TestGdprRetentionJob = Awaited<ReturnType<NonNullable<HandlerDependencies["listExpiredGdprExportArtifacts"]>>>[number];
type TestRetentionListInput = Parameters<NonNullable<HandlerDependencies["listSoftDeletedExportJobsForHardDelete"]>>[0];
type TestRetentionJob = Awaited<ReturnType<NonNullable<HandlerDependencies["listSoftDeletedExportJobsForHardDelete"]>>>[number];
const privacyAttempt = {
attemptId: "attempt_12345678",
artifactBucket: "exports",
artifactKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_job_1/attempt_12345678.zip",
};
function createPrivacyGdprService(
runExport: TestPrivacyGdprExportService["runExport"],
overrides: Partial<TestPrivacyGdprExportService> = {}
): TestPrivacyGdprExportService {
return {
withTenantBarrier: async (_payload, writer) => writer(async () => undefined),
resolveReady: vi.fn(async () => null),
resolveReclaimableProcessing: vi.fn(async () => null),
createAttempt: vi.fn(() => privacyAttempt),
runExport,
deleteArtifact: vi.fn(async () => undefined),
terminalizeReadyAfterQuiescence: vi.fn(async () => undefined),
...overrides,
};
}
const publicWebhookAddressResolver = vi.fn(async () => ["93.184.216.34"]);
const scanResult = <Result extends StorageScanResult>(result: Result): Result => result;
const retentionJob = (overrides: Partial<TestRetentionJob> & Pick<TestRetentionJob, "id" | "artifactBucket" | "artifactKey">): TestRetentionJob => ({
tenantId: "tenant_1",
status: "FAILED",
rowCount: 1,
expiresAt: new Date("2026-05-08T00:00:00.000Z"),
deletedAt: new Date("2026-05-09T00:00:00.000Z"),
requestId: `req_${overrides.id}`,
...overrides,
});
function createActiveTenantReader(ids: string[] = ["tenant_1"]): NonNullable<HandlerDependencies["activeTenantReader"]> {
return {
tenant: {
findMany: vi.fn(async (args: { take: number; cursor?: { id: string }; skip?: 1 }) => {
const start = args.cursor ? ids.indexOf(args.cursor.id) + (args.skip ?? 0) : 0;
return ids.slice(start, start + args.take).map((id) => ({ id }));
}),
},
};
}
function createOwnerDeliveryDeps() {
return {
claimOwnerInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) => ({
state: "claimed" as const,
payload: markWorkspaceOwnerInvitationDeliveryStarted(payload),
})),
acceptOwnerInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) => markWorkspaceOwnerInvitationProviderAccepted(payload)),
abandonOwnerInvitationDelivery: vi.fn(async () => ({ state: "invalidated" })),
};
}
function createPortalSetupMailFixture(token = "p".repeat(43), tokenId = "portal_token_1") {
const setupTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Kundenportal-Zugang einrichten";
const text = `Zugang einrichten: https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}`;
const html = `<a href="https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}">Zugang einrichten</a>`;
const encrypted = {
tenantId: "tenant_1",
to: ["customer@example.test"],
subject,
text,
html,
metadata: {
source: PORTAL_SETUP_MAIL_SOURCE,
invitationSource: "portal_admin_invite",
portalId: "portal_1",
portalUserId: "portal_user_1",
portalTokenId: tokenId,
setupTokenHash,
expiresAt: "2026-07-25T12:00:00.000Z",
mailContentHash: hashPortalSetupMailContent({
recipient: "customer@example.test",
subject,
text,
html,
}),
encryptedCapability: "v1.encrypted-capability",
},
};
const bound = bindEncryptedPortalSetupMailToOutbox(encrypted, {
outboxEventId: `outbox_${tokenId}`,
dispatchKey: `portal-setup:${tokenId}`,
});
const materialized = materializePortalSetupMailContent(bound, token);
const descriptor = buildTransactionalMailJob(materialized as Parameters<typeof buildTransactionalMailJob>[0]);
return { descriptor, token };
}
function createPortalSetupDeliveryDeps() {
return {
claimPortalSetupInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) => ({
state: "claimed" as const,
payload: markPortalSetupInvitationDeliveryStarted(payload),
})),
acceptPortalSetupInvitationDelivery: vi.fn(async ({ payload }: { tenantId: string; payload: unknown }) =>
markPortalSetupInvitationProviderAccepted(payload)
),
terminalizePortalSetupInvitation: vi.fn(async () => ({ state: "invalidated" })),
};
}
describe("worker handlers", () => {
test("sends queue-provider transactional email jobs through configured mail provider", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_1" })),
};
const registry = buildHandlerRegistry({ mailProvider });
const descriptor = buildTransactionalMailJob({
from: { email: "sender@example.com", name: "Sender" },
to: ["user@example.com", { email: "admin@example.com", name: "Admin" }],
cc: [{ email: "cc@example.com", name: "CC" }],
bcc: ["bcc@example.com"],
replyTo: "reply@example.com",
subject: "Welcome to HiveGrid",
text: "Hello from HiveGrid",
html: "<p>Hello from <strong>HiveGrid</strong></p>",
templateId: "tmpl_welcome",
attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }],
tags: ["welcome"],
precedenceBulk: true,
sendAt: 1_778_580_000,
inReplyTo: "message_1",
references: ["message_0"],
settings: { trackClicks: true, trackOpens: false, trackContent: true },
headers: [{ name: "X-Test", value: "1" }],
listUnsubscribe: "<mailto:unsubscribe@example.com>",
templateData: { firstName: "Ada", tenantName: "Example Co" },
tenantId: "tenant_1",
userId: "user_1",
metadata: { source: "queue-provider" },
});
expect(descriptor).toMatchObject({
queueName: queueNames.email,
jobName: "send-transactional-email",
payload: { kind: "mail.transactional" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "mail.sent",
details: { provider: "mailjet", messageId: "msg_1" },
});
expect(mailProvider.send).toHaveBeenCalledTimes(1);
expect(mailProvider.send).toHaveBeenCalledWith({
from: { email: "sender@example.com", name: "Sender" },
to: ["user@example.com", { email: "admin@example.com", name: "Admin" }],
cc: [{ email: "cc@example.com", name: "CC" }],
bcc: ["bcc@example.com"],
replyTo: "reply@example.com",
subject: "Welcome to HiveGrid",
text: "Hello from HiveGrid",
html: "<p>Hello from <strong>HiveGrid</strong></p>",
templateId: "tmpl_welcome",
attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }],
tags: ["welcome"],
precedenceBulk: true,
sendAt: 1_778_580_000,
inReplyTo: "message_1",
references: ["message_0"],
settings: { trackClicks: true, trackOpens: false, trackContent: true },
headers: [{ name: "X-Test", value: "1" }],
listUnsubscribe: "<mailto:unsubscribe@example.com>",
personalization: [
{ email: "user@example.com", data: { firstName: "Ada", tenantName: "Example Co" } },
{ email: "admin@example.com", data: { firstName: "Ada", tenantName: "Example Co" } },
],
});
});
test("claims and acknowledges a portal setup capability around exactly one provider call", async () => {
const { descriptor, token } = createPortalSetupMailFixture();
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_portal_setup" })),
};
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const delivery = createPortalSetupDeliveryDeps();
const registry = buildHandlerRegistry({ mailProvider, ...delivery });
await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload })).resolves.toEqual({
handled: true,
action: "mail.sent",
details: { provider: "mailjet", messageId: "msg_portal_setup" },
});
expect(delivery.claimPortalSetupInvitationDelivery).toHaveBeenCalledOnce();
expect(mailProvider.send).toHaveBeenCalledOnce();
expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ settings: { trackClicks: false, trackOpens: false } }));
expect(delivery.acceptPortalSetupInvitationDelivery).toHaveBeenCalledOnce();
expect(updatePayload).toHaveBeenCalledTimes(2);
expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token);
expect(updatePayload.mock.calls.at(-1)?.[0]).toMatchObject({
metadata: expect.objectContaining({ portalSetupDeliveryStatus: "provider_accepted", secretBodyRedacted: true }),
});
});
test("rotates an uncertain portal setup capability without a second provider call", async () => {
const { descriptor, token } = createPortalSetupMailFixture("u".repeat(43), "portal_token_uncertain");
const startedPayload = markPortalSetupInvitationDeliveryStarted(descriptor.payload);
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const terminalizePortalSetupInvitation = vi.fn(async () => ({ state: "invalidated" }));
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const registry = buildHandlerRegistry({
mailProvider,
claimPortalSetupInvitationDelivery: vi.fn(async () => ({ state: "uncertain" as const, payload: startedPayload })),
terminalizePortalSetupInvitation,
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload })).rejects.toBeInstanceOf(UnrecoverableError);
expect(mailProvider.send).not.toHaveBeenCalled();
expect(terminalizePortalSetupInvitation).toHaveBeenCalledOnce();
expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token);
});
test("deduplicates a provider-accepted portal setup retry after BullMQ persistence failed", async () => {
const { descriptor } = createPortalSetupMailFixture("a".repeat(43), "portal_token_accepted");
const acceptedPayload = markPortalSetupInvitationProviderAccepted(descriptor.payload);
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_once" })),
};
const claimPortalSetupInvitationDelivery = vi
.fn()
.mockResolvedValueOnce({ state: "claimed", payload: markPortalSetupInvitationDeliveryStarted(descriptor.payload) })
.mockResolvedValueOnce({ state: "provider_accepted", payload: acceptedPayload });
const acceptPortalSetupInvitationDelivery = vi.fn(async () => acceptedPayload);
const registry = buildHandlerRegistry({
mailProvider,
claimPortalSetupInvitationDelivery,
acceptPortalSetupInvitationDelivery,
});
const firstUpdatePayload = vi.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error("redis failed"));
await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: firstUpdatePayload })).rejects.toThrow("redis failed");
await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: vi.fn(async () => undefined) })).resolves.toMatchObject({
details: { deduplicated: true },
});
expect(mailProvider.send).toHaveBeenCalledOnce();
expect(acceptPortalSetupInvitationDelivery).toHaveBeenCalledOnce();
});
test("redacts and revokes a portal setup capability after an ambiguous provider failure", async () => {
const { descriptor, token } = createPortalSetupMailFixture("f".repeat(43), "portal_token_failure");
const providerError = new Error(`provider failed for /setup/${token}`);
const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(providerError)) };
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const delivery = createPortalSetupDeliveryDeps();
const registry = buildHandlerRegistry({ mailProvider, ...delivery });
let thrown: unknown;
try {
await registry[descriptor.queueName](descriptor.payload, {}, { updatePayload });
} catch (error) {
thrown = error;
}
expect(thrown).toBeInstanceOf(UnrecoverableError);
expect((thrown as Error).message).not.toContain(token);
expect(delivery.terminalizePortalSetupInvitation).toHaveBeenCalledOnce();
expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token);
expect(updatePayload.mock.calls.at(-1)?.[0]).toEqual(redactTerminalPortalSetupInvitationMailPayload(descriptor.payload));
});
test("persists a token-free OWNER invitation marker before the mail job can complete", async () => {
const token = "o".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_owner_1" })),
};
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const ownerDelivery = createOwnerDeliveryDeps();
const registry = buildHandlerRegistry({ mailProvider, ...ownerDelivery });
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_1",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload })).resolves.toEqual({
handled: true,
action: "mail.sent",
details: { provider: "mailjet", messageId: "msg_owner_1" },
});
expect(mailProvider.send).toHaveBeenCalledOnce();
expect(JSON.stringify(vi.mocked(mailProvider.send).mock.calls[0]?.[0])).toContain(token);
expect(updatePayload).toHaveBeenCalledOnce();
expect(JSON.stringify(updatePayload.mock.calls[0]?.[0])).not.toContain(token);
expect(updatePayload.mock.calls[0]?.[0]).toMatchObject({
kind: "mail.transactional",
metadata: expect.objectContaining({ source: "workspace-owner-invitation", secretBodyRedacted: true }),
});
});
test("deduplicates an OWNER retry when provider delivery succeeded but BullMQ redaction failed", async () => {
const token = "u".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_owner_2" })),
};
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_2",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
const acceptedPayload = markWorkspaceOwnerInvitationProviderAccepted(descriptor.payload);
const claimOwnerInvitationDelivery = vi
.fn()
.mockResolvedValueOnce({ state: "claimed", payload: markWorkspaceOwnerInvitationDeliveryStarted(descriptor.payload) })
.mockResolvedValueOnce({ state: "provider_accepted", payload: acceptedPayload });
const acceptOwnerInvitationDelivery = vi.fn(async () => acceptedPayload);
const registry = buildHandlerRegistry({ mailProvider, claimOwnerInvitationDelivery, acceptOwnerInvitationDelivery });
await expect(
registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: vi.fn(async () => Promise.reject(new Error("redis write failed"))) })
).rejects.toThrow("redis write failed");
const retryUpdatePayload = vi.fn(async () => undefined);
await expect(registry[descriptor.queueName](descriptor.payload, {}, { updatePayload: retryUpdatePayload })).resolves.toMatchObject({
details: { deduplicated: true },
});
expect(mailProvider.send).toHaveBeenCalledOnce();
expect(claimOwnerInvitationDelivery).toHaveBeenCalledTimes(2);
expect(acceptOwnerInvitationDelivery).toHaveBeenCalledOnce();
expect(JSON.stringify(retryUpdatePayload.mock.calls)).not.toContain(token);
});
test("fails closed before calling the provider for a corrupted OWNER mail contract", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const registry = buildHandlerRegistry({ mailProvider });
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject: "Workspace invitation",
html: `<a href="https://app.example.test/invitation/${"v".repeat(43)}">Accept</a>`,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_corrupt",
invitationTokenHash: "broken",
issuanceKeyHash: "b".repeat(64),
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow(/contract is invalid/u);
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("routes OWNER provider construction failures through terminal fail-closed cleanup", async () => {
const token = "c".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const terminalizeOwnerInvitation = vi.fn(async () => ({ state: "invalidated" }));
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const registry = buildHandlerRegistry({ env: {}, terminalizeOwnerInvitation });
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_provider_setup",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
outboxEventId: "outbox_provider_setup",
dispatchKey: "mail:workspace-owner-invite:invite_provider_setup",
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload })).rejects.toThrow(
/MAILJET_API_KEY/u
);
expect(terminalizeOwnerInvitation).toHaveBeenCalledWith({ tenantId: "tenant_1", payload: descriptor.payload });
expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token);
});
test("redacts and invalidates a permanently failed OWNER invitation before BullMQ dead-letters it", async () => {
const token = "p".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const providerFailure = new Error(`provider permanently unavailable html=${html} token=${token}`);
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => Promise.reject(providerFailure)),
};
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const ownerDelivery = createOwnerDeliveryDeps();
const registry = buildHandlerRegistry({ mailProvider, ...ownerDelivery });
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_terminal",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
let thrown: unknown;
try {
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload });
} catch (error) {
thrown = error;
}
expect(thrown).toBeInstanceOf(Error);
expect((thrown as Error).message).toContain("provider permanently unavailable");
expect((thrown as Error).message).not.toContain(token);
expect((thrown as Error).message).not.toContain(html);
expect(ownerDelivery.abandonOwnerInvitationDelivery).toHaveBeenCalledWith({
tenantId: "tenant_1",
payload: descriptor.payload,
lastError: expect.stringContaining("rotation is required"),
});
expect(updatePayload).toHaveBeenCalledOnce();
expect(JSON.stringify(updatePayload.mock.calls[0]?.[0])).not.toContain(token);
expect(updatePayload.mock.calls[0]?.[0]).toMatchObject({
metadata: expect.objectContaining({ secretBodyRedacted: true, ownerInvitationDeliveryStatus: "rotation_required" }),
});
});
test("rearms the durable outbox marker when final Bull invalidation transiently fails", async () => {
const token = "d".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const terminalizeOwnerInvitation = vi.fn(async () => Promise.reject(new Error("database unavailable")));
const scheduleOwnerInvitationRecovery = vi.fn(async () => undefined);
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const registry = buildHandlerRegistry({
mailProvider: { id: "mailjet", send: vi.fn(async () => Promise.reject(new Error("provider failed"))) },
terminalizeOwnerInvitation,
scheduleOwnerInvitationRecovery,
});
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_recovery",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
outboxEventId: "outbox_recovery",
dispatchKey: "mail:workspace-owner-invite:invite_recovery",
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload })).rejects.toThrow(
/cleanup failed/u
);
expect(scheduleOwnerInvitationRecovery).toHaveBeenCalledWith(expect.objectContaining({ tenantId: "tenant_1", outboxEventId: "outbox_recovery" }));
expect(JSON.stringify(scheduleOwnerInvitationRecovery.mock.calls)).not.toContain(token);
});
test("manually retrying a terminal-redacted Bull job retries invalidation instead of deduplicating it", async () => {
const token = "e".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const terminalizeOwnerInvitation = vi.fn(async () => ({ state: "invalidated" }));
const mailProvider: MailProvider = { id: "mailjet", send: vi.fn() };
const registry = buildHandlerRegistry({ mailProvider, terminalizeOwnerInvitation });
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_manual_retry",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
outboxEventId: "outbox_manual_retry",
dispatchKey: "mail:workspace-owner-invite:invite_manual_retry",
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload;
await expect(registry[descriptor.queueName](redacted, {})).resolves.toMatchObject({ details: { deduplicated: true } });
expect(terminalizeOwnerInvitation).toHaveBeenCalledWith({ tenantId: "tenant_1", payload: redacted });
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("retries durable recovery for a terminal-redacted Bull job after both final DB operations failed", async () => {
const token = "f".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_recovery_retry",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
outboxEventId: "outbox_recovery_retry",
dispatchKey: "mail:workspace-owner-invite:invite_recovery_retry",
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload;
const scheduleOwnerInvitationRecovery = vi.fn(async () => undefined);
const mailProvider: MailProvider = { id: "mailjet", send: vi.fn() };
const registry = buildHandlerRegistry({
mailProvider,
terminalizeOwnerInvitation: vi.fn(async () => Promise.reject(new Error("database still unavailable"))),
scheduleOwnerInvitationRecovery,
});
await expect(registry[descriptor.queueName](redacted, {})).resolves.toMatchObject({ details: { deduplicated: true } });
expect(scheduleOwnerInvitationRecovery).toHaveBeenCalledWith(
expect.objectContaining({ tenantId: "tenant_1", outboxEventId: "outbox_recovery_retry", redactedPayload: redacted })
);
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("does not deduplicate a terminal-redacted Bull job while invalidation and scheduling still fail", async () => {
const token = "g".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_recovery_blocked",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
outboxEventId: "outbox_recovery_blocked",
dispatchKey: "mail:workspace-owner-invite:invite_recovery_blocked",
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
const redacted = redactTerminalWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload;
const mailProvider: MailProvider = { id: "mailjet", send: vi.fn() };
const registry = buildHandlerRegistry({
mailProvider,
terminalizeOwnerInvitation: vi.fn(async () => Promise.reject(new Error("database unavailable"))),
scheduleOwnerInvitationRecovery: vi.fn(async () => Promise.reject(new Error("database unavailable"))),
});
await expect(registry[descriptor.queueName](redacted, {})).rejects.toThrow(/recovery failed/u);
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("never retries an OWNER provider attempt after its delivery claim became ambiguous", async () => {
const token = "r".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => Promise.reject(new Error(`provider temporarily unavailable html=${html} token=${token}`))),
};
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const ownerDelivery = createOwnerDeliveryDeps();
const registry = buildHandlerRegistry({ mailProvider, ...ownerDelivery });
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_retry",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
let thrown: unknown;
try {
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3, updatePayload });
} catch (error) {
thrown = error;
}
expect(thrown).toBeInstanceOf(Error);
expect((thrown as Error).message).toContain("temporarily unavailable");
expect((thrown as Error).message).not.toContain(token);
expect((thrown as Error).message).not.toContain(html);
expect(ownerDelivery.abandonOwnerInvitationDelivery).toHaveBeenCalledOnce();
expect(updatePayload).toHaveBeenCalledOnce();
expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token);
});
test("durably schedules terminal OWNER recovery when ambiguous-delivery invalidation keeps failing", async () => {
const token = "q".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => Promise.reject(new Error("provider outcome unavailable"))),
};
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_abandon_recovery",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
outboxEventId: "outbox_abandon_recovery",
dispatchKey: "mail:workspace-owner-invite:invite_abandon_recovery",
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
const claimOwnerInvitationDelivery = vi
.fn()
.mockResolvedValueOnce({ state: "claimed", payload: markWorkspaceOwnerInvitationDeliveryStarted(descriptor.payload) })
.mockResolvedValueOnce({ state: "uncertain", payload: markWorkspaceOwnerInvitationDeliveryStarted(descriptor.payload) });
const abandonOwnerInvitationDelivery = vi.fn(async () => Promise.reject(new Error("database invalidation unavailable")));
const terminalizeOwnerInvitation = vi.fn(async () => Promise.reject(new Error("database terminalization unavailable")));
const scheduleOwnerInvitationRecovery = vi.fn(async () => undefined);
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const registry = buildHandlerRegistry({
mailProvider,
claimOwnerInvitationDelivery,
abandonOwnerInvitationDelivery,
terminalizeOwnerInvitation,
scheduleOwnerInvitationRecovery,
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 2, updatePayload })).rejects.toThrow(
/ambiguous delivery cleanup failed/u
);
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 2, updatePayload })).rejects.toThrow(
/cleanup failed/u
);
expect(mailProvider.send).toHaveBeenCalledOnce();
expect(claimOwnerInvitationDelivery).toHaveBeenCalledTimes(2);
expect(abandonOwnerInvitationDelivery).toHaveBeenCalledTimes(2);
expect(terminalizeOwnerInvitation).toHaveBeenCalledOnce();
expect(scheduleOwnerInvitationRecovery).toHaveBeenCalledWith(
expect.objectContaining({
tenantId: "tenant_1",
outboxEventId: "outbox_abandon_recovery",
redactedPayload: expect.objectContaining({
metadata: expect.objectContaining({
ownerInvitationDeliveryStatus: "rotation_required",
secretBodyRedacted: true,
}),
}),
})
);
expect(JSON.stringify(updatePayload.mock.calls)).not.toContain(token);
expect(JSON.stringify(scheduleOwnerInvitationRecovery.mock.calls)).not.toContain(token);
});
test("keeps ordinary MEMBER transactional-mail failures outside OWNER recovery", async () => {
const providerFailure = new Error("member mail provider unavailable");
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => Promise.reject(providerFailure)),
};
const claimOwnerInvitationDelivery = vi.fn();
const abandonOwnerInvitationDelivery = vi.fn();
const terminalizeOwnerInvitation = vi.fn();
const scheduleOwnerInvitationRecovery = vi.fn();
const registry = buildHandlerRegistry({
mailProvider,
claimOwnerInvitationDelivery,
abandonOwnerInvitationDelivery,
terminalizeOwnerInvitation,
scheduleOwnerInvitationRecovery,
});
const descriptor = buildTransactionalMailJob({
to: ["member@example.test"],
subject: "Workspace invitation",
html: "<p>Ordinary member invitation</p>",
tenantId: "tenant_1",
metadata: { source: "member-invitation" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3, updatePayload: vi.fn() })).rejects.toBe(
providerFailure
);
expect(mailProvider.send).toHaveBeenCalledOnce();
expect(claimOwnerInvitationDelivery).not.toHaveBeenCalled();
expect(abandonOwnerInvitationDelivery).not.toHaveBeenCalled();
expect(terminalizeOwnerInvitation).not.toHaveBeenCalled();
expect(scheduleOwnerInvitationRecovery).not.toHaveBeenCalled();
});
test("completes a redacted OWNER invitation retry without sending the mail twice", async () => {
const token = "n".repeat(43);
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
const subject = "Workspace invitation";
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "unexpected" })),
};
const registry = buildHandlerRegistry({ mailProvider });
const descriptor = buildTransactionalMailJob({
to: ["owner@example.test"],
subject,
html,
tenantId: "tenant_1",
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_3",
invitationTokenHash,
issuanceKeyHash: "b".repeat(64),
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
recipient: "owner@example.test",
subject,
html,
rawToken: token,
invitationTokenHash,
}),
},
});
const redactedPayload = redactWorkspaceOwnerInvitationMailPayload(descriptor.payload) as typeof descriptor.payload;
await expect(registry[descriptor.queueName](redactedPayload, {})).resolves.toEqual({
handled: true,
action: "mail.sent",
details: { deduplicated: true },
});
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("preserves per-recipient transactional email template data", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_personalized" })),
};
const registry = buildHandlerRegistry({ mailProvider });
const descriptor = buildTransactionalMailJob({
to: ["user@example.com", "admin@example.com"],
subject: "Welcome to HiveGrid",
templateId: "tmpl_welcome",
templateData: {
"user@example.com": { firstName: "Ada" },
"admin@example.com": { firstName: "Grace" },
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
handled: true,
action: "mail.sent",
});
expect(mailProvider.send).toHaveBeenCalledWith({
to: ["user@example.com", "admin@example.com"],
subject: "Welcome to HiveGrid",
text: undefined,
html: undefined,
templateId: "tmpl_welcome",
personalization: [
{ email: "user@example.com", data: { firstName: "Ada" } },
{ email: "admin@example.com", data: { firstName: "Grace" } },
],
});
});
test("creates Mailjet provider from explicit worker env", async () => {
const fetchMock = vi.fn(
async () =>
new Response(JSON.stringify({ Messages: [{ Status: "success", To: [{ Email: "user@example.com", MessageUUID: "msg_env" }] }] }), {
status: 200,
headers: { "content-type": "application/json" },
})
);
const registry = buildHandlerRegistry({
env: {
MAILJET_API_KEY: "fixture-api-key",
MAILJET_SECRET_KEY: "fixture-secret-key",
MAILJET_FROM_EMAIL: "sender@example.com",
MAILJET_FROM_NAME: "Fixture Sender",
},
});
const originalFetch = globalThis.fetch;
vi.stubGlobal("fetch", fetchMock);
try {
await expect(
registry[queueNames.email](
{
kind: "mail.transactional",
to: ["user@example.com"],
subject: "Hello",
text: "Body",
},
{}
)
).resolves.toEqual({
handled: true,
action: "mail.sent",
details: { provider: "mailjet", messageId: "msg_env" },
});
} finally {
vi.stubGlobal("fetch", originalFetch);
}
expect(fetchMock).toHaveBeenCalledWith(
"https://api.mailjet.com/v3.1/send",
expect.objectContaining({
headers: expect.objectContaining({ Authorization: `Basic ${Buffer.from("fixture-api-key:fixture-secret-key", "utf8").toString("base64")}` }),
body: expect.stringContaining("Fixture Sender"),
})
);
});
test("reports missing Mailjet fixture env with key names only", async () => {
const registry = buildHandlerRegistry({ env: { MAILJET_API_KEY: "super-secret-api-key" } });
await expect(
registry[queueNames.email](
{
kind: "mail.transactional",
to: ["user@example.com"],
subject: "Hello",
text: "Body",
},
{}
)
).rejects.toMatchObject({ message: "MAILJET_SECRET_KEY is required for Mailjet email delivery." });
await expect(
registry[queueNames.email](
{
kind: "mail.transactional",
to: ["user@example.com"],
subject: "Hello",
text: "Body",
},
{}
)
).rejects.not.toThrow("super-secret-api-key");
});
test("propagates provider rejection diagnostics without exposing payload secrets", async () => {
const providerError = new MailProviderError("Mailjet rejected message", {
provider: "mailjet",
statusCode: 422,
messageId: "msg_rejected",
responseBody: { message: "Invalid recipient" },
});
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => {
throw providerError;
}),
};
const registry = buildHandlerRegistry({ mailProvider });
const descriptor = buildTransactionalMailJob({
to: ["user@example.com"],
subject: "Hello",
text: "Body",
templateData: { otp: "123456", apiKey: "payload-secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
message: "Mailjet rejected message",
provider: "mailjet",
statusCode: 422,
messageId: "msg_rejected",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("payload-secret");
expect(mailProvider.send).toHaveBeenCalledTimes(2);
});
test("notifies the responsible TenantUser after permanent portal status-mail delivery failure", async () => {
const providerError = new MailProviderError("Mailjet unavailable", { provider: "mailjet", statusCode: 503 });
const mailProvider: MailProvider = { id: "mailjet", send: vi.fn(async () => Promise.reject(providerError)) };
const notifyPortalStatusMailFailure = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ mailProvider, notifyPortalStatusMailFailure });
const descriptor = buildTransactionalMailJob({
tenantId: "tenant_1",
to: ["customer@example.test"],
subject: "Status",
text: "Body",
metadata: {
idempotencyKey: "portal-status:change_1:7f8b67d82471956e2fb6e2e39d4d9ae9",
recipientKind: "portal_user",
portalStatusMail: {
version: 1,
stageChangeId: "change_1",
projectId: "project_1",
expectedStageId: "stage_new",
internalRecipientUserId: "tenant_user_1",
},
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).rejects.toThrow("Mailjet unavailable");
expect(notifyPortalStatusMailFailure).not.toHaveBeenCalled();
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("Mailjet unavailable");
expect(notifyPortalStatusMailFailure).toHaveBeenCalledWith({
tenantId: "tenant_1",
projectId: "project_1",
expectedStageId: "stage_new",
stageChangeId: "change_1",
internalRecipientUserId: "tenant_user_1",
});
expect(JSON.stringify(notifyPortalStatusMailFailure.mock.calls)).not.toContain("portalUserId");
});
test("delivers queued webhook descriptors and updates bounded legacy attempt diagnostics", async () => {
const fetchMock = vi.fn(
async () =>
new Response("accepted Authorization: Bearer response-secret", { status: 202, statusText: "Accepted", headers: { "content-type": "text/plain" } })
);
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
const descriptor = {
queueName: queueNames.email,
jobName: "deliver-webhook",
payload: {
kind: "webhook.deliver",
url: "https://example.com/webhook",
method: "POST",
eventName: "legacy.event_webhook_attempt",
body: { id: "row_1" },
metadata: { legacyAttemptId: "attempt_1", legacyAttemptTenantId: "tenant_1" },
},
} satisfies WebhookDeliveryJobDescriptor;
await expect(registry[queueNames.email](descriptor.payload, {})).resolves.toMatchObject({
handled: true,
action: "webhook.delivered",
});
expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ method: "POST", redirect: "manual" }));
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_1", tenantId: "tenant_1" },
expect.objectContaining({
finishedAt: expect.any(Date),
success: true,
status: 202,
message: "Accepted",
body: expect.stringContaining('"status":202'),
})
);
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>;
const body = updateCalls[0]?.[1]?.body ?? "";
expect(body).toContain('"bodyBytes":');
expect(body).not.toContain("response-secret");
expect(body).not.toContain("Authorization");
});
test("rejects unsafe queued webhook URLs as non-retryable before fetching", async () => {
const fetchMock = vi.fn(async () => new Response(null, { status: 204 }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
const blockedUrls = [
"ftp://example.com/webhook",
"https://localhost/webhook",
"https://service.internal/webhook",
"https://service/webhook",
"http://127.0.0.1/webhook",
"http://10.1.2.3/webhook",
"http://169.254.169.254/latest/meta-data",
"http://[::1]/webhook",
"http://[fd00::1]/webhook",
];
for (const url of blockedUrls) {
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url,
eventName: "entity.created",
metadata: { legacyAttemptId: `attempt_${blockedUrls.indexOf(url)}`, legacyAttemptTenantId: "tenant_blocked" },
},
{}
)
).resolves.toMatchObject({
handled: true,
action: "webhook.rejected",
});
}
expect(fetchMock).not.toHaveBeenCalled();
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledTimes(blockedUrls.length);
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
expect.objectContaining({ tenantId: "tenant_blocked" }),
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
message: expect.stringMatching(/Webhook endpoint/),
})
);
});
test("rejects public webhook hostnames that resolve to private addresses before fetching", async () => {
const fetchMock = vi.fn(async () => new Response(null, { status: 204 }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const privateAddressResolver = vi.fn(async () => ["169.254.169.254"]);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: privateAddressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://public.example.test/latest/meta-data",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_dns_blocked", legacyAttemptTenantId: "tenant_dns" },
},
{}
)
).resolves.toMatchObject({
handled: true,
action: "webhook.rejected",
});
expect(privateAddressResolver).toHaveBeenCalledWith("public.example.test");
expect(fetchMock).not.toHaveBeenCalled();
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_dns_blocked", tenantId: "tenant_dns" },
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
message: "Webhook endpoint resolved to a blocked address.",
})
);
});
test("follows queued webhook redirects only after validating the target", async () => {
const fetchMock = vi
.fn()
.mockResolvedValueOnce(new Response(null, { status: 307, headers: { location: "https://hooks.example.com/next" } }))
.mockResolvedValueOnce(new Response("accepted", { status: 202, statusText: "Accepted" }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://example.com/webhook",
method: "POST",
headers: {
authorization: "Bearer customer-secret",
"x-hivegrid-signature": "sha256=customer-signature",
"x-delivery-id": "delivery-1",
},
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_redirect", legacyAttemptTenantId: "tenant_redirect" },
},
{}
)
).resolves.toMatchObject({
handled: true,
action: "webhook.delivered",
});
expect(fetchMock).toHaveBeenNthCalledWith(1, "https://example.com/webhook", expect.objectContaining({ method: "POST", redirect: "manual" }));
expect(fetchMock).toHaveBeenNthCalledWith(2, "https://hooks.example.com/next", expect.objectContaining({ method: "POST", redirect: "manual" }));
const redirectedHeaders = new Headers(fetchMock.mock.calls[1]?.[1]?.headers);
expect(redirectedHeaders.get("authorization")).toBeNull();
expect(redirectedHeaders.get("x-hivegrid-signature")).toBeNull();
expect(redirectedHeaders.get("x-delivery-id")).toBe("delivery-1");
expect(redirectedHeaders.get("content-type")).toBe("application/json");
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_redirect", tenantId: "tenant_redirect" },
expect.objectContaining({ success: true, status: 202 })
);
});
test("rejects queued webhook redirect hostnames that resolve to private addresses before following them", async () => {
const fetchMock = vi.fn(async () => new Response(null, { status: 302, headers: { location: "https://metadata.example.test/latest" } }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const addressResolver = vi.fn(async (hostname: string) => (hostname === "metadata.example.test" ? ["10.0.0.10"] : ["93.184.216.34"]));
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: addressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_redirect_dns_blocked", legacyAttemptTenantId: "tenant_redirect" },
},
{},
{ attemptsMade: 0, attemptsLimit: 3 }
)
).resolves.toMatchObject({
handled: true,
action: "webhook.rejected",
});
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(addressResolver).toHaveBeenCalledWith("example.com");
expect(addressResolver).toHaveBeenCalledWith("metadata.example.test");
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_redirect_dns_blocked", tenantId: "tenant_redirect" },
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
message: "Webhook endpoint resolved to a blocked address.",
})
);
});
test("rejects queued webhook redirects to unsafe targets without fetching them", async () => {
const fetchMock = vi.fn(async () => new Response(null, { status: 302, headers: { location: "http://127.0.0.1/webhook" } }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_redirect_blocked", legacyAttemptTenantId: "tenant_redirect" },
},
{},
{ attemptsMade: 0, attemptsLimit: 3 }
)
).resolves.toMatchObject({
handled: true,
action: "webhook.rejected",
});
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ redirect: "manual" }));
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_redirect_blocked", tenantId: "tenant_redirect" },
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
message: "Webhook endpoint host is not allowed.",
})
);
});
test("updates failed legacy attempt diagnostics and rejects non-2xx webhook responses", async () => {
const fetchMock = vi.fn(async () => new Response("upstream rejected token=upstream-secret body=payload", { status: 503, statusText: "Unavailable" }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_2", legacyAttemptTenantId: "tenant_2" },
},
{}
)
).rejects.toThrow("Webhook delivery failed with 503.");
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledTimes(1);
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_2", tenantId: "tenant_2" },
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
status: 503,
message: "Webhook delivery failed with 503.",
body: expect.stringContaining('"status":503'),
})
);
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>;
const body = updateCalls[0]?.[1]?.body ?? "";
expect(body).toContain('"bodyBytes":');
expect(body).not.toContain("upstream-secret");
expect(body).not.toContain("payload");
});
test("keeps queued webhook attempt diagnostics open until retries are exhausted", async () => {
const fetchMock = vi.fn(async () => new Response("temporarily unavailable", { status: 503, statusText: "Unavailable" }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
const payload = {
kind: "webhook.deliver" as const,
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_retry", legacyAttemptTenantId: "tenant_retry" },
};
await expect(registry[queueNames.email](payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow("Webhook delivery failed with 503.");
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_retry", tenantId: "tenant_retry" },
expect.not.objectContaining({
finishedAt: expect.any(Date),
success: false,
})
);
expect(updateLegacyEventWebhookAttempt).toHaveBeenLastCalledWith(
{ id: "attempt_retry", tenantId: "tenant_retry" },
expect.objectContaining({ status: 503, message: "Webhook delivery failed with 503." })
);
await expect(registry[queueNames.email](payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("Webhook delivery failed with 503.");
expect(updateLegacyEventWebhookAttempt).toHaveBeenLastCalledWith(
{ id: "attempt_retry", tenantId: "tenant_retry" },
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
status: 503,
})
);
});
test("does not retry permanent webhook client failures", async () => {
const fetchMock = vi.fn(async () => new Response("missing", { status: 404, statusText: "Not Found" }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
const payload = {
kind: "webhook.deliver" as const,
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_permanent", legacyAttemptTenantId: "tenant_permanent" },
};
await expect(registry[queueNames.email](payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({
handled: true,
action: "webhook.failed",
details: { status: 404, eventName: "entity.created", retryable: false },
});
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
{ id: "attempt_permanent", tenantId: "tenant_permanent" },
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
status: 404,
message: "Webhook delivery failed with 404.",
})
);
});
test("caps webhook response diagnostics before persisting legacy attempt bodies", async () => {
const oversizedBody = `${"a".repeat(4_096)}token=tail-secret`;
const fetchMock = vi.fn(async () => new Response(oversizedBody, { status: 502, statusText: "Bad Gateway" }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_capped", legacyAttemptTenantId: "tenant_capped" },
},
{}
)
).rejects.toThrow("Webhook delivery failed with 502.");
expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ signal: expect.any(AbortSignal) }));
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>;
const body = updateCalls[0]?.[1]?.body ?? "";
expect(body).toContain('"bodyBytes":4096');
expect(body).not.toContain("tail-secret");
});
test("updates redacted failed legacy attempt diagnostics when webhook fetch throws", async () => {
const fetchError = new Error("network unavailable token=fetch-secret Authorization: Bearer bearer-secret");
const fetchMock = vi.fn(async () => {
throw fetchError;
});
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: "attempt_3", legacyAttemptTenantId: "tenant_3" },
},
{}
)
).rejects.toThrow(fetchError);
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { message?: string }]>;
expect(updateCalls[0]?.[0]).toEqual({ id: "attempt_3", tenantId: "tenant_3" });
expect(updateCalls[0]?.[1]).toEqual(
expect.objectContaining({
finishedAt: expect.any(Date),
success: false,
message: expect.stringContaining("network unavailable token=[redacted] Authorization: [redacted]"),
})
);
expect(updateCalls[0]?.[1]?.message).not.toContain("fetch-secret");
expect(updateCalls[0]?.[1]?.message).not.toContain("bearer-secret");
});
test("does not update legacy attempts for generic webhook jobs without attempt metadata", async () => {
const fetchMock = vi.fn(async () => new Response(null, { status: 204 }));
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
await expect(
registry[queueNames.email](
{
kind: "webhook.deliver",
url: "https://example.com/webhook",
eventName: "entity.created",
metadata: { legacyAttemptId: 123 },
},
{}
)
).resolves.toMatchObject({
handled: true,
action: "webhook.delivered",
});
expect(updateLegacyEventWebhookAttempt).not.toHaveBeenCalled();
});
test("sends opted-in email notification jobs and marks delivery diagnostics sent", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_notification_1" })),
};
const getUserNotificationPreferences = vi.fn(async () => []);
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_1" }));
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery, markNotificationDeliveryStatus });
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
category: "billing",
email: "user@example.com",
messageKey: "billing.invoice-ready",
subject: "Invoice ready",
text: "Your invoice is ready",
templateData: { secretToken: "not-diagnostic" },
metadata: { requestId: "req_123", source: "billing", apiKey: "secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
handled: true,
action: "notification.email.sent",
details: { provider: "mailjet", messageId: "msg_notification_1" },
});
expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing" });
expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "default" });
expect(createNotificationDelivery).toHaveBeenCalledWith(
expect.objectContaining({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
category: "billing",
messageKey: "billing.invoice-ready",
sourceKey: "request:req_123",
status: "PENDING",
attemptsMade: 0,
payloadMetadata: { requestId: "req_123", source: "billing" },
})
);
expect(mailProvider.send).toHaveBeenCalledWith({
to: ["user@example.com"],
subject: "Invoice ready",
text: "Your invoice is ready",
html: undefined,
templateId: undefined,
idempotencyKey: "notification-delivery:delivery_1",
personalization: [{ email: "user@example.com", data: { secretToken: "not-diagnostic" } }],
});
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(
expect.objectContaining({
id: "delivery_1",
tenantId: "tenant_123",
status: "SENT",
attemptsMade: 1,
providerMessageId: "msg_notification_1",
sentAt: expect.any(Date),
})
);
});
test("uses worker job id as a stable notification delivery source key when payload persistence fails", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_retry_success" })),
};
const deliveries = new Map<string, { id: string }>();
const createNotificationDelivery = vi.fn(async (input: CreateNotificationDeliveryInput) => {
const key = input.sourceKey ?? "none";
const existing = deliveries.get(key);
if (existing) {
return existing;
}
const delivery = { id: `delivery_${deliveries.size + 1}` };
deliveries.set(key, delivery);
return delivery;
});
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences: vi.fn(async () => []),
createNotificationDelivery,
markNotificationDeliveryStatus: vi.fn(async () => undefined),
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
});
await expect(
registry[descriptor.queueName](
descriptor.payload,
{},
{
attemptsMade: 0,
attemptsLimit: 3,
jobId: "job_123",
updatePayload: vi.fn(async () => {
throw new Error("job data write failed");
}),
}
)
).rejects.toThrow("job data write failed");
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3, jobId: "job_123" });
expect(createNotificationDelivery).toHaveBeenCalledTimes(2);
expect(createNotificationDelivery).toHaveBeenCalledWith(expect.objectContaining({ sourceKey: "job:job_123" }));
expect(mailProvider.send).toHaveBeenCalledTimes(1);
expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:delivery_1" }));
});
test("persists delivery id in notification job metadata for retry-safe diagnostics", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_notification_1" })),
};
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_retry" }));
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const updatePayload = vi.fn(async (_payload: unknown) => undefined);
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences: vi.fn(async () => []),
createNotificationDelivery,
markNotificationDeliveryStatus,
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
});
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3, updatePayload });
expect(updatePayload).toHaveBeenCalledWith({
...descriptor.payload,
metadata: { deliveryId: "delivery_retry" },
});
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(expect.objectContaining({ id: "delivery_retry", status: "SENT", attemptsMade: 1 }));
});
test("reuses notification delivery id on email retry without creating duplicate diagnostics", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_retry_success" })),
};
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_new" }));
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences: vi.fn(async () => []),
createNotificationDelivery,
markNotificationDeliveryStatus,
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
metadata: { deliveryId: "delivery_existing" },
});
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 });
expect(createNotificationDelivery).not.toHaveBeenCalled();
expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:delivery_existing" }));
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(
expect.objectContaining({ id: "delivery_existing", tenantId: "tenant_123", status: "SENT", attemptsMade: 2, providerMessageId: "msg_retry_success" })
);
});
test("keeps email delivery pending until the terminal notification retry fails", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => {
throw new Error("provider token=secret failed");
}),
};
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences: vi.fn(async () => []),
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_existing" })),
markNotificationDeliveryStatus,
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
metadata: { deliveryId: "delivery_existing" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).rejects.toThrow("provider");
expect(markNotificationDeliveryStatus).toHaveBeenLastCalledWith({
id: "delivery_existing",
tenantId: "tenant_123",
status: "PENDING",
attemptsMade: 2,
lastError: "provider token=[redacted] failed",
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("provider");
expect(markNotificationDeliveryStatus).toHaveBeenLastCalledWith({
id: "delivery_existing",
tenantId: "tenant_123",
status: "FAILED",
attemptsMade: 3,
lastError: "provider token=[redacted] failed",
});
});
test("does not retry an already-sent email when sent status persistence fails", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_sent_once" })),
};
const markNotificationDeliveryStatus = vi.fn(async () => {
throw new Error("database token=secret unavailable");
});
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences: vi.fn(async () => []),
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_sent_once" })),
markNotificationDeliveryStatus,
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
metadata: { deliveryId: "delivery_sent_once" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({
handled: true,
action: "notification.email.sent.diagnostic_failed",
details: {
provider: "mailjet",
messageId: "msg_sent_once",
userId: "user_123",
messageKey: "billing.invoice-ready",
deliveryId: "delivery_sent_once",
diagnosticError: "database token=[redacted] unavailable",
},
});
expect(mailProvider.send).toHaveBeenCalledTimes(1);
});
test("skips opted-out email notification jobs without calling mail provider", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_skip" }));
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences: vi.fn(async () => [{ enabled: false }]),
createNotificationDelivery,
markNotificationDeliveryStatus,
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "notification.email.skipped",
details: {
userId: "user_123",
channel: "email",
messageKey: "billing.invoice-ready",
deliveryId: "delivery_skip",
reason: "preference_disabled",
},
});
expect(mailProvider.send).not.toHaveBeenCalled();
expect(createNotificationDelivery).toHaveBeenCalledWith(
expect.objectContaining({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
messageKey: "billing.invoice-ready",
status: "SKIPPED",
lastError: "preference_disabled",
payloadMetadata: { reason: "preference_disabled" },
})
);
expect(markNotificationDeliveryStatus).not.toHaveBeenCalled();
});
test("delivers a critical-category email even when a preference row disables it", async () => {
// AK-P21: GDPR export/deletion notices ride the locked "critical" category.
// foundation's isNotificationEnabled() never suppresses it, and the queue
// path must not either — otherwise a stale row silences a legal notice.
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const getUserNotificationPreferences = vi.fn(async () => [{ enabled: false }]);
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences,
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_critical" })),
markNotificationDeliveryStatus: vi.fn(async () => undefined),
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
category: "critical",
email: "user@example.com",
messageKey: "settings.privacy.notify.exportReady",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
handled: true,
action: "notification.email.sent",
});
expect(mailProvider.send).toHaveBeenCalledTimes(1);
// The lookup is skipped outright, so no stale row can reach the gate.
expect(getUserNotificationPreferences).not.toHaveBeenCalled();
});
test("lets explicit category notification preferences override disabled defaults", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_category_enabled" })),
};
const getUserNotificationPreferences = vi.fn().mockResolvedValueOnce([{ enabled: true, category: "billing" }]);
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_category_enabled" }));
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery, markNotificationDeliveryStatus });
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
category: "billing",
email: "user@example.com",
messageKey: "billing.invoice-ready",
subject: "Invoice ready",
text: "Your invoice is ready",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
handled: true,
action: "notification.email.sent",
details: { provider: "mailjet", messageId: "msg_category_enabled" },
});
expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing" });
expect(getUserNotificationPreferences).toHaveBeenCalledTimes(1);
expect(mailProvider.send).toHaveBeenCalledTimes(1);
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(
expect.objectContaining({ id: "delivery_category_enabled", tenantId: "tenant_123", status: "SENT" })
);
});
test("records skipped diagnostics for opted-out in-app notification jobs", async () => {
const getUserNotificationPreferences = vi.fn(async () => [{ enabled: false }]);
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_in_app_skip" }));
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery });
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "in-app",
category: "billing",
messageKey: "billing.invoice-ready",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "notification.in-app.skipped",
details: {
userId: "user_123",
channel: "in-app",
messageKey: "billing.invoice-ready",
deliveryId: "delivery_in_app_skip",
reason: "preference_disabled",
},
});
expect(getUserNotificationPreferences).toHaveBeenCalledWith({
tenantId: "tenant_123",
userId: "user_123",
channel: "in-app",
category: "billing",
});
expect(getUserNotificationPreferences).toHaveBeenCalledTimes(1);
expect(createNotificationDelivery).toHaveBeenCalledWith(
expect.objectContaining({
tenantId: "tenant_123",
userId: "user_123",
channel: "in-app",
category: "billing",
messageKey: "billing.invoice-ready",
status: "SKIPPED",
lastError: "preference_disabled",
payloadMetadata: { reason: "preference_disabled" },
})
);
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("marks email notification diagnostics failed when provider rejects", async () => {
const providerError = new Error("SMTP token=secret-token rejected for user@example.com with Authorization: Bearer abc123 and payload-secret");
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => {
throw providerError;
}),
};
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
mailProvider,
getUserNotificationPreferences: vi.fn(async () => [{ enabled: true }]),
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_failed" })),
markNotificationDeliveryStatus,
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
templateData: { apiKey: "payload-secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toBe(providerError);
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith({
id: "delivery_failed",
tenantId: "tenant_123",
status: "FAILED",
attemptsMade: 1,
lastError: "SMTP token=[redacted] rejected for [redacted-email] with Authorization: Bearer [redacted] and payload-[redacted]",
});
});
test("marks email notification diagnostics failed when provider setup rejects", async () => {
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
env: {},
getUserNotificationPreferences: vi.fn(async () => [{ enabled: true }]),
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_setup_failed" })),
markNotificationDeliveryStatus,
});
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "billing.invoice-ready",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("MAILJET_API_KEY is required");
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith({
id: "delivery_setup_failed",
tenantId: "tenant_123",
status: "FAILED",
attemptsMade: 1,
lastError: "MAILJET_API_KEY is required for Mailjet email delivery.",
});
});
test("rejects email notification jobs missing tenant or email before diagnostics", async () => {
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_missing" }));
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const registry = buildHandlerRegistry({ mailProvider, createNotificationDelivery });
await expect(
registry[queueNames.notifications](
{
kind: "notification.send",
userId: "user_123",
channel: "email",
email: "user@example.com",
messageKey: "account.updated",
} as Parameters<(typeof registry)[typeof queueNames.notifications]>[0],
{}
)
).rejects.toThrow("Notification jobs require tenantId.");
await expect(
registry[queueNames.notifications](
buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", messageKey: "account.updated" }).payload,
{}
)
).rejects.toThrow("Email notifications require email recipient.");
expect(createNotificationDelivery).not.toHaveBeenCalled();
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("rejects unsupported notification channels before diagnostics", async () => {
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_sms" }));
const registry = buildHandlerRegistry({ createNotificationDelivery });
await expect(
registry[queueNames.notifications](
{
kind: "notification.send",
tenantId: "tenant_123",
userId: "user_123",
channel: "sms",
messageKey: "account.updated",
} as unknown as Parameters<(typeof registry)[typeof queueNames.notifications]>[0],
{}
)
).rejects.toThrow("Unsupported notification channel: sms");
expect(createNotificationDelivery).not.toHaveBeenCalled();
});
test("accepts descriptor-derived non-email notification jobs through registry", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_in_app" }));
const registry = buildHandlerRegistry({ mailProvider, createNotificationDelivery, getUserNotificationPreferences: vi.fn(async () => []) });
const descriptor = buildNotificationJob({
tenantId: "tenant_123",
userId: "user_123",
channel: "in-app",
messageKey: "account.updated",
templateData: { ignoredBody: "not surfaced" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "notification.accepted",
details: {
userId: "user_123",
channel: "in-app",
messageKey: "account.updated",
deliveryId: "delivery_in_app",
},
});
expect(createNotificationDelivery).toHaveBeenCalledWith(
expect.objectContaining({
userId: "user_123",
channel: "in-app",
messageKey: "account.updated",
status: "SENT",
attemptsMade: 1,
sentAt: expect.any(Date),
})
);
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("processes tenantless platform in-app delivery through the notification worker", async () => {
const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "PENDING" }));
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ eligible: true, email: null }));
const getUserNotificationPreferences = vi.fn(async () => []);
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const registry = buildHandlerRegistry({
createPlatformNotificationDelivery,
markPlatformNotificationDeliveryStatus,
getUserNotificationPreferences,
mailProvider,
resolvePlatformLeadNotificationRecipient,
});
const descriptor = buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
category: "leads",
metadata: { leadId: "lead_1", sourceId: "hivegrid-website", deeplink: "/leads/lead_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({
handled: true,
action: "notification.platform.in-app.sent",
details: {
deliveryId: "platform_delivery_1",
userId: "user_haris",
channel: "in-app",
messageKey: "platform.leads.captured",
},
});
expect(createPlatformNotificationDelivery).toHaveBeenCalledWith({
userId: "user_haris",
channel: "IN_APP",
category: "leads",
messageKey: "platform.leads.captured",
sourceKey: "platform-lead:lead_1:in-app",
payloadMetadata: { leadId: "lead_1", sourceId: "hivegrid-website", deeplink: "/leads/lead_1" },
});
expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({
id: "platform_delivery_1",
userId: "user_haris",
status: "SENT",
attemptsMade: 1,
sentAt: expect.any(Date),
});
expect(getUserNotificationPreferences).not.toHaveBeenCalled();
expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledWith({ userId: "user_haris", leadId: "lead_1" });
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("delivers task reminders without coupling them to the lead recipient capability", async () => {
const createAdminPlatformNotificationDelivery = vi.fn(async () => ({
id: "platform_task_delivery_1",
status: "PENDING",
}));
const markAdminPlatformNotificationDeliveryStatus = vi.fn(async () => ({
count: 1,
}));
const resolvePlatformLeadNotificationRecipient = vi.fn();
const registry = buildHandlerRegistry({
createAdminPlatformNotificationDelivery,
markAdminPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient,
});
const descriptor = buildPlatformNotificationJob({
adminUserId: "admin_1",
sourceKey: "admin-sales:task-reminder:task_1:2026-07-25",
channel: "in-app",
messageKey: "admin.sales.notifications.taskreminder",
category: "sales",
metadata: {
taskId: "task_1",
deeplink: "/sales/tasks/today",
},
});
await expect(
registry[descriptor.queueName](descriptor.payload, {}, {
attemptsMade: 0,
attemptsLimit: 3,
})
).resolves.toMatchObject({
action: "notification.platform.in-app.sent",
details: {
deliveryId: "platform_task_delivery_1",
userId: "admin_1",
},
});
expect(resolvePlatformLeadNotificationRecipient).not.toHaveBeenCalled();
expect(createAdminPlatformNotificationDelivery).toHaveBeenCalledWith(
expect.objectContaining({
adminUserId: "admin_1",
messageKey: "admin.sales.notifications.taskreminder",
})
);
expect(markAdminPlatformNotificationDeliveryStatus).toHaveBeenCalledWith(
expect.objectContaining({
id: "platform_task_delivery_1",
adminUserId: "admin_1",
status: "SENT",
})
);
});
test("skips platform in-app delivery when current lead access was revoked", async () => {
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
const registry = buildHandlerRegistry({
createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_revoked", status: "PENDING" })),
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient: vi.fn(async () => ({ eligible: false, email: null })),
});
const descriptor = buildPlatformNotificationJob({
userId: "user_former_operator",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({
handled: true,
action: "notification.platform.in-app.skipped",
details: {
deliveryId: "platform_delivery_revoked",
userId: "user_former_operator",
channel: "in-app",
messageKey: "platform.leads.captured",
reason: "recipient_ineligible",
},
});
expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({
id: "platform_delivery_revoked",
userId: "user_former_operator",
status: "SKIPPED",
attemptsMade: 1,
lastError: "Platform lead notification recipient is no longer eligible.",
});
});
test("sends platform email with provider idempotency and PII-minimized diagnostics", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_platform_1" })),
};
const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "PENDING" }));
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ email: "current-haris@example.test" }));
const registry = buildHandlerRegistry({
mailProvider,
createPlatformNotificationDelivery,
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient,
});
const descriptor = buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
email: "haris@example.test",
subject: "Neuer Lead in HiveGrid",
text: "Ein neuer Lead ist eingegangen.",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1", sourceId: "hivegrid-website" },
});
const result = await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 });
expect(result).toEqual({
handled: true,
action: "notification.platform.email.sent",
details: {
provider: "mailjet",
messageId: "msg_platform_1",
userId: "user_haris",
messageKey: "platform.leads.captured",
deliveryId: "platform_delivery_1",
},
});
expect(mailProvider.send).toHaveBeenCalledWith(
expect.objectContaining({
to: ["current-haris@example.test"],
idempotencyKey: "notification-delivery:platform_delivery_1",
})
);
expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledWith({ userId: "user_haris", leadId: "lead_1" });
expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({
id: "platform_delivery_1",
userId: "user_haris",
status: "SENT",
attemptsMade: 1,
providerMessageId: "msg_platform_1",
sentAt: expect.any(Date),
});
expect(JSON.stringify(result)).not.toContain("haris@example.test");
});
test("retries a platform email when SENT persistence fails after provider acceptance", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_platform_retry" })),
};
const markPlatformNotificationDeliveryStatus = vi.fn(async () => {
throw new Error("database token=secret unavailable");
});
const registry = buildHandlerRegistry({
mailProvider,
createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_retry", status: "PENDING" })),
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient: vi.fn(async () => ({ email: "haris@example.test" })),
});
const descriptor = buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "platform-lead:lead_retry:email",
channel: "email",
email: "haris@example.test",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_retry" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow(
"Platform notification delivery status persistence failed: database token=[redacted] unavailable"
);
expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:platform_delivery_retry" }));
});
test("keeps platform email pending until terminal retry and redacts the failure", async () => {
const providerError = new Error("SMTP token=secret-token rejected for haris@example.test");
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => {
throw providerError;
}),
};
const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "PENDING" }));
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ email: "haris@example.test" }));
const registry = buildHandlerRegistry({
mailProvider,
createPlatformNotificationDelivery,
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient,
});
const descriptor = buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
email: "haris@example.test",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow(
"SMTP token=[redacted] rejected for [redacted-email]"
);
expect(markPlatformNotificationDeliveryStatus).toHaveBeenLastCalledWith({
id: "platform_delivery_1",
userId: "user_haris",
status: "PENDING",
attemptsMade: 1,
lastError: "SMTP token=[redacted] rejected for [redacted-email]",
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow(
"SMTP token=[redacted] rejected for [redacted-email]"
);
expect(markPlatformNotificationDeliveryStatus).toHaveBeenLastCalledWith({
id: "platform_delivery_1",
userId: "user_haris",
status: "FAILED",
attemptsMade: 3,
lastError: "SMTP token=[redacted] rejected for [redacted-email]",
});
expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledTimes(2);
});
test("skips platform lead email when the operator was demoted, deactivated, or lost record access", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ email: null }));
const registry = buildHandlerRegistry({
mailProvider,
createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_ineligible", status: "PENDING" })),
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient,
});
const descriptor = buildPlatformNotificationJob({
userId: "user_former_operator",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
email: "stale-address@example.test",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).resolves.toEqual({
handled: true,
action: "notification.platform.email.skipped",
details: {
deliveryId: "platform_delivery_ineligible",
userId: "user_former_operator",
channel: "email",
messageKey: "platform.leads.captured",
reason: "recipient_ineligible",
},
});
expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledWith({ userId: "user_former_operator", leadId: "lead_1" });
expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith({
id: "platform_delivery_ineligible",
userId: "user_former_operator",
status: "SKIPPED",
attemptsMade: 2,
lastError: "Platform lead notification recipient is no longer eligible.",
});
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("retries without sending when the recipient revalidation capability is unavailable", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
const registry = buildHandlerRegistry({
mailProvider,
createPlatformNotificationDelivery: vi.fn(async () => ({ id: "platform_delivery_retry_resolution", status: "PENDING" })),
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient: vi.fn(async () => {
throw new Error("recipient capability unavailable");
}),
});
const descriptor = buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
email: "stale-address@example.test",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow(
"recipient capability unavailable"
);
expect(mailProvider.send).not.toHaveBeenCalled();
expect(markPlatformNotificationDeliveryStatus).not.toHaveBeenCalled();
});
test("deduplicates an already-sent platform delivery without calling provider", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1", status: "SENT" }));
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery, markPlatformNotificationDeliveryStatus });
const descriptor = buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
email: "haris@example.test",
messageKey: "platform.leads.captured",
metadata: { deliveryId: "platform_delivery_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "notification.platform.deduplicated",
details: { deliveryId: "platform_delivery_1" },
});
expect(mailProvider.send).not.toHaveBeenCalled();
expect(markPlatformNotificationDeliveryStatus).not.toHaveBeenCalled();
});
test("rejects a queued delivery id that does not match the durable logical intent", async () => {
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
};
const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_actual", status: "PENDING" }));
const registry = buildHandlerRegistry({ mailProvider, createPlatformNotificationDelivery });
const descriptor = buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
email: "haris@example.test",
messageKey: "platform.leads.captured",
metadata: { deliveryId: "platform_delivery_queued" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("does not match the queued delivery id");
expect(mailProvider.send).not.toHaveBeenCalled();
});
test("rejects malformed platform notification scope before delivery", async () => {
const createPlatformNotificationDelivery = vi.fn(async () => ({ id: "platform_delivery_1" }));
const registry = buildHandlerRegistry({ createPlatformNotificationDelivery });
await expect(
registry[queueNames.notifications](
{
kind: "notification.platform.send",
tenantId: "tenant_escape",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
} as never,
{}
)
).rejects.toThrow("Platform notification jobs require tenantId=null");
expect(createPlatformNotificationDelivery).not.toHaveBeenCalled();
});
test("rejects unsupported cleanup jobs through registry", async () => {
const registry = buildHandlerRegistry();
const descriptor = buildCleanupJob({
task: "expired-sessions",
olderThanHours: 24,
requestedBy: "system-fixture",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "worker.unsupported",
message: "Unsupported maintenance task.",
details: {
queueName: queueNames.maintenance,
task: "expired-sessions",
olderThanHours: 24,
},
});
});
test("runs outbox DONE retention cleanup through maintenance handler", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-05-28T08:00:00.000Z"));
const deleteDoneOutboxEvents = vi.fn(async (input: { cutoff: Date; take?: number }) => ({
deleted: 17,
cutoff: input.cutoff,
take: input.take ?? 1000,
}));
const registry = buildHandlerRegistry({ deleteDoneOutboxEvents });
const descriptor = buildCleanupJob({
task: "outbox-done-retention",
olderThanHours: 48,
take: 500,
metadata: { requestId: "req_outbox_retention_1" },
});
try {
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "outbox.done-retention.completed",
details: {
task: "outbox-done-retention",
requestId: "req_outbox_retention_1",
olderThanHours: 48,
cutoff: "2026-05-26T08:00:00.000Z",
take: 500,
deleted: 17,
},
});
} finally {
vi.useRealTimers();
}
expect(deleteDoneOutboxEvents).toHaveBeenCalledWith({
cutoff: new Date("2026-05-26T08:00:00.000Z"),
take: 500,
});
});
test("runs workflow stalled execution reconciliation through maintenance handler", async () => {
const observedScopes: Array<string | null> = [];
const reconcileWorkflowExecutionStalls = vi.fn(async () => {
const tenantId = getRlsExecutionContext()?.tenantId ?? null;
observedScopes.push(tenantId);
return {
cutoff: new Date("2026-05-29T02:30:00.000Z"),
take: 100,
inspected: 1,
failedOutboxEvents: tenantId === "tenant_b" ? 0 : 1,
repaired: tenantId === null ? 0 : 1,
skippedWithoutFailedOutbox: tenantId === null ? 1 : 0,
updateMisses: 0,
};
});
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(["tenant_a", "tenant_b"]),
reconcileWorkflowExecutionStalls,
});
const descriptor = buildCleanupJob({
task: "workflow-execution-stalled-reconcile",
olderThanMinutes: 30,
take: 100,
metadata: { requestId: "req_workflow_stalled_reconcile_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "workflow.execution-stalled-reconcile.completed",
details: {
task: "workflow-execution-stalled-reconcile",
requestId: "req_workflow_stalled_reconcile_1",
olderThanMinutes: 30,
cutoff: "2026-05-29T02:30:00.000Z",
take: 100,
inspected: 3,
failedOutboxEvents: 2,
repaired: 2,
skippedWithoutFailedOutbox: 1,
updateMisses: 0,
},
});
expect(reconcileWorkflowExecutionStalls).toHaveBeenCalledTimes(3);
expect(reconcileWorkflowExecutionStalls).toHaveBeenNthCalledWith(1, {
olderThanMinutes: 30,
take: 100,
});
expect(reconcileWorkflowExecutionStalls).toHaveBeenNthCalledWith(2, {
olderThanMinutes: 30,
take: 100,
});
expect(reconcileWorkflowExecutionStalls).toHaveBeenNthCalledWith(3, {
olderThanMinutes: 30,
take: 100,
});
expect(observedScopes).toEqual(["tenant_a", "tenant_b", null]);
});
test("acknowledges an overlapping tenant maintenance sweep without touching the database", async () => {
const store = createSharedStore({ namespace: "worker-handler-lease-test", adapter: new MemoryAdapter() });
await store.set("job-lease:task-archive-retention", "other-replica", { ttlSeconds: 120 });
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, logger });
const descriptor = buildCleanupJob({ task: "task-archive-retention" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "maintenance.tenant-sweep.overlap-skipped",
details: { task: "task-archive-retention", reason: "active-lease" },
});
expect(logger.info).toHaveBeenCalledWith({ task: "task-archive-retention", requestId: null }, "maintenance.tenant-sweep.overlap-skipped");
});
test("dispatches billing-dunning-sweep as a leased tenant maintenance sweep", async () => {
// AK-P20 DoD 2 — proves the maintenance registry routes the dunning sweep
// into the leased sweep path (an unknown task would reject with worker.unsupported).
const store = createSharedStore({ namespace: "worker-handler-lease-test", adapter: new MemoryAdapter() });
await store.set("job-lease:billing-dunning-sweep", "other-replica", { ttlSeconds: 120 });
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, logger });
const descriptor = buildCleanupJob({ task: "billing-dunning-sweep" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "maintenance.tenant-sweep.overlap-skipped",
details: { task: "billing-dunning-sweep", reason: "active-lease" },
});
expect(logger.info).toHaveBeenCalledWith({ task: "billing-dunning-sweep", requestId: null }, "maintenance.tenant-sweep.overlap-skipped");
});
test("dispatches project-auto-archive as a leased tenant maintenance sweep", async () => {
// R-P9a DoD 4 — proves the maintenance registry routes the new task into the
// leased sweep path (an unknown task would reject with worker.unsupported).
const store = createSharedStore({ namespace: "worker-handler-lease-test", adapter: new MemoryAdapter() });
await store.set("job-lease:project-auto-archive", "other-replica", { ttlSeconds: 120 });
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, logger });
const descriptor = buildCleanupJob({ task: "project-auto-archive" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "maintenance.tenant-sweep.overlap-skipped",
details: { task: "project-auto-archive", reason: "active-lease" },
});
expect(logger.info).toHaveBeenCalledWith({ task: "project-auto-archive", requestId: null }, "maintenance.tenant-sweep.overlap-skipped");
});
test("runs the admin sales task reminder sweep through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-sales-task-reminder-handler-test",
adapter: new MemoryAdapter(),
});
const runAdminSalesTaskReminderSweep = vi.fn(async () => ({
scanned: 3,
dispatched: 2,
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
runAdminSalesTaskReminderSweep,
logger,
});
const descriptor = buildCleanupJob({
task: "admin-sales-task-reminder-sweep",
take: 75,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.task-reminder-sweep.completed",
details: { scanned: 3, dispatched: 2 },
});
expect(runAdminSalesTaskReminderSweep).toHaveBeenCalledWith({
now: expect.any(Date),
take: 75,
});
expect(logger.info).toHaveBeenCalledWith(
{
task: "admin-sales-task-reminder-sweep",
scanned: 3,
dispatched: 2,
},
"admin-sales.task-reminder-sweep.completed"
);
});
test("queues due admin booking reminders through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-booking-reminder-handler-test",
adapter: new MemoryAdapter(),
});
const runAdminBookingReminderSweep = vi.fn(async () => ({
scanned: 4,
dispatched: 3,
failed: 1,
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
runAdminBookingReminderSweep,
logger,
});
const descriptor = buildCleanupJob({
task: "admin-booking-reminder-sweep",
take: 80,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-booking.reminder-sweep.completed",
details: { scanned: 4, dispatched: 3, failed: 1 },
});
expect(runAdminBookingReminderSweep).toHaveBeenCalledWith({
now: expect.any(Date),
take: 80,
});
});
test("runs admin email delivery through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-email-delivery-handler-test",
adapter: new MemoryAdapter(),
});
const runAdminEmailDelivery = vi.fn(async () => ({
inspected: 4,
claimed: 2,
sent: 1,
failed: ["scheduled-2"],
blocked: 1,
deferred: 1,
recoveredUnknown: 0,
noReplyReminders: { scanned: 3, created: 2 },
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
runAdminEmailDelivery,
logger,
});
const descriptor = buildCleanupJob({
task: "admin-email-delivery",
take: 25,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.email-delivery.completed",
details: {
inspected: 4,
claimed: 2,
sent: 1,
failed: 1,
blocked: 1,
deferred: 1,
recoveredUnknown: 0,
noReplyReminderScanned: 3,
noReplyReminderCreated: 2,
},
});
expect(runAdminEmailDelivery).toHaveBeenCalledWith({
now: expect.any(Date),
take: 25,
});
});
test("runs admin mailbox sync through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-email-sync-handler-test",
adapter: new MemoryAdapter(),
});
const runAdminEmailSync = vi.fn(async () => ({
inspected: 3,
synced: 2,
upserted: 4,
replayed: 1,
failed: ["cursor-3"],
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
runAdminEmailSync,
logger,
});
const descriptor = buildCleanupJob({
task: "admin-email-sync",
take: 20,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.email-sync.completed",
details: {
inspected: 3,
synced: 2,
upserted: 4,
replayed: 1,
failed: 1,
},
});
expect(runAdminEmailSync).toHaveBeenCalledWith({
now: expect.any(Date),
take: 20,
});
});
test("checks admin sending domains through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-email-domain-handler-test",
adapter: new MemoryAdapter(),
});
const runAdminEmailDomainCheck = vi.fn(async () => ({
inspected: 3,
verified: 2,
degraded: 1,
failed: [],
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
runAdminEmailDomainCheck,
logger,
});
const descriptor = buildCleanupJob({
task: "admin-email-domain-check",
take: 20,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.email-domain-check.completed",
details: {
inspected: 3,
verified: 2,
degraded: 1,
failed: 0,
},
});
expect(runAdminEmailDomainCheck).toHaveBeenCalledWith({
now: expect.any(Date),
take: 20,
});
});
test("processes encrypted admin call recordings through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-call-recording-process-handler-test",
adapter: new MemoryAdapter(),
});
const runAdminCallRecordingProcessing = vi.fn(async () => ({
claimed: 2,
processed: 2,
}));
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
runAdminCallRecordingProcessing,
logger: { info: vi.fn(), error: vi.fn() },
});
const descriptor = buildCleanupJob({
task: "admin-call-recording-process",
take: 10,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.call-recording-process.completed",
details: { claimed: 2, processed: 2 },
});
expect(runAdminCallRecordingProcessing).toHaveBeenCalledWith({
take: 10,
});
});
test("expires admin call recordings through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-call-recording-retention-handler-test",
adapter: new MemoryAdapter(),
});
const runAdminCallRecordingRetention = vi.fn(async () => ({
claimed: 3,
deleted: 2,
failed: 1,
}));
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
runAdminCallRecordingRetention,
logger: { info: vi.fn(), error: vi.fn() },
});
const descriptor = buildCleanupJob({
task: "admin-call-recording-retention",
take: 50,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.call-recording-retention.completed",
details: { claimed: 3, deleted: 2, failed: 1 },
});
expect(runAdminCallRecordingRetention).toHaveBeenCalledWith({
now: expect.any(Date),
take: 50,
});
});
test("redacts expired admin email event raw data through the leased worker path", async () => {
const store = createSharedStore({
namespace: "admin-email-event-retention-handler-test",
adapter: new MemoryAdapter(),
});
const redactExpiredAdminEmailEventRawData = vi.fn(async () => ({
inspected: 8,
redacted: 7,
deletedLinks: 3,
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
redactExpiredAdminEmailEventRawData,
logger,
});
const descriptor = buildCleanupJob({
task: "admin-email-event-retention",
take: 50,
});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.email-event-retention.completed",
details: { inspected: 8, redacted: 7, deletedLinks: 3 },
});
expect(redactExpiredAdminEmailEventRawData).toHaveBeenCalledWith({
now: expect.any(Date),
take: 50,
});
});
test("runs telephony webhook maintenance through the injected general-group service", async () => {
const runTelephonyWebhookMaintenance = vi.fn(async () => ({
deletedCompleted: 3,
deletedFailed: 1,
recoveredPending: 2,
recoveredFailed: 1,
inspectedTenants: 4,
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({ runTelephonyWebhookMaintenance, logger });
const descriptor = buildCleanupJob({ task: "telephony-webhook-maintenance" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "telephony.webhook-maintenance.completed",
details: {
task: "telephony-webhook-maintenance",
deletedCompleted: 3,
deletedFailed: 1,
recoveredPending: 2,
recoveredFailed: 1,
inspectedTenants: 4,
},
});
expect(runTelephonyWebhookMaintenance).toHaveBeenCalledTimes(1);
expect(logger.info).toHaveBeenCalledWith(
{
task: "telephony-webhook-maintenance",
deletedCompleted: 3,
deletedFailed: 1,
recoveredPending: 2,
recoveredFailed: 1,
inspectedTenants: 4,
},
"telephony.webhook-maintenance.completed"
);
});
test("skips telephony webhook maintenance while another replica holds its lease", async () => {
const store = createSharedStore({ namespace: "telephony-maintenance-handler-lease-test", adapter: new MemoryAdapter() });
await store.set("job-lease:telephony-webhook-maintenance", "other-replica", { ttlSeconds: 120 });
const runTelephonyWebhookMaintenance = vi.fn(async () => ({
deletedCompleted: 0,
deletedFailed: 0,
recoveredPending: 0,
recoveredFailed: 0,
inspectedTenants: 0,
}));
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runTelephonyWebhookMaintenance });
const descriptor = buildCleanupJob({ task: "telephony-webhook-maintenance" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "maintenance.tenant-sweep.overlap-skipped",
details: { task: "telephony-webhook-maintenance", reason: "active-lease" },
});
expect(runTelephonyWebhookMaintenance).not.toHaveBeenCalled();
});
test("runs product acquisition reconciliation through the leased general-group service", async () => {
const runProductAcquisitionReconcile = vi.fn(async () => ({
inspected: 5,
expired: 1,
recoveredProviderPayments: 1,
refreshed: 1,
appliedTerminal: 2,
quarantined: 0,
raced: 0,
inspectedTenants: 3,
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({ runProductAcquisitionReconcile, logger });
const descriptor = buildCleanupJob({ task: "product-acquisition-reconcile", take: 25 });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "billing.product-acquisition-reconcile.completed",
details: {
task: "product-acquisition-reconcile",
inspected: 5,
expired: 1,
recoveredProviderPayments: 1,
refreshed: 1,
appliedTerminal: 2,
quarantined: 0,
raced: 0,
inspectedTenants: 3,
},
});
expect(runProductAcquisitionReconcile).toHaveBeenCalledWith({ take: 25 });
expect(logger.info).toHaveBeenCalledWith(
expect.objectContaining({ task: "product-acquisition-reconcile", inspectedTenants: 3, appliedTerminal: 2 }),
"billing.product-acquisition-reconcile.completed"
);
});
test("skips product acquisition reconciliation while another replica holds its lease", async () => {
const store = createSharedStore({ namespace: "product-acquisition-reconcile-handler-lease-test", adapter: new MemoryAdapter() });
await store.set("job-lease:product-acquisition-reconcile", "other-replica", { ttlSeconds: 120 });
const runProductAcquisitionReconcile = vi.fn(async () => ({
inspected: 0,
expired: 0,
recoveredProviderPayments: 0,
refreshed: 0,
appliedTerminal: 0,
quarantined: 0,
raced: 0,
inspectedTenants: 0,
}));
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runProductAcquisitionReconcile });
const descriptor = buildCleanupJob({ task: "product-acquisition-reconcile" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "maintenance.tenant-sweep.overlap-skipped",
details: { task: "product-acquisition-reconcile", reason: "active-lease" },
});
expect(runProductAcquisitionReconcile).not.toHaveBeenCalled();
});
test("runs the portal lifecycle scan through the leased general-group service", async () => {
const runScan = vi.fn(async () => ({
inspectedTenants: 2,
deletionScanned: 1,
deletionCompleted: 1,
exportArtifactsExpired: 2,
inactivityWarnings: 3,
inactivityDisabled: 1,
projectCompletionMails: 4,
projectAccessesClosed: 2,
failed: 0,
}));
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({ portalLifecycleService: { runScan }, logger });
const descriptor = buildCleanupJob({ task: "portal-lifecycle-scan" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "portal.lifecycle.scanned",
details: {
task: "portal-lifecycle-scan",
inspectedTenants: 2,
deletionScanned: 1,
deletionCompleted: 1,
exportArtifactsExpired: 2,
inactivityWarnings: 3,
inactivityDisabled: 1,
projectCompletionMails: 4,
projectAccessesClosed: 2,
failed: 0,
},
});
expect(runScan).toHaveBeenCalledTimes(1);
expect(logger.info).toHaveBeenCalledWith(
expect.objectContaining({ task: "portal-lifecycle-scan", inspectedTenants: 2 }),
"portal.lifecycle.scan-completed"
);
});
test("dispatches dsgvo-grace-period-scan through runDsgvoGraceCron with the configured service", async () => {
const listOverdueUsers = vi
.fn()
.mockResolvedValue([{ userId: "u-overdue-1", tenantId: "tenant_a", scheduledDeletionAt: new Date("2026-05-01T00:00:00Z"), deletionRequestId: "req-1" }]);
const enqueueDeletion = vi.fn().mockResolvedValue({ jobId: "dsgvo-cascade-1" });
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
dsgvoGraceCronService: { listOverdueUsers, enqueueDeletion },
logger,
});
const descriptor = buildCleanupJob({ task: "dsgvo-grace-period-scan" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "dsgvo.grace-period.scanned",
details: {
task: "dsgvo-grace-period-scan",
scanned: 1,
dispatched: 1,
failed: 0,
failures: [],
},
});
expect(listOverdueUsers).toHaveBeenCalledTimes(1);
expect(enqueueDeletion).toHaveBeenCalledTimes(1);
expect(enqueueDeletion).toHaveBeenCalledWith(
expect.objectContaining({
userId: "u-overdue-1",
tenantId: "tenant_a",
requestId: "req-1",
actorType: "system",
mode: "pseudonymize",
reason: "dsgvo-grace-period-elapsed",
})
);
});
test("acks dsgvo-grace-period-scan with skipped action when service dep is missing (no UnsupportedWorkerJobError loop)", async () => {
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({ logger });
const descriptor = buildCleanupJob({ task: "dsgvo-grace-period-scan", metadata: { requestId: "req_grace_1" } });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "dsgvo.grace-period.skipped",
details: {
task: "dsgvo-grace-period-scan",
reason: "dsgvoGraceCronService not configured",
scanned: 0,
dispatched: 0,
failed: 0,
},
});
expect(logger.info).toHaveBeenCalledWith(
expect.objectContaining({ task: "dsgvo-grace-period-scan", requestId: "req_grace_1" }),
expect.stringContaining("dsgvo-grace-cron.skipped")
);
});
test("hard-deletes expired export artifacts and reports scalar counts", async () => {
const jobs = [
retentionJob({ id: "export_job_1", artifactBucket: "exports", artifactKey: "tenant_1/export-1.csv" }),
retentionJob({ id: "export_job_2", artifactBucket: "exports", artifactKey: "tenant_1/export-2.csv" }),
];
const observedScopes: Array<string | undefined> = [];
const listExpiredCompletedExportJobs = vi.fn(async () => {
observedScopes.push(getRlsExecutionContext()?.tenantId);
return [];
});
const markExportJobExpired = vi.fn(async () => undefined);
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => {
observedScopes.push(getRlsExecutionContext()?.tenantId);
return jobs;
});
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async (input) => {
return { bucket: input.bucket ?? "exports", key: input.key };
}),
};
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
const markExportJobHardDeleteFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
listExpiredCompletedExportJobs,
markExportJobExpired,
listSoftDeletedExportJobsForHardDelete,
storageDeleteService,
markExportJobHardDeleteSucceeded,
markExportJobHardDeleteFailed,
});
const descriptor = buildCleanupJob({
task: "expired-export-artifacts",
olderThanHours: 2,
metadata: { requestId: "req_retention_1", token: "payload-secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "export-artifact-retention.completed",
details: {
task: "expired-export-artifacts",
requestId: "req_retention_1",
expired: 0,
processed: 2,
deleted: 2,
failed: 0,
jobs: [
{ exportJobId: "export_job_1", status: "deleted" },
{ exportJobId: "export_job_2", status: "deleted" },
],
},
});
expect(listExpiredCompletedExportJobs).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined });
expect(markExportJobExpired).not.toHaveBeenCalled();
expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined });
const hardDeleteListInput = (listSoftDeletedExportJobsForHardDelete.mock.calls as unknown as [TestRetentionListInput][])[0]?.[0];
expect(hardDeleteListInput?.now.getTime()).toBeLessThan(Date.now());
expect(storageDeleteService.deleteObject).toHaveBeenNthCalledWith(1, {
bucket: "exports",
key: "tenant_1/export-1.csv",
signal: expect.any(AbortSignal),
});
expect(storageDeleteService.deleteObject).toHaveBeenNthCalledWith(2, {
bucket: "exports",
key: "tenant_1/export-2.csv",
signal: expect.any(AbortSignal),
});
expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1" });
expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_2", tenantId: "tenant_1" });
expect(markExportJobHardDeleteFailed).not.toHaveBeenCalled();
expect(observedScopes).toEqual(["tenant_1", "tenant_1"]);
});
test("routes the dedicated portal GDPR retention task with its bounded take and request id", async () => {
const runSweep: TestPortalGdprExportRetentionService["runSweep"] = vi.fn(async () => ({
inspectedTenants: 2,
processed: 4,
claimed: 3,
deleted: 2,
raced: 1,
}));
const registry = buildHandlerRegistry({
portalGdprExportRetentionService: { runSweep },
});
const descriptor = buildCleanupJob({
task: "portal-gdpr-export-retention",
take: 25,
metadata: { requestId: "req_portal_gdpr_retention_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "portal.gdpr-export-retention.completed",
details: {
task: "portal-gdpr-export-retention",
requestId: "req_portal_gdpr_retention_1",
inspectedTenants: 2,
processed: 4,
claimed: 3,
deleted: 2,
raced: 1,
},
});
expect(runSweep).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 });
});
test("rejects the maintenance job when portal GDPR artifact deletion fails so BullMQ can retry", async () => {
const deleteFailure = new Error("portal GDPR retention deletion failed");
const runSweep: TestPortalGdprExportRetentionService["runSweep"] = vi.fn(async () => {
throw deleteFailure;
});
const registry = buildHandlerRegistry({
portalGdprExportRetentionService: { runSweep },
});
const descriptor = buildCleanupJob({ task: "portal-gdpr-export-retention" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toBe(deleteFailure);
});
test("owner-claims and hard-deletes persisted privacy GDPR attempts for active and inactive tenants", async () => {
const tenantFindMany = vi.fn(async () => [{ id: "tenant_1" }]);
const activeTenantReader = { tenant: { findMany: tenantFindMany } };
const artifact: TestGdprRetentionJob = {
id: "gdpr_export_1",
tenantId: "tenant_1",
status: "failed",
createdAt: new Date("2026-07-01T00:00:00.000Z"),
resultBucket: "exports",
resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_1/attempt_1.zip",
resultExpiresAt: null,
processingStartedAt: null,
};
const observedContexts: Array<string | undefined> = [];
const sequence: string[] = [];
const listExpiredGdprExportArtifacts = vi.fn(async () => {
observedContexts.push(getRlsExecutionContext()?.tenantId);
return [artifact];
});
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async (input) => {
sequence.push("storage-delete");
return { bucket: input.bucket ?? "exports", key: input.key };
}),
};
const markGdprExportArtifactExpired = vi.fn(async () => {
observedContexts.push(getRlsExecutionContext()?.tenantId);
sequence.push("db-claim");
});
const markGdprExportArtifactDeleted = vi.fn(async () => {
observedContexts.push(getRlsExecutionContext()?.tenantId);
sequence.push("db-finalize");
});
const registry = buildHandlerRegistry({
activeTenantReader,
env: { STORAGE_BUCKET: "exports" },
listExpiredGdprExportArtifacts,
markGdprExportArtifactDeleted,
markGdprExportArtifactExpired,
storageDeleteService,
});
const descriptor = buildCleanupJob({
task: "privacy-gdpr-export-retention",
take: 25,
metadata: { requestId: "req_gdpr_retention_1" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "privacy.gdpr-export-retention.completed",
details: {
task: "privacy-gdpr-export-retention",
requestId: "req_gdpr_retention_1",
processed: 1,
deleted: 1,
failed: 0,
jobs: [{ gdprExportJobId: "gdpr_export_1", status: "deleted" }],
},
});
expect(tenantFindMany).toHaveBeenCalledWith(expect.objectContaining({ where: {} }));
expect(listExpiredGdprExportArtifacts).toHaveBeenCalledWith({ tenantId: "tenant_1", now: expect.any(Date), take: 25 });
expect(storageDeleteService.deleteObject).toHaveBeenCalledWith({
bucket: "exports",
key: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_1/attempt_1.zip",
signal: expect.any(AbortSignal),
});
expect(markGdprExportArtifactExpired).toHaveBeenCalledWith({ ...artifact, now: expect.any(Date) });
expect(markGdprExportArtifactDeleted).toHaveBeenCalledWith({
id: "gdpr_export_1",
tenantId: "tenant_1",
resultBucket: "exports",
resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_1/attempt_1.zip",
});
expect(sequence).toEqual(["db-claim", "storage-delete", "db-finalize"]);
expect(observedContexts).toEqual(["tenant_1", "tenant_1", "tenant_1"]);
});
test("fails the maintenance job so failed privacy GDPR deletes retry on the next scheduler tick", async () => {
const listExpiredGdprExportArtifacts = vi.fn(
async (): Promise<TestGdprRetentionJob[]> => [
{
id: "gdpr_export_bad",
tenantId: "tenant_1",
status: "ready",
createdAt: new Date("2026-07-01T00:00:00.000Z"),
resultBucket: "exports",
resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_bad/attempt_bad.zip",
resultExpiresAt: new Date("2026-07-05T00:00:00.000Z"),
processingStartedAt: null,
},
]
);
const markGdprExportArtifactExpired = vi.fn(async () => undefined);
const markGdprExportArtifactDeleted = vi.fn(async () => undefined);
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async () => {
throw new Error("delete failed token=storage-secret authorization=Bearer-secret");
}),
};
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
env: { STORAGE_BUCKET: "exports" },
listExpiredGdprExportArtifacts,
markGdprExportArtifactDeleted,
markGdprExportArtifactExpired,
storageDeleteService,
});
const descriptor = buildCleanupJob({ task: "privacy-gdpr-export-retention" });
let failure: unknown;
try {
await registry[descriptor.queueName](descriptor.payload, {});
} catch (error) {
failure = error;
}
expect(failure).toBeInstanceOf(AggregateError);
expect(String(failure)).not.toContain("storage-secret");
expect(String(failure)).not.toContain("Bearer-secret");
expect(markGdprExportArtifactExpired).toHaveBeenCalledOnce();
expect(markGdprExportArtifactDeleted).not.toHaveBeenCalled();
});
test("continues a mixed privacy GDPR retention batch after one delete fails", async () => {
const artifacts: TestGdprRetentionJob[] = [
{
id: "gdpr_export_bad",
tenantId: "tenant_1",
status: "ready",
createdAt: new Date("2026-07-01T00:00:00.000Z"),
resultBucket: "exports",
resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_bad/attempt_bad.zip",
resultExpiresAt: new Date("2026-07-05T00:00:00.000Z"),
processingStartedAt: null,
},
{
id: "gdpr_export_good",
tenantId: "tenant_1",
status: "ready",
createdAt: new Date("2026-07-01T00:00:00.000Z"),
resultBucket: "exports",
resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_good/attempt_good.zip",
resultExpiresAt: new Date("2026-07-05T00:00:00.000Z"),
processingStartedAt: null,
},
];
const listExpiredGdprExportArtifacts = vi.fn(async () => artifacts);
const markGdprExportArtifactExpired = vi.fn(async () => undefined);
const markGdprExportArtifactDeleted = vi.fn(async () => undefined);
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async ({ bucket, key }) => {
if (key.includes("gdpr_export_bad")) throw new Error("delete failed token=storage-secret");
return { bucket, key };
}),
};
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
env: { STORAGE_BUCKET: "exports" },
listExpiredGdprExportArtifacts,
markGdprExportArtifactDeleted,
markGdprExportArtifactExpired,
storageDeleteService,
});
const descriptor = buildCleanupJob({ task: "privacy-gdpr-export-retention" });
let failure: unknown;
try {
await registry[descriptor.queueName](descriptor.payload, {});
} catch (error) {
failure = error;
}
expect(failure).toBeInstanceOf(AggregateError);
expect(String(failure)).not.toContain("storage-secret");
expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(2);
expect(markGdprExportArtifactExpired).toHaveBeenCalledTimes(2);
expect(markGdprExportArtifactDeleted).toHaveBeenCalledOnce();
expect(markGdprExportArtifactDeleted).toHaveBeenCalledWith({
id: "gdpr_export_good",
tenantId: "tenant_1",
resultBucket: "exports",
resultKey: "gdpr-exports/privacy/tenants/tenant_1/gdpr_export_good/attempt_good.zip",
});
});
test("caps export artifact hard-delete concurrency while preserving reported job order", async () => {
const jobs = Array.from({ length: 6 }, (_, index) =>
retentionJob({
id: `export_job_${index + 1}`,
artifactBucket: "exports",
artifactKey: `tenant_1/export-${index + 1}.csv`,
})
);
let activeDeletes = 0;
let maxActiveDeletes = 0;
let releaseFirstBatch: (() => void) | undefined;
const firstBatchWait = new Promise<void>((resolve) => {
releaseFirstBatch = resolve;
});
const listExpiredCompletedExportJobs = vi.fn(async () => []);
const markExportJobExpired = vi.fn(async () => undefined);
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs);
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async (input) => {
activeDeletes += 1;
maxActiveDeletes = Math.max(maxActiveDeletes, activeDeletes);
if (input.key !== "tenant_1/export-6.csv") {
await firstBatchWait;
}
activeDeletes -= 1;
return { bucket: input.bucket ?? "exports", key: input.key };
}),
};
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
listExpiredCompletedExportJobs,
markExportJobExpired,
listSoftDeletedExportJobsForHardDelete,
storageDeleteService,
markExportJobHardDeleteSucceeded,
});
const descriptor = buildCleanupJob({ task: "expired-export-artifacts" });
const run = registry[descriptor.queueName](descriptor.payload, {});
await vi.waitFor(() => expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(5));
expect(maxActiveDeletes).toBe(5);
expect(markExportJobHardDeleteSucceeded).not.toHaveBeenCalled();
releaseFirstBatch?.();
await expect(run).resolves.toEqual({
handled: true,
action: "export-artifact-retention.completed",
details: {
task: "expired-export-artifacts",
requestId: undefined,
expired: 0,
processed: 6,
deleted: 6,
failed: 0,
jobs: jobs.map((job) => ({ exportJobId: job.id, status: "deleted" })),
},
});
expect(maxActiveDeletes).toBe(5);
expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(6);
});
test("continues export artifact retention after storage failures with redacted diagnostics", async () => {
const jobs = [
retentionJob({ id: "export_job_ok", artifactBucket: "exports", artifactKey: "tenant_1/ok.csv" }),
retentionJob({ id: "export_job_bad", artifactBucket: "exports", artifactKey: "tenant_1/bad.csv" }),
];
const listExpiredCompletedExportJobs = vi.fn(async () => []);
const markExportJobExpired = vi.fn(async () => undefined);
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs);
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async (input) => {
if (input.key === "tenant_1/bad.csv") {
throw new Error("delete failed token=storage-secret body=payload-bytes authorization=Bearer-secret");
}
return { bucket: input.bucket ?? "exports", key: input.key };
}),
};
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
const markExportJobHardDeleteFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
listExpiredCompletedExportJobs,
markExportJobExpired,
listSoftDeletedExportJobsForHardDelete,
storageDeleteService,
markExportJobHardDeleteSucceeded,
markExportJobHardDeleteFailed,
});
const descriptor = buildCleanupJob({ task: "expired-export-artifacts", metadata: { requestId: "req_retention_partial" } });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "export-artifact-retention.completed",
details: {
task: "expired-export-artifacts",
requestId: "req_retention_partial",
expired: 0,
processed: 2,
deleted: 1,
failed: 1,
jobs: [
{ exportJobId: "export_job_ok", status: "deleted" },
{ exportJobId: "export_job_bad", status: "failed", message: "delete failed token=[redacted] body=[redacted] authorization=[redacted]" },
],
},
});
expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_ok", tenantId: "tenant_1" });
expect(markExportJobHardDeleteFailed).toHaveBeenCalledWith({
id: "export_job_bad",
tenantId: "tenant_1",
errorMessage: "delete failed token=[redacted] body=[redacted] authorization=[redacted]",
});
const resultText = JSON.stringify((await registry[descriptor.queueName](descriptor.payload, {})).details);
expect(resultText).not.toContain("storage-secret");
expect(resultText).not.toContain("payload-bytes");
expect(resultText).not.toContain("Bearer-secret");
});
test("fails export artifact retention when delete dependency is missing", async () => {
const jobs = [retentionJob({ id: "export_job_missing_delete", artifactBucket: "exports", artifactKey: "tenant_1/missing.csv" })];
const listExpiredCompletedExportJobs = vi.fn(async () => []);
const markExportJobExpired = vi.fn(async () => undefined);
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs);
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
const markExportJobHardDeleteFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
listExpiredCompletedExportJobs,
markExportJobExpired,
listSoftDeletedExportJobsForHardDelete,
markExportJobHardDeleteSucceeded,
markExportJobHardDeleteFailed,
});
const descriptor = buildCleanupJob({ task: "expired-export-artifacts" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("expired-export-artifacts failed for 1 tenant(s)");
expect(markExportJobHardDeleteFailed).not.toHaveBeenCalled();
expect(markExportJobHardDeleteSucceeded).not.toHaveBeenCalled();
});
test("marks expired completed exports before hard-delete retention scan", async () => {
const expiredJobs = [retentionJob({ id: "export_job_expired", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired.csv" })];
const listExpiredCompletedExportJobs = vi.fn(async () => expiredJobs);
const markExportJobExpired = vi.fn(async () => undefined);
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => []);
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async (input) => ({ bucket: input.bucket ?? "exports", key: input.key })),
};
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
listExpiredCompletedExportJobs,
markExportJobExpired,
listSoftDeletedExportJobsForHardDelete,
storageDeleteService,
});
const descriptor = buildCleanupJob({ task: "expired-export-artifacts", take: 25 });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "export-artifact-retention.completed",
details: {
expired: 1,
processed: 0,
deleted: 0,
failed: 0,
},
});
expect(listExpiredCompletedExportJobs).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 });
expect(markExportJobExpired).toHaveBeenCalledWith({ id: "export_job_expired", tenantId: "tenant_1", deletedAt: expect.any(Date) });
expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 });
expect(storageDeleteService.deleteObject).not.toHaveBeenCalled();
});
test("marks independent expired completed exports in parallel before hard-delete retention scan", async () => {
const expiredJobs = [
retentionJob({ id: "export_job_expired_1", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired-1.csv" }),
retentionJob({ id: "export_job_expired_2", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired-2.csv" }),
];
const listExpiredCompletedExportJobs = vi.fn(async () => expiredJobs);
let firstMarkCanFinish: (() => void) | undefined;
const firstMarkWait = new Promise<void>((resolve) => {
firstMarkCanFinish = resolve;
});
const markExportJobExpired = vi.fn(async ({ id }) => {
if (id === "export_job_expired_1") {
await firstMarkWait;
}
});
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => []);
const storageDeleteService: TestStorageDeleteService = {
deleteObject: vi.fn(async (input) => ({ bucket: input.bucket ?? "exports", key: input.key })),
};
const registry = buildHandlerRegistry({
activeTenantReader: createActiveTenantReader(),
listExpiredCompletedExportJobs,
markExportJobExpired,
listSoftDeletedExportJobsForHardDelete,
storageDeleteService,
});
const descriptor = buildCleanupJob({ task: "expired-export-artifacts" });
const run = registry[descriptor.queueName](descriptor.payload, {});
await vi.waitFor(() => expect(markExportJobExpired).toHaveBeenCalledTimes(2));
expect(listSoftDeletedExportJobsForHardDelete).not.toHaveBeenCalled();
firstMarkCanFinish?.();
await expect(run).resolves.toMatchObject({
action: "export-artifact-retention.completed",
details: { expired: 2, processed: 0 },
});
expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined });
});
test("rejects descriptor-derived browser automation jobs through registry without global fetch", async () => {
const fetchSpy = vi.spyOn(globalThis, "fetch");
const registry = buildHandlerRegistry();
const descriptors = [
buildBrowserSessionRunJob({
sessionId: "session_123",
tenantId: "tenant_123",
userId: "user_123",
allowedOrigins: ["https://example.com"],
task: { instruction: "fixture browser session task not surfaced" },
}),
buildBrowserSessionCleanupJob({
sessionId: "session_123",
tenantId: "tenant_123",
metadata: { reason: "fixture cleanup reason not surfaced" },
}),
buildBrowserUsageFlushJob({
sessionId: "session_123",
tenantId: "tenant_123",
metadata: { token: "fixture token not surfaced" },
}),
];
try {
await Promise.all(
descriptors.map((descriptor) =>
expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "worker.unsupported",
message: "Browser automation worker job is not configured.",
details: { queueName: queueNames.browserAutomation, kind: descriptor.payload.kind, sessionId: "session_123" },
})
)
);
expect(fetchSpy).not.toHaveBeenCalled();
} finally {
fetchSpy.mockRestore();
}
});
test("pauses gated export jobs without running the export service", async () => {
const exportService: TestExportService = {
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt: new Date() })),
};
const markExportJobPaused = vi.fn(async () => undefined);
const markExportJobRunning = vi.fn(async () => undefined);
const markExportJobSucceeded = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobPaused, markExportJobRunning, markExportJobSucceeded });
const descriptor = buildExportJob({
exportJobId: "export_job_paused",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "csv",
requestId: "req_export_paused",
complianceGate: {
state: "paused",
reason: "workspace-retention-pause",
requestId: "req_pause_1",
},
metadata: { attemptsMade: 3 },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "export.paused",
details: {
exportJobId: "export_job_paused",
tenantId: "tenant_1",
exportType: "members",
format: "csv",
requestId: "req_export_paused",
attemptsMade: 3,
pauseReason: "workspace-retention-pause",
pauseRequestId: "req_pause_1",
status: "paused",
},
});
expect(markExportJobPaused).toHaveBeenCalledWith({ id: "export_job_paused", tenantId: "tenant_1", attemptsMade: 3, reason: "workspace-retention-pause" });
expect(exportService.runExport).not.toHaveBeenCalled();
expect(markExportJobRunning).not.toHaveBeenCalled();
expect(markExportJobSucceeded).not.toHaveBeenCalled();
});
test("redacts secret-like pause reasons in persisted and returned diagnostics", async () => {
const exportService: TestExportService = {
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt: new Date() })),
};
const markExportJobPaused = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobPaused });
const descriptor = buildExportJob({
exportJobId: "export_job_secret_pause",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "json",
requestId: "req_export_secret_pause",
complianceGate: {
state: "paused",
reason: "pause token=service-secret body=payload-bytes authorization=Bearer-secret",
requestId: "req_pause_secret",
},
metadata: { attemptsMade: 1 },
filters: { apiKey: "payload-secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "export.paused",
details: {
pauseReason: "pause token=[redacted] body=[redacted] authorization=[redacted]",
},
});
expect(markExportJobPaused).toHaveBeenCalledWith({
id: "export_job_secret_pause",
tenantId: "tenant_1",
attemptsMade: 1,
reason: "pause token=[redacted] body=[redacted] authorization=[redacted]",
});
expect(exportService.runExport).not.toHaveBeenCalled();
expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("service-secret");
expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("payload-secret");
expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("Bearer-secret");
});
test("resumes active gated export jobs before running them", async () => {
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
const events: string[] = [];
const exportService: TestExportService = {
runExport: vi.fn(async () => {
events.push("export");
return { artifactBucket: "exports", artifactKey: "tenant_1/resumed.csv", rowCount: 7, expiresAt };
}),
};
const markExportJobResumed = vi.fn(async () => {
events.push("resume");
});
const markExportJobRunning = vi.fn(async () => {
events.push("running");
});
const markExportJobSucceeded = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobResumed, markExportJobRunning, markExportJobSucceeded });
const descriptor = buildExportJob({
exportJobId: "export_job_resumed",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "csv",
requestId: "req_export_resumed",
complianceGate: {
state: "active",
resumedAt: "2026-05-08T00:00:00.000Z",
requestId: "req_resume_1",
},
metadata: { attemptsMade: 2 },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "export.completed",
details: { exportJobId: "export_job_resumed", status: "completed", attemptsMade: 2 },
});
expect(markExportJobResumed).toHaveBeenCalledWith({ id: "export_job_resumed", tenantId: "tenant_1", attemptsMade: 2 });
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_resumed", tenantId: "tenant_1", attemptsMade: 2 });
expect(exportService.runExport).toHaveBeenCalledWith(descriptor.payload);
expect(events).toEqual(["resume", "running", "export"]);
});
test("runs export jobs through injected service and persists completed artifact diagnostics", async () => {
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
const exportService: TestExportService = {
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt })),
};
const markExportJobRunning = vi.fn(async () => undefined);
const markExportJobSucceeded = vi.fn(async () => undefined);
const markExportJobFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobSucceeded, markExportJobFailed });
const descriptor = buildExportJob({
exportJobId: "export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "csv",
requestId: "req_export_success",
metadata: { attemptsMade: 2, token: "payload-secret" },
filters: { authorization: "Bearer payload-secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "export.completed",
details: {
exportJobId: "export_job_1",
tenantId: "tenant_1",
exportType: "members",
format: "csv",
requestId: "req_export_success",
artifactBucket: "exports",
artifactKey: "tenant_1/export.csv",
attemptsMade: 2,
expiresAt: "2026-05-09T00:00:00.000Z",
rowCount: 42,
status: "completed",
},
});
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1", attemptsMade: 2 });
expect(exportService.runExport).toHaveBeenCalledWith(descriptor.payload);
expect(markExportJobSucceeded).toHaveBeenCalledWith({
id: "export_job_1",
tenantId: "tenant_1",
artifactBucket: "exports",
artifactKey: "tenant_1/export.csv",
rowCount: 42,
attemptsMade: 2,
expiresAt,
});
expect(markExportJobFailed).not.toHaveBeenCalled();
expect(JSON.stringify(markExportJobRunning.mock.calls)).not.toContain("payload-secret");
expect(JSON.stringify(markExportJobSucceeded.mock.calls)).not.toContain("payload-secret");
});
test("routes privacy GDPR export jobs through the GDPR service and status table", async () => {
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
const exportService: TestExportService = {
runExport: vi.fn(async () => ({ artifactBucket: "unused", artifactKey: "unused", rowCount: 0, expiresAt })),
};
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => ({
artifactBucket: "exports",
artifactKey: privacyAttempt.artifactKey,
rowCount: 7,
sizeBytes: 4096,
expiresAt,
}))
);
const markGdprExportProcessing = vi.fn<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>(async () => undefined);
const markGdprExportReady = vi.fn(async () => "committed" as const);
const markGdprExportFailed = vi.fn(async () => undefined);
const markExportJobRunning = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
exportService,
privacyGdprExportService,
markGdprExportProcessing,
markGdprExportReady,
markGdprExportFailed,
markExportJobRunning,
});
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
requestId: "req_gdpr_export_success",
metadata: { attemptsMade: 1, token: "payload-secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "privacy-gdpr-export.completed",
details: {
exportJobId: "gdpr_export_job_1",
tenantId: "tenant_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
requestId: "req_gdpr_export_success",
artifactBucket: "exports",
artifactKey: privacyAttempt.artifactKey,
attemptsMade: 1,
expiresAt: "2026-05-09T00:00:00.000Z",
rowCount: 7,
sizeBytes: 4096,
status: "ready",
},
});
expect(markGdprExportProcessing).toHaveBeenCalledWith(
expect.objectContaining({
id: "gdpr_export_job_1",
tenantId: "tenant_1",
resultBucket: "exports",
resultKey: privacyAttempt.artifactKey,
now: expect.any(Date),
})
);
expect(privacyGdprExportService.runExport).toHaveBeenCalledWith(descriptor.payload, privacyAttempt);
const processingStartedAt = markGdprExportProcessing.mock.calls[0]?.[0]?.now;
expect(markGdprExportReady).toHaveBeenCalledWith({
id: "gdpr_export_job_1",
tenantId: "tenant_1",
resultBucket: "exports",
resultKey: privacyAttempt.artifactKey,
resultSizeBytes: 4096,
resultExpiresAt: expiresAt,
processingStartedAt,
});
expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled();
expect(markGdprExportFailed).not.toHaveBeenCalled();
expect(exportService.runExport).not.toHaveBeenCalled();
expect(markExportJobRunning).not.toHaveBeenCalled();
});
test("does not run a privacy export whose queue subject loses the persisted-job claim", async () => {
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => {
throw new Error("must not read or store export data");
})
);
const markGdprExportProcessing = vi.fn<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>(async (input) => {
if (
input.requestedByUserId !== "user_1" ||
input.scope !== "user_self" ||
input.targetUserId !== "user_1"
) {
throw new Error("Privacy GDPR export subject does not match the persisted job.");
}
});
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_subject_bound",
tenantId: "tenant_1",
requestedByUserId: "user_stale",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
metadata: { targetUserId: "user_stale" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow(
"Privacy GDPR export subject does not match the persisted job."
);
expect(markGdprExportProcessing).toHaveBeenCalledWith(
expect.objectContaining({ requestedByUserId: "user_stale", scope: "user_self", targetUserId: "user_stale" })
);
expect(privacyGdprExportService.runExport).not.toHaveBeenCalled();
});
test("quiesces a privacy GDPR export after PUT and before the Ready transition", async () => {
const expiresAt = new Date("2026-07-23T10:00:00.000Z");
const result = {
artifactBucket: "exports",
artifactKey: privacyAttempt.artifactKey,
rowCount: 7,
sizeBytes: 4_096,
expiresAt,
};
let checkpointCount = 0;
const checkpoint = vi.fn(async () => {
checkpointCount += 1;
if (checkpointCount === 2) throw new TenantExportQuiescedError();
});
const withTenantBarrier: TestPrivacyGdprExportService["withTenantBarrier"] = async (_payload, writer) => writer(checkpoint);
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => result),
{ withTenantBarrier }
);
const markGdprExportProcessing = vi.fn<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>(async () => undefined);
const markGdprExportReady = vi.fn(async () => "committed" as const);
const markGdprExportFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_quiesced_after_put",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "export.error",
details: expect.objectContaining({ errorName: "TenantExportQuiescedError" }),
});
const processingStartedAt = markGdprExportProcessing.mock.calls[0]?.[0]?.now;
expect(checkpoint).toHaveBeenCalledTimes(2);
expect(privacyGdprExportService.runExport).toHaveBeenCalledWith(descriptor.payload, privacyAttempt);
expect(markGdprExportReady).not.toHaveBeenCalled();
expect(privacyGdprExportService.deleteArtifact).toHaveBeenCalledWith(privacyAttempt);
expect(markGdprExportFailed).toHaveBeenCalledWith({
id: "gdpr_export_job_quiesced_after_put",
tenantId: "tenant_1",
errorMessage: "Tenant exports are blocked because hard deletion has started.",
processingStartedAt,
resultBucket: privacyAttempt.artifactBucket,
resultKey: privacyAttempt.artifactKey,
clearResultLocation: true,
});
expect(privacyGdprExportService.terminalizeReadyAfterQuiescence).not.toHaveBeenCalled();
});
test("terminalizes the exact Ready artifact when quiescence starts after the Ready CAS", async () => {
const expiresAt = new Date("2026-07-23T10:00:00.000Z");
const result = {
artifactBucket: "exports",
artifactKey: privacyAttempt.artifactKey,
rowCount: 7,
sizeBytes: 4_096,
expiresAt,
};
let checkpointCount = 0;
const checkpoint = vi.fn(async () => {
checkpointCount += 1;
if (checkpointCount === 4) throw new TenantExportQuiescedError();
});
const withTenantBarrier: TestPrivacyGdprExportService["withTenantBarrier"] = async (_payload, writer) => writer(checkpoint);
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => result),
{ withTenantBarrier }
);
const markGdprExportProcessing = vi.fn<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>(async () => undefined);
const markGdprExportReady = vi.fn(async () => "committed" as const);
const markGdprExportFailed = vi.fn(async () => undefined);
const sendGdprExportReadyMail = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
privacyGdprExportService,
markGdprExportProcessing,
markGdprExportReady,
markGdprExportFailed,
sendGdprExportReadyMail,
});
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_quiesced_after_ready",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "export.error",
details: expect.objectContaining({ errorName: "TenantExportQuiescedError" }),
});
expect(checkpoint).toHaveBeenCalledTimes(4);
expect(markGdprExportReady).toHaveBeenCalledOnce();
expect(privacyGdprExportService.terminalizeReadyAfterQuiescence).toHaveBeenCalledWith({ payload: descriptor.payload, result });
expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled();
expect(markGdprExportFailed).not.toHaveBeenCalled();
expect(sendGdprExportReadyMail).not.toHaveBeenCalled();
});
test("short-circuits a retried privacy export when the ready row already owns a valid artifact", async () => {
const expiresAt = new Date("2026-07-23T10:00:00.000Z");
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => {
throw new Error("must not upload again");
})
);
vi.mocked(privacyGdprExportService.resolveReady).mockResolvedValue({
artifactBucket: "exports",
artifactKey: privacyAttempt.artifactKey,
rowCount: null,
sizeBytes: 4_096,
expiresAt,
});
const markGdprExportProcessing = vi.fn(async (_input: Parameters<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>[0]) => undefined);
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "privacy-gdpr-export.completed",
details: { artifactKey: privacyAttempt.artifactKey, rowCount: null, status: "ready" },
});
expect(privacyGdprExportService.createAttempt).not.toHaveBeenCalled();
expect(privacyGdprExportService.runExport).not.toHaveBeenCalled();
expect(markGdprExportProcessing).not.toHaveBeenCalled();
});
test("does not delete or downgrade an artifact when the ready commit outcome is ambiguous, then reconciles on retry", async () => {
const expiresAt = new Date("2026-07-23T10:00:00.000Z");
const result = {
artifactBucket: "exports",
artifactKey: privacyAttempt.artifactKey,
rowCount: 7,
sizeBytes: 4_096,
expiresAt,
};
const privacyGdprExportService = createPrivacyGdprService(vi.fn(async () => result));
const markGdprExportReady = vi.fn(async () => {
throw new Error("database response lost");
});
const markGdprExportProcessing = vi.fn(async (_input: Parameters<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>[0]) => undefined);
const markGdprExportFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error" });
expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled();
expect(markGdprExportFailed).not.toHaveBeenCalled();
vi.mocked(privacyGdprExportService.resolveReady).mockResolvedValue({ ...result, rowCount: null });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "privacy-gdpr-export.completed",
details: { artifactKey: privacyAttempt.artifactKey, rowCount: null, status: "ready" },
});
expect(privacyGdprExportService.createAttempt).toHaveBeenCalledTimes(1);
expect(privacyGdprExportService.runExport).toHaveBeenCalledTimes(1);
expect(markGdprExportReady).toHaveBeenCalledTimes(1);
});
test("does not delete an adopted attempt when the privacy ready transition explicitly loses ownership", async () => {
const expiresAt = new Date("2026-07-23T10:00:00.000Z");
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => ({ artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt }))
);
const markGdprExportReady = vi.fn(async () => "lost_owner" as const);
const markGdprExportProcessing = vi.fn(async (_input: Parameters<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>[0]) => undefined);
const markGdprExportFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({ action: "export.error" });
expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled();
expect(markGdprExportFailed).not.toHaveBeenCalled();
});
test("retries an ambiguously committed privacy PUT with the exact persisted attempt", async () => {
const expiresAt = new Date("2026-07-23T13:00:00.000Z");
const committedKeys: string[] = [];
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async (_payload, attempt) => {
committedKeys.push(attempt.artifactKey);
if (committedKeys.length === 1) throw new PrivacyGdprExportPutOutcomeUnknownError();
return { artifactBucket: attempt.artifactBucket, artifactKey: attempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt };
})
);
const markGdprExportProcessing = vi.fn<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>(async () => undefined);
const markGdprExportReady = vi.fn(async () => "committed" as const);
const markGdprExportFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady, markGdprExportFailed });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({
action: "export.error",
});
const firstClaim = markGdprExportProcessing.mock.calls[0]?.[0]?.now;
expect(firstClaim).toBeInstanceOf(Date);
expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled();
expect(markGdprExportFailed).not.toHaveBeenCalled();
vi.mocked(privacyGdprExportService.resolveReclaimableProcessing).mockResolvedValue({
processingStartedAt: firstClaim!,
attempt: privacyAttempt,
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).resolves.toMatchObject({
action: "privacy-gdpr-export.completed",
details: { artifactKey: privacyAttempt.artifactKey, status: "ready" },
});
expect(committedKeys).toEqual([privacyAttempt.artifactKey, privacyAttempt.artifactKey]);
expect(privacyGdprExportService.createAttempt).toHaveBeenCalledOnce();
expect(markGdprExportProcessing).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
previousProcessingStartedAt: firstClaim,
resultBucket: privacyAttempt.artifactBucket,
resultKey: privacyAttempt.artifactKey,
})
);
expect(markGdprExportReady).toHaveBeenCalledOnce();
expect(privacyGdprExportService.deleteArtifact).not.toHaveBeenCalled();
expect(privacyGdprExportService.terminalizeReadyAfterQuiescence).not.toHaveBeenCalled();
expect(markGdprExportFailed).not.toHaveBeenCalled();
});
test("reclaims a stale privacy processing claim on a BullMQ retry with an exact compare-and-swap token", async () => {
const previousProcessingStartedAt = new Date("2026-07-23T09:59:00.000Z");
const expiresAt = new Date("2026-07-23T13:00:00.000Z");
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => ({ artifactBucket: "exports", artifactKey: privacyAttempt.artifactKey, rowCount: 7, sizeBytes: 4_096, expiresAt }))
);
vi.mocked(privacyGdprExportService.resolveReclaimableProcessing).mockResolvedValue({
processingStartedAt: previousProcessingStartedAt,
attempt: privacyAttempt,
});
const markGdprExportProcessing = vi.fn(async (_input: Parameters<NonNullable<HandlerDependencies["markGdprExportProcessing"]>>[0]) => undefined);
const markGdprExportReady = vi.fn(async () => "committed" as const);
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportReady });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1 })).resolves.toMatchObject({
action: "privacy-gdpr-export.completed",
});
expect(markGdprExportProcessing).toHaveBeenCalledWith({
id: "gdpr_export_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
scope: "user_self",
targetUserId: "user_1",
resultBucket: "exports",
resultKey: privacyAttempt.artifactKey,
now: expect.any(Date),
previousProcessingStartedAt,
});
expect(markGdprExportReady).toHaveBeenCalledWith(expect.objectContaining({ processingStartedAt: expect.any(Date) }));
});
test("normalizes a PortalUser GDPR outbox envelope and routes only the safe export contract", async () => {
const expiresAt = new Date("2026-07-17T10:00:00.000Z");
const portalGdprExportService: TestPortalGdprExportService = {
runExport: vi.fn(async () => ({
artifactBucket: "private-exports",
artifactKey: "gdpr-exports/portal/tenants/tenant_1/export_1/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json",
rowCount: 9,
sizeBytes: 4096,
expiresAt,
})),
};
const registry = buildHandlerRegistry({ portalGdprExportService });
const outboxEnvelope = {
id: "outbox_1",
tenantId: "tenant_1",
eventType: "portal.gdpr_export.requested",
payload: { tenantId: "tenant_1", portalUserId: "portal_user_1", jobId: "export_1", password: "must-not-forward" },
};
await expect(registry[queueNames.exports](outboxEnvelope as never, {})).resolves.toEqual({
handled: true,
action: "portal-gdpr-export.completed",
details: {
exportJobId: "export_1",
tenantId: "tenant_1",
exportType: "gdpr_portal_user_self",
format: "json",
requestId: "outbox_1",
artifactBucket: "private-exports",
artifactKey: "gdpr-exports/portal/tenants/tenant_1/export_1/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json",
attemptsMade: 0,
expiresAt: "2026-07-17T10:00:00.000Z",
rowCount: 9,
sizeBytes: 4096,
status: "ready",
},
});
expect(portalGdprExportService.runExport).toHaveBeenCalledWith({
kind: "export.run",
exportJobId: "export_1",
tenantId: "tenant_1",
requestedByUserId: "portal_user_1",
exportType: "gdpr_portal_user_self",
format: "json",
requestId: "outbox_1",
metadata: { actorKind: "portal_user", sourceEventType: "portal.gdpr_export.requested", outboxEventId: "outbox_1" },
});
expect(JSON.stringify(vi.mocked(portalGdprExportService.runExport).mock.calls)).not.toContain("must-not-forward");
});
test("rejects a cross-tenant portal GDPR outbox envelope before the export service", async () => {
const portalGdprExportService: TestPortalGdprExportService = {
runExport: vi.fn(async () => {
throw new Error("must not run");
}),
};
const registry = buildHandlerRegistry({ portalGdprExportService });
const outboxEnvelope = {
id: "outbox_cross_tenant",
tenantId: "tenant_a",
eventType: "portal.gdpr_export.requested",
payload: { tenantId: "tenant_b", portalUserId: "portal_user_b", jobId: "export_b" },
};
await expect(registry[queueNames.exports](outboxEnvelope as never, {})).rejects.toMatchObject({
action: "export.error",
details: { code: "portal_gdpr_export_invalid_envelope", tenantMatch: false },
});
expect(portalGdprExportService.runExport).not.toHaveBeenCalled();
});
test("fails privacy GDPR export jobs when the dedicated service is missing", async () => {
const markGdprExportProcessing = vi.fn(async () => undefined);
const markGdprExportFailed = vi.fn(async () => undefined);
const markExportJobRunning = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ markGdprExportProcessing, markGdprExportFailed, markExportJobRunning });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_missing_service",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
requestId: "req_gdpr_export_missing_service",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "export.error",
message: "Privacy GDPR export service is not configured.",
details: {
exportJobId: "gdpr_export_job_missing_service",
tenantId: "tenant_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
requestId: "req_gdpr_export_missing_service",
attemptsMade: 0,
status: "failed",
code: "privacy_gdpr_export_service_missing",
},
});
expect(markGdprExportProcessing).toHaveBeenCalledWith({
id: "gdpr_export_job_missing_service",
tenantId: "tenant_1",
requestedByUserId: "user_1",
scope: "user_self",
targetUserId: "user_1",
});
expect(markGdprExportFailed).toHaveBeenCalledWith({
id: "gdpr_export_job_missing_service",
tenantId: "tenant_1",
errorMessage: "Privacy GDPR export service is not configured.",
});
expect(markExportJobRunning).not.toHaveBeenCalled();
});
test("persists the clear privacy GDPR size-limit diagnostic instead of an OOM-shaped failure", async () => {
const privacyGdprExportService = createPrivacyGdprService(
vi.fn(async () => {
throw new PrivacyGdprExportSizeLimitExceededError(2_048, 1_024);
})
);
const markGdprExportProcessing = vi.fn(async () => undefined);
const markGdprExportFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ privacyGdprExportService, markGdprExportProcessing, markGdprExportFailed });
const descriptor = buildExportJob({
exportJobId: "gdpr_export_job_too_large",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
format: "json",
});
const run = registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 2 });
await expect(run).rejects.toBeInstanceOf(UnrecoverableError);
await expect(run).rejects.toMatchObject({
action: "export.error",
details: expect.objectContaining({
errorName: "PrivacyGdprExportSizeLimitExceededError",
message: expect.stringContaining("stopped before a partial archive was stored"),
}),
});
expect(markGdprExportFailed).toHaveBeenCalledWith(
expect.objectContaining({
id: "gdpr_export_job_too_large",
tenantId: "tenant_1",
errorMessage: expect.stringContaining("streaming GDPR export path"),
processingStartedAt: expect.any(Date),
clearResultLocation: true,
})
);
expect(privacyGdprExportService.deleteArtifact).toHaveBeenCalledWith(privacyAttempt);
});
test("uses BullMQ attempt metadata for export job persistence when available", async () => {
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
const exportService: TestExportService = {
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt })),
};
const markExportJobRunning = vi.fn(async () => undefined);
const markExportJobSucceeded = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobSucceeded });
const descriptor = buildExportJob({
exportJobId: "export_job_bullmq_attempt",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "csv",
requestId: "req_export_bullmq_attempt",
metadata: { attemptsMade: 1 },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 4 })).resolves.toMatchObject({
action: "export.completed",
details: { attemptsMade: 5 },
});
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_bullmq_attempt", tenantId: "tenant_1", attemptsMade: 5 });
expect(markExportJobSucceeded).toHaveBeenCalledWith(expect.objectContaining({ id: "export_job_bullmq_attempt", tenantId: "tenant_1", attemptsMade: 5 }));
});
test("fails export jobs when service is missing", async () => {
const markExportJobRunning = vi.fn(async () => undefined);
const markExportJobSucceeded = vi.fn(async () => undefined);
const markExportJobFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ markExportJobRunning, markExportJobSucceeded, markExportJobFailed });
const descriptor = buildExportJob({
exportJobId: "export_job_noop",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "json",
requestId: "req_export_noop",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "export.error",
message: "Export service is not configured.",
details: {
exportJobId: "export_job_noop",
tenantId: "tenant_1",
exportType: "members",
format: "json",
requestId: "req_export_noop",
attemptsMade: 0,
status: "failed",
code: "export_service_missing",
},
});
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_noop", tenantId: "tenant_1", attemptsMade: 0 });
expect(markExportJobFailed).toHaveBeenCalledWith({
id: "export_job_noop",
tenantId: "tenant_1",
attemptsMade: 0,
errorName: "ExportHandlerError",
errorMessage: "Export service is not configured.",
});
expect(markExportJobSucceeded).not.toHaveBeenCalled();
});
test("marks export jobs failed with redacted scalar diagnostics when service rejects", async () => {
const exportService: TestExportService = {
runExport: vi.fn(async () => {
throw new Error("export failed token=service-secret authorization=Bearer-secret body=payload-bytes");
}),
};
const markExportJobRunning = vi.fn(async () => undefined);
const markExportJobFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobFailed });
const descriptor = buildExportJob({
exportJobId: "export_job_failed",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "csv",
metadata: { attemptsMade: 1, token: "payload-secret" },
filters: { apiKey: "payload-secret" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "export.error",
message: "Export job failed.",
details: expect.objectContaining({
status: "failed",
message: "export failed token=[redacted] authorization=[redacted] body=[redacted]",
}),
});
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_failed", tenantId: "tenant_1", attemptsMade: 1 });
expect(markExportJobFailed).toHaveBeenCalledWith({
id: "export_job_failed",
tenantId: "tenant_1",
attemptsMade: 1,
errorName: "Error",
errorMessage: "export failed token=[redacted] authorization=[redacted] body=[redacted]",
});
expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("payload-secret");
expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("service-secret");
expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("Bearer-secret");
});
test("keeps export jobs running until terminal retry failure", async () => {
const exportService: TestExportService = {
runExport: vi.fn(async () => {
throw new Error("temporary export outage");
}),
};
const markExportJobRunning = vi.fn(async () => undefined);
const markExportJobFailed = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobFailed });
const descriptor = buildExportJob({
exportJobId: "export_job_retrying",
tenantId: "tenant_1",
requestedByUserId: "user_1",
exportType: "members",
format: "csv",
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({
action: "export.error",
message: "Export job failed.",
});
expect(markExportJobFailed).not.toHaveBeenCalled();
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toMatchObject({
action: "export.error",
message: "Export job failed.",
});
expect(markExportJobFailed).toHaveBeenCalledWith({
id: "export_job_retrying",
tenantId: "tenant_1",
attemptsMade: 3,
errorName: "Error",
errorMessage: "temporary export outage",
});
});
test("rejects malformed export payload kinds", async () => {
const exportService: TestExportService = {
runExport: vi.fn(async () => ({ artifactBucket: "unused", artifactKey: "unused", rowCount: 0, expiresAt: new Date() })),
};
const markExportJobRunning = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ exportService, markExportJobRunning });
await expect(registry[queueNames.exports]({ kind: "export.other", exportJobId: "bad", tenantId: "tenant_1" } as never, {})).rejects.toMatchObject({
action: "export.error",
message: "Unsupported export payload kind.",
details: { code: "invalid_payload" },
});
expect(markExportJobRunning).not.toHaveBeenCalled();
expect(exportService.runExport).not.toHaveBeenCalled();
});
test("runs tenant-less admin sales import and export jobs through their dedicated service", async () => {
const adminSalesDataService = {
runImport: vi.fn(async () => ({
totalRows: 3,
acceptedRows: 2,
rejectedRows: 1,
})),
runExport: vi.fn(async () => ({
rowCount: 2,
artifactBucket: "hivegrid",
artifactKey: "admin-sales/exports/export_1.csv",
expiresAt: new Date("2026-08-01T00:00:00.000Z"),
})),
};
const registry = buildHandlerRegistry({ adminSalesDataService });
const importJob = buildAdminSalesContactsImportJob({
importJobId: "import_1",
requestedByAdminUserId: "admin_1",
});
const exportJob = buildAdminSalesContactsExportJob({
exportJobId: "export_1",
requestedByAdminUserId: "admin_1",
});
await expect(
registry[importJob.queueName](importJob.payload, {})
).resolves.toEqual({
handled: true,
action: "admin-sales.contacts-import.completed",
details: {
importJobId: "import_1",
requestedByAdminUserId: "admin_1",
totalRows: 3,
acceptedRows: 2,
rejectedRows: 1,
},
});
await expect(
registry[exportJob.queueName](exportJob.payload, {})
).resolves.toMatchObject({
handled: true,
action: "admin-sales.contacts-export.completed",
details: {
exportJobId: "export_1",
rowCount: 2,
expiresAt: "2026-08-01T00:00:00.000Z",
},
});
});
test("fails admin sales data jobs when their dedicated service is missing", async () => {
const descriptor = buildAdminSalesContactsImportJob({
importJobId: "import_missing",
requestedByAdminUserId: "admin_1",
});
const registry = buildHandlerRegistry({});
await expect(
registry[descriptor.queueName](descriptor.payload, {})
).rejects.toMatchObject({
action: "import.error",
message: "Admin sales data worker service is not configured.",
details: {
code: "admin_sales_data_service_missing",
kind: "admin-sales.contacts-import",
},
});
});
test("fails import jobs when service is missing", async () => {
const markImportJobRunning = vi.fn(async () => undefined);
const markImportJobSucceeded = vi.fn(async () => undefined);
const markImportJobFailed = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_1",
tenantId: "tenant_1",
requestedByUserId: "user_1",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_1/members.csv",
requestId: "req_import_1",
metadata: { attemptsMade: 2 },
});
const registry = buildHandlerRegistry({ markImportJobRunning, markImportJobSucceeded, markImportJobFailed });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "import.error",
message: "Import service is not configured.",
details: {
importJobId: "import_job_1",
tenantId: "tenant_1",
importType: "members",
format: "csv",
requestId: "req_import_1",
status: "failed",
attemptsMade: 2,
code: "import_service_missing",
},
});
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_1", tenantId: "tenant_1", attemptsMade: 2 });
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_1",
tenantId: "tenant_1",
attemptsMade: 2,
errorName: "ImportHandlerError",
errorMessage: "Import service is not configured.",
});
expect(markImportJobSucceeded).not.toHaveBeenCalled();
});
test("rejects import jobs whose source key is outside the tenant prefix before scanning", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
};
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" })
),
};
const markImportJobRunning = vi.fn(async () => undefined);
const markImportJobFailed = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_cross_tenant",
tenantId: "tenant_1",
requestedByUserId: "user_1",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_2/members.csv",
requestId: "req_import_cross_tenant",
});
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobFailed });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "import.error",
message: "Import source storage access denied.",
details: {
importJobId: "import_job_cross_tenant",
tenantId: "tenant_1",
requestId: "req_import_cross_tenant",
status: "failed",
code: "source_access_denied",
reason: "tenant_mismatch",
},
});
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_cross_tenant", tenantId: "tenant_1", attemptsMade: 0 });
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_cross_tenant",
tenantId: "tenant_1",
attemptsMade: 0,
errorName: "ImportHandlerError",
errorMessage: "Import source storage access denied. reason=tenant_mismatch",
});
expect(scanService.scanObject).not.toHaveBeenCalled();
expect(importService.runImport).not.toHaveBeenCalled();
});
test("runs import jobs through the injected service and persists duplicate-aware counters", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({
rowCount: 4,
acceptedRows: 2,
rejectedRows: 2,
duplicateCount: 1,
validationReport: {
summary: {
format: "csv" as const,
totalRows: 4,
acceptedRows: 2,
rejectedRows: 2,
diagnostics: [],
duplicateGroups: 1,
duplicateRows: 2,
},
errorReport: { diagnostics: [], rejectedRows: 2, errorReportKey: "reports/import_job_2.json" },
},
errorReportBucket: "diagnostics",
})),
};
const markImportJobRunning = vi.fn(async () => undefined);
const markImportJobSucceeded = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_2",
tenantId: "tenant_2",
requestedByUserId: "user_2",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_2/members.csv",
requestId: "req_import_2",
});
const scanService: TestScanService = {
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" })),
};
const events: string[] = [];
vi.mocked(scanService.scanObject).mockImplementation(async () => {
events.push("scan");
return scanResult({ verdict: "clean", bucket: "imports", key: "tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" });
});
vi.mocked(importService.runImport).mockImplementation(async () => {
events.push("import");
return {
rowCount: 4,
acceptedRows: 2,
rejectedRows: 2,
duplicateCount: 1,
validationReport: {
summary: {
format: "csv" as const,
totalRows: 4,
acceptedRows: 2,
rejectedRows: 2,
diagnostics: [],
duplicateGroups: 1,
duplicateRows: 2,
},
errorReport: { diagnostics: [], rejectedRows: 2, errorReportKey: "reports/import_job_2.json" },
},
errorReportBucket: "diagnostics",
};
});
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobSucceeded });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "import.completed",
details: {
importJobId: "import_job_2",
tenantId: "tenant_2",
importType: "members",
format: "csv",
requestId: "req_import_2",
status: "completed",
attemptsMade: 0,
rowCount: 4,
acceptedRows: 2,
rejectedRows: 2,
duplicateCount: 1,
errorReportBucket: "diagnostics",
errorReportKey: "reports/import_job_2.json",
},
});
expect(events).toEqual(["scan", "import"]);
expect(scanService.scanObject).toHaveBeenCalledWith({
bucket: "imports",
key: "tenants/tenant_2/members.csv",
tenantId: "tenant_2",
uploadId: "import_job_2",
contentType: undefined,
metadata: {
requestId: "req_import_2",
importJobId: "import_job_2",
importType: "members",
format: "csv",
},
});
expect(importService.runImport).toHaveBeenCalledWith(descriptor.payload);
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_2", tenantId: "tenant_2", attemptsMade: 0 });
expect(markImportJobSucceeded).toHaveBeenCalledWith({
id: "import_job_2",
tenantId: "tenant_2",
rowCount: 4,
acceptedRows: 2,
rejectedRows: 2,
attemptsMade: 0,
});
});
test("uses BullMQ attempt metadata for import job persistence when available", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({ rowCount: 10, acceptedRows: 9, rejectedRows: 1, duplicateCount: 0 })),
};
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_2/imports/import_job_bullmq_attempt.csv", scannedAt: new Date() })
),
};
const markImportJobRunning = vi.fn(async () => undefined);
const markImportJobSucceeded = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobSucceeded });
const descriptor = buildImportJob({
importJobId: "import_job_bullmq_attempt",
tenantId: "tenant_2",
requestedByUserId: "user_2",
importType: "members",
format: "csv",
requestId: "req_import_bullmq_attempt",
sourceBucket: "imports",
sourceKey: "tenants/tenant_2/imports/import_job_bullmq_attempt.csv",
metadata: { attemptsMade: 1 },
});
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 3 })).resolves.toMatchObject({
action: "import.completed",
details: { attemptsMade: 4 },
});
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_bullmq_attempt", tenantId: "tenant_2", attemptsMade: 4 });
expect(markImportJobSucceeded).toHaveBeenCalledWith(expect.objectContaining({ id: "import_job_bullmq_attempt", tenantId: "tenant_2", attemptsMade: 4 }));
});
test("persists failed import diagnostics and throws redacted handler errors", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => {
throw new Error("import failed token=secret-token body=row-secret password=hunter2");
}),
};
const markImportJobFailed = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_3",
tenantId: "tenant_3",
requestedByUserId: "user_3",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_3/members.csv",
requestId: "req_import_3",
metadata: { attemptsMade: 1 },
});
const scanService: TestScanService = {
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenant_3/members.csv", scannedAt: new Date(), engine: "clamav" })),
};
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "import.error",
message: "Import job failed.",
details: {
importJobId: "import_job_3",
tenantId: "tenant_3",
requestId: "req_import_3",
status: "failed",
attemptsMade: 1,
message: "import failed token=[redacted] body=[redacted] password=[redacted]",
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("secret-token");
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_3",
tenantId: "tenant_3",
attemptsMade: 1,
errorName: "Error",
errorMessage: "import failed token=[redacted] body=[redacted] password=[redacted]",
});
});
test("keeps import jobs running until terminal retry failure", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => {
throw new Error("temporary import outage");
}),
};
const scanService: TestScanService = {
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_3/members.csv", scannedAt: new Date() })),
};
const markImportJobFailed = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_retrying",
tenantId: "tenant_3",
requestedByUserId: "user_3",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_3/members.csv",
});
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed });
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({
action: "import.error",
message: "Import job failed.",
});
expect(markImportJobFailed).not.toHaveBeenCalled();
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toMatchObject({
action: "import.error",
message: "Import job failed.",
});
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_retrying",
tenantId: "tenant_3",
attemptsMade: 3,
errorName: "Error",
errorMessage: "temporary import outage",
});
});
test("rejects import jobs with configured import service when scan service is missing", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
};
const markImportJobFailed = vi.fn(async () => undefined);
const markImportJobSucceeded = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_missing_scan",
tenantId: "tenant_scan",
requestedByUserId: "user_scan",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_scan/members.csv",
requestId: "req_missing_scan",
metadata: { attemptsMade: 3 },
});
const registry = buildHandlerRegistry({
importService,
markImportJobRunning: vi.fn(async () => undefined),
markImportJobFailed,
markImportJobSucceeded,
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "import.error",
message: "Import source scan failed.",
details: {
importJobId: "import_job_missing_scan",
tenantId: "tenant_scan",
requestId: "req_missing_scan",
status: "failed",
attemptsMade: 3,
scan: {
verdict: "error",
code: "scan_service_missing",
message: "Import scan service is not configured.",
},
},
});
expect(importService.runImport).not.toHaveBeenCalled();
expect(markImportJobSucceeded).not.toHaveBeenCalled();
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_missing_scan",
tenantId: "tenant_scan",
attemptsMade: 3,
errorName: "ImportHandlerError",
errorMessage:
"Import source scan failed. verdict=error code=scan_service_missing requestId=req_missing_scan message=Import scan service is not configured.",
});
});
test("rejects infected import source scans without running the import", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
};
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({
verdict: "infected",
bucket: "imports",
key: "tenant_bad/members.csv",
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
engine: "clamav",
signature: "Eicar-Test-Signature",
quarantineKey: "quarantine/tenant_bad/members.csv",
})
),
};
const markImportJobFailed = vi.fn(async () => undefined);
const markImportJobSucceeded = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_infected",
tenantId: "tenant_bad",
requestedByUserId: "user_bad",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_bad/members.csv",
requestId: "req_infected_scan",
});
const registry = buildHandlerRegistry({
importService,
scanService,
markImportJobRunning: vi.fn(async () => undefined),
markImportJobFailed,
markImportJobSucceeded,
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "import.error",
message: "Import source scan failed.",
details: {
importJobId: "import_job_infected",
tenantId: "tenant_bad",
requestId: "req_infected_scan",
status: "failed",
attemptsMade: 0,
scan: {
verdict: "infected",
code: "source_infected",
engine: "clamav",
signature: "Eicar-Test-Signature",
quarantineKey: "quarantine/tenant_bad/members.csv",
message: "Import source object failed malware scan.",
},
},
});
expect(importService.runImport).not.toHaveBeenCalled();
expect(markImportJobSucceeded).not.toHaveBeenCalled();
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_infected",
tenantId: "tenant_bad",
attemptsMade: 0,
errorName: "ImportHandlerError",
errorMessage:
"Import source scan failed. verdict=infected code=source_infected requestId=req_infected_scan message=Import source object failed malware scan.",
});
});
test("rejects scanner errors for import sources with redacted diagnostics", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
};
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({
verdict: "error",
bucket: "imports",
key: "tenant_error/members.csv",
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
error: {
code: "scanner_failed",
message: "scanner failed token=top-secret body=file-bytes authorization=Bearer-secret",
upstream: {
name: "ScannerError",
message: "upstream password=hunter2 body=row-secret",
status: 503,
requestId: "scan_req_error",
},
},
})
),
};
const markImportJobFailed = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_scan_error",
tenantId: "tenant_error",
requestedByUserId: "user_error",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_error/members.csv",
requestId: "req_scan_error",
metadata: { attemptsMade: 5 },
});
const registry = buildHandlerRegistry({
importService,
scanService,
markImportJobRunning: vi.fn(async () => undefined),
markImportJobFailed,
markImportJobSucceeded: vi.fn(async () => undefined),
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "import.error",
details: {
importJobId: "import_job_scan_error",
tenantId: "tenant_error",
requestId: "req_scan_error",
status: "failed",
attemptsMade: 5,
scan: {
verdict: "error",
code: "scanner_failed",
message: "scanner failed token=[redacted] body=[redacted] authorization=[redacted]",
upstream: {
name: "ScannerError",
message: "upstream password=[redacted] body=[redacted]",
status: 503,
requestId: "scan_req_error",
},
},
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("top-secret");
expect(importService.runImport).not.toHaveBeenCalled();
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_scan_error",
tenantId: "tenant_error",
attemptsMade: 5,
errorName: "ImportHandlerError",
errorMessage:
"Import source scan failed. verdict=error code=scanner_failed requestId=req_scan_error message=scanner failed token=[redacted] body=[redacted] authorization=[redacted]",
});
expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("top-secret");
expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("Bearer-secret");
expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("hunter2");
});
test("rejects scanner promise rejections for import sources with redacted diagnostics", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
};
const scanService: TestScanService = {
scanObject: vi.fn(async () => {
throw new Error("scan rejected token=reject-secret body=file-bytes password=hunter2");
}),
};
const markImportJobFailed = vi.fn(async () => undefined);
const descriptor = buildImportJob({
importJobId: "import_job_scan_rejected",
tenantId: "tenant_rejected",
requestedByUserId: "user_rejected",
importType: "members",
format: "csv",
sourceBucket: "imports",
sourceKey: "tenants/tenant_rejected/members.csv",
requestId: "req_scan_rejected",
});
const registry = buildHandlerRegistry({
importService,
scanService,
markImportJobRunning: vi.fn(async () => undefined),
markImportJobFailed,
markImportJobSucceeded: vi.fn(async () => undefined),
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "import.error",
details: {
importJobId: "import_job_scan_rejected",
tenantId: "tenant_rejected",
requestId: "req_scan_rejected",
status: "failed",
attemptsMade: 0,
scan: {
verdict: "error",
code: "scanner_rejected",
message: "scan rejected token=[redacted] body=[redacted] password=[redacted]",
},
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("reject-secret");
expect(importService.runImport).not.toHaveBeenCalled();
expect(markImportJobFailed).toHaveBeenCalledWith({
id: "import_job_scan_rejected",
tenantId: "tenant_rejected",
attemptsMade: 0,
errorName: "ImportHandlerError",
errorMessage:
"Import source scan failed. verdict=error code=scanner_rejected requestId=req_scan_rejected message=scan rejected token=[redacted] body=[redacted] password=[redacted]",
});
});
test("rejects malformed import payload kinds without leaking source secrets", async () => {
const importService: TestImportService = {
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
};
const markImportJobRunning = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({ importService, markImportJobRunning });
await expect(
registry[queueNames.imports]({ kind: "import.other", importJobId: "bad", tenantId: "tenant_1", sourceKey: "secret/body.csv" } as never, {})
).rejects.toMatchObject({
action: "import.error",
message: "Unsupported import payload kind.",
details: { code: "invalid_payload" },
});
await expect(
registry[queueNames.imports]({ kind: "import.other", importJobId: "bad", tenantId: "tenant_1", sourceKey: "secret/body.csv" } as never, {})
).rejects.not.toThrow("secret/body.csv");
expect(markImportJobRunning).not.toHaveBeenCalled();
expect(importService.runImport).not.toHaveBeenCalled();
});
test("runs clean file scan jobs through the injected scan service", async () => {
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({
verdict: "clean",
bucket: "uploads",
key: "tenant_1/file.pdf",
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
engine: "clamav",
})
),
};
const registry = buildHandlerRegistry({ scanService });
const descriptor = buildFileScanJob({
bucket: "uploads",
key: "tenant_1/file.pdf",
tenantId: "tenant_1",
uploadId: "upload_1",
contentType: "application/pdf",
metadata: { requestId: "req_scan_clean" },
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "scan.clean",
details: {
verdict: "clean",
bucket: "uploads",
key: "tenant_1/file.pdf",
uploadId: "upload_1",
engine: "clamav",
},
});
expect(scanService.scanObject).toHaveBeenCalledWith({
bucket: "uploads",
key: "tenant_1/file.pdf",
tenantId: "tenant_1",
uploadId: "upload_1",
contentType: "application/pdf",
metadata: { requestId: "req_scan_clean" },
});
});
test("returns infected scan action with signature and quarantine key", async () => {
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({
verdict: "infected",
bucket: "uploads",
key: "tenant_1/infected.exe",
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
engine: "clamav",
signature: "Eicar-Test-Signature",
quarantineKey: "quarantine/tenant_1/infected.exe",
})
),
};
const registry = buildHandlerRegistry({ scanService });
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/infected.exe", uploadId: "upload_bad" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "scan.infected",
details: {
verdict: "infected",
bucket: "uploads",
key: "tenant_1/infected.exe",
uploadId: "upload_bad",
engine: "clamav",
signature: "Eicar-Test-Signature",
quarantineKey: "quarantine/tenant_1/infected.exe",
},
});
});
test("writes the verdict back onto File.scanStatus when fileId + writer are present", async () => {
const cleanScan: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({ verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav" })
),
};
const infectedScan: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({
verdict: "infected",
bucket: "uploads",
key: "tenant_1/bad.exe",
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
engine: "clamav",
signature: "Eicar-Test-Signature",
})
),
};
const fileScanStatusWriter = { recordVerdict: vi.fn(async () => true) };
const filePreviewEnqueuer = { enqueueAfterCleanScan: vi.fn(async () => undefined) };
const cleanRegistry = buildHandlerRegistry({ scanService: cleanScan, fileScanStatusWriter, filePreviewEnqueuer });
const cleanJob = buildFileScanJob({
bucket: "uploads",
key: "tenant_1/file.pdf",
tenantId: "tenant_1",
fileId: "file_1",
contentType: "application/pdf",
});
await cleanRegistry[cleanJob.queueName](cleanJob.payload, {});
expect(fileScanStatusWriter.recordVerdict).toHaveBeenCalledWith({
fileId: "file_1",
storageKey: "tenant_1/file.pdf",
tenantId: "tenant_1",
verdict: "clean",
});
expect(filePreviewEnqueuer.enqueueAfterCleanScan).toHaveBeenCalledWith({
bucket: "uploads",
contentType: "application/pdf",
fileId: "file_1",
storageKey: "tenant_1/file.pdf",
tenantId: "tenant_1",
});
const infectedRegistry = buildHandlerRegistry({ scanService: infectedScan, fileScanStatusWriter, filePreviewEnqueuer });
const infectedJob = buildFileScanJob({ bucket: "uploads", key: "tenant_1/bad.exe", tenantId: "tenant_1", fileId: "file_2" });
await infectedRegistry[infectedJob.queueName](infectedJob.payload, {});
expect(fileScanStatusWriter.recordVerdict).toHaveBeenCalledWith({
fileId: "file_2",
storageKey: "tenant_1/bad.exe",
tenantId: "tenant_1",
verdict: "infected",
});
expect(filePreviewEnqueuer.enqueueAfterCleanScan).toHaveBeenCalledTimes(1);
});
test("skips the verdict write-back when the job carries no fileId", async () => {
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({ verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav" })
),
};
const fileScanStatusWriter = { recordVerdict: vi.fn(async () => true) };
const registry = buildHandlerRegistry({ scanService, fileScanStatusWriter });
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/file.pdf", tenantId: "tenant_1" });
await registry[descriptor.queueName](descriptor.payload, {});
expect(fileScanStatusWriter.recordVerdict).not.toHaveBeenCalled();
});
test("rejects storage error scan verdicts with redacted diagnostics", async () => {
const scanService: TestScanService = {
scanObject: vi.fn(async () =>
scanResult({
verdict: "error",
bucket: "uploads",
key: "tenant_1/error.bin",
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
error: {
code: "scanner_failed",
message: "Storage scanner failed while scanning object.",
upstream: {
name: "ScannerError",
message: "scanner failed token=top-secret body=file-bytes credential=aws-secret",
status: 503,
requestId: "scan_req_1",
},
},
})
),
};
const registry = buildHandlerRegistry({ scanService });
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/error.bin", uploadId: "upload_error" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "scan.error",
details: {
verdict: "error",
bucket: "uploads",
key: "tenant_1/error.bin",
uploadId: "upload_error",
code: "scanner_failed",
message: "Storage scanner failed while scanning object.",
upstream: {
name: "ScannerError",
message: "scanner failed token=[redacted] body=[redacted] credential=[redacted]",
status: 503,
requestId: "scan_req_1",
},
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("top-secret");
});
test("rejects scanner service rejections without leaking secret-bearing messages", async () => {
const scanService: TestScanService = {
scanObject: vi.fn(async () => {
throw new Error("scanner rejected authorization=Bearer-secret body=file-bytes password=hunter2");
}),
};
const registry = buildHandlerRegistry({ scanService });
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/rejected.bin", uploadId: "upload_rejected" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "scan.error",
details: {
verdict: "error",
bucket: "uploads",
key: "tenant_1/rejected.bin",
uploadId: "upload_rejected",
code: "scanner_rejected",
message: "scanner rejected authorization=[redacted] body=[redacted] password=[redacted]",
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("Bearer-secret");
});
test("rejects file scan jobs when scan service is missing", async () => {
const registry = buildHandlerRegistry();
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/missing-service.bin", uploadId: "upload_missing" });
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
action: "scan.error",
message: "File scan service is not configured.",
details: {
verdict: "error",
bucket: "uploads",
key: "tenant_1/missing-service.bin",
uploadId: "upload_missing",
code: "scan_service_missing",
},
});
});
test("rejects malformed file scan payload kinds", async () => {
const scanService: TestScanService = {
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "unused", scannedAt: new Date() })),
};
const registry = buildHandlerRegistry({ scanService });
await expect(registry[queueNames.fileScan]({ kind: "scan.other", bucket: "uploads", key: "bad" } as never, {})).rejects.toMatchObject({
action: "scan.error",
message: "Unsupported file scan payload kind.",
details: { verdict: "error", code: "invalid_payload" },
});
expect(scanService.scanObject).not.toHaveBeenCalled();
});
test("runs KWP project sync on the dedicated KWP queue under a binding-scoped lease", async () => {
const store = createSharedStore({ namespace: "kwp-project-handler-lease-test", adapter: new MemoryAdapter() });
const syncKwpProjectBinding = vi.fn(async () => ({
runId: "run_1",
state: "synced" as const,
projectId: "project_1",
stageChanged: true,
}));
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectBinding });
const descriptor = buildKwpProjectSyncJob({
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "kwp.project-sync.completed",
details: {
kind: "kwp.project-sync",
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00",
runId: "run_1",
state: "synced",
projectId: "project_1",
stageChanged: true,
},
});
expect(syncKwpProjectBinding).toHaveBeenCalledOnce();
expect(await store.get("job-lease:kwp-project:tenant_1:binding_1")).toBeNull();
});
test("replays a pre-rollout KWP project job left on the generic import queue", async () => {
const store = createSharedStore({ namespace: "kwp-project-handler-legacy-queue-test", adapter: new MemoryAdapter() });
const syncKwpProjectBinding = vi.fn(async () => ({
runId: "run_legacy",
state: "unchanged" as const,
projectId: "project_1",
stageChanged: false,
}));
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectBinding });
const descriptor = buildKwpProjectSyncJob({
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "legacy",
});
await expect(registry[queueNames.imports](descriptor.payload as never, {})).resolves.toMatchObject({
action: "kwp.project-sync.completed",
details: { runId: "run_legacy" },
});
expect(syncKwpProjectBinding).toHaveBeenCalledOnce();
});
test("acknowledges overlapping KWP project replays without a second provider call", async () => {
const store = createSharedStore({ namespace: "kwp-project-handler-overlap-test", adapter: new MemoryAdapter() });
await store.set("job-lease:kwp-project:tenant_1:binding_1", "other-replica", { ttlSeconds: 120 });
const syncKwpProjectBinding = vi.fn();
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpProjectBinding });
const descriptor = buildKwpProjectSyncJob({
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
handled: true,
action: "kwp.import.overlap-skipped",
details: { kind: "kwp.project-sync", tenantId: "tenant_1", reason: "active-lease" },
});
expect(syncKwpProjectBinding).not.toHaveBeenCalled();
});
test("runs a bounded KWP project-catalog batch and schedules its continuation", async () => {
const store = createSharedStore({ namespace: "kwp-catalog-handler-test", adapter: new MemoryAdapter() });
const syncKwpProjectCatalog = vi.fn(async () => ({
discoveredCount: 50_000,
hydratedCount: 20,
processed: 20,
hasMore: true,
status: "syncing" as const,
}));
const enqueueKwpProjectCatalogContinuation = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
syncKwpProjectCatalog,
enqueueKwpProjectCatalogContinuation,
});
const descriptor = buildKwpProjectCatalogSyncJob({
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
handled: true,
action: "kwp.project-catalog-sync.completed",
details: {
kind: "kwp.project-catalog-sync",
discoveredCount: 50_000,
hydratedCount: 20,
processed: 20,
hasMore: true,
},
});
expect(syncKwpProjectCatalog).toHaveBeenCalledOnce();
expect(enqueueKwpProjectCatalogContinuation).toHaveBeenCalledWith(descriptor.payload);
expect(await store.get("job-lease:kwp-catalog:tenant_1:connection_1")).toBeNull();
});
test("runs core project inbound reconciliation and schedules archive discovery independently", async () => {
const store = createSharedStore({ namespace: "kwp-project-inbound-handler-test", adapter: new MemoryAdapter() });
const syncKwpProjectBinding = vi.fn(async () => ({
runId: "project-run",
state: "unchanged" as const,
projectId: "project_1",
stageChanged: false,
}));
const syncKwpProjectInbound = vi.fn(async () => ({
runId: "inbound-run",
projectId: "project_1",
appointments: 2,
tasks: 3,
materialPositions: 4,
failedAreas: [],
changed: true,
}));
const enqueueKwpArchiveDiscovery = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
syncKwpProjectBinding,
syncKwpProjectInbound,
enqueueKwpArchiveDiscovery,
});
const descriptor = buildKwpProjectSyncJob({
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "kwp.project-sync.completed",
details: {
inboundRunId: "inbound-run",
inboundChanged: true,
appointments: 2,
tasks: 3,
materialPositions: 4,
failedAreas: [],
archiveDiscoveryScheduled: true,
},
});
expect(syncKwpProjectBinding).toHaveBeenCalledBefore(syncKwpProjectInbound);
expect(enqueueKwpArchiveDiscovery).toHaveBeenCalledWith(descriptor.payload);
});
test("continues bounded archive discovery and fans files out without rerunning core inbound", async () => {
const store = createSharedStore({ namespace: "kwp-archive-discovery-handler-test", adapter: new MemoryAdapter() });
const archiveFile = {
externalId: "1:2:3",
headerId: 1,
revisionId: 3,
revision: 1,
folderId: 0,
displayName: "plan.pdf",
documentType: "PLAN",
providerFileType: "PDF",
providerSizeBytes: 123,
lastModifiedAt: null,
};
const cursor = { pendingFolderIds: [6], visitedFolderIds: [0, 1, 2, 3, 4, 5] };
const syncKwpArchiveDiscovery = vi.fn(async () => ({
archiveFiles: [archiveFile],
processedFolders: 5,
cursor,
}));
const enqueueKwpArchiveFileJobs = vi.fn(async () => 1);
const enqueueKwpArchiveDiscovery = vi.fn(async () => undefined);
const registry = buildHandlerRegistry({
tenantMaintenanceLeaseStore: store,
syncKwpArchiveDiscovery,
enqueueKwpArchiveFileJobs,
enqueueKwpArchiveDiscovery,
});
const descriptor = buildKwpArchiveDiscoveryJob({
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00:00.000Z",
batchNumber: 0,
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "kwp.archive-discovery.completed",
details: { processedFolders: 5, archiveJobs: 1, hasMore: true },
});
expect(enqueueKwpArchiveFileJobs).toHaveBeenCalledWith(descriptor.payload, [archiveFile]);
expect(enqueueKwpArchiveDiscovery).toHaveBeenCalledWith(descriptor.payload, cursor);
});
test("stops retrying deterministic KWP failures but keeps provider outages retryable", async () => {
const deterministic = Object.assign(new Error("schema changed"), { code: "contract_drift" });
const providerOutage = Object.assign(new Error("upstream unavailable"), { code: "provider_outage" });
const descriptor = buildKwpArchiveDiscoveryJob({
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00:00.000Z",
batchNumber: 0,
});
const deterministicRegistry = buildHandlerRegistry({ syncKwpArchiveDiscovery: vi.fn(async () => { throw deterministic; }) });
await expect(deterministicRegistry[descriptor.queueName](descriptor.payload, {})).rejects.toBeInstanceOf(UnrecoverableError);
const retryableRegistry = buildHandlerRegistry({ syncKwpArchiveDiscovery: vi.fn(async () => { throw providerOutage; }) });
await expect(retryableRegistry[descriptor.queueName](descriptor.payload, {})).rejects.toBe(providerOutage);
});
test("runs one KWP archive file under an external-file lease", async () => {
const store = createSharedStore({ namespace: "kwp-archive-handler-test", adapter: new MemoryAdapter() });
const syncKwpArchiveFile = vi.fn(async () => ({
runId: "archive-run",
fileId: "file_1",
projectId: "project_1",
storageKey: "tenants/tenant_1/kwp-archive/file.pdf",
state: "created" as const,
}));
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, syncKwpArchiveFile });
const descriptor = buildKwpArchiveFileSyncJob({
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T12:00",
archiveFile: {
externalId: "1:2:3",
headerId: 1,
revisionId: 3,
revision: 1,
folderId: 0,
displayName: "plan.pdf",
documentType: null,
providerFileType: "PDF",
providerSizeBytes: 123,
lastModifiedAt: null,
},
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
action: "kwp.archive-file-sync.completed",
details: {
kind: "kwp.archive-file-sync",
fileId: "file_1",
state: "created",
},
});
expect(syncKwpArchiveFile).toHaveBeenCalledOnce();
expect(await store.get("job-lease:kwp-archive:tenant_1:connection_1:1:2:3")).toBeNull();
});
test("runs the KWP contract sentinel independently of project capability state", async () => {
const store = createSharedStore({ namespace: "kwp-contract-handler-lease-test", adapter: new MemoryAdapter() });
const runKwpContractSentinel = vi.fn(async () => ({
status: "unchanged" as const,
fingerprint: "f".repeat(64),
}));
const registry = buildHandlerRegistry({ tenantMaintenanceLeaseStore: store, runKwpContractSentinel });
const descriptor = buildKwpContractSentinelJob({
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23",
});
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
handled: true,
action: "kwp.contract-sentinel.completed",
details: {
kind: "kwp.contract-sentinel",
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23",
status: "unchanged",
},
});
expect(runKwpContractSentinel).toHaveBeenCalledOnce();
});
// Phase 06 #21 / F-015 — plugin-invoke dispatch.
describe("plugin-invoke handler (F-015)", () => {
const samplePayload = {
kind: "plugin.invoke" as const,
tenantId: "tenant_abc",
pluginId: "com.acme.demo",
version: "1.0.0",
hookName: "tenant.created",
input: { tenantId: "tenant_abc", slug: "acme", ownerEmail: "owner@example.com", createdAt: "2026-05-28T10:00:00.000Z" },
metadata: { logicalHookName: "tenant.created" },
};
test("rejects malformed queue payloads before invoking plugin deps", async () => {
const errorLogs: Array<{ fields: Record<string, unknown>; msg: string }> = [];
const pluginInvokeDeps = {
loadPlugin: vi.fn(),
loadCapabilityGrants: vi.fn(),
loadBinary: vi.fn(),
verifySignature: vi.fn(),
consumeRateLimit: vi.fn(),
invokeInSandbox: vi.fn(),
recordAudit: vi.fn(),
trustedPublicKeysHex: [],
now: () => 1_000_000,
};
const registry = buildHandlerRegistry({
pluginInvokeDeps: pluginInvokeDeps as never,
logger: {
info: () => undefined,
error: (fields, msg) => errorLogs.push({ fields, msg: msg ?? "" }),
},
});
await expect(
registry[queueNames.pluginInvoke](
{
kind: "plugin.invoke",
tenantId: "tenant_abc",
pluginId: "com.acme.demo",
version: 1,
input: { secret: "not logged" },
} as never,
{} as never
)
).rejects.toMatchObject({
action: "plugin.invoke.invalid_payload",
message: "Invalid plugin-invoke queue payload.",
details: {
queueName: queueNames.pluginInvoke,
code: "invalid_payload",
invalidFields: ["version", "hookName"],
},
});
expect(pluginInvokeDeps.loadPlugin).not.toHaveBeenCalled();
expect(pluginInvokeDeps.invokeInSandbox).not.toHaveBeenCalled();
expect(errorLogs).toHaveLength(1);
expect(errorLogs[0]).toMatchObject({
msg: "Invalid plugin-invoke queue payload.",
fields: {
queueName: queueNames.pluginInvoke,
code: "invalid_payload",
invalidFields: ["version", "hookName"],
payload: {
kind: "plugin.invoke",
tenantId: "tenant_abc",
pluginId: "com.acme.demo",
version: "number",
hookName: "undefined",
},
},
});
expect(JSON.stringify(errorLogs[0])).not.toContain("not logged");
});
test("throws when pluginInvokeDeps is not configured (fail-loud, no silent ack)", async () => {
const errorLogs: Array<{ fields: Record<string, unknown>; msg: string }> = [];
const registry = buildHandlerRegistry({
logger: {
info: () => undefined,
error: (fields, msg) => errorLogs.push({ fields, msg: msg ?? "" }),
},
});
await expect(registry[queueNames.pluginInvoke](samplePayload, {} as never)).rejects.toThrow(/pluginInvokeDeps is not configured/);
expect(errorLogs).toHaveLength(1);
expect(errorLogs[0]!.fields).toMatchObject({
queue: queueNames.pluginInvoke,
pluginId: samplePayload.pluginId,
hookName: samplePayload.hookName,
});
});
test("dispatches to handlePluginInvokeJob and maps success outcome", async () => {
const pluginInvokeDeps = {
loadPlugin: vi.fn(async () => ({
tenantId: samplePayload.tenantId,
pluginId: samplePayload.pluginId,
version: samplePayload.version,
manifest: { capabilities: [], hooks: [{ name: "tenant.created", export: "tenant.created" }], wasmSha256: "x", signedBy: "y" } as never,
wasmSha256: "x",
signaturePublicKey: "y",
enabled: true,
})),
loadCapabilityGrants: vi.fn(async () => [{ capabilityName: "hook:tenant.created", scopeJson: null }]),
loadBinary: vi.fn(async () => new Uint8Array([0])),
verifySignature: vi.fn(() => ({ ok: true as const })),
consumeRateLimit: vi.fn(async () => ({ allowed: true, remaining: 99 })),
invokeInSandbox: vi.fn(async () => ({ ok: true as const, output: "ok", fuelUsed: 42 })),
recordAudit: vi.fn(async () => undefined),
trustedPublicKeysHex: [],
now: () => 1_000_000,
};
const registry = buildHandlerRegistry({ pluginInvokeDeps: pluginInvokeDeps as never });
const result = await registry[queueNames.pluginInvoke](samplePayload, {} as never);
expect(result).toMatchObject({
handled: true,
action: "plugin.invoke.completed",
details: { pluginId: samplePayload.pluginId, hookName: samplePayload.hookName, fuelUsed: 42 },
});
expect(pluginInvokeDeps.invokeInSandbox).toHaveBeenCalledTimes(1);
expect(pluginInvokeDeps.invokeInSandbox).toHaveBeenCalledWith(
expect.objectContaining({ capabilityGrants: [{ capabilityName: "hook:tenant.created", scopeJson: null }] })
);
expect(pluginInvokeDeps.recordAudit).toHaveBeenCalledTimes(1);
});
test("maps signature_invalid outcome to plugin.invoke.failed", async () => {
const pluginInvokeDeps = {
loadPlugin: vi.fn(async () => ({
tenantId: samplePayload.tenantId,
pluginId: samplePayload.pluginId,
version: samplePayload.version,
manifest: { capabilities: [], hooks: [{ name: "tenant.created", export: "tenant.created" }], wasmSha256: "x", signedBy: "y" } as never,
wasmSha256: "x",
signaturePublicKey: "y",
enabled: true,
})),
loadCapabilityGrants: vi.fn(async () => [{ capabilityName: "hook:tenant.created", scopeJson: null }]),
loadBinary: vi.fn(async () => new Uint8Array([0])),
verifySignature: vi.fn(() => ({ ok: false as const, reason: "binary_hash_mismatch" as const })),
consumeRateLimit: vi.fn(async () => ({ allowed: true, remaining: 99 })),
invokeInSandbox: vi.fn(),
recordAudit: vi.fn(async () => undefined),
trustedPublicKeysHex: [],
now: () => 1_000_000,
};
const registry = buildHandlerRegistry({ pluginInvokeDeps: pluginInvokeDeps as never });
const result = await registry[queueNames.pluginInvoke](samplePayload, {} as never);
expect(result).toMatchObject({
handled: true,
action: "plugin.invoke.failed",
details: { reason: "signature_invalid", message: "binary_hash_mismatch" },
});
expect(pluginInvokeDeps.invokeInSandbox).not.toHaveBeenCalled();
});
});
});