2973 lines
121 KiB
TypeScript
2973 lines
121 KiB
TypeScript
import { describe, expect, test, vi } from "vitest";
|
|
import {
|
|
buildAiBrowserTaskJob,
|
|
buildAiChatCompletionJob,
|
|
buildAiEmbeddingJob,
|
|
buildBrowserSessionCleanupJob,
|
|
buildBrowserSessionRunJob,
|
|
buildBrowserUsageFlushJob,
|
|
buildCleanupJob,
|
|
buildExportJob,
|
|
buildFileScanJob,
|
|
buildImportJob,
|
|
buildNotificationJob,
|
|
buildTransactionalMailJob,
|
|
queueNames,
|
|
type WebhookDeliveryJobDescriptor,
|
|
} from "@toolctrl/queue";
|
|
import { buildHandlerRegistry, type HandlerDependencies } from "./handlers.js";
|
|
import type { CreateNotificationDeliveryInput } from "@toolctrl/db/notifications.server";
|
|
import { MailProviderError, type MailProvider } from "@toolctrl/mail";
|
|
import type { StorageScanResult } from "@toolctrl/storage";
|
|
|
|
type TestScanService = NonNullable<HandlerDependencies["scanService"]>;
|
|
type TestExportService = NonNullable<HandlerDependencies["exportService"]>;
|
|
type TestImportService = NonNullable<HandlerDependencies["importService"]>;
|
|
type TestStorageDeleteService = NonNullable<HandlerDependencies["storageDeleteService"]>;
|
|
type TestRetentionListInput = Parameters<NonNullable<HandlerDependencies["listSoftDeletedExportJobsForHardDelete"]>>[0];
|
|
type TestRetentionJob = Awaited<ReturnType<NonNullable<HandlerDependencies["listSoftDeletedExportJobsForHardDelete"]>>>[number];
|
|
const publicWebhookAddressResolver = vi.fn(async () => ["93.184.216.34"]);
|
|
const scanResult = <Result extends StorageScanResult>(result: Result): Result => result;
|
|
const retentionJob = (overrides: Partial<TestRetentionJob> & Pick<TestRetentionJob, "id" | "artifactBucket" | "artifactKey">): TestRetentionJob => ({
|
|
tenantId: "tenant_1",
|
|
status: "FAILED",
|
|
rowCount: 1,
|
|
expiresAt: new Date("2026-05-08T00:00:00.000Z"),
|
|
deletedAt: new Date("2026-05-09T00:00:00.000Z"),
|
|
requestId: `req_${overrides.id}`,
|
|
...overrides,
|
|
});
|
|
|
|
describe("worker handlers", () => {
|
|
test("sends queue-provider transactional email jobs through configured mail provider", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_1" })),
|
|
};
|
|
const registry = buildHandlerRegistry({ mailProvider });
|
|
const descriptor = buildTransactionalMailJob({
|
|
from: { email: "sender@example.com", name: "Sender" },
|
|
to: ["user@example.com", { email: "admin@example.com", name: "Admin" }],
|
|
cc: [{ email: "cc@example.com", name: "CC" }],
|
|
bcc: ["bcc@example.com"],
|
|
replyTo: "reply@example.com",
|
|
subject: "Welcome to ToolCTRL",
|
|
text: "Hello from ToolCTRL",
|
|
html: "<p>Hello from <strong>ToolCTRL</strong></p>",
|
|
templateId: "tmpl_welcome",
|
|
attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }],
|
|
tags: ["welcome"],
|
|
precedenceBulk: true,
|
|
sendAt: 1_778_580_000,
|
|
inReplyTo: "message_1",
|
|
references: ["message_0"],
|
|
settings: { trackClicks: true, trackOpens: false, trackContent: true },
|
|
headers: [{ name: "X-Test", value: "1" }],
|
|
listUnsubscribe: "<mailto:unsubscribe@example.com>",
|
|
templateData: { firstName: "Ada", tenantName: "Example Co" },
|
|
tenantId: "tenant_1",
|
|
userId: "user_1",
|
|
metadata: { source: "queue-provider" },
|
|
});
|
|
|
|
expect(descriptor).toMatchObject({
|
|
queueName: queueNames.email,
|
|
jobName: "send-transactional-email",
|
|
payload: { kind: "mail.transactional" },
|
|
});
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { provider: "mailjet", messageId: "msg_1" },
|
|
});
|
|
expect(mailProvider.send).toHaveBeenCalledTimes(1);
|
|
expect(mailProvider.send).toHaveBeenCalledWith({
|
|
from: { email: "sender@example.com", name: "Sender" },
|
|
to: ["user@example.com", { email: "admin@example.com", name: "Admin" }],
|
|
cc: [{ email: "cc@example.com", name: "CC" }],
|
|
bcc: ["bcc@example.com"],
|
|
replyTo: "reply@example.com",
|
|
subject: "Welcome to ToolCTRL",
|
|
text: "Hello from ToolCTRL",
|
|
html: "<p>Hello from <strong>ToolCTRL</strong></p>",
|
|
templateId: "tmpl_welcome",
|
|
attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }],
|
|
tags: ["welcome"],
|
|
precedenceBulk: true,
|
|
sendAt: 1_778_580_000,
|
|
inReplyTo: "message_1",
|
|
references: ["message_0"],
|
|
settings: { trackClicks: true, trackOpens: false, trackContent: true },
|
|
headers: [{ name: "X-Test", value: "1" }],
|
|
listUnsubscribe: "<mailto:unsubscribe@example.com>",
|
|
personalization: [
|
|
{ email: "user@example.com", data: { firstName: "Ada", tenantName: "Example Co" } },
|
|
{ email: "admin@example.com", data: { firstName: "Ada", tenantName: "Example Co" } },
|
|
],
|
|
});
|
|
});
|
|
|
|
test("preserves per-recipient transactional email template data", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_personalized" })),
|
|
};
|
|
const registry = buildHandlerRegistry({ mailProvider });
|
|
const descriptor = buildTransactionalMailJob({
|
|
to: ["user@example.com", "admin@example.com"],
|
|
subject: "Welcome to ToolCTRL",
|
|
templateId: "tmpl_welcome",
|
|
templateData: {
|
|
"user@example.com": { firstName: "Ada" },
|
|
"admin@example.com": { firstName: "Grace" },
|
|
},
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "mail.sent",
|
|
});
|
|
expect(mailProvider.send).toHaveBeenCalledWith({
|
|
to: ["user@example.com", "admin@example.com"],
|
|
subject: "Welcome to ToolCTRL",
|
|
text: undefined,
|
|
html: undefined,
|
|
templateId: "tmpl_welcome",
|
|
personalization: [
|
|
{ email: "user@example.com", data: { firstName: "Ada" } },
|
|
{ email: "admin@example.com", data: { firstName: "Grace" } },
|
|
],
|
|
});
|
|
});
|
|
|
|
test("creates Mailjet provider from explicit worker env", async () => {
|
|
const fetchMock = vi.fn(
|
|
async () =>
|
|
new Response(JSON.stringify({ Messages: [{ Status: "success", To: [{ Email: "user@example.com", MessageUUID: "msg_env" }] }] }), {
|
|
status: 200,
|
|
headers: { "content-type": "application/json" },
|
|
})
|
|
);
|
|
const registry = buildHandlerRegistry({
|
|
env: {
|
|
MAILJET_API_KEY: "fixture-api-key",
|
|
MAILJET_SECRET_KEY: "fixture-secret-key",
|
|
MAILJET_FROM_EMAIL: "sender@example.com",
|
|
MAILJET_FROM_NAME: "Fixture Sender",
|
|
},
|
|
});
|
|
const originalFetch = globalThis.fetch;
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
|
|
try {
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "mail.transactional",
|
|
to: ["user@example.com"],
|
|
subject: "Hello",
|
|
text: "Body",
|
|
},
|
|
{}
|
|
)
|
|
).resolves.toEqual({
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { provider: "mailjet", messageId: "msg_env" },
|
|
});
|
|
} finally {
|
|
vi.stubGlobal("fetch", originalFetch);
|
|
}
|
|
|
|
expect(fetchMock).toHaveBeenCalledWith(
|
|
"https://api.mailjet.com/v3.1/send",
|
|
expect.objectContaining({
|
|
headers: expect.objectContaining({ Authorization: `Basic ${Buffer.from("fixture-api-key:fixture-secret-key", "utf8").toString("base64")}` }),
|
|
body: expect.stringContaining("Fixture Sender"),
|
|
})
|
|
);
|
|
});
|
|
|
|
test("reports missing Mailjet fixture env with key names only", async () => {
|
|
const registry = buildHandlerRegistry({ env: { MAILJET_API_KEY: "super-secret-api-key" } });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "mail.transactional",
|
|
to: ["user@example.com"],
|
|
subject: "Hello",
|
|
text: "Body",
|
|
},
|
|
{}
|
|
)
|
|
).rejects.toMatchObject({ message: "MAILJET_SECRET_KEY is required for Mailjet email delivery." });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "mail.transactional",
|
|
to: ["user@example.com"],
|
|
subject: "Hello",
|
|
text: "Body",
|
|
},
|
|
{}
|
|
)
|
|
).rejects.not.toThrow("super-secret-api-key");
|
|
});
|
|
|
|
test("propagates provider rejection diagnostics without exposing payload secrets", async () => {
|
|
const providerError = new MailProviderError("Mailjet rejected message", {
|
|
provider: "mailjet",
|
|
statusCode: 422,
|
|
messageId: "msg_rejected",
|
|
responseBody: { message: "Invalid recipient" },
|
|
});
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => {
|
|
throw providerError;
|
|
}),
|
|
};
|
|
const registry = buildHandlerRegistry({ mailProvider });
|
|
const descriptor = buildTransactionalMailJob({
|
|
to: ["user@example.com"],
|
|
subject: "Hello",
|
|
text: "Body",
|
|
templateData: { otp: "123456", apiKey: "payload-secret" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
message: "Mailjet rejected message",
|
|
provider: "mailjet",
|
|
statusCode: 422,
|
|
messageId: "msg_rejected",
|
|
});
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("payload-secret");
|
|
expect(mailProvider.send).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
test("delivers queued webhook descriptors and updates bounded legacy attempt diagnostics", async () => {
|
|
const fetchMock = vi.fn(
|
|
async () =>
|
|
new Response("accepted Authorization: Bearer response-secret", { status: 202, statusText: "Accepted", headers: { "content-type": "text/plain" } })
|
|
);
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
const descriptor = {
|
|
queueName: queueNames.email,
|
|
jobName: "deliver-webhook",
|
|
payload: {
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
method: "POST",
|
|
eventName: "legacy.event_webhook_attempt",
|
|
body: { id: "row_1" },
|
|
metadata: { legacyAttemptId: "attempt_1", legacyAttemptTenantId: "tenant_1" },
|
|
},
|
|
} satisfies WebhookDeliveryJobDescriptor;
|
|
|
|
await expect(registry[queueNames.email](descriptor.payload, {})).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "webhook.delivered",
|
|
});
|
|
expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ method: "POST", redirect: "manual" }));
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_1", tenantId: "tenant_1" },
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: true,
|
|
status: 202,
|
|
message: "Accepted",
|
|
body: expect.stringContaining('"status":202'),
|
|
})
|
|
);
|
|
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>;
|
|
const body = updateCalls[0]?.[1]?.body ?? "";
|
|
expect(body).toContain('"bodyBytes":');
|
|
expect(body).not.toContain("response-secret");
|
|
expect(body).not.toContain("Authorization");
|
|
});
|
|
|
|
test("rejects unsafe queued webhook URLs as non-retryable before fetching", async () => {
|
|
const fetchMock = vi.fn(async () => new Response(null, { status: 204 }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
const blockedUrls = [
|
|
"ftp://example.com/webhook",
|
|
"https://localhost/webhook",
|
|
"https://service.internal/webhook",
|
|
"https://service/webhook",
|
|
"http://127.0.0.1/webhook",
|
|
"http://10.1.2.3/webhook",
|
|
"http://169.254.169.254/latest/meta-data",
|
|
"http://[::1]/webhook",
|
|
"http://[fd00::1]/webhook",
|
|
];
|
|
|
|
for (const url of blockedUrls) {
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url,
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: `attempt_${blockedUrls.indexOf(url)}`, legacyAttemptTenantId: "tenant_blocked" },
|
|
},
|
|
{}
|
|
)
|
|
).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "webhook.rejected",
|
|
});
|
|
}
|
|
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledTimes(blockedUrls.length);
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
expect.objectContaining({ tenantId: "tenant_blocked" }),
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
message: expect.stringMatching(/Webhook endpoint/),
|
|
})
|
|
);
|
|
});
|
|
|
|
test("rejects public webhook hostnames that resolve to private addresses before fetching", async () => {
|
|
const fetchMock = vi.fn(async () => new Response(null, { status: 204 }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const privateAddressResolver = vi.fn(async () => ["169.254.169.254"]);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: privateAddressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://public.example.test/latest/meta-data",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_dns_blocked", legacyAttemptTenantId: "tenant_dns" },
|
|
},
|
|
{}
|
|
)
|
|
).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "webhook.rejected",
|
|
});
|
|
|
|
expect(privateAddressResolver).toHaveBeenCalledWith("public.example.test");
|
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_dns_blocked", tenantId: "tenant_dns" },
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
message: "Webhook endpoint resolved to a blocked address.",
|
|
})
|
|
);
|
|
});
|
|
|
|
test("follows queued webhook redirects only after validating the target", async () => {
|
|
const fetchMock = vi
|
|
.fn()
|
|
.mockResolvedValueOnce(new Response(null, { status: 307, headers: { location: "https://hooks.example.com/next" } }))
|
|
.mockResolvedValueOnce(new Response("accepted", { status: 202, statusText: "Accepted" }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
method: "POST",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_redirect", legacyAttemptTenantId: "tenant_redirect" },
|
|
},
|
|
{}
|
|
)
|
|
).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "webhook.delivered",
|
|
});
|
|
|
|
expect(fetchMock).toHaveBeenNthCalledWith(1, "https://example.com/webhook", expect.objectContaining({ method: "POST", redirect: "manual" }));
|
|
expect(fetchMock).toHaveBeenNthCalledWith(2, "https://hooks.example.com/next", expect.objectContaining({ method: "POST", redirect: "manual" }));
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_redirect", tenantId: "tenant_redirect" },
|
|
expect.objectContaining({ success: true, status: 202 })
|
|
);
|
|
});
|
|
|
|
test("rejects queued webhook redirect hostnames that resolve to private addresses before following them", async () => {
|
|
const fetchMock = vi.fn(async () => new Response(null, { status: 302, headers: { location: "https://metadata.example.test/latest" } }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const addressResolver = vi.fn(async (hostname: string) => (hostname === "metadata.example.test" ? ["10.0.0.10"] : ["93.184.216.34"]));
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: addressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_redirect_dns_blocked", legacyAttemptTenantId: "tenant_redirect" },
|
|
},
|
|
{},
|
|
{ attemptsMade: 0, attemptsLimit: 3 }
|
|
)
|
|
).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "webhook.rejected",
|
|
});
|
|
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
expect(addressResolver).toHaveBeenCalledWith("example.com");
|
|
expect(addressResolver).toHaveBeenCalledWith("metadata.example.test");
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_redirect_dns_blocked", tenantId: "tenant_redirect" },
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
message: "Webhook endpoint resolved to a blocked address.",
|
|
})
|
|
);
|
|
});
|
|
|
|
test("rejects queued webhook redirects to unsafe targets without fetching them", async () => {
|
|
const fetchMock = vi.fn(async () => new Response(null, { status: 302, headers: { location: "http://127.0.0.1/webhook" } }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_redirect_blocked", legacyAttemptTenantId: "tenant_redirect" },
|
|
},
|
|
{},
|
|
{ attemptsMade: 0, attemptsLimit: 3 }
|
|
)
|
|
).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "webhook.rejected",
|
|
});
|
|
|
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
|
expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ redirect: "manual" }));
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_redirect_blocked", tenantId: "tenant_redirect" },
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
message: "Webhook endpoint host is not allowed.",
|
|
})
|
|
);
|
|
});
|
|
|
|
test("updates failed legacy attempt diagnostics and rejects non-2xx webhook responses", async () => {
|
|
const fetchMock = vi.fn(async () => new Response("upstream rejected token=upstream-secret body=payload", { status: 503, statusText: "Unavailable" }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_2", legacyAttemptTenantId: "tenant_2" },
|
|
},
|
|
{}
|
|
)
|
|
).rejects.toThrow("Webhook delivery failed with 503.");
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledTimes(1);
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_2", tenantId: "tenant_2" },
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
status: 503,
|
|
message: "Webhook delivery failed with 503.",
|
|
body: expect.stringContaining('"status":503'),
|
|
})
|
|
);
|
|
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>;
|
|
const body = updateCalls[0]?.[1]?.body ?? "";
|
|
expect(body).toContain('"bodyBytes":');
|
|
expect(body).not.toContain("upstream-secret");
|
|
expect(body).not.toContain("payload");
|
|
});
|
|
|
|
test("keeps queued webhook attempt diagnostics open until retries are exhausted", async () => {
|
|
const fetchMock = vi.fn(async () => new Response("temporarily unavailable", { status: 503, statusText: "Unavailable" }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
const payload = {
|
|
kind: "webhook.deliver" as const,
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_retry", legacyAttemptTenantId: "tenant_retry" },
|
|
};
|
|
|
|
await expect(registry[queueNames.email](payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toThrow("Webhook delivery failed with 503.");
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_retry", tenantId: "tenant_retry" },
|
|
expect.not.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
})
|
|
);
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenLastCalledWith(
|
|
{ id: "attempt_retry", tenantId: "tenant_retry" },
|
|
expect.objectContaining({ status: 503, message: "Webhook delivery failed with 503." })
|
|
);
|
|
|
|
await expect(registry[queueNames.email](payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("Webhook delivery failed with 503.");
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenLastCalledWith(
|
|
{ id: "attempt_retry", tenantId: "tenant_retry" },
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
status: 503,
|
|
})
|
|
);
|
|
});
|
|
|
|
test("does not retry permanent webhook client failures", async () => {
|
|
const fetchMock = vi.fn(async () => new Response("missing", { status: 404, statusText: "Not Found" }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
const payload = {
|
|
kind: "webhook.deliver" as const,
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_permanent", legacyAttemptTenantId: "tenant_permanent" },
|
|
};
|
|
|
|
await expect(registry[queueNames.email](payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({
|
|
handled: true,
|
|
action: "webhook.failed",
|
|
details: { status: 404, eventName: "entity.created", retryable: false },
|
|
});
|
|
expect(updateLegacyEventWebhookAttempt).toHaveBeenCalledWith(
|
|
{ id: "attempt_permanent", tenantId: "tenant_permanent" },
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
status: 404,
|
|
message: "Webhook delivery failed with 404.",
|
|
})
|
|
);
|
|
});
|
|
|
|
test("caps webhook response diagnostics before persisting legacy attempt bodies", async () => {
|
|
const oversizedBody = `${"a".repeat(4_096)}token=tail-secret`;
|
|
const fetchMock = vi.fn(async () => new Response(oversizedBody, { status: 502, statusText: "Bad Gateway" }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_capped", legacyAttemptTenantId: "tenant_capped" },
|
|
},
|
|
{}
|
|
)
|
|
).rejects.toThrow("Webhook delivery failed with 502.");
|
|
|
|
expect(fetchMock).toHaveBeenCalledWith("https://example.com/webhook", expect.objectContaining({ signal: expect.any(AbortSignal) }));
|
|
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { body?: string }]>;
|
|
const body = updateCalls[0]?.[1]?.body ?? "";
|
|
expect(body).toContain('"bodyBytes":4096');
|
|
expect(body).not.toContain("tail-secret");
|
|
});
|
|
|
|
test("updates redacted failed legacy attempt diagnostics when webhook fetch throws", async () => {
|
|
const fetchError = new Error("network unavailable token=fetch-secret Authorization: Bearer bearer-secret");
|
|
const fetchMock = vi.fn(async () => {
|
|
throw fetchError;
|
|
});
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: "attempt_3", legacyAttemptTenantId: "tenant_3" },
|
|
},
|
|
{}
|
|
)
|
|
).rejects.toThrow(fetchError);
|
|
const updateCalls = updateLegacyEventWebhookAttempt.mock.calls as unknown as Array<[{ id: string; tenantId: string | null }, { message?: string }]>;
|
|
expect(updateCalls[0]?.[0]).toEqual({ id: "attempt_3", tenantId: "tenant_3" });
|
|
expect(updateCalls[0]?.[1]).toEqual(
|
|
expect.objectContaining({
|
|
finishedAt: expect.any(Date),
|
|
success: false,
|
|
message: expect.stringContaining("network unavailable token=[redacted] Authorization: [redacted]"),
|
|
})
|
|
);
|
|
expect(updateCalls[0]?.[1]?.message).not.toContain("fetch-secret");
|
|
expect(updateCalls[0]?.[1]?.message).not.toContain("bearer-secret");
|
|
});
|
|
|
|
test("does not update legacy attempts for generic webhook jobs without attempt metadata", async () => {
|
|
const fetchMock = vi.fn(async () => new Response(null, { status: 204 }));
|
|
const updateLegacyEventWebhookAttempt = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ fetch: fetchMock, updateLegacyEventWebhookAttempt, webhookAddressResolver: publicWebhookAddressResolver });
|
|
|
|
await expect(
|
|
registry[queueNames.email](
|
|
{
|
|
kind: "webhook.deliver",
|
|
url: "https://example.com/webhook",
|
|
eventName: "entity.created",
|
|
metadata: { legacyAttemptId: 123 },
|
|
},
|
|
{}
|
|
)
|
|
).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "webhook.delivered",
|
|
});
|
|
expect(updateLegacyEventWebhookAttempt).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("sends opted-in email notification jobs and marks delivery diagnostics sent", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_notification_1" })),
|
|
};
|
|
const getUserNotificationPreferences = vi.fn(async () => []);
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_1" }));
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery, markNotificationDeliveryStatus });
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
category: "billing",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
subject: "Invoice ready",
|
|
text: "Your invoice is ready",
|
|
templateData: { secretToken: "not-diagnostic" },
|
|
metadata: { requestId: "req_123", source: "billing", apiKey: "secret" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "notification.email.sent",
|
|
details: { provider: "mailjet", messageId: "msg_notification_1" },
|
|
});
|
|
expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing" });
|
|
expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "default" });
|
|
expect(createNotificationDelivery).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
category: "billing",
|
|
messageKey: "billing.invoice-ready",
|
|
sourceKey: "request:req_123",
|
|
status: "PENDING",
|
|
attemptsMade: 0,
|
|
payloadMetadata: { requestId: "req_123", source: "billing" },
|
|
})
|
|
);
|
|
expect(mailProvider.send).toHaveBeenCalledWith({
|
|
to: ["user@example.com"],
|
|
subject: "Invoice ready",
|
|
text: "Your invoice is ready",
|
|
html: undefined,
|
|
templateId: undefined,
|
|
idempotencyKey: "notification-delivery:delivery_1",
|
|
personalization: [{ email: "user@example.com", data: { secretToken: "not-diagnostic" } }],
|
|
});
|
|
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
id: "delivery_1",
|
|
tenantId: "tenant_123",
|
|
status: "SENT",
|
|
attemptsMade: 1,
|
|
providerMessageId: "msg_notification_1",
|
|
sentAt: expect.any(Date),
|
|
})
|
|
);
|
|
});
|
|
|
|
test("uses worker job id as a stable notification delivery source key when payload persistence fails", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_retry_success" })),
|
|
};
|
|
const deliveries = new Map<string, { id: string }>();
|
|
const createNotificationDelivery = vi.fn(async (input: CreateNotificationDeliveryInput) => {
|
|
const key = input.sourceKey ?? "none";
|
|
const existing = deliveries.get(key);
|
|
if (existing) {
|
|
return existing;
|
|
}
|
|
const delivery = { id: `delivery_${deliveries.size + 1}` };
|
|
deliveries.set(key, delivery);
|
|
return delivery;
|
|
});
|
|
const registry = buildHandlerRegistry({
|
|
mailProvider,
|
|
getUserNotificationPreferences: vi.fn(async () => []),
|
|
createNotificationDelivery,
|
|
markNotificationDeliveryStatus: vi.fn(async () => undefined),
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
});
|
|
|
|
await expect(
|
|
registry[descriptor.queueName](
|
|
descriptor.payload,
|
|
{},
|
|
{
|
|
attemptsMade: 0,
|
|
attemptsLimit: 3,
|
|
jobId: "job_123",
|
|
updatePayload: vi.fn(async () => {
|
|
throw new Error("job data write failed");
|
|
}),
|
|
}
|
|
)
|
|
).rejects.toThrow("job data write failed");
|
|
|
|
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3, jobId: "job_123" });
|
|
|
|
expect(createNotificationDelivery).toHaveBeenCalledTimes(2);
|
|
expect(createNotificationDelivery).toHaveBeenCalledWith(expect.objectContaining({ sourceKey: "job:job_123" }));
|
|
expect(mailProvider.send).toHaveBeenCalledTimes(1);
|
|
expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:delivery_1" }));
|
|
});
|
|
|
|
test("persists delivery id in notification job metadata for retry-safe diagnostics", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_notification_1" })),
|
|
};
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_retry" }));
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const updatePayload = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
mailProvider,
|
|
getUserNotificationPreferences: vi.fn(async () => []),
|
|
createNotificationDelivery,
|
|
markNotificationDeliveryStatus,
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
});
|
|
|
|
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3, updatePayload });
|
|
|
|
expect(updatePayload).toHaveBeenCalledWith({
|
|
...descriptor.payload,
|
|
metadata: { deliveryId: "delivery_retry" },
|
|
});
|
|
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(expect.objectContaining({ id: "delivery_retry", status: "SENT", attemptsMade: 1 }));
|
|
});
|
|
|
|
test("reuses notification delivery id on email retry without creating duplicate diagnostics", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_retry_success" })),
|
|
};
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_new" }));
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
mailProvider,
|
|
getUserNotificationPreferences: vi.fn(async () => []),
|
|
createNotificationDelivery,
|
|
markNotificationDeliveryStatus,
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
metadata: { deliveryId: "delivery_existing" },
|
|
});
|
|
|
|
await registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 });
|
|
|
|
expect(createNotificationDelivery).not.toHaveBeenCalled();
|
|
expect(mailProvider.send).toHaveBeenCalledWith(expect.objectContaining({ idempotencyKey: "notification-delivery:delivery_existing" }));
|
|
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(
|
|
expect.objectContaining({ id: "delivery_existing", tenantId: "tenant_123", status: "SENT", attemptsMade: 2, providerMessageId: "msg_retry_success" })
|
|
);
|
|
});
|
|
|
|
test("keeps email delivery pending until the terminal notification retry fails", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => {
|
|
throw new Error("provider token=secret failed");
|
|
}),
|
|
};
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
mailProvider,
|
|
getUserNotificationPreferences: vi.fn(async () => []),
|
|
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_existing" })),
|
|
markNotificationDeliveryStatus,
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
metadata: { deliveryId: "delivery_existing" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 1, attemptsLimit: 3 })).rejects.toThrow("provider");
|
|
expect(markNotificationDeliveryStatus).toHaveBeenLastCalledWith({
|
|
id: "delivery_existing",
|
|
tenantId: "tenant_123",
|
|
status: "PENDING",
|
|
attemptsMade: 2,
|
|
lastError: "provider token=[redacted] failed",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toThrow("provider");
|
|
expect(markNotificationDeliveryStatus).toHaveBeenLastCalledWith({
|
|
id: "delivery_existing",
|
|
tenantId: "tenant_123",
|
|
status: "FAILED",
|
|
attemptsMade: 3,
|
|
lastError: "provider token=[redacted] failed",
|
|
});
|
|
});
|
|
|
|
test("does not retry an already-sent email when sent status persistence fails", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_sent_once" })),
|
|
};
|
|
const markNotificationDeliveryStatus = vi.fn(async () => {
|
|
throw new Error("database token=secret unavailable");
|
|
});
|
|
const registry = buildHandlerRegistry({
|
|
mailProvider,
|
|
getUserNotificationPreferences: vi.fn(async () => []),
|
|
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_sent_once" })),
|
|
markNotificationDeliveryStatus,
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
metadata: { deliveryId: "delivery_sent_once" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toEqual({
|
|
handled: true,
|
|
action: "notification.email.sent.diagnostic_failed",
|
|
details: {
|
|
provider: "mailjet",
|
|
messageId: "msg_sent_once",
|
|
userId: "user_123",
|
|
messageKey: "billing.invoice-ready",
|
|
deliveryId: "delivery_sent_once",
|
|
diagnosticError: "database token=[redacted] unavailable",
|
|
},
|
|
});
|
|
expect(mailProvider.send).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
test("skips opted-out email notification jobs without calling mail provider", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
|
|
};
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_skip" }));
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
mailProvider,
|
|
getUserNotificationPreferences: vi.fn(async () => [{ enabled: false }]),
|
|
createNotificationDelivery,
|
|
markNotificationDeliveryStatus,
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "notification.email.skipped",
|
|
details: {
|
|
userId: "user_123",
|
|
channel: "email",
|
|
messageKey: "billing.invoice-ready",
|
|
deliveryId: "delivery_skip",
|
|
reason: "preference_disabled",
|
|
},
|
|
});
|
|
expect(mailProvider.send).not.toHaveBeenCalled();
|
|
expect(createNotificationDelivery).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
messageKey: "billing.invoice-ready",
|
|
status: "SKIPPED",
|
|
lastError: "preference_disabled",
|
|
payloadMetadata: { reason: "preference_disabled" },
|
|
})
|
|
);
|
|
expect(markNotificationDeliveryStatus).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("lets explicit category notification preferences override disabled defaults", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "msg_category_enabled" })),
|
|
};
|
|
const getUserNotificationPreferences = vi
|
|
.fn()
|
|
.mockResolvedValueOnce([{ enabled: true, category: "billing" }])
|
|
.mockResolvedValueOnce([{ enabled: false, category: "default" }]);
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_category_enabled" }));
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery, markNotificationDeliveryStatus });
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
category: "billing",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
subject: "Invoice ready",
|
|
text: "Your invoice is ready",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
|
|
handled: true,
|
|
action: "notification.email.sent",
|
|
details: { provider: "mailjet", messageId: "msg_category_enabled" },
|
|
});
|
|
expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "billing" });
|
|
expect(getUserNotificationPreferences).toHaveBeenCalledWith({ tenantId: "tenant_123", userId: "user_123", channel: "email", category: "default" });
|
|
expect(mailProvider.send).toHaveBeenCalledTimes(1);
|
|
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith(
|
|
expect.objectContaining({ id: "delivery_category_enabled", tenantId: "tenant_123", status: "SENT" })
|
|
);
|
|
});
|
|
|
|
test("records skipped diagnostics for opted-out in-app notification jobs", async () => {
|
|
const getUserNotificationPreferences = vi.fn(async () => [{ enabled: false }]);
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_in_app_skip" }));
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
|
|
};
|
|
const registry = buildHandlerRegistry({ mailProvider, getUserNotificationPreferences, createNotificationDelivery });
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
category: "billing",
|
|
messageKey: "billing.invoice-ready",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "notification.in-app.skipped",
|
|
details: {
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
messageKey: "billing.invoice-ready",
|
|
deliveryId: "delivery_in_app_skip",
|
|
reason: "preference_disabled",
|
|
},
|
|
});
|
|
expect(getUserNotificationPreferences).toHaveBeenCalledWith({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
category: "billing",
|
|
});
|
|
expect(getUserNotificationPreferences).toHaveBeenCalledWith({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
category: "default",
|
|
});
|
|
expect(createNotificationDelivery).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
category: "billing",
|
|
messageKey: "billing.invoice-ready",
|
|
status: "SKIPPED",
|
|
lastError: "preference_disabled",
|
|
payloadMetadata: { reason: "preference_disabled" },
|
|
})
|
|
);
|
|
expect(mailProvider.send).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("marks email notification diagnostics failed when provider rejects", async () => {
|
|
const providerError = new Error("SMTP token=secret-token rejected for user@example.com with Authorization: Bearer abc123 and payload-secret");
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => {
|
|
throw providerError;
|
|
}),
|
|
};
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
mailProvider,
|
|
getUserNotificationPreferences: vi.fn(async () => [{ enabled: true }]),
|
|
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_failed" })),
|
|
markNotificationDeliveryStatus,
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
templateData: { apiKey: "payload-secret" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toBe(providerError);
|
|
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith({
|
|
id: "delivery_failed",
|
|
tenantId: "tenant_123",
|
|
status: "FAILED",
|
|
attemptsMade: 1,
|
|
lastError: "SMTP token=[redacted] rejected for [redacted-email] with Authorization: Bearer [redacted] and payload-[redacted]",
|
|
});
|
|
});
|
|
|
|
test("marks email notification diagnostics failed when provider setup rejects", async () => {
|
|
const markNotificationDeliveryStatus = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
env: {},
|
|
getUserNotificationPreferences: vi.fn(async () => [{ enabled: true }]),
|
|
createNotificationDelivery: vi.fn(async () => ({ id: "delivery_setup_failed" })),
|
|
markNotificationDeliveryStatus,
|
|
});
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "billing.invoice-ready",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("MAILJET_API_KEY is required");
|
|
expect(markNotificationDeliveryStatus).toHaveBeenCalledWith({
|
|
id: "delivery_setup_failed",
|
|
tenantId: "tenant_123",
|
|
status: "FAILED",
|
|
attemptsMade: 1,
|
|
lastError: "MAILJET_API_KEY is required for Mailjet email delivery.",
|
|
});
|
|
});
|
|
|
|
test("rejects email notification jobs missing tenant or email before diagnostics", async () => {
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_missing" }));
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
|
|
};
|
|
const registry = buildHandlerRegistry({ mailProvider, createNotificationDelivery });
|
|
|
|
await expect(
|
|
registry[queueNames.notifications](
|
|
{
|
|
kind: "notification.send",
|
|
userId: "user_123",
|
|
channel: "email",
|
|
email: "user@example.com",
|
|
messageKey: "account.updated",
|
|
} as Parameters<(typeof registry)[typeof queueNames.notifications]>[0],
|
|
{}
|
|
)
|
|
).rejects.toThrow("Notification jobs require tenantId.");
|
|
await expect(
|
|
registry[queueNames.notifications](
|
|
buildNotificationJob({ tenantId: "tenant_123", userId: "user_123", channel: "email", messageKey: "account.updated" }).payload,
|
|
{}
|
|
)
|
|
).rejects.toThrow("Email notifications require email recipient.");
|
|
expect(createNotificationDelivery).not.toHaveBeenCalled();
|
|
expect(mailProvider.send).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("rejects unsupported notification channels before diagnostics", async () => {
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_sms" }));
|
|
const registry = buildHandlerRegistry({ createNotificationDelivery });
|
|
|
|
await expect(
|
|
registry[queueNames.notifications](
|
|
{
|
|
kind: "notification.send",
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "sms",
|
|
messageKey: "account.updated",
|
|
} as unknown as Parameters<(typeof registry)[typeof queueNames.notifications]>[0],
|
|
{}
|
|
)
|
|
).rejects.toThrow("Unsupported notification channel: sms");
|
|
expect(createNotificationDelivery).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("accepts descriptor-derived non-email notification jobs through registry", async () => {
|
|
const mailProvider: MailProvider = {
|
|
id: "mailjet",
|
|
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const })),
|
|
};
|
|
const createNotificationDelivery = vi.fn(async () => ({ id: "delivery_in_app" }));
|
|
const registry = buildHandlerRegistry({ mailProvider, createNotificationDelivery, getUserNotificationPreferences: vi.fn(async () => []) });
|
|
const descriptor = buildNotificationJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
messageKey: "account.updated",
|
|
templateData: { ignoredBody: "not surfaced" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "notification.accepted",
|
|
details: {
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
messageKey: "account.updated",
|
|
deliveryId: "delivery_in_app",
|
|
},
|
|
});
|
|
expect(createNotificationDelivery).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
userId: "user_123",
|
|
channel: "in-app",
|
|
messageKey: "account.updated",
|
|
status: "SENT",
|
|
attemptsMade: 1,
|
|
sentAt: expect.any(Date),
|
|
})
|
|
);
|
|
expect(mailProvider.send).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("rejects unsupported cleanup jobs through registry", async () => {
|
|
const registry = buildHandlerRegistry();
|
|
const descriptor = buildCleanupJob({
|
|
task: "expired-sessions",
|
|
olderThanHours: 24,
|
|
requestedBy: "system-fixture",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "worker.unsupported",
|
|
message: "Unsupported maintenance task.",
|
|
details: {
|
|
queueName: queueNames.maintenance,
|
|
task: "expired-sessions",
|
|
olderThanHours: 24,
|
|
},
|
|
});
|
|
});
|
|
|
|
test("runs outbox DONE retention cleanup through maintenance handler", async () => {
|
|
vi.useFakeTimers();
|
|
vi.setSystemTime(new Date("2026-05-28T08:00:00.000Z"));
|
|
const deleteDoneOutboxEvents = vi.fn(async (input: { cutoff: Date; take?: number }) => ({
|
|
deleted: 17,
|
|
cutoff: input.cutoff,
|
|
take: input.take ?? 1000,
|
|
}));
|
|
const registry = buildHandlerRegistry({ deleteDoneOutboxEvents });
|
|
const descriptor = buildCleanupJob({
|
|
task: "outbox-done-retention",
|
|
olderThanHours: 48,
|
|
take: 500,
|
|
metadata: { requestId: "req_outbox_retention_1" },
|
|
});
|
|
|
|
try {
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "outbox.done-retention.completed",
|
|
details: {
|
|
task: "outbox-done-retention",
|
|
requestId: "req_outbox_retention_1",
|
|
olderThanHours: 48,
|
|
cutoff: "2026-05-26T08:00:00.000Z",
|
|
take: 500,
|
|
deleted: 17,
|
|
},
|
|
});
|
|
} finally {
|
|
vi.useRealTimers();
|
|
}
|
|
|
|
expect(deleteDoneOutboxEvents).toHaveBeenCalledWith({
|
|
cutoff: new Date("2026-05-26T08:00:00.000Z"),
|
|
take: 500,
|
|
});
|
|
});
|
|
|
|
test("runs workflow stalled execution reconciliation through maintenance handler", async () => {
|
|
const reconcileWorkflowExecutionStalls = vi.fn(async () => ({
|
|
cutoff: new Date("2026-05-29T02:30:00.000Z"),
|
|
take: 100,
|
|
inspected: 3,
|
|
failedOutboxEvents: 2,
|
|
repaired: 2,
|
|
skippedWithoutFailedOutbox: 1,
|
|
updateMisses: 0,
|
|
}));
|
|
const registry = buildHandlerRegistry({ reconcileWorkflowExecutionStalls });
|
|
const descriptor = buildCleanupJob({
|
|
task: "workflow-execution-stalled-reconcile",
|
|
olderThanMinutes: 30,
|
|
take: 100,
|
|
metadata: { requestId: "req_workflow_stalled_reconcile_1" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "workflow.execution-stalled-reconcile.completed",
|
|
details: {
|
|
task: "workflow-execution-stalled-reconcile",
|
|
requestId: "req_workflow_stalled_reconcile_1",
|
|
olderThanMinutes: 30,
|
|
cutoff: "2026-05-29T02:30:00.000Z",
|
|
take: 100,
|
|
inspected: 3,
|
|
failedOutboxEvents: 2,
|
|
repaired: 2,
|
|
skippedWithoutFailedOutbox: 1,
|
|
updateMisses: 0,
|
|
},
|
|
});
|
|
|
|
expect(reconcileWorkflowExecutionStalls).toHaveBeenCalledWith({
|
|
olderThanMinutes: 30,
|
|
take: 100,
|
|
});
|
|
});
|
|
|
|
test("dispatches dsgvo-grace-period-scan through runDsgvoGraceCron with the configured service", async () => {
|
|
const listOverdueUsers = vi
|
|
.fn()
|
|
.mockResolvedValue([{ userId: "u-overdue-1", tenantId: "tenant_a", scheduledDeletionAt: new Date("2026-05-01T00:00:00Z"), deletionRequestId: "req-1" }]);
|
|
const enqueueDeletion = vi.fn().mockResolvedValue({ jobId: "dsgvo-cascade-1" });
|
|
const logger = { info: vi.fn(), error: vi.fn() };
|
|
const registry = buildHandlerRegistry({
|
|
dsgvoGraceCronService: { listOverdueUsers, enqueueDeletion },
|
|
logger,
|
|
});
|
|
const descriptor = buildCleanupJob({ task: "dsgvo-grace-period-scan" });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "dsgvo.grace-period.scanned",
|
|
details: {
|
|
task: "dsgvo-grace-period-scan",
|
|
scanned: 1,
|
|
dispatched: 1,
|
|
failed: 0,
|
|
failures: [],
|
|
},
|
|
});
|
|
expect(listOverdueUsers).toHaveBeenCalledTimes(1);
|
|
expect(enqueueDeletion).toHaveBeenCalledTimes(1);
|
|
expect(enqueueDeletion).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
userId: "u-overdue-1",
|
|
tenantId: "tenant_a",
|
|
requestId: "req-1",
|
|
actorType: "system",
|
|
mode: "delete",
|
|
reason: "dsgvo-grace-period-elapsed",
|
|
})
|
|
);
|
|
});
|
|
|
|
test("acks dsgvo-grace-period-scan with skipped action when service dep is missing (no UnsupportedWorkerJobError loop)", async () => {
|
|
const logger = { info: vi.fn(), error: vi.fn() };
|
|
const registry = buildHandlerRegistry({ logger });
|
|
const descriptor = buildCleanupJob({ task: "dsgvo-grace-period-scan", metadata: { requestId: "req_grace_1" } });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "dsgvo.grace-period.skipped",
|
|
details: {
|
|
task: "dsgvo-grace-period-scan",
|
|
reason: "dsgvoGraceCronService not configured",
|
|
scanned: 0,
|
|
dispatched: 0,
|
|
failed: 0,
|
|
},
|
|
});
|
|
expect(logger.info).toHaveBeenCalledWith(
|
|
expect.objectContaining({ task: "dsgvo-grace-period-scan", requestId: "req_grace_1" }),
|
|
expect.stringContaining("dsgvo-grace-cron.skipped")
|
|
);
|
|
});
|
|
|
|
test("hard-deletes expired export artifacts and reports scalar counts", async () => {
|
|
const jobs = [
|
|
retentionJob({ id: "export_job_1", artifactBucket: "exports", artifactKey: "tenant_1/export-1.csv" }),
|
|
retentionJob({ id: "export_job_2", artifactBucket: "exports", artifactKey: "tenant_1/export-2.csv" }),
|
|
];
|
|
const listExpiredCompletedExportJobs = vi.fn(async () => []);
|
|
const markExportJobExpired = vi.fn(async () => undefined);
|
|
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs);
|
|
const storageDeleteService: TestStorageDeleteService = {
|
|
deleteObject: vi.fn(async (input) => {
|
|
return { bucket: input.bucket ?? "exports", key: input.key };
|
|
}),
|
|
};
|
|
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
|
|
const markExportJobHardDeleteFailed = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
listExpiredCompletedExportJobs,
|
|
markExportJobExpired,
|
|
listSoftDeletedExportJobsForHardDelete,
|
|
storageDeleteService,
|
|
markExportJobHardDeleteSucceeded,
|
|
markExportJobHardDeleteFailed,
|
|
});
|
|
const descriptor = buildCleanupJob({
|
|
task: "expired-export-artifacts",
|
|
olderThanHours: 2,
|
|
metadata: { requestId: "req_retention_1", token: "payload-secret" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "export-artifact-retention.completed",
|
|
details: {
|
|
task: "expired-export-artifacts",
|
|
requestId: "req_retention_1",
|
|
expired: 0,
|
|
processed: 2,
|
|
deleted: 2,
|
|
failed: 0,
|
|
jobs: [
|
|
{ exportJobId: "export_job_1", status: "deleted" },
|
|
{ exportJobId: "export_job_2", status: "deleted" },
|
|
],
|
|
},
|
|
});
|
|
expect(listExpiredCompletedExportJobs).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined });
|
|
expect(markExportJobExpired).not.toHaveBeenCalled();
|
|
expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined });
|
|
const hardDeleteListInput = (listSoftDeletedExportJobsForHardDelete.mock.calls as unknown as [TestRetentionListInput][])[0]?.[0];
|
|
expect(hardDeleteListInput?.now.getTime()).toBeLessThan(Date.now());
|
|
expect(storageDeleteService.deleteObject).toHaveBeenNthCalledWith(1, { bucket: "exports", key: "tenant_1/export-1.csv" });
|
|
expect(storageDeleteService.deleteObject).toHaveBeenNthCalledWith(2, { bucket: "exports", key: "tenant_1/export-2.csv" });
|
|
expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1" });
|
|
expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_2", tenantId: "tenant_1" });
|
|
expect(markExportJobHardDeleteFailed).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("caps export artifact hard-delete concurrency while preserving reported job order", async () => {
|
|
const jobs = Array.from({ length: 6 }, (_, index) =>
|
|
retentionJob({
|
|
id: `export_job_${index + 1}`,
|
|
artifactBucket: "exports",
|
|
artifactKey: `tenant_1/export-${index + 1}.csv`,
|
|
})
|
|
);
|
|
let activeDeletes = 0;
|
|
let maxActiveDeletes = 0;
|
|
let releaseFirstBatch: (() => void) | undefined;
|
|
const firstBatchWait = new Promise<void>((resolve) => {
|
|
releaseFirstBatch = resolve;
|
|
});
|
|
const listExpiredCompletedExportJobs = vi.fn(async () => []);
|
|
const markExportJobExpired = vi.fn(async () => undefined);
|
|
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs);
|
|
const storageDeleteService: TestStorageDeleteService = {
|
|
deleteObject: vi.fn(async (input) => {
|
|
activeDeletes += 1;
|
|
maxActiveDeletes = Math.max(maxActiveDeletes, activeDeletes);
|
|
if (input.key !== "tenant_1/export-6.csv") {
|
|
await firstBatchWait;
|
|
}
|
|
activeDeletes -= 1;
|
|
return { bucket: input.bucket ?? "exports", key: input.key };
|
|
}),
|
|
};
|
|
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
listExpiredCompletedExportJobs,
|
|
markExportJobExpired,
|
|
listSoftDeletedExportJobsForHardDelete,
|
|
storageDeleteService,
|
|
markExportJobHardDeleteSucceeded,
|
|
});
|
|
const descriptor = buildCleanupJob({ task: "expired-export-artifacts" });
|
|
const run = registry[descriptor.queueName](descriptor.payload, {});
|
|
|
|
await vi.waitFor(() => expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(5));
|
|
expect(maxActiveDeletes).toBe(5);
|
|
expect(markExportJobHardDeleteSucceeded).not.toHaveBeenCalled();
|
|
releaseFirstBatch?.();
|
|
|
|
await expect(run).resolves.toEqual({
|
|
handled: true,
|
|
action: "export-artifact-retention.completed",
|
|
details: {
|
|
task: "expired-export-artifacts",
|
|
requestId: undefined,
|
|
expired: 0,
|
|
processed: 6,
|
|
deleted: 6,
|
|
failed: 0,
|
|
jobs: jobs.map((job) => ({ exportJobId: job.id, status: "deleted" })),
|
|
},
|
|
});
|
|
expect(maxActiveDeletes).toBe(5);
|
|
expect(storageDeleteService.deleteObject).toHaveBeenCalledTimes(6);
|
|
});
|
|
|
|
test("continues export artifact retention after storage failures with redacted diagnostics", async () => {
|
|
const jobs = [
|
|
retentionJob({ id: "export_job_ok", artifactBucket: "exports", artifactKey: "tenant_1/ok.csv" }),
|
|
retentionJob({ id: "export_job_bad", artifactBucket: "exports", artifactKey: "tenant_1/bad.csv" }),
|
|
];
|
|
const listExpiredCompletedExportJobs = vi.fn(async () => []);
|
|
const markExportJobExpired = vi.fn(async () => undefined);
|
|
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs);
|
|
const storageDeleteService: TestStorageDeleteService = {
|
|
deleteObject: vi.fn(async (input) => {
|
|
if (input.key === "tenant_1/bad.csv") {
|
|
throw new Error("delete failed token=storage-secret body=payload-bytes authorization=Bearer-secret");
|
|
}
|
|
return { bucket: input.bucket ?? "exports", key: input.key };
|
|
}),
|
|
};
|
|
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
|
|
const markExportJobHardDeleteFailed = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
listExpiredCompletedExportJobs,
|
|
markExportJobExpired,
|
|
listSoftDeletedExportJobsForHardDelete,
|
|
storageDeleteService,
|
|
markExportJobHardDeleteSucceeded,
|
|
markExportJobHardDeleteFailed,
|
|
});
|
|
const descriptor = buildCleanupJob({ task: "expired-export-artifacts", metadata: { requestId: "req_retention_partial" } });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "export-artifact-retention.completed",
|
|
details: {
|
|
task: "expired-export-artifacts",
|
|
requestId: "req_retention_partial",
|
|
expired: 0,
|
|
processed: 2,
|
|
deleted: 1,
|
|
failed: 1,
|
|
jobs: [
|
|
{ exportJobId: "export_job_ok", status: "deleted" },
|
|
{ exportJobId: "export_job_bad", status: "failed", message: "delete failed token=[redacted] body=[redacted] authorization=[redacted]" },
|
|
],
|
|
},
|
|
});
|
|
expect(markExportJobHardDeleteSucceeded).toHaveBeenCalledWith({ id: "export_job_ok", tenantId: "tenant_1" });
|
|
expect(markExportJobHardDeleteFailed).toHaveBeenCalledWith({
|
|
id: "export_job_bad",
|
|
tenantId: "tenant_1",
|
|
errorMessage: "delete failed token=[redacted] body=[redacted] authorization=[redacted]",
|
|
});
|
|
const resultText = JSON.stringify((await registry[descriptor.queueName](descriptor.payload, {})).details);
|
|
expect(resultText).not.toContain("storage-secret");
|
|
expect(resultText).not.toContain("payload-bytes");
|
|
expect(resultText).not.toContain("Bearer-secret");
|
|
});
|
|
|
|
test("fails export artifact retention when delete dependency is missing", async () => {
|
|
const jobs = [retentionJob({ id: "export_job_missing_delete", artifactBucket: "exports", artifactKey: "tenant_1/missing.csv" })];
|
|
const listExpiredCompletedExportJobs = vi.fn(async () => []);
|
|
const markExportJobExpired = vi.fn(async () => undefined);
|
|
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => jobs);
|
|
const markExportJobHardDeleteSucceeded = vi.fn(async () => undefined);
|
|
const markExportJobHardDeleteFailed = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({
|
|
listExpiredCompletedExportJobs,
|
|
markExportJobExpired,
|
|
listSoftDeletedExportJobsForHardDelete,
|
|
markExportJobHardDeleteSucceeded,
|
|
markExportJobHardDeleteFailed,
|
|
});
|
|
const descriptor = buildCleanupJob({ task: "expired-export-artifacts" });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toThrow("Export artifact storage delete service is not configured.");
|
|
expect(markExportJobHardDeleteFailed).not.toHaveBeenCalled();
|
|
expect(markExportJobHardDeleteSucceeded).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("marks expired completed exports before hard-delete retention scan", async () => {
|
|
const expiredJobs = [retentionJob({ id: "export_job_expired", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired.csv" })];
|
|
const listExpiredCompletedExportJobs = vi.fn(async () => expiredJobs);
|
|
const markExportJobExpired = vi.fn(async () => undefined);
|
|
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => []);
|
|
const storageDeleteService: TestStorageDeleteService = {
|
|
deleteObject: vi.fn(async (input) => ({ bucket: input.bucket ?? "exports", key: input.key })),
|
|
};
|
|
const registry = buildHandlerRegistry({
|
|
listExpiredCompletedExportJobs,
|
|
markExportJobExpired,
|
|
listSoftDeletedExportJobsForHardDelete,
|
|
storageDeleteService,
|
|
});
|
|
const descriptor = buildCleanupJob({ task: "expired-export-artifacts", take: 25 });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
|
|
action: "export-artifact-retention.completed",
|
|
details: {
|
|
expired: 1,
|
|
processed: 0,
|
|
deleted: 0,
|
|
failed: 0,
|
|
},
|
|
});
|
|
expect(listExpiredCompletedExportJobs).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 });
|
|
expect(markExportJobExpired).toHaveBeenCalledWith({ id: "export_job_expired", tenantId: "tenant_1", deletedAt: expect.any(Date) });
|
|
expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: 25 });
|
|
expect(storageDeleteService.deleteObject).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("marks independent expired completed exports in parallel before hard-delete retention scan", async () => {
|
|
const expiredJobs = [
|
|
retentionJob({ id: "export_job_expired_1", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired-1.csv" }),
|
|
retentionJob({ id: "export_job_expired_2", status: "COMPLETED", artifactBucket: "exports", artifactKey: "tenant_1/expired-2.csv" }),
|
|
];
|
|
const listExpiredCompletedExportJobs = vi.fn(async () => expiredJobs);
|
|
let firstMarkCanFinish: (() => void) | undefined;
|
|
const firstMarkWait = new Promise<void>((resolve) => {
|
|
firstMarkCanFinish = resolve;
|
|
});
|
|
const markExportJobExpired = vi.fn(async ({ id }) => {
|
|
if (id === "export_job_expired_1") {
|
|
await firstMarkWait;
|
|
}
|
|
});
|
|
const listSoftDeletedExportJobsForHardDelete = vi.fn(async () => []);
|
|
const storageDeleteService: TestStorageDeleteService = {
|
|
deleteObject: vi.fn(async (input) => ({ bucket: input.bucket ?? "exports", key: input.key })),
|
|
};
|
|
const registry = buildHandlerRegistry({
|
|
listExpiredCompletedExportJobs,
|
|
markExportJobExpired,
|
|
listSoftDeletedExportJobsForHardDelete,
|
|
storageDeleteService,
|
|
});
|
|
const descriptor = buildCleanupJob({ task: "expired-export-artifacts" });
|
|
const run = registry[descriptor.queueName](descriptor.payload, {});
|
|
|
|
await vi.waitFor(() => expect(markExportJobExpired).toHaveBeenCalledTimes(2));
|
|
expect(listSoftDeletedExportJobsForHardDelete).not.toHaveBeenCalled();
|
|
firstMarkCanFinish?.();
|
|
|
|
await expect(run).resolves.toMatchObject({
|
|
action: "export-artifact-retention.completed",
|
|
details: { expired: 2, processed: 0 },
|
|
});
|
|
expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined });
|
|
});
|
|
|
|
test("accepts descriptor-derived AI jobs through registry", async () => {
|
|
const registry = buildHandlerRegistry();
|
|
const descriptors = [
|
|
buildAiChatCompletionJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
conversationId: "conversation_123",
|
|
messages: [{ role: "user", content: "fixture prompt not surfaced" }],
|
|
}),
|
|
buildAiEmbeddingJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
model: "fixture-embedding-model",
|
|
task: { text: "fixture embedding text not surfaced" },
|
|
}),
|
|
buildAiBrowserTaskJob({
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
task: { instruction: "fixture browser task not surfaced" },
|
|
}),
|
|
];
|
|
|
|
await Promise.all(
|
|
descriptors.map((descriptor) =>
|
|
expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "worker.unsupported",
|
|
message: "AI worker job is not configured.",
|
|
details: { queueName: queueNames.aiChatbot, kind: descriptor.payload.kind, tenantId: "tenant_123", userId: "user_123" },
|
|
})
|
|
)
|
|
);
|
|
});
|
|
|
|
test("rejects descriptor-derived browser automation jobs through registry without global fetch", async () => {
|
|
const fetchSpy = vi.spyOn(globalThis, "fetch");
|
|
const registry = buildHandlerRegistry();
|
|
const descriptors = [
|
|
buildBrowserSessionRunJob({
|
|
sessionId: "session_123",
|
|
tenantId: "tenant_123",
|
|
userId: "user_123",
|
|
allowedOrigins: ["https://example.com"],
|
|
task: { instruction: "fixture browser session task not surfaced" },
|
|
}),
|
|
buildBrowserSessionCleanupJob({
|
|
sessionId: "session_123",
|
|
tenantId: "tenant_123",
|
|
metadata: { reason: "fixture cleanup reason not surfaced" },
|
|
}),
|
|
buildBrowserUsageFlushJob({
|
|
sessionId: "session_123",
|
|
tenantId: "tenant_123",
|
|
metadata: { token: "fixture token not surfaced" },
|
|
}),
|
|
];
|
|
|
|
try {
|
|
await Promise.all(
|
|
descriptors.map((descriptor) =>
|
|
expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "worker.unsupported",
|
|
message: "Browser automation worker job is not configured.",
|
|
details: { queueName: queueNames.browserAutomation, kind: descriptor.payload.kind, sessionId: "session_123" },
|
|
})
|
|
)
|
|
);
|
|
expect(fetchSpy).not.toHaveBeenCalled();
|
|
} finally {
|
|
fetchSpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
test("pauses gated export jobs without running the export service", async () => {
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt: new Date() })),
|
|
};
|
|
const markExportJobPaused = vi.fn(async () => undefined);
|
|
const markExportJobRunning = vi.fn(async () => undefined);
|
|
const markExportJobSucceeded = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobPaused, markExportJobRunning, markExportJobSucceeded });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_paused",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
requestId: "req_export_paused",
|
|
complianceGate: {
|
|
state: "paused",
|
|
reason: "workspace-retention-pause",
|
|
requestId: "req_pause_1",
|
|
},
|
|
metadata: { attemptsMade: 3 },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "export.paused",
|
|
details: {
|
|
exportJobId: "export_job_paused",
|
|
tenantId: "tenant_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
requestId: "req_export_paused",
|
|
attemptsMade: 3,
|
|
pauseReason: "workspace-retention-pause",
|
|
pauseRequestId: "req_pause_1",
|
|
status: "paused",
|
|
},
|
|
});
|
|
expect(markExportJobPaused).toHaveBeenCalledWith({ id: "export_job_paused", tenantId: "tenant_1", attemptsMade: 3, reason: "workspace-retention-pause" });
|
|
expect(exportService.runExport).not.toHaveBeenCalled();
|
|
expect(markExportJobRunning).not.toHaveBeenCalled();
|
|
expect(markExportJobSucceeded).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("redacts secret-like pause reasons in persisted and returned diagnostics", async () => {
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt: new Date() })),
|
|
};
|
|
const markExportJobPaused = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobPaused });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_secret_pause",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "json",
|
|
requestId: "req_export_secret_pause",
|
|
complianceGate: {
|
|
state: "paused",
|
|
reason: "pause token=service-secret body=payload-bytes authorization=Bearer-secret",
|
|
requestId: "req_pause_secret",
|
|
},
|
|
metadata: { attemptsMade: 1 },
|
|
filters: { apiKey: "payload-secret" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
|
|
action: "export.paused",
|
|
details: {
|
|
pauseReason: "pause token=[redacted] body=[redacted] authorization=[redacted]",
|
|
},
|
|
});
|
|
expect(markExportJobPaused).toHaveBeenCalledWith({
|
|
id: "export_job_secret_pause",
|
|
tenantId: "tenant_1",
|
|
attemptsMade: 1,
|
|
reason: "pause token=[redacted] body=[redacted] authorization=[redacted]",
|
|
});
|
|
expect(exportService.runExport).not.toHaveBeenCalled();
|
|
expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("service-secret");
|
|
expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("payload-secret");
|
|
expect(JSON.stringify(markExportJobPaused.mock.calls)).not.toContain("Bearer-secret");
|
|
});
|
|
|
|
test("resumes active gated export jobs before running them", async () => {
|
|
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
|
|
const events: string[] = [];
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => {
|
|
events.push("export");
|
|
return { artifactBucket: "exports", artifactKey: "tenant_1/resumed.csv", rowCount: 7, expiresAt };
|
|
}),
|
|
};
|
|
const markExportJobResumed = vi.fn(async () => {
|
|
events.push("resume");
|
|
});
|
|
const markExportJobRunning = vi.fn(async () => {
|
|
events.push("running");
|
|
});
|
|
const markExportJobSucceeded = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobResumed, markExportJobRunning, markExportJobSucceeded });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_resumed",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
requestId: "req_export_resumed",
|
|
complianceGate: {
|
|
state: "active",
|
|
resumedAt: "2026-05-08T00:00:00.000Z",
|
|
requestId: "req_resume_1",
|
|
},
|
|
metadata: { attemptsMade: 2 },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toMatchObject({
|
|
action: "export.completed",
|
|
details: { exportJobId: "export_job_resumed", status: "completed", attemptsMade: 2 },
|
|
});
|
|
expect(markExportJobResumed).toHaveBeenCalledWith({ id: "export_job_resumed", tenantId: "tenant_1", attemptsMade: 2 });
|
|
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_resumed", tenantId: "tenant_1", attemptsMade: 2 });
|
|
expect(exportService.runExport).toHaveBeenCalledWith(descriptor.payload);
|
|
expect(events).toEqual(["resume", "running", "export"]);
|
|
});
|
|
|
|
test("runs export jobs through injected service and persists completed artifact diagnostics", async () => {
|
|
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt })),
|
|
};
|
|
const markExportJobRunning = vi.fn(async () => undefined);
|
|
const markExportJobSucceeded = vi.fn(async () => undefined);
|
|
const markExportJobFailed = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobSucceeded, markExportJobFailed });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_1",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
requestId: "req_export_success",
|
|
metadata: { attemptsMade: 2, token: "payload-secret" },
|
|
filters: { authorization: "Bearer payload-secret" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "export.completed",
|
|
details: {
|
|
exportJobId: "export_job_1",
|
|
tenantId: "tenant_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
requestId: "req_export_success",
|
|
artifactBucket: "exports",
|
|
artifactKey: "tenant_1/export.csv",
|
|
attemptsMade: 2,
|
|
expiresAt: "2026-05-09T00:00:00.000Z",
|
|
rowCount: 42,
|
|
status: "completed",
|
|
},
|
|
});
|
|
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1", attemptsMade: 2 });
|
|
expect(exportService.runExport).toHaveBeenCalledWith(descriptor.payload);
|
|
expect(markExportJobSucceeded).toHaveBeenCalledWith({
|
|
id: "export_job_1",
|
|
tenantId: "tenant_1",
|
|
artifactBucket: "exports",
|
|
artifactKey: "tenant_1/export.csv",
|
|
rowCount: 42,
|
|
attemptsMade: 2,
|
|
expiresAt,
|
|
});
|
|
expect(markExportJobFailed).not.toHaveBeenCalled();
|
|
expect(JSON.stringify(markExportJobRunning.mock.calls)).not.toContain("payload-secret");
|
|
expect(JSON.stringify(markExportJobSucceeded.mock.calls)).not.toContain("payload-secret");
|
|
});
|
|
|
|
test("uses BullMQ attempt metadata for export job persistence when available", async () => {
|
|
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => ({ artifactBucket: "exports", artifactKey: "tenant_1/export.csv", rowCount: 42, expiresAt })),
|
|
};
|
|
const markExportJobRunning = vi.fn(async () => undefined);
|
|
const markExportJobSucceeded = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobSucceeded });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_bullmq_attempt",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
requestId: "req_export_bullmq_attempt",
|
|
metadata: { attemptsMade: 1 },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 4 })).resolves.toMatchObject({
|
|
action: "export.completed",
|
|
details: { attemptsMade: 5 },
|
|
});
|
|
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_bullmq_attempt", tenantId: "tenant_1", attemptsMade: 5 });
|
|
expect(markExportJobSucceeded).toHaveBeenCalledWith(expect.objectContaining({ id: "export_job_bullmq_attempt", tenantId: "tenant_1", attemptsMade: 5 }));
|
|
});
|
|
|
|
test("fails export jobs when service is missing", async () => {
|
|
const markExportJobRunning = vi.fn(async () => undefined);
|
|
const markExportJobSucceeded = vi.fn(async () => undefined);
|
|
const markExportJobFailed = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ markExportJobRunning, markExportJobSucceeded, markExportJobFailed });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_noop",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "json",
|
|
requestId: "req_export_noop",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "export.error",
|
|
message: "Export service is not configured.",
|
|
details: {
|
|
exportJobId: "export_job_noop",
|
|
tenantId: "tenant_1",
|
|
exportType: "members",
|
|
format: "json",
|
|
requestId: "req_export_noop",
|
|
attemptsMade: 0,
|
|
status: "failed",
|
|
code: "export_service_missing",
|
|
},
|
|
});
|
|
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_noop", tenantId: "tenant_1", attemptsMade: 0 });
|
|
expect(markExportJobFailed).toHaveBeenCalledWith({
|
|
id: "export_job_noop",
|
|
tenantId: "tenant_1",
|
|
attemptsMade: 0,
|
|
errorName: "ExportHandlerError",
|
|
errorMessage: "Export service is not configured.",
|
|
});
|
|
expect(markExportJobSucceeded).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("marks export jobs failed with redacted scalar diagnostics when service rejects", async () => {
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => {
|
|
throw new Error("export failed token=service-secret authorization=Bearer-secret body=payload-bytes");
|
|
}),
|
|
};
|
|
const markExportJobRunning = vi.fn(async () => undefined);
|
|
const markExportJobFailed = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobFailed });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_failed",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
metadata: { attemptsMade: 1, token: "payload-secret" },
|
|
filters: { apiKey: "payload-secret" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "export.error",
|
|
message: "Export job failed.",
|
|
details: expect.objectContaining({
|
|
status: "failed",
|
|
message: "export failed token=[redacted] authorization=[redacted] body=[redacted]",
|
|
}),
|
|
});
|
|
expect(markExportJobRunning).toHaveBeenCalledWith({ id: "export_job_failed", tenantId: "tenant_1", attemptsMade: 1 });
|
|
expect(markExportJobFailed).toHaveBeenCalledWith({
|
|
id: "export_job_failed",
|
|
tenantId: "tenant_1",
|
|
attemptsMade: 1,
|
|
errorName: "Error",
|
|
errorMessage: "export failed token=[redacted] authorization=[redacted] body=[redacted]",
|
|
});
|
|
expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("payload-secret");
|
|
expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("service-secret");
|
|
expect(JSON.stringify(markExportJobFailed.mock.calls)).not.toContain("Bearer-secret");
|
|
});
|
|
|
|
test("keeps export jobs running until terminal retry failure", async () => {
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => {
|
|
throw new Error("temporary export outage");
|
|
}),
|
|
};
|
|
const markExportJobRunning = vi.fn(async () => undefined);
|
|
const markExportJobFailed = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobRunning, markExportJobFailed });
|
|
const descriptor = buildExportJob({
|
|
exportJobId: "export_job_retrying",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
exportType: "members",
|
|
format: "csv",
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({
|
|
action: "export.error",
|
|
message: "Export job failed.",
|
|
});
|
|
expect(markExportJobFailed).not.toHaveBeenCalled();
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toMatchObject({
|
|
action: "export.error",
|
|
message: "Export job failed.",
|
|
});
|
|
expect(markExportJobFailed).toHaveBeenCalledWith({
|
|
id: "export_job_retrying",
|
|
tenantId: "tenant_1",
|
|
attemptsMade: 3,
|
|
errorName: "Error",
|
|
errorMessage: "temporary export outage",
|
|
});
|
|
});
|
|
|
|
test("rejects malformed export payload kinds", async () => {
|
|
const exportService: TestExportService = {
|
|
runExport: vi.fn(async () => ({ artifactBucket: "unused", artifactKey: "unused", rowCount: 0, expiresAt: new Date() })),
|
|
};
|
|
const markExportJobRunning = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ exportService, markExportJobRunning });
|
|
|
|
await expect(registry[queueNames.exports]({ kind: "export.other", exportJobId: "bad", tenantId: "tenant_1" } as never, {})).rejects.toMatchObject({
|
|
action: "export.error",
|
|
message: "Unsupported export payload kind.",
|
|
details: { code: "invalid_payload" },
|
|
});
|
|
expect(markExportJobRunning).not.toHaveBeenCalled();
|
|
expect(exportService.runExport).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("fails import jobs when service is missing", async () => {
|
|
const markImportJobRunning = vi.fn(async () => undefined);
|
|
const markImportJobSucceeded = vi.fn(async () => undefined);
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_1",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_1/members.csv",
|
|
requestId: "req_import_1",
|
|
metadata: { attemptsMade: 2 },
|
|
});
|
|
const registry = buildHandlerRegistry({ markImportJobRunning, markImportJobSucceeded, markImportJobFailed });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Import service is not configured.",
|
|
details: {
|
|
importJobId: "import_job_1",
|
|
tenantId: "tenant_1",
|
|
importType: "members",
|
|
format: "csv",
|
|
requestId: "req_import_1",
|
|
status: "failed",
|
|
attemptsMade: 2,
|
|
code: "import_service_missing",
|
|
},
|
|
});
|
|
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_1", tenantId: "tenant_1", attemptsMade: 2 });
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_1",
|
|
tenantId: "tenant_1",
|
|
attemptsMade: 2,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: "Import service is not configured.",
|
|
});
|
|
expect(markImportJobSucceeded).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("rejects import jobs whose source key is outside the tenant prefix before scanning", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
|
|
};
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () =>
|
|
scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" })
|
|
),
|
|
};
|
|
const markImportJobRunning = vi.fn(async () => undefined);
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_cross_tenant",
|
|
tenantId: "tenant_1",
|
|
requestedByUserId: "user_1",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_2/members.csv",
|
|
requestId: "req_import_cross_tenant",
|
|
});
|
|
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobFailed });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Import source storage access denied.",
|
|
details: {
|
|
importJobId: "import_job_cross_tenant",
|
|
tenantId: "tenant_1",
|
|
requestId: "req_import_cross_tenant",
|
|
status: "failed",
|
|
code: "source_access_denied",
|
|
reason: "tenant_mismatch",
|
|
},
|
|
});
|
|
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_cross_tenant", tenantId: "tenant_1", attemptsMade: 0 });
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_cross_tenant",
|
|
tenantId: "tenant_1",
|
|
attemptsMade: 0,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: "Import source storage access denied. reason=tenant_mismatch",
|
|
});
|
|
expect(scanService.scanObject).not.toHaveBeenCalled();
|
|
expect(importService.runImport).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("runs import jobs through the injected service and persists duplicate-aware counters", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({
|
|
rowCount: 4,
|
|
acceptedRows: 2,
|
|
rejectedRows: 2,
|
|
duplicateCount: 1,
|
|
validationReport: {
|
|
summary: {
|
|
format: "csv" as const,
|
|
totalRows: 4,
|
|
acceptedRows: 2,
|
|
rejectedRows: 2,
|
|
diagnostics: [],
|
|
duplicateGroups: 1,
|
|
duplicateRows: 2,
|
|
},
|
|
errorReport: { diagnostics: [], rejectedRows: 2, errorReportKey: "reports/import_job_2.json" },
|
|
},
|
|
errorReportBucket: "diagnostics",
|
|
})),
|
|
};
|
|
const markImportJobRunning = vi.fn(async () => undefined);
|
|
const markImportJobSucceeded = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_2",
|
|
tenantId: "tenant_2",
|
|
requestedByUserId: "user_2",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_2/members.csv",
|
|
requestId: "req_import_2",
|
|
});
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" })),
|
|
};
|
|
const events: string[] = [];
|
|
vi.mocked(scanService.scanObject).mockImplementation(async () => {
|
|
events.push("scan");
|
|
return scanResult({ verdict: "clean", bucket: "imports", key: "tenant_2/members.csv", scannedAt: new Date(), engine: "clamav" });
|
|
});
|
|
vi.mocked(importService.runImport).mockImplementation(async () => {
|
|
events.push("import");
|
|
return {
|
|
rowCount: 4,
|
|
acceptedRows: 2,
|
|
rejectedRows: 2,
|
|
duplicateCount: 1,
|
|
validationReport: {
|
|
summary: {
|
|
format: "csv" as const,
|
|
totalRows: 4,
|
|
acceptedRows: 2,
|
|
rejectedRows: 2,
|
|
diagnostics: [],
|
|
duplicateGroups: 1,
|
|
duplicateRows: 2,
|
|
},
|
|
errorReport: { diagnostics: [], rejectedRows: 2, errorReportKey: "reports/import_job_2.json" },
|
|
},
|
|
errorReportBucket: "diagnostics",
|
|
};
|
|
});
|
|
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobSucceeded });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "import.completed",
|
|
details: {
|
|
importJobId: "import_job_2",
|
|
tenantId: "tenant_2",
|
|
importType: "members",
|
|
format: "csv",
|
|
requestId: "req_import_2",
|
|
status: "completed",
|
|
attemptsMade: 0,
|
|
rowCount: 4,
|
|
acceptedRows: 2,
|
|
rejectedRows: 2,
|
|
duplicateCount: 1,
|
|
errorReportBucket: "diagnostics",
|
|
errorReportKey: "reports/import_job_2.json",
|
|
},
|
|
});
|
|
expect(events).toEqual(["scan", "import"]);
|
|
expect(scanService.scanObject).toHaveBeenCalledWith({
|
|
bucket: "imports",
|
|
key: "tenants/tenant_2/members.csv",
|
|
tenantId: "tenant_2",
|
|
uploadId: "import_job_2",
|
|
contentType: undefined,
|
|
metadata: {
|
|
requestId: "req_import_2",
|
|
importJobId: "import_job_2",
|
|
importType: "members",
|
|
format: "csv",
|
|
},
|
|
});
|
|
expect(importService.runImport).toHaveBeenCalledWith(descriptor.payload);
|
|
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_2", tenantId: "tenant_2", attemptsMade: 0 });
|
|
expect(markImportJobSucceeded).toHaveBeenCalledWith({
|
|
id: "import_job_2",
|
|
tenantId: "tenant_2",
|
|
rowCount: 4,
|
|
acceptedRows: 2,
|
|
rejectedRows: 2,
|
|
attemptsMade: 0,
|
|
});
|
|
});
|
|
|
|
test("uses BullMQ attempt metadata for import job persistence when available", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({ rowCount: 10, acceptedRows: 9, rejectedRows: 1, duplicateCount: 0 })),
|
|
};
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () =>
|
|
scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_2/imports/import_job_bullmq_attempt.csv", scannedAt: new Date() })
|
|
),
|
|
};
|
|
const markImportJobRunning = vi.fn(async () => undefined);
|
|
const markImportJobSucceeded = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning, markImportJobSucceeded });
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_bullmq_attempt",
|
|
tenantId: "tenant_2",
|
|
requestedByUserId: "user_2",
|
|
importType: "members",
|
|
format: "csv",
|
|
requestId: "req_import_bullmq_attempt",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_2/imports/import_job_bullmq_attempt.csv",
|
|
metadata: { attemptsMade: 1 },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 3 })).resolves.toMatchObject({
|
|
action: "import.completed",
|
|
details: { attemptsMade: 4 },
|
|
});
|
|
expect(markImportJobRunning).toHaveBeenCalledWith({ id: "import_job_bullmq_attempt", tenantId: "tenant_2", attemptsMade: 4 });
|
|
expect(markImportJobSucceeded).toHaveBeenCalledWith(expect.objectContaining({ id: "import_job_bullmq_attempt", tenantId: "tenant_2", attemptsMade: 4 }));
|
|
});
|
|
|
|
test("persists failed import diagnostics and throws redacted handler errors", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => {
|
|
throw new Error("import failed token=secret-token body=row-secret password=hunter2");
|
|
}),
|
|
};
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_3",
|
|
tenantId: "tenant_3",
|
|
requestedByUserId: "user_3",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_3/members.csv",
|
|
requestId: "req_import_3",
|
|
metadata: { attemptsMade: 1 },
|
|
});
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenant_3/members.csv", scannedAt: new Date(), engine: "clamav" })),
|
|
};
|
|
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Import job failed.",
|
|
details: {
|
|
importJobId: "import_job_3",
|
|
tenantId: "tenant_3",
|
|
requestId: "req_import_3",
|
|
status: "failed",
|
|
attemptsMade: 1,
|
|
message: "import failed token=[redacted] body=[redacted] password=[redacted]",
|
|
},
|
|
});
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("secret-token");
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_3",
|
|
tenantId: "tenant_3",
|
|
attemptsMade: 1,
|
|
errorName: "Error",
|
|
errorMessage: "import failed token=[redacted] body=[redacted] password=[redacted]",
|
|
});
|
|
});
|
|
|
|
test("keeps import jobs running until terminal retry failure", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => {
|
|
throw new Error("temporary import outage");
|
|
}),
|
|
};
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "imports", key: "tenants/tenant_3/members.csv", scannedAt: new Date() })),
|
|
};
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_retrying",
|
|
tenantId: "tenant_3",
|
|
requestedByUserId: "user_3",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_3/members.csv",
|
|
});
|
|
const registry = buildHandlerRegistry({ importService, scanService, markImportJobRunning: vi.fn(async () => undefined), markImportJobFailed });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Import job failed.",
|
|
});
|
|
expect(markImportJobFailed).not.toHaveBeenCalled();
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {}, { attemptsMade: 2, attemptsLimit: 3 })).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Import job failed.",
|
|
});
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_retrying",
|
|
tenantId: "tenant_3",
|
|
attemptsMade: 3,
|
|
errorName: "Error",
|
|
errorMessage: "temporary import outage",
|
|
});
|
|
});
|
|
|
|
test("rejects import jobs with configured import service when scan service is missing", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
|
|
};
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const markImportJobSucceeded = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_missing_scan",
|
|
tenantId: "tenant_scan",
|
|
requestedByUserId: "user_scan",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_scan/members.csv",
|
|
requestId: "req_missing_scan",
|
|
metadata: { attemptsMade: 3 },
|
|
});
|
|
const registry = buildHandlerRegistry({
|
|
importService,
|
|
markImportJobRunning: vi.fn(async () => undefined),
|
|
markImportJobFailed,
|
|
markImportJobSucceeded,
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Import source scan failed.",
|
|
details: {
|
|
importJobId: "import_job_missing_scan",
|
|
tenantId: "tenant_scan",
|
|
requestId: "req_missing_scan",
|
|
status: "failed",
|
|
attemptsMade: 3,
|
|
scan: {
|
|
verdict: "error",
|
|
code: "scan_service_missing",
|
|
message: "Import scan service is not configured.",
|
|
},
|
|
},
|
|
});
|
|
expect(importService.runImport).not.toHaveBeenCalled();
|
|
expect(markImportJobSucceeded).not.toHaveBeenCalled();
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_missing_scan",
|
|
tenantId: "tenant_scan",
|
|
attemptsMade: 3,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage:
|
|
"Import source scan failed. verdict=error code=scan_service_missing requestId=req_missing_scan message=Import scan service is not configured.",
|
|
});
|
|
});
|
|
|
|
test("rejects infected import source scans without running the import", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
|
|
};
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () =>
|
|
scanResult({
|
|
verdict: "infected",
|
|
bucket: "imports",
|
|
key: "tenant_bad/members.csv",
|
|
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
|
|
engine: "clamav",
|
|
signature: "Eicar-Test-Signature",
|
|
quarantineKey: "quarantine/tenant_bad/members.csv",
|
|
})
|
|
),
|
|
};
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const markImportJobSucceeded = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_infected",
|
|
tenantId: "tenant_bad",
|
|
requestedByUserId: "user_bad",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_bad/members.csv",
|
|
requestId: "req_infected_scan",
|
|
});
|
|
const registry = buildHandlerRegistry({
|
|
importService,
|
|
scanService,
|
|
markImportJobRunning: vi.fn(async () => undefined),
|
|
markImportJobFailed,
|
|
markImportJobSucceeded,
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Import source scan failed.",
|
|
details: {
|
|
importJobId: "import_job_infected",
|
|
tenantId: "tenant_bad",
|
|
requestId: "req_infected_scan",
|
|
status: "failed",
|
|
attemptsMade: 0,
|
|
scan: {
|
|
verdict: "infected",
|
|
code: "source_infected",
|
|
engine: "clamav",
|
|
signature: "Eicar-Test-Signature",
|
|
quarantineKey: "quarantine/tenant_bad/members.csv",
|
|
message: "Import source object failed malware scan.",
|
|
},
|
|
},
|
|
});
|
|
expect(importService.runImport).not.toHaveBeenCalled();
|
|
expect(markImportJobSucceeded).not.toHaveBeenCalled();
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_infected",
|
|
tenantId: "tenant_bad",
|
|
attemptsMade: 0,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage:
|
|
"Import source scan failed. verdict=infected code=source_infected requestId=req_infected_scan message=Import source object failed malware scan.",
|
|
});
|
|
});
|
|
|
|
test("rejects scanner errors for import sources with redacted diagnostics", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
|
|
};
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () =>
|
|
scanResult({
|
|
verdict: "error",
|
|
bucket: "imports",
|
|
key: "tenant_error/members.csv",
|
|
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
|
|
error: {
|
|
code: "scanner_failed",
|
|
message: "scanner failed token=top-secret body=file-bytes authorization=Bearer-secret",
|
|
upstream: {
|
|
name: "ScannerError",
|
|
message: "upstream password=hunter2 body=row-secret",
|
|
status: 503,
|
|
requestId: "scan_req_error",
|
|
},
|
|
},
|
|
})
|
|
),
|
|
};
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_scan_error",
|
|
tenantId: "tenant_error",
|
|
requestedByUserId: "user_error",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_error/members.csv",
|
|
requestId: "req_scan_error",
|
|
metadata: { attemptsMade: 5 },
|
|
});
|
|
const registry = buildHandlerRegistry({
|
|
importService,
|
|
scanService,
|
|
markImportJobRunning: vi.fn(async () => undefined),
|
|
markImportJobFailed,
|
|
markImportJobSucceeded: vi.fn(async () => undefined),
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "import.error",
|
|
details: {
|
|
importJobId: "import_job_scan_error",
|
|
tenantId: "tenant_error",
|
|
requestId: "req_scan_error",
|
|
status: "failed",
|
|
attemptsMade: 5,
|
|
scan: {
|
|
verdict: "error",
|
|
code: "scanner_failed",
|
|
message: "scanner failed token=[redacted] body=[redacted] authorization=[redacted]",
|
|
upstream: {
|
|
name: "ScannerError",
|
|
message: "upstream password=[redacted] body=[redacted]",
|
|
status: 503,
|
|
requestId: "scan_req_error",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("top-secret");
|
|
expect(importService.runImport).not.toHaveBeenCalled();
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_scan_error",
|
|
tenantId: "tenant_error",
|
|
attemptsMade: 5,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage:
|
|
"Import source scan failed. verdict=error code=scanner_failed requestId=req_scan_error message=scanner failed token=[redacted] body=[redacted] authorization=[redacted]",
|
|
});
|
|
expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("top-secret");
|
|
expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("Bearer-secret");
|
|
expect(JSON.stringify(markImportJobFailed.mock.calls)).not.toContain("hunter2");
|
|
});
|
|
|
|
test("rejects scanner promise rejections for import sources with redacted diagnostics", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
|
|
};
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () => {
|
|
throw new Error("scan rejected token=reject-secret body=file-bytes password=hunter2");
|
|
}),
|
|
};
|
|
const markImportJobFailed = vi.fn(async () => undefined);
|
|
const descriptor = buildImportJob({
|
|
importJobId: "import_job_scan_rejected",
|
|
tenantId: "tenant_rejected",
|
|
requestedByUserId: "user_rejected",
|
|
importType: "members",
|
|
format: "csv",
|
|
sourceBucket: "imports",
|
|
sourceKey: "tenants/tenant_rejected/members.csv",
|
|
requestId: "req_scan_rejected",
|
|
});
|
|
const registry = buildHandlerRegistry({
|
|
importService,
|
|
scanService,
|
|
markImportJobRunning: vi.fn(async () => undefined),
|
|
markImportJobFailed,
|
|
markImportJobSucceeded: vi.fn(async () => undefined),
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "import.error",
|
|
details: {
|
|
importJobId: "import_job_scan_rejected",
|
|
tenantId: "tenant_rejected",
|
|
requestId: "req_scan_rejected",
|
|
status: "failed",
|
|
attemptsMade: 0,
|
|
scan: {
|
|
verdict: "error",
|
|
code: "scanner_rejected",
|
|
message: "scan rejected token=[redacted] body=[redacted] password=[redacted]",
|
|
},
|
|
},
|
|
});
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("reject-secret");
|
|
expect(importService.runImport).not.toHaveBeenCalled();
|
|
expect(markImportJobFailed).toHaveBeenCalledWith({
|
|
id: "import_job_scan_rejected",
|
|
tenantId: "tenant_rejected",
|
|
attemptsMade: 0,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage:
|
|
"Import source scan failed. verdict=error code=scanner_rejected requestId=req_scan_rejected message=scan rejected token=[redacted] body=[redacted] password=[redacted]",
|
|
});
|
|
});
|
|
|
|
test("rejects malformed import payload kinds without leaking source secrets", async () => {
|
|
const importService: TestImportService = {
|
|
runImport: vi.fn(async () => ({ rowCount: 0, acceptedRows: 0, rejectedRows: 0, duplicateCount: 0 })),
|
|
};
|
|
const markImportJobRunning = vi.fn(async () => undefined);
|
|
const registry = buildHandlerRegistry({ importService, markImportJobRunning });
|
|
|
|
await expect(
|
|
registry[queueNames.imports]({ kind: "import.other", importJobId: "bad", tenantId: "tenant_1", sourceKey: "secret/body.csv" } as never, {})
|
|
).rejects.toMatchObject({
|
|
action: "import.error",
|
|
message: "Unsupported import payload kind.",
|
|
details: { code: "invalid_payload" },
|
|
});
|
|
await expect(
|
|
registry[queueNames.imports]({ kind: "import.other", importJobId: "bad", tenantId: "tenant_1", sourceKey: "secret/body.csv" } as never, {})
|
|
).rejects.not.toThrow("secret/body.csv");
|
|
expect(markImportJobRunning).not.toHaveBeenCalled();
|
|
expect(importService.runImport).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("runs clean file scan jobs through the injected scan service", async () => {
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () =>
|
|
scanResult({
|
|
verdict: "clean",
|
|
bucket: "uploads",
|
|
key: "tenant_1/file.pdf",
|
|
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
|
|
engine: "clamav",
|
|
})
|
|
),
|
|
};
|
|
const registry = buildHandlerRegistry({ scanService });
|
|
const descriptor = buildFileScanJob({
|
|
bucket: "uploads",
|
|
key: "tenant_1/file.pdf",
|
|
tenantId: "tenant_1",
|
|
uploadId: "upload_1",
|
|
contentType: "application/pdf",
|
|
metadata: { requestId: "req_scan_clean" },
|
|
});
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "scan.clean",
|
|
details: {
|
|
verdict: "clean",
|
|
bucket: "uploads",
|
|
key: "tenant_1/file.pdf",
|
|
uploadId: "upload_1",
|
|
engine: "clamav",
|
|
},
|
|
});
|
|
expect(scanService.scanObject).toHaveBeenCalledWith({
|
|
bucket: "uploads",
|
|
key: "tenant_1/file.pdf",
|
|
tenantId: "tenant_1",
|
|
uploadId: "upload_1",
|
|
contentType: "application/pdf",
|
|
metadata: { requestId: "req_scan_clean" },
|
|
});
|
|
});
|
|
|
|
test("returns infected scan action with signature and quarantine key", async () => {
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () =>
|
|
scanResult({
|
|
verdict: "infected",
|
|
bucket: "uploads",
|
|
key: "tenant_1/infected.exe",
|
|
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
|
|
engine: "clamav",
|
|
signature: "Eicar-Test-Signature",
|
|
quarantineKey: "quarantine/tenant_1/infected.exe",
|
|
})
|
|
),
|
|
};
|
|
const registry = buildHandlerRegistry({ scanService });
|
|
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/infected.exe", uploadId: "upload_bad" });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
|
handled: true,
|
|
action: "scan.infected",
|
|
details: {
|
|
verdict: "infected",
|
|
bucket: "uploads",
|
|
key: "tenant_1/infected.exe",
|
|
uploadId: "upload_bad",
|
|
engine: "clamav",
|
|
signature: "Eicar-Test-Signature",
|
|
quarantineKey: "quarantine/tenant_1/infected.exe",
|
|
},
|
|
});
|
|
});
|
|
|
|
test("writes the verdict back onto File.scanStatus when fileId + writer are present", async () => {
|
|
const cleanScan: TestScanService = {
|
|
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav" })),
|
|
};
|
|
const infectedScan: TestScanService = {
|
|
scanObject: vi.fn(async () => scanResult({ verdict: "infected", bucket: "uploads", key: "tenant_1/bad.exe", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav", signature: "Eicar-Test-Signature" })),
|
|
};
|
|
const fileScanStatusWriter = { recordVerdict: vi.fn(async () => undefined) };
|
|
|
|
const cleanRegistry = buildHandlerRegistry({ scanService: cleanScan, fileScanStatusWriter });
|
|
const cleanJob = buildFileScanJob({ bucket: "uploads", key: "tenant_1/file.pdf", tenantId: "tenant_1", fileId: "file_1" });
|
|
await cleanRegistry[cleanJob.queueName](cleanJob.payload, {});
|
|
expect(fileScanStatusWriter.recordVerdict).toHaveBeenCalledWith({ fileId: "file_1", tenantId: "tenant_1", verdict: "clean" });
|
|
|
|
const infectedRegistry = buildHandlerRegistry({ scanService: infectedScan, fileScanStatusWriter });
|
|
const infectedJob = buildFileScanJob({ bucket: "uploads", key: "tenant_1/bad.exe", tenantId: "tenant_1", fileId: "file_2" });
|
|
await infectedRegistry[infectedJob.queueName](infectedJob.payload, {});
|
|
expect(fileScanStatusWriter.recordVerdict).toHaveBeenCalledWith({ fileId: "file_2", tenantId: "tenant_1", verdict: "infected" });
|
|
});
|
|
|
|
test("skips the verdict write-back when the job carries no fileId", async () => {
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "tenant_1/file.pdf", scannedAt: new Date("2026-05-08T00:00:00.000Z"), engine: "clamav" })),
|
|
};
|
|
const fileScanStatusWriter = { recordVerdict: vi.fn(async () => undefined) };
|
|
const registry = buildHandlerRegistry({ scanService, fileScanStatusWriter });
|
|
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/file.pdf", tenantId: "tenant_1" });
|
|
|
|
await registry[descriptor.queueName](descriptor.payload, {});
|
|
expect(fileScanStatusWriter.recordVerdict).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("rejects storage error scan verdicts with redacted diagnostics", async () => {
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () =>
|
|
scanResult({
|
|
verdict: "error",
|
|
bucket: "uploads",
|
|
key: "tenant_1/error.bin",
|
|
scannedAt: new Date("2026-05-08T00:00:00.000Z"),
|
|
error: {
|
|
code: "scanner_failed",
|
|
message: "Storage scanner failed while scanning object.",
|
|
upstream: {
|
|
name: "ScannerError",
|
|
message: "scanner failed token=top-secret body=file-bytes credential=aws-secret",
|
|
status: 503,
|
|
requestId: "scan_req_1",
|
|
},
|
|
},
|
|
})
|
|
),
|
|
};
|
|
const registry = buildHandlerRegistry({ scanService });
|
|
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/error.bin", uploadId: "upload_error" });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "scan.error",
|
|
details: {
|
|
verdict: "error",
|
|
bucket: "uploads",
|
|
key: "tenant_1/error.bin",
|
|
uploadId: "upload_error",
|
|
code: "scanner_failed",
|
|
message: "Storage scanner failed while scanning object.",
|
|
upstream: {
|
|
name: "ScannerError",
|
|
message: "scanner failed token=[redacted] body=[redacted] credential=[redacted]",
|
|
status: 503,
|
|
requestId: "scan_req_1",
|
|
},
|
|
},
|
|
});
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("top-secret");
|
|
});
|
|
|
|
test("rejects scanner service rejections without leaking secret-bearing messages", async () => {
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () => {
|
|
throw new Error("scanner rejected authorization=Bearer-secret body=file-bytes password=hunter2");
|
|
}),
|
|
};
|
|
const registry = buildHandlerRegistry({ scanService });
|
|
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/rejected.bin", uploadId: "upload_rejected" });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "scan.error",
|
|
details: {
|
|
verdict: "error",
|
|
bucket: "uploads",
|
|
key: "tenant_1/rejected.bin",
|
|
uploadId: "upload_rejected",
|
|
code: "scanner_rejected",
|
|
message: "scanner rejected authorization=[redacted] body=[redacted] password=[redacted]",
|
|
},
|
|
});
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.not.toThrow("Bearer-secret");
|
|
});
|
|
|
|
test("rejects file scan jobs when scan service is missing", async () => {
|
|
const registry = buildHandlerRegistry();
|
|
const descriptor = buildFileScanJob({ bucket: "uploads", key: "tenant_1/missing-service.bin", uploadId: "upload_missing" });
|
|
|
|
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
|
action: "scan.error",
|
|
message: "File scan service is not configured.",
|
|
details: {
|
|
verdict: "error",
|
|
bucket: "uploads",
|
|
key: "tenant_1/missing-service.bin",
|
|
uploadId: "upload_missing",
|
|
code: "scan_service_missing",
|
|
},
|
|
});
|
|
});
|
|
|
|
test("rejects malformed file scan payload kinds", async () => {
|
|
const scanService: TestScanService = {
|
|
scanObject: vi.fn(async () => scanResult({ verdict: "clean", bucket: "uploads", key: "unused", scannedAt: new Date() })),
|
|
};
|
|
const registry = buildHandlerRegistry({ scanService });
|
|
|
|
await expect(registry[queueNames.fileScan]({ kind: "scan.other", bucket: "uploads", key: "bad" } as never, {})).rejects.toMatchObject({
|
|
action: "scan.error",
|
|
message: "Unsupported file scan payload kind.",
|
|
details: { verdict: "error", code: "invalid_payload" },
|
|
});
|
|
expect(scanService.scanObject).not.toHaveBeenCalled();
|
|
});
|
|
|
|
// Phase 06 #21 / F-015 — plugin-invoke dispatch.
|
|
describe("plugin-invoke handler (F-015)", () => {
|
|
const samplePayload = {
|
|
kind: "plugin.invoke" as const,
|
|
tenantId: "tenant_abc",
|
|
pluginId: "com.acme.demo",
|
|
version: "1.0.0",
|
|
hookName: "tenant.created",
|
|
input: { tenantId: "tenant_abc", slug: "acme", ownerEmail: "owner@example.com", createdAt: "2026-05-28T10:00:00.000Z" },
|
|
metadata: { logicalHookName: "tenant.created" },
|
|
};
|
|
|
|
test("rejects malformed queue payloads before invoking plugin deps", async () => {
|
|
const errorLogs: Array<{ fields: Record<string, unknown>; msg: string }> = [];
|
|
const pluginInvokeDeps = {
|
|
loadPlugin: vi.fn(),
|
|
loadCapabilityGrants: vi.fn(),
|
|
loadBinary: vi.fn(),
|
|
verifySignature: vi.fn(),
|
|
consumeRateLimit: vi.fn(),
|
|
invokeInSandbox: vi.fn(),
|
|
recordAudit: vi.fn(),
|
|
trustedPublicKeysHex: [],
|
|
now: () => 1_000_000,
|
|
};
|
|
const registry = buildHandlerRegistry({
|
|
pluginInvokeDeps: pluginInvokeDeps as never,
|
|
logger: {
|
|
info: () => undefined,
|
|
error: (fields, msg) => errorLogs.push({ fields, msg: msg ?? "" }),
|
|
},
|
|
});
|
|
|
|
await expect(
|
|
registry[queueNames.pluginInvoke](
|
|
{
|
|
kind: "plugin.invoke",
|
|
tenantId: "tenant_abc",
|
|
pluginId: "com.acme.demo",
|
|
version: 1,
|
|
input: { secret: "not logged" },
|
|
} as never,
|
|
{} as never
|
|
)
|
|
).rejects.toMatchObject({
|
|
action: "plugin.invoke.invalid_payload",
|
|
message: "Invalid plugin-invoke queue payload.",
|
|
details: {
|
|
queueName: queueNames.pluginInvoke,
|
|
code: "invalid_payload",
|
|
invalidFields: ["version", "hookName"],
|
|
},
|
|
});
|
|
|
|
expect(pluginInvokeDeps.loadPlugin).not.toHaveBeenCalled();
|
|
expect(pluginInvokeDeps.invokeInSandbox).not.toHaveBeenCalled();
|
|
expect(errorLogs).toHaveLength(1);
|
|
expect(errorLogs[0]).toMatchObject({
|
|
msg: "Invalid plugin-invoke queue payload.",
|
|
fields: {
|
|
queueName: queueNames.pluginInvoke,
|
|
code: "invalid_payload",
|
|
invalidFields: ["version", "hookName"],
|
|
payload: {
|
|
kind: "plugin.invoke",
|
|
tenantId: "tenant_abc",
|
|
pluginId: "com.acme.demo",
|
|
version: "number",
|
|
hookName: "undefined",
|
|
},
|
|
},
|
|
});
|
|
expect(JSON.stringify(errorLogs[0])).not.toContain("not logged");
|
|
});
|
|
|
|
test("throws when pluginInvokeDeps is not configured (fail-loud, no silent ack)", async () => {
|
|
const errorLogs: Array<{ fields: Record<string, unknown>; msg: string }> = [];
|
|
const registry = buildHandlerRegistry({
|
|
logger: {
|
|
info: () => undefined,
|
|
error: (fields, msg) => errorLogs.push({ fields, msg: msg ?? "" }),
|
|
},
|
|
});
|
|
await expect(registry[queueNames.pluginInvoke](samplePayload, {} as never)).rejects.toThrow(/pluginInvokeDeps is not configured/);
|
|
expect(errorLogs).toHaveLength(1);
|
|
expect(errorLogs[0]!.fields).toMatchObject({
|
|
queue: queueNames.pluginInvoke,
|
|
pluginId: samplePayload.pluginId,
|
|
hookName: samplePayload.hookName,
|
|
});
|
|
});
|
|
|
|
test("dispatches to handlePluginInvokeJob and maps success outcome", async () => {
|
|
const pluginInvokeDeps = {
|
|
loadPlugin: vi.fn(async () => ({
|
|
tenantId: samplePayload.tenantId,
|
|
pluginId: samplePayload.pluginId,
|
|
version: samplePayload.version,
|
|
manifest: { capabilities: [], hooks: [{ name: "tenant.created", export: "tenant.created" }], wasmSha256: "x", signedBy: "y" } as never,
|
|
wasmSha256: "x",
|
|
signaturePublicKey: "y",
|
|
enabled: true,
|
|
})),
|
|
loadCapabilityGrants: vi.fn(async () => [{ capabilityName: "hook:tenant.created", scopeJson: null }]),
|
|
loadBinary: vi.fn(async () => new Uint8Array([0])),
|
|
verifySignature: vi.fn(() => ({ ok: true as const })),
|
|
consumeRateLimit: vi.fn(async () => ({ allowed: true, remaining: 99 })),
|
|
invokeInSandbox: vi.fn(async () => ({ ok: true as const, output: "ok", fuelUsed: 42 })),
|
|
recordAudit: vi.fn(async () => undefined),
|
|
trustedPublicKeysHex: [],
|
|
now: () => 1_000_000,
|
|
};
|
|
const registry = buildHandlerRegistry({ pluginInvokeDeps: pluginInvokeDeps as never });
|
|
const result = await registry[queueNames.pluginInvoke](samplePayload, {} as never);
|
|
expect(result).toMatchObject({
|
|
handled: true,
|
|
action: "plugin.invoke.completed",
|
|
details: { pluginId: samplePayload.pluginId, hookName: samplePayload.hookName, fuelUsed: 42 },
|
|
});
|
|
expect(pluginInvokeDeps.invokeInSandbox).toHaveBeenCalledTimes(1);
|
|
expect(pluginInvokeDeps.invokeInSandbox).toHaveBeenCalledWith(
|
|
expect.objectContaining({ capabilityGrants: [{ capabilityName: "hook:tenant.created", scopeJson: null }] })
|
|
);
|
|
expect(pluginInvokeDeps.recordAudit).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
test("maps signature_invalid outcome to plugin.invoke.failed", async () => {
|
|
const pluginInvokeDeps = {
|
|
loadPlugin: vi.fn(async () => ({
|
|
tenantId: samplePayload.tenantId,
|
|
pluginId: samplePayload.pluginId,
|
|
version: samplePayload.version,
|
|
manifest: { capabilities: [], hooks: [{ name: "tenant.created", export: "tenant.created" }], wasmSha256: "x", signedBy: "y" } as never,
|
|
wasmSha256: "x",
|
|
signaturePublicKey: "y",
|
|
enabled: true,
|
|
})),
|
|
loadCapabilityGrants: vi.fn(async () => [{ capabilityName: "hook:tenant.created", scopeJson: null }]),
|
|
loadBinary: vi.fn(async () => new Uint8Array([0])),
|
|
verifySignature: vi.fn(() => ({ ok: false as const, reason: "binary_hash_mismatch" as const })),
|
|
consumeRateLimit: vi.fn(async () => ({ allowed: true, remaining: 99 })),
|
|
invokeInSandbox: vi.fn(),
|
|
recordAudit: vi.fn(async () => undefined),
|
|
trustedPublicKeysHex: [],
|
|
now: () => 1_000_000,
|
|
};
|
|
const registry = buildHandlerRegistry({ pluginInvokeDeps: pluginInvokeDeps as never });
|
|
const result = await registry[queueNames.pluginInvoke](samplePayload, {} as never);
|
|
expect(result).toMatchObject({
|
|
handled: true,
|
|
action: "plugin.invoke.failed",
|
|
details: { reason: "signature_invalid", message: "binary_hash_mismatch" },
|
|
});
|
|
expect(pluginInvokeDeps.invokeInSandbox).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
});
|