Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+192
-23
@@ -7,6 +7,9 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { bootstrapOutboxDispatcher, type OutboxDispatcherBootstrapDeps } from "./bootstrap.js";
|
||||
import type { DispatcherLogger } from "./dispatcher.js";
|
||||
import type { OutboxEvent } from "@hivegrid/db/outbox";
|
||||
import { hashPortalSetupMailContent, PORTAL_SETUP_TOKEN_PLACEHOLDER } from "@hivegrid/db/portal-setup-mail-contract.server";
|
||||
import { parsePlatformLeadCapturedOutboxJob } from "../jobs/platform-lead-captured-outbox.js";
|
||||
import { parseProductAcquisitionProviderSideEffectJob } from "../jobs/product-acquisition-provider-side-effect.js";
|
||||
|
||||
const sourceDir = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
@@ -19,15 +22,23 @@ const queueRuntimeMocks = vi.hoisted(() => ({
|
||||
})),
|
||||
}));
|
||||
|
||||
const portalTerminalizationMocks = vi.hoisted(() => ({
|
||||
terminalize: vi.fn(async () => ({ portalTokenId: "portal_token_1", state: "invalidated" as const })),
|
||||
}));
|
||||
|
||||
vi.mock("@hivegrid/queue/runtime", () => ({
|
||||
createQueue: queueRuntimeMocks.createQueue,
|
||||
}));
|
||||
|
||||
vi.mock("../security/portal-setup-invitation-terminalization.js", () => ({
|
||||
terminalizePortalSetupInvitationDelivery: portalTerminalizationMocks.terminalize,
|
||||
}));
|
||||
|
||||
function createPrismaStub(): OutboxDispatcherBootstrapDeps["prisma"] {
|
||||
return {
|
||||
$queryRaw: vi.fn(async () => []) as unknown as OutboxDispatcherBootstrapDeps["prisma"]["$queryRaw"],
|
||||
outboxEvent: {
|
||||
update: vi.fn(async () => ({}) as unknown),
|
||||
updateMany: vi.fn(async () => ({ count: 1 })),
|
||||
},
|
||||
} as unknown as OutboxDispatcherBootstrapDeps["prisma"];
|
||||
}
|
||||
@@ -46,6 +57,7 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
queueRuntimeMocks.add.mockClear();
|
||||
queueRuntimeMocks.close.mockClear();
|
||||
queueRuntimeMocks.createQueue.mockClear();
|
||||
portalTerminalizationMocks.terminalize.mockClear();
|
||||
});
|
||||
|
||||
it("returns disabled when OUTBOX_DISPATCHER_ENABLED is unset", async () => {
|
||||
@@ -108,10 +120,7 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
);
|
||||
expect(result.status).toBe("started");
|
||||
expect(result.handle).toBeDefined();
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ component: "outbox-dispatcher", pollIntervalMs: 10000 }),
|
||||
expect.any(String)
|
||||
);
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ component: "outbox-dispatcher", pollIntervalMs: 10000 }), expect.any(String));
|
||||
await result.handle?.close();
|
||||
});
|
||||
|
||||
@@ -127,20 +136,14 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
);
|
||||
|
||||
expect(result.status).toBe("started");
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ component: "outbox-dispatcher", dispatchConcurrency: 7 }),
|
||||
expect.any(String)
|
||||
);
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ component: "outbox-dispatcher", dispatchConcurrency: 7 }), expect.any(String));
|
||||
await result.handle?.close();
|
||||
});
|
||||
|
||||
it("throws when enabled but no redis URL and no enqueue override", async () => {
|
||||
await expect(
|
||||
bootstrapOutboxDispatcher(
|
||||
{ OUTBOX_DISPATCHER_ENABLED: "true" },
|
||||
{ prisma: createPrismaStub(), logger: createLogger() }
|
||||
)
|
||||
).rejects.toThrow(/BULLMQ_REDIS_URL or REDIS_URL/);
|
||||
await expect(bootstrapOutboxDispatcher({ OUTBOX_DISPATCHER_ENABLED: "true" }, { prisma: createPrismaStub(), logger: createLogger() })).rejects.toThrow(
|
||||
/BULLMQ_REDIS_URL or REDIS_URL/
|
||||
);
|
||||
});
|
||||
|
||||
it("wires the default Redis enqueue path without forwarding raw outbox priority", async () => {
|
||||
@@ -175,9 +178,11 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
payload: { workflowId: "workflow-1" },
|
||||
dispatchKey: "workflow:outbox-1",
|
||||
retries: 0,
|
||||
deliveryGeneration: 0,
|
||||
lockedUntil: event.lockedUntil?.toISOString() ?? null,
|
||||
},
|
||||
{
|
||||
jobId: "outbox-outbox-1",
|
||||
jobId: "outbox-outbox-1-0",
|
||||
attempts: 5,
|
||||
backoff: { type: "exponential", delay: 15_000 },
|
||||
removeOnComplete: 1_000,
|
||||
@@ -188,8 +193,13 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
// BullMQ rejects custom job ids containing ":" at runtime, which the mocked
|
||||
// queue here would not surface — guard the separator explicitly.
|
||||
expect((queueRuntimeMocks.add.mock.calls[0][2] as { jobId: string }).jobId).not.toContain(":");
|
||||
expect(prisma.outboxEvent.update).toHaveBeenCalledWith({
|
||||
where: { id: "outbox-1" },
|
||||
expect(prisma.outboxEvent.updateMany).toHaveBeenCalledWith({
|
||||
where: {
|
||||
id: "outbox-1",
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: 0,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: expect.objectContaining({
|
||||
status: "DONE",
|
||||
lastError: null,
|
||||
@@ -223,6 +233,56 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
expect(metrics.recordDispatch).toHaveBeenCalledWith("workflow.trigger.event", "ok");
|
||||
});
|
||||
|
||||
it("wires terminal portal setup failures through the atomic production terminalizer", async () => {
|
||||
vi.useFakeTimers();
|
||||
const event = createPortalSetupOutboxEvent({ retries: 1, deliveryGeneration: 3 });
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([event]);
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
OUTBOX_DISPATCHER_ENABLED: "true",
|
||||
OUTBOX_DISPATCH_INTERVAL_MS: "5",
|
||||
OUTBOX_MAX_RETRIES: "1",
|
||||
},
|
||||
{
|
||||
prisma,
|
||||
enqueue: vi.fn(async () => Promise.reject(new Error(`queue rejected ${JSON.stringify(event.payload)}`))),
|
||||
logger: createLogger(),
|
||||
}
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await result.handle?.close();
|
||||
|
||||
expect(portalTerminalizationMocks.terminalize).toHaveBeenCalledOnce();
|
||||
expect(portalTerminalizationMocks.terminalize).toHaveBeenCalledWith(
|
||||
{
|
||||
tenantId: "tenant-1",
|
||||
payload: event.payload,
|
||||
outboxFailure: {
|
||||
eventId: event.id,
|
||||
deliveryGeneration: 3,
|
||||
lockedUntil: event.lockedUntil,
|
||||
redactedPayload: expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
source: "portal_setup_invite",
|
||||
outboxEventId: event.id,
|
||||
dispatchKey: event.dispatchKey,
|
||||
secretBodyRedacted: true,
|
||||
portalSetupDeliveryStatus: "rotation_required",
|
||||
}),
|
||||
}),
|
||||
retries: 2,
|
||||
lastError: expect.not.stringContaining("encrypted-bootstrap-portal-capability"),
|
||||
processedAt: expect.any(Date),
|
||||
},
|
||||
},
|
||||
prisma
|
||||
);
|
||||
expect(prisma.outboxEvent.updateMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("closes cached BullMQ queues when the default enqueue handle closes", async () => {
|
||||
vi.useFakeTimers();
|
||||
const prisma = createPrismaStub();
|
||||
@@ -250,9 +310,7 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
it("uses deterministic BullMQ job ids so redelivered outbox events dedupe", async () => {
|
||||
vi.useFakeTimers();
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw)
|
||||
.mockResolvedValueOnce([createOutboxEvent()])
|
||||
.mockResolvedValueOnce([createOutboxEvent()]);
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([createOutboxEvent()]).mockResolvedValueOnce([createOutboxEvent()]);
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
@@ -272,11 +330,86 @@ describe("bootstrapOutboxDispatcher", () => {
|
||||
|
||||
expect(queueRuntimeMocks.add).toHaveBeenCalledTimes(2);
|
||||
expect(queueRuntimeMocks.add.mock.calls.map((call) => call[2])).toEqual([
|
||||
expect.objectContaining({ jobId: "outbox-outbox-1" }),
|
||||
expect.objectContaining({ jobId: "outbox-outbox-1" }),
|
||||
expect.objectContaining({ jobId: "outbox-outbox-1-0" }),
|
||||
expect.objectContaining({ jobId: "outbox-outbox-1-0" }),
|
||||
]);
|
||||
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps provider failures inspectable while replay generations avoid retained-job dedupe", async () => {
|
||||
vi.useFakeTimers();
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([
|
||||
createOutboxEvent({
|
||||
eventType: "billing.product-acquisition.provider-side-effect",
|
||||
dispatchKey: "billing-product:acquisition_1:1:active:abcdef1234567890abcd",
|
||||
payload: {
|
||||
acquisitionId: "acquisition_1",
|
||||
tenantId: "tenant-1",
|
||||
productKind: "one_time_setup",
|
||||
acquisitionStatus: "active",
|
||||
providerAttempt: 1,
|
||||
providerPaymentId: "tr_payment_1",
|
||||
invoiceExternalId: "invoice_1",
|
||||
lagoSubscriptionId: null,
|
||||
action: "mark_lago_invoice_succeeded",
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
OUTBOX_DISPATCHER_ENABLED: "true",
|
||||
OUTBOX_DISPATCH_INTERVAL_MS: "5",
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
},
|
||||
{ prisma, logger: createLogger() }
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await result.handle?.close();
|
||||
|
||||
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledWith("outbox.billing-product-provider", expect.any(Object));
|
||||
expect(queueRuntimeMocks.add).toHaveBeenCalledWith(
|
||||
"billing.product-acquisition.provider-side-effect",
|
||||
expect.any(Object),
|
||||
expect.objectContaining({ jobId: "outbox-outbox-1-0", removeOnFail: 10_000 })
|
||||
);
|
||||
expect(prisma.outboxEvent.updateMany).not.toHaveBeenCalled();
|
||||
expect(() => parseProductAcquisitionProviderSideEffectJob(queueRuntimeMocks.add.mock.calls[0]?.[1])).not.toThrow();
|
||||
});
|
||||
|
||||
it("emits a platform-lead envelope accepted by the strict consumer", async () => {
|
||||
vi.useFakeTimers();
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([
|
||||
createOutboxEvent({
|
||||
tenantId: null,
|
||||
eventType: "platform.admin_lead.captured.v1",
|
||||
dispatchKey: "platform-lead-captured:lead_1",
|
||||
payload: {
|
||||
leadId: "lead_1",
|
||||
sourceId: "hivegrid-website",
|
||||
eventTime: "2026-07-16T10:20:30.123Z",
|
||||
routingUserIds: ["user_haris"],
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
OUTBOX_DISPATCHER_ENABLED: "true",
|
||||
OUTBOX_DISPATCH_INTERVAL_MS: "5",
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
},
|
||||
{ prisma, logger: createLogger() }
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await result.handle?.close();
|
||||
|
||||
expect(() => parsePlatformLeadCapturedOutboxJob(queueRuntimeMocks.add.mock.calls[0]?.[1])).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
function createOutboxEvent(overrides: Partial<OutboxEvent> = {}): OutboxEvent {
|
||||
@@ -290,6 +423,7 @@ function createOutboxEvent(overrides: Partial<OutboxEvent> = {}): OutboxEvent {
|
||||
createdAt: now,
|
||||
processedAt: null,
|
||||
retries: 0,
|
||||
deliveryGeneration: 0,
|
||||
lastError: null,
|
||||
scheduledAt: now,
|
||||
dispatchKey: "workflow:outbox-1",
|
||||
@@ -298,3 +432,38 @@ function createOutboxEvent(overrides: Partial<OutboxEvent> = {}): OutboxEvent {
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createPortalSetupOutboxEvent(overrides: Partial<OutboxEvent> = {}): OutboxEvent {
|
||||
const subject = "Kundenportal-Zugang einrichten";
|
||||
const text = `Richten Sie Ihren Zugang ein: https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}`;
|
||||
const html = `<a href="https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}">Zugang einrichten</a>`;
|
||||
return createOutboxEvent({
|
||||
tenantId: "tenant-1",
|
||||
eventType: "mail.transactional.send",
|
||||
dispatchKey: "portal-setup:portal_token_1",
|
||||
payload: {
|
||||
tenantId: "tenant-1",
|
||||
to: ["portal.user@example.test"],
|
||||
subject,
|
||||
text,
|
||||
html,
|
||||
metadata: {
|
||||
source: "portal_setup_invite",
|
||||
invitationSource: "portal_admin_invite",
|
||||
portalId: "portal_1",
|
||||
portalUserId: "portal_user_1",
|
||||
portalTokenId: "portal_token_1",
|
||||
setupTokenHash: "a".repeat(64),
|
||||
expiresAt: "2026-06-03T01:00:00.000Z",
|
||||
mailContentHash: hashPortalSetupMailContent({
|
||||
recipient: "portal.user@example.test",
|
||||
subject,
|
||||
text,
|
||||
html,
|
||||
}),
|
||||
encryptedCapability: "encrypted-bootstrap-portal-capability",
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
@@ -1,11 +1,15 @@
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import { claimPendingOutboxEvents, OUTBOX_CLAIM_DEFAULT_LIMIT, type OutboxClaimRunner } from "@hivegrid/db/outbox-claim";
|
||||
import type { OutboxEvent } from "@hivegrid/db/outbox";
|
||||
import type { DbClient } from "@hivegrid/db/client.server";
|
||||
import type { Prisma } from "@hivegrid/db/prisma-browser";
|
||||
import type { ErrorReporter } from "@hivegrid/error-reporter";
|
||||
import { parseRedisUrl } from "@hivegrid/queue";
|
||||
import { workerLogger } from "../logger.js";
|
||||
import { getWorkerMetrics } from "../metrics.js";
|
||||
import { workerErrorReporter } from "../sentry.js";
|
||||
import { terminalizePortalSetupInvitationDelivery } from "../security/portal-setup-invitation-terminalization.js";
|
||||
import { terminalizeWorkspaceOwnerInvitationDelivery } from "../security/workspace-owner-invitation-terminalization.js";
|
||||
import {
|
||||
OUTBOX_DEFAULT_BATCH_SIZE,
|
||||
OUTBOX_DEFAULT_DISPATCH_CONCURRENCY,
|
||||
@@ -77,7 +81,10 @@ function parseInteger(value: string | undefined, fallback: number): number {
|
||||
async function createDefaultEnqueue(redisUrl: string): Promise<ClosableDispatcherQueueEnqueuer> {
|
||||
const connection = parseRedisUrl(redisUrl);
|
||||
const { createQueue } = (await import("@hivegrid/queue/runtime")) as {
|
||||
createQueue: (name: string, options: { connection: typeof connection }) => {
|
||||
createQueue: (
|
||||
name: string,
|
||||
options: { connection: typeof connection }
|
||||
) => {
|
||||
add(name: string, data: unknown, options?: unknown): Promise<unknown>;
|
||||
close?: () => Promise<void>;
|
||||
};
|
||||
@@ -99,12 +106,14 @@ async function createDefaultEnqueue(redisUrl: string): Promise<ClosableDispatche
|
||||
payload: event.payload,
|
||||
dispatchKey: event.dispatchKey,
|
||||
retries: event.retries,
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil?.toISOString() ?? null,
|
||||
},
|
||||
{
|
||||
// BullMQ rejects custom job ids containing ":" ("Custom Id cannot
|
||||
// contain :"), which silently failed every outbox dispatch. Use "-";
|
||||
// workflow-execution-stalled-reconciler.buildOutboxJobId must match.
|
||||
jobId: `outbox-${event.id}`,
|
||||
jobId: `outbox-${event.id}-${event.deliveryGeneration}`,
|
||||
attempts: OUTBOX_QUEUE_JOB_ATTEMPTS,
|
||||
backoff: { type: "exponential", delay: OUTBOX_QUEUE_JOB_BACKOFF_MS },
|
||||
removeOnComplete: OUTBOX_QUEUE_REMOVE_ON_COMPLETE,
|
||||
@@ -189,6 +198,48 @@ export async function bootstrapOutboxDispatcher(
|
||||
}
|
||||
},
|
||||
},
|
||||
terminalizeOwnerInvitation: async ({ event, redactedPayload, retries, lastError, processedAt }) => {
|
||||
if (!event.tenantId) {
|
||||
throw new Error("Workspace-owner invitation outbox event is missing its tenant.");
|
||||
}
|
||||
await terminalizeWorkspaceOwnerInvitationDelivery(
|
||||
{
|
||||
tenantId: event.tenantId,
|
||||
payload: event.payload,
|
||||
outboxFailure: {
|
||||
eventId: event.id,
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
redactedPayload: redactedPayload as Prisma.InputJsonValue,
|
||||
retries,
|
||||
lastError,
|
||||
processedAt,
|
||||
},
|
||||
},
|
||||
deps.prisma as unknown as Pick<DbClient, "$transaction">
|
||||
);
|
||||
},
|
||||
terminalizePortalSetupInvitation: async ({ event, redactedPayload, retries, lastError, processedAt }) => {
|
||||
if (!event.tenantId) {
|
||||
throw new Error("Portal setup invitation outbox event is missing its tenant.");
|
||||
}
|
||||
await terminalizePortalSetupInvitationDelivery(
|
||||
{
|
||||
tenantId: event.tenantId,
|
||||
payload: event.payload,
|
||||
outboxFailure: {
|
||||
eventId: event.id,
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
redactedPayload: redactedPayload as Prisma.InputJsonValue,
|
||||
retries,
|
||||
lastError,
|
||||
processedAt,
|
||||
},
|
||||
},
|
||||
deps.prisma as unknown as Pick<DbClient, "$transaction">
|
||||
);
|
||||
},
|
||||
});
|
||||
const handle = closeDefaultEnqueue
|
||||
? {
|
||||
|
||||
+647
-5
@@ -1,6 +1,10 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { OutboxEvent } from "@hivegrid/db/outbox";
|
||||
import { hashPortalSetupMailContent, PORTAL_SETUP_TOKEN_PLACEHOLDER } from "@hivegrid/db/portal-setup-mail-contract.server";
|
||||
import { hashWorkspaceOwnerInvitationMailContent } from "@hivegrid/db/workspace-owner-invite-mail-contract.server";
|
||||
import {
|
||||
OUTBOX_DEFAULT_DISPATCH_CONCURRENCY,
|
||||
OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
@@ -12,20 +16,47 @@ import {
|
||||
runDispatcherTick,
|
||||
startOutboxDispatcher,
|
||||
type DispatcherClaim,
|
||||
type DispatcherOwnerInvitationTerminalizer,
|
||||
type DispatcherPortalSetupInvitationTerminalizer,
|
||||
type DispatcherQueueEnqueuer,
|
||||
type DispatcherTransactionRunner,
|
||||
} from "./dispatcher.js";
|
||||
|
||||
type FakeEvent = OutboxEvent;
|
||||
|
||||
type Updates = Array<{ id: string; data: Record<string, unknown> }>;
|
||||
type Updates = Array<{
|
||||
where: { id: string; status: "PROCESSING"; deliveryGeneration: number; lockedUntil: Date | null };
|
||||
data: Record<string, unknown>;
|
||||
}>;
|
||||
|
||||
function makeStore(updates: Updates): DispatcherTransactionRunner {
|
||||
function makeStore(updates: Updates, counts: number[] = []): DispatcherTransactionRunner {
|
||||
return {
|
||||
outboxEvent: {
|
||||
update: vi.fn(async ({ where, data }) => {
|
||||
updates.push({ id: where.id, data });
|
||||
return { id: where.id, ...data } as unknown as FakeEvent;
|
||||
updateMany: vi.fn(async ({ where, data }) => {
|
||||
updates.push({ where, data });
|
||||
return { count: counts.shift() ?? 1 };
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeStatefulStore(
|
||||
state: { id: string; status: string; deliveryGeneration: number; lockedUntil: Date | null },
|
||||
updates: Updates
|
||||
): DispatcherTransactionRunner {
|
||||
return {
|
||||
outboxEvent: {
|
||||
updateMany: vi.fn(async ({ where, data }) => {
|
||||
updates.push({ where, data });
|
||||
const ownsLease =
|
||||
state.id === where.id &&
|
||||
state.status === where.status &&
|
||||
state.deliveryGeneration === where.deliveryGeneration &&
|
||||
state.lockedUntil?.getTime() === where.lockedUntil?.getTime();
|
||||
if (!ownsLease) return { count: 0 };
|
||||
if (typeof data.status === "string") state.status = data.status;
|
||||
if (data.lockedUntil === null || data.lockedUntil instanceof Date) state.lockedUntil = data.lockedUntil;
|
||||
return { count: 1 };
|
||||
}),
|
||||
},
|
||||
};
|
||||
@@ -38,6 +69,7 @@ function makeEvent(overrides: Partial<FakeEvent> = {}): FakeEvent {
|
||||
eventType: "mail.welcome.send",
|
||||
payload: { to: "user@example.com" },
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: 0,
|
||||
createdAt: new Date("2026-05-17T08:00:00.000Z"),
|
||||
processedAt: null,
|
||||
retries: 0,
|
||||
@@ -79,6 +111,82 @@ describe("computeOutboxRetryDelayMs", () => {
|
||||
|
||||
describe("runDispatcherTick", () => {
|
||||
const now = new Date("2026-05-17T09:00:00.000Z");
|
||||
const ownerToken = "owner-capability-token-fixture";
|
||||
|
||||
function ownerInvitationPayload() {
|
||||
return {
|
||||
tenantId: "tenant_1",
|
||||
to: ["owner@example.test"],
|
||||
subject: "Workspace invitation",
|
||||
html: `<a href="https://app.example.test/invitation/${ownerToken}">Accept</a>`,
|
||||
metadata: {
|
||||
source: "workspace-owner-invitation",
|
||||
invitationId: "invite_1",
|
||||
invitationTokenHash: "a".repeat(64),
|
||||
issuanceKeyHash: "b".repeat(64),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function exactOwnerInvitationPayload() {
|
||||
const token = "x".repeat(43);
|
||||
const invitationTokenHash = createHash("sha256").update(token).digest("hex");
|
||||
const subject = "Workspace invitation";
|
||||
const html = `<a href="https://app.example.test/invitation/${token}">Accept</a>`;
|
||||
return {
|
||||
tenantId: "tenant_1",
|
||||
to: ["owner@example.test"],
|
||||
subject,
|
||||
html,
|
||||
metadata: {
|
||||
source: "workspace-owner-invitation",
|
||||
invitationId: "invite_1",
|
||||
invitationTokenHash,
|
||||
issuanceKeyHash: "b".repeat(64),
|
||||
mailContentHash: hashWorkspaceOwnerInvitationMailContent({
|
||||
recipient: "owner@example.test",
|
||||
subject,
|
||||
html,
|
||||
rawToken: token,
|
||||
invitationTokenHash,
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function exactPortalSetupInvitationPayload() {
|
||||
const rawToken = "p".repeat(43);
|
||||
const setupTokenHash = createHash("sha256").update(rawToken).digest("hex");
|
||||
const subject = "Kundenportal-Zugang einrichten";
|
||||
const text = `Richten Sie Ihren Zugang ein: https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}`;
|
||||
const html = `<a href="https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}">Zugang einrichten</a>`;
|
||||
return {
|
||||
rawToken,
|
||||
payload: {
|
||||
tenantId: "tenant_1",
|
||||
to: ["portal.user@example.test"],
|
||||
subject,
|
||||
text,
|
||||
html,
|
||||
metadata: {
|
||||
source: "portal_setup_invite",
|
||||
invitationSource: "portal_admin_invite",
|
||||
portalId: "portal_1",
|
||||
portalUserId: "portal_user_1",
|
||||
portalTokenId: "portal_token_1",
|
||||
setupTokenHash,
|
||||
expiresAt: "2026-05-24T09:00:00.000Z",
|
||||
mailContentHash: hashPortalSetupMailContent({
|
||||
recipient: "portal.user@example.test",
|
||||
subject,
|
||||
text,
|
||||
html,
|
||||
}),
|
||||
encryptedCapability: "encrypted-portal-setup-capability",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("returns empty result when claim throws", async () => {
|
||||
const claim: DispatcherClaim = vi.fn(async () => {
|
||||
@@ -118,6 +226,244 @@ describe("runDispatcherTick", () => {
|
||||
expect(updates[0]?.data.status).toBe("DONE");
|
||||
expect(updates[0]?.data.processedAt).toBe(now);
|
||||
expect(updates[0]?.data.lockedUntil).toBeNull();
|
||||
expect(updates[0]?.where).toEqual({
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves financial provider terminal state to the generation-bound consumer receipt", async () => {
|
||||
const event = makeEvent({
|
||||
eventType: "billing.product-acquisition.provider-side-effect",
|
||||
payload: { acquisitionId: "acquisition_1" },
|
||||
});
|
||||
const updates: Updates = [];
|
||||
const enqueue = vi.fn(async () => undefined);
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
|
||||
expect(enqueue).toHaveBeenCalledWith("outbox.billing-product-provider", event);
|
||||
expect(updates).toEqual([]);
|
||||
});
|
||||
|
||||
it("leaves audit terminal state to the atomic consumer receipt", async () => {
|
||||
const event = makeEvent({
|
||||
eventType: "audit.event.create",
|
||||
payload: { tenantId: "tenant_1", action: "widget.created" },
|
||||
});
|
||||
const updates: Updates = [];
|
||||
const enqueue = vi.fn(async () => undefined);
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
|
||||
expect(enqueue).toHaveBeenCalledWith("outbox.audit", event);
|
||||
expect(updates).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves a terminal worker receipt that wins before dispatcher completion", async () => {
|
||||
const event = makeEvent({ deliveryGeneration: 7 });
|
||||
const updates: Updates = [];
|
||||
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
const state = {
|
||||
id: event.id,
|
||||
status: event.status,
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
};
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => {
|
||||
state.status = "FAILED";
|
||||
state.lockedUntil = null;
|
||||
}),
|
||||
store: makeStatefulStore(state, updates),
|
||||
now: () => now,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ claimed: 1, ok: 0, errors: 1, deadLetters: 0 });
|
||||
expect(updates[0]?.where).toEqual({
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: 7,
|
||||
lockedUntil: event.lockedUntil,
|
||||
});
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ eventId: event.id, deliveryGeneration: 7 }), expect.stringContaining("race"));
|
||||
});
|
||||
|
||||
it("cannot complete a newer replay generation from an old dispatcher claim", async () => {
|
||||
const event = makeEvent({ deliveryGeneration: 2 });
|
||||
const updates: Updates = [];
|
||||
const state = {
|
||||
id: event.id,
|
||||
status: event.status,
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
};
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => {
|
||||
state.status = "PENDING";
|
||||
state.deliveryGeneration = 3;
|
||||
state.lockedUntil = null;
|
||||
}),
|
||||
store: makeStatefulStore(state, updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(0);
|
||||
expect(result.errors).toBe(1);
|
||||
expect(updates[0]?.where.deliveryGeneration).toBe(2);
|
||||
});
|
||||
|
||||
it("cannot complete a reclaimed lease with the same delivery generation", async () => {
|
||||
const oldLease = new Date("2026-05-17T08:05:00.000Z");
|
||||
const event = makeEvent({ lockedUntil: oldLease });
|
||||
const updates: Updates = [];
|
||||
const state = {
|
||||
id: event.id,
|
||||
status: event.status,
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
};
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => {
|
||||
state.lockedUntil = new Date("2026-05-17T08:10:00.000Z");
|
||||
}),
|
||||
store: makeStatefulStore(state, updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(0);
|
||||
expect(result.errors).toBe(1);
|
||||
expect(updates[0]?.where.lockedUntil).toBe(oldLease);
|
||||
});
|
||||
|
||||
it("redacts the OWNER capability from the DB outbox only after enqueue succeeds", async () => {
|
||||
const event = makeEvent({ eventType: "mail.transactional.send", payload: exactOwnerInvitationPayload() });
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
|
||||
const updates: Updates = [];
|
||||
|
||||
await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(JSON.stringify(vi.mocked(enqueue).mock.calls[0]?.[1].payload)).toContain("x".repeat(43));
|
||||
expect(JSON.stringify(updates[0]?.data.payload)).not.toContain("x".repeat(43));
|
||||
expect(updates[0]?.data.payload).toMatchObject({
|
||||
metadata: expect.objectContaining({ source: "workspace-owner-invitation", secretBodyRedacted: true }),
|
||||
});
|
||||
expect(updates[0]?.data).toMatchObject({ status: "DONE", processedAt: now });
|
||||
});
|
||||
|
||||
it("validates and redacts an encrypted portal setup invitation after enqueue", async () => {
|
||||
const { payload, rawToken } = exactPortalSetupInvitationPayload();
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload,
|
||||
dispatchKey: "portal-setup:portal_token_1",
|
||||
});
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
|
||||
const updates: Updates = [];
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
|
||||
expect(enqueue).toHaveBeenCalledWith("outbox.mail", event);
|
||||
expect(JSON.stringify(vi.mocked(enqueue).mock.calls[0]?.[1].payload)).toContain("encrypted-portal-setup-capability");
|
||||
expect(JSON.stringify(vi.mocked(enqueue).mock.calls[0]?.[1].payload)).not.toContain(rawToken);
|
||||
expect(updates[0]?.data).toMatchObject({
|
||||
status: "DONE",
|
||||
processedAt: now,
|
||||
payload: {
|
||||
metadata: expect.objectContaining({
|
||||
source: "portal_setup_invite",
|
||||
secretBodyRedacted: true,
|
||||
}),
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(updates[0]?.data.payload)).not.toContain("encrypted-portal-setup-capability");
|
||||
expect(JSON.stringify(updates[0]?.data.payload)).not.toContain(PORTAL_SETUP_TOKEN_PLACEHOLDER);
|
||||
expect(JSON.stringify(updates[0]?.data.payload)).not.toContain(rawToken);
|
||||
});
|
||||
|
||||
it("rejects an invalid portal setup invitation contract before enqueue", async () => {
|
||||
const { payload } = exactPortalSetupInvitationPayload();
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload: { ...payload, subject: "tampered subject" },
|
||||
dispatchKey: "portal-setup:portal_token_1",
|
||||
});
|
||||
const enqueue = vi.fn(async () => undefined);
|
||||
const updates: Updates = [];
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ claimed: 1, ok: 0, errors: 1, deadLetters: 0 });
|
||||
expect(enqueue).not.toHaveBeenCalled();
|
||||
expect(updates[0]?.data).toMatchObject({
|
||||
status: "PENDING",
|
||||
retries: 1,
|
||||
lastError: "Portal setup invitation encrypted mail contract is invalid.",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the OWNER capability in the DB outbox when enqueue must retry", async () => {
|
||||
const payload = ownerInvitationPayload();
|
||||
const event = makeEvent({ eventType: "mail.transactional.send", payload });
|
||||
const updates: Updates = [];
|
||||
|
||||
await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => {
|
||||
throw new Error("redis offline");
|
||||
}),
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(JSON.stringify(event.payload)).toContain(ownerToken);
|
||||
expect(updates[0]?.data).toMatchObject({ status: "PENDING", retries: 1 });
|
||||
expect(updates[0]?.data).not.toHaveProperty("payload");
|
||||
});
|
||||
|
||||
it("routes workflow trigger events to the workflow outbox receiver queue", async () => {
|
||||
@@ -143,6 +489,34 @@ describe("runDispatcherTick", () => {
|
||||
expect(updates[0]?.data).toMatchObject({ status: "DONE", processedAt: now, lockedUntil: null });
|
||||
});
|
||||
|
||||
it("dispatches the platform lead capture marker instead of dead-lettering it", async () => {
|
||||
const event = makeEvent({
|
||||
tenantId: null,
|
||||
eventType: "platform.admin_lead.captured.v1",
|
||||
payload: {
|
||||
leadId: "lead_1",
|
||||
sourceId: "hivegrid-website",
|
||||
eventTime: "2026-07-16T10:20:30.123Z",
|
||||
routingUserIds: ["user_haris"],
|
||||
},
|
||||
dispatchKey: "platform-lead-captured:lead_1",
|
||||
});
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
|
||||
const updates: Updates = [];
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
|
||||
expect(enqueue).toHaveBeenCalledWith("outbox.notification", event);
|
||||
expect(updates[0]?.data).toMatchObject({ status: "DONE", processedAt: now, lockedUntil: null });
|
||||
});
|
||||
|
||||
it("dead-letters unknown event types without enqueueing", async () => {
|
||||
const event = makeEvent({ eventType: "no.routing.here" });
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
@@ -191,6 +565,39 @@ describe("runDispatcherTick", () => {
|
||||
expect(scheduled.getTime() - now.getTime()).toBe(2 * 60_000);
|
||||
});
|
||||
|
||||
it("retains a retryable OWNER capability but scrubs queue diagnostics and lastError", async () => {
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload: exactOwnerInvitationPayload(),
|
||||
retries: 1,
|
||||
});
|
||||
const updates: Updates = [];
|
||||
const logger = { error: vi.fn() };
|
||||
const reporter = { captureException: vi.fn() };
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => Promise.reject(new Error(`queue rejected ${JSON.stringify(event.payload)}`))),
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger,
|
||||
reporter,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ errors: 1, deadLetters: 0 });
|
||||
expect(updates[0]?.data).toMatchObject({ status: "PENDING", retries: 2 });
|
||||
expect(updates[0]?.data).not.toHaveProperty("payload");
|
||||
expect(updates[0]?.data.lastError).not.toContain("x".repeat(43));
|
||||
expect(updates[0]?.data.lastError).not.toContain("<a");
|
||||
const loggedError = logger.error.mock.calls[0]?.[0].error as Error;
|
||||
const reportedError = reporter.captureException.mock.calls[0]?.[0] as Error;
|
||||
expect(loggedError.message).not.toContain("x".repeat(43));
|
||||
expect(loggedError.message).not.toContain("<a");
|
||||
expect(reportedError.message).not.toContain("x".repeat(43));
|
||||
expect(reportedError.message).not.toContain("<a");
|
||||
expect(JSON.stringify(event.payload)).toContain("x".repeat(43));
|
||||
});
|
||||
|
||||
it("dead-letters after retries exceed the limit", async () => {
|
||||
const event = makeEvent({ retries: OUTBOX_DEFAULT_MAX_RETRIES });
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
@@ -212,6 +619,241 @@ describe("runDispatcherTick", () => {
|
||||
expect(updates[0]?.data.retries).toBe(OUTBOX_DEFAULT_MAX_RETRIES + 1);
|
||||
});
|
||||
|
||||
it("atomically invalidates and redacts a terminal OWNER outbox capability", async () => {
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload: exactOwnerInvitationPayload(),
|
||||
retries: OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
dispatchKey: "mail:workspace-owner-invite:invite_1",
|
||||
});
|
||||
const updates: Updates = [];
|
||||
const terminalizeOwnerInvitation = vi.fn<DispatcherOwnerInvitationTerminalizer>(async (_input) => undefined);
|
||||
const logger = { error: vi.fn() };
|
||||
const reporter = { captureException: vi.fn() };
|
||||
const queueError = new Error(`queue rejected ${JSON.stringify(event.payload)}`);
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => Promise.reject(queueError)),
|
||||
store: makeStore(updates),
|
||||
terminalizeOwnerInvitation,
|
||||
now: () => now,
|
||||
logger,
|
||||
reporter,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ errors: 1, deadLetters: 1 });
|
||||
expect(terminalizeOwnerInvitation).toHaveBeenCalledOnce();
|
||||
const terminalInput = terminalizeOwnerInvitation.mock.calls[0]?.[0];
|
||||
expect(JSON.stringify(terminalInput?.redactedPayload)).not.toContain("x".repeat(43));
|
||||
expect(terminalInput?.lastError).not.toContain("x".repeat(43));
|
||||
expect(terminalInput?.lastError).not.toContain("<a");
|
||||
expect(terminalInput?.redactedPayload).toMatchObject({
|
||||
metadata: expect.objectContaining({ secretBodyRedacted: true, ownerInvitationDeliveryStatus: "rotation_required" }),
|
||||
});
|
||||
expect(updates).toEqual([]);
|
||||
const loggedError = logger.error.mock.calls[0]?.[0].error as Error;
|
||||
const reportedError = reporter.captureException.mock.calls[0]?.[0] as Error;
|
||||
expect(loggedError.message).not.toContain("x".repeat(43));
|
||||
expect(loggedError.message).not.toContain("<a");
|
||||
expect(reportedError.message).not.toContain("x".repeat(43));
|
||||
expect(reportedError.message).not.toContain("<a");
|
||||
});
|
||||
|
||||
it("atomically revokes and redacts a terminal portal setup capability", async () => {
|
||||
const { payload, rawToken } = exactPortalSetupInvitationPayload();
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload,
|
||||
retries: OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
dispatchKey: "portal-setup:portal_token_1",
|
||||
deliveryGeneration: 4,
|
||||
});
|
||||
const updates: Updates = [];
|
||||
const terminalizePortalSetupInvitation = vi.fn<DispatcherPortalSetupInvitationTerminalizer>(async () => undefined);
|
||||
const logger = { error: vi.fn() };
|
||||
const reporter = { captureException: vi.fn() };
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => Promise.reject(new Error(`queue rejected ${JSON.stringify(payload)}`))),
|
||||
store: makeStore(updates),
|
||||
terminalizePortalSetupInvitation,
|
||||
now: () => now,
|
||||
logger,
|
||||
reporter,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ errors: 1, deadLetters: 1 });
|
||||
expect(terminalizePortalSetupInvitation).toHaveBeenCalledOnce();
|
||||
const terminalInput = terminalizePortalSetupInvitation.mock.calls[0]?.[0];
|
||||
expect(terminalInput?.event).toBe(event);
|
||||
expect(terminalInput?.redactedPayload).toMatchObject({
|
||||
tenantId: "tenant_1",
|
||||
metadata: expect.objectContaining({
|
||||
source: "portal_setup_invite",
|
||||
outboxEventId: event.id,
|
||||
dispatchKey: event.dispatchKey,
|
||||
secretBodyRedacted: true,
|
||||
portalSetupDeliveryStatus: "rotation_required",
|
||||
}),
|
||||
});
|
||||
expect(JSON.stringify(terminalInput?.redactedPayload)).not.toContain("encrypted-portal-setup-capability");
|
||||
expect(JSON.stringify(terminalInput?.redactedPayload)).not.toContain(PORTAL_SETUP_TOKEN_PLACEHOLDER);
|
||||
expect(JSON.stringify(terminalInput?.redactedPayload)).not.toContain(rawToken);
|
||||
expect(terminalInput?.lastError).not.toContain("encrypted-portal-setup-capability");
|
||||
expect(terminalInput?.lastError).not.toContain("<a");
|
||||
expect(updates).toEqual([]);
|
||||
const loggedError = logger.error.mock.calls[0]?.[0].error as Error;
|
||||
const reportedError = reporter.captureException.mock.calls[0]?.[0] as Error;
|
||||
expect(loggedError.message).not.toContain("encrypted-portal-setup-capability");
|
||||
expect(loggedError.message).not.toContain("<a");
|
||||
expect(reportedError.message).not.toContain("encrypted-portal-setup-capability");
|
||||
expect(reportedError.message).not.toContain("<a");
|
||||
});
|
||||
|
||||
it("persists and recovers a durable redacted portal setup rotation marker", async () => {
|
||||
const { payload, rawToken } = exactPortalSetupInvitationPayload();
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload,
|
||||
retries: OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
dispatchKey: "portal-setup:portal_token_1",
|
||||
});
|
||||
const firstUpdates: Updates = [];
|
||||
|
||||
await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => Promise.reject(new Error(`queue rejected ${JSON.stringify(payload)}`))),
|
||||
store: makeStore(firstUpdates),
|
||||
terminalizePortalSetupInvitation: vi.fn(async () => Promise.reject(new Error(`db rejected encrypted-portal-setup-capability`))),
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(firstUpdates[0]?.data).toMatchObject({
|
||||
status: "PENDING",
|
||||
payload: {
|
||||
metadata: expect.objectContaining({
|
||||
outboxEventId: event.id,
|
||||
dispatchKey: event.dispatchKey,
|
||||
portalSetupDeliveryStatus: "rotation_required",
|
||||
}),
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(firstUpdates[0]?.data.payload)).not.toContain("encrypted-portal-setup-capability");
|
||||
expect(JSON.stringify(firstUpdates[0]?.data.payload)).not.toContain(PORTAL_SETUP_TOKEN_PLACEHOLDER);
|
||||
expect(JSON.stringify(firstUpdates[0]?.data.payload)).not.toContain(rawToken);
|
||||
expect(firstUpdates[0]?.data.lastError).not.toContain("encrypted-portal-setup-capability");
|
||||
|
||||
const recoveryEvent = makeEvent({
|
||||
...event,
|
||||
payload: firstUpdates[0]?.data.payload as never,
|
||||
retries: firstUpdates[0]?.data.retries as number,
|
||||
lastError: firstUpdates[0]?.data.lastError as string,
|
||||
});
|
||||
const enqueue = vi.fn(async () => undefined);
|
||||
const terminalizePortalSetupInvitation = vi.fn<DispatcherPortalSetupInvitationTerminalizer>(async () => undefined);
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [recoveryEvent]),
|
||||
enqueue,
|
||||
store: makeStore([]),
|
||||
terminalizePortalSetupInvitation,
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ errors: 0, deadLetters: 1 });
|
||||
expect(enqueue).not.toHaveBeenCalled();
|
||||
expect(terminalizePortalSetupInvitation).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
event: recoveryEvent,
|
||||
redactedPayload: recoveryEvent.payload,
|
||||
retries: OUTBOX_DEFAULT_MAX_RETRIES + 2,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("persists a durable redacted recovery marker when atomic invalidation fails", async () => {
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload: exactOwnerInvitationPayload(),
|
||||
retries: OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
dispatchKey: "mail:workspace-owner-invite:invite_1",
|
||||
});
|
||||
const updates: Updates = [];
|
||||
const logger = { error: vi.fn() };
|
||||
const reporter = { captureException: vi.fn() };
|
||||
const terminalizationError = new Error(`db rejected ${JSON.stringify(event.payload)}`);
|
||||
|
||||
await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => Promise.reject(new Error("redis permanently unavailable"))),
|
||||
store: makeStore(updates),
|
||||
terminalizeOwnerInvitation: vi.fn(async () => Promise.reject(terminalizationError)),
|
||||
now: () => now,
|
||||
logger,
|
||||
reporter,
|
||||
});
|
||||
|
||||
expect(updates[0]?.data).toMatchObject({
|
||||
status: "PENDING",
|
||||
payload: { metadata: expect.objectContaining({ ownerInvitationDeliveryStatus: "rotation_required" }) },
|
||||
});
|
||||
expect(JSON.stringify(updates[0]?.data.payload)).not.toContain("x".repeat(43));
|
||||
const terminalLogError = logger.error.mock.calls[1]?.[0].error as Error;
|
||||
const terminalReportedError = reporter.captureException.mock.calls[1]?.[0] as Error;
|
||||
expect(terminalLogError.message).not.toContain("x".repeat(43));
|
||||
expect(terminalLogError.message).not.toContain("<a");
|
||||
expect(terminalReportedError.message).not.toContain("x".repeat(43));
|
||||
expect(terminalReportedError.message).not.toContain("<a");
|
||||
});
|
||||
|
||||
it("recovers a durable OWNER rotation marker without enqueueing the redacted mail", async () => {
|
||||
const event = makeEvent({
|
||||
eventType: "mail.transactional.send",
|
||||
payload: exactOwnerInvitationPayload(),
|
||||
retries: OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
dispatchKey: "mail:workspace-owner-invite:invite_1",
|
||||
});
|
||||
const firstUpdates: Updates = [];
|
||||
await runDispatcherTick({
|
||||
claim: vi.fn(async () => [event]),
|
||||
enqueue: vi.fn(async () => Promise.reject(new Error("redis permanently unavailable"))),
|
||||
store: makeStore(firstUpdates),
|
||||
terminalizeOwnerInvitation: vi.fn(async () => Promise.reject(new Error("database temporarily unavailable"))),
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
const recoveryEvent = makeEvent({
|
||||
...event,
|
||||
payload: firstUpdates[0]?.data.payload as never,
|
||||
retries: firstUpdates[0]?.data.retries as number,
|
||||
lastError: firstUpdates[0]?.data.lastError as string,
|
||||
});
|
||||
const enqueue = vi.fn(async () => undefined);
|
||||
const terminalizeOwnerInvitation = vi.fn<DispatcherOwnerInvitationTerminalizer>(async () => undefined);
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim: vi.fn(async () => [recoveryEvent]),
|
||||
enqueue,
|
||||
store: makeStore([]),
|
||||
terminalizeOwnerInvitation,
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ errors: 0, deadLetters: 1 });
|
||||
expect(enqueue).not.toHaveBeenCalled();
|
||||
expect(terminalizeOwnerInvitation).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
event: recoveryEvent,
|
||||
redactedPayload: recoveryEvent.payload,
|
||||
retries: OUTBOX_DEFAULT_MAX_RETRIES + 2,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("records metrics per event", async () => {
|
||||
const event = makeEvent();
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
|
||||
@@ -1,5 +1,26 @@
|
||||
import type { OutboxEvent } from "@hivegrid/db/outbox";
|
||||
import { AUDIT_EVENT_OUTBOX_TYPE } from "@hivegrid/db/audit-events.server";
|
||||
import { PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_EVENT } from "@hivegrid/db/product-acquisition-provider-side-effect.server";
|
||||
import { RECURRING_INVOICE_COLLECTION_EVENT } from "@hivegrid/db/recurring-invoice-collection.server";
|
||||
import { workerLogger } from "../logger.js";
|
||||
import {
|
||||
bindTerminalPortalSetupInvitationMailPayload,
|
||||
isEncryptedPortalSetupInvitationMailPayload,
|
||||
isPortalSetupMailIntent,
|
||||
isTerminalPortalSetupInvitationMailPayload,
|
||||
redactPortalSetupInvitationError,
|
||||
redactPortalSetupInvitationMailPayload,
|
||||
redactTerminalPortalSetupInvitationMailPayload,
|
||||
} from "../security/portal-setup-invitation-secret.js";
|
||||
import {
|
||||
bindTerminalWorkspaceOwnerInvitationMailPayload,
|
||||
hasValidWorkspaceOwnerInvitationMailContract,
|
||||
isTerminalWorkspaceOwnerInvitationMailPayload,
|
||||
isWorkspaceOwnerInvitationMailIntent,
|
||||
redactTerminalWorkspaceOwnerInvitationMailPayload,
|
||||
redactWorkspaceOwnerInvitationMailPayload,
|
||||
redactWorkspaceOwnerInvitationError,
|
||||
} from "../security/workspace-owner-invitation-secret.js";
|
||||
import { resolveOutboxQueueName, type EventRoutingRule } from "./event-router.js";
|
||||
|
||||
/**
|
||||
@@ -31,10 +52,7 @@ export const OUTBOX_RETRY_BACKOFF_MINUTES: readonly number[] = [1, 2, 4, 8, 16,
|
||||
* Compute the next `scheduledAt` for a failed dispatch. `retries` is the
|
||||
* count AFTER incrementing the counter (so retries=1 → 1 minute delay).
|
||||
*/
|
||||
export function computeOutboxRetryDelayMs(
|
||||
retries: number,
|
||||
schedule: readonly number[] = OUTBOX_RETRY_BACKOFF_MINUTES
|
||||
): number {
|
||||
export function computeOutboxRetryDelayMs(retries: number, schedule: readonly number[] = OUTBOX_RETRY_BACKOFF_MINUTES): number {
|
||||
if (retries < 1) return schedule[0] !== undefined ? schedule[0] * 60 * 1000 : 60_000;
|
||||
const idx = Math.min(retries - 1, schedule.length - 1);
|
||||
return (schedule[idx] ?? schedule.at(-1) ?? 1) * 60 * 1000;
|
||||
@@ -60,15 +78,31 @@ export type DispatcherQueueEnqueuer = (queueName: string, event: OutboxEvent) =>
|
||||
|
||||
export type DispatcherTransactionRunner = {
|
||||
outboxEvent: {
|
||||
update(args: {
|
||||
where: { id: string };
|
||||
updateMany(args: {
|
||||
where: { id: string; status: "PROCESSING"; deliveryGeneration: number; lockedUntil: Date | null };
|
||||
data: Record<string, unknown>;
|
||||
}): Promise<OutboxEvent>;
|
||||
}): Promise<{ count: number }>;
|
||||
};
|
||||
};
|
||||
|
||||
export type DispatcherClaim = (limit: number, now: Date) => Promise<OutboxEvent[]>;
|
||||
|
||||
export type DispatcherOwnerInvitationTerminalizer = (input: {
|
||||
event: OutboxEvent;
|
||||
redactedPayload: unknown;
|
||||
retries: number;
|
||||
lastError: string;
|
||||
processedAt: Date;
|
||||
}) => Promise<void>;
|
||||
|
||||
export type DispatcherPortalSetupInvitationTerminalizer = (input: {
|
||||
event: OutboxEvent;
|
||||
redactedPayload: unknown;
|
||||
retries: number;
|
||||
lastError: string;
|
||||
processedAt: Date;
|
||||
}) => Promise<void>;
|
||||
|
||||
export type DispatcherOptions = {
|
||||
/** Atomic batch-claim function (bound to a Prisma client). */
|
||||
claim: DispatcherClaim;
|
||||
@@ -92,6 +126,10 @@ export type DispatcherOptions = {
|
||||
logger?: DispatcherLogger;
|
||||
metrics?: DispatcherMetrics;
|
||||
reporter?: DispatcherReporter;
|
||||
/** Atomically invalidates an OWNER capability with its terminal outbox row. */
|
||||
terminalizeOwnerInvitation?: DispatcherOwnerInvitationTerminalizer;
|
||||
/** Atomically revokes a portal setup capability with its terminal outbox row. */
|
||||
terminalizePortalSetupInvitation?: DispatcherPortalSetupInvitationTerminalizer;
|
||||
};
|
||||
|
||||
export type DispatcherTickResult = {
|
||||
@@ -135,91 +173,360 @@ export async function runDispatcherTick(options: DispatcherOptions): Promise<Dis
|
||||
const dispatchConcurrency = clampDispatchConcurrency(options.dispatchConcurrency);
|
||||
|
||||
for (let i = 0; i < claimed.length; i += dispatchConcurrency) {
|
||||
await Promise.all(claimed.slice(i, i + dispatchConcurrency).map(async (event) => {
|
||||
const targetQueue = resolveOutboxQueueName(event.eventType, routingRules ? { rules: routingRules } : undefined);
|
||||
await Promise.all(
|
||||
claimed.slice(i, i + dispatchConcurrency).map(async (event) => {
|
||||
const targetQueue = resolveOutboxQueueName(event.eventType, routingRules ? { rules: routingRules } : undefined);
|
||||
|
||||
if (!targetQueue) {
|
||||
// Unknown event-type → dead-letter immediately so it shows up in
|
||||
// admin UI and Sentry alerts. Do NOT retry — the type will never
|
||||
// route until the operator adds a rule, and silent retries waste
|
||||
// throughput.
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
const message = `No routing rule matched eventType ${event.eventType}.`;
|
||||
logger.error({ eventId: event.id, eventType: event.eventType }, message);
|
||||
reporter?.captureException?.(new Error(message), {
|
||||
component: "outbox-dispatcher",
|
||||
op: "route",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
});
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
lastError: message,
|
||||
processedAt: nowDate,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (isTerminalPortalSetupInvitationMailPayload(event.payload)) {
|
||||
const nextRetries = event.retries + 1;
|
||||
const recoveryError = await recoverTerminalPortalSetupInvitation({ event, nextRetries, nowDate, options, logger, reporter });
|
||||
if (recoveryError) {
|
||||
errors += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "error");
|
||||
await store.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: {
|
||||
status: "PENDING",
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(recoveryError.message),
|
||||
scheduledAt: new Date(nowDate.getTime() + computeOutboxRetryDelayMs(nextRetries)),
|
||||
processedAt: null,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await enqueue(targetQueue, event);
|
||||
ok += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "ok");
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
data: {
|
||||
status: "DONE",
|
||||
processedAt: nowDate,
|
||||
lastError: null,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
errors += 1;
|
||||
const nextRetries = event.retries + 1;
|
||||
const isTerminal = nextRetries > maxRetries;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
metrics?.recordDispatch?.(event.eventType, isTerminal ? "dead_letter" : "error");
|
||||
logger.error({ eventId: event.id, eventType: event.eventType, retries: nextRetries, error }, "Outbox dispatch failed.");
|
||||
reporter?.captureException?.(error, {
|
||||
component: "outbox-dispatcher",
|
||||
op: "enqueue",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
retries: nextRetries,
|
||||
});
|
||||
if (isTerminalWorkspaceOwnerInvitationMailPayload(event.payload)) {
|
||||
const nextRetries = event.retries + 1;
|
||||
const recoveryError = await recoverTerminalOwnerInvitation({ event, nextRetries, nowDate, options, logger, reporter });
|
||||
if (recoveryError) {
|
||||
errors += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "error");
|
||||
await store.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: {
|
||||
status: "PENDING",
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(recoveryError.message),
|
||||
scheduledAt: new Date(nowDate.getTime() + computeOutboxRetryDelayMs(nextRetries)),
|
||||
processedAt: null,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (isTerminal) {
|
||||
deadLetters += 1;
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
if (!targetQueue) {
|
||||
// Unknown event-type → dead-letter immediately so it shows up in
|
||||
// admin UI and Sentry alerts. Do NOT retry — the type will never
|
||||
// route until the operator adds a rule, and silent retries waste
|
||||
// throughput.
|
||||
const message = `No routing rule matched eventType ${event.eventType}.`;
|
||||
logger.error({ eventId: event.id, eventType: event.eventType }, message);
|
||||
reporter?.captureException?.(new Error(message), {
|
||||
component: "outbox-dispatcher",
|
||||
op: "route",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
});
|
||||
const terminal = await store.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: {
|
||||
status: "FAILED",
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(message),
|
||||
lastError: message,
|
||||
processedAt: nowDate,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
const delayMs = computeOutboxRetryDelayMs(nextRetries);
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
if (terminal.count === 1) {
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (isPortalSetupMailIntent(event.payload) && !isEncryptedPortalSetupInvitationMailPayload(event.payload)) {
|
||||
throw new Error("Portal setup invitation encrypted mail contract is invalid.");
|
||||
}
|
||||
if (isWorkspaceOwnerInvitationMailIntent(event.payload) && !hasValidWorkspaceOwnerInvitationMailContract(event.payload)) {
|
||||
throw new Error("Workspace-owner invitation mail contract is invalid.");
|
||||
}
|
||||
await enqueue(targetQueue, event);
|
||||
// Privileged consumers own their durable terminal receipt. BullMQ may
|
||||
// run the job before `queue.add` returns, so dispatcher-side DONE
|
||||
// would erase the durable authority before the consumer can prove the
|
||||
// exact tenant/generation/payload/lease. The audit consumer also
|
||||
// commits its AuditEvent and DONE receipt in one DB transaction.
|
||||
if (
|
||||
event.eventType === PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_EVENT ||
|
||||
event.eventType === RECURRING_INVOICE_COLLECTION_EVENT ||
|
||||
event.eventType === AUDIT_EVENT_OUTBOX_TYPE
|
||||
) {
|
||||
ok += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "ok");
|
||||
return;
|
||||
}
|
||||
const redactedPayload = redactPortalSetupInvitationMailPayload(redactWorkspaceOwnerInvitationMailPayload(event.payload));
|
||||
const completed = await store.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: {
|
||||
status: "PENDING",
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(message),
|
||||
scheduledAt: new Date(nowDate.getTime() + delayMs),
|
||||
status: "DONE",
|
||||
...(redactedPayload !== event.payload ? { payload: redactedPayload } : {}),
|
||||
processedAt: nowDate,
|
||||
lastError: null,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
if (completed.count === 1) {
|
||||
ok += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "ok");
|
||||
} else {
|
||||
errors += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "error");
|
||||
logger.warn?.(
|
||||
{ eventId: event.id, eventType: event.eventType, deliveryGeneration: event.deliveryGeneration },
|
||||
"Outbox completion lost its generation/status race; terminal or replay state was preserved."
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
errors += 1;
|
||||
const nextRetries = event.retries + 1;
|
||||
const isTerminal = nextRetries > maxRetries;
|
||||
const safeError = redactInvitationDispatchError(error, event.payload);
|
||||
const message = safeError.message;
|
||||
logger.error({ eventId: event.id, eventType: event.eventType, retries: nextRetries, error: safeError }, "Outbox dispatch failed.");
|
||||
reporter?.captureException?.(safeError, {
|
||||
component: "outbox-dispatcher",
|
||||
op: "enqueue",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
retries: nextRetries,
|
||||
});
|
||||
|
||||
if (isTerminal) {
|
||||
const lastError = truncateError(message);
|
||||
if (isPortalSetupMailIntent(event.payload)) {
|
||||
let redactedPayload = redactPortalSetupInvitationMailPayload(event.payload);
|
||||
let terminalizedAtomically = false;
|
||||
let recoveryError: Error | null = null;
|
||||
const terminalizePortalSetupInvitation = options.terminalizePortalSetupInvitation;
|
||||
let hasRecoverableContract = false;
|
||||
if (terminalizePortalSetupInvitation && event.tenantId && event.dispatchKey && isEncryptedPortalSetupInvitationMailPayload(event.payload)) {
|
||||
try {
|
||||
redactedPayload = bindTerminalPortalSetupInvitationMailPayload(event.payload, {
|
||||
tenantId: event.tenantId,
|
||||
outboxEventId: event.id,
|
||||
dispatchKey: event.dispatchKey,
|
||||
});
|
||||
hasRecoverableContract = true;
|
||||
} catch (bindingError) {
|
||||
recoveryError = redactPortalSetupInvitationError(bindingError, event.payload);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
redactedPayload = redactTerminalPortalSetupInvitationMailPayload(event.payload);
|
||||
} catch (redactionError) {
|
||||
recoveryError = redactPortalSetupInvitationError(redactionError, event.payload);
|
||||
}
|
||||
}
|
||||
if (terminalizePortalSetupInvitation && hasRecoverableContract) {
|
||||
try {
|
||||
await terminalizePortalSetupInvitation({
|
||||
event,
|
||||
redactedPayload,
|
||||
retries: nextRetries,
|
||||
lastError,
|
||||
processedAt: nowDate,
|
||||
});
|
||||
terminalizedAtomically = true;
|
||||
} catch (terminalizationError) {
|
||||
const safeTerminalizationError = redactPortalSetupInvitationError(terminalizationError, event.payload);
|
||||
recoveryError = safeTerminalizationError;
|
||||
logger.error(
|
||||
{ eventId: event.id, eventType: event.eventType, error: safeTerminalizationError },
|
||||
"Terminal portal setup invitation invalidation failed; scheduling a durable redacted retry."
|
||||
);
|
||||
reporter?.captureException?.(safeTerminalizationError, {
|
||||
component: "outbox-dispatcher",
|
||||
op: "portal-setup-invitation-terminalize",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!terminalizedAtomically) {
|
||||
const recoveryPending = hasRecoverableContract && recoveryError !== null;
|
||||
const terminal = await store.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: {
|
||||
status: recoveryPending ? "PENDING" : "FAILED",
|
||||
...(redactedPayload !== event.payload ? { payload: redactedPayload } : {}),
|
||||
retries: nextRetries,
|
||||
lastError: recoveryError ? truncateError(recoveryError.message) : lastError,
|
||||
...(recoveryPending
|
||||
? {
|
||||
scheduledAt: new Date(nowDate.getTime() + computeOutboxRetryDelayMs(nextRetries)),
|
||||
processedAt: null,
|
||||
}
|
||||
: { processedAt: nowDate }),
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
if (terminal.count !== 1) {
|
||||
logger.warn?.(
|
||||
{ eventId: event.id, eventType: event.eventType, deliveryGeneration: event.deliveryGeneration },
|
||||
"Portal setup outbox terminalization lost its generation/status race; newer state was preserved."
|
||||
);
|
||||
} else if (recoveryPending) {
|
||||
metrics?.recordDispatch?.(event.eventType, "error");
|
||||
} else {
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
}
|
||||
} else {
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let redactedPayload = redactTerminalWorkspaceOwnerInvitationMailPayload(event.payload);
|
||||
let terminalizedAtomically = false;
|
||||
let recoveryError: Error | null = null;
|
||||
const terminalizeOwnerInvitation = options.terminalizeOwnerInvitation;
|
||||
let hasRecoverableContract = false;
|
||||
if (terminalizeOwnerInvitation && event.tenantId && event.dispatchKey && hasValidWorkspaceOwnerInvitationMailContract(event.payload)) {
|
||||
try {
|
||||
redactedPayload = bindTerminalWorkspaceOwnerInvitationMailPayload(event.payload, {
|
||||
tenantId: event.tenantId,
|
||||
outboxEventId: event.id,
|
||||
dispatchKey: event.dispatchKey,
|
||||
});
|
||||
hasRecoverableContract = true;
|
||||
} catch (bindingError) {
|
||||
recoveryError = redactWorkspaceOwnerInvitationError(bindingError, event.payload);
|
||||
}
|
||||
}
|
||||
if (terminalizeOwnerInvitation && hasRecoverableContract) {
|
||||
try {
|
||||
await terminalizeOwnerInvitation({
|
||||
event,
|
||||
redactedPayload,
|
||||
retries: nextRetries,
|
||||
lastError,
|
||||
processedAt: nowDate,
|
||||
});
|
||||
terminalizedAtomically = true;
|
||||
} catch (terminalizationError) {
|
||||
const safeTerminalizationError = redactWorkspaceOwnerInvitationError(terminalizationError, event.payload);
|
||||
recoveryError = safeTerminalizationError;
|
||||
logger.error(
|
||||
{ eventId: event.id, eventType: event.eventType, error: safeTerminalizationError },
|
||||
"Terminal workspace-owner invitation invalidation failed; scheduling a durable redacted retry."
|
||||
);
|
||||
reporter?.captureException?.(safeTerminalizationError, {
|
||||
component: "outbox-dispatcher",
|
||||
op: "owner-invitation-terminalize",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!terminalizedAtomically) {
|
||||
const recoveryPending = hasRecoverableContract && recoveryError !== null;
|
||||
const terminal = await store.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: {
|
||||
status: recoveryPending ? "PENDING" : "FAILED",
|
||||
...(redactedPayload !== event.payload ? { payload: redactedPayload } : {}),
|
||||
retries: nextRetries,
|
||||
lastError: recoveryError ? truncateError(recoveryError.message) : lastError,
|
||||
...(recoveryPending
|
||||
? {
|
||||
scheduledAt: new Date(nowDate.getTime() + computeOutboxRetryDelayMs(nextRetries)),
|
||||
processedAt: null,
|
||||
}
|
||||
: { processedAt: nowDate }),
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
if (terminal.count !== 1) {
|
||||
logger.warn?.(
|
||||
{ eventId: event.id, eventType: event.eventType, deliveryGeneration: event.deliveryGeneration },
|
||||
"Outbox terminalization lost its generation/status race; newer state was preserved."
|
||||
);
|
||||
} else if (recoveryPending) {
|
||||
metrics?.recordDispatch?.(event.eventType, "error");
|
||||
} else {
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
}
|
||||
} else {
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
}
|
||||
} else {
|
||||
metrics?.recordDispatch?.(event.eventType, "error");
|
||||
const delayMs = computeOutboxRetryDelayMs(nextRetries);
|
||||
await store.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: event.id,
|
||||
status: "PROCESSING",
|
||||
deliveryGeneration: event.deliveryGeneration,
|
||||
lockedUntil: event.lockedUntil,
|
||||
},
|
||||
data: {
|
||||
status: "PENDING",
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(message),
|
||||
scheduledAt: new Date(nowDate.getTime() + delayMs),
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
if (claimed.length > 0) {
|
||||
@@ -229,6 +536,86 @@ export async function runDispatcherTick(options: DispatcherOptions): Promise<Dis
|
||||
return { claimed: claimed.length, ok, errors, deadLetters };
|
||||
}
|
||||
|
||||
async function recoverTerminalPortalSetupInvitation(input: {
|
||||
event: OutboxEvent;
|
||||
nextRetries: number;
|
||||
nowDate: Date;
|
||||
options: DispatcherOptions;
|
||||
logger: DispatcherLogger;
|
||||
reporter: DispatcherReporter | undefined;
|
||||
}): Promise<Error | null> {
|
||||
const { event, nextRetries, nowDate, options, logger, reporter } = input;
|
||||
try {
|
||||
if (!options.terminalizePortalSetupInvitation) {
|
||||
throw new Error("Portal setup invitation terminal recovery is not wired.");
|
||||
}
|
||||
await options.terminalizePortalSetupInvitation({
|
||||
event,
|
||||
redactedPayload: event.payload,
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(event.lastError ?? "Portal setup invitation delivery failed; capability rotation required."),
|
||||
processedAt: nowDate,
|
||||
});
|
||||
return null;
|
||||
} catch (error) {
|
||||
const safeError = redactPortalSetupInvitationError(error, event.payload);
|
||||
logger.error(
|
||||
{ eventId: event.id, eventType: event.eventType, retries: nextRetries, error: safeError },
|
||||
"Portal setup invitation terminal recovery failed; retry remains durable."
|
||||
);
|
||||
reporter?.captureException?.(safeError, {
|
||||
component: "outbox-dispatcher",
|
||||
op: "portal-setup-invitation-terminal-recovery",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
retries: nextRetries,
|
||||
});
|
||||
return safeError;
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverTerminalOwnerInvitation(input: {
|
||||
event: OutboxEvent;
|
||||
nextRetries: number;
|
||||
nowDate: Date;
|
||||
options: DispatcherOptions;
|
||||
logger: DispatcherLogger;
|
||||
reporter: DispatcherReporter | undefined;
|
||||
}): Promise<Error | null> {
|
||||
const { event, nextRetries, nowDate, options, logger, reporter } = input;
|
||||
try {
|
||||
if (!options.terminalizeOwnerInvitation) {
|
||||
throw new Error("Workspace-owner invitation terminal recovery is not wired.");
|
||||
}
|
||||
await options.terminalizeOwnerInvitation({
|
||||
event,
|
||||
redactedPayload: event.payload,
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(event.lastError ?? "Workspace-owner invitation delivery failed; capability rotation required."),
|
||||
processedAt: nowDate,
|
||||
});
|
||||
return null;
|
||||
} catch (error) {
|
||||
const safeError = redactWorkspaceOwnerInvitationError(error, event.payload);
|
||||
logger.error(
|
||||
{ eventId: event.id, eventType: event.eventType, retries: nextRetries, error: safeError },
|
||||
"Workspace-owner invitation terminal recovery failed; retry remains durable."
|
||||
);
|
||||
reporter?.captureException?.(safeError, {
|
||||
component: "outbox-dispatcher",
|
||||
op: "owner-invitation-terminal-recovery",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
retries: nextRetries,
|
||||
});
|
||||
return safeError;
|
||||
}
|
||||
}
|
||||
|
||||
function redactInvitationDispatchError(error: unknown, payload: unknown): Error {
|
||||
return redactPortalSetupInvitationError(redactWorkspaceOwnerInvitationError(error, payload), payload);
|
||||
}
|
||||
|
||||
const MAX_LAST_ERROR_LENGTH = 1_000;
|
||||
|
||||
function truncateError(message: string): string {
|
||||
|
||||
+48
-2
@@ -1,6 +1,12 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { PORTAL_STATUS_MAIL_OUTBOX_QUEUE } from "@hivegrid/foundation";
|
||||
import { queueNames } from "@hivegrid/queue";
|
||||
|
||||
import { EMAIL_SEQUENCE_TRIGGER_QUEUE } from "../jobs/email-sequence-trigger.js";
|
||||
import { CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE } from "../jobs/chat-realtime-wakeup.js";
|
||||
import { CHAT_LINK_PREVIEW_QUEUE } from "../jobs/chat-link-preview.js";
|
||||
import { PORTAL_DOMAIN_COMMAND_OUTBOX_QUEUE } from "../jobs/portal-domain-orchestration.js";
|
||||
import { PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_QUEUE } from "../jobs/product-acquisition-provider-side-effect.js";
|
||||
import { WORKFLOW_TRIGGER_OUTBOX_QUEUE } from "../jobs/workflow-trigger.js";
|
||||
import { DEFAULT_OUTBOX_ROUTING, resolveOutboxQueueName } from "./event-router.js";
|
||||
|
||||
@@ -9,6 +15,16 @@ describe("resolveOutboxQueueName", () => {
|
||||
expect(resolveOutboxQueueName("mail.welcome.send")).toBe("outbox.mail");
|
||||
});
|
||||
|
||||
it("routes portal status mail to its revalidation worker before generic mail", () => {
|
||||
expect(resolveOutboxQueueName("mail.portal-status.stage-changed")).toBe(PORTAL_STATUS_MAIL_OUTBOX_QUEUE);
|
||||
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({ prefix: "mail.portal-status.", queue: PORTAL_STATUS_MAIL_OUTBOX_QUEUE });
|
||||
});
|
||||
|
||||
it("routes portal GDPR requests to the private export worker", () => {
|
||||
expect(resolveOutboxQueueName("portal.gdpr_export.requested")).toBe(queueNames.exports);
|
||||
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({ prefix: "portal.gdpr_export.", queue: queueNames.exports });
|
||||
});
|
||||
|
||||
it("routes webhook.* to outbox.webhook", () => {
|
||||
expect(resolveOutboxQueueName("webhook.delivery.attempted")).toBe("outbox.webhook");
|
||||
expect(resolveOutboxQueueName("webhook.form-submission.deliver")).toBe("outbox.webhook");
|
||||
@@ -16,6 +32,11 @@ describe("resolveOutboxQueueName", () => {
|
||||
|
||||
it("routes notification.* to outbox.notification", () => {
|
||||
expect(resolveOutboxQueueName("notification.in_app.created")).toBe("outbox.notification");
|
||||
expect(resolveOutboxQueueName("notification.platform.requested.v1")).toBe("outbox.notification");
|
||||
});
|
||||
|
||||
it("routes the platform lead capture marker to its existing notification subscriber", () => {
|
||||
expect(resolveOutboxQueueName("platform.admin_lead.captured.v1")).toBe("outbox.notification");
|
||||
});
|
||||
|
||||
it("routes audit.* to outbox.audit", () => {
|
||||
@@ -34,6 +55,11 @@ describe("resolveOutboxQueueName", () => {
|
||||
expect(resolveOutboxQueueName("billing.invoice.paid")).toBe("outbox.notification");
|
||||
});
|
||||
|
||||
it("routes only the exact product-acquisition provider command to its privileged consumer", () => {
|
||||
expect(resolveOutboxQueueName("billing.product-acquisition.provider-side-effect")).toBe(PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_QUEUE);
|
||||
expect(resolveOutboxQueueName("billing.product-acquisition.provider-side-effect.untrusted")).toBe("outbox.notification");
|
||||
});
|
||||
|
||||
it("routes marketing.lead.* to the email-sequence trigger queue", () => {
|
||||
expect(resolveOutboxQueueName("marketing.lead.captured")).toBe(EMAIL_SEQUENCE_TRIGGER_QUEUE);
|
||||
expect(resolveOutboxQueueName("marketing.lead.convertedtocustomer")).toBe(EMAIL_SEQUENCE_TRIGGER_QUEUE);
|
||||
@@ -60,6 +86,26 @@ describe("resolveOutboxQueueName", () => {
|
||||
expect(resolveOutboxQueueName("plugin.hook.workflow.deleted")).toBe("outbox.plugin-hook");
|
||||
});
|
||||
|
||||
it("routes persistent chat realtime wakeups to their dedicated queue", () => {
|
||||
expect(resolveOutboxQueueName("chat.realtime.persistent")).toBe(CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE);
|
||||
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({
|
||||
prefix: "chat.realtime.",
|
||||
queue: CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE,
|
||||
});
|
||||
});
|
||||
|
||||
it("routes asynchronous chat link previews to their dedicated queue", () => {
|
||||
expect(resolveOutboxQueueName("chat.linkpreview.requested")).toBe(CHAT_LINK_PREVIEW_QUEUE);
|
||||
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({
|
||||
prefix: "chat.linkpreview.",
|
||||
queue: CHAT_LINK_PREVIEW_QUEUE,
|
||||
});
|
||||
});
|
||||
|
||||
it("routes custom-domain command intents to the private signing worker", () => {
|
||||
expect(resolveOutboxQueueName("portal.domain.command.requested")).toBe(PORTAL_DOMAIN_COMMAND_OUTBOX_QUEUE);
|
||||
});
|
||||
|
||||
it("returns undefined when no rule matches", () => {
|
||||
expect(resolveOutboxQueueName("unknown.entity.verb")).toBeUndefined();
|
||||
});
|
||||
@@ -80,8 +126,8 @@ describe("resolveOutboxQueueName", () => {
|
||||
it("exposes a frozen-style default routing list", () => {
|
||||
expect(DEFAULT_OUTBOX_ROUTING.length).toBeGreaterThan(0);
|
||||
for (const rule of DEFAULT_OUTBOX_ROUTING) {
|
||||
expect(rule.prefix.endsWith(".")).toBe(true);
|
||||
expect(rule.queue.startsWith("outbox.")).toBe(true);
|
||||
expect(rule.exact === true || rule.prefix.endsWith(".")).toBe(true);
|
||||
expect(rule.queue.startsWith("outbox.") || rule.queue === queueNames.exports).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -9,24 +9,54 @@
|
||||
* See docs/architecture/outbox-pattern.md.
|
||||
*/
|
||||
|
||||
/** Built-in routing prefixes. Order matters: longest-prefix wins. */
|
||||
export const DEFAULT_OUTBOX_ROUTING: ReadonlyArray<{ prefix: string; queue: string }> = [
|
||||
import { PORTAL_STATUS_MAIL_OUTBOX_QUEUE } from "@hivegrid/foundation";
|
||||
import { queueNames } from "@hivegrid/queue";
|
||||
|
||||
import { PORTAL_DOMAIN_COMMAND_OUTBOX_QUEUE } from "../jobs/portal-domain-orchestration.js";
|
||||
import { PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_QUEUE } from "../jobs/product-acquisition-provider-side-effect.js";
|
||||
import { RECURRING_INVOICE_COLLECTION_QUEUE } from "../jobs/recurring-invoice-collection.js";
|
||||
|
||||
/** Built-in routing rules. Order matters: longest matching rule wins. */
|
||||
export type EventRoutingRule = { prefix: string; queue: string; exact?: boolean };
|
||||
|
||||
export const DEFAULT_OUTBOX_ROUTING: ReadonlyArray<EventRoutingRule> = [
|
||||
{ prefix: "platform.admin_lead.captured.", queue: "outbox.notification" },
|
||||
{ prefix: "tenant.subscription.", queue: "outbox.notification" },
|
||||
// Portal GDPR exports are created by the portal SECURITY DEFINER
|
||||
// capability. Route the durable request straight to the import-group
|
||||
// exports queue; its handler validates and normalizes the outbox envelope
|
||||
// before touching the private artifact store.
|
||||
{ prefix: "portal.gdpr_export.", queue: queueNames.exports },
|
||||
{ prefix: "portal.domain.command.", queue: PORTAL_DOMAIN_COMMAND_OUTBOX_QUEUE },
|
||||
{ prefix: "mail.portal-status.", queue: PORTAL_STATUS_MAIL_OUTBOX_QUEUE },
|
||||
{ prefix: "mail.", queue: "outbox.mail" },
|
||||
{ prefix: "webhook.", queue: "outbox.webhook" },
|
||||
{ prefix: "notification.", queue: "outbox.notification" },
|
||||
{ prefix: "audit.", queue: "outbox.audit" },
|
||||
{ prefix: "workflow.", queue: "outbox.workflow" },
|
||||
// R-P12c: forward-executor continuations (initial dispatch, retry, delay).
|
||||
{ prefix: "automation.execution.", queue: "outbox.automation-executor" },
|
||||
{ prefix: "booking.", queue: "outbox.booking" },
|
||||
{ prefix: "search.index.", queue: "outbox.search-index" },
|
||||
{ prefix: "marketing.lead.", queue: "outbox.marketing" },
|
||||
{ prefix: "form.submission.", queue: "outbox.form-submission-route" },
|
||||
{
|
||||
prefix: "billing.product-acquisition.provider-side-effect",
|
||||
queue: PRODUCT_ACQUISITION_PROVIDER_SIDE_EFFECT_QUEUE,
|
||||
exact: true,
|
||||
},
|
||||
{
|
||||
prefix: "billing.recurring-invoice.collect",
|
||||
queue: RECURRING_INVOICE_COLLECTION_QUEUE,
|
||||
exact: true,
|
||||
},
|
||||
{ prefix: "billing.", queue: "outbox.notification" },
|
||||
{ prefix: "plugin.hook.", queue: "outbox.plugin-hook" },
|
||||
{ prefix: "plugin.capability-use.", queue: "outbox.plugin-capability-use" },
|
||||
{ prefix: "chat.realtime.", queue: "outbox.chat-realtime" },
|
||||
{ prefix: "chat.linkpreview.", queue: "outbox.chat-link-preview" },
|
||||
];
|
||||
|
||||
export type EventRoutingRule = { prefix: string; queue: string };
|
||||
|
||||
export type EventRouterResolveOptions = {
|
||||
rules?: ReadonlyArray<EventRoutingRule>;
|
||||
};
|
||||
@@ -36,18 +66,16 @@ export type EventRouterResolveOptions = {
|
||||
* routing rule matches — the caller (dispatcher) treats this as a
|
||||
* dead-letter condition.
|
||||
*/
|
||||
export function resolveOutboxQueueName(
|
||||
eventType: string,
|
||||
options: EventRouterResolveOptions = {}
|
||||
): string | undefined {
|
||||
export function resolveOutboxQueueName(eventType: string, options: EventRouterResolveOptions = {}): string | undefined {
|
||||
if (typeof eventType !== "string" || eventType.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const rules = options.rules ?? DEFAULT_OUTBOX_ROUTING;
|
||||
// Longest matching prefix wins.
|
||||
// Longest matching prefix or exact rule wins.
|
||||
let best: { prefix: string; queue: string } | undefined;
|
||||
for (const rule of rules) {
|
||||
if (eventType.startsWith(rule.prefix)) {
|
||||
const matches = rule.exact ? eventType === rule.prefix : eventType.startsWith(rule.prefix);
|
||||
if (matches) {
|
||||
if (!best || rule.prefix.length > best.prefix.length) {
|
||||
best = rule;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user