Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,300 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { bootstrapOutboxDispatcher, type OutboxDispatcherBootstrapDeps } from "./bootstrap.js";
|
||||
import type { DispatcherLogger } from "./dispatcher.js";
|
||||
import type { OutboxEvent } from "@toolctrl/db/outbox";
|
||||
|
||||
const sourceDir = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const queueRuntimeMocks = vi.hoisted(() => ({
|
||||
add: vi.fn<(name: string, data: unknown, options?: unknown) => Promise<void>>(async () => undefined),
|
||||
close: vi.fn(async () => undefined),
|
||||
createQueue: vi.fn(() => ({
|
||||
add: queueRuntimeMocks.add,
|
||||
close: queueRuntimeMocks.close,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/queue/runtime", () => ({
|
||||
createQueue: queueRuntimeMocks.createQueue,
|
||||
}));
|
||||
|
||||
function createPrismaStub(): OutboxDispatcherBootstrapDeps["prisma"] {
|
||||
return {
|
||||
$queryRaw: vi.fn(async () => []) as unknown as OutboxDispatcherBootstrapDeps["prisma"]["$queryRaw"],
|
||||
outboxEvent: {
|
||||
update: vi.fn(async () => ({}) as unknown),
|
||||
},
|
||||
} as unknown as OutboxDispatcherBootstrapDeps["prisma"];
|
||||
}
|
||||
|
||||
function createLogger(): DispatcherLogger {
|
||||
return {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("bootstrapOutboxDispatcher", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
queueRuntimeMocks.add.mockClear();
|
||||
queueRuntimeMocks.close.mockClear();
|
||||
queueRuntimeMocks.createQueue.mockClear();
|
||||
});
|
||||
|
||||
it("returns disabled when OUTBOX_DISPATCHER_ENABLED is unset", async () => {
|
||||
const logger = createLogger();
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{},
|
||||
{
|
||||
prisma: createPrismaStub(),
|
||||
enqueue: async () => undefined,
|
||||
logger,
|
||||
}
|
||||
);
|
||||
expect(result.status).toBe("disabled");
|
||||
expect(result.handle).toBeUndefined();
|
||||
expect(logger.info).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("imports outbox claim helpers through the dedicated db subpath", async () => {
|
||||
const source = await readFile(join(sourceDir, "bootstrap.ts"), "utf8");
|
||||
|
||||
expect(source).toContain('from "@toolctrl/db/outbox-claim"');
|
||||
expect(source).not.toMatch(/from\s+["']@toolctrl\/db["']/);
|
||||
});
|
||||
|
||||
it("returns disabled when OUTBOX_DISPATCHER_ENABLED is 'false'", async () => {
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{ OUTBOX_DISPATCHER_ENABLED: "false" },
|
||||
{
|
||||
prisma: createPrismaStub(),
|
||||
enqueue: async () => undefined,
|
||||
logger: createLogger(),
|
||||
}
|
||||
);
|
||||
expect(result.status).toBe("disabled");
|
||||
});
|
||||
|
||||
it("throws when disabled in required mode", async () => {
|
||||
await expect(
|
||||
bootstrapOutboxDispatcher(
|
||||
{ OUTBOX_DISPATCHER_ENABLED: "false", OUTBOX_DISPATCHER_REQUIRED: "true" },
|
||||
{
|
||||
prisma: createPrismaStub(),
|
||||
enqueue: async () => undefined,
|
||||
logger: createLogger(),
|
||||
}
|
||||
)
|
||||
).rejects.toThrow("Outbox dispatcher is required");
|
||||
});
|
||||
|
||||
it("starts the dispatcher when enabled and enqueue is provided", async () => {
|
||||
const enqueue = vi.fn(async () => undefined);
|
||||
const logger = createLogger();
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{ OUTBOX_DISPATCHER_ENABLED: "true", OUTBOX_DISPATCH_INTERVAL_MS: "10000" },
|
||||
{
|
||||
prisma: createPrismaStub(),
|
||||
enqueue,
|
||||
logger,
|
||||
}
|
||||
);
|
||||
expect(result.status).toBe("started");
|
||||
expect(result.handle).toBeDefined();
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ component: "outbox-dispatcher", pollIntervalMs: 10000 }),
|
||||
expect.any(String)
|
||||
);
|
||||
await result.handle?.close();
|
||||
});
|
||||
|
||||
it("passes configured dispatch concurrency into startup logs", async () => {
|
||||
const logger = createLogger();
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{ OUTBOX_DISPATCHER_ENABLED: "true", OUTBOX_DISPATCH_INTERVAL_MS: "10000", OUTBOX_DISPATCH_CONCURRENCY: "7" },
|
||||
{
|
||||
prisma: createPrismaStub(),
|
||||
enqueue: async () => undefined,
|
||||
logger,
|
||||
}
|
||||
);
|
||||
|
||||
expect(result.status).toBe("started");
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ component: "outbox-dispatcher", dispatchConcurrency: 7 }),
|
||||
expect.any(String)
|
||||
);
|
||||
await result.handle?.close();
|
||||
});
|
||||
|
||||
it("throws when enabled but no redis URL and no enqueue override", async () => {
|
||||
await expect(
|
||||
bootstrapOutboxDispatcher(
|
||||
{ OUTBOX_DISPATCHER_ENABLED: "true" },
|
||||
{ prisma: createPrismaStub(), logger: createLogger() }
|
||||
)
|
||||
).rejects.toThrow(/BULLMQ_REDIS_URL or REDIS_URL/);
|
||||
});
|
||||
|
||||
it("wires the default Redis enqueue path without forwarding raw outbox priority", async () => {
|
||||
vi.useFakeTimers();
|
||||
const event = createOutboxEvent();
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([event]);
|
||||
const logger = createLogger();
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
OUTBOX_DISPATCHER_ENABLED: "true",
|
||||
OUTBOX_DISPATCH_INTERVAL_MS: "5",
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
},
|
||||
{
|
||||
prisma,
|
||||
logger,
|
||||
}
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await result.handle?.close();
|
||||
|
||||
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledWith("outbox.workflow", expect.any(Object));
|
||||
expect(queueRuntimeMocks.add).toHaveBeenCalledWith(
|
||||
"workflow.trigger.event",
|
||||
{
|
||||
outboxEventId: "outbox-1",
|
||||
tenantId: "tenant-1",
|
||||
eventType: "workflow.trigger.event",
|
||||
payload: { workflowId: "workflow-1" },
|
||||
dispatchKey: "workflow:outbox-1",
|
||||
retries: 0,
|
||||
},
|
||||
{
|
||||
jobId: "outbox-outbox-1",
|
||||
attempts: 5,
|
||||
backoff: { type: "exponential", delay: 15_000 },
|
||||
removeOnComplete: 1_000,
|
||||
removeOnFail: 10_000,
|
||||
}
|
||||
);
|
||||
expect(queueRuntimeMocks.add.mock.calls[0]).toHaveLength(3);
|
||||
// BullMQ rejects custom job ids containing ":" at runtime, which the mocked
|
||||
// queue here would not surface — guard the separator explicitly.
|
||||
expect((queueRuntimeMocks.add.mock.calls[0][2] as { jobId: string }).jobId).not.toContain(":");
|
||||
expect(prisma.outboxEvent.update).toHaveBeenCalledWith({
|
||||
where: { id: "outbox-1" },
|
||||
data: expect.objectContaining({
|
||||
status: "DONE",
|
||||
lastError: null,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("wires dispatcher metrics through the bootstrap path", async () => {
|
||||
vi.useFakeTimers();
|
||||
const event = createOutboxEvent({ eventType: "workflow.trigger.event" });
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([event]);
|
||||
const metrics = { recordDispatch: vi.fn() };
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
OUTBOX_DISPATCHER_ENABLED: "true",
|
||||
OUTBOX_DISPATCH_INTERVAL_MS: "5",
|
||||
},
|
||||
{
|
||||
prisma,
|
||||
enqueue: vi.fn(async () => undefined),
|
||||
logger: createLogger(),
|
||||
metrics,
|
||||
}
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await result.handle?.close();
|
||||
|
||||
expect(metrics.recordDispatch).toHaveBeenCalledWith("workflow.trigger.event", "ok");
|
||||
});
|
||||
|
||||
it("closes cached BullMQ queues when the default enqueue handle closes", async () => {
|
||||
vi.useFakeTimers();
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([createOutboxEvent()]);
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
OUTBOX_DISPATCHER_ENABLED: "true",
|
||||
OUTBOX_DISPATCH_INTERVAL_MS: "5",
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
},
|
||||
{
|
||||
prisma,
|
||||
logger: createLogger(),
|
||||
}
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await result.handle?.close();
|
||||
|
||||
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledTimes(1);
|
||||
expect(queueRuntimeMocks.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uses deterministic BullMQ job ids so redelivered outbox events dedupe", async () => {
|
||||
vi.useFakeTimers();
|
||||
const prisma = createPrismaStub();
|
||||
vi.mocked(prisma.$queryRaw)
|
||||
.mockResolvedValueOnce([createOutboxEvent()])
|
||||
.mockResolvedValueOnce([createOutboxEvent()]);
|
||||
|
||||
const result = await bootstrapOutboxDispatcher(
|
||||
{
|
||||
OUTBOX_DISPATCHER_ENABLED: "true",
|
||||
OUTBOX_DISPATCH_INTERVAL_MS: "5",
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
},
|
||||
{
|
||||
prisma,
|
||||
logger: createLogger(),
|
||||
}
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
await result.handle?.close();
|
||||
|
||||
expect(queueRuntimeMocks.add).toHaveBeenCalledTimes(2);
|
||||
expect(queueRuntimeMocks.add.mock.calls.map((call) => call[2])).toEqual([
|
||||
expect.objectContaining({ jobId: "outbox-outbox-1" }),
|
||||
expect.objectContaining({ jobId: "outbox-outbox-1" }),
|
||||
]);
|
||||
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
function createOutboxEvent(overrides: Partial<OutboxEvent> = {}): OutboxEvent {
|
||||
const now = new Date("2026-05-27T01:00:00.000Z");
|
||||
return {
|
||||
id: "outbox-1",
|
||||
tenantId: "tenant-1",
|
||||
eventType: "workflow.trigger.event",
|
||||
payload: { workflowId: "workflow-1" },
|
||||
status: "PROCESSING",
|
||||
createdAt: now,
|
||||
processedAt: null,
|
||||
retries: 0,
|
||||
lastError: null,
|
||||
scheduledAt: now,
|
||||
dispatchKey: "workflow:outbox-1",
|
||||
priority: 50,
|
||||
lockedUntil: new Date(now.getTime() + 60_000),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import { claimPendingOutboxEvents, OUTBOX_CLAIM_DEFAULT_LIMIT, type OutboxClaimRunner } from "@toolctrl/db/outbox-claim";
|
||||
import type { OutboxEvent } from "@toolctrl/db/outbox";
|
||||
import type { ErrorReporter } from "@toolctrl/error-reporter";
|
||||
import { parseRedisUrl } from "@toolctrl/queue";
|
||||
import { workerLogger } from "../logger.js";
|
||||
import { getWorkerMetrics } from "../metrics.js";
|
||||
import { workerErrorReporter } from "../sentry.js";
|
||||
import {
|
||||
OUTBOX_DEFAULT_BATCH_SIZE,
|
||||
OUTBOX_DEFAULT_DISPATCH_CONCURRENCY,
|
||||
OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
OUTBOX_DEFAULT_POLL_INTERVAL_MS,
|
||||
isOutboxDispatcherEnabled,
|
||||
isOutboxDispatcherRequired,
|
||||
startOutboxDispatcher,
|
||||
type DispatcherHandle,
|
||||
type DispatcherLogger,
|
||||
type DispatcherMetrics,
|
||||
type DispatcherQueueEnqueuer,
|
||||
type DispatcherTransactionRunner,
|
||||
} from "./dispatcher.js";
|
||||
|
||||
/**
|
||||
* Bootstrap helper that wires the outbox dispatcher into the worker
|
||||
* runtime. Returns `{ status: "disabled" }` when `OUTBOX_DISPATCHER_ENABLED`
|
||||
* is not exactly `"true"` so dev environments default to "dispatcher off".
|
||||
*
|
||||
* See docs/architecture/outbox-pattern.md.
|
||||
*/
|
||||
|
||||
export type OutboxDispatcherBootstrapEnv = EnvRecord & {
|
||||
OUTBOX_DISPATCHER_ENABLED?: string;
|
||||
OUTBOX_DISPATCHER_REQUIRED?: string;
|
||||
OUTBOX_DISPATCH_BATCH_SIZE?: string;
|
||||
OUTBOX_DISPATCH_CONCURRENCY?: string;
|
||||
OUTBOX_DISPATCH_INTERVAL_MS?: string;
|
||||
OUTBOX_MAX_RETRIES?: string;
|
||||
BULLMQ_REDIS_URL?: string;
|
||||
REDIS_URL?: string;
|
||||
};
|
||||
|
||||
export type OutboxDispatcherBootstrapDeps = {
|
||||
prisma: OutboxClaimRunner & DispatcherTransactionRunner;
|
||||
enqueue?: DispatcherQueueEnqueuer;
|
||||
logger?: DispatcherLogger;
|
||||
metrics?: DispatcherMetrics;
|
||||
reporter?: ErrorReporter;
|
||||
};
|
||||
|
||||
export type OutboxDispatcherBootstrapResult = {
|
||||
status: "started" | "disabled";
|
||||
handle?: DispatcherHandle;
|
||||
};
|
||||
|
||||
const OUTBOX_QUEUE_JOB_ATTEMPTS = 5;
|
||||
const OUTBOX_QUEUE_JOB_BACKOFF_MS = 15_000;
|
||||
const OUTBOX_QUEUE_REMOVE_ON_COMPLETE = 1_000;
|
||||
const OUTBOX_QUEUE_REMOVE_ON_FAIL = 10_000;
|
||||
|
||||
type ClosableDispatcherQueueEnqueuer = DispatcherQueueEnqueuer & {
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
function parseInteger(value: string | undefined, fallback: number): number {
|
||||
if (!value) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a default BullMQ enqueue function that fans events out to the
|
||||
* target queue determined by the event router. The implementation lazy-
|
||||
* imports the bullmq runtime so tests can inject `enqueue` without
|
||||
* loading bullmq at all.
|
||||
*/
|
||||
async function createDefaultEnqueue(redisUrl: string): Promise<ClosableDispatcherQueueEnqueuer> {
|
||||
const connection = parseRedisUrl(redisUrl);
|
||||
const { createQueue } = (await import("@toolctrl/queue/runtime")) as {
|
||||
createQueue: (name: string, options: { connection: typeof connection }) => {
|
||||
add(name: string, data: unknown, options?: unknown): Promise<unknown>;
|
||||
close?: () => Promise<void>;
|
||||
};
|
||||
};
|
||||
const queueCache = new Map<string, { add(name: string, data: unknown, options?: unknown): Promise<unknown>; close?: () => Promise<void> }>();
|
||||
|
||||
const enqueue = (async (queueName: string, event: OutboxEvent) => {
|
||||
let queue = queueCache.get(queueName);
|
||||
if (!queue) {
|
||||
queue = createQueue(queueName, { connection });
|
||||
queueCache.set(queueName, queue);
|
||||
}
|
||||
await queue.add(
|
||||
event.eventType,
|
||||
{
|
||||
outboxEventId: event.id,
|
||||
tenantId: event.tenantId,
|
||||
eventType: event.eventType,
|
||||
payload: event.payload,
|
||||
dispatchKey: event.dispatchKey,
|
||||
retries: event.retries,
|
||||
},
|
||||
{
|
||||
// BullMQ rejects custom job ids containing ":" ("Custom Id cannot
|
||||
// contain :"), which silently failed every outbox dispatch. Use "-";
|
||||
// workflow-execution-stalled-reconciler.buildOutboxJobId must match.
|
||||
jobId: `outbox-${event.id}`,
|
||||
attempts: OUTBOX_QUEUE_JOB_ATTEMPTS,
|
||||
backoff: { type: "exponential", delay: OUTBOX_QUEUE_JOB_BACKOFF_MS },
|
||||
removeOnComplete: OUTBOX_QUEUE_REMOVE_ON_COMPLETE,
|
||||
removeOnFail: OUTBOX_QUEUE_REMOVE_ON_FAIL,
|
||||
}
|
||||
);
|
||||
// OutboxEvent.priority is claim-only here: Postgres dispatches higher
|
||||
// values first, while BullMQ treats 1 as highest priority. Passing the
|
||||
// raw value would invert semantics after claim, so keep queue enqueue
|
||||
// FIFO until a reviewed mapping is introduced.
|
||||
}) as ClosableDispatcherQueueEnqueuer;
|
||||
|
||||
enqueue.close = async () => {
|
||||
const queues = [...queueCache.values()];
|
||||
queueCache.clear();
|
||||
await Promise.all(queues.map((queue) => queue.close?.() ?? Promise.resolve()));
|
||||
};
|
||||
|
||||
return enqueue;
|
||||
}
|
||||
|
||||
export async function bootstrapOutboxDispatcher(
|
||||
env: OutboxDispatcherBootstrapEnv,
|
||||
deps: OutboxDispatcherBootstrapDeps
|
||||
): Promise<OutboxDispatcherBootstrapResult> {
|
||||
const logger = deps.logger ?? workerLogger;
|
||||
if (!isOutboxDispatcherEnabled(env)) {
|
||||
if (isOutboxDispatcherRequired(env)) {
|
||||
throw new Error("Outbox dispatcher is required but OUTBOX_DISPATCHER_ENABLED is not true.");
|
||||
}
|
||||
logger.info?.({ component: "outbox-dispatcher" }, "Outbox dispatcher disabled (OUTBOX_DISPATCHER_ENABLED!=true).");
|
||||
return { status: "disabled" };
|
||||
}
|
||||
|
||||
const batchSize = parseInteger(env.OUTBOX_DISPATCH_BATCH_SIZE, OUTBOX_DEFAULT_BATCH_SIZE);
|
||||
const dispatchConcurrency = parseInteger(env.OUTBOX_DISPATCH_CONCURRENCY, OUTBOX_DEFAULT_DISPATCH_CONCURRENCY);
|
||||
const pollIntervalMs = parseInteger(env.OUTBOX_DISPATCH_INTERVAL_MS, OUTBOX_DEFAULT_POLL_INTERVAL_MS);
|
||||
const maxRetries = parseInteger(env.OUTBOX_MAX_RETRIES, OUTBOX_DEFAULT_MAX_RETRIES);
|
||||
|
||||
let enqueue: DispatcherQueueEnqueuer | undefined = deps.enqueue;
|
||||
let closeDefaultEnqueue: (() => Promise<void>) | undefined;
|
||||
if (!enqueue) {
|
||||
const redisUrl = env.BULLMQ_REDIS_URL ?? env.REDIS_URL;
|
||||
if (!redisUrl) {
|
||||
throw new Error("Outbox dispatcher requires BULLMQ_REDIS_URL or REDIS_URL when no custom enqueue is provided.");
|
||||
}
|
||||
const defaultEnqueue = await createDefaultEnqueue(redisUrl);
|
||||
enqueue = defaultEnqueue;
|
||||
closeDefaultEnqueue = defaultEnqueue.close;
|
||||
}
|
||||
|
||||
const reporter = deps.reporter ?? workerErrorReporter;
|
||||
const metrics =
|
||||
deps.metrics ??
|
||||
({
|
||||
recordDispatch(eventType, status) {
|
||||
getWorkerMetrics().metrics.outboxDispatchTotal.labels({ event_type: eventType, status }).inc();
|
||||
},
|
||||
} satisfies DispatcherMetrics);
|
||||
const dispatcherHandle = startOutboxDispatcher({
|
||||
claim: (limit, now) => claimPendingOutboxEvents(deps.prisma, { limit, now }),
|
||||
enqueue,
|
||||
store: deps.prisma,
|
||||
batchSize,
|
||||
dispatchConcurrency,
|
||||
pollIntervalMs,
|
||||
maxRetries,
|
||||
logger,
|
||||
metrics,
|
||||
reporter: {
|
||||
captureException: (error, context) => {
|
||||
try {
|
||||
reporter.report({
|
||||
component: "outbox-dispatcher",
|
||||
operation: typeof context?.op === "string" ? context.op : "dispatch",
|
||||
severity: "error",
|
||||
error,
|
||||
...(context ?? {}),
|
||||
});
|
||||
} catch {
|
||||
// Reporter failures must never mask dispatcher progress.
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
const handle = closeDefaultEnqueue
|
||||
? {
|
||||
get running() {
|
||||
return dispatcherHandle.running;
|
||||
},
|
||||
async close() {
|
||||
try {
|
||||
await dispatcherHandle.close();
|
||||
} finally {
|
||||
await closeDefaultEnqueue?.();
|
||||
}
|
||||
},
|
||||
}
|
||||
: dispatcherHandle;
|
||||
|
||||
logger.info?.(
|
||||
{
|
||||
component: "outbox-dispatcher",
|
||||
batchSize,
|
||||
dispatchConcurrency,
|
||||
pollIntervalMs,
|
||||
maxRetries,
|
||||
},
|
||||
"Outbox dispatcher started."
|
||||
);
|
||||
|
||||
return { status: "started", handle };
|
||||
}
|
||||
|
||||
export { OUTBOX_CLAIM_DEFAULT_LIMIT };
|
||||
@@ -0,0 +1,320 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { OutboxEvent } from "@toolctrl/db/outbox";
|
||||
import {
|
||||
OUTBOX_DEFAULT_DISPATCH_CONCURRENCY,
|
||||
OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
OUTBOX_DEFAULT_POLL_INTERVAL_MS,
|
||||
OUTBOX_RETRY_BACKOFF_MINUTES,
|
||||
computeOutboxRetryDelayMs,
|
||||
isOutboxDispatcherEnabled,
|
||||
isOutboxDispatcherRequired,
|
||||
runDispatcherTick,
|
||||
startOutboxDispatcher,
|
||||
type DispatcherClaim,
|
||||
type DispatcherQueueEnqueuer,
|
||||
type DispatcherTransactionRunner,
|
||||
} from "./dispatcher.js";
|
||||
|
||||
type FakeEvent = OutboxEvent;
|
||||
|
||||
type Updates = Array<{ id: string; data: Record<string, unknown> }>;
|
||||
|
||||
function makeStore(updates: Updates): DispatcherTransactionRunner {
|
||||
return {
|
||||
outboxEvent: {
|
||||
update: vi.fn(async ({ where, data }) => {
|
||||
updates.push({ id: where.id, data });
|
||||
return { id: where.id, ...data } as unknown as FakeEvent;
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeEvent(overrides: Partial<FakeEvent> = {}): FakeEvent {
|
||||
return {
|
||||
id: "evt_1",
|
||||
tenantId: "tenant_1",
|
||||
eventType: "mail.welcome.send",
|
||||
payload: { to: "user@example.com" },
|
||||
status: "PROCESSING",
|
||||
createdAt: new Date("2026-05-17T08:00:00.000Z"),
|
||||
processedAt: null,
|
||||
retries: 0,
|
||||
lastError: null,
|
||||
scheduledAt: new Date("2026-05-17T08:00:00.000Z"),
|
||||
dispatchKey: null,
|
||||
priority: 0,
|
||||
lockedUntil: new Date("2026-05-17T08:05:00.000Z"),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("computeOutboxRetryDelayMs", () => {
|
||||
it("returns 1 minute on the first retry", () => {
|
||||
expect(computeOutboxRetryDelayMs(1)).toBe(60_000);
|
||||
});
|
||||
|
||||
it("doubles per retry up to the schedule cap", () => {
|
||||
expect(computeOutboxRetryDelayMs(2)).toBe(2 * 60_000);
|
||||
expect(computeOutboxRetryDelayMs(3)).toBe(4 * 60_000);
|
||||
expect(computeOutboxRetryDelayMs(4)).toBe(8 * 60_000);
|
||||
expect(computeOutboxRetryDelayMs(6)).toBe(32 * 60_000);
|
||||
});
|
||||
|
||||
it("caps at the last schedule entry for retries beyond the schedule", () => {
|
||||
expect(computeOutboxRetryDelayMs(99)).toBe(32 * 60_000);
|
||||
});
|
||||
|
||||
it("falls back to the first entry when retries < 1", () => {
|
||||
expect(computeOutboxRetryDelayMs(0)).toBe(60_000);
|
||||
expect(computeOutboxRetryDelayMs(-3)).toBe(60_000);
|
||||
});
|
||||
|
||||
it("exposes the documented schedule", () => {
|
||||
expect([...OUTBOX_RETRY_BACKOFF_MINUTES]).toEqual([1, 2, 4, 8, 16, 32]);
|
||||
expect(OUTBOX_DEFAULT_MAX_RETRIES).toBe(6);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runDispatcherTick", () => {
|
||||
const now = new Date("2026-05-17T09:00:00.000Z");
|
||||
|
||||
it("returns empty result when claim throws", async () => {
|
||||
const claim: DispatcherClaim = vi.fn(async () => {
|
||||
throw new Error("db down");
|
||||
});
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn();
|
||||
const updates: Updates = [];
|
||||
const result = await runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
expect(result).toEqual({ claimed: 0, ok: 0, errors: 0, deadLetters: 0 });
|
||||
expect(enqueue).not.toHaveBeenCalled();
|
||||
expect(updates).toEqual([]);
|
||||
});
|
||||
|
||||
it("marks events DONE on successful enqueue", async () => {
|
||||
const event = makeEvent();
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
|
||||
const updates: Updates = [];
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
|
||||
expect(enqueue).toHaveBeenCalledWith("outbox.mail", event);
|
||||
expect(updates).toHaveLength(1);
|
||||
expect(updates[0]?.data.status).toBe("DONE");
|
||||
expect(updates[0]?.data.processedAt).toBe(now);
|
||||
expect(updates[0]?.data.lockedUntil).toBeNull();
|
||||
});
|
||||
|
||||
it("routes workflow trigger events to the workflow outbox receiver queue", async () => {
|
||||
const event = makeEvent({
|
||||
eventType: "workflow.trigger.row.created",
|
||||
payload: { event: "row.created", data: { id: "row_1" } },
|
||||
dispatchKey: "workflow:row.created:row_1",
|
||||
});
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
|
||||
const updates: Updates = [];
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
|
||||
expect(enqueue).toHaveBeenCalledWith("outbox.workflow", event);
|
||||
expect(updates[0]?.data).toMatchObject({ status: "DONE", processedAt: now, lockedUntil: null });
|
||||
});
|
||||
|
||||
it("dead-letters unknown event types without enqueueing", async () => {
|
||||
const event = makeEvent({ eventType: "no.routing.here" });
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn();
|
||||
const updates: Updates = [];
|
||||
const reporter = { captureException: vi.fn() };
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
reporter,
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.deadLetters).toBe(1);
|
||||
expect(enqueue).not.toHaveBeenCalled();
|
||||
expect(updates[0]?.data.status).toBe("FAILED");
|
||||
expect(reporter.captureException).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries with exponential backoff on enqueue failure", async () => {
|
||||
const event = makeEvent({ retries: 1 });
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => {
|
||||
throw new Error("redis offline");
|
||||
});
|
||||
const updates: Updates = [];
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.errors).toBe(1);
|
||||
expect(result.deadLetters).toBe(0);
|
||||
expect(updates[0]?.data.status).toBe("PENDING");
|
||||
expect(updates[0]?.data.retries).toBe(2);
|
||||
expect(updates[0]?.data.lastError).toContain("redis offline");
|
||||
// retries=2 → 2 minute backoff.
|
||||
const scheduled = updates[0]?.data.scheduledAt as Date;
|
||||
expect(scheduled.getTime() - now.getTime()).toBe(2 * 60_000);
|
||||
});
|
||||
|
||||
it("dead-letters after retries exceed the limit", async () => {
|
||||
const event = makeEvent({ retries: OUTBOX_DEFAULT_MAX_RETRIES });
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
const updates: Updates = [];
|
||||
|
||||
const result = await runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
now: () => now,
|
||||
logger: { error: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.deadLetters).toBe(1);
|
||||
expect(updates[0]?.data.status).toBe("FAILED");
|
||||
expect(updates[0]?.data.retries).toBe(OUTBOX_DEFAULT_MAX_RETRIES + 1);
|
||||
});
|
||||
|
||||
it("records metrics per event", async () => {
|
||||
const event = makeEvent();
|
||||
const claim: DispatcherClaim = vi.fn(async () => [event]);
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
|
||||
const metrics = { recordDispatch: vi.fn() };
|
||||
|
||||
await runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore([]),
|
||||
metrics,
|
||||
now: () => now,
|
||||
});
|
||||
|
||||
expect(metrics.recordDispatch).toHaveBeenCalledWith("mail.welcome.send", "ok");
|
||||
});
|
||||
|
||||
it("dispatches claimed events with bounded concurrency", async () => {
|
||||
const events = Array.from({ length: 4 }, (_, index) => makeEvent({ id: `evt_${index + 1}` }));
|
||||
const claim: DispatcherClaim = vi.fn(async () => events);
|
||||
const updates: Updates = [];
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const release: Array<() => void> = [];
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => {
|
||||
inFlight += 1;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise<void>((resolve) => release.push(resolve));
|
||||
inFlight -= 1;
|
||||
});
|
||||
|
||||
const tick = runDispatcherTick({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore(updates),
|
||||
dispatchConcurrency: 2,
|
||||
now: () => now,
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(release).toHaveLength(2));
|
||||
expect(maxInFlight).toBe(2);
|
||||
release.splice(0).forEach((resolve) => resolve());
|
||||
await vi.waitFor(() => expect(release).toHaveLength(2));
|
||||
expect(maxInFlight).toBe(2);
|
||||
release.splice(0).forEach((resolve) => resolve());
|
||||
|
||||
await expect(tick).resolves.toEqual({ claimed: 4, ok: 4, errors: 0, deadLetters: 0 });
|
||||
expect(updates).toHaveLength(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("startOutboxDispatcher", () => {
|
||||
it("invokes runDispatcherTick on the configured interval until close()", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const claim: DispatcherClaim = vi.fn(async () => []);
|
||||
const enqueue: DispatcherQueueEnqueuer = vi.fn();
|
||||
|
||||
const handle = startOutboxDispatcher({
|
||||
claim,
|
||||
enqueue,
|
||||
store: makeStore([]),
|
||||
pollIntervalMs: 100,
|
||||
logger: { error: vi.fn(), info: vi.fn() },
|
||||
});
|
||||
|
||||
expect(handle.running).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(350);
|
||||
// ~3 ticks should have fired
|
||||
expect((claim as unknown as { mock: { calls: unknown[] } }).mock.calls.length).toBeGreaterThanOrEqual(3);
|
||||
await handle.close();
|
||||
const callsAfterClose = (claim as unknown as { mock: { calls: unknown[] } }).mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
expect((claim as unknown as { mock: { calls: unknown[] } }).mock.calls.length).toBe(callsAfterClose);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("isOutboxDispatcherEnabled", () => {
|
||||
it("returns true only when OUTBOX_DISPATCHER_ENABLED is exactly 'true'", () => {
|
||||
expect(isOutboxDispatcherEnabled({ OUTBOX_DISPATCHER_ENABLED: "true" })).toBe(true);
|
||||
expect(isOutboxDispatcherEnabled({ OUTBOX_DISPATCHER_ENABLED: "TRUE" })).toBe(false);
|
||||
expect(isOutboxDispatcherEnabled({ OUTBOX_DISPATCHER_ENABLED: "1" })).toBe(false);
|
||||
expect(isOutboxDispatcherEnabled({})).toBe(false);
|
||||
});
|
||||
|
||||
it("returns true only when OUTBOX_DISPATCHER_REQUIRED is exactly 'true'", () => {
|
||||
expect(isOutboxDispatcherRequired({ OUTBOX_DISPATCHER_REQUIRED: "true" })).toBe(true);
|
||||
expect(isOutboxDispatcherRequired({ OUTBOX_DISPATCHER_REQUIRED: "TRUE" })).toBe(false);
|
||||
expect(isOutboxDispatcherRequired({ OUTBOX_DISPATCHER_REQUIRED: "1" })).toBe(false);
|
||||
expect(isOutboxDispatcherRequired({})).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("constants", () => {
|
||||
it("publishes a sensible default poll interval", () => {
|
||||
expect(OUTBOX_DEFAULT_POLL_INTERVAL_MS).toBe(1_000);
|
||||
});
|
||||
|
||||
it("publishes a conservative default dispatch concurrency", () => {
|
||||
expect(OUTBOX_DEFAULT_DISPATCH_CONCURRENCY).toBe(10);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,315 @@
|
||||
import type { OutboxEvent } from "@toolctrl/db/outbox";
|
||||
import { workerLogger } from "../logger.js";
|
||||
import { resolveOutboxQueueName, type EventRoutingRule } from "./event-router.js";
|
||||
|
||||
/**
|
||||
* Outbox dispatcher worker.
|
||||
*
|
||||
* Polls the OutboxEvent table on a fixed interval (default 1s), atomically
|
||||
* claims a batch via `claimPendingOutboxEvents`, routes each event to the
|
||||
* appropriate BullMQ queue via the event router, and marks the row DONE.
|
||||
*
|
||||
* Failures trigger a retry with exponential backoff (1, 2, 4, 8, 16, 32
|
||||
* minutes). After `OUTBOX_MAX_RETRIES` failures the row is marked FAILED
|
||||
* and stays in the dead-letter list for an operator to re-queue via CLI.
|
||||
*
|
||||
* The dispatcher is bootstrapped by the `WORKER_QUEUE_GROUP=general` worker
|
||||
* so it can scale independently from mail/import/webhook/plugin workers.
|
||||
*
|
||||
* See docs/architecture/outbox-pattern.md.
|
||||
*/
|
||||
|
||||
export const OUTBOX_DEFAULT_MAX_RETRIES = 6;
|
||||
export const OUTBOX_DEFAULT_BATCH_SIZE = 100;
|
||||
export const OUTBOX_DEFAULT_POLL_INTERVAL_MS = 1_000;
|
||||
export const OUTBOX_DEFAULT_DISPATCH_CONCURRENCY = 10;
|
||||
|
||||
/** Minutes — `2^retry`. Capped at OUTBOX_DEFAULT_MAX_RETRIES entries. */
|
||||
export const OUTBOX_RETRY_BACKOFF_MINUTES: readonly number[] = [1, 2, 4, 8, 16, 32];
|
||||
|
||||
/**
|
||||
* Compute the next `scheduledAt` for a failed dispatch. `retries` is the
|
||||
* count AFTER incrementing the counter (so retries=1 → 1 minute delay).
|
||||
*/
|
||||
export function computeOutboxRetryDelayMs(
|
||||
retries: number,
|
||||
schedule: readonly number[] = OUTBOX_RETRY_BACKOFF_MINUTES
|
||||
): number {
|
||||
if (retries < 1) return schedule[0] !== undefined ? schedule[0] * 60 * 1000 : 60_000;
|
||||
const idx = Math.min(retries - 1, schedule.length - 1);
|
||||
return (schedule[idx] ?? schedule.at(-1) ?? 1) * 60 * 1000;
|
||||
}
|
||||
|
||||
export type DispatcherLogger = {
|
||||
child?: (bindings: Record<string, unknown>) => DispatcherLogger;
|
||||
info?: (record: Record<string, unknown>, message?: string) => void;
|
||||
warn?: (record: Record<string, unknown>, message?: string) => void;
|
||||
error: (record: Record<string, unknown>, message?: string) => void;
|
||||
};
|
||||
|
||||
export type DispatcherMetrics = {
|
||||
/** Increment with labels `{event_type, status}`. */
|
||||
recordDispatch?: (event_type: string, status: "ok" | "error" | "dead_letter") => void;
|
||||
};
|
||||
|
||||
export type DispatcherReporter = {
|
||||
captureException?: (error: unknown, context?: Record<string, unknown>) => void;
|
||||
};
|
||||
|
||||
export type DispatcherQueueEnqueuer = (queueName: string, event: OutboxEvent) => Promise<void>;
|
||||
|
||||
export type DispatcherTransactionRunner = {
|
||||
outboxEvent: {
|
||||
update(args: {
|
||||
where: { id: string };
|
||||
data: Record<string, unknown>;
|
||||
}): Promise<OutboxEvent>;
|
||||
};
|
||||
};
|
||||
|
||||
export type DispatcherClaim = (limit: number, now: Date) => Promise<OutboxEvent[]>;
|
||||
|
||||
export type DispatcherOptions = {
|
||||
/** Atomic batch-claim function (bound to a Prisma client). */
|
||||
claim: DispatcherClaim;
|
||||
/** Transaction-style update hook used to flip event status. */
|
||||
store: DispatcherTransactionRunner;
|
||||
/** Enqueues a claimed event to its target BullMQ queue. */
|
||||
enqueue: DispatcherQueueEnqueuer;
|
||||
/** Optional max retries before dead-lettering (default 6). */
|
||||
maxRetries?: number;
|
||||
/** Optional batch size (default 100). */
|
||||
batchSize?: number;
|
||||
/** Optional poll interval ms (default 1000). */
|
||||
pollIntervalMs?: number;
|
||||
/** Optional per-tick enqueue/update concurrency (default 10). */
|
||||
dispatchConcurrency?: number;
|
||||
/** Optional routing rule overrides. */
|
||||
routingRules?: ReadonlyArray<EventRoutingRule>;
|
||||
/** Optional clock for deterministic tests. */
|
||||
now?: () => Date;
|
||||
/** Optional logger / metrics / Sentry-like reporter. */
|
||||
logger?: DispatcherLogger;
|
||||
metrics?: DispatcherMetrics;
|
||||
reporter?: DispatcherReporter;
|
||||
};
|
||||
|
||||
export type DispatcherTickResult = {
|
||||
claimed: number;
|
||||
ok: number;
|
||||
errors: number;
|
||||
deadLetters: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Execute one polling pass: claim a batch, enqueue each event, update
|
||||
* statuses. Returns counters for tests / metrics.
|
||||
*/
|
||||
export async function runDispatcherTick(options: DispatcherOptions): Promise<DispatcherTickResult> {
|
||||
const {
|
||||
claim,
|
||||
store,
|
||||
enqueue,
|
||||
maxRetries = OUTBOX_DEFAULT_MAX_RETRIES,
|
||||
batchSize = OUTBOX_DEFAULT_BATCH_SIZE,
|
||||
routingRules,
|
||||
now = () => new Date(),
|
||||
logger = workerLogger,
|
||||
metrics,
|
||||
reporter,
|
||||
} = options;
|
||||
|
||||
const nowDate = now();
|
||||
let claimed: OutboxEvent[] = [];
|
||||
try {
|
||||
claimed = await claim(batchSize, nowDate);
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Outbox dispatcher claim failed.");
|
||||
reporter?.captureException?.(error, { component: "outbox-dispatcher", op: "claim" });
|
||||
return { claimed: 0, ok: 0, errors: 0, deadLetters: 0 };
|
||||
}
|
||||
|
||||
let ok = 0;
|
||||
let errors = 0;
|
||||
let deadLetters = 0;
|
||||
const dispatchConcurrency = clampDispatchConcurrency(options.dispatchConcurrency);
|
||||
|
||||
for (let i = 0; i < claimed.length; i += dispatchConcurrency) {
|
||||
await Promise.all(claimed.slice(i, i + dispatchConcurrency).map(async (event) => {
|
||||
const targetQueue = resolveOutboxQueueName(event.eventType, routingRules ? { rules: routingRules } : undefined);
|
||||
|
||||
if (!targetQueue) {
|
||||
// Unknown event-type → dead-letter immediately so it shows up in
|
||||
// admin UI and Sentry alerts. Do NOT retry — the type will never
|
||||
// route until the operator adds a rule, and silent retries waste
|
||||
// throughput.
|
||||
deadLetters += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "dead_letter");
|
||||
const message = `No routing rule matched eventType ${event.eventType}.`;
|
||||
logger.error({ eventId: event.id, eventType: event.eventType }, message);
|
||||
reporter?.captureException?.(new Error(message), {
|
||||
component: "outbox-dispatcher",
|
||||
op: "route",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
});
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
lastError: message,
|
||||
processedAt: nowDate,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await enqueue(targetQueue, event);
|
||||
ok += 1;
|
||||
metrics?.recordDispatch?.(event.eventType, "ok");
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
data: {
|
||||
status: "DONE",
|
||||
processedAt: nowDate,
|
||||
lastError: null,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
errors += 1;
|
||||
const nextRetries = event.retries + 1;
|
||||
const isTerminal = nextRetries > maxRetries;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
metrics?.recordDispatch?.(event.eventType, isTerminal ? "dead_letter" : "error");
|
||||
logger.error({ eventId: event.id, eventType: event.eventType, retries: nextRetries, error }, "Outbox dispatch failed.");
|
||||
reporter?.captureException?.(error, {
|
||||
component: "outbox-dispatcher",
|
||||
op: "enqueue",
|
||||
eventId: event.id,
|
||||
eventType: event.eventType,
|
||||
retries: nextRetries,
|
||||
});
|
||||
|
||||
if (isTerminal) {
|
||||
deadLetters += 1;
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
data: {
|
||||
status: "FAILED",
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(message),
|
||||
processedAt: nowDate,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
const delayMs = computeOutboxRetryDelayMs(nextRetries);
|
||||
await store.outboxEvent.update({
|
||||
where: { id: event.id },
|
||||
data: {
|
||||
status: "PENDING",
|
||||
retries: nextRetries,
|
||||
lastError: truncateError(message),
|
||||
scheduledAt: new Date(nowDate.getTime() + delayMs),
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
if (claimed.length > 0) {
|
||||
logger.info?.({ claimed: claimed.length, ok, errors, deadLetters }, "Outbox dispatch tick complete.");
|
||||
}
|
||||
|
||||
return { claimed: claimed.length, ok, errors, deadLetters };
|
||||
}
|
||||
|
||||
const MAX_LAST_ERROR_LENGTH = 1_000;
|
||||
|
||||
function truncateError(message: string): string {
|
||||
if (message.length <= MAX_LAST_ERROR_LENGTH) return message;
|
||||
return `${message.slice(0, MAX_LAST_ERROR_LENGTH - 16)}… (truncated)`;
|
||||
}
|
||||
|
||||
function clampDispatchConcurrency(value: number | undefined): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return OUTBOX_DEFAULT_DISPATCH_CONCURRENCY;
|
||||
return Math.max(1, Math.min(100, Math.floor(value)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a long-running polling loop on top of {@link runDispatcherTick}.
|
||||
*
|
||||
* Returns a handle that the worker bootstrap can `close()` on shutdown.
|
||||
* The loop uses `setTimeout` (not `setInterval`) so a slow tick can't
|
||||
* pile up overlapping invocations.
|
||||
*/
|
||||
export type DispatcherHandle = {
|
||||
readonly running: boolean;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export type StartDispatcherOptions = DispatcherOptions & {
|
||||
pollIntervalMs?: number;
|
||||
};
|
||||
|
||||
export function startOutboxDispatcher(options: StartDispatcherOptions): DispatcherHandle {
|
||||
const pollIntervalMs = options.pollIntervalMs ?? OUTBOX_DEFAULT_POLL_INTERVAL_MS;
|
||||
const logger = options.logger ?? workerLogger;
|
||||
let running = true;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
let inflight: Promise<unknown> | undefined;
|
||||
|
||||
const schedule = (): void => {
|
||||
if (!running) return;
|
||||
timer = setTimeout(() => {
|
||||
inflight = runDispatcherTick(options)
|
||||
.catch((error) => {
|
||||
logger.error({ error }, "Outbox dispatcher tick threw.");
|
||||
})
|
||||
.finally(() => {
|
||||
inflight = undefined;
|
||||
schedule();
|
||||
});
|
||||
}, pollIntervalMs);
|
||||
};
|
||||
|
||||
schedule();
|
||||
|
||||
return {
|
||||
get running() {
|
||||
return running;
|
||||
},
|
||||
async close(): Promise<void> {
|
||||
running = false;
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = undefined;
|
||||
}
|
||||
if (inflight) {
|
||||
try {
|
||||
await inflight;
|
||||
} catch {
|
||||
// already logged above
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type IsDispatcherEnabledEnv = {
|
||||
OUTBOX_DISPATCHER_ENABLED?: string;
|
||||
OUTBOX_DISPATCHER_REQUIRED?: string;
|
||||
};
|
||||
|
||||
export function isOutboxDispatcherEnabled(env: IsDispatcherEnabledEnv = process.env): boolean {
|
||||
return env.OUTBOX_DISPATCHER_ENABLED === "true";
|
||||
}
|
||||
|
||||
export function isOutboxDispatcherRequired(env: IsDispatcherEnabledEnv = process.env): boolean {
|
||||
return env.OUTBOX_DISPATCHER_REQUIRED === "true";
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { EMAIL_SEQUENCE_TRIGGER_QUEUE } from "../jobs/email-sequence-trigger.js";
|
||||
import { WORKFLOW_TRIGGER_OUTBOX_QUEUE } from "../jobs/workflow-trigger.js";
|
||||
import { DEFAULT_OUTBOX_ROUTING, resolveOutboxQueueName } from "./event-router.js";
|
||||
|
||||
describe("resolveOutboxQueueName", () => {
|
||||
it("routes mail.* to outbox.mail", () => {
|
||||
expect(resolveOutboxQueueName("mail.welcome.send")).toBe("outbox.mail");
|
||||
});
|
||||
|
||||
it("routes webhook.* to outbox.webhook", () => {
|
||||
expect(resolveOutboxQueueName("webhook.delivery.attempted")).toBe("outbox.webhook");
|
||||
expect(resolveOutboxQueueName("webhook.form-submission.deliver")).toBe("outbox.webhook");
|
||||
});
|
||||
|
||||
it("routes notification.* to outbox.notification", () => {
|
||||
expect(resolveOutboxQueueName("notification.in_app.created")).toBe("outbox.notification");
|
||||
});
|
||||
|
||||
it("routes audit.* to outbox.audit", () => {
|
||||
expect(resolveOutboxQueueName("audit.user.created")).toBe("outbox.audit");
|
||||
});
|
||||
|
||||
it("routes workflow.* to outbox.workflow", () => {
|
||||
expect(resolveOutboxQueueName("workflow.trigger.fired")).toBe(WORKFLOW_TRIGGER_OUTBOX_QUEUE);
|
||||
});
|
||||
|
||||
it("keeps the workflow routing rule aligned with the workflow trigger queue constant", () => {
|
||||
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({ prefix: "workflow.", queue: WORKFLOW_TRIGGER_OUTBOX_QUEUE });
|
||||
});
|
||||
|
||||
it("routes billing.* to outbox.notification", () => {
|
||||
expect(resolveOutboxQueueName("billing.invoice.paid")).toBe("outbox.notification");
|
||||
});
|
||||
|
||||
it("routes marketing.lead.* to the email-sequence trigger queue", () => {
|
||||
expect(resolveOutboxQueueName("marketing.lead.captured")).toBe(EMAIL_SEQUENCE_TRIGGER_QUEUE);
|
||||
expect(resolveOutboxQueueName("marketing.lead.convertedtocustomer")).toBe(EMAIL_SEQUENCE_TRIGGER_QUEUE);
|
||||
});
|
||||
|
||||
it("keeps the marketing lead routing rule aligned with the email-sequence trigger owner", () => {
|
||||
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({ prefix: "marketing.lead.", queue: EMAIL_SEQUENCE_TRIGGER_QUEUE });
|
||||
});
|
||||
|
||||
it("prefers the longest matching prefix (tenant.subscription. beats no prefix)", () => {
|
||||
expect(resolveOutboxQueueName("tenant.subscription.updated")).toBe("outbox.notification");
|
||||
});
|
||||
|
||||
it("routes form.submission.* to outbox.form-submission-route", () => {
|
||||
expect(resolveOutboxQueueName("form.submission.process")).toBe("outbox.form-submission-route");
|
||||
expect(resolveOutboxQueueName("form.submission.webhook")).toBe("outbox.form-submission-route");
|
||||
});
|
||||
|
||||
it("routes plugin hook events to the plugin hook fanout queue", () => {
|
||||
expect(resolveOutboxQueueName("plugin.hook.tenant.created")).toBe("outbox.plugin-hook");
|
||||
expect(resolveOutboxQueueName("plugin.hook.entity.updated")).toBe("outbox.plugin-hook");
|
||||
expect(resolveOutboxQueueName("plugin.hook.workflow.created")).toBe("outbox.plugin-hook");
|
||||
expect(resolveOutboxQueueName("plugin.hook.workflow.updated")).toBe("outbox.plugin-hook");
|
||||
expect(resolveOutboxQueueName("plugin.hook.workflow.deleted")).toBe("outbox.plugin-hook");
|
||||
});
|
||||
|
||||
it("returns undefined when no rule matches", () => {
|
||||
expect(resolveOutboxQueueName("unknown.entity.verb")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns undefined for invalid input", () => {
|
||||
expect(resolveOutboxQueueName("")).toBeUndefined();
|
||||
expect(resolveOutboxQueueName(undefined as unknown as string)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("respects custom rules", () => {
|
||||
expect(
|
||||
resolveOutboxQueueName("custom.thing.happened", {
|
||||
rules: [{ prefix: "custom.", queue: "outbox.custom" }],
|
||||
})
|
||||
).toBe("outbox.custom");
|
||||
});
|
||||
|
||||
it("exposes a frozen-style default routing list", () => {
|
||||
expect(DEFAULT_OUTBOX_ROUTING.length).toBeGreaterThan(0);
|
||||
for (const rule of DEFAULT_OUTBOX_ROUTING) {
|
||||
expect(rule.prefix.endsWith(".")).toBe(true);
|
||||
expect(rule.queue.startsWith("outbox.")).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Maps OutboxEvent eventType strings to BullMQ queue names.
|
||||
*
|
||||
* The convention is `<domain>.<entity>.<verb>`, e.g.
|
||||
* `billing.subscription.updated` or `mail.welcome.send`. The router
|
||||
* picks the queue by the first segment (domain) but composite prefixes
|
||||
* such as `tenant.subscription.` can override individual mappings.
|
||||
*
|
||||
* See docs/architecture/outbox-pattern.md.
|
||||
*/
|
||||
|
||||
/** Built-in routing prefixes. Order matters: longest-prefix wins. */
|
||||
export const DEFAULT_OUTBOX_ROUTING: ReadonlyArray<{ prefix: string; queue: string }> = [
|
||||
{ prefix: "tenant.subscription.", queue: "outbox.notification" },
|
||||
{ prefix: "mail.", queue: "outbox.mail" },
|
||||
{ prefix: "webhook.", queue: "outbox.webhook" },
|
||||
{ prefix: "notification.", queue: "outbox.notification" },
|
||||
{ prefix: "audit.", queue: "outbox.audit" },
|
||||
{ prefix: "workflow.", queue: "outbox.workflow" },
|
||||
{ prefix: "search.index.", queue: "outbox.search-index" },
|
||||
{ prefix: "marketing.lead.", queue: "outbox.marketing" },
|
||||
{ prefix: "form.submission.", queue: "outbox.form-submission-route" },
|
||||
{ prefix: "billing.", queue: "outbox.notification" },
|
||||
{ prefix: "plugin.hook.", queue: "outbox.plugin-hook" },
|
||||
{ prefix: "plugin.capability-use.", queue: "outbox.plugin-capability-use" },
|
||||
];
|
||||
|
||||
export type EventRoutingRule = { prefix: string; queue: string };
|
||||
|
||||
export type EventRouterResolveOptions = {
|
||||
rules?: ReadonlyArray<EventRoutingRule>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve the queue name for an eventType. Returns `undefined` when no
|
||||
* routing rule matches — the caller (dispatcher) treats this as a
|
||||
* dead-letter condition.
|
||||
*/
|
||||
export function resolveOutboxQueueName(
|
||||
eventType: string,
|
||||
options: EventRouterResolveOptions = {}
|
||||
): string | undefined {
|
||||
if (typeof eventType !== "string" || eventType.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const rules = options.rules ?? DEFAULT_OUTBOX_ROUTING;
|
||||
// Longest matching prefix wins.
|
||||
let best: { prefix: string; queue: string } | undefined;
|
||||
for (const rule of rules) {
|
||||
if (eventType.startsWith(rule.prefix)) {
|
||||
if (!best || rule.prefix.length > best.prefix.length) {
|
||||
best = rule;
|
||||
}
|
||||
}
|
||||
}
|
||||
return best?.queue;
|
||||
}
|
||||
Reference in new issue
Block a user