Reorganize repo structure and standards references

This commit is contained in:
david committed 2026-06-29 05:03:42 +02:00
1 parent 3cb93a81be
commit be2054274d
5885 files changed
+2588 -195

No files matched your search

@@ -0,0 +1,300 @@
import { readFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it, vi } from "vitest";
import { bootstrapOutboxDispatcher, type OutboxDispatcherBootstrapDeps } from "./bootstrap.js";
import type { DispatcherLogger } from "./dispatcher.js";
import type { OutboxEvent } from "@toolctrl/db/outbox";
const sourceDir = dirname(fileURLToPath(import.meta.url));
const queueRuntimeMocks = vi.hoisted(() => ({
add: vi.fn<(name: string, data: unknown, options?: unknown) => Promise<void>>(async () => undefined),
close: vi.fn(async () => undefined),
createQueue: vi.fn(() => ({
add: queueRuntimeMocks.add,
close: queueRuntimeMocks.close,
})),
}));
vi.mock("@toolctrl/queue/runtime", () => ({
createQueue: queueRuntimeMocks.createQueue,
}));
function createPrismaStub(): OutboxDispatcherBootstrapDeps["prisma"] {
return {
$queryRaw: vi.fn(async () => []) as unknown as OutboxDispatcherBootstrapDeps["prisma"]["$queryRaw"],
outboxEvent: {
update: vi.fn(async () => ({}) as unknown),
},
} as unknown as OutboxDispatcherBootstrapDeps["prisma"];
}
function createLogger(): DispatcherLogger {
return {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
}
describe("bootstrapOutboxDispatcher", () => {
afterEach(() => {
vi.useRealTimers();
queueRuntimeMocks.add.mockClear();
queueRuntimeMocks.close.mockClear();
queueRuntimeMocks.createQueue.mockClear();
});
it("returns disabled when OUTBOX_DISPATCHER_ENABLED is unset", async () => {
const logger = createLogger();
const result = await bootstrapOutboxDispatcher(
{},
{
prisma: createPrismaStub(),
enqueue: async () => undefined,
logger,
}
);
expect(result.status).toBe("disabled");
expect(result.handle).toBeUndefined();
expect(logger.info).toHaveBeenCalled();
});
it("imports outbox claim helpers through the dedicated db subpath", async () => {
const source = await readFile(join(sourceDir, "bootstrap.ts"), "utf8");
expect(source).toContain('from "@toolctrl/db/outbox-claim"');
expect(source).not.toMatch(/from\s+["']@toolctrl\/db["']/);
});
it("returns disabled when OUTBOX_DISPATCHER_ENABLED is 'false'", async () => {
const result = await bootstrapOutboxDispatcher(
{ OUTBOX_DISPATCHER_ENABLED: "false" },
{
prisma: createPrismaStub(),
enqueue: async () => undefined,
logger: createLogger(),
}
);
expect(result.status).toBe("disabled");
});
it("throws when disabled in required mode", async () => {
await expect(
bootstrapOutboxDispatcher(
{ OUTBOX_DISPATCHER_ENABLED: "false", OUTBOX_DISPATCHER_REQUIRED: "true" },
{
prisma: createPrismaStub(),
enqueue: async () => undefined,
logger: createLogger(),
}
)
).rejects.toThrow("Outbox dispatcher is required");
});
it("starts the dispatcher when enabled and enqueue is provided", async () => {
const enqueue = vi.fn(async () => undefined);
const logger = createLogger();
const result = await bootstrapOutboxDispatcher(
{ OUTBOX_DISPATCHER_ENABLED: "true", OUTBOX_DISPATCH_INTERVAL_MS: "10000" },
{
prisma: createPrismaStub(),
enqueue,
logger,
}
);
expect(result.status).toBe("started");
expect(result.handle).toBeDefined();
expect(logger.info).toHaveBeenCalledWith(
expect.objectContaining({ component: "outbox-dispatcher", pollIntervalMs: 10000 }),
expect.any(String)
);
await result.handle?.close();
});
it("passes configured dispatch concurrency into startup logs", async () => {
const logger = createLogger();
const result = await bootstrapOutboxDispatcher(
{ OUTBOX_DISPATCHER_ENABLED: "true", OUTBOX_DISPATCH_INTERVAL_MS: "10000", OUTBOX_DISPATCH_CONCURRENCY: "7" },
{
prisma: createPrismaStub(),
enqueue: async () => undefined,
logger,
}
);
expect(result.status).toBe("started");
expect(logger.info).toHaveBeenCalledWith(
expect.objectContaining({ component: "outbox-dispatcher", dispatchConcurrency: 7 }),
expect.any(String)
);
await result.handle?.close();
});
it("throws when enabled but no redis URL and no enqueue override", async () => {
await expect(
bootstrapOutboxDispatcher(
{ OUTBOX_DISPATCHER_ENABLED: "true" },
{ prisma: createPrismaStub(), logger: createLogger() }
)
).rejects.toThrow(/BULLMQ_REDIS_URL or REDIS_URL/);
});
it("wires the default Redis enqueue path without forwarding raw outbox priority", async () => {
vi.useFakeTimers();
const event = createOutboxEvent();
const prisma = createPrismaStub();
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([event]);
const logger = createLogger();
const result = await bootstrapOutboxDispatcher(
{
OUTBOX_DISPATCHER_ENABLED: "true",
OUTBOX_DISPATCH_INTERVAL_MS: "5",
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
},
{
prisma,
logger,
}
);
await vi.advanceTimersByTimeAsync(5);
await result.handle?.close();
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledWith("outbox.workflow", expect.any(Object));
expect(queueRuntimeMocks.add).toHaveBeenCalledWith(
"workflow.trigger.event",
{
outboxEventId: "outbox-1",
tenantId: "tenant-1",
eventType: "workflow.trigger.event",
payload: { workflowId: "workflow-1" },
dispatchKey: "workflow:outbox-1",
retries: 0,
},
{
jobId: "outbox-outbox-1",
attempts: 5,
backoff: { type: "exponential", delay: 15_000 },
removeOnComplete: 1_000,
removeOnFail: 10_000,
}
);
expect(queueRuntimeMocks.add.mock.calls[0]).toHaveLength(3);
// BullMQ rejects custom job ids containing ":" at runtime, which the mocked
// queue here would not surface — guard the separator explicitly.
expect((queueRuntimeMocks.add.mock.calls[0][2] as { jobId: string }).jobId).not.toContain(":");
expect(prisma.outboxEvent.update).toHaveBeenCalledWith({
where: { id: "outbox-1" },
data: expect.objectContaining({
status: "DONE",
lastError: null,
}),
});
});
it("wires dispatcher metrics through the bootstrap path", async () => {
vi.useFakeTimers();
const event = createOutboxEvent({ eventType: "workflow.trigger.event" });
const prisma = createPrismaStub();
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([event]);
const metrics = { recordDispatch: vi.fn() };
const result = await bootstrapOutboxDispatcher(
{
OUTBOX_DISPATCHER_ENABLED: "true",
OUTBOX_DISPATCH_INTERVAL_MS: "5",
},
{
prisma,
enqueue: vi.fn(async () => undefined),
logger: createLogger(),
metrics,
}
);
await vi.advanceTimersByTimeAsync(5);
await result.handle?.close();
expect(metrics.recordDispatch).toHaveBeenCalledWith("workflow.trigger.event", "ok");
});
it("closes cached BullMQ queues when the default enqueue handle closes", async () => {
vi.useFakeTimers();
const prisma = createPrismaStub();
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([createOutboxEvent()]);
const result = await bootstrapOutboxDispatcher(
{
OUTBOX_DISPATCHER_ENABLED: "true",
OUTBOX_DISPATCH_INTERVAL_MS: "5",
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
},
{
prisma,
logger: createLogger(),
}
);
await vi.advanceTimersByTimeAsync(5);
await result.handle?.close();
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledTimes(1);
expect(queueRuntimeMocks.close).toHaveBeenCalledTimes(1);
});
it("uses deterministic BullMQ job ids so redelivered outbox events dedupe", async () => {
vi.useFakeTimers();
const prisma = createPrismaStub();
vi.mocked(prisma.$queryRaw)
.mockResolvedValueOnce([createOutboxEvent()])
.mockResolvedValueOnce([createOutboxEvent()]);
const result = await bootstrapOutboxDispatcher(
{
OUTBOX_DISPATCHER_ENABLED: "true",
OUTBOX_DISPATCH_INTERVAL_MS: "5",
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
},
{
prisma,
logger: createLogger(),
}
);
await vi.advanceTimersByTimeAsync(5);
await vi.advanceTimersByTimeAsync(5);
await result.handle?.close();
expect(queueRuntimeMocks.add).toHaveBeenCalledTimes(2);
expect(queueRuntimeMocks.add.mock.calls.map((call) => call[2])).toEqual([
expect.objectContaining({ jobId: "outbox-outbox-1" }),
expect.objectContaining({ jobId: "outbox-outbox-1" }),
]);
expect(queueRuntimeMocks.createQueue).toHaveBeenCalledTimes(1);
});
});
function createOutboxEvent(overrides: Partial<OutboxEvent> = {}): OutboxEvent {
const now = new Date("2026-05-27T01:00:00.000Z");
return {
id: "outbox-1",
tenantId: "tenant-1",
eventType: "workflow.trigger.event",
payload: { workflowId: "workflow-1" },
status: "PROCESSING",
createdAt: now,
processedAt: null,
retries: 0,
lastError: null,
scheduledAt: now,
dispatchKey: "workflow:outbox-1",
priority: 50,
lockedUntil: new Date(now.getTime() + 60_000),
...overrides,
};
}
@@ -0,0 +1,222 @@
import type { EnvRecord } from "@toolctrl/config";
import { claimPendingOutboxEvents, OUTBOX_CLAIM_DEFAULT_LIMIT, type OutboxClaimRunner } from "@toolctrl/db/outbox-claim";
import type { OutboxEvent } from "@toolctrl/db/outbox";
import type { ErrorReporter } from "@toolctrl/error-reporter";
import { parseRedisUrl } from "@toolctrl/queue";
import { workerLogger } from "../logger.js";
import { getWorkerMetrics } from "../metrics.js";
import { workerErrorReporter } from "../sentry.js";
import {
OUTBOX_DEFAULT_BATCH_SIZE,
OUTBOX_DEFAULT_DISPATCH_CONCURRENCY,
OUTBOX_DEFAULT_MAX_RETRIES,
OUTBOX_DEFAULT_POLL_INTERVAL_MS,
isOutboxDispatcherEnabled,
isOutboxDispatcherRequired,
startOutboxDispatcher,
type DispatcherHandle,
type DispatcherLogger,
type DispatcherMetrics,
type DispatcherQueueEnqueuer,
type DispatcherTransactionRunner,
} from "./dispatcher.js";
/**
* Bootstrap helper that wires the outbox dispatcher into the worker
* runtime. Returns `{ status: "disabled" }` when `OUTBOX_DISPATCHER_ENABLED`
* is not exactly `"true"` so dev environments default to "dispatcher off".
*
* See docs/architecture/outbox-pattern.md.
*/
export type OutboxDispatcherBootstrapEnv = EnvRecord & {
OUTBOX_DISPATCHER_ENABLED?: string;
OUTBOX_DISPATCHER_REQUIRED?: string;
OUTBOX_DISPATCH_BATCH_SIZE?: string;
OUTBOX_DISPATCH_CONCURRENCY?: string;
OUTBOX_DISPATCH_INTERVAL_MS?: string;
OUTBOX_MAX_RETRIES?: string;
BULLMQ_REDIS_URL?: string;
REDIS_URL?: string;
};
export type OutboxDispatcherBootstrapDeps = {
prisma: OutboxClaimRunner & DispatcherTransactionRunner;
enqueue?: DispatcherQueueEnqueuer;
logger?: DispatcherLogger;
metrics?: DispatcherMetrics;
reporter?: ErrorReporter;
};
export type OutboxDispatcherBootstrapResult = {
status: "started" | "disabled";
handle?: DispatcherHandle;
};
const OUTBOX_QUEUE_JOB_ATTEMPTS = 5;
const OUTBOX_QUEUE_JOB_BACKOFF_MS = 15_000;
const OUTBOX_QUEUE_REMOVE_ON_COMPLETE = 1_000;
const OUTBOX_QUEUE_REMOVE_ON_FAIL = 10_000;
type ClosableDispatcherQueueEnqueuer = DispatcherQueueEnqueuer & {
close(): Promise<void>;
};
function parseInteger(value: string | undefined, fallback: number): number {
if (!value) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
/**
* Build a default BullMQ enqueue function that fans events out to the
* target queue determined by the event router. The implementation lazy-
* imports the bullmq runtime so tests can inject `enqueue` without
* loading bullmq at all.
*/
async function createDefaultEnqueue(redisUrl: string): Promise<ClosableDispatcherQueueEnqueuer> {
const connection = parseRedisUrl(redisUrl);
const { createQueue } = (await import("@toolctrl/queue/runtime")) as {
createQueue: (name: string, options: { connection: typeof connection }) => {
add(name: string, data: unknown, options?: unknown): Promise<unknown>;
close?: () => Promise<void>;
};
};
const queueCache = new Map<string, { add(name: string, data: unknown, options?: unknown): Promise<unknown>; close?: () => Promise<void> }>();
const enqueue = (async (queueName: string, event: OutboxEvent) => {
let queue = queueCache.get(queueName);
if (!queue) {
queue = createQueue(queueName, { connection });
queueCache.set(queueName, queue);
}
await queue.add(
event.eventType,
{
outboxEventId: event.id,
tenantId: event.tenantId,
eventType: event.eventType,
payload: event.payload,
dispatchKey: event.dispatchKey,
retries: event.retries,
},
{
// BullMQ rejects custom job ids containing ":" ("Custom Id cannot
// contain :"), which silently failed every outbox dispatch. Use "-";
// workflow-execution-stalled-reconciler.buildOutboxJobId must match.
jobId: `outbox-${event.id}`,
attempts: OUTBOX_QUEUE_JOB_ATTEMPTS,
backoff: { type: "exponential", delay: OUTBOX_QUEUE_JOB_BACKOFF_MS },
removeOnComplete: OUTBOX_QUEUE_REMOVE_ON_COMPLETE,
removeOnFail: OUTBOX_QUEUE_REMOVE_ON_FAIL,
}
);
// OutboxEvent.priority is claim-only here: Postgres dispatches higher
// values first, while BullMQ treats 1 as highest priority. Passing the
// raw value would invert semantics after claim, so keep queue enqueue
// FIFO until a reviewed mapping is introduced.
}) as ClosableDispatcherQueueEnqueuer;
enqueue.close = async () => {
const queues = [...queueCache.values()];
queueCache.clear();
await Promise.all(queues.map((queue) => queue.close?.() ?? Promise.resolve()));
};
return enqueue;
}
export async function bootstrapOutboxDispatcher(
env: OutboxDispatcherBootstrapEnv,
deps: OutboxDispatcherBootstrapDeps
): Promise<OutboxDispatcherBootstrapResult> {
const logger = deps.logger ?? workerLogger;
if (!isOutboxDispatcherEnabled(env)) {
if (isOutboxDispatcherRequired(env)) {
throw new Error("Outbox dispatcher is required but OUTBOX_DISPATCHER_ENABLED is not true.");
}
logger.info?.({ component: "outbox-dispatcher" }, "Outbox dispatcher disabled (OUTBOX_DISPATCHER_ENABLED!=true).");
return { status: "disabled" };
}
const batchSize = parseInteger(env.OUTBOX_DISPATCH_BATCH_SIZE, OUTBOX_DEFAULT_BATCH_SIZE);
const dispatchConcurrency = parseInteger(env.OUTBOX_DISPATCH_CONCURRENCY, OUTBOX_DEFAULT_DISPATCH_CONCURRENCY);
const pollIntervalMs = parseInteger(env.OUTBOX_DISPATCH_INTERVAL_MS, OUTBOX_DEFAULT_POLL_INTERVAL_MS);
const maxRetries = parseInteger(env.OUTBOX_MAX_RETRIES, OUTBOX_DEFAULT_MAX_RETRIES);
let enqueue: DispatcherQueueEnqueuer | undefined = deps.enqueue;
let closeDefaultEnqueue: (() => Promise<void>) | undefined;
if (!enqueue) {
const redisUrl = env.BULLMQ_REDIS_URL ?? env.REDIS_URL;
if (!redisUrl) {
throw new Error("Outbox dispatcher requires BULLMQ_REDIS_URL or REDIS_URL when no custom enqueue is provided.");
}
const defaultEnqueue = await createDefaultEnqueue(redisUrl);
enqueue = defaultEnqueue;
closeDefaultEnqueue = defaultEnqueue.close;
}
const reporter = deps.reporter ?? workerErrorReporter;
const metrics =
deps.metrics ??
({
recordDispatch(eventType, status) {
getWorkerMetrics().metrics.outboxDispatchTotal.labels({ event_type: eventType, status }).inc();
},
} satisfies DispatcherMetrics);
const dispatcherHandle = startOutboxDispatcher({
claim: (limit, now) => claimPendingOutboxEvents(deps.prisma, { limit, now }),
enqueue,
store: deps.prisma,
batchSize,
dispatchConcurrency,
pollIntervalMs,
maxRetries,
logger,
metrics,
reporter: {
captureException: (error, context) => {
try {
reporter.report({
component: "outbox-dispatcher",
operation: typeof context?.op === "string" ? context.op : "dispatch",
severity: "error",
error,
...(context ?? {}),
});
} catch {
// Reporter failures must never mask dispatcher progress.
}
},
},
});
const handle = closeDefaultEnqueue
? {
get running() {
return dispatcherHandle.running;
},
async close() {
try {
await dispatcherHandle.close();
} finally {
await closeDefaultEnqueue?.();
}
},
}
: dispatcherHandle;
logger.info?.(
{
component: "outbox-dispatcher",
batchSize,
dispatchConcurrency,
pollIntervalMs,
maxRetries,
},
"Outbox dispatcher started."
);
return { status: "started", handle };
}
export { OUTBOX_CLAIM_DEFAULT_LIMIT };
@@ -0,0 +1,320 @@
import { describe, expect, it, vi } from "vitest";
import type { OutboxEvent } from "@toolctrl/db/outbox";
import {
OUTBOX_DEFAULT_DISPATCH_CONCURRENCY,
OUTBOX_DEFAULT_MAX_RETRIES,
OUTBOX_DEFAULT_POLL_INTERVAL_MS,
OUTBOX_RETRY_BACKOFF_MINUTES,
computeOutboxRetryDelayMs,
isOutboxDispatcherEnabled,
isOutboxDispatcherRequired,
runDispatcherTick,
startOutboxDispatcher,
type DispatcherClaim,
type DispatcherQueueEnqueuer,
type DispatcherTransactionRunner,
} from "./dispatcher.js";
type FakeEvent = OutboxEvent;
type Updates = Array<{ id: string; data: Record<string, unknown> }>;
function makeStore(updates: Updates): DispatcherTransactionRunner {
return {
outboxEvent: {
update: vi.fn(async ({ where, data }) => {
updates.push({ id: where.id, data });
return { id: where.id, ...data } as unknown as FakeEvent;
}),
},
};
}
function makeEvent(overrides: Partial<FakeEvent> = {}): FakeEvent {
return {
id: "evt_1",
tenantId: "tenant_1",
eventType: "mail.welcome.send",
payload: { to: "user@example.com" },
status: "PROCESSING",
createdAt: new Date("2026-05-17T08:00:00.000Z"),
processedAt: null,
retries: 0,
lastError: null,
scheduledAt: new Date("2026-05-17T08:00:00.000Z"),
dispatchKey: null,
priority: 0,
lockedUntil: new Date("2026-05-17T08:05:00.000Z"),
...overrides,
};
}
describe("computeOutboxRetryDelayMs", () => {
it("returns 1 minute on the first retry", () => {
expect(computeOutboxRetryDelayMs(1)).toBe(60_000);
});
it("doubles per retry up to the schedule cap", () => {
expect(computeOutboxRetryDelayMs(2)).toBe(2 * 60_000);
expect(computeOutboxRetryDelayMs(3)).toBe(4 * 60_000);
expect(computeOutboxRetryDelayMs(4)).toBe(8 * 60_000);
expect(computeOutboxRetryDelayMs(6)).toBe(32 * 60_000);
});
it("caps at the last schedule entry for retries beyond the schedule", () => {
expect(computeOutboxRetryDelayMs(99)).toBe(32 * 60_000);
});
it("falls back to the first entry when retries < 1", () => {
expect(computeOutboxRetryDelayMs(0)).toBe(60_000);
expect(computeOutboxRetryDelayMs(-3)).toBe(60_000);
});
it("exposes the documented schedule", () => {
expect([...OUTBOX_RETRY_BACKOFF_MINUTES]).toEqual([1, 2, 4, 8, 16, 32]);
expect(OUTBOX_DEFAULT_MAX_RETRIES).toBe(6);
});
});
describe("runDispatcherTick", () => {
const now = new Date("2026-05-17T09:00:00.000Z");
it("returns empty result when claim throws", async () => {
const claim: DispatcherClaim = vi.fn(async () => {
throw new Error("db down");
});
const enqueue: DispatcherQueueEnqueuer = vi.fn();
const updates: Updates = [];
const result = await runDispatcherTick({
claim,
enqueue,
store: makeStore(updates),
now: () => now,
logger: { error: vi.fn() },
});
expect(result).toEqual({ claimed: 0, ok: 0, errors: 0, deadLetters: 0 });
expect(enqueue).not.toHaveBeenCalled();
expect(updates).toEqual([]);
});
it("marks events DONE on successful enqueue", async () => {
const event = makeEvent();
const claim: DispatcherClaim = vi.fn(async () => [event]);
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
const updates: Updates = [];
const result = await runDispatcherTick({
claim,
enqueue,
store: makeStore(updates),
now: () => now,
logger: { info: vi.fn(), error: vi.fn() },
});
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
expect(enqueue).toHaveBeenCalledWith("outbox.mail", event);
expect(updates).toHaveLength(1);
expect(updates[0]?.data.status).toBe("DONE");
expect(updates[0]?.data.processedAt).toBe(now);
expect(updates[0]?.data.lockedUntil).toBeNull();
});
it("routes workflow trigger events to the workflow outbox receiver queue", async () => {
const event = makeEvent({
eventType: "workflow.trigger.row.created",
payload: { event: "row.created", data: { id: "row_1" } },
dispatchKey: "workflow:row.created:row_1",
});
const claim: DispatcherClaim = vi.fn(async () => [event]);
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
const updates: Updates = [];
const result = await runDispatcherTick({
claim,
enqueue,
store: makeStore(updates),
now: () => now,
logger: { info: vi.fn(), error: vi.fn() },
});
expect(result).toEqual({ claimed: 1, ok: 1, errors: 0, deadLetters: 0 });
expect(enqueue).toHaveBeenCalledWith("outbox.workflow", event);
expect(updates[0]?.data).toMatchObject({ status: "DONE", processedAt: now, lockedUntil: null });
});
it("dead-letters unknown event types without enqueueing", async () => {
const event = makeEvent({ eventType: "no.routing.here" });
const claim: DispatcherClaim = vi.fn(async () => [event]);
const enqueue: DispatcherQueueEnqueuer = vi.fn();
const updates: Updates = [];
const reporter = { captureException: vi.fn() };
const result = await runDispatcherTick({
claim,
enqueue,
store: makeStore(updates),
reporter,
now: () => now,
logger: { error: vi.fn() },
});
expect(result.deadLetters).toBe(1);
expect(enqueue).not.toHaveBeenCalled();
expect(updates[0]?.data.status).toBe("FAILED");
expect(reporter.captureException).toHaveBeenCalled();
});
it("retries with exponential backoff on enqueue failure", async () => {
const event = makeEvent({ retries: 1 });
const claim: DispatcherClaim = vi.fn(async () => [event]);
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => {
throw new Error("redis offline");
});
const updates: Updates = [];
const result = await runDispatcherTick({
claim,
enqueue,
store: makeStore(updates),
now: () => now,
logger: { error: vi.fn() },
});
expect(result.errors).toBe(1);
expect(result.deadLetters).toBe(0);
expect(updates[0]?.data.status).toBe("PENDING");
expect(updates[0]?.data.retries).toBe(2);
expect(updates[0]?.data.lastError).toContain("redis offline");
// retries=2 → 2 minute backoff.
const scheduled = updates[0]?.data.scheduledAt as Date;
expect(scheduled.getTime() - now.getTime()).toBe(2 * 60_000);
});
it("dead-letters after retries exceed the limit", async () => {
const event = makeEvent({ retries: OUTBOX_DEFAULT_MAX_RETRIES });
const claim: DispatcherClaim = vi.fn(async () => [event]);
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => {
throw new Error("boom");
});
const updates: Updates = [];
const result = await runDispatcherTick({
claim,
enqueue,
store: makeStore(updates),
now: () => now,
logger: { error: vi.fn() },
});
expect(result.deadLetters).toBe(1);
expect(updates[0]?.data.status).toBe("FAILED");
expect(updates[0]?.data.retries).toBe(OUTBOX_DEFAULT_MAX_RETRIES + 1);
});
it("records metrics per event", async () => {
const event = makeEvent();
const claim: DispatcherClaim = vi.fn(async () => [event]);
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => undefined);
const metrics = { recordDispatch: vi.fn() };
await runDispatcherTick({
claim,
enqueue,
store: makeStore([]),
metrics,
now: () => now,
});
expect(metrics.recordDispatch).toHaveBeenCalledWith("mail.welcome.send", "ok");
});
it("dispatches claimed events with bounded concurrency", async () => {
const events = Array.from({ length: 4 }, (_, index) => makeEvent({ id: `evt_${index + 1}` }));
const claim: DispatcherClaim = vi.fn(async () => events);
const updates: Updates = [];
let inFlight = 0;
let maxInFlight = 0;
const release: Array<() => void> = [];
const enqueue: DispatcherQueueEnqueuer = vi.fn(async () => {
inFlight += 1;
maxInFlight = Math.max(maxInFlight, inFlight);
await new Promise<void>((resolve) => release.push(resolve));
inFlight -= 1;
});
const tick = runDispatcherTick({
claim,
enqueue,
store: makeStore(updates),
dispatchConcurrency: 2,
now: () => now,
logger: { info: vi.fn(), error: vi.fn() },
});
await vi.waitFor(() => expect(release).toHaveLength(2));
expect(maxInFlight).toBe(2);
release.splice(0).forEach((resolve) => resolve());
await vi.waitFor(() => expect(release).toHaveLength(2));
expect(maxInFlight).toBe(2);
release.splice(0).forEach((resolve) => resolve());
await expect(tick).resolves.toEqual({ claimed: 4, ok: 4, errors: 0, deadLetters: 0 });
expect(updates).toHaveLength(4);
});
});
describe("startOutboxDispatcher", () => {
it("invokes runDispatcherTick on the configured interval until close()", async () => {
vi.useFakeTimers();
try {
const claim: DispatcherClaim = vi.fn(async () => []);
const enqueue: DispatcherQueueEnqueuer = vi.fn();
const handle = startOutboxDispatcher({
claim,
enqueue,
store: makeStore([]),
pollIntervalMs: 100,
logger: { error: vi.fn(), info: vi.fn() },
});
expect(handle.running).toBe(true);
await vi.advanceTimersByTimeAsync(350);
// ~3 ticks should have fired
expect((claim as unknown as { mock: { calls: unknown[] } }).mock.calls.length).toBeGreaterThanOrEqual(3);
await handle.close();
const callsAfterClose = (claim as unknown as { mock: { calls: unknown[] } }).mock.calls.length;
await vi.advanceTimersByTimeAsync(500);
expect((claim as unknown as { mock: { calls: unknown[] } }).mock.calls.length).toBe(callsAfterClose);
} finally {
vi.useRealTimers();
}
});
});
describe("isOutboxDispatcherEnabled", () => {
it("returns true only when OUTBOX_DISPATCHER_ENABLED is exactly 'true'", () => {
expect(isOutboxDispatcherEnabled({ OUTBOX_DISPATCHER_ENABLED: "true" })).toBe(true);
expect(isOutboxDispatcherEnabled({ OUTBOX_DISPATCHER_ENABLED: "TRUE" })).toBe(false);
expect(isOutboxDispatcherEnabled({ OUTBOX_DISPATCHER_ENABLED: "1" })).toBe(false);
expect(isOutboxDispatcherEnabled({})).toBe(false);
});
it("returns true only when OUTBOX_DISPATCHER_REQUIRED is exactly 'true'", () => {
expect(isOutboxDispatcherRequired({ OUTBOX_DISPATCHER_REQUIRED: "true" })).toBe(true);
expect(isOutboxDispatcherRequired({ OUTBOX_DISPATCHER_REQUIRED: "TRUE" })).toBe(false);
expect(isOutboxDispatcherRequired({ OUTBOX_DISPATCHER_REQUIRED: "1" })).toBe(false);
expect(isOutboxDispatcherRequired({})).toBe(false);
});
});
describe("constants", () => {
it("publishes a sensible default poll interval", () => {
expect(OUTBOX_DEFAULT_POLL_INTERVAL_MS).toBe(1_000);
});
it("publishes a conservative default dispatch concurrency", () => {
expect(OUTBOX_DEFAULT_DISPATCH_CONCURRENCY).toBe(10);
});
});
@@ -0,0 +1,315 @@
import type { OutboxEvent } from "@toolctrl/db/outbox";
import { workerLogger } from "../logger.js";
import { resolveOutboxQueueName, type EventRoutingRule } from "./event-router.js";
/**
* Outbox dispatcher worker.
*
* Polls the OutboxEvent table on a fixed interval (default 1s), atomically
* claims a batch via `claimPendingOutboxEvents`, routes each event to the
* appropriate BullMQ queue via the event router, and marks the row DONE.
*
* Failures trigger a retry with exponential backoff (1, 2, 4, 8, 16, 32
* minutes). After `OUTBOX_MAX_RETRIES` failures the row is marked FAILED
* and stays in the dead-letter list for an operator to re-queue via CLI.
*
* The dispatcher is bootstrapped by the `WORKER_QUEUE_GROUP=general` worker
* so it can scale independently from mail/import/webhook/plugin workers.
*
* See docs/architecture/outbox-pattern.md.
*/
export const OUTBOX_DEFAULT_MAX_RETRIES = 6;
export const OUTBOX_DEFAULT_BATCH_SIZE = 100;
export const OUTBOX_DEFAULT_POLL_INTERVAL_MS = 1_000;
export const OUTBOX_DEFAULT_DISPATCH_CONCURRENCY = 10;
/** Minutes — `2^retry`. Capped at OUTBOX_DEFAULT_MAX_RETRIES entries. */
export const OUTBOX_RETRY_BACKOFF_MINUTES: readonly number[] = [1, 2, 4, 8, 16, 32];
/**
* Compute the next `scheduledAt` for a failed dispatch. `retries` is the
* count AFTER incrementing the counter (so retries=1 → 1 minute delay).
*/
export function computeOutboxRetryDelayMs(
retries: number,
schedule: readonly number[] = OUTBOX_RETRY_BACKOFF_MINUTES
): number {
if (retries < 1) return schedule[0] !== undefined ? schedule[0] * 60 * 1000 : 60_000;
const idx = Math.min(retries - 1, schedule.length - 1);
return (schedule[idx] ?? schedule.at(-1) ?? 1) * 60 * 1000;
}
export type DispatcherLogger = {
child?: (bindings: Record<string, unknown>) => DispatcherLogger;
info?: (record: Record<string, unknown>, message?: string) => void;
warn?: (record: Record<string, unknown>, message?: string) => void;
error: (record: Record<string, unknown>, message?: string) => void;
};
export type DispatcherMetrics = {
/** Increment with labels `{event_type, status}`. */
recordDispatch?: (event_type: string, status: "ok" | "error" | "dead_letter") => void;
};
export type DispatcherReporter = {
captureException?: (error: unknown, context?: Record<string, unknown>) => void;
};
export type DispatcherQueueEnqueuer = (queueName: string, event: OutboxEvent) => Promise<void>;
export type DispatcherTransactionRunner = {
outboxEvent: {
update(args: {
where: { id: string };
data: Record<string, unknown>;
}): Promise<OutboxEvent>;
};
};
export type DispatcherClaim = (limit: number, now: Date) => Promise<OutboxEvent[]>;
export type DispatcherOptions = {
/** Atomic batch-claim function (bound to a Prisma client). */
claim: DispatcherClaim;
/** Transaction-style update hook used to flip event status. */
store: DispatcherTransactionRunner;
/** Enqueues a claimed event to its target BullMQ queue. */
enqueue: DispatcherQueueEnqueuer;
/** Optional max retries before dead-lettering (default 6). */
maxRetries?: number;
/** Optional batch size (default 100). */
batchSize?: number;
/** Optional poll interval ms (default 1000). */
pollIntervalMs?: number;
/** Optional per-tick enqueue/update concurrency (default 10). */
dispatchConcurrency?: number;
/** Optional routing rule overrides. */
routingRules?: ReadonlyArray<EventRoutingRule>;
/** Optional clock for deterministic tests. */
now?: () => Date;
/** Optional logger / metrics / Sentry-like reporter. */
logger?: DispatcherLogger;
metrics?: DispatcherMetrics;
reporter?: DispatcherReporter;
};
export type DispatcherTickResult = {
claimed: number;
ok: number;
errors: number;
deadLetters: number;
};
/**
* Execute one polling pass: claim a batch, enqueue each event, update
* statuses. Returns counters for tests / metrics.
*/
export async function runDispatcherTick(options: DispatcherOptions): Promise<DispatcherTickResult> {
const {
claim,
store,
enqueue,
maxRetries = OUTBOX_DEFAULT_MAX_RETRIES,
batchSize = OUTBOX_DEFAULT_BATCH_SIZE,
routingRules,
now = () => new Date(),
logger = workerLogger,
metrics,
reporter,
} = options;
const nowDate = now();
let claimed: OutboxEvent[] = [];
try {
claimed = await claim(batchSize, nowDate);
} catch (error) {
logger.error({ error }, "Outbox dispatcher claim failed.");
reporter?.captureException?.(error, { component: "outbox-dispatcher", op: "claim" });
return { claimed: 0, ok: 0, errors: 0, deadLetters: 0 };
}
let ok = 0;
let errors = 0;
let deadLetters = 0;
const dispatchConcurrency = clampDispatchConcurrency(options.dispatchConcurrency);
for (let i = 0; i < claimed.length; i += dispatchConcurrency) {
await Promise.all(claimed.slice(i, i + dispatchConcurrency).map(async (event) => {
const targetQueue = resolveOutboxQueueName(event.eventType, routingRules ? { rules: routingRules } : undefined);
if (!targetQueue) {
// Unknown event-type → dead-letter immediately so it shows up in
// admin UI and Sentry alerts. Do NOT retry — the type will never
// route until the operator adds a rule, and silent retries waste
// throughput.
deadLetters += 1;
metrics?.recordDispatch?.(event.eventType, "dead_letter");
const message = `No routing rule matched eventType ${event.eventType}.`;
logger.error({ eventId: event.id, eventType: event.eventType }, message);
reporter?.captureException?.(new Error(message), {
component: "outbox-dispatcher",
op: "route",
eventId: event.id,
eventType: event.eventType,
});
await store.outboxEvent.update({
where: { id: event.id },
data: {
status: "FAILED",
lastError: message,
processedAt: nowDate,
lockedUntil: null,
},
});
return;
}
try {
await enqueue(targetQueue, event);
ok += 1;
metrics?.recordDispatch?.(event.eventType, "ok");
await store.outboxEvent.update({
where: { id: event.id },
data: {
status: "DONE",
processedAt: nowDate,
lastError: null,
lockedUntil: null,
},
});
} catch (error) {
errors += 1;
const nextRetries = event.retries + 1;
const isTerminal = nextRetries > maxRetries;
const message = error instanceof Error ? error.message : String(error);
metrics?.recordDispatch?.(event.eventType, isTerminal ? "dead_letter" : "error");
logger.error({ eventId: event.id, eventType: event.eventType, retries: nextRetries, error }, "Outbox dispatch failed.");
reporter?.captureException?.(error, {
component: "outbox-dispatcher",
op: "enqueue",
eventId: event.id,
eventType: event.eventType,
retries: nextRetries,
});
if (isTerminal) {
deadLetters += 1;
await store.outboxEvent.update({
where: { id: event.id },
data: {
status: "FAILED",
retries: nextRetries,
lastError: truncateError(message),
processedAt: nowDate,
lockedUntil: null,
},
});
} else {
const delayMs = computeOutboxRetryDelayMs(nextRetries);
await store.outboxEvent.update({
where: { id: event.id },
data: {
status: "PENDING",
retries: nextRetries,
lastError: truncateError(message),
scheduledAt: new Date(nowDate.getTime() + delayMs),
lockedUntil: null,
},
});
}
}
}));
}
if (claimed.length > 0) {
logger.info?.({ claimed: claimed.length, ok, errors, deadLetters }, "Outbox dispatch tick complete.");
}
return { claimed: claimed.length, ok, errors, deadLetters };
}
const MAX_LAST_ERROR_LENGTH = 1_000;
function truncateError(message: string): string {
if (message.length <= MAX_LAST_ERROR_LENGTH) return message;
return `${message.slice(0, MAX_LAST_ERROR_LENGTH - 16)}… (truncated)`;
}
function clampDispatchConcurrency(value: number | undefined): number {
if (typeof value !== "number" || !Number.isFinite(value)) return OUTBOX_DEFAULT_DISPATCH_CONCURRENCY;
return Math.max(1, Math.min(100, Math.floor(value)));
}
/**
* Start a long-running polling loop on top of {@link runDispatcherTick}.
*
* Returns a handle that the worker bootstrap can `close()` on shutdown.
* The loop uses `setTimeout` (not `setInterval`) so a slow tick can't
* pile up overlapping invocations.
*/
export type DispatcherHandle = {
readonly running: boolean;
close(): Promise<void>;
};
export type StartDispatcherOptions = DispatcherOptions & {
pollIntervalMs?: number;
};
export function startOutboxDispatcher(options: StartDispatcherOptions): DispatcherHandle {
const pollIntervalMs = options.pollIntervalMs ?? OUTBOX_DEFAULT_POLL_INTERVAL_MS;
const logger = options.logger ?? workerLogger;
let running = true;
let timer: NodeJS.Timeout | undefined;
let inflight: Promise<unknown> | undefined;
const schedule = (): void => {
if (!running) return;
timer = setTimeout(() => {
inflight = runDispatcherTick(options)
.catch((error) => {
logger.error({ error }, "Outbox dispatcher tick threw.");
})
.finally(() => {
inflight = undefined;
schedule();
});
}, pollIntervalMs);
};
schedule();
return {
get running() {
return running;
},
async close(): Promise<void> {
running = false;
if (timer) {
clearTimeout(timer);
timer = undefined;
}
if (inflight) {
try {
await inflight;
} catch {
// already logged above
}
}
},
};
}
export type IsDispatcherEnabledEnv = {
OUTBOX_DISPATCHER_ENABLED?: string;
OUTBOX_DISPATCHER_REQUIRED?: string;
};
export function isOutboxDispatcherEnabled(env: IsDispatcherEnabledEnv = process.env): boolean {
return env.OUTBOX_DISPATCHER_ENABLED === "true";
}
export function isOutboxDispatcherRequired(env: IsDispatcherEnabledEnv = process.env): boolean {
return env.OUTBOX_DISPATCHER_REQUIRED === "true";
}
@@ -0,0 +1,87 @@
import { describe, expect, it } from "vitest";
import { EMAIL_SEQUENCE_TRIGGER_QUEUE } from "../jobs/email-sequence-trigger.js";
import { WORKFLOW_TRIGGER_OUTBOX_QUEUE } from "../jobs/workflow-trigger.js";
import { DEFAULT_OUTBOX_ROUTING, resolveOutboxQueueName } from "./event-router.js";
describe("resolveOutboxQueueName", () => {
it("routes mail.* to outbox.mail", () => {
expect(resolveOutboxQueueName("mail.welcome.send")).toBe("outbox.mail");
});
it("routes webhook.* to outbox.webhook", () => {
expect(resolveOutboxQueueName("webhook.delivery.attempted")).toBe("outbox.webhook");
expect(resolveOutboxQueueName("webhook.form-submission.deliver")).toBe("outbox.webhook");
});
it("routes notification.* to outbox.notification", () => {
expect(resolveOutboxQueueName("notification.in_app.created")).toBe("outbox.notification");
});
it("routes audit.* to outbox.audit", () => {
expect(resolveOutboxQueueName("audit.user.created")).toBe("outbox.audit");
});
it("routes workflow.* to outbox.workflow", () => {
expect(resolveOutboxQueueName("workflow.trigger.fired")).toBe(WORKFLOW_TRIGGER_OUTBOX_QUEUE);
});
it("keeps the workflow routing rule aligned with the workflow trigger queue constant", () => {
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({ prefix: "workflow.", queue: WORKFLOW_TRIGGER_OUTBOX_QUEUE });
});
it("routes billing.* to outbox.notification", () => {
expect(resolveOutboxQueueName("billing.invoice.paid")).toBe("outbox.notification");
});
it("routes marketing.lead.* to the email-sequence trigger queue", () => {
expect(resolveOutboxQueueName("marketing.lead.captured")).toBe(EMAIL_SEQUENCE_TRIGGER_QUEUE);
expect(resolveOutboxQueueName("marketing.lead.convertedtocustomer")).toBe(EMAIL_SEQUENCE_TRIGGER_QUEUE);
});
it("keeps the marketing lead routing rule aligned with the email-sequence trigger owner", () => {
expect(DEFAULT_OUTBOX_ROUTING).toContainEqual({ prefix: "marketing.lead.", queue: EMAIL_SEQUENCE_TRIGGER_QUEUE });
});
it("prefers the longest matching prefix (tenant.subscription. beats no prefix)", () => {
expect(resolveOutboxQueueName("tenant.subscription.updated")).toBe("outbox.notification");
});
it("routes form.submission.* to outbox.form-submission-route", () => {
expect(resolveOutboxQueueName("form.submission.process")).toBe("outbox.form-submission-route");
expect(resolveOutboxQueueName("form.submission.webhook")).toBe("outbox.form-submission-route");
});
it("routes plugin hook events to the plugin hook fanout queue", () => {
expect(resolveOutboxQueueName("plugin.hook.tenant.created")).toBe("outbox.plugin-hook");
expect(resolveOutboxQueueName("plugin.hook.entity.updated")).toBe("outbox.plugin-hook");
expect(resolveOutboxQueueName("plugin.hook.workflow.created")).toBe("outbox.plugin-hook");
expect(resolveOutboxQueueName("plugin.hook.workflow.updated")).toBe("outbox.plugin-hook");
expect(resolveOutboxQueueName("plugin.hook.workflow.deleted")).toBe("outbox.plugin-hook");
});
it("returns undefined when no rule matches", () => {
expect(resolveOutboxQueueName("unknown.entity.verb")).toBeUndefined();
});
it("returns undefined for invalid input", () => {
expect(resolveOutboxQueueName("")).toBeUndefined();
expect(resolveOutboxQueueName(undefined as unknown as string)).toBeUndefined();
});
it("respects custom rules", () => {
expect(
resolveOutboxQueueName("custom.thing.happened", {
rules: [{ prefix: "custom.", queue: "outbox.custom" }],
})
).toBe("outbox.custom");
});
it("exposes a frozen-style default routing list", () => {
expect(DEFAULT_OUTBOX_ROUTING.length).toBeGreaterThan(0);
for (const rule of DEFAULT_OUTBOX_ROUTING) {
expect(rule.prefix.endsWith(".")).toBe(true);
expect(rule.queue.startsWith("outbox.")).toBe(true);
}
});
});
@@ -0,0 +1,57 @@
/**
* Maps OutboxEvent eventType strings to BullMQ queue names.
*
* The convention is `<domain>.<entity>.<verb>`, e.g.
* `billing.subscription.updated` or `mail.welcome.send`. The router
* picks the queue by the first segment (domain) but composite prefixes
* such as `tenant.subscription.` can override individual mappings.
*
* See docs/architecture/outbox-pattern.md.
*/
/** Built-in routing prefixes. Order matters: longest-prefix wins. */
export const DEFAULT_OUTBOX_ROUTING: ReadonlyArray<{ prefix: string; queue: string }> = [
{ prefix: "tenant.subscription.", queue: "outbox.notification" },
{ prefix: "mail.", queue: "outbox.mail" },
{ prefix: "webhook.", queue: "outbox.webhook" },
{ prefix: "notification.", queue: "outbox.notification" },
{ prefix: "audit.", queue: "outbox.audit" },
{ prefix: "workflow.", queue: "outbox.workflow" },
{ prefix: "search.index.", queue: "outbox.search-index" },
{ prefix: "marketing.lead.", queue: "outbox.marketing" },
{ prefix: "form.submission.", queue: "outbox.form-submission-route" },
{ prefix: "billing.", queue: "outbox.notification" },
{ prefix: "plugin.hook.", queue: "outbox.plugin-hook" },
{ prefix: "plugin.capability-use.", queue: "outbox.plugin-capability-use" },
];
export type EventRoutingRule = { prefix: string; queue: string };
export type EventRouterResolveOptions = {
rules?: ReadonlyArray<EventRoutingRule>;
};
/**
* Resolve the queue name for an eventType. Returns `undefined` when no
* routing rule matches — the caller (dispatcher) treats this as a
* dead-letter condition.
*/
export function resolveOutboxQueueName(
eventType: string,
options: EventRouterResolveOptions = {}
): string | undefined {
if (typeof eventType !== "string" || eventType.length === 0) {
return undefined;
}
const rules = options.rules ?? DEFAULT_OUTBOX_ROUTING;
// Longest matching prefix wins.
let best: { prefix: string; queue: string } | undefined;
for (const rule of rules) {
if (eventType.startsWith(rule.prefix)) {
if (!best || rule.prefix.length > best.prefix.length) {
best = rule;
}
}
}
return best?.queue;
}