94 lines
3.7 KiB
TypeScript
94 lines
3.7 KiB
TypeScript
import {
|
|
productAcquisitionProviderOutboxQueueName,
|
|
recurringInvoiceCollectionOutboxQueueName,
|
|
queueNames,
|
|
type OperationalQueueName,
|
|
type QueueName,
|
|
} from "./names.js";
|
|
|
|
/**
|
|
* Worker queue groups — single source of truth for which BullMQ queues a
|
|
* given worker process should subscribe to.
|
|
*
|
|
* The five groups exist to break head-of-line-blocking between unrelated
|
|
* workloads and to allow independent horizontal scaling per workload type
|
|
* in docker-compose / Kubernetes.
|
|
*
|
|
* - `general` — short, mixed I/O jobs (notifications, retention, scan,
|
|
* cleanup, outbox dispatcher cron). Concurrency ~10.
|
|
* - `mail` — outbound mail and webhook delivery (SMTP/HTTP throughput-
|
|
* bound). Concurrency ~5; replicate horizontally for volume.
|
|
* - `import` — provider sync plus long-running CPU/memory-bound
|
|
* imports/exports. KWP uses a dedicated queue so its
|
|
* read-only synchronization can run while generic imports
|
|
* remain gated off.
|
|
* - `webhook` — external API integrations (browser automation) that are
|
|
* I/O-bound and benefit from high parallelism.
|
|
* - `plugin` — third-party WebAssembly plugin invocations (Phase 06
|
|
* #21). Each job spins up a fresh WASM sandbox with hard
|
|
* memory + fuel limits; isolation from other groups is
|
|
* the whole point. Concurrency ~5, replicate ~2.
|
|
*
|
|
* Mapping is exhaustive: every queue name declared in `names.ts` is
|
|
* assigned to exactly one group. The exhaustiveness check lives in
|
|
* `groups.test.ts`.
|
|
*/
|
|
export const WORKER_QUEUE_GROUPS = {
|
|
general: [queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview],
|
|
mail: [queueNames.email],
|
|
import: [queueNames.kwpImports, queueNames.imports, queueNames.exports],
|
|
webhook: [queueNames.browserAutomation],
|
|
plugin: [queueNames.pluginInvoke],
|
|
} as const satisfies Record<string, readonly QueueName[]>;
|
|
|
|
export type WorkerQueueGroup = keyof typeof WORKER_QUEUE_GROUPS;
|
|
|
|
export const WORKER_QUEUE_GROUP_NAMES = Object.keys(WORKER_QUEUE_GROUPS) as readonly WorkerQueueGroup[];
|
|
|
|
const QUEUE_TO_GROUP: Readonly<Record<QueueName, WorkerQueueGroup>> = (() => {
|
|
const map = {} as Record<QueueName, WorkerQueueGroup>;
|
|
for (const group of WORKER_QUEUE_GROUP_NAMES) {
|
|
for (const queue of WORKER_QUEUE_GROUPS[group]) {
|
|
map[queue] = group;
|
|
}
|
|
}
|
|
return map;
|
|
})();
|
|
|
|
/**
|
|
* Returns the queues that belong to `group`. The result is a frozen
|
|
* readonly tuple — callers must not mutate it.
|
|
*/
|
|
export function getQueueNamesForGroup(group: WorkerQueueGroup): readonly QueueName[] {
|
|
return WORKER_QUEUE_GROUPS[group];
|
|
}
|
|
|
|
/**
|
|
* Reverse lookup: which group does this queue belong to? Useful for
|
|
* tagging metrics and Sentry events with the worker group when the
|
|
* producer side already knows the queue name.
|
|
*/
|
|
export function getGroupForQueue(queue: QueueName): WorkerQueueGroup {
|
|
return QUEUE_TO_GROUP[queue];
|
|
}
|
|
|
|
/**
|
|
* Reverse lookup for operator-visible queues. Dedicated outbox consumers are
|
|
* operationally owned by the general worker group without being added to the
|
|
* generic group subscription list.
|
|
*/
|
|
export function getOperationalGroupForQueue(queue: OperationalQueueName): WorkerQueueGroup {
|
|
if (queue === productAcquisitionProviderOutboxQueueName || queue === recurringInvoiceCollectionOutboxQueueName) {
|
|
return "general";
|
|
}
|
|
return getGroupForQueue(queue);
|
|
}
|
|
|
|
/**
|
|
* Type guard — does this string identify a known worker queue group?
|
|
* Used by the runtime entry point to validate `WORKER_QUEUE_GROUP` env.
|
|
*/
|
|
export function isWorkerQueueGroup(value: string): value is WorkerQueueGroup {
|
|
return (WORKER_QUEUE_GROUP_NAMES as readonly string[]).includes(value);
|
|
}
|