Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,75 @@
|
||||
import { queueNames, type QueueName } from "./names.js";
|
||||
|
||||
/**
|
||||
* Worker queue groups — single source of truth for which BullMQ queues a
|
||||
* given worker process should subscribe to.
|
||||
*
|
||||
* The five groups exist to break head-of-line-blocking between unrelated
|
||||
* workloads and to allow independent horizontal scaling per workload type
|
||||
* in docker-compose / Kubernetes.
|
||||
*
|
||||
* - `general` — short, mixed I/O jobs (notifications, retention, scan,
|
||||
* cleanup, outbox dispatcher cron). Concurrency ~10.
|
||||
* - `mail` — outbound mail and webhook delivery (SMTP/HTTP throughput-
|
||||
* bound). Concurrency ~5; replicate horizontally for volume.
|
||||
* - `import` — long-running CPU/memory-bound imports/exports. Concurrency
|
||||
* 1 — head-of-line-blocking is acceptable inside this group
|
||||
* because all jobs are heavy; we isolate them so they don't
|
||||
* stall the rest of the system.
|
||||
* - `webhook` — external API integrations (AI chat, browser automation)
|
||||
* that are I/O-bound and benefit from high parallelism.
|
||||
* - `plugin` — third-party WebAssembly plugin invocations (Phase 06
|
||||
* #21). Each job spins up a fresh WASM sandbox with hard
|
||||
* memory + fuel limits; isolation from other groups is
|
||||
* the whole point. Concurrency ~5, replicate ~2.
|
||||
*
|
||||
* Mapping is exhaustive: every queue name declared in `names.ts` is
|
||||
* assigned to exactly one group. The exhaustiveness check lives in
|
||||
* `groups.test.ts`.
|
||||
*/
|
||||
export const WORKER_QUEUE_GROUPS = {
|
||||
general: [queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview],
|
||||
mail: [queueNames.email],
|
||||
import: [queueNames.imports, queueNames.exports],
|
||||
webhook: [queueNames.aiChatbot, queueNames.browserAutomation],
|
||||
plugin: [queueNames.pluginInvoke],
|
||||
} as const satisfies Record<string, readonly QueueName[]>;
|
||||
|
||||
export type WorkerQueueGroup = keyof typeof WORKER_QUEUE_GROUPS;
|
||||
|
||||
export const WORKER_QUEUE_GROUP_NAMES = Object.keys(WORKER_QUEUE_GROUPS) as readonly WorkerQueueGroup[];
|
||||
|
||||
const QUEUE_TO_GROUP: Readonly<Record<QueueName, WorkerQueueGroup>> = (() => {
|
||||
const map = {} as Record<QueueName, WorkerQueueGroup>;
|
||||
for (const group of WORKER_QUEUE_GROUP_NAMES) {
|
||||
for (const queue of WORKER_QUEUE_GROUPS[group]) {
|
||||
map[queue] = group;
|
||||
}
|
||||
}
|
||||
return map;
|
||||
})();
|
||||
|
||||
/**
|
||||
* Returns the queues that belong to `group`. The result is a frozen
|
||||
* readonly tuple — callers must not mutate it.
|
||||
*/
|
||||
export function getQueueNamesForGroup(group: WorkerQueueGroup): readonly QueueName[] {
|
||||
return WORKER_QUEUE_GROUPS[group];
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse lookup: which group does this queue belong to? Useful for
|
||||
* tagging metrics and Sentry events with the worker group when the
|
||||
* producer side already knows the queue name.
|
||||
*/
|
||||
export function getGroupForQueue(queue: QueueName): WorkerQueueGroup {
|
||||
return QUEUE_TO_GROUP[queue];
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard — does this string identify a known worker queue group?
|
||||
* Used by the runtime entry point to validate `WORKER_QUEUE_GROUP` env.
|
||||
*/
|
||||
export function isWorkerQueueGroup(value: string): value is WorkerQueueGroup {
|
||||
return (WORKER_QUEUE_GROUP_NAMES as readonly string[]).includes(value);
|
||||
}
|
||||
Reference in new issue
Block a user