/** * AK-P19 — Queue-Monitor service (DoD #6). * * Reads BullMQ job counts for the worker fleet and exposes the operator * controls (pause / resume / retry). The aggregation + audited actions are * unit-testable through an injected {@link QueueInspector}; the concrete * BullMQ adapter ({@link createBullMqQueueInspector}) is thin glue that * needs a live Redis and is verified on staging (no Redis in unit tests). * * Pure presentation (counts -> health) lives in the queue-monitor VM; this * module only gathers raw counts and performs the side-effecting controls. */ import { getGroupForQueue, queueNames, resolveRedisConnectionOptionsFromProcessEnv } from "@hivegrid/queue"; import { createQueue } from "@hivegrid/queue/runtime"; import type { QueueCountsInput } from "@hivegrid/view-models"; import { recordAdminAction, type AdminActionAuditContext, type AdminOperatorAction } from "./admin-action-audit.server"; export type QueueRawCounts = { readonly active: number; readonly waiting: number; readonly delayed: number; readonly failed: number; readonly completed: number; readonly paused: boolean; }; export type QueueFailedJobDetail = { readonly id: string; readonly name: string; readonly failedReason: string | null; readonly attemptsMade: number; readonly timestamp: number | null; }; /** Live-queue port. The production impl wraps BullMQ; tests pass a fake. */ export interface QueueInspector { getCounts(queueName: string): Promise; listFailedJobs(queueName: string, limit: number): Promise; pauseQueue(queueName: string): Promise; resumeQueue(queueName: string): Promise; retryJob(queueName: string, jobId: string): Promise; } export interface QueueMonitorDeps { readonly inspector: QueueInspector; readonly queueNames: readonly string[]; readonly recordAction: (context: AdminActionAuditContext) => Promise; } const ALL_QUEUE_NAMES: readonly string[] = Object.values(queueNames); function defaultQueueMonitorDeps(): QueueMonitorDeps { return { inspector: createBullMqQueueInspector(), queueNames: ALL_QUEUE_NAMES, recordAction: recordAdminAction }; } /** Gathers raw counts for every queue into the VM input shape. */ export async function getQueueMonitorSnapshot( deps: QueueMonitorDeps = defaultQueueMonitorDeps() ): Promise { return Promise.all( deps.queueNames.map(async (queueName) => { const counts = await deps.inspector.getCounts(queueName); const group = isKnownQueueName(queueName) ? getGroupForQueue(queueName) : "general"; return { queueName, group, ...counts } satisfies QueueCountsInput; }) ); } function isKnownQueueName(value: string): value is (typeof queueNames)[keyof typeof queueNames] { return (ALL_QUEUE_NAMES as readonly string[]).includes(value); } export interface QueueControlInput { readonly queueName: string; readonly actorUserId: string | null; readonly actorEmail?: string | null; readonly request?: Request; } async function auditQueueControl( deps: QueueMonitorDeps, action: Extract, input: QueueControlInput, reason: string, extra?: Record ): Promise { await deps.recordAction({ action, actorUserId: input.actorUserId, actorEmail: input.actorEmail ?? null, targetKind: "queue", targetId: input.queueName, reason, after: { queueName: input.queueName, ...extra }, request: input.request, }); } export async function pauseQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise { await deps.inspector.pauseQueue(input.queueName); await auditQueueControl(deps, "queue.paused", input, "Operator hat die Queue pausiert"); } export async function resumeQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise { await deps.inspector.resumeQueue(input.queueName); await auditQueueControl(deps, "queue.resumed", input, "Operator hat die Queue fortgesetzt"); } export async function retryQueueJob( input: QueueControlInput & { readonly jobId: string }, deps: QueueMonitorDeps = defaultQueueMonitorDeps() ): Promise { await deps.inspector.retryJob(input.queueName, input.jobId); await auditQueueControl(deps, "queue.jobRetried", input, "Operator hat einen fehlgeschlagenen Job erneut gestartet", { jobId: input.jobId, }); } /** * Structural view of the BullMQ Queue read/control surface we use. The * shared `TypedQueue` contract is intentionally narrowed to `add`/`close` * (enqueue-only); the runtime object is a real BullMQ Queue that also * carries these methods, so we widen structurally at the adapter boundary. */ interface BullJobControl { readonly id?: string | number; readonly name: string; readonly failedReason?: string; readonly attemptsMade: number; readonly timestamp?: number; retry(): Promise; } interface BullQueueReadControl { getJobCounts(...types: string[]): Promise>; isPaused(): Promise; pause(): Promise; resume(): Promise; getJobs(types: string[], start?: number, end?: number): Promise; getJob(jobId: string): Promise; close(): Promise; } /** * BullMQ-backed inspector. Constructs a short-lived Queue per call against * the worker Redis and closes it again — read traffic only, no consumers. * Requires BULLMQ_REDIS_URL/REDIS_URL; runtime-verified on staging. */ export function createBullMqQueueInspector(): QueueInspector { const connection = resolveRedisConnectionOptionsFromProcessEnv({ required: true }); async function withQueue(queueName: string, fn: (queue: BullQueueReadControl) => Promise): Promise { const queue = createQueue(queueName as (typeof queueNames)[keyof typeof queueNames], { connection, }) as unknown as BullQueueReadControl; try { return await fn(queue); } finally { await queue.close(); } } return { async getCounts(queueName) { return withQueue(queueName, async (queue) => { const [counts, paused] = await Promise.all([ queue.getJobCounts("active", "waiting", "delayed", "failed", "completed"), queue.isPaused(), ]); return { active: counts.active ?? 0, waiting: counts.waiting ?? 0, delayed: counts.delayed ?? 0, failed: counts.failed ?? 0, completed: counts.completed ?? 0, paused, }; }); }, async listFailedJobs(queueName, limit) { return withQueue(queueName, async (queue) => { const jobs = await queue.getJobs(["failed"], 0, Math.max(0, limit - 1)); return jobs.map((job) => ({ id: String(job.id ?? ""), name: job.name, failedReason: job.failedReason ?? null, attemptsMade: job.attemptsMade, timestamp: job.timestamp ?? null, })); }); }, async pauseQueue(queueName) { await withQueue(queueName, (queue) => queue.pause()); }, async resumeQueue(queueName) { await withQueue(queueName, (queue) => queue.resume()); }, async retryJob(queueName, jobId) { await withQueue(queueName, async (queue) => { const job = await queue.getJob(jobId); if (job) await job.retry(); }); }, }; }