/** * AK-P19 — Queue-Monitor service (DoD #6). * * Reads BullMQ job counts for the worker fleet and exposes the operator * controls (pause / resume / retry). The aggregation + audited actions are * unit-testable through an injected {@link QueueInspector}; the concrete * BullMQ adapter ({@link createBullMqQueueInspector}) is thin glue that * needs a live Redis and is verified on staging (no Redis in unit tests). * * Pure presentation (counts -> health) lives in the queue-monitor VM; this * module only gathers raw counts and performs the side-effecting controls. */ import { getOperationalGroupForQueue, fromBullMqJobId, isOperationalQueueName, operationalQueueNames, productAcquisitionProviderOutboxQueueName, queueNames, resolveRedisConnectionOptionsFromProcessEnv, type OperationalQueueName, } from "@hivegrid/queue"; import type { QueueCountsInput } from "@hivegrid/view-models"; import { recordAdminAction, type AdminActionAuditContext, type AdminOperatorAction } from "./admin-action-audit.server"; export type QueueRawCounts = { readonly active: number; readonly waiting: number; readonly delayed: number; readonly failed: number; readonly completed: number; readonly paused: boolean; }; export type QueueFailedJobDetail = { /** BullMQ wire id — the value every control action (retry, detail) must send. */ readonly id: string; /** Same id with the BullMQ colon escaping reversed, for operator-facing text. */ readonly displayId: string; readonly name: string; readonly failedReason: string | null; readonly attemptsMade: number; readonly timestamp: number | null; }; /** * Full single-job view for the operator job detail (DoD #6): includes the * complete BullMQ `stacktrace` (one string per failed attempt), not just * the last `failedReason` line. */ export type QueueJobDetail = { /** BullMQ wire id — the value every control action (retry, detail) must send. */ readonly id: string; /** Same id with the BullMQ colon escaping reversed, for operator-facing text. */ readonly displayId: string; readonly name: string; readonly failedReason: string | null; readonly stacktrace: readonly string[]; readonly attemptsMade: number; readonly timestamp: number | null; readonly processedOn: number | null; readonly finishedOn: number | null; }; /** Live-queue port. The production impl wraps BullMQ; tests pass a fake. */ export interface QueueInspector { getCounts(queueName: string): Promise; listFailedJobs(queueName: string, limit: number): Promise; getJobDetail(queueName: string, jobId: string): Promise; pauseQueue(queueName: string): Promise; resumeQueue(queueName: string): Promise; retryJob(queueName: string, jobId: string): Promise; } export interface QueueMonitorDeps { readonly inspector: QueueInspector; readonly queueNames: readonly string[]; readonly recordAction: (context: AdminActionAuditContext) => Promise; } const ALL_QUEUE_NAMES: readonly OperationalQueueName[] = operationalQueueNames; export class QueueMonitorControlError extends Error { constructor( readonly code: "QUEUE_NOT_ALLOWED" | "JOB_ID_INVALID" | "DIRECT_OUTBOX_RETRY_FORBIDDEN", message: string ) { super(message); this.name = "QueueMonitorControlError"; } } function requireOperationalQueueName(value: string): OperationalQueueName { if (!isOperationalQueueName(value)) { throw new QueueMonitorControlError( "QUEUE_NOT_ALLOWED", "Diese Queue ist nicht für Operator-Aktionen freigegeben." ); } return value; } function requireJobId(value: string): string { const normalized = value.trim(); if (normalized.length === 0 || normalized.length > 191 || normalized.includes("\u0000")) { throw new QueueMonitorControlError("JOB_ID_INVALID", "Die Job-ID ist ungültig."); } return normalized; } function defaultQueueMonitorDeps(): QueueMonitorDeps { return { inspector: createBullMqQueueInspector(), queueNames: ALL_QUEUE_NAMES, recordAction: recordAdminAction }; } /** Gathers raw counts for every queue into the VM input shape. */ export async function getQueueMonitorSnapshot( deps: QueueMonitorDeps = defaultQueueMonitorDeps() ): Promise { return Promise.all( deps.queueNames.map(async (queueName) => { const allowedQueueName = requireOperationalQueueName(queueName); const counts = await deps.inspector.getCounts(allowedQueueName); const group = getOperationalGroupForQueue(allowedQueueName); return { queueName: allowedQueueName, group, ...counts } satisfies QueueCountsInput; }) ); } export type QueueFailedJobsSnapshot = { readonly queueName: string; readonly jobs: readonly QueueFailedJobDetail[]; }; export async function getQueueFailedJobsSnapshot( limitPerQueue = 5, deps: QueueMonitorDeps = defaultQueueMonitorDeps() ): Promise { return Promise.all( deps.queueNames.map(async (queueName) => { const allowedQueueName = requireOperationalQueueName(queueName); return { queueName: allowedQueueName, jobs: await deps.inspector.listFailedJobs(allowedQueueName, limitPerQueue), }; }) ); } /** Reads one job's full detail (error stack + timeline) for the detail dialog. */ export async function getQueueJobDetail( queueName: string, jobId: string, deps: QueueMonitorDeps = defaultQueueMonitorDeps() ): Promise { const allowedQueueName = requireOperationalQueueName(queueName); return deps.inspector.getJobDetail(allowedQueueName, requireJobId(jobId)); } export interface QueueControlInput { readonly queueName: string; readonly actorUserId: string | null; readonly actorEmail?: string | null; readonly request?: Request; } async function auditQueueControl( deps: QueueMonitorDeps, action: Extract, input: QueueControlInput, reason: string, extra?: Record ): Promise { await deps.recordAction({ action, actorUserId: input.actorUserId, actorEmail: input.actorEmail ?? null, targetKind: "queue", targetId: input.queueName, reason, after: { queueName: input.queueName, ...extra }, request: input.request, }); } export async function pauseQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise { const queueName = requireOperationalQueueName(input.queueName); const allowedInput = { ...input, queueName }; await deps.inspector.pauseQueue(queueName); await auditQueueControl(deps, "queue.paused", allowedInput, "Operator hat die Queue pausiert"); } export async function resumeQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise { const queueName = requireOperationalQueueName(input.queueName); const allowedInput = { ...input, queueName }; await deps.inspector.resumeQueue(queueName); await auditQueueControl(deps, "queue.resumed", allowedInput, "Operator hat die Queue fortgesetzt"); } export async function retryQueueJob( input: QueueControlInput & { readonly jobId: string }, deps: QueueMonitorDeps = defaultQueueMonitorDeps() ): Promise { const queueName = requireOperationalQueueName(input.queueName); if (queueName === productAcquisitionProviderOutboxQueueName) { throw new QueueMonitorControlError( "DIRECT_OUTBOX_RETRY_FORBIDDEN", "Direktes BullMQ-Retry ist für diese Finanz-Queue gesperrt. Nutze `hivegrid-cli outbox requeue --tenant --event-id `, damit Tenant-Scope und Delivery-Generation atomar fortgeschrieben werden." ); } const jobId = requireJobId(input.jobId); const allowedInput = { ...input, queueName }; await deps.inspector.retryJob(queueName, jobId); await auditQueueControl(deps, "queue.jobRetried", allowedInput, "Operator hat einen fehlgeschlagenen Job erneut gestartet", { jobId, }); } /** * Structural view of the BullMQ Queue read/control surface we use. The * shared `TypedQueue` contract is intentionally narrowed to `add`/`close` * (enqueue-only); the runtime object is a real BullMQ Queue that also * carries these methods, so we widen structurally at the adapter boundary. */ interface BullJobControl { readonly id?: string | number; readonly name: string; readonly failedReason?: string; readonly stacktrace?: readonly string[]; readonly attemptsMade: number; readonly timestamp?: number; readonly processedOn?: number; readonly finishedOn?: number; retry(): Promise; } interface BullQueueReadControl { getJobCounts(...types: string[]): Promise>; isPaused(): Promise; pause(): Promise; resume(): Promise; getJobs(types: string[], start?: number, end?: number): Promise; getJob(jobId: string): Promise; close(): Promise; } /** * BullMQ-backed inspector. Constructs a short-lived Queue per call against * the worker Redis and closes it again — read traffic only, no consumers. * Requires BULLMQ_REDIS_URL/REDIS_URL; runtime-verified on staging. */ export function createBullMqQueueInspector(): QueueInspector { const connection = resolveRedisConnectionOptionsFromProcessEnv({ required: true }); async function withQueue(queueName: string, fn: (queue: BullQueueReadControl) => Promise): Promise { const allowedQueueName = requireOperationalQueueName(queueName); // Lazy-load the BullMQ runtime through an indirect specifier so the web build // never statically resolves the `@hivegrid/queue/runtime` -> /dev/null alias // (see apps/web/vite.config.ts). Matches every other server-only queue producer. const queueRuntimeModule = "@hivegrid/queue/runtime"; const { createQueue } = (await import(/* @vite-ignore */ queueRuntimeModule)) as typeof import("@hivegrid/queue/runtime"); const queue = createQueue(allowedQueueName as (typeof queueNames)[keyof typeof queueNames], { connection, }) as unknown as BullQueueReadControl; try { return await fn(queue); } finally { await queue.close(); } } return { async getCounts(queueName) { return withQueue(queueName, async (queue) => { const [counts, paused] = await Promise.all([ queue.getJobCounts("active", "waiting", "delayed", "failed", "completed"), queue.isPaused(), ]); return { active: counts.active ?? 0, waiting: counts.waiting ?? 0, delayed: counts.delayed ?? 0, failed: counts.failed ?? 0, completed: counts.completed ?? 0, paused, }; }); }, async listFailedJobs(queueName, limit) { return withQueue(queueName, async (queue) => { const jobs = await queue.getJobs(["failed"], 0, Math.max(0, limit - 1)); return jobs.map((job) => { const id = String(job.id ?? ""); return { id, displayId: fromBullMqJobId(id), name: job.name, failedReason: job.failedReason ?? null, attemptsMade: job.attemptsMade, timestamp: job.timestamp ?? null, }; }); }); }, async getJobDetail(queueName, jobId) { return withQueue(queueName, async (queue) => { const job = await queue.getJob(jobId); if (!job) return null; const id = String(job.id ?? jobId); return { id, displayId: fromBullMqJobId(id), name: job.name, failedReason: job.failedReason ?? null, stacktrace: job.stacktrace ?? [], attemptsMade: job.attemptsMade, timestamp: job.timestamp ?? null, processedOn: job.processedOn ?? null, finishedOn: job.finishedOn ?? null, }; }); }, async pauseQueue(queueName) { await withQueue(queueName, (queue) => queue.pause()); }, async resumeQueue(queueName) { await withQueue(queueName, (queue) => queue.resume()); }, async retryJob(queueName, jobId) { await withQueue(queueName, async (queue) => { const job = await queue.getJob(jobId); if (job) await job.retry(); }); }, }; }