Files

335 lines
12 KiB
TypeScript

/**
* AK-P19 — Queue-Monitor service (DoD #6).
*
* Reads BullMQ job counts for the worker fleet and exposes the operator
* controls (pause / resume / retry). The aggregation + audited actions are
* unit-testable through an injected {@link QueueInspector}; the concrete
* BullMQ adapter ({@link createBullMqQueueInspector}) is thin glue that
* needs a live Redis and is verified on staging (no Redis in unit tests).
*
* Pure presentation (counts -> health) lives in the queue-monitor VM; this
* module only gathers raw counts and performs the side-effecting controls.
*/
import {
getOperationalGroupForQueue,
fromBullMqJobId,
isOperationalQueueName,
operationalQueueNames,
productAcquisitionProviderOutboxQueueName,
queueNames,
resolveRedisConnectionOptionsFromProcessEnv,
type OperationalQueueName,
} from "@hivegrid/queue";
import type { QueueCountsInput } from "@hivegrid/view-models";
import { recordAdminAction, type AdminActionAuditContext, type AdminOperatorAction } from "./admin-action-audit.server";
export type QueueRawCounts = {
readonly active: number;
readonly waiting: number;
readonly delayed: number;
readonly failed: number;
readonly completed: number;
readonly paused: boolean;
};
export type QueueFailedJobDetail = {
/** BullMQ wire id — the value every control action (retry, detail) must send. */
readonly id: string;
/** Same id with the BullMQ colon escaping reversed, for operator-facing text. */
readonly displayId: string;
readonly name: string;
readonly failedReason: string | null;
readonly attemptsMade: number;
readonly timestamp: number | null;
};
/**
* Full single-job view for the operator job detail (DoD #6): includes the
* complete BullMQ `stacktrace` (one string per failed attempt), not just
* the last `failedReason` line.
*/
export type QueueJobDetail = {
/** BullMQ wire id — the value every control action (retry, detail) must send. */
readonly id: string;
/** Same id with the BullMQ colon escaping reversed, for operator-facing text. */
readonly displayId: string;
readonly name: string;
readonly failedReason: string | null;
readonly stacktrace: readonly string[];
readonly attemptsMade: number;
readonly timestamp: number | null;
readonly processedOn: number | null;
readonly finishedOn: number | null;
};
/** Live-queue port. The production impl wraps BullMQ; tests pass a fake. */
export interface QueueInspector {
getCounts(queueName: string): Promise<QueueRawCounts>;
listFailedJobs(queueName: string, limit: number): Promise<readonly QueueFailedJobDetail[]>;
getJobDetail(queueName: string, jobId: string): Promise<QueueJobDetail | null>;
pauseQueue(queueName: string): Promise<void>;
resumeQueue(queueName: string): Promise<void>;
retryJob(queueName: string, jobId: string): Promise<void>;
}
export interface QueueMonitorDeps {
readonly inspector: QueueInspector;
readonly queueNames: readonly string[];
readonly recordAction: (context: AdminActionAuditContext) => Promise<void>;
}
const ALL_QUEUE_NAMES: readonly OperationalQueueName[] = operationalQueueNames;
export class QueueMonitorControlError extends Error {
constructor(
readonly code: "QUEUE_NOT_ALLOWED" | "JOB_ID_INVALID" | "DIRECT_OUTBOX_RETRY_FORBIDDEN",
message: string
) {
super(message);
this.name = "QueueMonitorControlError";
}
}
function requireOperationalQueueName(value: string): OperationalQueueName {
if (!isOperationalQueueName(value)) {
throw new QueueMonitorControlError(
"QUEUE_NOT_ALLOWED",
"Diese Queue ist nicht für Operator-Aktionen freigegeben."
);
}
return value;
}
function requireJobId(value: string): string {
const normalized = value.trim();
if (normalized.length === 0 || normalized.length > 191 || normalized.includes("\u0000")) {
throw new QueueMonitorControlError("JOB_ID_INVALID", "Die Job-ID ist ungültig.");
}
return normalized;
}
function defaultQueueMonitorDeps(): QueueMonitorDeps {
return { inspector: createBullMqQueueInspector(), queueNames: ALL_QUEUE_NAMES, recordAction: recordAdminAction };
}
/** Gathers raw counts for every queue into the VM input shape. */
export async function getQueueMonitorSnapshot(
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
): Promise<QueueCountsInput[]> {
return Promise.all(
deps.queueNames.map(async (queueName) => {
const allowedQueueName = requireOperationalQueueName(queueName);
const counts = await deps.inspector.getCounts(allowedQueueName);
const group = getOperationalGroupForQueue(allowedQueueName);
return { queueName: allowedQueueName, group, ...counts } satisfies QueueCountsInput;
})
);
}
export type QueueFailedJobsSnapshot = {
readonly queueName: string;
readonly jobs: readonly QueueFailedJobDetail[];
};
export async function getQueueFailedJobsSnapshot(
limitPerQueue = 5,
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
): Promise<QueueFailedJobsSnapshot[]> {
return Promise.all(
deps.queueNames.map(async (queueName) => {
const allowedQueueName = requireOperationalQueueName(queueName);
return {
queueName: allowedQueueName,
jobs: await deps.inspector.listFailedJobs(allowedQueueName, limitPerQueue),
};
})
);
}
/** Reads one job's full detail (error stack + timeline) for the detail dialog. */
export async function getQueueJobDetail(
queueName: string,
jobId: string,
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
): Promise<QueueJobDetail | null> {
const allowedQueueName = requireOperationalQueueName(queueName);
return deps.inspector.getJobDetail(allowedQueueName, requireJobId(jobId));
}
export interface QueueControlInput {
readonly queueName: string;
readonly actorUserId: string | null;
readonly actorEmail?: string | null;
readonly request?: Request;
}
async function auditQueueControl(
deps: QueueMonitorDeps,
action: Extract<AdminOperatorAction, "queue.paused" | "queue.resumed" | "queue.jobRetried">,
input: QueueControlInput,
reason: string,
extra?: Record<string, unknown>
): Promise<void> {
await deps.recordAction({
action,
actorUserId: input.actorUserId,
actorEmail: input.actorEmail ?? null,
targetKind: "queue",
targetId: input.queueName,
reason,
after: { queueName: input.queueName, ...extra },
request: input.request,
});
}
export async function pauseQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise<void> {
const queueName = requireOperationalQueueName(input.queueName);
const allowedInput = { ...input, queueName };
await deps.inspector.pauseQueue(queueName);
await auditQueueControl(deps, "queue.paused", allowedInput, "Operator hat die Queue pausiert");
}
export async function resumeQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise<void> {
const queueName = requireOperationalQueueName(input.queueName);
const allowedInput = { ...input, queueName };
await deps.inspector.resumeQueue(queueName);
await auditQueueControl(deps, "queue.resumed", allowedInput, "Operator hat die Queue fortgesetzt");
}
export async function retryQueueJob(
input: QueueControlInput & { readonly jobId: string },
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
): Promise<void> {
const queueName = requireOperationalQueueName(input.queueName);
if (queueName === productAcquisitionProviderOutboxQueueName) {
throw new QueueMonitorControlError(
"DIRECT_OUTBOX_RETRY_FORBIDDEN",
"Direktes BullMQ-Retry ist für diese Finanz-Queue gesperrt. Nutze `hivegrid-cli outbox requeue --tenant <tenant-id> --event-id <outbox-event-id>`, damit Tenant-Scope und Delivery-Generation atomar fortgeschrieben werden."
);
}
const jobId = requireJobId(input.jobId);
const allowedInput = { ...input, queueName };
await deps.inspector.retryJob(queueName, jobId);
await auditQueueControl(deps, "queue.jobRetried", allowedInput, "Operator hat einen fehlgeschlagenen Job erneut gestartet", {
jobId,
});
}
/**
* Structural view of the BullMQ Queue read/control surface we use. The
* shared `TypedQueue` contract is intentionally narrowed to `add`/`close`
* (enqueue-only); the runtime object is a real BullMQ Queue that also
* carries these methods, so we widen structurally at the adapter boundary.
*/
interface BullJobControl {
readonly id?: string | number;
readonly name: string;
readonly failedReason?: string;
readonly stacktrace?: readonly string[];
readonly attemptsMade: number;
readonly timestamp?: number;
readonly processedOn?: number;
readonly finishedOn?: number;
retry(): Promise<void>;
}
interface BullQueueReadControl {
getJobCounts(...types: string[]): Promise<Record<string, number>>;
isPaused(): Promise<boolean>;
pause(): Promise<void>;
resume(): Promise<void>;
getJobs(types: string[], start?: number, end?: number): Promise<BullJobControl[]>;
getJob(jobId: string): Promise<BullJobControl | null | undefined>;
close(): Promise<void>;
}
/**
* BullMQ-backed inspector. Constructs a short-lived Queue per call against
* the worker Redis and closes it again — read traffic only, no consumers.
* Requires BULLMQ_REDIS_URL/REDIS_URL; runtime-verified on staging.
*/
export function createBullMqQueueInspector(): QueueInspector {
const connection = resolveRedisConnectionOptionsFromProcessEnv({ required: true });
async function withQueue<T>(queueName: string, fn: (queue: BullQueueReadControl) => Promise<T>): Promise<T> {
const allowedQueueName = requireOperationalQueueName(queueName);
// Lazy-load the BullMQ runtime through an indirect specifier so the web build
// never statically resolves the `@hivegrid/queue/runtime` -> /dev/null alias
// (see apps/web/vite.config.ts). Matches every other server-only queue producer.
const queueRuntimeModule = "@hivegrid/queue/runtime";
const { createQueue } = (await import(/* @vite-ignore */ queueRuntimeModule)) as typeof import("@hivegrid/queue/runtime");
const queue = createQueue(allowedQueueName as (typeof queueNames)[keyof typeof queueNames], {
connection,
}) as unknown as BullQueueReadControl;
try {
return await fn(queue);
} finally {
await queue.close();
}
}
return {
async getCounts(queueName) {
return withQueue(queueName, async (queue) => {
const [counts, paused] = await Promise.all([
queue.getJobCounts("active", "waiting", "delayed", "failed", "completed"),
queue.isPaused(),
]);
return {
active: counts.active ?? 0,
waiting: counts.waiting ?? 0,
delayed: counts.delayed ?? 0,
failed: counts.failed ?? 0,
completed: counts.completed ?? 0,
paused,
};
});
},
async listFailedJobs(queueName, limit) {
return withQueue(queueName, async (queue) => {
const jobs = await queue.getJobs(["failed"], 0, Math.max(0, limit - 1));
return jobs.map((job) => {
const id = String(job.id ?? "");
return {
id,
displayId: fromBullMqJobId(id),
name: job.name,
failedReason: job.failedReason ?? null,
attemptsMade: job.attemptsMade,
timestamp: job.timestamp ?? null,
};
});
});
},
async getJobDetail(queueName, jobId) {
return withQueue(queueName, async (queue) => {
const job = await queue.getJob(jobId);
if (!job) return null;
const id = String(job.id ?? jobId);
return {
id,
displayId: fromBullMqJobId(id),
name: job.name,
failedReason: job.failedReason ?? null,
stacktrace: job.stacktrace ?? [],
attemptsMade: job.attemptsMade,
timestamp: job.timestamp ?? null,
processedOn: job.processedOn ?? null,
finishedOn: job.finishedOn ?? null,
};
});
},
async pauseQueue(queueName) {
await withQueue(queueName, (queue) => queue.pause());
},
async resumeQueue(queueName) {
await withQueue(queueName, (queue) => queue.resume());
},
async retryJob(queueName, jobId) {
await withQueue(queueName, async (queue) => {
const job = await queue.getJob(jobId);
if (job) await job.retry();
});
},
};
}