Update hivegrid-cloud reference to latest main
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e98d38e716
commit
7b21524c86
3781 files changed
+373484
-225607
No files matched your search
+204
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* AK-P19 — Queue-Monitor service (DoD #6).
|
||||
*
|
||||
* Reads BullMQ job counts for the worker fleet and exposes the operator
|
||||
* controls (pause / resume / retry). The aggregation + audited actions are
|
||||
* unit-testable through an injected {@link QueueInspector}; the concrete
|
||||
* BullMQ adapter ({@link createBullMqQueueInspector}) is thin glue that
|
||||
* needs a live Redis and is verified on staging (no Redis in unit tests).
|
||||
*
|
||||
* Pure presentation (counts -> health) lives in the queue-monitor VM; this
|
||||
* module only gathers raw counts and performs the side-effecting controls.
|
||||
*/
|
||||
import { getGroupForQueue, queueNames, resolveRedisConnectionOptionsFromProcessEnv } from "@hivegrid/queue";
|
||||
import { createQueue } from "@hivegrid/queue/runtime";
|
||||
import type { QueueCountsInput } from "@hivegrid/view-models";
|
||||
import { recordAdminAction, type AdminActionAuditContext, type AdminOperatorAction } from "./admin-action-audit.server";
|
||||
|
||||
export type QueueRawCounts = {
|
||||
readonly active: number;
|
||||
readonly waiting: number;
|
||||
readonly delayed: number;
|
||||
readonly failed: number;
|
||||
readonly completed: number;
|
||||
readonly paused: boolean;
|
||||
};
|
||||
|
||||
export type QueueFailedJobDetail = {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly failedReason: string | null;
|
||||
readonly attemptsMade: number;
|
||||
readonly timestamp: number | null;
|
||||
};
|
||||
|
||||
/** Live-queue port. The production impl wraps BullMQ; tests pass a fake. */
|
||||
export interface QueueInspector {
|
||||
getCounts(queueName: string): Promise<QueueRawCounts>;
|
||||
listFailedJobs(queueName: string, limit: number): Promise<readonly QueueFailedJobDetail[]>;
|
||||
pauseQueue(queueName: string): Promise<void>;
|
||||
resumeQueue(queueName: string): Promise<void>;
|
||||
retryJob(queueName: string, jobId: string): Promise<void>;
|
||||
}
|
||||
|
||||
export interface QueueMonitorDeps {
|
||||
readonly inspector: QueueInspector;
|
||||
readonly queueNames: readonly string[];
|
||||
readonly recordAction: (context: AdminActionAuditContext) => Promise<void>;
|
||||
}
|
||||
|
||||
const ALL_QUEUE_NAMES: readonly string[] = Object.values(queueNames);
|
||||
|
||||
function defaultQueueMonitorDeps(): QueueMonitorDeps {
|
||||
return { inspector: createBullMqQueueInspector(), queueNames: ALL_QUEUE_NAMES, recordAction: recordAdminAction };
|
||||
}
|
||||
|
||||
/** Gathers raw counts for every queue into the VM input shape. */
|
||||
export async function getQueueMonitorSnapshot(
|
||||
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
|
||||
): Promise<QueueCountsInput[]> {
|
||||
return Promise.all(
|
||||
deps.queueNames.map(async (queueName) => {
|
||||
const counts = await deps.inspector.getCounts(queueName);
|
||||
const group = isKnownQueueName(queueName) ? getGroupForQueue(queueName) : "general";
|
||||
return { queueName, group, ...counts } satisfies QueueCountsInput;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function isKnownQueueName(value: string): value is (typeof queueNames)[keyof typeof queueNames] {
|
||||
return (ALL_QUEUE_NAMES as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
export interface QueueControlInput {
|
||||
readonly queueName: string;
|
||||
readonly actorUserId: string | null;
|
||||
readonly actorEmail?: string | null;
|
||||
readonly request?: Request;
|
||||
}
|
||||
|
||||
async function auditQueueControl(
|
||||
deps: QueueMonitorDeps,
|
||||
action: Extract<AdminOperatorAction, "queue.paused" | "queue.resumed" | "queue.jobRetried">,
|
||||
input: QueueControlInput,
|
||||
reason: string,
|
||||
extra?: Record<string, unknown>
|
||||
): Promise<void> {
|
||||
await deps.recordAction({
|
||||
action,
|
||||
actorUserId: input.actorUserId,
|
||||
actorEmail: input.actorEmail ?? null,
|
||||
targetKind: "queue",
|
||||
targetId: input.queueName,
|
||||
reason,
|
||||
after: { queueName: input.queueName, ...extra },
|
||||
request: input.request,
|
||||
});
|
||||
}
|
||||
|
||||
export async function pauseQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise<void> {
|
||||
await deps.inspector.pauseQueue(input.queueName);
|
||||
await auditQueueControl(deps, "queue.paused", input, "Operator hat die Queue pausiert");
|
||||
}
|
||||
|
||||
export async function resumeQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise<void> {
|
||||
await deps.inspector.resumeQueue(input.queueName);
|
||||
await auditQueueControl(deps, "queue.resumed", input, "Operator hat die Queue fortgesetzt");
|
||||
}
|
||||
|
||||
export async function retryQueueJob(
|
||||
input: QueueControlInput & { readonly jobId: string },
|
||||
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
|
||||
): Promise<void> {
|
||||
await deps.inspector.retryJob(input.queueName, input.jobId);
|
||||
await auditQueueControl(deps, "queue.jobRetried", input, "Operator hat einen fehlgeschlagenen Job erneut gestartet", {
|
||||
jobId: input.jobId,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural view of the BullMQ Queue read/control surface we use. The
|
||||
* shared `TypedQueue` contract is intentionally narrowed to `add`/`close`
|
||||
* (enqueue-only); the runtime object is a real BullMQ Queue that also
|
||||
* carries these methods, so we widen structurally at the adapter boundary.
|
||||
*/
|
||||
interface BullJobControl {
|
||||
readonly id?: string | number;
|
||||
readonly name: string;
|
||||
readonly failedReason?: string;
|
||||
readonly attemptsMade: number;
|
||||
readonly timestamp?: number;
|
||||
retry(): Promise<void>;
|
||||
}
|
||||
interface BullQueueReadControl {
|
||||
getJobCounts(...types: string[]): Promise<Record<string, number>>;
|
||||
isPaused(): Promise<boolean>;
|
||||
pause(): Promise<void>;
|
||||
resume(): Promise<void>;
|
||||
getJobs(types: string[], start?: number, end?: number): Promise<BullJobControl[]>;
|
||||
getJob(jobId: string): Promise<BullJobControl | null | undefined>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* BullMQ-backed inspector. Constructs a short-lived Queue per call against
|
||||
* the worker Redis and closes it again — read traffic only, no consumers.
|
||||
* Requires BULLMQ_REDIS_URL/REDIS_URL; runtime-verified on staging.
|
||||
*/
|
||||
export function createBullMqQueueInspector(): QueueInspector {
|
||||
const connection = resolveRedisConnectionOptionsFromProcessEnv({ required: true });
|
||||
|
||||
async function withQueue<T>(queueName: string, fn: (queue: BullQueueReadControl) => Promise<T>): Promise<T> {
|
||||
const queue = createQueue(queueName as (typeof queueNames)[keyof typeof queueNames], {
|
||||
connection,
|
||||
}) as unknown as BullQueueReadControl;
|
||||
try {
|
||||
return await fn(queue);
|
||||
} finally {
|
||||
await queue.close();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async getCounts(queueName) {
|
||||
return withQueue(queueName, async (queue) => {
|
||||
const [counts, paused] = await Promise.all([
|
||||
queue.getJobCounts("active", "waiting", "delayed", "failed", "completed"),
|
||||
queue.isPaused(),
|
||||
]);
|
||||
return {
|
||||
active: counts.active ?? 0,
|
||||
waiting: counts.waiting ?? 0,
|
||||
delayed: counts.delayed ?? 0,
|
||||
failed: counts.failed ?? 0,
|
||||
completed: counts.completed ?? 0,
|
||||
paused,
|
||||
};
|
||||
});
|
||||
},
|
||||
async listFailedJobs(queueName, limit) {
|
||||
return withQueue(queueName, async (queue) => {
|
||||
const jobs = await queue.getJobs(["failed"], 0, Math.max(0, limit - 1));
|
||||
return jobs.map((job) => ({
|
||||
id: String(job.id ?? ""),
|
||||
name: job.name,
|
||||
failedReason: job.failedReason ?? null,
|
||||
attemptsMade: job.attemptsMade,
|
||||
timestamp: job.timestamp ?? null,
|
||||
}));
|
||||
});
|
||||
},
|
||||
async pauseQueue(queueName) {
|
||||
await withQueue(queueName, (queue) => queue.pause());
|
||||
},
|
||||
async resumeQueue(queueName) {
|
||||
await withQueue(queueName, (queue) => queue.resume());
|
||||
},
|
||||
async retryJob(queueName, jobId) {
|
||||
await withQueue(queueName, async (queue) => {
|
||||
const job = await queue.getJob(jobId);
|
||||
if (job) await job.retry();
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user