Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+149
-19
@@ -10,8 +10,16 @@
|
||||
* Pure presentation (counts -> health) lives in the queue-monitor VM; this
|
||||
* module only gathers raw counts and performs the side-effecting controls.
|
||||
*/
|
||||
import { getGroupForQueue, queueNames, resolveRedisConnectionOptionsFromProcessEnv } from "@hivegrid/queue";
|
||||
import { createQueue } from "@hivegrid/queue/runtime";
|
||||
import {
|
||||
getOperationalGroupForQueue,
|
||||
fromBullMqJobId,
|
||||
isOperationalQueueName,
|
||||
operationalQueueNames,
|
||||
productAcquisitionProviderOutboxQueueName,
|
||||
queueNames,
|
||||
resolveRedisConnectionOptionsFromProcessEnv,
|
||||
type OperationalQueueName,
|
||||
} from "@hivegrid/queue";
|
||||
import type { QueueCountsInput } from "@hivegrid/view-models";
|
||||
import { recordAdminAction, type AdminActionAuditContext, type AdminOperatorAction } from "./admin-action-audit.server";
|
||||
|
||||
@@ -25,17 +33,40 @@ export type QueueRawCounts = {
|
||||
};
|
||||
|
||||
export type QueueFailedJobDetail = {
|
||||
/** BullMQ wire id — the value every control action (retry, detail) must send. */
|
||||
readonly id: string;
|
||||
/** Same id with the BullMQ colon escaping reversed, for operator-facing text. */
|
||||
readonly displayId: string;
|
||||
readonly name: string;
|
||||
readonly failedReason: string | null;
|
||||
readonly attemptsMade: number;
|
||||
readonly timestamp: number | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Full single-job view for the operator job detail (DoD #6): includes the
|
||||
* complete BullMQ `stacktrace` (one string per failed attempt), not just
|
||||
* the last `failedReason` line.
|
||||
*/
|
||||
export type QueueJobDetail = {
|
||||
/** BullMQ wire id — the value every control action (retry, detail) must send. */
|
||||
readonly id: string;
|
||||
/** Same id with the BullMQ colon escaping reversed, for operator-facing text. */
|
||||
readonly displayId: string;
|
||||
readonly name: string;
|
||||
readonly failedReason: string | null;
|
||||
readonly stacktrace: readonly string[];
|
||||
readonly attemptsMade: number;
|
||||
readonly timestamp: number | null;
|
||||
readonly processedOn: number | null;
|
||||
readonly finishedOn: number | null;
|
||||
};
|
||||
|
||||
/** Live-queue port. The production impl wraps BullMQ; tests pass a fake. */
|
||||
export interface QueueInspector {
|
||||
getCounts(queueName: string): Promise<QueueRawCounts>;
|
||||
listFailedJobs(queueName: string, limit: number): Promise<readonly QueueFailedJobDetail[]>;
|
||||
getJobDetail(queueName: string, jobId: string): Promise<QueueJobDetail | null>;
|
||||
pauseQueue(queueName: string): Promise<void>;
|
||||
resumeQueue(queueName: string): Promise<void>;
|
||||
retryJob(queueName: string, jobId: string): Promise<void>;
|
||||
@@ -47,7 +78,35 @@ export interface QueueMonitorDeps {
|
||||
readonly recordAction: (context: AdminActionAuditContext) => Promise<void>;
|
||||
}
|
||||
|
||||
const ALL_QUEUE_NAMES: readonly string[] = Object.values(queueNames);
|
||||
const ALL_QUEUE_NAMES: readonly OperationalQueueName[] = operationalQueueNames;
|
||||
|
||||
export class QueueMonitorControlError extends Error {
|
||||
constructor(
|
||||
readonly code: "QUEUE_NOT_ALLOWED" | "JOB_ID_INVALID" | "DIRECT_OUTBOX_RETRY_FORBIDDEN",
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "QueueMonitorControlError";
|
||||
}
|
||||
}
|
||||
|
||||
function requireOperationalQueueName(value: string): OperationalQueueName {
|
||||
if (!isOperationalQueueName(value)) {
|
||||
throw new QueueMonitorControlError(
|
||||
"QUEUE_NOT_ALLOWED",
|
||||
"Diese Queue ist nicht für Operator-Aktionen freigegeben."
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireJobId(value: string): string {
|
||||
const normalized = value.trim();
|
||||
if (normalized.length === 0 || normalized.length > 191 || normalized.includes("\u0000")) {
|
||||
throw new QueueMonitorControlError("JOB_ID_INVALID", "Die Job-ID ist ungültig.");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function defaultQueueMonitorDeps(): QueueMonitorDeps {
|
||||
return { inspector: createBullMqQueueInspector(), queueNames: ALL_QUEUE_NAMES, recordAction: recordAdminAction };
|
||||
@@ -59,15 +118,42 @@ export async function getQueueMonitorSnapshot(
|
||||
): Promise<QueueCountsInput[]> {
|
||||
return Promise.all(
|
||||
deps.queueNames.map(async (queueName) => {
|
||||
const counts = await deps.inspector.getCounts(queueName);
|
||||
const group = isKnownQueueName(queueName) ? getGroupForQueue(queueName) : "general";
|
||||
return { queueName, group, ...counts } satisfies QueueCountsInput;
|
||||
const allowedQueueName = requireOperationalQueueName(queueName);
|
||||
const counts = await deps.inspector.getCounts(allowedQueueName);
|
||||
const group = getOperationalGroupForQueue(allowedQueueName);
|
||||
return { queueName: allowedQueueName, group, ...counts } satisfies QueueCountsInput;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function isKnownQueueName(value: string): value is (typeof queueNames)[keyof typeof queueNames] {
|
||||
return (ALL_QUEUE_NAMES as readonly string[]).includes(value);
|
||||
export type QueueFailedJobsSnapshot = {
|
||||
readonly queueName: string;
|
||||
readonly jobs: readonly QueueFailedJobDetail[];
|
||||
};
|
||||
|
||||
export async function getQueueFailedJobsSnapshot(
|
||||
limitPerQueue = 5,
|
||||
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
|
||||
): Promise<QueueFailedJobsSnapshot[]> {
|
||||
return Promise.all(
|
||||
deps.queueNames.map(async (queueName) => {
|
||||
const allowedQueueName = requireOperationalQueueName(queueName);
|
||||
return {
|
||||
queueName: allowedQueueName,
|
||||
jobs: await deps.inspector.listFailedJobs(allowedQueueName, limitPerQueue),
|
||||
};
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/** Reads one job's full detail (error stack + timeline) for the detail dialog. */
|
||||
export async function getQueueJobDetail(
|
||||
queueName: string,
|
||||
jobId: string,
|
||||
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
|
||||
): Promise<QueueJobDetail | null> {
|
||||
const allowedQueueName = requireOperationalQueueName(queueName);
|
||||
return deps.inspector.getJobDetail(allowedQueueName, requireJobId(jobId));
|
||||
}
|
||||
|
||||
export interface QueueControlInput {
|
||||
@@ -97,22 +183,35 @@ async function auditQueueControl(
|
||||
}
|
||||
|
||||
export async function pauseQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise<void> {
|
||||
await deps.inspector.pauseQueue(input.queueName);
|
||||
await auditQueueControl(deps, "queue.paused", input, "Operator hat die Queue pausiert");
|
||||
const queueName = requireOperationalQueueName(input.queueName);
|
||||
const allowedInput = { ...input, queueName };
|
||||
await deps.inspector.pauseQueue(queueName);
|
||||
await auditQueueControl(deps, "queue.paused", allowedInput, "Operator hat die Queue pausiert");
|
||||
}
|
||||
|
||||
export async function resumeQueue(input: QueueControlInput, deps: QueueMonitorDeps = defaultQueueMonitorDeps()): Promise<void> {
|
||||
await deps.inspector.resumeQueue(input.queueName);
|
||||
await auditQueueControl(deps, "queue.resumed", input, "Operator hat die Queue fortgesetzt");
|
||||
const queueName = requireOperationalQueueName(input.queueName);
|
||||
const allowedInput = { ...input, queueName };
|
||||
await deps.inspector.resumeQueue(queueName);
|
||||
await auditQueueControl(deps, "queue.resumed", allowedInput, "Operator hat die Queue fortgesetzt");
|
||||
}
|
||||
|
||||
export async function retryQueueJob(
|
||||
input: QueueControlInput & { readonly jobId: string },
|
||||
deps: QueueMonitorDeps = defaultQueueMonitorDeps()
|
||||
): Promise<void> {
|
||||
await deps.inspector.retryJob(input.queueName, input.jobId);
|
||||
await auditQueueControl(deps, "queue.jobRetried", input, "Operator hat einen fehlgeschlagenen Job erneut gestartet", {
|
||||
jobId: input.jobId,
|
||||
const queueName = requireOperationalQueueName(input.queueName);
|
||||
if (queueName === productAcquisitionProviderOutboxQueueName) {
|
||||
throw new QueueMonitorControlError(
|
||||
"DIRECT_OUTBOX_RETRY_FORBIDDEN",
|
||||
"Direktes BullMQ-Retry ist für diese Finanz-Queue gesperrt. Nutze `hivegrid-cli outbox requeue --tenant <tenant-id> --event-id <outbox-event-id>`, damit Tenant-Scope und Delivery-Generation atomar fortgeschrieben werden."
|
||||
);
|
||||
}
|
||||
const jobId = requireJobId(input.jobId);
|
||||
const allowedInput = { ...input, queueName };
|
||||
await deps.inspector.retryJob(queueName, jobId);
|
||||
await auditQueueControl(deps, "queue.jobRetried", allowedInput, "Operator hat einen fehlgeschlagenen Job erneut gestartet", {
|
||||
jobId,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -126,8 +225,11 @@ interface BullJobControl {
|
||||
readonly id?: string | number;
|
||||
readonly name: string;
|
||||
readonly failedReason?: string;
|
||||
readonly stacktrace?: readonly string[];
|
||||
readonly attemptsMade: number;
|
||||
readonly timestamp?: number;
|
||||
readonly processedOn?: number;
|
||||
readonly finishedOn?: number;
|
||||
retry(): Promise<void>;
|
||||
}
|
||||
interface BullQueueReadControl {
|
||||
@@ -149,7 +251,13 @@ export function createBullMqQueueInspector(): QueueInspector {
|
||||
const connection = resolveRedisConnectionOptionsFromProcessEnv({ required: true });
|
||||
|
||||
async function withQueue<T>(queueName: string, fn: (queue: BullQueueReadControl) => Promise<T>): Promise<T> {
|
||||
const queue = createQueue(queueName as (typeof queueNames)[keyof typeof queueNames], {
|
||||
const allowedQueueName = requireOperationalQueueName(queueName);
|
||||
// Lazy-load the BullMQ runtime through an indirect specifier so the web build
|
||||
// never statically resolves the `@hivegrid/queue/runtime` -> /dev/null alias
|
||||
// (see apps/web/vite.config.ts). Matches every other server-only queue producer.
|
||||
const queueRuntimeModule = "@hivegrid/queue/runtime";
|
||||
const { createQueue } = (await import(/* @vite-ignore */ queueRuntimeModule)) as typeof import("@hivegrid/queue/runtime");
|
||||
const queue = createQueue(allowedQueueName as (typeof queueNames)[keyof typeof queueNames], {
|
||||
connection,
|
||||
}) as unknown as BullQueueReadControl;
|
||||
try {
|
||||
@@ -179,13 +287,35 @@ export function createBullMqQueueInspector(): QueueInspector {
|
||||
async listFailedJobs(queueName, limit) {
|
||||
return withQueue(queueName, async (queue) => {
|
||||
const jobs = await queue.getJobs(["failed"], 0, Math.max(0, limit - 1));
|
||||
return jobs.map((job) => ({
|
||||
id: String(job.id ?? ""),
|
||||
return jobs.map((job) => {
|
||||
const id = String(job.id ?? "");
|
||||
return {
|
||||
id,
|
||||
displayId: fromBullMqJobId(id),
|
||||
name: job.name,
|
||||
failedReason: job.failedReason ?? null,
|
||||
attemptsMade: job.attemptsMade,
|
||||
timestamp: job.timestamp ?? null,
|
||||
};
|
||||
});
|
||||
});
|
||||
},
|
||||
async getJobDetail(queueName, jobId) {
|
||||
return withQueue(queueName, async (queue) => {
|
||||
const job = await queue.getJob(jobId);
|
||||
if (!job) return null;
|
||||
const id = String(job.id ?? jobId);
|
||||
return {
|
||||
id,
|
||||
displayId: fromBullMqJobId(id),
|
||||
name: job.name,
|
||||
failedReason: job.failedReason ?? null,
|
||||
stacktrace: job.stacktrace ?? [],
|
||||
attemptsMade: job.attemptsMade,
|
||||
timestamp: job.timestamp ?? null,
|
||||
}));
|
||||
processedOn: job.processedOn ?? null,
|
||||
finishedOn: job.finishedOn ?? null,
|
||||
};
|
||||
});
|
||||
},
|
||||
async pauseQueue(queueName) {
|
||||
|
||||
Reference in new issue
Block a user