174 lines
7.0 KiB
TypeScript
174 lines
7.0 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
import {
|
|
getQueueFailedJobsSnapshot,
|
|
getQueueJobDetail,
|
|
getQueueMonitorSnapshot,
|
|
pauseQueue,
|
|
resumeQueue,
|
|
retryQueueJob,
|
|
type QueueInspector,
|
|
type QueueMonitorDeps,
|
|
} from "./queue-monitor.server";
|
|
|
|
function makeInspector(): QueueInspector {
|
|
return {
|
|
getCounts: vi.fn(async () => ({ active: 1, waiting: 2, delayed: 0, failed: 3, completed: 4, paused: false })),
|
|
listFailedJobs: vi.fn(async () => []),
|
|
getJobDetail: vi.fn(async () => ({
|
|
id: "job_1",
|
|
name: "send-email",
|
|
failedReason: "SMTP timeout",
|
|
stacktrace: ["Error: SMTP timeout\n at send (mailer.ts:10:3)"],
|
|
attemptsMade: 2,
|
|
timestamp: 1,
|
|
processedOn: 2,
|
|
finishedOn: 3,
|
|
})),
|
|
pauseQueue: vi.fn(async () => {}),
|
|
resumeQueue: vi.fn(async () => {}),
|
|
retryJob: vi.fn(async () => {}),
|
|
};
|
|
}
|
|
|
|
function makeDeps(inspector: QueueInspector, queueNames: readonly string[]): {
|
|
deps: QueueMonitorDeps;
|
|
recordAction: ReturnType<typeof vi.fn>;
|
|
} {
|
|
const recordAction = vi.fn(async () => {});
|
|
return { deps: { inspector, queueNames, recordAction }, recordAction };
|
|
}
|
|
|
|
describe("getQueueMonitorSnapshot", () => {
|
|
it("gathers counts per queue and resolves the worker group", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps } = makeDeps(inspector, ["hivegrid.email", "hivegrid.imports"]);
|
|
const rows = await getQueueMonitorSnapshot(deps);
|
|
expect(rows).toHaveLength(2);
|
|
expect(rows[0]).toMatchObject({ queueName: "hivegrid.email", failed: 3, paused: false });
|
|
expect(rows[0].group).toBeTypeOf("string");
|
|
expect(inspector.getCounts).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it("assigns the dedicated provider outbox queue to the general group", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps } = makeDeps(inspector, ["outbox.billing-product-provider"]);
|
|
const rows = await getQueueMonitorSnapshot(deps);
|
|
expect(rows[0].group).toBe("general");
|
|
});
|
|
|
|
it("rejects queue names outside the operational allowlist", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps } = makeDeps(inspector, ["totally.unknown.queue"]);
|
|
|
|
await expect(getQueueMonitorSnapshot(deps)).rejects.toMatchObject({ code: "QUEUE_NOT_ALLOWED" });
|
|
expect(inspector.getCounts).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe("getQueueFailedJobsSnapshot", () => {
|
|
it("lists failed jobs per queue", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps } = makeDeps(inspector, ["hivegrid.email", "hivegrid.imports"]);
|
|
|
|
const rows = await getQueueFailedJobsSnapshot(3, deps);
|
|
|
|
expect(rows).toEqual([
|
|
{ queueName: "hivegrid.email", jobs: [] },
|
|
{ queueName: "hivegrid.imports", jobs: [] },
|
|
]);
|
|
expect(inspector.listFailedJobs).toHaveBeenCalledWith("hivegrid.email", 3);
|
|
expect(inspector.listFailedJobs).toHaveBeenCalledWith("hivegrid.imports", 3);
|
|
});
|
|
});
|
|
|
|
describe("queue controls", () => {
|
|
it("pauseQueue calls the inspector and audits queue.paused", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps, recordAction } = makeDeps(inspector, ["hivegrid.email"]);
|
|
await pauseQueue({ queueName: "hivegrid.email", actorUserId: "a1" }, deps);
|
|
expect(inspector.pauseQueue).toHaveBeenCalledWith("hivegrid.email");
|
|
expect(recordAction.mock.calls[0][0]).toMatchObject({ action: "queue.paused", targetKind: "queue", targetId: "hivegrid.email" });
|
|
});
|
|
|
|
it("resumeQueue calls the inspector and audits queue.resumed", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps, recordAction } = makeDeps(inspector, ["hivegrid.email"]);
|
|
await resumeQueue({ queueName: "hivegrid.email", actorUserId: "a1" }, deps);
|
|
expect(inspector.resumeQueue).toHaveBeenCalledOnce();
|
|
expect(recordAction.mock.calls[0][0]).toMatchObject({ action: "queue.resumed" });
|
|
});
|
|
|
|
it("retryQueueJob retries the job and audits with the jobId", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps, recordAction } = makeDeps(inspector, ["hivegrid.email"]);
|
|
await retryQueueJob({ queueName: "hivegrid.email", jobId: "job-7", actorUserId: "a1" }, deps);
|
|
expect(inspector.retryJob).toHaveBeenCalledWith("hivegrid.email", "job-7");
|
|
expect(recordAction.mock.calls[0][0]).toMatchObject({ action: "queue.jobRetried", after: { queueName: "hivegrid.email", jobId: "job-7" } });
|
|
});
|
|
|
|
it("blocks direct BullMQ retry for provider outbox jobs and points to generation-aware replay", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps, recordAction } = makeDeps(inspector, ["outbox.billing-product-provider"]);
|
|
|
|
await expect(
|
|
retryQueueJob(
|
|
{ queueName: "outbox.billing-product-provider", jobId: "outbox-event-1-0", actorUserId: "a1" },
|
|
deps
|
|
)
|
|
).rejects.toMatchObject({
|
|
code: "DIRECT_OUTBOX_RETRY_FORBIDDEN",
|
|
message: expect.stringContaining("hivegrid-cli outbox requeue --tenant"),
|
|
});
|
|
expect(inspector.retryJob).not.toHaveBeenCalled();
|
|
expect(recordAction).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("rejects retry controls for queues outside the operational allowlist", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps, recordAction } = makeDeps(inspector, ["hivegrid.email"]);
|
|
|
|
const promise = retryQueueJob({ queueName: "rogue.queue", jobId: "job-7", actorUserId: "a1" }, deps);
|
|
await expect(promise).rejects.toMatchObject({ name: "QueueMonitorControlError", code: "QUEUE_NOT_ALLOWED" });
|
|
expect(inspector.retryJob).not.toHaveBeenCalled();
|
|
expect(recordAction).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("rejects empty job ids before touching BullMQ", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps, recordAction } = makeDeps(inspector, ["hivegrid.email"]);
|
|
|
|
await expect(
|
|
retryQueueJob({ queueName: "hivegrid.email", jobId: " ", actorUserId: "a1" }, deps)
|
|
).rejects.toMatchObject({ code: "JOB_ID_INVALID" });
|
|
expect(inspector.retryJob).not.toHaveBeenCalled();
|
|
expect(recordAction).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe("getQueueJobDetail", () => {
|
|
it("returns the full stacktrace for an allowed queue", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps } = makeDeps(inspector, ["hivegrid.email"]);
|
|
|
|
const detail = await getQueueJobDetail("hivegrid.email", "job_1", deps);
|
|
|
|
expect(inspector.getJobDetail).toHaveBeenCalledWith("hivegrid.email", "job_1");
|
|
expect(detail).toMatchObject({
|
|
id: "job_1",
|
|
failedReason: "SMTP timeout",
|
|
stacktrace: ["Error: SMTP timeout\n at send (mailer.ts:10:3)"],
|
|
processedOn: 2,
|
|
finishedOn: 3,
|
|
});
|
|
});
|
|
|
|
it("rejects non-operational queues and invalid job ids", async () => {
|
|
const inspector = makeInspector();
|
|
const { deps } = makeDeps(inspector, ["hivegrid.email"]);
|
|
|
|
await expect(getQueueJobDetail("not-a-queue", "job_1", deps)).rejects.toMatchObject({ code: "QUEUE_NOT_ALLOWED" });
|
|
await expect(getQueueJobDetail("hivegrid.email", "", deps)).rejects.toMatchObject({ code: "JOB_ID_INVALID" });
|
|
expect(inspector.getJobDetail).not.toHaveBeenCalled();
|
|
});
|
|
});
|