750 lines
31 KiB
TypeScript
750 lines
31 KiB
TypeScript
import { WorkerConfigError } from "@hivegrid/config";
|
|
import type { ErrorReporter } from "@hivegrid/error-reporter";
|
|
import { describe, expect, test, vi } from "vitest";
|
|
import { queueNames, type QueueName, type TypedProcessor } from "@hivegrid/queue";
|
|
import {
|
|
shutdownWorkerRuntime,
|
|
startWorkerRuntime,
|
|
getWorkerQueueNames,
|
|
workerQueueNames,
|
|
type StartWorkerRuntimeOptions,
|
|
type WorkerRuntimeLogger,
|
|
type WorkerRuntimeWorker,
|
|
} from "./runtime.js";
|
|
|
|
const mailjetEnv = {
|
|
MAILJET_API_KEY: "mailer-key",
|
|
MAILJET_SECRET_KEY: "mailer-secret",
|
|
MAILJET_FROM_EMAIL: "worker@example.com",
|
|
};
|
|
const databaseEnv = {
|
|
DATABASE_URL: "postgresql://localhost/app",
|
|
};
|
|
const workerGroupEnv = {
|
|
WORKER_QUEUE_GROUP: "general",
|
|
};
|
|
|
|
function createLogger() {
|
|
return { error: vi.fn(), info: vi.fn(), log: vi.fn(), warn: vi.fn() };
|
|
}
|
|
|
|
function createChildLogger() {
|
|
const childLoggers: Array<ReturnType<typeof createLogger>> = [];
|
|
const logger = {
|
|
...createLogger(),
|
|
child: vi.fn(() => {
|
|
const child = createLogger();
|
|
childLoggers.push(child);
|
|
return child;
|
|
}),
|
|
};
|
|
return { childLoggers, logger };
|
|
}
|
|
|
|
type MockWorkerJob = {
|
|
attemptsMade?: number;
|
|
data?: unknown;
|
|
id?: string | number;
|
|
name: string;
|
|
opts?: { attempts?: number; backoff?: number | { delay?: number; type?: string } };
|
|
queueName?: string;
|
|
};
|
|
|
|
type MockWorker = WorkerRuntimeWorker & {
|
|
emitCompleted(job: MockWorkerJob): void;
|
|
emitFailed(job: MockWorkerJob | undefined, error: Error): void;
|
|
};
|
|
|
|
type TestCreateWorker = NonNullable<StartWorkerRuntimeOptions["createWorkerImpl"]>;
|
|
|
|
function createMockWorker(name: QueueName, close = vi.fn(async () => undefined)): MockWorker {
|
|
const listeners = new Map<string, Function>();
|
|
const worker: MockWorker = {
|
|
close,
|
|
name,
|
|
on: vi.fn((event: "completed" | "failed", listener: Function) => {
|
|
listeners.set(event, listener);
|
|
return worker;
|
|
}),
|
|
emitCompleted: (job) => listeners.get("completed")?.(job),
|
|
emitFailed: (job, error) => listeners.get("failed")?.(job, error),
|
|
};
|
|
return worker;
|
|
}
|
|
|
|
function startOneWorker(options: {
|
|
createProcessor?: StartWorkerRuntimeOptions["createProcessor"];
|
|
logger: WorkerRuntimeLogger;
|
|
recordFailedJob?: StartWorkerRuntimeOptions["recordFailedJob"];
|
|
reporter?: ErrorReporter;
|
|
sentry?: StartWorkerRuntimeOptions["sentry"];
|
|
}) {
|
|
const workers: MockWorker[] = [];
|
|
let registeredProcessor: TypedProcessor<QueueName> | undefined;
|
|
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name, processor: TypedProcessor<Name>) => {
|
|
registeredProcessor = processor as TypedProcessor<QueueName>;
|
|
const worker = createMockWorker(queueName);
|
|
workers.push(worker);
|
|
return worker;
|
|
}) as TestCreateWorker;
|
|
const runtime = startWorkerRuntime({
|
|
createProcessor: options.createProcessor ?? ((queueName) => vi.fn(async () => ({ queueName })) as TypedProcessor<typeof queueName>),
|
|
createWorkerImpl,
|
|
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
|
|
logger: options.logger,
|
|
queueNames: [queueNames.email],
|
|
recordFailedJob: options.recordFailedJob,
|
|
reporter: options.reporter,
|
|
sentry: options.sentry,
|
|
});
|
|
return { createWorkerImpl, registeredProcessor, runtime, worker: workers[0] };
|
|
}
|
|
|
|
async function flushMicrotasks() {
|
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
}
|
|
|
|
describe("worker runtime", () => {
|
|
test("returns disabled status without resolving Redis or creating workers", () => {
|
|
const logger = createLogger();
|
|
const createWorkerImpl = vi.fn();
|
|
const runtime = startWorkerRuntime({ createWorkerImpl, env: { WORKER_ENABLED: "false" }, logger });
|
|
expect(runtime).toMatchObject({ configured: false, enabled: false, status: "disabled", workers: [] });
|
|
expect(runtime.queueNames).toEqual(workerQueueNames);
|
|
expect(createWorkerImpl).not.toHaveBeenCalled();
|
|
expect(logger.info).toHaveBeenCalledWith({ queueNames: workerQueueNames }, "Worker disabled; queues not started.");
|
|
});
|
|
|
|
test("rejects enabled startup without Redis using centralized redacted diagnostics", () => {
|
|
expect(() => startWorkerRuntime({ env: { ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv }, logger: createLogger() })).toThrow(WorkerConfigError);
|
|
});
|
|
|
|
test("rejects enabled startup without DATABASE_URL using centralized redacted diagnostics", () => {
|
|
expect(() =>
|
|
startWorkerRuntime({
|
|
createWorkerImpl: vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName)) as TestCreateWorker,
|
|
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
|
|
logger: createLogger(),
|
|
})
|
|
).toThrow(WorkerConfigError);
|
|
});
|
|
|
|
test("starts webhook-capable queue workers without Mailjet config", () => {
|
|
const logger = createLogger();
|
|
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
|
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
|
|
const runtime = startWorkerRuntime({
|
|
createWorkerImpl,
|
|
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "webhook" },
|
|
logger,
|
|
queueNames: [queueNames.email],
|
|
});
|
|
|
|
expect(runtime.status).toBe("started");
|
|
expect(createWorkerImpl).toHaveBeenCalledWith(queueNames.email, expect.any(Function), expect.objectContaining({ concurrency: 1 }));
|
|
});
|
|
|
|
test("registers one worker per queue and logs startup", () => {
|
|
const logger = createLogger();
|
|
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
|
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
|
|
const runtime = startWorkerRuntime({
|
|
createWorkerImpl,
|
|
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...mailjetEnv, WORKER_ENABLED: "true", WORKER_CONCURRENCY: "3", ...workerGroupEnv },
|
|
logger,
|
|
});
|
|
expect(runtime.status).toBe("started");
|
|
expect(runtime.queueNames).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
|
|
expect(workerQueueNames).toEqual([queueNames.email, queueNames.notifications]);
|
|
expect(workerQueueNames).not.toContain(queueNames.exports);
|
|
expect(workerQueueNames).not.toContain(queueNames.browserAutomation);
|
|
expect(createWorkerImpl).toHaveBeenCalledTimes(runtime.queueNames.length);
|
|
expect(vi.mocked(createWorkerImpl).mock.calls.map(([queueName]) => queueName)).toEqual(runtime.queueNames);
|
|
expect(logger.info).toHaveBeenCalledWith({ workerNames: runtime.queueNames }, "Worker started.");
|
|
});
|
|
|
|
test("adds the maintenance queue when export retention processing is explicitly enabled", () => {
|
|
const logger = createLogger();
|
|
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName)) as TestCreateWorker;
|
|
const runtime = startWorkerRuntime({
|
|
createWorkerImpl,
|
|
env: {
|
|
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
|
...databaseEnv,
|
|
EXPORT_RETENTION_QUEUE_ENABLED: "true",
|
|
...mailjetEnv,
|
|
STORAGE_ACCESS_KEY_ID: "access-key",
|
|
STORAGE_BUCKET: "exports",
|
|
STORAGE_ENDPOINT: "https://storage.example.com",
|
|
STORAGE_REGION: "us-east-1",
|
|
STORAGE_SECRET_ACCESS_KEY: "secret-key",
|
|
WORKER_ENABLED: "true",
|
|
...workerGroupEnv,
|
|
},
|
|
logger,
|
|
});
|
|
|
|
expect(runtime.queueNames).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
|
|
expect(getWorkerQueueNames({ EXPORT_RETENTION_QUEUE_ENABLED: "true", WORKER_QUEUE_GROUP: "general" })).toEqual([
|
|
queueNames.notifications,
|
|
queueNames.maintenance,
|
|
queueNames.fileScan,
|
|
queueNames.filePreview,
|
|
]);
|
|
expect(vi.mocked(createWorkerImpl).mock.calls.map(([queueName]) => queueName)).toEqual(runtime.queueNames);
|
|
});
|
|
|
|
test("keeps the import worker group idle until import runtime gates are enabled", () => {
|
|
expect(getWorkerQueueNames({ WORKER_QUEUE_GROUP: "import" })).toEqual([]);
|
|
expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]);
|
|
expect(getWorkerQueueNames({ IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]);
|
|
expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([
|
|
queueNames.imports,
|
|
queueNames.exports,
|
|
]);
|
|
});
|
|
|
|
test("rejects import worker startup without wired import/export services", () => {
|
|
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
|
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
|
|
|
|
expect(() =>
|
|
startWorkerRuntime({
|
|
createWorkerImpl,
|
|
env: {
|
|
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
|
...databaseEnv,
|
|
IMPORT_QUEUE_ENABLED: "true",
|
|
IMPORT_QUEUE_RUNTIME_READY: "true",
|
|
...mailjetEnv,
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "import",
|
|
},
|
|
logger: createLogger(),
|
|
})
|
|
).toThrow(/missing importService, scanService, exportService/u);
|
|
expect(createWorkerImplMock).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("starts import worker queues only with wired import/export services", () => {
|
|
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
|
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
|
|
|
|
const runtime = startWorkerRuntime({
|
|
createWorkerImpl,
|
|
env: {
|
|
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
|
...databaseEnv,
|
|
IMPORT_QUEUE_ENABLED: "true",
|
|
IMPORT_QUEUE_RUNTIME_READY: "true",
|
|
...mailjetEnv,
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "import",
|
|
},
|
|
extraHandlerDependencies: {
|
|
exportService: { runExport: vi.fn() },
|
|
importService: { runImport: vi.fn() },
|
|
scanService: { scanObject: vi.fn() },
|
|
},
|
|
logger: createLogger(),
|
|
});
|
|
|
|
expect(runtime.status).toBe("started");
|
|
expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([queueNames.imports, queueNames.exports]);
|
|
});
|
|
|
|
test("rejects export retention startup without storage delete configuration", () => {
|
|
expect(() =>
|
|
startWorkerRuntime({
|
|
createWorkerImpl: vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName)) as TestCreateWorker,
|
|
env: {
|
|
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
|
...databaseEnv,
|
|
EXPORT_RETENTION_QUEUE_ENABLED: "true",
|
|
...mailjetEnv,
|
|
WORKER_ENABLED: "true",
|
|
...workerGroupEnv,
|
|
},
|
|
logger: createLogger(),
|
|
})
|
|
).toThrow("Export retention storage delete is missing required env");
|
|
});
|
|
|
|
test("binds requestId and retry fields through child loggers for completed and failed jobs", async () => {
|
|
const { childLoggers, logger } = createChildLogger();
|
|
const recordFailedJob = vi.fn(async () => undefined);
|
|
const { worker } = startOneWorker({ logger, recordFailedJob });
|
|
const failure = new Error("handler failed");
|
|
worker.emitCompleted({
|
|
attemptsMade: 1,
|
|
data: { metadata: { requestId: "req_completed" } },
|
|
id: "job_1",
|
|
name: "send-transactional-email",
|
|
opts: { attempts: 3, backoff: 500 },
|
|
queueName: queueNames.email,
|
|
});
|
|
worker.emitFailed(
|
|
{
|
|
attemptsMade: 3,
|
|
data: { metadata: { requestId: "req_failed" } },
|
|
id: "job_2",
|
|
name: "deliver-webhook",
|
|
opts: { attempts: 3, backoff: { delay: 2_000, type: "exponential" } },
|
|
queueName: queueNames.email,
|
|
},
|
|
failure
|
|
);
|
|
await flushMicrotasks();
|
|
expect(logger.child).toHaveBeenCalledWith({
|
|
attemptsLimit: 3,
|
|
attemptsMade: 1,
|
|
backoffDelay: 500,
|
|
backoffType: "fixed",
|
|
jobId: "job_1",
|
|
jobName: "send-transactional-email",
|
|
queueName: queueNames.email,
|
|
requestId: "req_completed",
|
|
});
|
|
expect(logger.child).toHaveBeenCalledWith(expect.objectContaining({ jobId: "job_2", requestId: "req_failed", backoffType: "exponential" }));
|
|
expect(childLoggers[0]?.info).toHaveBeenCalledWith({}, "Worker job completed.");
|
|
expect(childLoggers[1]?.error).toHaveBeenCalledWith({ error: expect.objectContaining({ message: failure.message }) }, "Worker job failed.");
|
|
expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ jobId: "job_2" }));
|
|
});
|
|
|
|
test("falls back to structured calls without child and ignores invalid requestId metadata", () => {
|
|
const logger = createLogger();
|
|
const { worker } = startOneWorker({ logger });
|
|
worker.emitCompleted({
|
|
data: { metadata: { requestId: 12345 } },
|
|
id: "job-no-child",
|
|
name: "deliver-webhook",
|
|
queueName: queueNames.email,
|
|
});
|
|
expect(logger.info).toHaveBeenCalledWith(
|
|
{ attemptsLimit: null, attemptsMade: 0, jobId: "job-no-child", jobName: "deliver-webhook", queueName: queueNames.email },
|
|
"Worker job completed."
|
|
);
|
|
});
|
|
|
|
test("logs retry diagnostics without leaking payload data or persisting non-terminal failures", () => {
|
|
const logger = createLogger();
|
|
const recordFailedJob = vi.fn(async () => undefined);
|
|
const { worker } = startOneWorker({ logger, recordFailedJob });
|
|
const failure = new Error("remote service unavailable");
|
|
worker.emitFailed(
|
|
{
|
|
attemptsMade: 2,
|
|
data: { body: "fixture-body-secret", headers: { authorization: "Bearer token-secret" }, metadata: { requestId: "req_retry" } },
|
|
id: "webhook:tenant_1:event_1",
|
|
name: "deliver-webhook",
|
|
opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } },
|
|
queueName: queueNames.email,
|
|
},
|
|
failure
|
|
);
|
|
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ error: expect.objectContaining({ message: "remote service unavailable" }), requestId: "req_retry", attemptsLimit: 5 }), "Worker job failed.");
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("fixture-body-secret");
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("token-secret");
|
|
expect(recordFailedJob).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("reports failed jobs with correlation fields and redacted payload secrets", async () => {
|
|
const logger = createLogger();
|
|
const reporter: ErrorReporter = { report: vi.fn() };
|
|
const recordFailedJob = vi.fn(async () => undefined);
|
|
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
|
|
const failure = new Error("remote service unavailable token=runtime-secret Authorization: Bearer auth-secret");
|
|
Object.assign(failure, { token: "runtime-secret" });
|
|
|
|
worker.emitFailed(
|
|
{
|
|
attemptsMade: 2,
|
|
data: {
|
|
body: "fixture-body-secret",
|
|
headers: { authorization: "Bearer auth-secret" },
|
|
metadata: { requestId: "req_reported_retry" },
|
|
recipient: "private@example.com",
|
|
templateData: { token: "template-secret" },
|
|
},
|
|
id: "job-reporter-1",
|
|
name: "deliver-webhook",
|
|
opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } },
|
|
queueName: queueNames.email,
|
|
},
|
|
failure
|
|
);
|
|
await flushMicrotasks();
|
|
|
|
expect(reporter.report).toHaveBeenCalledTimes(1);
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
component: "worker",
|
|
operation: "job.failed",
|
|
severity: "error",
|
|
queueName: queueNames.email,
|
|
jobName: "deliver-webhook",
|
|
jobId: "job-reporter-1",
|
|
requestId: "req_reported_retry",
|
|
attemptsMade: 2,
|
|
attemptsLimit: 5,
|
|
retryCount: 2,
|
|
backoffDelay: 2000,
|
|
backoffType: "exponential",
|
|
error: expect.objectContaining({
|
|
errorName: "Error",
|
|
errorMessage: expect.stringContaining("remote service unavailable"),
|
|
}),
|
|
})
|
|
);
|
|
const serializedReport = JSON.stringify(vi.mocked(reporter.report).mock.calls);
|
|
expect(serializedReport).not.toContain("fixture-body-secret");
|
|
expect(serializedReport).not.toContain("auth-secret");
|
|
expect(serializedReport).not.toContain("private@example.com");
|
|
expect(serializedReport).not.toContain("template-secret");
|
|
expect(serializedReport).not.toContain("runtime-secret");
|
|
expect(recordFailedJob).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("persists terminal failures and logs persistence errors without throwing", async () => {
|
|
const logger = createLogger();
|
|
const persistenceError = new Error("database unavailable");
|
|
Object.assign(persistenceError, { token: "persistence-secret-token" });
|
|
const reporter: ErrorReporter = { report: vi.fn() };
|
|
const recordFailedJob = vi.fn(async () => {
|
|
throw persistenceError;
|
|
});
|
|
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
|
|
expect(() =>
|
|
worker.emitFailed(
|
|
{ attemptsMade: 1, data: { token: "persist-secret" }, id: "job-persist-error", name: "deliver-webhook", opts: { attempts: 1 }, queueName: queueNames.email },
|
|
new TypeError("terminal failure")
|
|
)
|
|
).not.toThrow();
|
|
await flushMicrotasks();
|
|
expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ attemptsLimit: 1, errorName: "TypeError", errorMessage: "terminal failure", jobId: "job-persist-error" }));
|
|
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ error: expect.objectContaining({ message: "database unavailable" }), jobId: "job-persist-error" }), "Failed to persist terminal worker failure.");
|
|
expect(reporter.report).toHaveBeenCalledWith(expect.objectContaining({ operation: "job.failed", jobId: "job-persist-error", requestId: undefined }));
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
component: "worker",
|
|
operation: "job.persistence_failed",
|
|
jobId: "job-persist-error",
|
|
error: expect.objectContaining({ errorName: "Error", errorMessage: "database unavailable" }),
|
|
})
|
|
);
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("persist-secret");
|
|
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("persist-secret");
|
|
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("persistence-secret-token");
|
|
});
|
|
|
|
test("logs handler failures with requestId and rethrows processor errors", async () => {
|
|
const logger = createLogger();
|
|
const handlerError = new Error("handler exploded");
|
|
const reporter: ErrorReporter = { report: vi.fn() };
|
|
const { registeredProcessor } = startOneWorker({
|
|
createProcessor: () => vi.fn(async () => {
|
|
throw handlerError;
|
|
}) as TypedProcessor<QueueName>,
|
|
logger,
|
|
reporter,
|
|
});
|
|
await expect(
|
|
registeredProcessor?.({
|
|
attemptsMade: 1,
|
|
data: { metadata: { requestId: "req_handler" }, templateData: "fixture-secret", to: "private@example.com" },
|
|
id: "job_3",
|
|
name: "deliver-webhook",
|
|
opts: { attempts: 3, backoff: 750 },
|
|
} as unknown as Parameters<TypedProcessor<QueueName>>[0])
|
|
).rejects.toThrow(handlerError);
|
|
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ error: expect.objectContaining({ message: "handler exploded" }), requestId: "req_handler", backoffType: "fixed" }), "Worker handler failed.");
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
component: "worker",
|
|
operation: "handler.failed",
|
|
severity: "error",
|
|
requestId: "req_handler",
|
|
jobId: "job_3",
|
|
retryCount: 1,
|
|
error: expect.objectContaining({ errorName: "Error" }),
|
|
})
|
|
);
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("fixture-secret");
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("private@example.com");
|
|
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("fixture-secret");
|
|
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("private@example.com");
|
|
});
|
|
|
|
test("shutdown closes workers and reports close rejections or success as structured records", async () => {
|
|
const logger = createLogger();
|
|
const closeError = new Error("close failed");
|
|
const reporter: ErrorReporter = { report: vi.fn() };
|
|
const emailWorker = createMockWorker(queueNames.email);
|
|
const notificationWorker = createMockWorker(queueNames.notifications, vi.fn(async () => { throw closeError; }));
|
|
await expect(shutdownWorkerRuntime({ workers: [emailWorker, notificationWorker] }, { logger, reason: "SIGTERM", reporter })).rejects.toThrow("Failed to close workers: hivegrid.notifications.");
|
|
expect(logger.error).toHaveBeenCalledWith({ error: expect.objectContaining({ message: "close failed" }), reason: "SIGTERM", workerName: queueNames.notifications }, "Failed to close worker.");
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
component: "worker",
|
|
operation: "shutdown.close_failed",
|
|
severity: "error",
|
|
queueName: queueNames.notifications,
|
|
workerName: queueNames.notifications,
|
|
reason: "SIGTERM",
|
|
error: expect.objectContaining({ errorName: "Error" }),
|
|
})
|
|
);
|
|
await expect(shutdownWorkerRuntime({ workers: [emailWorker] }, { logger, reason: "SIGINT", reporter })).resolves.toEqual({ closedWorkers: [queueNames.email] });
|
|
expect(logger.info).toHaveBeenCalledWith({ closedWorkers: [queueNames.email], reason: "SIGINT" }, "Worker shutdown complete.");
|
|
});
|
|
|
|
test("swallows reporter failures without changing retry or shutdown behavior", async () => {
|
|
const logger = createLogger();
|
|
const reporter: ErrorReporter = { report: vi.fn(async () => { throw new Error("reporter failed"); }) };
|
|
const recordFailedJob = vi.fn(async () => undefined);
|
|
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
|
|
|
|
expect(() =>
|
|
worker.emitFailed(
|
|
{ attemptsMade: 1, data: { metadata: { requestId: "req_reporter_failure" } }, id: "job-report-fail", name: "deliver-webhook", opts: { attempts: 3 }, queueName: queueNames.email },
|
|
new Error("retry failure")
|
|
)
|
|
).not.toThrow();
|
|
await flushMicrotasks();
|
|
|
|
expect(recordFailedJob).not.toHaveBeenCalled();
|
|
const closingWorker = createMockWorker(queueNames.email, vi.fn(async () => { throw new Error("close failed"); }));
|
|
await expect(shutdownWorkerRuntime({ workers: [closingWorker] }, { logger, reporter })).rejects.toThrow("Failed to close workers: hivegrid.email.");
|
|
});
|
|
|
|
test("wraps each job in a Sentry queue.process span with queue/job/tenant tags", async () => {
|
|
type Span = { args: { op: string; name: string; attributes?: Record<string, unknown> }; status?: { code: number; message?: string } };
|
|
const spans: Span[] = [];
|
|
const captureException = vi.fn();
|
|
const sentry = {
|
|
captureException,
|
|
async startSpan<T>(
|
|
args: { op: string; name: string; attributes?: Record<string, unknown> },
|
|
callback: (s: { setAttribute(k: string, v: unknown): void; setStatus(s: { code: number; message?: string }): void }) => Promise<T> | T
|
|
): Promise<T> {
|
|
const record: Span = { args, status: undefined };
|
|
spans.push(record);
|
|
return callback({
|
|
setAttribute() { /* no-op for assertion brevity */ },
|
|
setStatus(s) { record.status = s; },
|
|
});
|
|
},
|
|
};
|
|
|
|
const logger = createLogger();
|
|
const processor = vi.fn(async () => ({ ok: true }));
|
|
const { registeredProcessor } = startOneWorker({
|
|
logger,
|
|
sentry,
|
|
createProcessor: () => processor as unknown as TypedProcessor<QueueName>,
|
|
});
|
|
expect(registeredProcessor).toBeDefined();
|
|
|
|
await registeredProcessor?.({
|
|
attemptsMade: 0,
|
|
data: { tenantId: "tnt_42" },
|
|
id: "job-abc",
|
|
name: "deliver-webhook",
|
|
opts: { attempts: 3 },
|
|
queueName: queueNames.email,
|
|
} as never);
|
|
|
|
expect(spans).toHaveLength(1);
|
|
expect(spans[0].args.op).toBe("queue.process");
|
|
expect(spans[0].args.name).toBe(`${queueNames.email}/deliver-webhook`);
|
|
expect(spans[0].args.attributes?.["messaging.destination.name"]).toBe(queueNames.email);
|
|
expect(spans[0].args.attributes?.["messaging.bullmq.job.name"]).toBe("deliver-webhook");
|
|
expect(spans[0].args.attributes?.["messaging.bullmq.attempts_made"]).toBe(0);
|
|
expect(spans[0].args.attributes?.["messaging.bullmq.attempts_limit"]).toBe(3);
|
|
expect(spans[0].args.attributes?.["hivegrid.tenant_id"]).toBe("tnt_42");
|
|
expect(spans[0].status?.code).toBe(1);
|
|
expect(captureException).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("captures handler exceptions via Sentry.captureException with queue/job/tenant tags", async () => {
|
|
const spans: Array<{ status?: { code: number; message?: string } }> = [];
|
|
const captureException = vi.fn();
|
|
const sentry = {
|
|
captureException,
|
|
async startSpan<T>(
|
|
_args: unknown,
|
|
callback: (s: { setAttribute(k: string, v: unknown): void; setStatus(s: { code: number; message?: string }): void }) => Promise<T> | T
|
|
): Promise<T> {
|
|
const record: { status?: { code: number; message?: string } } = {};
|
|
spans.push(record);
|
|
return callback({
|
|
setAttribute() { /* no-op */ },
|
|
setStatus(s) { record.status = s; },
|
|
});
|
|
},
|
|
};
|
|
|
|
const logger = createLogger();
|
|
const failing = vi.fn(async () => {
|
|
throw new Error("kaboom");
|
|
});
|
|
const { registeredProcessor } = startOneWorker({
|
|
logger,
|
|
sentry,
|
|
createProcessor: () => failing as unknown as TypedProcessor<QueueName>,
|
|
});
|
|
|
|
await expect(
|
|
registeredProcessor?.({
|
|
attemptsMade: 1,
|
|
data: { metadata: { tenantId: "tnt_meta" } },
|
|
id: 7,
|
|
name: "send-email",
|
|
opts: { attempts: 2 },
|
|
queueName: queueNames.email,
|
|
} as never)
|
|
).rejects.toThrow("kaboom");
|
|
|
|
expect(spans).toHaveLength(1);
|
|
expect(spans[0].status?.code).toBe(2);
|
|
expect(captureException).toHaveBeenCalledTimes(1);
|
|
const [thrown, ctx] = captureException.mock.calls[0] ?? [];
|
|
expect((thrown as Error).message).toBe("kaboom");
|
|
const tags = (ctx as { tags: Record<string, string> } | undefined)?.tags;
|
|
expect(tags?.component).toBe("worker");
|
|
expect(tags?.queue).toBe(queueNames.email);
|
|
expect(tags?.job_name).toBe("send-email");
|
|
expect(tags?.job_id).toBe("7");
|
|
expect(tags?.tenant_id).toBe("tnt_meta");
|
|
});
|
|
|
|
test("does not invoke sentry when options.sentry is omitted (default pass-through)", async () => {
|
|
const logger = createLogger();
|
|
const processor = vi.fn(async () => ({ ok: true }));
|
|
const { registeredProcessor } = startOneWorker({
|
|
logger,
|
|
createProcessor: () => processor as unknown as TypedProcessor<QueueName>,
|
|
});
|
|
|
|
await registeredProcessor?.({
|
|
attemptsMade: 0,
|
|
name: "ping",
|
|
queueName: queueNames.email,
|
|
} as never);
|
|
|
|
expect(processor).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
describe("metrics instrumentation", () => {
|
|
function createMetricsStub() {
|
|
const samples = {
|
|
completed: [] as Array<Record<string, string>>,
|
|
failed: [] as Array<Record<string, string>>,
|
|
duration: [] as Array<{ labels: Record<string, string>; value: number }>,
|
|
};
|
|
const metrics = {
|
|
queueJobsCompletedTotal: {
|
|
labels: (labels: Record<string, string>) => ({
|
|
inc: () => {
|
|
samples.completed.push(labels);
|
|
},
|
|
}),
|
|
},
|
|
queueJobsFailedTotal: {
|
|
labels: (labels: Record<string, string>) => ({
|
|
inc: () => {
|
|
samples.failed.push(labels);
|
|
},
|
|
}),
|
|
},
|
|
queueJobDurationSeconds: {
|
|
labels: (labels: Record<string, string>) => ({
|
|
observe: (value: number) => {
|
|
samples.duration.push({ labels, value });
|
|
},
|
|
}),
|
|
},
|
|
};
|
|
return { metrics, samples };
|
|
}
|
|
|
|
test("records completion and duration while leaving queue-depth gauges to the scheduler", async () => {
|
|
const logger = createLogger();
|
|
const { metrics, samples } = createMetricsStub();
|
|
let registered: TypedProcessor<QueueName> | undefined;
|
|
const captureProcessor = vi.fn(<Name extends QueueName>(queueName: Name, processor: TypedProcessor<Name>) => {
|
|
registered = processor as TypedProcessor<QueueName>;
|
|
const worker = createMockWorker(queueName);
|
|
return worker;
|
|
}) as TestCreateWorker;
|
|
|
|
let tick = 0;
|
|
startWorkerRuntime({
|
|
createProcessor: () =>
|
|
vi.fn(async () => {
|
|
return undefined;
|
|
}) as TypedProcessor<QueueName>,
|
|
createWorkerImpl: captureProcessor,
|
|
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
|
|
logger,
|
|
queueNames: [queueNames.email],
|
|
metrics: metrics as never,
|
|
now: () => {
|
|
const next = tick;
|
|
tick += 250;
|
|
return next;
|
|
},
|
|
});
|
|
|
|
await registered?.({
|
|
attemptsMade: 0,
|
|
name: "send-mail",
|
|
queueName: queueNames.email,
|
|
} as never);
|
|
|
|
expect("queueJobsActive" in metrics).toBe(false);
|
|
expect(samples.completed).toEqual([{ queue: queueNames.email, job_name: "send-mail" }]);
|
|
expect(samples.failed).toEqual([]);
|
|
expect(samples.duration).toEqual([{ labels: { queue: queueNames.email, job_name: "send-mail" }, value: 0.25 }]);
|
|
});
|
|
|
|
test("records failed counter and duration on throw", async () => {
|
|
const logger = createLogger();
|
|
const { metrics, samples } = createMetricsStub();
|
|
let registered: TypedProcessor<QueueName> | undefined;
|
|
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name, processor: TypedProcessor<Name>) => {
|
|
registered = processor as TypedProcessor<QueueName>;
|
|
return createMockWorker(queueName);
|
|
}) as TestCreateWorker;
|
|
|
|
class TimeoutError extends Error {
|
|
override name = "TimeoutError";
|
|
}
|
|
|
|
startWorkerRuntime({
|
|
createProcessor: () =>
|
|
vi.fn(async () => {
|
|
throw new TimeoutError("oh no");
|
|
}) as TypedProcessor<QueueName>,
|
|
createWorkerImpl,
|
|
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
|
|
logger,
|
|
queueNames: [queueNames.email],
|
|
metrics: metrics as never,
|
|
});
|
|
|
|
await expect(
|
|
registered?.({
|
|
attemptsMade: 0,
|
|
name: "send-mail",
|
|
queueName: queueNames.email,
|
|
} as never),
|
|
).rejects.toBeInstanceOf(TimeoutError);
|
|
|
|
expect(samples.completed).toEqual([]);
|
|
expect(samples.failed).toEqual([{ queue: queueNames.email, job_name: "send-mail", error_type: "TimeoutError" }]);
|
|
expect(samples.duration.length).toBe(1);
|
|
});
|
|
});
|
|
});
|