import type { ErrorReporter } from "@toolctrl/error-reporter"; import { describe, expect, test, vi } from "vitest"; import { queueNames, type QueueName, type TypedProcessor } from "@toolctrl/queue"; import { startWorkerRuntime, type StartWorkerRuntimeOptions, type WorkerRuntimeWorker } from "../src/runtime.js"; type MockWorkerJob = { attemptsMade?: number; data?: unknown; id?: string | number; name: string; opts?: { attempts?: number; backoff?: number | { delay?: number; type?: string } }; queueName?: string; }; type MockWorker = WorkerRuntimeWorker & { emitFailed(job: MockWorkerJob | undefined, error: Error): void; }; type TestCreateWorker = NonNullable; function createLogger() { return { error: vi.fn(), info: vi.fn(), log: vi.fn() }; } function createMockWorker(name: QueueName): MockWorker { let failedListener: ((job: MockWorkerJob | undefined, error: Error) => void) | undefined; const worker: MockWorker = { close: vi.fn(async () => undefined), name, on: vi.fn((event: "completed" | "failed", listener: (job: MockWorkerJob | undefined, error: Error) => void) => { if (event === "failed") failedListener = listener; return worker; }) as WorkerRuntimeWorker["on"], emitFailed: (job, error) => failedListener?.(job, error), }; return worker; } function createStartedRuntime(options: { logger: ReturnType; recordFailedJob?: StartWorkerRuntimeOptions["recordFailedJob"]; reporter?: ErrorReporter; }) { const workers: MockWorker[] = []; const createWorkerImpl = vi.fn((queueName: Name) => { const worker = createMockWorker(queueName); workers.push(worker); return worker; }) as TestCreateWorker; startWorkerRuntime({ createProcessor: (queueName) => vi.fn(async () => ({ queueName })) as TypedProcessor, createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", DATABASE_URL: "postgresql://user:pass@localhost:5432/toolctrl", WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "mail", }, logger: options.logger, queueNames: [queueNames.email], recordFailedJob: options.recordFailedJob, reporter: options.reporter, }); return { worker: workers[0] }; } async function flushMicrotasks() { await new Promise((resolve) => setTimeout(resolve, 0)); } describe("worker forced-failure diagnostics", () => { test("logs retry diagnostics for non-terminal failures without persisting payload data", async () => { const logger = createLogger(); const reporter: ErrorReporter = { report: vi.fn() }; const recordFailedJob = vi.fn(async () => undefined); const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter }); const failure = new Error("upstream timed out"); worker.emitFailed( { attemptsMade: 2, data: { body: "secret-webhook-body", headers: { authorization: "Bearer non-terminal-token" }, metadata: { requestId: "req_non_terminal" }, recipient: "private@example.com", templateData: { code: "123456" }, }, id: "job-retry-1", name: "deliver-webhook", opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } }, queueName: queueNames.email, }, failure ); await flushMicrotasks(); expect(logger.error).toHaveBeenCalledWith( expect.objectContaining({ attemptsLimit: 5, attemptsMade: 2, backoffDelay: 2000, backoffType: "exponential", error: expect.objectContaining({ message: failure.message }), jobId: "job-retry-1", jobName: "deliver-webhook", queueName: queueNames.email, requestId: "req_non_terminal", }), "Worker job failed." ); expect(recordFailedJob).not.toHaveBeenCalled(); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ attemptsLimit: 5, attemptsMade: 2, backoffDelay: 2000, backoffType: "exponential", component: "worker", error: expect.objectContaining({ errorMessage: "upstream timed out", errorName: "Error" }), jobId: "job-retry-1", jobName: "deliver-webhook", operation: "job.failed", queueName: queueNames.email, requestId: "req_non_terminal", retryCount: 2, severity: "error", }) ); const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls); expect(reporterPayload).not.toContain("secret-webhook-body"); expect(reporterPayload).not.toContain("non-terminal-token"); expect(reporterPayload).not.toContain("private@example.com"); expect(reporterPayload).not.toContain("templateData"); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("secret-webhook-body"); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("non-terminal-token"); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("private@example.com"); }); test("persists terminal failures with redacted scalar diagnostics only", async () => { const logger = createLogger(); const reporter: ErrorReporter = { report: vi.fn() }; const recordFailedJob = vi.fn(async () => undefined); const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter }); const failure = new TypeError("render failed for request"); worker.emitFailed( { attemptsMade: 3, data: { body: "secret-email-body", headers: { authorization: "Bearer terminal-token" }, metadata: { requestId: "req_terminal" }, recipient: "customer@example.com", templateData: { resetToken: "template-secret-token" }, }, id: 42, name: "send-transactional-email", opts: { attempts: 3 }, queueName: queueNames.email, }, failure ); await flushMicrotasks(); expect(recordFailedJob).toHaveBeenCalledTimes(1); expect(recordFailedJob).toHaveBeenCalledWith({ attemptsLimit: 3, attemptsMade: 3, errorMessage: "render failed for request", errorName: "TypeError", failedAt: expect.any(Date), jobId: "42", jobName: "send-transactional-email", queueName: queueNames.email, }); const persistedPayload = JSON.stringify(vi.mocked(recordFailedJob).mock.calls); expect(persistedPayload).not.toContain("data"); expect(persistedPayload).not.toContain("headers"); expect(persistedPayload).not.toContain("body"); expect(persistedPayload).not.toContain("templateData"); expect(persistedPayload).not.toContain("recipient"); expect(persistedPayload).not.toContain("terminal-token"); expect(persistedPayload).not.toContain("template-secret-token"); expect(persistedPayload).not.toContain("customer@example.com"); expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ requestId: "req_terminal" }), "Worker job failed."); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ attemptsLimit: 3, attemptsMade: 3, component: "worker", error: expect.objectContaining({ errorMessage: "render failed for request", errorName: "TypeError" }), jobId: "42", jobName: "send-transactional-email", operation: "job.failed", queueName: queueNames.email, requestId: "req_terminal", retryCount: 3, severity: "error", }) ); const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls); expect(reporterPayload).not.toContain("secret-email-body"); expect(reporterPayload).not.toContain("terminal-token"); expect(reporterPayload).not.toContain("template-secret-token"); expect(reporterPayload).not.toContain("customer@example.com"); }); test("logs persistence rejections from terminal failures without throwing", async () => { const logger = createLogger(); const reporter: ErrorReporter = { report: vi.fn() }; const persistenceError = new Error("database unavailable"); const recordFailedJob = vi.fn(async () => { throw persistenceError; }); const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter }); expect(() => worker.emitFailed( { attemptsMade: 1, data: { metadata: { requestId: "req_persist" }, token: "persist-secret-token" }, id: "job-persist-error", name: "deliver-webhook", opts: { attempts: 1 }, queueName: queueNames.email, }, new Error("terminal failure") ) ).not.toThrow(); await flushMicrotasks(); expect(logger.error).toHaveBeenCalledWith( expect.objectContaining({ attemptsLimit: 1, error: expect.objectContaining({ message: persistenceError.message }), jobId: "job-persist-error", requestId: "req_persist", }), "Failed to persist terminal worker failure." ); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ attemptsLimit: 1, attemptsMade: 1, component: "worker", error: expect.objectContaining({ errorMessage: "terminal failure", errorName: "Error" }), jobId: "job-persist-error", jobName: "deliver-webhook", operation: "job.failed", queueName: queueNames.email, requestId: "req_persist", retryCount: 1, severity: "error", }) ); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ attemptsLimit: 1, attemptsMade: 1, component: "worker", error: expect.objectContaining({ errorMessage: "database unavailable", errorName: "Error" }), jobId: "job-persist-error", jobName: "deliver-webhook", operation: "job.persistence_failed", queueName: queueNames.email, requestId: "req_persist", retryCount: 1, severity: "error", }) ); const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls); expect(reporterPayload).not.toContain("persist-secret-token"); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("persist-secret-token"); }); });