279 lines
10 KiB
TypeScript
279 lines
10 KiB
TypeScript
import type { ErrorReporter } from "@toolctrl/error-reporter";
|
|
import { describe, expect, test, vi } from "vitest";
|
|
import { queueNames, type QueueName, type TypedProcessor } from "@toolctrl/queue";
|
|
import { startWorkerRuntime, type StartWorkerRuntimeOptions, type WorkerRuntimeWorker } from "../src/runtime.js";
|
|
|
|
type MockWorkerJob = {
|
|
attemptsMade?: number;
|
|
data?: unknown;
|
|
id?: string | number;
|
|
name: string;
|
|
opts?: { attempts?: number; backoff?: number | { delay?: number; type?: string } };
|
|
queueName?: string;
|
|
};
|
|
|
|
type MockWorker = WorkerRuntimeWorker & {
|
|
emitFailed(job: MockWorkerJob | undefined, error: Error): void;
|
|
};
|
|
|
|
type TestCreateWorker = NonNullable<StartWorkerRuntimeOptions["createWorkerImpl"]>;
|
|
|
|
function createLogger() {
|
|
return { error: vi.fn(), info: vi.fn(), log: vi.fn() };
|
|
}
|
|
|
|
function createMockWorker(name: QueueName): MockWorker {
|
|
let failedListener: ((job: MockWorkerJob | undefined, error: Error) => void) | undefined;
|
|
const worker: MockWorker = {
|
|
close: vi.fn(async () => undefined),
|
|
name,
|
|
on: vi.fn((event: "completed" | "failed", listener: (job: MockWorkerJob | undefined, error: Error) => void) => {
|
|
if (event === "failed") failedListener = listener;
|
|
return worker;
|
|
}) as WorkerRuntimeWorker["on"],
|
|
emitFailed: (job, error) => failedListener?.(job, error),
|
|
};
|
|
return worker;
|
|
}
|
|
|
|
function createStartedRuntime(options: {
|
|
logger: ReturnType<typeof createLogger>;
|
|
recordFailedJob?: StartWorkerRuntimeOptions["recordFailedJob"];
|
|
reporter?: ErrorReporter;
|
|
}) {
|
|
const workers: MockWorker[] = [];
|
|
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => {
|
|
const worker = createMockWorker(queueName);
|
|
workers.push(worker);
|
|
return worker;
|
|
}) as TestCreateWorker;
|
|
|
|
startWorkerRuntime({
|
|
createProcessor: (queueName) => vi.fn(async () => ({ queueName })) as TypedProcessor<typeof queueName>,
|
|
createWorkerImpl,
|
|
env: {
|
|
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
|
DATABASE_URL: "postgresql://user:pass@localhost:5432/toolctrl",
|
|
WORKER_ENABLED: "true",
|
|
WORKER_QUEUE_GROUP: "mail",
|
|
},
|
|
logger: options.logger,
|
|
queueNames: [queueNames.email],
|
|
recordFailedJob: options.recordFailedJob,
|
|
reporter: options.reporter,
|
|
});
|
|
|
|
return { worker: workers[0] };
|
|
}
|
|
|
|
async function flushMicrotasks() {
|
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
}
|
|
|
|
describe("worker forced-failure diagnostics", () => {
|
|
test("logs retry diagnostics for non-terminal failures without persisting payload data", async () => {
|
|
const logger = createLogger();
|
|
const reporter: ErrorReporter = { report: vi.fn() };
|
|
const recordFailedJob = vi.fn(async () => undefined);
|
|
const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter });
|
|
const failure = new Error("upstream timed out");
|
|
|
|
worker.emitFailed(
|
|
{
|
|
attemptsMade: 2,
|
|
data: {
|
|
body: "secret-webhook-body",
|
|
headers: { authorization: "Bearer non-terminal-token" },
|
|
metadata: { requestId: "req_non_terminal" },
|
|
recipient: "private@example.com",
|
|
templateData: { code: "123456" },
|
|
},
|
|
id: "job-retry-1",
|
|
name: "deliver-webhook",
|
|
opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } },
|
|
queueName: queueNames.email,
|
|
},
|
|
failure
|
|
);
|
|
await flushMicrotasks();
|
|
|
|
expect(logger.error).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
attemptsLimit: 5,
|
|
attemptsMade: 2,
|
|
backoffDelay: 2000,
|
|
backoffType: "exponential",
|
|
error: expect.objectContaining({ message: failure.message }),
|
|
jobId: "job-retry-1",
|
|
jobName: "deliver-webhook",
|
|
queueName: queueNames.email,
|
|
requestId: "req_non_terminal",
|
|
}),
|
|
"Worker job failed."
|
|
);
|
|
expect(recordFailedJob).not.toHaveBeenCalled();
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
attemptsLimit: 5,
|
|
attemptsMade: 2,
|
|
backoffDelay: 2000,
|
|
backoffType: "exponential",
|
|
component: "worker",
|
|
error: expect.objectContaining({ errorMessage: "upstream timed out", errorName: "Error" }),
|
|
jobId: "job-retry-1",
|
|
jobName: "deliver-webhook",
|
|
operation: "job.failed",
|
|
queueName: queueNames.email,
|
|
requestId: "req_non_terminal",
|
|
retryCount: 2,
|
|
severity: "error",
|
|
})
|
|
);
|
|
const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls);
|
|
expect(reporterPayload).not.toContain("secret-webhook-body");
|
|
expect(reporterPayload).not.toContain("non-terminal-token");
|
|
expect(reporterPayload).not.toContain("private@example.com");
|
|
expect(reporterPayload).not.toContain("templateData");
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("secret-webhook-body");
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("non-terminal-token");
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("private@example.com");
|
|
});
|
|
|
|
test("persists terminal failures with redacted scalar diagnostics only", async () => {
|
|
const logger = createLogger();
|
|
const reporter: ErrorReporter = { report: vi.fn() };
|
|
const recordFailedJob = vi.fn(async () => undefined);
|
|
const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter });
|
|
const failure = new TypeError("render failed for request");
|
|
|
|
worker.emitFailed(
|
|
{
|
|
attemptsMade: 3,
|
|
data: {
|
|
body: "secret-email-body",
|
|
headers: { authorization: "Bearer terminal-token" },
|
|
metadata: { requestId: "req_terminal" },
|
|
recipient: "customer@example.com",
|
|
templateData: { resetToken: "template-secret-token" },
|
|
},
|
|
id: 42,
|
|
name: "send-transactional-email",
|
|
opts: { attempts: 3 },
|
|
queueName: queueNames.email,
|
|
},
|
|
failure
|
|
);
|
|
await flushMicrotasks();
|
|
|
|
expect(recordFailedJob).toHaveBeenCalledTimes(1);
|
|
expect(recordFailedJob).toHaveBeenCalledWith({
|
|
attemptsLimit: 3,
|
|
attemptsMade: 3,
|
|
errorMessage: "render failed for request",
|
|
errorName: "TypeError",
|
|
failedAt: expect.any(Date),
|
|
jobId: "42",
|
|
jobName: "send-transactional-email",
|
|
queueName: queueNames.email,
|
|
});
|
|
const persistedPayload = JSON.stringify(vi.mocked(recordFailedJob).mock.calls);
|
|
expect(persistedPayload).not.toContain("data");
|
|
expect(persistedPayload).not.toContain("headers");
|
|
expect(persistedPayload).not.toContain("body");
|
|
expect(persistedPayload).not.toContain("templateData");
|
|
expect(persistedPayload).not.toContain("recipient");
|
|
expect(persistedPayload).not.toContain("terminal-token");
|
|
expect(persistedPayload).not.toContain("template-secret-token");
|
|
expect(persistedPayload).not.toContain("customer@example.com");
|
|
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ requestId: "req_terminal" }), "Worker job failed.");
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
attemptsLimit: 3,
|
|
attemptsMade: 3,
|
|
component: "worker",
|
|
error: expect.objectContaining({ errorMessage: "render failed for request", errorName: "TypeError" }),
|
|
jobId: "42",
|
|
jobName: "send-transactional-email",
|
|
operation: "job.failed",
|
|
queueName: queueNames.email,
|
|
requestId: "req_terminal",
|
|
retryCount: 3,
|
|
severity: "error",
|
|
})
|
|
);
|
|
const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls);
|
|
expect(reporterPayload).not.toContain("secret-email-body");
|
|
expect(reporterPayload).not.toContain("terminal-token");
|
|
expect(reporterPayload).not.toContain("template-secret-token");
|
|
expect(reporterPayload).not.toContain("customer@example.com");
|
|
});
|
|
|
|
test("logs persistence rejections from terminal failures without throwing", async () => {
|
|
const logger = createLogger();
|
|
const reporter: ErrorReporter = { report: vi.fn() };
|
|
const persistenceError = new Error("database unavailable");
|
|
const recordFailedJob = vi.fn(async () => {
|
|
throw persistenceError;
|
|
});
|
|
const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter });
|
|
|
|
expect(() =>
|
|
worker.emitFailed(
|
|
{
|
|
attemptsMade: 1,
|
|
data: { metadata: { requestId: "req_persist" }, token: "persist-secret-token" },
|
|
id: "job-persist-error",
|
|
name: "deliver-webhook",
|
|
opts: { attempts: 1 },
|
|
queueName: queueNames.email,
|
|
},
|
|
new Error("terminal failure")
|
|
)
|
|
).not.toThrow();
|
|
await flushMicrotasks();
|
|
|
|
expect(logger.error).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
attemptsLimit: 1,
|
|
error: expect.objectContaining({ message: persistenceError.message }),
|
|
jobId: "job-persist-error",
|
|
requestId: "req_persist",
|
|
}),
|
|
"Failed to persist terminal worker failure."
|
|
);
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
attemptsLimit: 1,
|
|
attemptsMade: 1,
|
|
component: "worker",
|
|
error: expect.objectContaining({ errorMessage: "terminal failure", errorName: "Error" }),
|
|
jobId: "job-persist-error",
|
|
jobName: "deliver-webhook",
|
|
operation: "job.failed",
|
|
queueName: queueNames.email,
|
|
requestId: "req_persist",
|
|
retryCount: 1,
|
|
severity: "error",
|
|
})
|
|
);
|
|
expect(reporter.report).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
attemptsLimit: 1,
|
|
attemptsMade: 1,
|
|
component: "worker",
|
|
error: expect.objectContaining({ errorMessage: "database unavailable", errorName: "Error" }),
|
|
jobId: "job-persist-error",
|
|
jobName: "deliver-webhook",
|
|
operation: "job.persistence_failed",
|
|
queueName: queueNames.email,
|
|
requestId: "req_persist",
|
|
retryCount: 1,
|
|
severity: "error",
|
|
})
|
|
);
|
|
const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls);
|
|
expect(reporterPayload).not.toContain("persist-secret-token");
|
|
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("persist-secret-token");
|
|
});
|
|
});
|