Track HiveGrid Cloud planning reference
This commit is contained in:
david committed
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,278 @@
|
||||
import type { ErrorReporter } from "@toolctrl/error-reporter";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { queueNames, type QueueName, type TypedProcessor } from "@toolctrl/queue";
|
||||
import { startWorkerRuntime, type StartWorkerRuntimeOptions, type WorkerRuntimeWorker } from "../src/runtime.js";
|
||||
|
||||
type MockWorkerJob = {
|
||||
attemptsMade?: number;
|
||||
data?: unknown;
|
||||
id?: string | number;
|
||||
name: string;
|
||||
opts?: { attempts?: number; backoff?: number | { delay?: number; type?: string } };
|
||||
queueName?: string;
|
||||
};
|
||||
|
||||
type MockWorker = WorkerRuntimeWorker & {
|
||||
emitFailed(job: MockWorkerJob | undefined, error: Error): void;
|
||||
};
|
||||
|
||||
type TestCreateWorker = NonNullable<StartWorkerRuntimeOptions["createWorkerImpl"]>;
|
||||
|
||||
function createLogger() {
|
||||
return { error: vi.fn(), info: vi.fn(), log: vi.fn() };
|
||||
}
|
||||
|
||||
function createMockWorker(name: QueueName): MockWorker {
|
||||
let failedListener: ((job: MockWorkerJob | undefined, error: Error) => void) | undefined;
|
||||
const worker: MockWorker = {
|
||||
close: vi.fn(async () => undefined),
|
||||
name,
|
||||
on: vi.fn((event: "completed" | "failed", listener: (job: MockWorkerJob | undefined, error: Error) => void) => {
|
||||
if (event === "failed") failedListener = listener;
|
||||
return worker;
|
||||
}) as WorkerRuntimeWorker["on"],
|
||||
emitFailed: (job, error) => failedListener?.(job, error),
|
||||
};
|
||||
return worker;
|
||||
}
|
||||
|
||||
function createStartedRuntime(options: {
|
||||
logger: ReturnType<typeof createLogger>;
|
||||
recordFailedJob?: StartWorkerRuntimeOptions["recordFailedJob"];
|
||||
reporter?: ErrorReporter;
|
||||
}) {
|
||||
const workers: MockWorker[] = [];
|
||||
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => {
|
||||
const worker = createMockWorker(queueName);
|
||||
workers.push(worker);
|
||||
return worker;
|
||||
}) as TestCreateWorker;
|
||||
|
||||
startWorkerRuntime({
|
||||
createProcessor: (queueName) => vi.fn(async () => ({ queueName })) as TypedProcessor<typeof queueName>,
|
||||
createWorkerImpl,
|
||||
env: {
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
DATABASE_URL: "postgresql://user:pass@localhost:5432/toolctrl",
|
||||
WORKER_ENABLED: "true",
|
||||
WORKER_QUEUE_GROUP: "mail",
|
||||
},
|
||||
logger: options.logger,
|
||||
queueNames: [queueNames.email],
|
||||
recordFailedJob: options.recordFailedJob,
|
||||
reporter: options.reporter,
|
||||
});
|
||||
|
||||
return { worker: workers[0] };
|
||||
}
|
||||
|
||||
async function flushMicrotasks() {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
describe("worker forced-failure diagnostics", () => {
|
||||
test("logs retry diagnostics for non-terminal failures without persisting payload data", async () => {
|
||||
const logger = createLogger();
|
||||
const reporter: ErrorReporter = { report: vi.fn() };
|
||||
const recordFailedJob = vi.fn(async () => undefined);
|
||||
const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter });
|
||||
const failure = new Error("upstream timed out");
|
||||
|
||||
worker.emitFailed(
|
||||
{
|
||||
attemptsMade: 2,
|
||||
data: {
|
||||
body: "secret-webhook-body",
|
||||
headers: { authorization: "Bearer non-terminal-token" },
|
||||
metadata: { requestId: "req_non_terminal" },
|
||||
recipient: "private@example.com",
|
||||
templateData: { code: "123456" },
|
||||
},
|
||||
id: "job-retry-1",
|
||||
name: "deliver-webhook",
|
||||
opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } },
|
||||
queueName: queueNames.email,
|
||||
},
|
||||
failure
|
||||
);
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
attemptsLimit: 5,
|
||||
attemptsMade: 2,
|
||||
backoffDelay: 2000,
|
||||
backoffType: "exponential",
|
||||
error: expect.objectContaining({ message: failure.message }),
|
||||
jobId: "job-retry-1",
|
||||
jobName: "deliver-webhook",
|
||||
queueName: queueNames.email,
|
||||
requestId: "req_non_terminal",
|
||||
}),
|
||||
"Worker job failed."
|
||||
);
|
||||
expect(recordFailedJob).not.toHaveBeenCalled();
|
||||
expect(reporter.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
attemptsLimit: 5,
|
||||
attemptsMade: 2,
|
||||
backoffDelay: 2000,
|
||||
backoffType: "exponential",
|
||||
component: "worker",
|
||||
error: expect.objectContaining({ errorMessage: "upstream timed out", errorName: "Error" }),
|
||||
jobId: "job-retry-1",
|
||||
jobName: "deliver-webhook",
|
||||
operation: "job.failed",
|
||||
queueName: queueNames.email,
|
||||
requestId: "req_non_terminal",
|
||||
retryCount: 2,
|
||||
severity: "error",
|
||||
})
|
||||
);
|
||||
const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls);
|
||||
expect(reporterPayload).not.toContain("secret-webhook-body");
|
||||
expect(reporterPayload).not.toContain("non-terminal-token");
|
||||
expect(reporterPayload).not.toContain("private@example.com");
|
||||
expect(reporterPayload).not.toContain("templateData");
|
||||
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("secret-webhook-body");
|
||||
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("non-terminal-token");
|
||||
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("private@example.com");
|
||||
});
|
||||
|
||||
test("persists terminal failures with redacted scalar diagnostics only", async () => {
|
||||
const logger = createLogger();
|
||||
const reporter: ErrorReporter = { report: vi.fn() };
|
||||
const recordFailedJob = vi.fn(async () => undefined);
|
||||
const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter });
|
||||
const failure = new TypeError("render failed for request");
|
||||
|
||||
worker.emitFailed(
|
||||
{
|
||||
attemptsMade: 3,
|
||||
data: {
|
||||
body: "secret-email-body",
|
||||
headers: { authorization: "Bearer terminal-token" },
|
||||
metadata: { requestId: "req_terminal" },
|
||||
recipient: "customer@example.com",
|
||||
templateData: { resetToken: "template-secret-token" },
|
||||
},
|
||||
id: 42,
|
||||
name: "send-transactional-email",
|
||||
opts: { attempts: 3 },
|
||||
queueName: queueNames.email,
|
||||
},
|
||||
failure
|
||||
);
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(recordFailedJob).toHaveBeenCalledTimes(1);
|
||||
expect(recordFailedJob).toHaveBeenCalledWith({
|
||||
attemptsLimit: 3,
|
||||
attemptsMade: 3,
|
||||
errorMessage: "render failed for request",
|
||||
errorName: "TypeError",
|
||||
failedAt: expect.any(Date),
|
||||
jobId: "42",
|
||||
jobName: "send-transactional-email",
|
||||
queueName: queueNames.email,
|
||||
});
|
||||
const persistedPayload = JSON.stringify(vi.mocked(recordFailedJob).mock.calls);
|
||||
expect(persistedPayload).not.toContain("data");
|
||||
expect(persistedPayload).not.toContain("headers");
|
||||
expect(persistedPayload).not.toContain("body");
|
||||
expect(persistedPayload).not.toContain("templateData");
|
||||
expect(persistedPayload).not.toContain("recipient");
|
||||
expect(persistedPayload).not.toContain("terminal-token");
|
||||
expect(persistedPayload).not.toContain("template-secret-token");
|
||||
expect(persistedPayload).not.toContain("customer@example.com");
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ requestId: "req_terminal" }), "Worker job failed.");
|
||||
expect(reporter.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
attemptsLimit: 3,
|
||||
attemptsMade: 3,
|
||||
component: "worker",
|
||||
error: expect.objectContaining({ errorMessage: "render failed for request", errorName: "TypeError" }),
|
||||
jobId: "42",
|
||||
jobName: "send-transactional-email",
|
||||
operation: "job.failed",
|
||||
queueName: queueNames.email,
|
||||
requestId: "req_terminal",
|
||||
retryCount: 3,
|
||||
severity: "error",
|
||||
})
|
||||
);
|
||||
const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls);
|
||||
expect(reporterPayload).not.toContain("secret-email-body");
|
||||
expect(reporterPayload).not.toContain("terminal-token");
|
||||
expect(reporterPayload).not.toContain("template-secret-token");
|
||||
expect(reporterPayload).not.toContain("customer@example.com");
|
||||
});
|
||||
|
||||
test("logs persistence rejections from terminal failures without throwing", async () => {
|
||||
const logger = createLogger();
|
||||
const reporter: ErrorReporter = { report: vi.fn() };
|
||||
const persistenceError = new Error("database unavailable");
|
||||
const recordFailedJob = vi.fn(async () => {
|
||||
throw persistenceError;
|
||||
});
|
||||
const { worker } = createStartedRuntime({ logger, recordFailedJob, reporter });
|
||||
|
||||
expect(() =>
|
||||
worker.emitFailed(
|
||||
{
|
||||
attemptsMade: 1,
|
||||
data: { metadata: { requestId: "req_persist" }, token: "persist-secret-token" },
|
||||
id: "job-persist-error",
|
||||
name: "deliver-webhook",
|
||||
opts: { attempts: 1 },
|
||||
queueName: queueNames.email,
|
||||
},
|
||||
new Error("terminal failure")
|
||||
)
|
||||
).not.toThrow();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
attemptsLimit: 1,
|
||||
error: expect.objectContaining({ message: persistenceError.message }),
|
||||
jobId: "job-persist-error",
|
||||
requestId: "req_persist",
|
||||
}),
|
||||
"Failed to persist terminal worker failure."
|
||||
);
|
||||
expect(reporter.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
attemptsLimit: 1,
|
||||
attemptsMade: 1,
|
||||
component: "worker",
|
||||
error: expect.objectContaining({ errorMessage: "terminal failure", errorName: "Error" }),
|
||||
jobId: "job-persist-error",
|
||||
jobName: "deliver-webhook",
|
||||
operation: "job.failed",
|
||||
queueName: queueNames.email,
|
||||
requestId: "req_persist",
|
||||
retryCount: 1,
|
||||
severity: "error",
|
||||
})
|
||||
);
|
||||
expect(reporter.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
attemptsLimit: 1,
|
||||
attemptsMade: 1,
|
||||
component: "worker",
|
||||
error: expect.objectContaining({ errorMessage: "database unavailable", errorName: "Error" }),
|
||||
jobId: "job-persist-error",
|
||||
jobName: "deliver-webhook",
|
||||
operation: "job.persistence_failed",
|
||||
queueName: queueNames.email,
|
||||
requestId: "req_persist",
|
||||
retryCount: 1,
|
||||
severity: "error",
|
||||
})
|
||||
);
|
||||
const reporterPayload = JSON.stringify(vi.mocked(reporter.report).mock.calls);
|
||||
expect(reporterPayload).not.toContain("persist-secret-token");
|
||||
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("persist-secret-token");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user