Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,628 @@
|
||||
import { loadRequiredWorkerConfig, type EnvRecord } from "@toolctrl/config";
|
||||
import { recordQueueFailedJob, type QueueFailedJobRecordInput } from "@toolctrl/db/queue-failed-jobs.server";
|
||||
import { createNoopErrorReporter, reportError, type ErrorReporter, type ErrorReportSeverity } from "@toolctrl/error-reporter";
|
||||
import type { DefaultMetrics } from "@toolctrl/metrics";
|
||||
import { parseRedisUrl, queueNames, type QueueName, type TypedProcessor } from "@toolctrl/queue";
|
||||
import { createWorker } from "@toolctrl/queue/runtime";
|
||||
import { createRegisteredProcessor, type HandlerDependencies } from "./handlers.js";
|
||||
import { defaultWorkerQueueNames, getWorkerQueueNames } from "./queues.js";
|
||||
import { createWorkerStorageDeleteService } from "./storage.js";
|
||||
import { createWorkerScanService } from "./scanService.js";
|
||||
|
||||
/**
|
||||
* Subset of the shared metrics bundle the queue runtime actually writes to.
|
||||
* Keeping this narrow in the public signature lets tests inject a single
|
||||
* counter without instantiating the entire prom-client registry.
|
||||
*/
|
||||
export type WorkerRuntimeMetrics = Pick<
|
||||
DefaultMetrics,
|
||||
"queueJobsCompletedTotal" | "queueJobsFailedTotal" | "queueJobDurationSeconds" | "mailSentTotal" | "webhookDeliveryAttemptsTotal"
|
||||
>;
|
||||
|
||||
export const workerQueueNames = defaultWorkerQueueNames;
|
||||
export { getWorkerQueueNames };
|
||||
|
||||
type WorkerBackoffOptions = {
|
||||
delay?: number;
|
||||
type?: string;
|
||||
};
|
||||
|
||||
type WorkerJob = {
|
||||
attemptsMade?: number;
|
||||
data?: unknown;
|
||||
id?: string | number;
|
||||
name: string;
|
||||
opts?: {
|
||||
attempts?: number;
|
||||
backoff?: number | WorkerBackoffOptions;
|
||||
};
|
||||
queueName?: string;
|
||||
};
|
||||
|
||||
type WorkerLogBindings = {
|
||||
attemptsLimit?: number | null;
|
||||
attemptsMade?: number;
|
||||
backoffDelay?: number;
|
||||
backoffType?: string;
|
||||
jobId?: string | null;
|
||||
jobName?: string;
|
||||
queueName?: string;
|
||||
requestId?: string;
|
||||
};
|
||||
|
||||
export type WorkerRuntimeLogger = {
|
||||
child?: (bindings: Record<string, unknown>) => WorkerRuntimeLogger;
|
||||
error: (message?: unknown, ...optionalParams: unknown[]) => void;
|
||||
info?: (message?: unknown, ...optionalParams: unknown[]) => void;
|
||||
log?: (message?: unknown, ...optionalParams: unknown[]) => void;
|
||||
warn?: (message?: unknown, ...optionalParams: unknown[]) => void;
|
||||
};
|
||||
|
||||
export type WorkerRuntimeWorker = {
|
||||
close(): Promise<void>;
|
||||
name: string;
|
||||
on(event: "completed", listener: (job: WorkerJob) => void): WorkerRuntimeWorker;
|
||||
on(event: "failed", listener: (job: WorkerJob | undefined, error: Error) => void): WorkerRuntimeWorker;
|
||||
};
|
||||
|
||||
export type WorkerRuntimeStatus =
|
||||
| {
|
||||
configured: false;
|
||||
enabled: false;
|
||||
queueNames: readonly QueueName[];
|
||||
status: "disabled";
|
||||
workers: readonly [];
|
||||
}
|
||||
| {
|
||||
configured: true;
|
||||
enabled: true;
|
||||
queueNames: readonly QueueName[];
|
||||
status: "started";
|
||||
workers: readonly WorkerRuntimeWorker[];
|
||||
};
|
||||
|
||||
export type SentryStartSpanArgs = {
|
||||
op: string;
|
||||
name: string;
|
||||
attributes?: Record<string, string | number | boolean | undefined>;
|
||||
};
|
||||
|
||||
export type SentrySpan = {
|
||||
setAttribute(key: string, value: string | number | boolean | undefined): void;
|
||||
setStatus?(status: { code: number; message?: string }): void;
|
||||
};
|
||||
|
||||
export type SentryLikeForWorkerTracing = {
|
||||
startSpan: <T>(args: SentryStartSpanArgs, callback: (span: SentrySpan) => Promise<T> | T) => Promise<T> | T;
|
||||
captureException?: (error: unknown, captureContext?: Record<string, unknown>) => void;
|
||||
};
|
||||
|
||||
export type StartWorkerRuntimeOptions = {
|
||||
createProcessor?: <Name extends QueueName>(queueName: Name, dependencies: HandlerDependencies) => TypedProcessor<Name>;
|
||||
createWorkerImpl?: <Name extends QueueName>(
|
||||
queueName: Name,
|
||||
processor: TypedProcessor<Name>,
|
||||
options: Parameters<typeof createWorker<Name>>[2]
|
||||
) => WorkerRuntimeWorker;
|
||||
env?: EnvRecord;
|
||||
logger?: WorkerRuntimeLogger;
|
||||
queueNames?: readonly QueueName[];
|
||||
recordFailedJob?: (data: QueueFailedJobRecordInput) => Promise<void>;
|
||||
reporter?: ErrorReporter;
|
||||
/**
|
||||
* Optional Sentry-like module for queue.process span tracing. When omitted,
|
||||
* the runtime falls back to a pass-through processor. The production caller
|
||||
* lazy-loads `@sentry/node`; tests inject a deterministic fake.
|
||||
*/
|
||||
sentry?: SentryLikeForWorkerTracing;
|
||||
/**
|
||||
* Optional Prometheus metrics. When set, the runtime records
|
||||
* `queue_jobs_completed_total` on success or `queue_jobs_failed_total`
|
||||
* on throw, and observes `queue_job_duration_seconds`. Queue-depth gauges
|
||||
* such as `queue_jobs_active` are owned by the metrics scheduler.
|
||||
* Omitting this is a no-op so existing callers stay byte-identical.
|
||||
*/
|
||||
metrics?: WorkerRuntimeMetrics;
|
||||
/**
|
||||
* Optional time source for duration histograms. Defaults to
|
||||
* `performance.now()`; tests inject a deterministic clock.
|
||||
*/
|
||||
now?: () => number;
|
||||
/**
|
||||
* Optional extra HandlerDependencies merged into the runtime-built
|
||||
* dependency object. The worker bootstrap uses this to inject
|
||||
* Prisma-backed services that the runtime layer cannot construct on
|
||||
* its own.
|
||||
*/
|
||||
extraHandlerDependencies?: Partial<HandlerDependencies>;
|
||||
};
|
||||
|
||||
export type ShutdownWorkerRuntimeResult = {
|
||||
closedWorkers: readonly string[];
|
||||
};
|
||||
|
||||
export function startWorkerRuntime(options: StartWorkerRuntimeOptions = {}): WorkerRuntimeStatus {
|
||||
const env = options.env ?? (process.env as EnvRecord);
|
||||
const logger = options.logger ?? console;
|
||||
const queues = options.queueNames ?? getWorkerQueueNames(env);
|
||||
const config = loadRequiredWorkerConfig(env);
|
||||
|
||||
if (!config.enabled) {
|
||||
logInfo(logger, { queueNames: queues }, "Worker disabled; queues not started.");
|
||||
return {
|
||||
configured: false,
|
||||
enabled: false,
|
||||
queueNames: queues,
|
||||
status: "disabled",
|
||||
workers: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (!config.redis.url) {
|
||||
throw new Error("Worker Redis URL is required when worker runtime is enabled.");
|
||||
}
|
||||
const connection = parseRedisUrl(config.redis.url);
|
||||
const createWorkerImpl = options.createWorkerImpl ?? createWorker;
|
||||
const createProcessor = options.createProcessor ?? ((queueName, dependencies) => createRegisteredProcessor(queueName, { dependencies }));
|
||||
const recordFailedJob = options.recordFailedJob ?? recordQueueFailedJob;
|
||||
const reporter = options.reporter ?? createNoopErrorReporter();
|
||||
const sentry = options.sentry;
|
||||
const scanService = env.SCAN_QUEUE_ENABLED === "true" ? createWorkerScanService(env) : undefined;
|
||||
const metrics = options.metrics;
|
||||
const mailMetricsHook = metrics
|
||||
? ({ provider, status }: { provider: "mailjet"; status: "delivered" | "failed" }) => {
|
||||
try {
|
||||
metrics.mailSentTotal.labels({ provider, status }).inc();
|
||||
} catch {
|
||||
// Metric writes never block mail.
|
||||
}
|
||||
}
|
||||
: undefined;
|
||||
const webhookDeliveryMetricsHook = metrics
|
||||
? ({ status }: { status: "success" | "retry" | "dlq" | "rejected" }) => {
|
||||
try {
|
||||
metrics.webhookDeliveryAttemptsTotal.labels({ status }).inc();
|
||||
} catch {
|
||||
// Metric writes never block delivery.
|
||||
}
|
||||
}
|
||||
: undefined;
|
||||
const handlerDependencies: HandlerDependencies = {
|
||||
env,
|
||||
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? { storageDeleteService: createWorkerStorageDeleteService(env) } : {}),
|
||||
...(scanService ? { scanService } : {}),
|
||||
...(mailMetricsHook ? { mailMetricsHook } : {}),
|
||||
...(webhookDeliveryMetricsHook ? { webhookDeliveryMetricsHook } : {}),
|
||||
...(options.extraHandlerDependencies ?? {}),
|
||||
};
|
||||
assertQueueServicesWired(queues, handlerDependencies);
|
||||
const now = options.now ?? (() => performance.now());
|
||||
const workers = queues.map((queueName) => {
|
||||
const baseProcessor = createProcessor(queueName, handlerDependencies);
|
||||
const instrumented = metrics ? wrapProcessorWithMetrics(queueName, baseProcessor, metrics, now) : baseProcessor;
|
||||
const processor = createLoggedProcessor(queueName, instrumented, logger, reporter, sentry);
|
||||
const worker = createWorkerImpl(queueName, processor, { connection, concurrency: config.concurrency });
|
||||
|
||||
worker.on("completed", (job: WorkerJob) => {
|
||||
logInfo(createJobLogger(logger, queueName, job), {}, "Worker job completed.");
|
||||
});
|
||||
worker.on("failed", (job: WorkerJob | undefined, error: Error) => {
|
||||
const jobLogger = createJobLogger(logger, queueName, job);
|
||||
const redactedError = redactWorkerError(error);
|
||||
logError(jobLogger, { error: redactedError }, "Worker job failed.");
|
||||
reportWorkerError(reporter, "job.failed", "error", queueName, job, redactedError);
|
||||
void persistTerminalFailedJob({ queueName, job, error: redactedError, logger: jobLogger, recordFailedJob, reporter });
|
||||
});
|
||||
|
||||
return worker;
|
||||
});
|
||||
|
||||
logInfo(logger, { workerNames: workers.map((worker) => worker.name) }, "Worker started.");
|
||||
return {
|
||||
configured: true,
|
||||
enabled: true,
|
||||
queueNames: queues,
|
||||
status: "started",
|
||||
workers,
|
||||
};
|
||||
}
|
||||
|
||||
function assertQueueServicesWired(queues: readonly QueueName[], dependencies: HandlerDependencies): void {
|
||||
const ownsImports = queues.includes(queueNames.imports);
|
||||
const ownsExports = queues.includes(queueNames.exports);
|
||||
const missing: string[] = [];
|
||||
|
||||
if (ownsImports) {
|
||||
if (!dependencies.importService) missing.push("importService");
|
||||
if (!dependencies.scanService) missing.push("scanService");
|
||||
}
|
||||
if (ownsExports && !dependencies.exportService) {
|
||||
missing.push("exportService");
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`Worker import/export queues require wired production services: missing ${missing.join(", ")}.`);
|
||||
}
|
||||
}
|
||||
|
||||
function createJobLogger(logger: WorkerRuntimeLogger, fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerRuntimeLogger {
|
||||
const bindings = buildWorkerJobBindings(fallbackQueueName, job);
|
||||
return typeof logger.child === "function" ? logger.child(bindings) : createBoundFallbackLogger(logger, bindings);
|
||||
}
|
||||
|
||||
function createBoundFallbackLogger(logger: WorkerRuntimeLogger, bindings: WorkerLogBindings): WorkerRuntimeLogger {
|
||||
return {
|
||||
child: (childBindings) => createBoundFallbackLogger(logger, { ...bindings, ...childBindings }),
|
||||
error: (record?: unknown, ...optionalParams: unknown[]) => logger.error(mergeLogRecord(bindings, record), ...optionalParams),
|
||||
info: (record?: unknown, ...optionalParams: unknown[]) => {
|
||||
if (logger.info) {
|
||||
logger.info(mergeLogRecord(bindings, record), ...optionalParams);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.log?.(mergeLogRecord(bindings, record), ...optionalParams);
|
||||
},
|
||||
log: (record?: unknown, ...optionalParams: unknown[]) => logger.log?.(mergeLogRecord(bindings, record), ...optionalParams),
|
||||
warn: (record?: unknown, ...optionalParams: unknown[]) => {
|
||||
if (logger.warn) {
|
||||
logger.warn(mergeLogRecord(bindings, record), ...optionalParams);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.log?.(mergeLogRecord(bindings, record), ...optionalParams);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildWorkerJobBindings(fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerLogBindings {
|
||||
const backoff = parseWorkerBackoff(job?.opts?.backoff);
|
||||
|
||||
return {
|
||||
...extractRequestIdBinding(job),
|
||||
queueName: job?.queueName ?? fallbackQueueName,
|
||||
jobName: job?.name ?? "unknown",
|
||||
jobId: job?.id === undefined ? null : String(job.id),
|
||||
attemptsMade: job?.attemptsMade ?? 0,
|
||||
attemptsLimit: job?.opts?.attempts ?? null,
|
||||
...backoff,
|
||||
};
|
||||
}
|
||||
|
||||
function extractRequestIdBinding(job: WorkerJob | undefined): Pick<WorkerLogBindings, "requestId"> {
|
||||
if (!job || !isRecord(job.data)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const metadata = job.data.metadata;
|
||||
if (!isRecord(metadata) || typeof metadata.requestId !== "string") {
|
||||
return {};
|
||||
}
|
||||
|
||||
return { requestId: metadata.requestId };
|
||||
}
|
||||
|
||||
async function persistTerminalFailedJob({
|
||||
queueName,
|
||||
job,
|
||||
error,
|
||||
logger,
|
||||
recordFailedJob,
|
||||
reporter,
|
||||
}: {
|
||||
queueName: QueueName;
|
||||
job: WorkerJob | undefined;
|
||||
error: Error;
|
||||
logger: WorkerRuntimeLogger;
|
||||
recordFailedJob: (data: QueueFailedJobRecordInput) => Promise<void>;
|
||||
reporter: ErrorReporter;
|
||||
}): Promise<void> {
|
||||
if (!isTerminalFailure(job)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await recordFailedJob({
|
||||
queueName: job?.queueName ?? queueName,
|
||||
jobName: job?.name ?? "unknown",
|
||||
jobId: job?.id === undefined ? null : String(job.id),
|
||||
attemptsMade: job?.attemptsMade ?? 0,
|
||||
attemptsLimit: job?.opts?.attempts ?? null,
|
||||
errorName: error.name,
|
||||
errorMessage: error.message,
|
||||
failedAt: new Date(),
|
||||
});
|
||||
} catch (persistenceError) {
|
||||
const redactedPersistenceError = redactWorkerError(persistenceError);
|
||||
logError(logger, { error: redactedPersistenceError }, "Failed to persist terminal worker failure.");
|
||||
reportWorkerError(reporter, "job.persistence_failed", "error", queueName, job, redactedPersistenceError);
|
||||
}
|
||||
}
|
||||
|
||||
function isTerminalFailure(job: WorkerJob | undefined): boolean {
|
||||
const attemptsLimit = job?.opts?.attempts;
|
||||
if (attemptsLimit === undefined) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return (job?.attemptsMade ?? 0) >= attemptsLimit;
|
||||
}
|
||||
|
||||
function parseWorkerBackoff(backoff: NonNullable<WorkerJob["opts"]>["backoff"]): Pick<WorkerLogBindings, "backoffDelay" | "backoffType"> {
|
||||
if (typeof backoff === "number") {
|
||||
return { backoffDelay: backoff, backoffType: "fixed" };
|
||||
}
|
||||
|
||||
if (backoff && typeof backoff === "object") {
|
||||
return {
|
||||
...(typeof backoff.delay === "number" ? { backoffDelay: backoff.delay } : {}),
|
||||
...(typeof backoff.type === "string" ? { backoffType: backoff.type } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
export async function shutdownWorkerRuntime(
|
||||
runtime: Pick<WorkerRuntimeStatus, "workers">,
|
||||
options: { logger?: WorkerRuntimeLogger; reason?: string; reporter?: ErrorReporter } = {}
|
||||
): Promise<ShutdownWorkerRuntimeResult> {
|
||||
const logger = options.logger ?? console;
|
||||
const reason = options.reason ?? "shutdown";
|
||||
const reporter = options.reporter ?? createNoopErrorReporter();
|
||||
const closeResults = await Promise.allSettled(runtime.workers.map((worker) => worker.close()));
|
||||
const failedCloses = closeResults.flatMap((result, index) => {
|
||||
if (result.status === "fulfilled") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const worker = runtime.workers[index];
|
||||
const workerName = worker?.name ?? "unknown";
|
||||
const redactedError = redactWorkerError(result.reason);
|
||||
logError(logger, { error: redactedError, reason, workerName }, "Failed to close worker.");
|
||||
reportError(reporter, {
|
||||
component: "worker",
|
||||
operation: "shutdown.close_failed",
|
||||
severity: "error",
|
||||
queueName: workerName,
|
||||
workerName,
|
||||
reason,
|
||||
error: redactedError,
|
||||
});
|
||||
return [workerName];
|
||||
});
|
||||
|
||||
if (failedCloses.length > 0) {
|
||||
throw new Error(`Failed to close workers: ${failedCloses.join(", ")}.`);
|
||||
}
|
||||
|
||||
const closedWorkers = runtime.workers.map((worker) => worker.name);
|
||||
logInfo(logger, { closedWorkers, reason }, "Worker shutdown complete.");
|
||||
return { closedWorkers };
|
||||
}
|
||||
|
||||
function createLoggedProcessor<Name extends QueueName>(
|
||||
queueName: Name,
|
||||
processor: TypedProcessor<Name>,
|
||||
logger: WorkerRuntimeLogger,
|
||||
reporter: ErrorReporter,
|
||||
sentry: SentryLikeForWorkerTracing | undefined
|
||||
): TypedProcessor<Name> {
|
||||
const tracedCore: TypedProcessor<Name> = async (job) => {
|
||||
try {
|
||||
return await processor(job);
|
||||
} catch (error) {
|
||||
const redactedError = redactWorkerError(error);
|
||||
logError(createJobLogger(logger, queueName, job), { error: redactedError }, "Worker handler failed.");
|
||||
reportWorkerError(reporter, "handler.failed", "error", queueName, job, redactedError);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
if (!sentry) {
|
||||
return tracedCore;
|
||||
}
|
||||
|
||||
return wrapProcessorWithSentryTracing(queueName, tracedCore, sentry);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps the BullMQ processor in a `Sentry.startSpan({ op: 'queue.process' })`
|
||||
* span tagged with queue + job + tenant context. Sentry's @sentry/node v8/v10
|
||||
* has no BullMQ auto-instrumentation, so we do this manually.
|
||||
*/
|
||||
function wrapProcessorWithSentryTracing<Name extends QueueName>(
|
||||
queueName: Name,
|
||||
inner: TypedProcessor<Name>,
|
||||
sentry: SentryLikeForWorkerTracing
|
||||
): TypedProcessor<Name> {
|
||||
return async (job) => {
|
||||
const jobShape = (job ?? {}) as Partial<WorkerJob> & { data?: unknown };
|
||||
const jobName = jobShape.name ?? "unknown";
|
||||
const jobId = jobShape.id !== undefined && jobShape.id !== null ? String(jobShape.id) : undefined;
|
||||
const attemptsMade = jobShape.attemptsMade ?? 0;
|
||||
const attemptsLimit = jobShape.opts?.attempts;
|
||||
const tenantId = extractTenantId(jobShape.data);
|
||||
|
||||
return sentry.startSpan(
|
||||
{
|
||||
op: "queue.process",
|
||||
name: `${queueName}/${jobName}`,
|
||||
attributes: {
|
||||
"messaging.system": "bullmq",
|
||||
"messaging.destination.name": queueName,
|
||||
"messaging.operation": "process",
|
||||
"messaging.message.id": jobId,
|
||||
"messaging.bullmq.job.name": jobName,
|
||||
"messaging.bullmq.attempts_made": attemptsMade,
|
||||
...(typeof attemptsLimit === "number" ? { "messaging.bullmq.attempts_limit": attemptsLimit } : {}),
|
||||
...(tenantId ? { "toolctrl.tenant_id": tenantId } : {}),
|
||||
},
|
||||
},
|
||||
async (span) => {
|
||||
try {
|
||||
const result = await inner(job);
|
||||
span.setStatus?.({ code: 1, message: "ok" });
|
||||
return result;
|
||||
} catch (error) {
|
||||
span.setStatus?.({ code: 2, message: "internal_error" });
|
||||
if (typeof sentry.captureException === "function") {
|
||||
try {
|
||||
sentry.captureException(error, {
|
||||
tags: {
|
||||
component: "worker",
|
||||
queue: queueName,
|
||||
job_name: jobName,
|
||||
...(jobId ? { job_id: jobId } : {}),
|
||||
...(tenantId ? { tenant_id: tenantId } : {}),
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// Sentry transport failures must never mask the runtime failure.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a BullMQ processor in a Prometheus-instrumented closure.
|
||||
*
|
||||
* Side effects per invocation:
|
||||
* - `queue_jobs_completed_total{queue,job_name}` +1 on success.
|
||||
* - `queue_jobs_failed_total{queue,job_name,error_type}` +1 on throw.
|
||||
* `error_type` is the redacted `error.name` or `"Error"` to keep
|
||||
* cardinality bounded (no message strings).
|
||||
* - `queue_job_duration_seconds{queue,job_name}` observation in seconds.
|
||||
*
|
||||
* Metric writes are wrapped in try/catch so a registry failure can never
|
||||
* mask a queue handler error.
|
||||
*/
|
||||
function wrapProcessorWithMetrics<Name extends QueueName>(
|
||||
queueName: Name,
|
||||
inner: TypedProcessor<Name>,
|
||||
metrics: WorkerRuntimeMetrics,
|
||||
now: () => number
|
||||
): TypedProcessor<Name> {
|
||||
return async (job) => {
|
||||
const jobShape = (job ?? {}) as Partial<WorkerJob>;
|
||||
const jobName = jobShape.name ?? "unknown";
|
||||
const started = now();
|
||||
let failed = false;
|
||||
let errorType = "Error";
|
||||
try {
|
||||
const result = await inner(job);
|
||||
try {
|
||||
metrics.queueJobsCompletedTotal.labels({ queue: queueName, job_name: jobName }).inc();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
failed = true;
|
||||
errorType = pickErrorType(error);
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
metrics.queueJobDurationSeconds.labels({ queue: queueName, job_name: jobName }).observe((now() - started) / 1000);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (failed) {
|
||||
try {
|
||||
metrics.queueJobsFailedTotal.labels({ queue: queueName, job_name: jobName, error_type: errorType }).inc();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function pickErrorType(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
const name = error.name.trim();
|
||||
return name.length > 0 ? name : "Error";
|
||||
}
|
||||
return "Error";
|
||||
}
|
||||
|
||||
function extractTenantId(data: unknown): string | undefined {
|
||||
if (!isRecord(data)) return undefined;
|
||||
const direct = data.tenantId;
|
||||
if (typeof direct === "string" && direct) return direct;
|
||||
const metadata = data.metadata;
|
||||
if (isRecord(metadata) && typeof metadata.tenantId === "string" && metadata.tenantId) {
|
||||
return metadata.tenantId;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function reportWorkerError(
|
||||
reporter: ErrorReporter,
|
||||
operation: string,
|
||||
severity: ErrorReportSeverity,
|
||||
fallbackQueueName: QueueName,
|
||||
job: WorkerJob | undefined,
|
||||
error: unknown
|
||||
): void {
|
||||
const bindings = buildWorkerJobBindings(fallbackQueueName, job);
|
||||
reportError(reporter, {
|
||||
component: "worker",
|
||||
operation,
|
||||
severity,
|
||||
queueName: bindings.queueName,
|
||||
jobName: bindings.jobName,
|
||||
jobId: bindings.jobId ?? undefined,
|
||||
requestId: bindings.requestId,
|
||||
attemptsMade: bindings.attemptsMade,
|
||||
attemptsLimit: bindings.attemptsLimit,
|
||||
retryCount: bindings.attemptsMade,
|
||||
backoffDelay: bindings.backoffDelay,
|
||||
backoffType: bindings.backoffType,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
function logInfo(logger: WorkerRuntimeLogger, record: Record<string, unknown>, message: string): void {
|
||||
if (logger.info) {
|
||||
logger.info(record, message);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.log?.(record, message);
|
||||
}
|
||||
|
||||
function logError(logger: WorkerRuntimeLogger, record: Record<string, unknown>, message: string): void {
|
||||
logger.error(record, message);
|
||||
}
|
||||
|
||||
function mergeLogRecord(bindings: WorkerLogBindings, record: unknown): Record<string, unknown> {
|
||||
if (isRecord(record)) {
|
||||
return { ...bindings, ...record };
|
||||
}
|
||||
|
||||
return bindings;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function redactWorkerError(error: unknown): Error {
|
||||
const source = error instanceof Error ? error : new Error(String(error));
|
||||
const redacted = new Error(redactWorkerDiagnosticText(source.message || source.name));
|
||||
redacted.name = source.name || "Error";
|
||||
return redacted;
|
||||
}
|
||||
|
||||
function redactWorkerDiagnosticText(value: string): string {
|
||||
return value
|
||||
.replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]")
|
||||
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
||||
.replace(/(authorization|access[_-]?key(?:id)?|api[-_]?key|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
|
||||
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]")
|
||||
.slice(0, 1_000);
|
||||
}
|
||||
Reference in new issue
Block a user