629 lines
22 KiB
TypeScript
629 lines
22 KiB
TypeScript
import { loadRequiredWorkerConfig, type EnvRecord } from "@hivegrid/config";
|
|
import { recordQueueFailedJob, type QueueFailedJobRecordInput } from "@hivegrid/db/queue-failed-jobs.server";
|
|
import { createNoopErrorReporter, reportError, type ErrorReporter, type ErrorReportSeverity } from "@hivegrid/error-reporter";
|
|
import type { DefaultMetrics } from "@hivegrid/metrics";
|
|
import { parseRedisUrl, queueNames, type QueueName, type TypedProcessor } from "@hivegrid/queue";
|
|
import { createWorker } from "@hivegrid/queue/runtime";
|
|
import { createRegisteredProcessor, type HandlerDependencies } from "./handlers.js";
|
|
import { defaultWorkerQueueNames, getWorkerQueueNames } from "./queues.js";
|
|
import { createWorkerStorageDeleteService } from "./storage.js";
|
|
import { createWorkerScanService } from "./scanService.js";
|
|
|
|
/**
|
|
* Subset of the shared metrics bundle the queue runtime actually writes to.
|
|
* Keeping this narrow in the public signature lets tests inject a single
|
|
* counter without instantiating the entire prom-client registry.
|
|
*/
|
|
export type WorkerRuntimeMetrics = Pick<
|
|
DefaultMetrics,
|
|
"queueJobsCompletedTotal" | "queueJobsFailedTotal" | "queueJobDurationSeconds" | "mailSentTotal" | "webhookDeliveryAttemptsTotal"
|
|
>;
|
|
|
|
export const workerQueueNames = defaultWorkerQueueNames;
|
|
export { getWorkerQueueNames };
|
|
|
|
type WorkerBackoffOptions = {
|
|
delay?: number;
|
|
type?: string;
|
|
};
|
|
|
|
type WorkerJob = {
|
|
attemptsMade?: number;
|
|
data?: unknown;
|
|
id?: string | number;
|
|
name: string;
|
|
opts?: {
|
|
attempts?: number;
|
|
backoff?: number | WorkerBackoffOptions;
|
|
};
|
|
queueName?: string;
|
|
};
|
|
|
|
type WorkerLogBindings = {
|
|
attemptsLimit?: number | null;
|
|
attemptsMade?: number;
|
|
backoffDelay?: number;
|
|
backoffType?: string;
|
|
jobId?: string | null;
|
|
jobName?: string;
|
|
queueName?: string;
|
|
requestId?: string;
|
|
};
|
|
|
|
export type WorkerRuntimeLogger = {
|
|
child?: (bindings: Record<string, unknown>) => WorkerRuntimeLogger;
|
|
error: (message?: unknown, ...optionalParams: unknown[]) => void;
|
|
info?: (message?: unknown, ...optionalParams: unknown[]) => void;
|
|
log?: (message?: unknown, ...optionalParams: unknown[]) => void;
|
|
warn?: (message?: unknown, ...optionalParams: unknown[]) => void;
|
|
};
|
|
|
|
export type WorkerRuntimeWorker = {
|
|
close(): Promise<void>;
|
|
name: string;
|
|
on(event: "completed", listener: (job: WorkerJob) => void): WorkerRuntimeWorker;
|
|
on(event: "failed", listener: (job: WorkerJob | undefined, error: Error) => void): WorkerRuntimeWorker;
|
|
};
|
|
|
|
export type WorkerRuntimeStatus =
|
|
| {
|
|
configured: false;
|
|
enabled: false;
|
|
queueNames: readonly QueueName[];
|
|
status: "disabled";
|
|
workers: readonly [];
|
|
}
|
|
| {
|
|
configured: true;
|
|
enabled: true;
|
|
queueNames: readonly QueueName[];
|
|
status: "started";
|
|
workers: readonly WorkerRuntimeWorker[];
|
|
};
|
|
|
|
export type SentryStartSpanArgs = {
|
|
op: string;
|
|
name: string;
|
|
attributes?: Record<string, string | number | boolean | undefined>;
|
|
};
|
|
|
|
export type SentrySpan = {
|
|
setAttribute(key: string, value: string | number | boolean | undefined): void;
|
|
setStatus?(status: { code: number; message?: string }): void;
|
|
};
|
|
|
|
export type SentryLikeForWorkerTracing = {
|
|
startSpan: <T>(args: SentryStartSpanArgs, callback: (span: SentrySpan) => Promise<T> | T) => Promise<T> | T;
|
|
captureException?: (error: unknown, captureContext?: Record<string, unknown>) => void;
|
|
};
|
|
|
|
export type StartWorkerRuntimeOptions = {
|
|
createProcessor?: <Name extends QueueName>(queueName: Name, dependencies: HandlerDependencies) => TypedProcessor<Name>;
|
|
createWorkerImpl?: <Name extends QueueName>(
|
|
queueName: Name,
|
|
processor: TypedProcessor<Name>,
|
|
options: Parameters<typeof createWorker<Name>>[2]
|
|
) => WorkerRuntimeWorker;
|
|
env?: EnvRecord;
|
|
logger?: WorkerRuntimeLogger;
|
|
queueNames?: readonly QueueName[];
|
|
recordFailedJob?: (data: QueueFailedJobRecordInput) => Promise<void>;
|
|
reporter?: ErrorReporter;
|
|
/**
|
|
* Optional Sentry-like module for queue.process span tracing. When omitted,
|
|
* the runtime falls back to a pass-through processor. The production caller
|
|
* lazy-loads `@sentry/node`; tests inject a deterministic fake.
|
|
*/
|
|
sentry?: SentryLikeForWorkerTracing;
|
|
/**
|
|
* Optional Prometheus metrics. When set, the runtime records
|
|
* `queue_jobs_completed_total` on success or `queue_jobs_failed_total`
|
|
* on throw, and observes `queue_job_duration_seconds`. Queue-depth gauges
|
|
* such as `queue_jobs_active` are owned by the metrics scheduler.
|
|
* Omitting this is a no-op so existing callers stay byte-identical.
|
|
*/
|
|
metrics?: WorkerRuntimeMetrics;
|
|
/**
|
|
* Optional time source for duration histograms. Defaults to
|
|
* `performance.now()`; tests inject a deterministic clock.
|
|
*/
|
|
now?: () => number;
|
|
/**
|
|
* Optional extra HandlerDependencies merged into the runtime-built
|
|
* dependency object. The worker bootstrap uses this to inject
|
|
* Prisma-backed services that the runtime layer cannot construct on
|
|
* its own.
|
|
*/
|
|
extraHandlerDependencies?: Partial<HandlerDependencies>;
|
|
};
|
|
|
|
export type ShutdownWorkerRuntimeResult = {
|
|
closedWorkers: readonly string[];
|
|
};
|
|
|
|
export function startWorkerRuntime(options: StartWorkerRuntimeOptions = {}): WorkerRuntimeStatus {
|
|
const env = options.env ?? (process.env as EnvRecord);
|
|
const logger = options.logger ?? console;
|
|
const queues = options.queueNames ?? getWorkerQueueNames(env);
|
|
const config = loadRequiredWorkerConfig(env);
|
|
|
|
if (!config.enabled) {
|
|
logInfo(logger, { queueNames: queues }, "Worker disabled; queues not started.");
|
|
return {
|
|
configured: false,
|
|
enabled: false,
|
|
queueNames: queues,
|
|
status: "disabled",
|
|
workers: [],
|
|
};
|
|
}
|
|
|
|
if (!config.redis.url) {
|
|
throw new Error("Worker Redis URL is required when worker runtime is enabled.");
|
|
}
|
|
const connection = parseRedisUrl(config.redis.url);
|
|
const createWorkerImpl = options.createWorkerImpl ?? createWorker;
|
|
const createProcessor = options.createProcessor ?? ((queueName, dependencies) => createRegisteredProcessor(queueName, { dependencies }));
|
|
const recordFailedJob = options.recordFailedJob ?? recordQueueFailedJob;
|
|
const reporter = options.reporter ?? createNoopErrorReporter();
|
|
const sentry = options.sentry;
|
|
const scanService = env.SCAN_QUEUE_ENABLED === "true" ? createWorkerScanService(env) : undefined;
|
|
const metrics = options.metrics;
|
|
const mailMetricsHook = metrics
|
|
? ({ provider, status }: { provider: "mailjet"; status: "delivered" | "failed" }) => {
|
|
try {
|
|
metrics.mailSentTotal.labels({ provider, status }).inc();
|
|
} catch {
|
|
// Metric writes never block mail.
|
|
}
|
|
}
|
|
: undefined;
|
|
const webhookDeliveryMetricsHook = metrics
|
|
? ({ status }: { status: "success" | "retry" | "dlq" | "rejected" }) => {
|
|
try {
|
|
metrics.webhookDeliveryAttemptsTotal.labels({ status }).inc();
|
|
} catch {
|
|
// Metric writes never block delivery.
|
|
}
|
|
}
|
|
: undefined;
|
|
const handlerDependencies: HandlerDependencies = {
|
|
env,
|
|
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? { storageDeleteService: createWorkerStorageDeleteService(env) } : {}),
|
|
...(scanService ? { scanService } : {}),
|
|
...(mailMetricsHook ? { mailMetricsHook } : {}),
|
|
...(webhookDeliveryMetricsHook ? { webhookDeliveryMetricsHook } : {}),
|
|
...(options.extraHandlerDependencies ?? {}),
|
|
};
|
|
assertQueueServicesWired(queues, handlerDependencies);
|
|
const now = options.now ?? (() => performance.now());
|
|
const workers = queues.map((queueName) => {
|
|
const baseProcessor = createProcessor(queueName, handlerDependencies);
|
|
const instrumented = metrics ? wrapProcessorWithMetrics(queueName, baseProcessor, metrics, now) : baseProcessor;
|
|
const processor = createLoggedProcessor(queueName, instrumented, logger, reporter, sentry);
|
|
const worker = createWorkerImpl(queueName, processor, { connection, concurrency: config.concurrency });
|
|
|
|
worker.on("completed", (job: WorkerJob) => {
|
|
logInfo(createJobLogger(logger, queueName, job), {}, "Worker job completed.");
|
|
});
|
|
worker.on("failed", (job: WorkerJob | undefined, error: Error) => {
|
|
const jobLogger = createJobLogger(logger, queueName, job);
|
|
const redactedError = redactWorkerError(error);
|
|
logError(jobLogger, { error: redactedError }, "Worker job failed.");
|
|
reportWorkerError(reporter, "job.failed", "error", queueName, job, redactedError);
|
|
void persistTerminalFailedJob({ queueName, job, error: redactedError, logger: jobLogger, recordFailedJob, reporter });
|
|
});
|
|
|
|
return worker;
|
|
});
|
|
|
|
logInfo(logger, { workerNames: workers.map((worker) => worker.name) }, "Worker started.");
|
|
return {
|
|
configured: true,
|
|
enabled: true,
|
|
queueNames: queues,
|
|
status: "started",
|
|
workers,
|
|
};
|
|
}
|
|
|
|
function assertQueueServicesWired(queues: readonly QueueName[], dependencies: HandlerDependencies): void {
|
|
const ownsImports = queues.includes(queueNames.imports);
|
|
const ownsExports = queues.includes(queueNames.exports);
|
|
const missing: string[] = [];
|
|
|
|
if (ownsImports) {
|
|
if (!dependencies.importService) missing.push("importService");
|
|
if (!dependencies.scanService) missing.push("scanService");
|
|
}
|
|
if (ownsExports && !dependencies.exportService) {
|
|
missing.push("exportService");
|
|
}
|
|
|
|
if (missing.length > 0) {
|
|
throw new Error(`Worker import/export queues require wired production services: missing ${missing.join(", ")}.`);
|
|
}
|
|
}
|
|
|
|
function createJobLogger(logger: WorkerRuntimeLogger, fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerRuntimeLogger {
|
|
const bindings = buildWorkerJobBindings(fallbackQueueName, job);
|
|
return typeof logger.child === "function" ? logger.child(bindings) : createBoundFallbackLogger(logger, bindings);
|
|
}
|
|
|
|
function createBoundFallbackLogger(logger: WorkerRuntimeLogger, bindings: WorkerLogBindings): WorkerRuntimeLogger {
|
|
return {
|
|
child: (childBindings) => createBoundFallbackLogger(logger, { ...bindings, ...childBindings }),
|
|
error: (record?: unknown, ...optionalParams: unknown[]) => logger.error(mergeLogRecord(bindings, record), ...optionalParams),
|
|
info: (record?: unknown, ...optionalParams: unknown[]) => {
|
|
if (logger.info) {
|
|
logger.info(mergeLogRecord(bindings, record), ...optionalParams);
|
|
return;
|
|
}
|
|
|
|
logger.log?.(mergeLogRecord(bindings, record), ...optionalParams);
|
|
},
|
|
log: (record?: unknown, ...optionalParams: unknown[]) => logger.log?.(mergeLogRecord(bindings, record), ...optionalParams),
|
|
warn: (record?: unknown, ...optionalParams: unknown[]) => {
|
|
if (logger.warn) {
|
|
logger.warn(mergeLogRecord(bindings, record), ...optionalParams);
|
|
return;
|
|
}
|
|
|
|
logger.log?.(mergeLogRecord(bindings, record), ...optionalParams);
|
|
},
|
|
};
|
|
}
|
|
|
|
function buildWorkerJobBindings(fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerLogBindings {
|
|
const backoff = parseWorkerBackoff(job?.opts?.backoff);
|
|
|
|
return {
|
|
...extractRequestIdBinding(job),
|
|
queueName: job?.queueName ?? fallbackQueueName,
|
|
jobName: job?.name ?? "unknown",
|
|
jobId: job?.id === undefined ? null : String(job.id),
|
|
attemptsMade: job?.attemptsMade ?? 0,
|
|
attemptsLimit: job?.opts?.attempts ?? null,
|
|
...backoff,
|
|
};
|
|
}
|
|
|
|
function extractRequestIdBinding(job: WorkerJob | undefined): Pick<WorkerLogBindings, "requestId"> {
|
|
if (!job || !isRecord(job.data)) {
|
|
return {};
|
|
}
|
|
|
|
const metadata = job.data.metadata;
|
|
if (!isRecord(metadata) || typeof metadata.requestId !== "string") {
|
|
return {};
|
|
}
|
|
|
|
return { requestId: metadata.requestId };
|
|
}
|
|
|
|
async function persistTerminalFailedJob({
|
|
queueName,
|
|
job,
|
|
error,
|
|
logger,
|
|
recordFailedJob,
|
|
reporter,
|
|
}: {
|
|
queueName: QueueName;
|
|
job: WorkerJob | undefined;
|
|
error: Error;
|
|
logger: WorkerRuntimeLogger;
|
|
recordFailedJob: (data: QueueFailedJobRecordInput) => Promise<void>;
|
|
reporter: ErrorReporter;
|
|
}): Promise<void> {
|
|
if (!isTerminalFailure(job)) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
await recordFailedJob({
|
|
queueName: job?.queueName ?? queueName,
|
|
jobName: job?.name ?? "unknown",
|
|
jobId: job?.id === undefined ? null : String(job.id),
|
|
attemptsMade: job?.attemptsMade ?? 0,
|
|
attemptsLimit: job?.opts?.attempts ?? null,
|
|
errorName: error.name,
|
|
errorMessage: error.message,
|
|
failedAt: new Date(),
|
|
});
|
|
} catch (persistenceError) {
|
|
const redactedPersistenceError = redactWorkerError(persistenceError);
|
|
logError(logger, { error: redactedPersistenceError }, "Failed to persist terminal worker failure.");
|
|
reportWorkerError(reporter, "job.persistence_failed", "error", queueName, job, redactedPersistenceError);
|
|
}
|
|
}
|
|
|
|
function isTerminalFailure(job: WorkerJob | undefined): boolean {
|
|
const attemptsLimit = job?.opts?.attempts;
|
|
if (attemptsLimit === undefined) {
|
|
return true;
|
|
}
|
|
|
|
return (job?.attemptsMade ?? 0) >= attemptsLimit;
|
|
}
|
|
|
|
function parseWorkerBackoff(backoff: NonNullable<WorkerJob["opts"]>["backoff"]): Pick<WorkerLogBindings, "backoffDelay" | "backoffType"> {
|
|
if (typeof backoff === "number") {
|
|
return { backoffDelay: backoff, backoffType: "fixed" };
|
|
}
|
|
|
|
if (backoff && typeof backoff === "object") {
|
|
return {
|
|
...(typeof backoff.delay === "number" ? { backoffDelay: backoff.delay } : {}),
|
|
...(typeof backoff.type === "string" ? { backoffType: backoff.type } : {}),
|
|
};
|
|
}
|
|
|
|
return {};
|
|
}
|
|
|
|
export async function shutdownWorkerRuntime(
|
|
runtime: Pick<WorkerRuntimeStatus, "workers">,
|
|
options: { logger?: WorkerRuntimeLogger; reason?: string; reporter?: ErrorReporter } = {}
|
|
): Promise<ShutdownWorkerRuntimeResult> {
|
|
const logger = options.logger ?? console;
|
|
const reason = options.reason ?? "shutdown";
|
|
const reporter = options.reporter ?? createNoopErrorReporter();
|
|
const closeResults = await Promise.allSettled(runtime.workers.map((worker) => worker.close()));
|
|
const failedCloses = closeResults.flatMap((result, index) => {
|
|
if (result.status === "fulfilled") {
|
|
return [];
|
|
}
|
|
|
|
const worker = runtime.workers[index];
|
|
const workerName = worker?.name ?? "unknown";
|
|
const redactedError = redactWorkerError(result.reason);
|
|
logError(logger, { error: redactedError, reason, workerName }, "Failed to close worker.");
|
|
reportError(reporter, {
|
|
component: "worker",
|
|
operation: "shutdown.close_failed",
|
|
severity: "error",
|
|
queueName: workerName,
|
|
workerName,
|
|
reason,
|
|
error: redactedError,
|
|
});
|
|
return [workerName];
|
|
});
|
|
|
|
if (failedCloses.length > 0) {
|
|
throw new Error(`Failed to close workers: ${failedCloses.join(", ")}.`);
|
|
}
|
|
|
|
const closedWorkers = runtime.workers.map((worker) => worker.name);
|
|
logInfo(logger, { closedWorkers, reason }, "Worker shutdown complete.");
|
|
return { closedWorkers };
|
|
}
|
|
|
|
function createLoggedProcessor<Name extends QueueName>(
|
|
queueName: Name,
|
|
processor: TypedProcessor<Name>,
|
|
logger: WorkerRuntimeLogger,
|
|
reporter: ErrorReporter,
|
|
sentry: SentryLikeForWorkerTracing | undefined
|
|
): TypedProcessor<Name> {
|
|
const tracedCore: TypedProcessor<Name> = async (job) => {
|
|
try {
|
|
return await processor(job);
|
|
} catch (error) {
|
|
const redactedError = redactWorkerError(error);
|
|
logError(createJobLogger(logger, queueName, job), { error: redactedError }, "Worker handler failed.");
|
|
reportWorkerError(reporter, "handler.failed", "error", queueName, job, redactedError);
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
if (!sentry) {
|
|
return tracedCore;
|
|
}
|
|
|
|
return wrapProcessorWithSentryTracing(queueName, tracedCore, sentry);
|
|
}
|
|
|
|
/**
|
|
* Wraps the BullMQ processor in a `Sentry.startSpan({ op: 'queue.process' })`
|
|
* span tagged with queue + job + tenant context. Sentry's @sentry/node v8/v10
|
|
* has no BullMQ auto-instrumentation, so we do this manually.
|
|
*/
|
|
function wrapProcessorWithSentryTracing<Name extends QueueName>(
|
|
queueName: Name,
|
|
inner: TypedProcessor<Name>,
|
|
sentry: SentryLikeForWorkerTracing
|
|
): TypedProcessor<Name> {
|
|
return async (job) => {
|
|
const jobShape = (job ?? {}) as Partial<WorkerJob> & { data?: unknown };
|
|
const jobName = jobShape.name ?? "unknown";
|
|
const jobId = jobShape.id !== undefined && jobShape.id !== null ? String(jobShape.id) : undefined;
|
|
const attemptsMade = jobShape.attemptsMade ?? 0;
|
|
const attemptsLimit = jobShape.opts?.attempts;
|
|
const tenantId = extractTenantId(jobShape.data);
|
|
|
|
return sentry.startSpan(
|
|
{
|
|
op: "queue.process",
|
|
name: `${queueName}/${jobName}`,
|
|
attributes: {
|
|
"messaging.system": "bullmq",
|
|
"messaging.destination.name": queueName,
|
|
"messaging.operation": "process",
|
|
"messaging.message.id": jobId,
|
|
"messaging.bullmq.job.name": jobName,
|
|
"messaging.bullmq.attempts_made": attemptsMade,
|
|
...(typeof attemptsLimit === "number" ? { "messaging.bullmq.attempts_limit": attemptsLimit } : {}),
|
|
...(tenantId ? { "hivegrid.tenant_id": tenantId } : {}),
|
|
},
|
|
},
|
|
async (span) => {
|
|
try {
|
|
const result = await inner(job);
|
|
span.setStatus?.({ code: 1, message: "ok" });
|
|
return result;
|
|
} catch (error) {
|
|
span.setStatus?.({ code: 2, message: "internal_error" });
|
|
if (typeof sentry.captureException === "function") {
|
|
try {
|
|
sentry.captureException(error, {
|
|
tags: {
|
|
component: "worker",
|
|
queue: queueName,
|
|
job_name: jobName,
|
|
...(jobId ? { job_id: jobId } : {}),
|
|
...(tenantId ? { tenant_id: tenantId } : {}),
|
|
},
|
|
});
|
|
} catch {
|
|
// Sentry transport failures must never mask the runtime failure.
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
);
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Wraps a BullMQ processor in a Prometheus-instrumented closure.
|
|
*
|
|
* Side effects per invocation:
|
|
* - `queue_jobs_completed_total{queue,job_name}` +1 on success.
|
|
* - `queue_jobs_failed_total{queue,job_name,error_type}` +1 on throw.
|
|
* `error_type` is the redacted `error.name` or `"Error"` to keep
|
|
* cardinality bounded (no message strings).
|
|
* - `queue_job_duration_seconds{queue,job_name}` observation in seconds.
|
|
*
|
|
* Metric writes are wrapped in try/catch so a registry failure can never
|
|
* mask a queue handler error.
|
|
*/
|
|
function wrapProcessorWithMetrics<Name extends QueueName>(
|
|
queueName: Name,
|
|
inner: TypedProcessor<Name>,
|
|
metrics: WorkerRuntimeMetrics,
|
|
now: () => number
|
|
): TypedProcessor<Name> {
|
|
return async (job) => {
|
|
const jobShape = (job ?? {}) as Partial<WorkerJob>;
|
|
const jobName = jobShape.name ?? "unknown";
|
|
const started = now();
|
|
let failed = false;
|
|
let errorType = "Error";
|
|
try {
|
|
const result = await inner(job);
|
|
try {
|
|
metrics.queueJobsCompletedTotal.labels({ queue: queueName, job_name: jobName }).inc();
|
|
} catch {
|
|
// ignore
|
|
}
|
|
return result;
|
|
} catch (error) {
|
|
failed = true;
|
|
errorType = pickErrorType(error);
|
|
throw error;
|
|
} finally {
|
|
try {
|
|
metrics.queueJobDurationSeconds.labels({ queue: queueName, job_name: jobName }).observe((now() - started) / 1000);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
if (failed) {
|
|
try {
|
|
metrics.queueJobsFailedTotal.labels({ queue: queueName, job_name: jobName, error_type: errorType }).inc();
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
function pickErrorType(error: unknown): string {
|
|
if (error instanceof Error) {
|
|
const name = error.name.trim();
|
|
return name.length > 0 ? name : "Error";
|
|
}
|
|
return "Error";
|
|
}
|
|
|
|
function extractTenantId(data: unknown): string | undefined {
|
|
if (!isRecord(data)) return undefined;
|
|
const direct = data.tenantId;
|
|
if (typeof direct === "string" && direct) return direct;
|
|
const metadata = data.metadata;
|
|
if (isRecord(metadata) && typeof metadata.tenantId === "string" && metadata.tenantId) {
|
|
return metadata.tenantId;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function reportWorkerError(
|
|
reporter: ErrorReporter,
|
|
operation: string,
|
|
severity: ErrorReportSeverity,
|
|
fallbackQueueName: QueueName,
|
|
job: WorkerJob | undefined,
|
|
error: unknown
|
|
): void {
|
|
const bindings = buildWorkerJobBindings(fallbackQueueName, job);
|
|
reportError(reporter, {
|
|
component: "worker",
|
|
operation,
|
|
severity,
|
|
queueName: bindings.queueName,
|
|
jobName: bindings.jobName,
|
|
jobId: bindings.jobId ?? undefined,
|
|
requestId: bindings.requestId,
|
|
attemptsMade: bindings.attemptsMade,
|
|
attemptsLimit: bindings.attemptsLimit,
|
|
retryCount: bindings.attemptsMade,
|
|
backoffDelay: bindings.backoffDelay,
|
|
backoffType: bindings.backoffType,
|
|
error,
|
|
});
|
|
}
|
|
|
|
function logInfo(logger: WorkerRuntimeLogger, record: Record<string, unknown>, message: string): void {
|
|
if (logger.info) {
|
|
logger.info(record, message);
|
|
return;
|
|
}
|
|
|
|
logger.log?.(record, message);
|
|
}
|
|
|
|
function logError(logger: WorkerRuntimeLogger, record: Record<string, unknown>, message: string): void {
|
|
logger.error(record, message);
|
|
}
|
|
|
|
function mergeLogRecord(bindings: WorkerLogBindings, record: unknown): Record<string, unknown> {
|
|
if (isRecord(record)) {
|
|
return { ...bindings, ...record };
|
|
}
|
|
|
|
return bindings;
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function redactWorkerError(error: unknown): Error {
|
|
const source = error instanceof Error ? error : new Error(String(error));
|
|
const redacted = new Error(redactWorkerDiagnosticText(source.message || source.name));
|
|
redacted.name = source.name || "Error";
|
|
return redacted;
|
|
}
|
|
|
|
function redactWorkerDiagnosticText(value: string): string {
|
|
return value
|
|
.replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]")
|
|
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
|
.replace(/(authorization|access[_-]?key(?:id)?|api[-_]?key|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
|
|
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]")
|
|
.slice(0, 1_000);
|
|
}
|