Reorganize repo structure and standards references

This commit is contained in:
david committed 2026-06-29 05:03:42 +02:00
1 parent 3cb93a81be
commit be2054274d
5885 files changed
+2588 -195

No files matched your search

@@ -0,0 +1,628 @@
import { loadRequiredWorkerConfig, type EnvRecord } from "@toolctrl/config";
import { recordQueueFailedJob, type QueueFailedJobRecordInput } from "@toolctrl/db/queue-failed-jobs.server";
import { createNoopErrorReporter, reportError, type ErrorReporter, type ErrorReportSeverity } from "@toolctrl/error-reporter";
import type { DefaultMetrics } from "@toolctrl/metrics";
import { parseRedisUrl, queueNames, type QueueName, type TypedProcessor } from "@toolctrl/queue";
import { createWorker } from "@toolctrl/queue/runtime";
import { createRegisteredProcessor, type HandlerDependencies } from "./handlers.js";
import { defaultWorkerQueueNames, getWorkerQueueNames } from "./queues.js";
import { createWorkerStorageDeleteService } from "./storage.js";
import { createWorkerScanService } from "./scanService.js";
/**
* Subset of the shared metrics bundle the queue runtime actually writes to.
* Keeping this narrow in the public signature lets tests inject a single
* counter without instantiating the entire prom-client registry.
*/
export type WorkerRuntimeMetrics = Pick<
DefaultMetrics,
"queueJobsCompletedTotal" | "queueJobsFailedTotal" | "queueJobDurationSeconds" | "mailSentTotal" | "webhookDeliveryAttemptsTotal"
>;
export const workerQueueNames = defaultWorkerQueueNames;
export { getWorkerQueueNames };
type WorkerBackoffOptions = {
delay?: number;
type?: string;
};
type WorkerJob = {
attemptsMade?: number;
data?: unknown;
id?: string | number;
name: string;
opts?: {
attempts?: number;
backoff?: number | WorkerBackoffOptions;
};
queueName?: string;
};
type WorkerLogBindings = {
attemptsLimit?: number | null;
attemptsMade?: number;
backoffDelay?: number;
backoffType?: string;
jobId?: string | null;
jobName?: string;
queueName?: string;
requestId?: string;
};
export type WorkerRuntimeLogger = {
child?: (bindings: Record<string, unknown>) => WorkerRuntimeLogger;
error: (message?: unknown, ...optionalParams: unknown[]) => void;
info?: (message?: unknown, ...optionalParams: unknown[]) => void;
log?: (message?: unknown, ...optionalParams: unknown[]) => void;
warn?: (message?: unknown, ...optionalParams: unknown[]) => void;
};
export type WorkerRuntimeWorker = {
close(): Promise<void>;
name: string;
on(event: "completed", listener: (job: WorkerJob) => void): WorkerRuntimeWorker;
on(event: "failed", listener: (job: WorkerJob | undefined, error: Error) => void): WorkerRuntimeWorker;
};
export type WorkerRuntimeStatus =
| {
configured: false;
enabled: false;
queueNames: readonly QueueName[];
status: "disabled";
workers: readonly [];
}
| {
configured: true;
enabled: true;
queueNames: readonly QueueName[];
status: "started";
workers: readonly WorkerRuntimeWorker[];
};
export type SentryStartSpanArgs = {
op: string;
name: string;
attributes?: Record<string, string | number | boolean | undefined>;
};
export type SentrySpan = {
setAttribute(key: string, value: string | number | boolean | undefined): void;
setStatus?(status: { code: number; message?: string }): void;
};
export type SentryLikeForWorkerTracing = {
startSpan: <T>(args: SentryStartSpanArgs, callback: (span: SentrySpan) => Promise<T> | T) => Promise<T> | T;
captureException?: (error: unknown, captureContext?: Record<string, unknown>) => void;
};
export type StartWorkerRuntimeOptions = {
createProcessor?: <Name extends QueueName>(queueName: Name, dependencies: HandlerDependencies) => TypedProcessor<Name>;
createWorkerImpl?: <Name extends QueueName>(
queueName: Name,
processor: TypedProcessor<Name>,
options: Parameters<typeof createWorker<Name>>[2]
) => WorkerRuntimeWorker;
env?: EnvRecord;
logger?: WorkerRuntimeLogger;
queueNames?: readonly QueueName[];
recordFailedJob?: (data: QueueFailedJobRecordInput) => Promise<void>;
reporter?: ErrorReporter;
/**
* Optional Sentry-like module for queue.process span tracing. When omitted,
* the runtime falls back to a pass-through processor. The production caller
* lazy-loads `@sentry/node`; tests inject a deterministic fake.
*/
sentry?: SentryLikeForWorkerTracing;
/**
* Optional Prometheus metrics. When set, the runtime records
* `queue_jobs_completed_total` on success or `queue_jobs_failed_total`
* on throw, and observes `queue_job_duration_seconds`. Queue-depth gauges
* such as `queue_jobs_active` are owned by the metrics scheduler.
* Omitting this is a no-op so existing callers stay byte-identical.
*/
metrics?: WorkerRuntimeMetrics;
/**
* Optional time source for duration histograms. Defaults to
* `performance.now()`; tests inject a deterministic clock.
*/
now?: () => number;
/**
* Optional extra HandlerDependencies merged into the runtime-built
* dependency object. The worker bootstrap uses this to inject
* Prisma-backed services that the runtime layer cannot construct on
* its own.
*/
extraHandlerDependencies?: Partial<HandlerDependencies>;
};
export type ShutdownWorkerRuntimeResult = {
closedWorkers: readonly string[];
};
export function startWorkerRuntime(options: StartWorkerRuntimeOptions = {}): WorkerRuntimeStatus {
const env = options.env ?? (process.env as EnvRecord);
const logger = options.logger ?? console;
const queues = options.queueNames ?? getWorkerQueueNames(env);
const config = loadRequiredWorkerConfig(env);
if (!config.enabled) {
logInfo(logger, { queueNames: queues }, "Worker disabled; queues not started.");
return {
configured: false,
enabled: false,
queueNames: queues,
status: "disabled",
workers: [],
};
}
if (!config.redis.url) {
throw new Error("Worker Redis URL is required when worker runtime is enabled.");
}
const connection = parseRedisUrl(config.redis.url);
const createWorkerImpl = options.createWorkerImpl ?? createWorker;
const createProcessor = options.createProcessor ?? ((queueName, dependencies) => createRegisteredProcessor(queueName, { dependencies }));
const recordFailedJob = options.recordFailedJob ?? recordQueueFailedJob;
const reporter = options.reporter ?? createNoopErrorReporter();
const sentry = options.sentry;
const scanService = env.SCAN_QUEUE_ENABLED === "true" ? createWorkerScanService(env) : undefined;
const metrics = options.metrics;
const mailMetricsHook = metrics
? ({ provider, status }: { provider: "mailjet"; status: "delivered" | "failed" }) => {
try {
metrics.mailSentTotal.labels({ provider, status }).inc();
} catch {
// Metric writes never block mail.
}
}
: undefined;
const webhookDeliveryMetricsHook = metrics
? ({ status }: { status: "success" | "retry" | "dlq" | "rejected" }) => {
try {
metrics.webhookDeliveryAttemptsTotal.labels({ status }).inc();
} catch {
// Metric writes never block delivery.
}
}
: undefined;
const handlerDependencies: HandlerDependencies = {
env,
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? { storageDeleteService: createWorkerStorageDeleteService(env) } : {}),
...(scanService ? { scanService } : {}),
...(mailMetricsHook ? { mailMetricsHook } : {}),
...(webhookDeliveryMetricsHook ? { webhookDeliveryMetricsHook } : {}),
...(options.extraHandlerDependencies ?? {}),
};
assertQueueServicesWired(queues, handlerDependencies);
const now = options.now ?? (() => performance.now());
const workers = queues.map((queueName) => {
const baseProcessor = createProcessor(queueName, handlerDependencies);
const instrumented = metrics ? wrapProcessorWithMetrics(queueName, baseProcessor, metrics, now) : baseProcessor;
const processor = createLoggedProcessor(queueName, instrumented, logger, reporter, sentry);
const worker = createWorkerImpl(queueName, processor, { connection, concurrency: config.concurrency });
worker.on("completed", (job: WorkerJob) => {
logInfo(createJobLogger(logger, queueName, job), {}, "Worker job completed.");
});
worker.on("failed", (job: WorkerJob | undefined, error: Error) => {
const jobLogger = createJobLogger(logger, queueName, job);
const redactedError = redactWorkerError(error);
logError(jobLogger, { error: redactedError }, "Worker job failed.");
reportWorkerError(reporter, "job.failed", "error", queueName, job, redactedError);
void persistTerminalFailedJob({ queueName, job, error: redactedError, logger: jobLogger, recordFailedJob, reporter });
});
return worker;
});
logInfo(logger, { workerNames: workers.map((worker) => worker.name) }, "Worker started.");
return {
configured: true,
enabled: true,
queueNames: queues,
status: "started",
workers,
};
}
function assertQueueServicesWired(queues: readonly QueueName[], dependencies: HandlerDependencies): void {
const ownsImports = queues.includes(queueNames.imports);
const ownsExports = queues.includes(queueNames.exports);
const missing: string[] = [];
if (ownsImports) {
if (!dependencies.importService) missing.push("importService");
if (!dependencies.scanService) missing.push("scanService");
}
if (ownsExports && !dependencies.exportService) {
missing.push("exportService");
}
if (missing.length > 0) {
throw new Error(`Worker import/export queues require wired production services: missing ${missing.join(", ")}.`);
}
}
function createJobLogger(logger: WorkerRuntimeLogger, fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerRuntimeLogger {
const bindings = buildWorkerJobBindings(fallbackQueueName, job);
return typeof logger.child === "function" ? logger.child(bindings) : createBoundFallbackLogger(logger, bindings);
}
function createBoundFallbackLogger(logger: WorkerRuntimeLogger, bindings: WorkerLogBindings): WorkerRuntimeLogger {
return {
child: (childBindings) => createBoundFallbackLogger(logger, { ...bindings, ...childBindings }),
error: (record?: unknown, ...optionalParams: unknown[]) => logger.error(mergeLogRecord(bindings, record), ...optionalParams),
info: (record?: unknown, ...optionalParams: unknown[]) => {
if (logger.info) {
logger.info(mergeLogRecord(bindings, record), ...optionalParams);
return;
}
logger.log?.(mergeLogRecord(bindings, record), ...optionalParams);
},
log: (record?: unknown, ...optionalParams: unknown[]) => logger.log?.(mergeLogRecord(bindings, record), ...optionalParams),
warn: (record?: unknown, ...optionalParams: unknown[]) => {
if (logger.warn) {
logger.warn(mergeLogRecord(bindings, record), ...optionalParams);
return;
}
logger.log?.(mergeLogRecord(bindings, record), ...optionalParams);
},
};
}
function buildWorkerJobBindings(fallbackQueueName: QueueName, job: WorkerJob | undefined): WorkerLogBindings {
const backoff = parseWorkerBackoff(job?.opts?.backoff);
return {
...extractRequestIdBinding(job),
queueName: job?.queueName ?? fallbackQueueName,
jobName: job?.name ?? "unknown",
jobId: job?.id === undefined ? null : String(job.id),
attemptsMade: job?.attemptsMade ?? 0,
attemptsLimit: job?.opts?.attempts ?? null,
...backoff,
};
}
function extractRequestIdBinding(job: WorkerJob | undefined): Pick<WorkerLogBindings, "requestId"> {
if (!job || !isRecord(job.data)) {
return {};
}
const metadata = job.data.metadata;
if (!isRecord(metadata) || typeof metadata.requestId !== "string") {
return {};
}
return { requestId: metadata.requestId };
}
async function persistTerminalFailedJob({
queueName,
job,
error,
logger,
recordFailedJob,
reporter,
}: {
queueName: QueueName;
job: WorkerJob | undefined;
error: Error;
logger: WorkerRuntimeLogger;
recordFailedJob: (data: QueueFailedJobRecordInput) => Promise<void>;
reporter: ErrorReporter;
}): Promise<void> {
if (!isTerminalFailure(job)) {
return;
}
try {
await recordFailedJob({
queueName: job?.queueName ?? queueName,
jobName: job?.name ?? "unknown",
jobId: job?.id === undefined ? null : String(job.id),
attemptsMade: job?.attemptsMade ?? 0,
attemptsLimit: job?.opts?.attempts ?? null,
errorName: error.name,
errorMessage: error.message,
failedAt: new Date(),
});
} catch (persistenceError) {
const redactedPersistenceError = redactWorkerError(persistenceError);
logError(logger, { error: redactedPersistenceError }, "Failed to persist terminal worker failure.");
reportWorkerError(reporter, "job.persistence_failed", "error", queueName, job, redactedPersistenceError);
}
}
function isTerminalFailure(job: WorkerJob | undefined): boolean {
const attemptsLimit = job?.opts?.attempts;
if (attemptsLimit === undefined) {
return true;
}
return (job?.attemptsMade ?? 0) >= attemptsLimit;
}
function parseWorkerBackoff(backoff: NonNullable<WorkerJob["opts"]>["backoff"]): Pick<WorkerLogBindings, "backoffDelay" | "backoffType"> {
if (typeof backoff === "number") {
return { backoffDelay: backoff, backoffType: "fixed" };
}
if (backoff && typeof backoff === "object") {
return {
...(typeof backoff.delay === "number" ? { backoffDelay: backoff.delay } : {}),
...(typeof backoff.type === "string" ? { backoffType: backoff.type } : {}),
};
}
return {};
}
export async function shutdownWorkerRuntime(
runtime: Pick<WorkerRuntimeStatus, "workers">,
options: { logger?: WorkerRuntimeLogger; reason?: string; reporter?: ErrorReporter } = {}
): Promise<ShutdownWorkerRuntimeResult> {
const logger = options.logger ?? console;
const reason = options.reason ?? "shutdown";
const reporter = options.reporter ?? createNoopErrorReporter();
const closeResults = await Promise.allSettled(runtime.workers.map((worker) => worker.close()));
const failedCloses = closeResults.flatMap((result, index) => {
if (result.status === "fulfilled") {
return [];
}
const worker = runtime.workers[index];
const workerName = worker?.name ?? "unknown";
const redactedError = redactWorkerError(result.reason);
logError(logger, { error: redactedError, reason, workerName }, "Failed to close worker.");
reportError(reporter, {
component: "worker",
operation: "shutdown.close_failed",
severity: "error",
queueName: workerName,
workerName,
reason,
error: redactedError,
});
return [workerName];
});
if (failedCloses.length > 0) {
throw new Error(`Failed to close workers: ${failedCloses.join(", ")}.`);
}
const closedWorkers = runtime.workers.map((worker) => worker.name);
logInfo(logger, { closedWorkers, reason }, "Worker shutdown complete.");
return { closedWorkers };
}
function createLoggedProcessor<Name extends QueueName>(
queueName: Name,
processor: TypedProcessor<Name>,
logger: WorkerRuntimeLogger,
reporter: ErrorReporter,
sentry: SentryLikeForWorkerTracing | undefined
): TypedProcessor<Name> {
const tracedCore: TypedProcessor<Name> = async (job) => {
try {
return await processor(job);
} catch (error) {
const redactedError = redactWorkerError(error);
logError(createJobLogger(logger, queueName, job), { error: redactedError }, "Worker handler failed.");
reportWorkerError(reporter, "handler.failed", "error", queueName, job, redactedError);
throw error;
}
};
if (!sentry) {
return tracedCore;
}
return wrapProcessorWithSentryTracing(queueName, tracedCore, sentry);
}
/**
* Wraps the BullMQ processor in a `Sentry.startSpan({ op: 'queue.process' })`
* span tagged with queue + job + tenant context. Sentry's @sentry/node v8/v10
* has no BullMQ auto-instrumentation, so we do this manually.
*/
function wrapProcessorWithSentryTracing<Name extends QueueName>(
queueName: Name,
inner: TypedProcessor<Name>,
sentry: SentryLikeForWorkerTracing
): TypedProcessor<Name> {
return async (job) => {
const jobShape = (job ?? {}) as Partial<WorkerJob> & { data?: unknown };
const jobName = jobShape.name ?? "unknown";
const jobId = jobShape.id !== undefined && jobShape.id !== null ? String(jobShape.id) : undefined;
const attemptsMade = jobShape.attemptsMade ?? 0;
const attemptsLimit = jobShape.opts?.attempts;
const tenantId = extractTenantId(jobShape.data);
return sentry.startSpan(
{
op: "queue.process",
name: `${queueName}/${jobName}`,
attributes: {
"messaging.system": "bullmq",
"messaging.destination.name": queueName,
"messaging.operation": "process",
"messaging.message.id": jobId,
"messaging.bullmq.job.name": jobName,
"messaging.bullmq.attempts_made": attemptsMade,
...(typeof attemptsLimit === "number" ? { "messaging.bullmq.attempts_limit": attemptsLimit } : {}),
...(tenantId ? { "toolctrl.tenant_id": tenantId } : {}),
},
},
async (span) => {
try {
const result = await inner(job);
span.setStatus?.({ code: 1, message: "ok" });
return result;
} catch (error) {
span.setStatus?.({ code: 2, message: "internal_error" });
if (typeof sentry.captureException === "function") {
try {
sentry.captureException(error, {
tags: {
component: "worker",
queue: queueName,
job_name: jobName,
...(jobId ? { job_id: jobId } : {}),
...(tenantId ? { tenant_id: tenantId } : {}),
},
});
} catch {
// Sentry transport failures must never mask the runtime failure.
}
}
throw error;
}
}
);
};
}
/**
* Wraps a BullMQ processor in a Prometheus-instrumented closure.
*
* Side effects per invocation:
* - `queue_jobs_completed_total{queue,job_name}` +1 on success.
* - `queue_jobs_failed_total{queue,job_name,error_type}` +1 on throw.
* `error_type` is the redacted `error.name` or `"Error"` to keep
* cardinality bounded (no message strings).
* - `queue_job_duration_seconds{queue,job_name}` observation in seconds.
*
* Metric writes are wrapped in try/catch so a registry failure can never
* mask a queue handler error.
*/
function wrapProcessorWithMetrics<Name extends QueueName>(
queueName: Name,
inner: TypedProcessor<Name>,
metrics: WorkerRuntimeMetrics,
now: () => number
): TypedProcessor<Name> {
return async (job) => {
const jobShape = (job ?? {}) as Partial<WorkerJob>;
const jobName = jobShape.name ?? "unknown";
const started = now();
let failed = false;
let errorType = "Error";
try {
const result = await inner(job);
try {
metrics.queueJobsCompletedTotal.labels({ queue: queueName, job_name: jobName }).inc();
} catch {
// ignore
}
return result;
} catch (error) {
failed = true;
errorType = pickErrorType(error);
throw error;
} finally {
try {
metrics.queueJobDurationSeconds.labels({ queue: queueName, job_name: jobName }).observe((now() - started) / 1000);
} catch {
// ignore
}
if (failed) {
try {
metrics.queueJobsFailedTotal.labels({ queue: queueName, job_name: jobName, error_type: errorType }).inc();
} catch {
// ignore
}
}
}
};
}
function pickErrorType(error: unknown): string {
if (error instanceof Error) {
const name = error.name.trim();
return name.length > 0 ? name : "Error";
}
return "Error";
}
function extractTenantId(data: unknown): string | undefined {
if (!isRecord(data)) return undefined;
const direct = data.tenantId;
if (typeof direct === "string" && direct) return direct;
const metadata = data.metadata;
if (isRecord(metadata) && typeof metadata.tenantId === "string" && metadata.tenantId) {
return metadata.tenantId;
}
return undefined;
}
function reportWorkerError(
reporter: ErrorReporter,
operation: string,
severity: ErrorReportSeverity,
fallbackQueueName: QueueName,
job: WorkerJob | undefined,
error: unknown
): void {
const bindings = buildWorkerJobBindings(fallbackQueueName, job);
reportError(reporter, {
component: "worker",
operation,
severity,
queueName: bindings.queueName,
jobName: bindings.jobName,
jobId: bindings.jobId ?? undefined,
requestId: bindings.requestId,
attemptsMade: bindings.attemptsMade,
attemptsLimit: bindings.attemptsLimit,
retryCount: bindings.attemptsMade,
backoffDelay: bindings.backoffDelay,
backoffType: bindings.backoffType,
error,
});
}
function logInfo(logger: WorkerRuntimeLogger, record: Record<string, unknown>, message: string): void {
if (logger.info) {
logger.info(record, message);
return;
}
logger.log?.(record, message);
}
function logError(logger: WorkerRuntimeLogger, record: Record<string, unknown>, message: string): void {
logger.error(record, message);
}
function mergeLogRecord(bindings: WorkerLogBindings, record: unknown): Record<string, unknown> {
if (isRecord(record)) {
return { ...bindings, ...record };
}
return bindings;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function redactWorkerError(error: unknown): Error {
const source = error instanceof Error ? error : new Error(String(error));
const redacted = new Error(redactWorkerDiagnosticText(source.message || source.name));
redacted.name = source.name || "Error";
return redacted;
}
function redactWorkerDiagnosticText(value: string): string {
return value
.replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]")
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
.replace(/(authorization|access[_-]?key(?:id)?|api[-_]?key|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]")
.slice(0, 1_000);
}