import { createHash } from "node:crypto"; import { WorkerConfigError } from "@hivegrid/config"; import { bindEncryptedPortalSetupMailToOutbox, hashPortalSetupMailContent, materializePortalSetupMailContent, PORTAL_SETUP_MAIL_SOURCE, PORTAL_SETUP_TOKEN_PLACEHOLDER, } from "@hivegrid/db/portal-setup-mail-contract.server"; import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server"; import type { ErrorReporter } from "@hivegrid/error-reporter"; import { describe, expect, test, vi } from "vitest"; import { queueNames, type QueueName, type TypedProcessor } from "@hivegrid/queue"; import { shutdownWorkerRuntime, startWorkerRuntime, getWorkerQueueNames, workerQueueNames, type StartWorkerRuntimeOptions, type WorkerRuntimeLogger, type WorkerRuntimeWorker, } from "./runtime.js"; const mailjetEnv = { MAILJET_API_KEY: "mailer-key", MAILJET_SECRET_KEY: "mailer-secret", MAILJET_FROM_EMAIL: "worker@example.com", }; const databaseEnv = { DATABASE_URL: "postgresql://localhost/app", }; const adminConnectionEnv = { ADMIN_CONNECTION_MASTER_KEY: "admin-connection-key-with-at-least-32-characters", }; const tenantHardDeleteStorageEnv = { STORAGE_ACCESS_KEY_ID: "access-key", STORAGE_BUCKET: "exports", STORAGE_ENDPOINT: "https://storage.example.com", STORAGE_REGION: "eu-central-1", STORAGE_SECRET_ACCESS_KEY: "secret-key", }; const portalGdprExportStorageEnv = { ...tenantHardDeleteStorageEnv, GDPR_EXPORT_SSE_C_KEY_BASE64: "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=", }; const workerGroupEnv = { WORKER_QUEUE_GROUP: "general", }; const tenantHardDeleteSearchEnv = { MEILI_API_KEY: "test-meili-key", MEILI_URL: "http://meili.internal.test:7700", }; function createPortalSetupJobData(token: string, tokenId: string) { const subject = "Kundenportal-Zugang einrichten"; const text = `https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}`; const html = `Einrichten`; const encrypted = { kind: "mail.transactional", tenantId: "tenant_1", to: ["customer@example.test"], subject, text, html, metadata: { source: PORTAL_SETUP_MAIL_SOURCE, invitationSource: "portal_admin_invite", portalId: "portal_1", portalUserId: "portal_user_1", portalTokenId: tokenId, setupTokenHash: createHash("sha256").update(token).digest("hex"), expiresAt: "2026-07-25T12:00:00.000Z", mailContentHash: hashPortalSetupMailContent({ recipient: "customer@example.test", subject, text, html }), encryptedCapability: "v1.encrypted-capability", }, }; return materializePortalSetupMailContent( bindEncryptedPortalSetupMailToOutbox(encrypted, { outboxEventId: `outbox_${tokenId}`, dispatchKey: `portal-setup:${tokenId}`, }), token ); } function createLogger() { return { error: vi.fn(), info: vi.fn(), log: vi.fn(), warn: vi.fn() }; } function createKwpHandlerDependencies() { return { tenantMaintenanceLeaseStore: {} as never, syncKwpProjectBinding: vi.fn(), syncKwpProjectCatalog: vi.fn(), enqueueKwpProjectCatalogContinuation: vi.fn(), syncKwpProjectInbound: vi.fn(), syncKwpArchiveDiscovery: vi.fn(), enqueueKwpArchiveDiscovery: vi.fn(), enqueueKwpArchiveFileJobs: vi.fn(), syncKwpArchiveFile: vi.fn(), runKwpContractSentinel: vi.fn(), }; } function createChildLogger() { const childLoggers: Array> = []; const logger = { ...createLogger(), child: vi.fn(() => { const child = createLogger(); childLoggers.push(child); return child; }), }; return { childLoggers, logger }; } type MockWorkerJob = { attemptsMade?: number; data?: unknown; id?: string | number; name: string; opts?: { attempts?: number; backoff?: number | { delay?: number; type?: string } }; queueName?: string; updateData?: (data: never) => Promise; }; type MockWorker = WorkerRuntimeWorker & { emitCompleted(job: MockWorkerJob): void; emitFailed(job: MockWorkerJob | undefined, error: Error): void; }; type TestCreateWorker = NonNullable; function createMockWorker(name: QueueName, close = vi.fn(async () => undefined)): MockWorker { const listeners = new Map(); const worker: MockWorker = { close, name, on: vi.fn((event: "completed" | "failed", listener: Function) => { listeners.set(event, listener); return worker; }), emitCompleted: (job) => listeners.get("completed")?.(job), emitFailed: (job, error) => listeners.get("failed")?.(job, error), }; return worker; } function startOneWorker(options: { createProcessor?: StartWorkerRuntimeOptions["createProcessor"]; logger: WorkerRuntimeLogger; recordFailedJob?: StartWorkerRuntimeOptions["recordFailedJob"]; reporter?: ErrorReporter; sentry?: StartWorkerRuntimeOptions["sentry"]; }) { const workers: MockWorker[] = []; let registeredProcessor: TypedProcessor | undefined; const createWorkerImpl = vi.fn((queueName: Name, processor: TypedProcessor) => { registeredProcessor = processor as TypedProcessor; const worker = createMockWorker(queueName); workers.push(worker); return worker; }) as TestCreateWorker; const runtime = startWorkerRuntime({ createProcessor: options.createProcessor ?? ((queueName) => vi.fn(async () => ({ queueName })) as TypedProcessor), createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv }, logger: options.logger, queueNames: [queueNames.email], recordFailedJob: options.recordFailedJob, reporter: options.reporter, sentry: options.sentry, }); return { createWorkerImpl, registeredProcessor, runtime, worker: workers[0] }; } async function flushMicrotasks() { await new Promise((resolve) => setTimeout(resolve, 0)); } describe("worker runtime", () => { test("wires handler diagnostics into the structured runtime logger", () => { const logger = createLogger(); let handlerLogger: { info: (fields: Record, message: string) => void } | undefined; startOneWorker({ logger, createProcessor: (queueName, dependencies) => { handlerLogger = dependencies.logger; return vi.fn(async () => ({ queueName })) as TypedProcessor; }, }); handlerLogger?.info({ reason: "provider_outage", connectionId: "kwp_1" }, "integrations.token-refresh.provider-failed"); expect(logger.info).toHaveBeenCalledWith( { reason: "provider_outage", connectionId: "kwp_1" }, "integrations.token-refresh.provider-failed", ); }); test("binds each registered processor to its queue payload tenant", async () => { const contexts = new Map>(); const { registeredProcessor } = startOneWorker({ logger: createLogger(), createProcessor: () => (async (job) => { await Promise.resolve(); const tenantId = (job.data as { tenantId?: string }).tenantId ?? "system"; contexts.set(tenantId, getRlsExecutionContext()); }) as TypedProcessor, }); await Promise.all([ registeredProcessor?.({ name: "send-transactional-email", data: { kind: "mail.transactional", tenantId: "tenant-a", userId: "user-a", to: ["a@example.test"] }, }), registeredProcessor?.({ name: "send-transactional-email", data: { kind: "mail.transactional", tenantId: "tenant-b", userId: "user-b", to: ["b@example.test"] }, }), registeredProcessor?.({ name: "deliver-webhook", data: { kind: "webhook.deliver", url: "https://example.test/hook", eventName: "system.event" }, }), ]); expect(contexts.get("tenant-a")).toEqual({ tenantId: "tenant-a" }); expect(contexts.get("tenant-b")).toEqual({ tenantId: "tenant-b" }); expect(contexts.get("system")).toBeNull(); expect(getRlsExecutionContext()).toBeNull(); }); test("returns disabled status without resolving Redis or creating workers", () => { const logger = createLogger(); const createWorkerImpl = vi.fn(); const runtime = startWorkerRuntime({ createWorkerImpl, env: { WORKER_ENABLED: "false" }, logger }); expect(runtime).toMatchObject({ configured: false, enabled: false, status: "disabled", workers: [] }); expect(runtime.queueNames).toEqual(workerQueueNames); expect(createWorkerImpl).not.toHaveBeenCalled(); expect(logger.info).toHaveBeenCalledWith({ queueNames: workerQueueNames }, "Worker disabled; queues not started."); }); test("rejects enabled startup without Redis using centralized redacted diagnostics", () => { expect(() => startWorkerRuntime({ env: { ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv }, logger: createLogger() })).toThrow(WorkerConfigError); }); test("rejects enabled startup without DATABASE_URL using centralized redacted diagnostics", () => { expect(() => startWorkerRuntime({ createWorkerImpl: vi.fn((queueName: Name) => createMockWorker(queueName)) as TestCreateWorker, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv }, logger: createLogger(), }) ).toThrow(WorkerConfigError); }); test("starts webhook-capable queue workers without Mailjet config", () => { const logger = createLogger(); const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); const createWorkerImpl = createWorkerImplMock as TestCreateWorker; const runtime = startWorkerRuntime({ createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "webhook" }, logger, queueNames: [queueNames.email], }); expect(runtime.status).toBe("started"); expect(createWorkerImpl).toHaveBeenCalledWith(queueNames.email, expect.any(Function), expect.objectContaining({ concurrency: 1 })); }); test("registers one worker per queue and logs startup", () => { const logger = createLogger(); const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); const createWorkerImpl = createWorkerImplMock as TestCreateWorker; const runtime = startWorkerRuntime({ createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...mailjetEnv, WORKER_ENABLED: "true", WORKER_CONCURRENCY: "3", ...workerGroupEnv }, logger, }); expect(runtime.status).toBe("started"); expect(runtime.queueNames).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]); expect(workerQueueNames).toEqual([queueNames.email, queueNames.notifications]); expect(workerQueueNames).not.toContain(queueNames.exports); expect(workerQueueNames).not.toContain(queueNames.browserAutomation); expect(createWorkerImpl).toHaveBeenCalledTimes(runtime.queueNames.length); expect(vi.mocked(createWorkerImpl).mock.calls.map(([queueName]) => queueName)).toEqual(runtime.queueNames); expect(logger.info).toHaveBeenCalledWith({ workerNames: runtime.queueNames }, "Worker started."); }); test("adds the maintenance queue when export retention processing is explicitly enabled", () => { const logger = createLogger(); const createWorkerImpl = vi.fn((queueName: Name) => createMockWorker(queueName)) as TestCreateWorker; const runtime = startWorkerRuntime({ createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, EXPORT_RETENTION_QUEUE_ENABLED: "true", ...mailjetEnv, STORAGE_ACCESS_KEY_ID: "access-key", STORAGE_BUCKET: "exports", STORAGE_ENDPOINT: "https://storage.example.com", STORAGE_REGION: "us-east-1", STORAGE_SECRET_ACCESS_KEY: "secret-key", WORKER_ENABLED: "true", ...workerGroupEnv, }, logger, }); expect(runtime.queueNames).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]); expect(getWorkerQueueNames({ EXPORT_RETENTION_QUEUE_ENABLED: "true", WORKER_QUEUE_GROUP: "general" })).toEqual([ queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview, ]); expect(vi.mocked(createWorkerImpl).mock.calls.map(([queueName]) => queueName)).toEqual(runtime.queueNames); }); test("keeps KWP synchronization active while generic import runtime gates are disabled", () => { expect(getWorkerQueueNames({ WORKER_QUEUE_GROUP: "import" })).toEqual([queueNames.kwpImports]); expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]); expect(getWorkerQueueNames({ IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]); expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([ queueNames.kwpImports, queueNames.imports, queueNames.exports, ]); }); test("starts KWP plus exports with portal and privacy GDPR services in dedicated mode", () => { const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); const createWorkerImpl = createWorkerImplMock as TestCreateWorker; const runtime = startWorkerRuntime({ createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true", ...portalGdprExportStorageEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "import", }, extraHandlerDependencies: { ...createKwpHandlerDependencies(), portalGdprExportService: { runExport: vi.fn() }, privacyGdprExportService: { withTenantBarrier: async (_payload, writer) => writer(async () => undefined), resolveReady: vi.fn(), resolveReclaimableProcessing: vi.fn(), createAttempt: vi.fn(), runExport: vi.fn(), deleteArtifact: vi.fn(), terminalizeReadyAfterQuiescence: vi.fn(), }, }, logger: createLogger(), }); expect(runtime.queueNames).toEqual([queueNames.kwpImports, queueNames.exports]); expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([queueNames.kwpImports, queueNames.exports]); }); test("fails closed when dedicated GDPR export services are not wired", () => { const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); expect(() => startWorkerRuntime({ createWorkerImpl: createWorkerImplMock as TestCreateWorker, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true", ...portalGdprExportStorageEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "import", }, extraHandlerDependencies: createKwpHandlerDependencies(), logger: createLogger(), }) ).toThrow(/KWP and portal\/privacy GDPR queues requires wired production services: missing portalGdprExportService, privacyGdprExportService/u); expect(createWorkerImplMock).not.toHaveBeenCalled(); }); test("fails closed when the dedicated export gate is assigned to another worker group", () => { const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); expect(() => startWorkerRuntime({ createWorkerImpl: createWorkerImplMock as TestCreateWorker, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true", ...portalGdprExportStorageEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "general", }, extraHandlerDependencies: createKwpHandlerDependencies(), logger: createLogger(), }) ).toThrow(/PORTAL_GDPR_EXPORT_QUEUE_ENABLED:invalid-worker-queue-group/u); expect(createWorkerImplMock).not.toHaveBeenCalled(); }); test("rejects import worker startup without wired import/export services", () => { const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); const createWorkerImpl = createWorkerImplMock as TestCreateWorker; expect(() => startWorkerRuntime({ createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", ...portalGdprExportStorageEnv, ...mailjetEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "import", }, extraHandlerDependencies: createKwpHandlerDependencies(), logger: createLogger(), }) ).toThrow( /missing importService, scanService, adminSalesDataService, exportService/u ); expect(createWorkerImplMock).not.toHaveBeenCalled(); }); test("rejects full import startup without the GDPR SSE-C key before queue ownership", () => { const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); const { GDPR_EXPORT_SSE_C_KEY_BASE64: _key, ...storageWithoutKey } = portalGdprExportStorageEnv; expect(() => startWorkerRuntime({ createWorkerImpl: createWorkerImplMock as TestCreateWorker, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", ...storageWithoutKey, ...mailjetEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "import", }, extraHandlerDependencies: { ...createKwpHandlerDependencies(), exportService: { runExport: vi.fn() }, importService: { runImport: vi.fn() }, portalGdprExportService: { runExport: vi.fn() }, privacyGdprExportService: { withTenantBarrier: async (_payload, writer) => writer(async () => undefined), resolveReady: vi.fn(), resolveReclaimableProcessing: vi.fn(), createAttempt: vi.fn(), runExport: vi.fn(), deleteArtifact: vi.fn(), terminalizeReadyAfterQuiescence: vi.fn(), }, scanService: { scanObject: vi.fn() }, }, logger: createLogger(), }) ).toThrow(/GDPR_EXPORT_SSE_C_KEY_BASE64:missing-required/u); expect(createWorkerImplMock).not.toHaveBeenCalled(); }); test("starts import worker queues only with wired import/export services", () => { const createWorkerImplMock = vi.fn((queueName: Name) => createMockWorker(queueName)); const createWorkerImpl = createWorkerImplMock as TestCreateWorker; const runtime = startWorkerRuntime({ createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", ...portalGdprExportStorageEnv, ...mailjetEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "import", }, extraHandlerDependencies: { ...createKwpHandlerDependencies(), adminSalesDataService: { runImport: vi.fn(), runExport: vi.fn(), }, exportService: { runExport: vi.fn() }, importService: { runImport: vi.fn() }, portalGdprExportService: { runExport: vi.fn() }, privacyGdprExportService: { withTenantBarrier: async (_payload, writer) => writer(async () => undefined), resolveReady: vi.fn(), resolveReclaimableProcessing: vi.fn(), createAttempt: vi.fn(), runExport: vi.fn(), deleteArtifact: vi.fn(), terminalizeReadyAfterQuiescence: vi.fn(), }, scanService: { scanObject: vi.fn() }, }, logger: createLogger(), }); expect(runtime.status).toBe("started"); expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([ queueNames.kwpImports, queueNames.imports, queueNames.exports, ]); }); test("rejects export retention startup without storage delete configuration", () => { expect(() => startWorkerRuntime({ createWorkerImpl: vi.fn((queueName: Name) => createMockWorker(queueName)) as TestCreateWorker, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, EXPORT_RETENTION_QUEUE_ENABLED: "true", ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv, }, logger: createLogger(), }) ).toThrow(/Invalid worker configuration \(STORAGE_ENDPOINT:missing-required/u); }); test("rejects production general-worker startup before queue ownership without tenant hard-delete storage", () => { const createWorkerImpl = vi.fn((queueName: Name) => createMockWorker(queueName)); expect(() => startWorkerRuntime({ createWorkerImpl: createWorkerImpl as TestCreateWorker, env: { NODE_ENV: "production", APP_BASE_DOMAIN: "example.test", AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes", ...adminConnectionEnv, BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv, }, logger: createLogger(), }) ).toThrow(/STORAGE_BUCKET:missing-required/u); expect(createWorkerImpl).not.toHaveBeenCalled(); }); test("rejects production general-worker queue ownership without the tenant hard-delete service", () => { const createWorkerImpl = vi.fn((queueName: Name) => createMockWorker(queueName)); expect(() => startWorkerRuntime({ createWorkerImpl: createWorkerImpl as TestCreateWorker, env: { NODE_ENV: "production", APP_BASE_DOMAIN: "example.test", AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes", ...adminConnectionEnv, BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...tenantHardDeleteStorageEnv, ...tenantHardDeleteSearchEnv, ...mailjetEnv, EXPORT_RETENTION_QUEUE_ENABLED: "true", WORKER_ENABLED: "true", ...workerGroupEnv, }, logger: createLogger(), queueNames: [queueNames.maintenance], }) ).toThrow(/missing privacyTenantDeletionService/u); expect(createWorkerImpl).not.toHaveBeenCalled(); }); test.each(["MEILI_URL", "MEILI_API_KEY"] as const)( "rejects production general-worker queue ownership without %s for tenant search cleanup", (missingName) => { const createWorkerImpl = vi.fn((queueName: Name) => createMockWorker(queueName)); const env = { NODE_ENV: "production", APP_BASE_DOMAIN: "example.test", AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes", ...adminConnectionEnv, BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...tenantHardDeleteStorageEnv, ...tenantHardDeleteSearchEnv, ...mailjetEnv, EXPORT_RETENTION_QUEUE_ENABLED: "true", WORKER_ENABLED: "true", ...workerGroupEnv, }; delete env[missingName]; expect(() => startWorkerRuntime({ createWorkerImpl: createWorkerImpl as TestCreateWorker, env, extraHandlerDependencies: { privacyTenantDeletionService: { runScan: vi.fn() } }, logger: createLogger(), queueNames: [queueNames.maintenance], }) ).toThrow(new RegExp(`missing ${missingName}`, "u")); expect(createWorkerImpl).not.toHaveBeenCalled(); } ); test("binds requestId and retry fields through child loggers for completed and failed jobs", async () => { const { childLoggers, logger } = createChildLogger(); const recordFailedJob = vi.fn(async () => undefined); const { worker } = startOneWorker({ logger, recordFailedJob }); const failure = new Error("handler failed"); worker.emitCompleted({ attemptsMade: 1, data: { metadata: { requestId: "req_completed" } }, id: "job_1", name: "send-transactional-email", opts: { attempts: 3, backoff: 500 }, queueName: queueNames.email, }); worker.emitFailed( { attemptsMade: 3, data: { metadata: { requestId: "req_failed" } }, id: "job_2", name: "deliver-webhook", opts: { attempts: 3, backoff: { delay: 2_000, type: "exponential" } }, queueName: queueNames.email, }, failure ); await flushMicrotasks(); expect(logger.child).toHaveBeenCalledWith({ attemptsLimit: 3, attemptsMade: 1, backoffDelay: 500, backoffType: "fixed", jobId: "job_1", jobName: "send-transactional-email", queueName: queueNames.email, requestId: "req_completed", }); expect(logger.child).toHaveBeenCalledWith(expect.objectContaining({ jobId: "job_2", requestId: "req_failed", backoffType: "exponential" })); expect(childLoggers[0]?.info).toHaveBeenCalledWith({}, "Worker job completed."); expect(childLoggers[1]?.error).toHaveBeenCalledWith({ error: expect.objectContaining({ message: failure.message }) }, "Worker job failed."); expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ jobId: "job_2" })); }); test("falls back to structured calls without child and ignores invalid requestId metadata", () => { const logger = createLogger(); const { worker } = startOneWorker({ logger }); worker.emitCompleted({ data: { metadata: { requestId: 12345 } }, id: "job-no-child", name: "deliver-webhook", queueName: queueNames.email, }); expect(logger.info).toHaveBeenCalledWith( { attemptsLimit: null, attemptsMade: 0, jobId: "job-no-child", jobName: "deliver-webhook", queueName: queueNames.email }, "Worker job completed." ); }); test("redacts completed OWNER mail payloads while failed attempts retain the token for retry", async () => { const token = "owner-capability-token-fixture"; const data = { kind: "mail.transactional", tenantId: "tenant_1", to: ["owner@example.test"], html: `Accept`, metadata: { source: "workspace-owner-invitation", invitationId: "invite_1", invitationTokenHash: "a".repeat(64), issuanceKeyHash: "b".repeat(64), }, }; const completedUpdateData = vi.fn(async (_data: unknown) => undefined); const failedUpdateData = vi.fn(async (_data: unknown) => undefined); const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob: vi.fn(async () => undefined) }); worker.emitFailed( { attemptsMade: 1, data, id: "owner-mail-retry", name: "send-transactional-email", opts: { attempts: 3 }, queueName: queueNames.email, updateData: failedUpdateData, }, new Error("mail provider unavailable") ); worker.emitCompleted({ attemptsMade: 1, data, id: "owner-mail-complete", name: "send-transactional-email", opts: { attempts: 3 }, queueName: queueNames.email, updateData: completedUpdateData, }); await vi.waitFor(() => expect(completedUpdateData).toHaveBeenCalledOnce()); expect(failedUpdateData).not.toHaveBeenCalled(); expect(JSON.stringify(data)).toContain(token); expect(JSON.stringify(completedUpdateData.mock.calls[0]?.[0])).not.toContain(token); expect(completedUpdateData.mock.calls[0]?.[0]).toMatchObject({ metadata: expect.objectContaining({ source: "workspace-owner-invitation", secretBodyRedacted: true }), }); }); test("redacts a terminal failed OWNER job so the Redis DLQ cannot retain its capability", async () => { const token = "terminal-owner-capability-token-fixture"; const data = { kind: "mail.transactional", tenantId: "tenant_1", to: ["owner@example.test"], html: `Accept`, metadata: { source: "workspace-owner-invitation", invitationId: "invite_terminal", invitationTokenHash: "a".repeat(64), issuanceKeyHash: "b".repeat(64), }, }; const updateData = vi.fn(async (_data: unknown) => undefined); const recordFailedJob = vi.fn(async () => undefined); const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob }); worker.emitFailed( { attemptsMade: 3, data, id: "owner-mail-terminal", name: "send-transactional-email", opts: { attempts: 3 }, queueName: queueNames.email, updateData, }, new Error("mail provider permanently unavailable") ); await vi.waitFor(() => expect(updateData).toHaveBeenCalledOnce()); expect(JSON.stringify(updateData.mock.calls[0]?.[0])).not.toContain(token); expect(updateData.mock.calls[0]?.[0]).toMatchObject({ metadata: expect.objectContaining({ secretBodyRedacted: true, ownerInvitationDeliveryStatus: "rotation_required" }), }); expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ jobId: "owner-mail-terminal" })); }); test("redacts completed and terminal portal setup jobs so Redis retains no setup capability", async () => { const completedToken = "c".repeat(43); const failedToken = "f".repeat(43); const completedUpdateData = vi.fn(async (_data: unknown) => undefined); const failedUpdateData = vi.fn(async (_data: unknown) => undefined); const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob: vi.fn(async () => undefined) }); worker.emitCompleted({ attemptsMade: 0, data: createPortalSetupJobData(completedToken, "portal_token_completed"), id: "portal-setup-completed", name: "send-transactional-email", opts: { attempts: 3 }, queueName: queueNames.email, updateData: completedUpdateData, }); worker.emitFailed( { attemptsMade: 3, data: createPortalSetupJobData(failedToken, "portal_token_failed"), id: "portal-setup-failed", name: "send-transactional-email", opts: { attempts: 3 }, queueName: queueNames.email, updateData: failedUpdateData, }, new Error("provider unavailable") ); await vi.waitFor(() => expect(completedUpdateData).toHaveBeenCalledOnce()); await vi.waitFor(() => expect(failedUpdateData).toHaveBeenCalledOnce()); expect(JSON.stringify(completedUpdateData.mock.calls)).not.toContain(completedToken); expect(JSON.stringify(failedUpdateData.mock.calls)).not.toContain(failedToken); expect(completedUpdateData.mock.calls[0]?.[0]).toMatchObject({ metadata: expect.objectContaining({ secretBodyRedacted: true }), }); expect(failedUpdateData.mock.calls[0]?.[0]).toMatchObject({ metadata: expect.objectContaining({ secretBodyRedacted: true, portalSetupDeliveryStatus: "rotation_required" }), }); }); test("logs retry diagnostics without leaking payload data or persisting non-terminal failures", () => { const logger = createLogger(); const recordFailedJob = vi.fn(async () => undefined); const { worker } = startOneWorker({ logger, recordFailedJob }); const failure = new Error("remote service unavailable"); worker.emitFailed( { attemptsMade: 2, data: { body: "fixture-body-secret", headers: { authorization: "Bearer token-secret" }, metadata: { requestId: "req_retry" } }, id: "webhook:tenant_1:event_1", name: "deliver-webhook", opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } }, queueName: queueNames.email, }, failure ); expect(logger.error).toHaveBeenCalledWith( expect.objectContaining({ error: expect.objectContaining({ message: "remote service unavailable" }), requestId: "req_retry", attemptsLimit: 5 }), "Worker job failed." ); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("fixture-body-secret"); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("token-secret"); expect(recordFailedJob).not.toHaveBeenCalled(); }); test("reports failed jobs with correlation fields and redacted payload secrets", async () => { const logger = createLogger(); const reporter: ErrorReporter = { report: vi.fn() }; const recordFailedJob = vi.fn(async () => undefined); const { worker } = startOneWorker({ logger, recordFailedJob, reporter }); const failure = new Error("remote service unavailable token=runtime-secret Authorization: Bearer auth-secret"); Object.assign(failure, { token: "runtime-secret" }); worker.emitFailed( { attemptsMade: 2, data: { body: "fixture-body-secret", headers: { authorization: "Bearer auth-secret" }, metadata: { requestId: "req_reported_retry" }, recipient: "private@example.com", templateData: { token: "template-secret" }, }, id: "job-reporter-1", name: "deliver-webhook", opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } }, queueName: queueNames.email, }, failure ); await flushMicrotasks(); expect(reporter.report).toHaveBeenCalledTimes(1); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ component: "worker", operation: "job.failed", severity: "error", queueName: queueNames.email, jobName: "deliver-webhook", jobId: "job-reporter-1", requestId: "req_reported_retry", attemptsMade: 2, attemptsLimit: 5, retryCount: 2, backoffDelay: 2000, backoffType: "exponential", error: expect.objectContaining({ errorName: "Error", errorMessage: expect.stringContaining("remote service unavailable"), }), }) ); const serializedReport = JSON.stringify(vi.mocked(reporter.report).mock.calls); expect(serializedReport).not.toContain("fixture-body-secret"); expect(serializedReport).not.toContain("auth-secret"); expect(serializedReport).not.toContain("private@example.com"); expect(serializedReport).not.toContain("template-secret"); expect(serializedReport).not.toContain("runtime-secret"); expect(recordFailedJob).not.toHaveBeenCalled(); }); test("persists terminal failures and logs persistence errors without throwing", async () => { const logger = createLogger(); const persistenceError = new Error("database unavailable"); Object.assign(persistenceError, { token: "persistence-secret-token" }); const reporter: ErrorReporter = { report: vi.fn() }; const recordFailedJob = vi.fn(async () => { throw persistenceError; }); const { worker } = startOneWorker({ logger, recordFailedJob, reporter }); expect(() => worker.emitFailed( { attemptsMade: 1, data: { token: "persist-secret" }, id: "job-persist-error", name: "deliver-webhook", opts: { attempts: 1 }, queueName: queueNames.email, }, new TypeError("terminal failure") ) ).not.toThrow(); await flushMicrotasks(); expect(recordFailedJob).toHaveBeenCalledWith( expect.objectContaining({ attemptsLimit: 1, errorName: "TypeError", errorMessage: "terminal failure", jobId: "job-persist-error" }) ); expect(logger.error).toHaveBeenCalledWith( expect.objectContaining({ error: expect.objectContaining({ message: "database unavailable" }), jobId: "job-persist-error" }), "Failed to persist terminal worker failure." ); expect(reporter.report).toHaveBeenCalledWith(expect.objectContaining({ operation: "job.failed", jobId: "job-persist-error", requestId: undefined })); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ component: "worker", operation: "job.persistence_failed", jobId: "job-persist-error", error: expect.objectContaining({ errorName: "Error", errorMessage: "database unavailable" }), }) ); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("persist-secret"); expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("persist-secret"); expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("persistence-secret-token"); }); test("logs handler failures with requestId and rethrows processor errors", async () => { const logger = createLogger(); const handlerError = new Error("handler exploded"); const reporter: ErrorReporter = { report: vi.fn() }; const { registeredProcessor } = startOneWorker({ createProcessor: () => vi.fn(async () => { throw handlerError; }) as TypedProcessor, logger, reporter, }); await expect( registeredProcessor?.({ attemptsMade: 1, data: { metadata: { requestId: "req_handler" }, templateData: "fixture-secret", to: "private@example.com" }, id: "job_3", name: "deliver-webhook", opts: { attempts: 3, backoff: 750 }, } as unknown as Parameters>[0]) ).rejects.toThrow(handlerError); expect(logger.error).toHaveBeenCalledWith( expect.objectContaining({ error: expect.objectContaining({ message: "handler exploded" }), requestId: "req_handler", backoffType: "fixed" }), "Worker handler failed." ); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ component: "worker", operation: "handler.failed", severity: "error", requestId: "req_handler", jobId: "job_3", retryCount: 1, error: expect.objectContaining({ errorName: "Error" }), }) ); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("fixture-secret"); expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("private@example.com"); expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("fixture-secret"); expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("private@example.com"); }); test("shutdown closes workers and reports close rejections or success as structured records", async () => { const logger = createLogger(); const closeError = new Error("close failed"); const reporter: ErrorReporter = { report: vi.fn() }; const emailWorker = createMockWorker(queueNames.email); const notificationWorker = createMockWorker( queueNames.notifications, vi.fn(async () => { throw closeError; }) ); await expect(shutdownWorkerRuntime({ workers: [emailWorker, notificationWorker] }, { logger, reason: "SIGTERM", reporter })).rejects.toThrow( "Failed to close workers: hivegrid.notifications." ); expect(logger.error).toHaveBeenCalledWith( { error: expect.objectContaining({ message: "close failed" }), reason: "SIGTERM", workerName: queueNames.notifications }, "Failed to close worker." ); expect(reporter.report).toHaveBeenCalledWith( expect.objectContaining({ component: "worker", operation: "shutdown.close_failed", severity: "error", queueName: queueNames.notifications, workerName: queueNames.notifications, reason: "SIGTERM", error: expect.objectContaining({ errorName: "Error" }), }) ); await expect(shutdownWorkerRuntime({ workers: [emailWorker] }, { logger, reason: "SIGINT", reporter })).resolves.toEqual({ closedWorkers: [queueNames.email], }); expect(logger.info).toHaveBeenCalledWith({ closedWorkers: [queueNames.email], reason: "SIGINT" }, "Worker shutdown complete."); }); test("swallows reporter failures without changing retry or shutdown behavior", async () => { const logger = createLogger(); const reporter: ErrorReporter = { report: vi.fn(async () => { throw new Error("reporter failed"); }), }; const recordFailedJob = vi.fn(async () => undefined); const { worker } = startOneWorker({ logger, recordFailedJob, reporter }); expect(() => worker.emitFailed( { attemptsMade: 1, data: { metadata: { requestId: "req_reporter_failure" } }, id: "job-report-fail", name: "deliver-webhook", opts: { attempts: 3 }, queueName: queueNames.email, }, new Error("retry failure") ) ).not.toThrow(); await flushMicrotasks(); expect(recordFailedJob).not.toHaveBeenCalled(); const closingWorker = createMockWorker( queueNames.email, vi.fn(async () => { throw new Error("close failed"); }) ); await expect(shutdownWorkerRuntime({ workers: [closingWorker] }, { logger, reporter })).rejects.toThrow("Failed to close workers: hivegrid.email."); }); test("wraps each job in a Sentry queue.process span with queue/job/tenant tags", async () => { type Span = { args: { op: string; name: string; attributes?: Record }; status?: { code: number; message?: string } }; const spans: Span[] = []; const captureException = vi.fn(); const sentry = { captureException, async startSpan( args: { op: string; name: string; attributes?: Record }, callback: (s: { setAttribute(k: string, v: unknown): void; setStatus(s: { code: number; message?: string }): void }) => Promise | T ): Promise { const record: Span = { args, status: undefined }; spans.push(record); return callback({ setAttribute() { /* no-op for assertion brevity */ }, setStatus(s) { record.status = s; }, }); }, }; const logger = createLogger(); const processor = vi.fn(async () => ({ ok: true })); const { registeredProcessor } = startOneWorker({ logger, sentry, createProcessor: () => processor as unknown as TypedProcessor, }); expect(registeredProcessor).toBeDefined(); await registeredProcessor?.({ attemptsMade: 0, data: { tenantId: "tnt_42" }, id: "job-abc", name: "deliver-webhook", opts: { attempts: 3 }, queueName: queueNames.email, } as never); expect(spans).toHaveLength(1); expect(spans[0].args.op).toBe("queue.process"); expect(spans[0].args.name).toBe(`${queueNames.email}/deliver-webhook`); expect(spans[0].args.attributes?.["messaging.destination.name"]).toBe(queueNames.email); expect(spans[0].args.attributes?.["messaging.bullmq.job.name"]).toBe("deliver-webhook"); expect(spans[0].args.attributes?.["messaging.bullmq.attempts_made"]).toBe(0); expect(spans[0].args.attributes?.["messaging.bullmq.attempts_limit"]).toBe(3); expect(spans[0].args.attributes?.["hivegrid.tenant_id"]).toBe("tnt_42"); expect(spans[0].status?.code).toBe(1); expect(captureException).not.toHaveBeenCalled(); }); test("captures handler exceptions via Sentry.captureException with queue/job/tenant tags", async () => { const spans: Array<{ status?: { code: number; message?: string } }> = []; const captureException = vi.fn(); const sentry = { captureException, async startSpan( _args: unknown, callback: (s: { setAttribute(k: string, v: unknown): void; setStatus(s: { code: number; message?: string }): void }) => Promise | T ): Promise { const record: { status?: { code: number; message?: string } } = {}; spans.push(record); return callback({ setAttribute() { /* no-op */ }, setStatus(s) { record.status = s; }, }); }, }; const logger = createLogger(); const failing = vi.fn(async () => { throw new Error("kaboom"); }); const { registeredProcessor } = startOneWorker({ logger, sentry, createProcessor: () => failing as unknown as TypedProcessor, }); await expect( registeredProcessor?.({ attemptsMade: 1, data: { metadata: { tenantId: "tnt_meta" } }, id: 7, name: "send-email", opts: { attempts: 2 }, queueName: queueNames.email, } as never) ).rejects.toThrow("kaboom"); expect(spans).toHaveLength(1); expect(spans[0].status?.code).toBe(2); expect(captureException).toHaveBeenCalledTimes(1); const [thrown, ctx] = captureException.mock.calls[0] ?? []; expect((thrown as Error).message).toBe("kaboom"); const tags = (ctx as { tags: Record } | undefined)?.tags; expect(tags?.component).toBe("worker"); expect(tags?.queue).toBe(queueNames.email); expect(tags?.job_name).toBe("send-email"); expect(tags?.job_id).toBe("7"); expect(tags?.tenant_id).toBe("tnt_meta"); }); test("does not invoke sentry when options.sentry is omitted (default pass-through)", async () => { const logger = createLogger(); const processor = vi.fn(async () => ({ ok: true })); const { registeredProcessor } = startOneWorker({ logger, createProcessor: () => processor as unknown as TypedProcessor, }); await registeredProcessor?.({ attemptsMade: 0, name: "ping", queueName: queueNames.email, } as never); expect(processor).toHaveBeenCalledTimes(1); }); describe("metrics instrumentation", () => { function createMetricsStub() { const samples = { completed: [] as Array>, failed: [] as Array>, duration: [] as Array<{ labels: Record; value: number }>, }; const metrics = { queueJobsCompletedTotal: { labels: (labels: Record) => ({ inc: () => { samples.completed.push(labels); }, }), }, queueJobsFailedTotal: { labels: (labels: Record) => ({ inc: () => { samples.failed.push(labels); }, }), }, queueJobDurationSeconds: { labels: (labels: Record) => ({ observe: (value: number) => { samples.duration.push({ labels, value }); }, }), }, }; return { metrics, samples }; } test("records completion and duration while leaving queue-depth gauges to the scheduler", async () => { const logger = createLogger(); const { metrics, samples } = createMetricsStub(); let registered: TypedProcessor | undefined; const captureProcessor = vi.fn((queueName: Name, processor: TypedProcessor) => { registered = processor as TypedProcessor; const worker = createMockWorker(queueName); return worker; }) as TestCreateWorker; let tick = 0; startWorkerRuntime({ createProcessor: () => vi.fn(async () => { return undefined; }) as TypedProcessor, createWorkerImpl: captureProcessor, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv }, logger, queueNames: [queueNames.email], metrics: metrics as never, now: () => { const next = tick; tick += 250; return next; }, }); await registered?.({ attemptsMade: 0, name: "send-mail", queueName: queueNames.email, } as never); expect("queueJobsActive" in metrics).toBe(false); expect(samples.completed).toEqual([{ queue: queueNames.email, job_name: "send-mail" }]); expect(samples.failed).toEqual([]); expect(samples.duration).toEqual([{ labels: { queue: queueNames.email, job_name: "send-mail" }, value: 0.25 }]); }); test("records failed counter and duration on throw", async () => { const logger = createLogger(); const { metrics, samples } = createMetricsStub(); let registered: TypedProcessor | undefined; const createWorkerImpl = vi.fn((queueName: Name, processor: TypedProcessor) => { registered = processor as TypedProcessor; return createMockWorker(queueName); }) as TestCreateWorker; class TimeoutError extends Error { override name = "TimeoutError"; } startWorkerRuntime({ createProcessor: () => vi.fn(async () => { throw new TimeoutError("oh no"); }) as TypedProcessor, createWorkerImpl, env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv }, logger, queueNames: [queueNames.email], metrics: metrics as never, }); await expect( registered?.({ attemptsMade: 0, name: "send-mail", queueName: queueNames.email, } as never) ).rejects.toBeInstanceOf(TimeoutError); expect(samples.completed).toEqual([]); expect(samples.failed).toEqual([{ queue: queueNames.email, job_name: "send-mail", error_type: "TimeoutError" }]); expect(samples.duration.length).toBe(1); }); }); });