Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/worker/src/runtime.test.ts
T

1286 lines
50 KiB
TypeScript

import { createHash } from "node:crypto";
import { WorkerConfigError } from "@hivegrid/config";
import {
bindEncryptedPortalSetupMailToOutbox,
hashPortalSetupMailContent,
materializePortalSetupMailContent,
PORTAL_SETUP_MAIL_SOURCE,
PORTAL_SETUP_TOKEN_PLACEHOLDER,
} from "@hivegrid/db/portal-setup-mail-contract.server";
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
import type { ErrorReporter } from "@hivegrid/error-reporter";
import { describe, expect, test, vi } from "vitest";
import { queueNames, type QueueName, type TypedProcessor } from "@hivegrid/queue";
import {
shutdownWorkerRuntime,
startWorkerRuntime,
getWorkerQueueNames,
workerQueueNames,
type StartWorkerRuntimeOptions,
type WorkerRuntimeLogger,
type WorkerRuntimeWorker,
} from "./runtime.js";
const mailjetEnv = {
MAILJET_API_KEY: "mailer-key",
MAILJET_SECRET_KEY: "mailer-secret",
MAILJET_FROM_EMAIL: "worker@example.com",
};
const databaseEnv = {
DATABASE_URL: "postgresql://localhost/app",
};
const adminConnectionEnv = {
ADMIN_CONNECTION_MASTER_KEY: "admin-connection-key-with-at-least-32-characters",
};
const tenantHardDeleteStorageEnv = {
STORAGE_ACCESS_KEY_ID: "access-key",
STORAGE_BUCKET: "exports",
STORAGE_ENDPOINT: "https://storage.example.com",
STORAGE_REGION: "eu-central-1",
STORAGE_SECRET_ACCESS_KEY: "secret-key",
};
const portalGdprExportStorageEnv = {
...tenantHardDeleteStorageEnv,
GDPR_EXPORT_SSE_C_KEY_BASE64: "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=",
};
const workerGroupEnv = {
WORKER_QUEUE_GROUP: "general",
};
const tenantHardDeleteSearchEnv = {
MEILI_API_KEY: "test-meili-key",
MEILI_URL: "http://meili.internal.test:7700",
};
function createPortalSetupJobData(token: string, tokenId: string) {
const subject = "Kundenportal-Zugang einrichten";
const text = `https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}`;
const html = `<a href="https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}">Einrichten</a>`;
const encrypted = {
kind: "mail.transactional",
tenantId: "tenant_1",
to: ["customer@example.test"],
subject,
text,
html,
metadata: {
source: PORTAL_SETUP_MAIL_SOURCE,
invitationSource: "portal_admin_invite",
portalId: "portal_1",
portalUserId: "portal_user_1",
portalTokenId: tokenId,
setupTokenHash: createHash("sha256").update(token).digest("hex"),
expiresAt: "2026-07-25T12:00:00.000Z",
mailContentHash: hashPortalSetupMailContent({ recipient: "customer@example.test", subject, text, html }),
encryptedCapability: "v1.encrypted-capability",
},
};
return materializePortalSetupMailContent(
bindEncryptedPortalSetupMailToOutbox(encrypted, {
outboxEventId: `outbox_${tokenId}`,
dispatchKey: `portal-setup:${tokenId}`,
}),
token
);
}
function createLogger() {
return { error: vi.fn(), info: vi.fn(), log: vi.fn(), warn: vi.fn() };
}
function createKwpHandlerDependencies() {
return {
tenantMaintenanceLeaseStore: {} as never,
syncKwpProjectBinding: vi.fn(),
syncKwpProjectCatalog: vi.fn(),
enqueueKwpProjectCatalogContinuation: vi.fn(),
syncKwpProjectInbound: vi.fn(),
syncKwpArchiveDiscovery: vi.fn(),
enqueueKwpArchiveDiscovery: vi.fn(),
enqueueKwpArchiveFileJobs: vi.fn(),
syncKwpArchiveFile: vi.fn(),
runKwpContractSentinel: vi.fn(),
};
}
function createChildLogger() {
const childLoggers: Array<ReturnType<typeof createLogger>> = [];
const logger = {
...createLogger(),
child: vi.fn(() => {
const child = createLogger();
childLoggers.push(child);
return child;
}),
};
return { childLoggers, logger };
}
type MockWorkerJob = {
attemptsMade?: number;
data?: unknown;
id?: string | number;
name: string;
opts?: { attempts?: number; backoff?: number | { delay?: number; type?: string } };
queueName?: string;
updateData?: (data: never) => Promise<void>;
};
type MockWorker = WorkerRuntimeWorker & {
emitCompleted(job: MockWorkerJob): void;
emitFailed(job: MockWorkerJob | undefined, error: Error): void;
};
type TestCreateWorker = NonNullable<StartWorkerRuntimeOptions["createWorkerImpl"]>;
function createMockWorker(name: QueueName, close = vi.fn(async () => undefined)): MockWorker {
const listeners = new Map<string, Function>();
const worker: MockWorker = {
close,
name,
on: vi.fn((event: "completed" | "failed", listener: Function) => {
listeners.set(event, listener);
return worker;
}),
emitCompleted: (job) => listeners.get("completed")?.(job),
emitFailed: (job, error) => listeners.get("failed")?.(job, error),
};
return worker;
}
function startOneWorker(options: {
createProcessor?: StartWorkerRuntimeOptions["createProcessor"];
logger: WorkerRuntimeLogger;
recordFailedJob?: StartWorkerRuntimeOptions["recordFailedJob"];
reporter?: ErrorReporter;
sentry?: StartWorkerRuntimeOptions["sentry"];
}) {
const workers: MockWorker[] = [];
let registeredProcessor: TypedProcessor<QueueName> | undefined;
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name, processor: TypedProcessor<Name>) => {
registeredProcessor = processor as TypedProcessor<QueueName>;
const worker = createMockWorker(queueName);
workers.push(worker);
return worker;
}) as TestCreateWorker;
const runtime = startWorkerRuntime({
createProcessor: options.createProcessor ?? ((queueName) => vi.fn(async () => ({ queueName })) as TypedProcessor<typeof queueName>),
createWorkerImpl,
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
logger: options.logger,
queueNames: [queueNames.email],
recordFailedJob: options.recordFailedJob,
reporter: options.reporter,
sentry: options.sentry,
});
return { createWorkerImpl, registeredProcessor, runtime, worker: workers[0] };
}
async function flushMicrotasks() {
await new Promise((resolve) => setTimeout(resolve, 0));
}
describe("worker runtime", () => {
test("wires handler diagnostics into the structured runtime logger", () => {
const logger = createLogger();
let handlerLogger: { info: (fields: Record<string, unknown>, message: string) => void } | undefined;
startOneWorker({
logger,
createProcessor: (queueName, dependencies) => {
handlerLogger = dependencies.logger;
return vi.fn(async () => ({ queueName })) as TypedProcessor<typeof queueName>;
},
});
handlerLogger?.info({ reason: "provider_outage", connectionId: "kwp_1" }, "integrations.token-refresh.provider-failed");
expect(logger.info).toHaveBeenCalledWith(
{ reason: "provider_outage", connectionId: "kwp_1" },
"integrations.token-refresh.provider-failed",
);
});
test("binds each registered processor to its queue payload tenant", async () => {
const contexts = new Map<string, ReturnType<typeof getRlsExecutionContext>>();
const { registeredProcessor } = startOneWorker({
logger: createLogger(),
createProcessor: () =>
(async (job) => {
await Promise.resolve();
const tenantId = (job.data as { tenantId?: string }).tenantId ?? "system";
contexts.set(tenantId, getRlsExecutionContext());
}) as TypedProcessor<QueueName>,
});
await Promise.all([
registeredProcessor?.({
name: "send-transactional-email",
data: { kind: "mail.transactional", tenantId: "tenant-a", userId: "user-a", to: ["a@example.test"] },
}),
registeredProcessor?.({
name: "send-transactional-email",
data: { kind: "mail.transactional", tenantId: "tenant-b", userId: "user-b", to: ["b@example.test"] },
}),
registeredProcessor?.({
name: "deliver-webhook",
data: { kind: "webhook.deliver", url: "https://example.test/hook", eventName: "system.event" },
}),
]);
expect(contexts.get("tenant-a")).toEqual({ tenantId: "tenant-a" });
expect(contexts.get("tenant-b")).toEqual({ tenantId: "tenant-b" });
expect(contexts.get("system")).toBeNull();
expect(getRlsExecutionContext()).toBeNull();
});
test("returns disabled status without resolving Redis or creating workers", () => {
const logger = createLogger();
const createWorkerImpl = vi.fn();
const runtime = startWorkerRuntime({ createWorkerImpl, env: { WORKER_ENABLED: "false" }, logger });
expect(runtime).toMatchObject({ configured: false, enabled: false, status: "disabled", workers: [] });
expect(runtime.queueNames).toEqual(workerQueueNames);
expect(createWorkerImpl).not.toHaveBeenCalled();
expect(logger.info).toHaveBeenCalledWith({ queueNames: workerQueueNames }, "Worker disabled; queues not started.");
});
test("rejects enabled startup without Redis using centralized redacted diagnostics", () => {
expect(() => startWorkerRuntime({ env: { ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv }, logger: createLogger() })).toThrow(WorkerConfigError);
});
test("rejects enabled startup without DATABASE_URL using centralized redacted diagnostics", () => {
expect(() =>
startWorkerRuntime({
createWorkerImpl: vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName)) as TestCreateWorker,
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...mailjetEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
logger: createLogger(),
})
).toThrow(WorkerConfigError);
});
test("starts webhook-capable queue workers without Mailjet config", () => {
const logger = createLogger();
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
const runtime = startWorkerRuntime({
createWorkerImpl,
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", WORKER_QUEUE_GROUP: "webhook" },
logger,
queueNames: [queueNames.email],
});
expect(runtime.status).toBe("started");
expect(createWorkerImpl).toHaveBeenCalledWith(queueNames.email, expect.any(Function), expect.objectContaining({ concurrency: 1 }));
});
test("registers one worker per queue and logs startup", () => {
const logger = createLogger();
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
const runtime = startWorkerRuntime({
createWorkerImpl,
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, ...mailjetEnv, WORKER_ENABLED: "true", WORKER_CONCURRENCY: "3", ...workerGroupEnv },
logger,
});
expect(runtime.status).toBe("started");
expect(runtime.queueNames).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
expect(workerQueueNames).toEqual([queueNames.email, queueNames.notifications]);
expect(workerQueueNames).not.toContain(queueNames.exports);
expect(workerQueueNames).not.toContain(queueNames.browserAutomation);
expect(createWorkerImpl).toHaveBeenCalledTimes(runtime.queueNames.length);
expect(vi.mocked(createWorkerImpl).mock.calls.map(([queueName]) => queueName)).toEqual(runtime.queueNames);
expect(logger.info).toHaveBeenCalledWith({ workerNames: runtime.queueNames }, "Worker started.");
});
test("adds the maintenance queue when export retention processing is explicitly enabled", () => {
const logger = createLogger();
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName)) as TestCreateWorker;
const runtime = startWorkerRuntime({
createWorkerImpl,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
EXPORT_RETENTION_QUEUE_ENABLED: "true",
...mailjetEnv,
STORAGE_ACCESS_KEY_ID: "access-key",
STORAGE_BUCKET: "exports",
STORAGE_ENDPOINT: "https://storage.example.com",
STORAGE_REGION: "us-east-1",
STORAGE_SECRET_ACCESS_KEY: "secret-key",
WORKER_ENABLED: "true",
...workerGroupEnv,
},
logger,
});
expect(runtime.queueNames).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
expect(getWorkerQueueNames({ EXPORT_RETENTION_QUEUE_ENABLED: "true", WORKER_QUEUE_GROUP: "general" })).toEqual([
queueNames.notifications,
queueNames.maintenance,
queueNames.fileScan,
queueNames.filePreview,
]);
expect(vi.mocked(createWorkerImpl).mock.calls.map(([queueName]) => queueName)).toEqual(runtime.queueNames);
});
test("keeps KWP synchronization active while generic import runtime gates are disabled", () => {
expect(getWorkerQueueNames({ WORKER_QUEUE_GROUP: "import" })).toEqual([queueNames.kwpImports]);
expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]);
expect(getWorkerQueueNames({ IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]);
expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([
queueNames.kwpImports,
queueNames.imports,
queueNames.exports,
]);
});
test("starts KWP plus exports with portal and privacy GDPR services in dedicated mode", () => {
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
const runtime = startWorkerRuntime({
createWorkerImpl,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
...portalGdprExportStorageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
extraHandlerDependencies: {
...createKwpHandlerDependencies(),
portalGdprExportService: { runExport: vi.fn() },
privacyGdprExportService: {
withTenantBarrier: async (_payload, writer) => writer(async () => undefined),
resolveReady: vi.fn(),
resolveReclaimableProcessing: vi.fn(),
createAttempt: vi.fn(),
runExport: vi.fn(),
deleteArtifact: vi.fn(),
terminalizeReadyAfterQuiescence: vi.fn(),
},
},
logger: createLogger(),
});
expect(runtime.queueNames).toEqual([queueNames.kwpImports, queueNames.exports]);
expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([queueNames.kwpImports, queueNames.exports]);
});
test("fails closed when dedicated GDPR export services are not wired", () => {
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
expect(() =>
startWorkerRuntime({
createWorkerImpl: createWorkerImplMock as TestCreateWorker,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
...portalGdprExportStorageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
extraHandlerDependencies: createKwpHandlerDependencies(),
logger: createLogger(),
})
).toThrow(/KWP and portal\/privacy GDPR queues requires wired production services: missing portalGdprExportService, privacyGdprExportService/u);
expect(createWorkerImplMock).not.toHaveBeenCalled();
});
test("fails closed when the dedicated export gate is assigned to another worker group", () => {
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
expect(() =>
startWorkerRuntime({
createWorkerImpl: createWorkerImplMock as TestCreateWorker,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
...portalGdprExportStorageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "general",
},
extraHandlerDependencies: createKwpHandlerDependencies(),
logger: createLogger(),
})
).toThrow(/PORTAL_GDPR_EXPORT_QUEUE_ENABLED:invalid-worker-queue-group/u);
expect(createWorkerImplMock).not.toHaveBeenCalled();
});
test("rejects import worker startup without wired import/export services", () => {
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
expect(() =>
startWorkerRuntime({
createWorkerImpl,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
IMPORT_QUEUE_ENABLED: "true",
IMPORT_QUEUE_RUNTIME_READY: "true",
...portalGdprExportStorageEnv,
...mailjetEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
extraHandlerDependencies: createKwpHandlerDependencies(),
logger: createLogger(),
})
).toThrow(
/missing importService, scanService, adminSalesDataService, exportService/u
);
expect(createWorkerImplMock).not.toHaveBeenCalled();
});
test("rejects full import startup without the GDPR SSE-C key before queue ownership", () => {
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
const { GDPR_EXPORT_SSE_C_KEY_BASE64: _key, ...storageWithoutKey } = portalGdprExportStorageEnv;
expect(() =>
startWorkerRuntime({
createWorkerImpl: createWorkerImplMock as TestCreateWorker,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
IMPORT_QUEUE_ENABLED: "true",
IMPORT_QUEUE_RUNTIME_READY: "true",
...storageWithoutKey,
...mailjetEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
extraHandlerDependencies: {
...createKwpHandlerDependencies(),
exportService: { runExport: vi.fn() },
importService: { runImport: vi.fn() },
portalGdprExportService: { runExport: vi.fn() },
privacyGdprExportService: {
withTenantBarrier: async (_payload, writer) => writer(async () => undefined),
resolveReady: vi.fn(),
resolveReclaimableProcessing: vi.fn(),
createAttempt: vi.fn(),
runExport: vi.fn(),
deleteArtifact: vi.fn(),
terminalizeReadyAfterQuiescence: vi.fn(),
},
scanService: { scanObject: vi.fn() },
},
logger: createLogger(),
})
).toThrow(/GDPR_EXPORT_SSE_C_KEY_BASE64:missing-required/u);
expect(createWorkerImplMock).not.toHaveBeenCalled();
});
test("starts import worker queues only with wired import/export services", () => {
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
const runtime = startWorkerRuntime({
createWorkerImpl,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
IMPORT_QUEUE_ENABLED: "true",
IMPORT_QUEUE_RUNTIME_READY: "true",
...portalGdprExportStorageEnv,
...mailjetEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
extraHandlerDependencies: {
...createKwpHandlerDependencies(),
adminSalesDataService: {
runImport: vi.fn(),
runExport: vi.fn(),
},
exportService: { runExport: vi.fn() },
importService: { runImport: vi.fn() },
portalGdprExportService: { runExport: vi.fn() },
privacyGdprExportService: {
withTenantBarrier: async (_payload, writer) => writer(async () => undefined),
resolveReady: vi.fn(),
resolveReclaimableProcessing: vi.fn(),
createAttempt: vi.fn(),
runExport: vi.fn(),
deleteArtifact: vi.fn(),
terminalizeReadyAfterQuiescence: vi.fn(),
},
scanService: { scanObject: vi.fn() },
},
logger: createLogger(),
});
expect(runtime.status).toBe("started");
expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([
queueNames.kwpImports,
queueNames.imports,
queueNames.exports,
]);
});
test("rejects export retention startup without storage delete configuration", () => {
expect(() =>
startWorkerRuntime({
createWorkerImpl: vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName)) as TestCreateWorker,
env: {
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
EXPORT_RETENTION_QUEUE_ENABLED: "true",
...mailjetEnv,
WORKER_ENABLED: "true",
...workerGroupEnv,
},
logger: createLogger(),
})
).toThrow(/Invalid worker configuration \(STORAGE_ENDPOINT:missing-required/u);
});
test("rejects production general-worker startup before queue ownership without tenant hard-delete storage", () => {
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
expect(() =>
startWorkerRuntime({
createWorkerImpl: createWorkerImpl as TestCreateWorker,
env: {
NODE_ENV: "production",
APP_BASE_DOMAIN: "example.test",
AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes",
...adminConnectionEnv,
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
...mailjetEnv,
WORKER_ENABLED: "true",
...workerGroupEnv,
},
logger: createLogger(),
})
).toThrow(/STORAGE_BUCKET:missing-required/u);
expect(createWorkerImpl).not.toHaveBeenCalled();
});
test("rejects production general-worker queue ownership without the tenant hard-delete service", () => {
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
expect(() =>
startWorkerRuntime({
createWorkerImpl: createWorkerImpl as TestCreateWorker,
env: {
NODE_ENV: "production",
APP_BASE_DOMAIN: "example.test",
AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes",
...adminConnectionEnv,
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
...tenantHardDeleteStorageEnv,
...tenantHardDeleteSearchEnv,
...mailjetEnv,
EXPORT_RETENTION_QUEUE_ENABLED: "true",
WORKER_ENABLED: "true",
...workerGroupEnv,
},
logger: createLogger(),
queueNames: [queueNames.maintenance],
})
).toThrow(/missing privacyTenantDeletionService/u);
expect(createWorkerImpl).not.toHaveBeenCalled();
});
test.each(["MEILI_URL", "MEILI_API_KEY"] as const)(
"rejects production general-worker queue ownership without %s for tenant search cleanup",
(missingName) => {
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
const env = {
NODE_ENV: "production",
APP_BASE_DOMAIN: "example.test",
AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes",
...adminConnectionEnv,
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
...databaseEnv,
...tenantHardDeleteStorageEnv,
...tenantHardDeleteSearchEnv,
...mailjetEnv,
EXPORT_RETENTION_QUEUE_ENABLED: "true",
WORKER_ENABLED: "true",
...workerGroupEnv,
};
delete env[missingName];
expect(() =>
startWorkerRuntime({
createWorkerImpl: createWorkerImpl as TestCreateWorker,
env,
extraHandlerDependencies: { privacyTenantDeletionService: { runScan: vi.fn() } },
logger: createLogger(),
queueNames: [queueNames.maintenance],
})
).toThrow(new RegExp(`missing ${missingName}`, "u"));
expect(createWorkerImpl).not.toHaveBeenCalled();
}
);
test("binds requestId and retry fields through child loggers for completed and failed jobs", async () => {
const { childLoggers, logger } = createChildLogger();
const recordFailedJob = vi.fn(async () => undefined);
const { worker } = startOneWorker({ logger, recordFailedJob });
const failure = new Error("handler failed");
worker.emitCompleted({
attemptsMade: 1,
data: { metadata: { requestId: "req_completed" } },
id: "job_1",
name: "send-transactional-email",
opts: { attempts: 3, backoff: 500 },
queueName: queueNames.email,
});
worker.emitFailed(
{
attemptsMade: 3,
data: { metadata: { requestId: "req_failed" } },
id: "job_2",
name: "deliver-webhook",
opts: { attempts: 3, backoff: { delay: 2_000, type: "exponential" } },
queueName: queueNames.email,
},
failure
);
await flushMicrotasks();
expect(logger.child).toHaveBeenCalledWith({
attemptsLimit: 3,
attemptsMade: 1,
backoffDelay: 500,
backoffType: "fixed",
jobId: "job_1",
jobName: "send-transactional-email",
queueName: queueNames.email,
requestId: "req_completed",
});
expect(logger.child).toHaveBeenCalledWith(expect.objectContaining({ jobId: "job_2", requestId: "req_failed", backoffType: "exponential" }));
expect(childLoggers[0]?.info).toHaveBeenCalledWith({}, "Worker job completed.");
expect(childLoggers[1]?.error).toHaveBeenCalledWith({ error: expect.objectContaining({ message: failure.message }) }, "Worker job failed.");
expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ jobId: "job_2" }));
});
test("falls back to structured calls without child and ignores invalid requestId metadata", () => {
const logger = createLogger();
const { worker } = startOneWorker({ logger });
worker.emitCompleted({
data: { metadata: { requestId: 12345 } },
id: "job-no-child",
name: "deliver-webhook",
queueName: queueNames.email,
});
expect(logger.info).toHaveBeenCalledWith(
{ attemptsLimit: null, attemptsMade: 0, jobId: "job-no-child", jobName: "deliver-webhook", queueName: queueNames.email },
"Worker job completed."
);
});
test("redacts completed OWNER mail payloads while failed attempts retain the token for retry", async () => {
const token = "owner-capability-token-fixture";
const data = {
kind: "mail.transactional",
tenantId: "tenant_1",
to: ["owner@example.test"],
html: `<a href="https://app.example.test/invitation/${token}">Accept</a>`,
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_1",
invitationTokenHash: "a".repeat(64),
issuanceKeyHash: "b".repeat(64),
},
};
const completedUpdateData = vi.fn(async (_data: unknown) => undefined);
const failedUpdateData = vi.fn(async (_data: unknown) => undefined);
const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob: vi.fn(async () => undefined) });
worker.emitFailed(
{
attemptsMade: 1,
data,
id: "owner-mail-retry",
name: "send-transactional-email",
opts: { attempts: 3 },
queueName: queueNames.email,
updateData: failedUpdateData,
},
new Error("mail provider unavailable")
);
worker.emitCompleted({
attemptsMade: 1,
data,
id: "owner-mail-complete",
name: "send-transactional-email",
opts: { attempts: 3 },
queueName: queueNames.email,
updateData: completedUpdateData,
});
await vi.waitFor(() => expect(completedUpdateData).toHaveBeenCalledOnce());
expect(failedUpdateData).not.toHaveBeenCalled();
expect(JSON.stringify(data)).toContain(token);
expect(JSON.stringify(completedUpdateData.mock.calls[0]?.[0])).not.toContain(token);
expect(completedUpdateData.mock.calls[0]?.[0]).toMatchObject({
metadata: expect.objectContaining({ source: "workspace-owner-invitation", secretBodyRedacted: true }),
});
});
test("redacts a terminal failed OWNER job so the Redis DLQ cannot retain its capability", async () => {
const token = "terminal-owner-capability-token-fixture";
const data = {
kind: "mail.transactional",
tenantId: "tenant_1",
to: ["owner@example.test"],
html: `<a href="https://app.example.test/invitation/${token}">Accept</a>`,
metadata: {
source: "workspace-owner-invitation",
invitationId: "invite_terminal",
invitationTokenHash: "a".repeat(64),
issuanceKeyHash: "b".repeat(64),
},
};
const updateData = vi.fn(async (_data: unknown) => undefined);
const recordFailedJob = vi.fn(async () => undefined);
const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob });
worker.emitFailed(
{
attemptsMade: 3,
data,
id: "owner-mail-terminal",
name: "send-transactional-email",
opts: { attempts: 3 },
queueName: queueNames.email,
updateData,
},
new Error("mail provider permanently unavailable")
);
await vi.waitFor(() => expect(updateData).toHaveBeenCalledOnce());
expect(JSON.stringify(updateData.mock.calls[0]?.[0])).not.toContain(token);
expect(updateData.mock.calls[0]?.[0]).toMatchObject({
metadata: expect.objectContaining({ secretBodyRedacted: true, ownerInvitationDeliveryStatus: "rotation_required" }),
});
expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ jobId: "owner-mail-terminal" }));
});
test("redacts completed and terminal portal setup jobs so Redis retains no setup capability", async () => {
const completedToken = "c".repeat(43);
const failedToken = "f".repeat(43);
const completedUpdateData = vi.fn(async (_data: unknown) => undefined);
const failedUpdateData = vi.fn(async (_data: unknown) => undefined);
const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob: vi.fn(async () => undefined) });
worker.emitCompleted({
attemptsMade: 0,
data: createPortalSetupJobData(completedToken, "portal_token_completed"),
id: "portal-setup-completed",
name: "send-transactional-email",
opts: { attempts: 3 },
queueName: queueNames.email,
updateData: completedUpdateData,
});
worker.emitFailed(
{
attemptsMade: 3,
data: createPortalSetupJobData(failedToken, "portal_token_failed"),
id: "portal-setup-failed",
name: "send-transactional-email",
opts: { attempts: 3 },
queueName: queueNames.email,
updateData: failedUpdateData,
},
new Error("provider unavailable")
);
await vi.waitFor(() => expect(completedUpdateData).toHaveBeenCalledOnce());
await vi.waitFor(() => expect(failedUpdateData).toHaveBeenCalledOnce());
expect(JSON.stringify(completedUpdateData.mock.calls)).not.toContain(completedToken);
expect(JSON.stringify(failedUpdateData.mock.calls)).not.toContain(failedToken);
expect(completedUpdateData.mock.calls[0]?.[0]).toMatchObject({
metadata: expect.objectContaining({ secretBodyRedacted: true }),
});
expect(failedUpdateData.mock.calls[0]?.[0]).toMatchObject({
metadata: expect.objectContaining({ secretBodyRedacted: true, portalSetupDeliveryStatus: "rotation_required" }),
});
});
test("logs retry diagnostics without leaking payload data or persisting non-terminal failures", () => {
const logger = createLogger();
const recordFailedJob = vi.fn(async () => undefined);
const { worker } = startOneWorker({ logger, recordFailedJob });
const failure = new Error("remote service unavailable");
worker.emitFailed(
{
attemptsMade: 2,
data: { body: "fixture-body-secret", headers: { authorization: "Bearer token-secret" }, metadata: { requestId: "req_retry" } },
id: "webhook:tenant_1:event_1",
name: "deliver-webhook",
opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } },
queueName: queueNames.email,
},
failure
);
expect(logger.error).toHaveBeenCalledWith(
expect.objectContaining({ error: expect.objectContaining({ message: "remote service unavailable" }), requestId: "req_retry", attemptsLimit: 5 }),
"Worker job failed."
);
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("fixture-body-secret");
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("token-secret");
expect(recordFailedJob).not.toHaveBeenCalled();
});
test("reports failed jobs with correlation fields and redacted payload secrets", async () => {
const logger = createLogger();
const reporter: ErrorReporter = { report: vi.fn() };
const recordFailedJob = vi.fn(async () => undefined);
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
const failure = new Error("remote service unavailable token=runtime-secret Authorization: Bearer auth-secret");
Object.assign(failure, { token: "runtime-secret" });
worker.emitFailed(
{
attemptsMade: 2,
data: {
body: "fixture-body-secret",
headers: { authorization: "Bearer auth-secret" },
metadata: { requestId: "req_reported_retry" },
recipient: "private@example.com",
templateData: { token: "template-secret" },
},
id: "job-reporter-1",
name: "deliver-webhook",
opts: { attempts: 5, backoff: { delay: 2_000, type: "exponential" } },
queueName: queueNames.email,
},
failure
);
await flushMicrotasks();
expect(reporter.report).toHaveBeenCalledTimes(1);
expect(reporter.report).toHaveBeenCalledWith(
expect.objectContaining({
component: "worker",
operation: "job.failed",
severity: "error",
queueName: queueNames.email,
jobName: "deliver-webhook",
jobId: "job-reporter-1",
requestId: "req_reported_retry",
attemptsMade: 2,
attemptsLimit: 5,
retryCount: 2,
backoffDelay: 2000,
backoffType: "exponential",
error: expect.objectContaining({
errorName: "Error",
errorMessage: expect.stringContaining("remote service unavailable"),
}),
})
);
const serializedReport = JSON.stringify(vi.mocked(reporter.report).mock.calls);
expect(serializedReport).not.toContain("fixture-body-secret");
expect(serializedReport).not.toContain("auth-secret");
expect(serializedReport).not.toContain("private@example.com");
expect(serializedReport).not.toContain("template-secret");
expect(serializedReport).not.toContain("runtime-secret");
expect(recordFailedJob).not.toHaveBeenCalled();
});
test("persists terminal failures and logs persistence errors without throwing", async () => {
const logger = createLogger();
const persistenceError = new Error("database unavailable");
Object.assign(persistenceError, { token: "persistence-secret-token" });
const reporter: ErrorReporter = { report: vi.fn() };
const recordFailedJob = vi.fn(async () => {
throw persistenceError;
});
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
expect(() =>
worker.emitFailed(
{
attemptsMade: 1,
data: { token: "persist-secret" },
id: "job-persist-error",
name: "deliver-webhook",
opts: { attempts: 1 },
queueName: queueNames.email,
},
new TypeError("terminal failure")
)
).not.toThrow();
await flushMicrotasks();
expect(recordFailedJob).toHaveBeenCalledWith(
expect.objectContaining({ attemptsLimit: 1, errorName: "TypeError", errorMessage: "terminal failure", jobId: "job-persist-error" })
);
expect(logger.error).toHaveBeenCalledWith(
expect.objectContaining({ error: expect.objectContaining({ message: "database unavailable" }), jobId: "job-persist-error" }),
"Failed to persist terminal worker failure."
);
expect(reporter.report).toHaveBeenCalledWith(expect.objectContaining({ operation: "job.failed", jobId: "job-persist-error", requestId: undefined }));
expect(reporter.report).toHaveBeenCalledWith(
expect.objectContaining({
component: "worker",
operation: "job.persistence_failed",
jobId: "job-persist-error",
error: expect.objectContaining({ errorName: "Error", errorMessage: "database unavailable" }),
})
);
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("persist-secret");
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("persist-secret");
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("persistence-secret-token");
});
test("logs handler failures with requestId and rethrows processor errors", async () => {
const logger = createLogger();
const handlerError = new Error("handler exploded");
const reporter: ErrorReporter = { report: vi.fn() };
const { registeredProcessor } = startOneWorker({
createProcessor: () =>
vi.fn(async () => {
throw handlerError;
}) as TypedProcessor<QueueName>,
logger,
reporter,
});
await expect(
registeredProcessor?.({
attemptsMade: 1,
data: { metadata: { requestId: "req_handler" }, templateData: "fixture-secret", to: "private@example.com" },
id: "job_3",
name: "deliver-webhook",
opts: { attempts: 3, backoff: 750 },
} as unknown as Parameters<TypedProcessor<QueueName>>[0])
).rejects.toThrow(handlerError);
expect(logger.error).toHaveBeenCalledWith(
expect.objectContaining({ error: expect.objectContaining({ message: "handler exploded" }), requestId: "req_handler", backoffType: "fixed" }),
"Worker handler failed."
);
expect(reporter.report).toHaveBeenCalledWith(
expect.objectContaining({
component: "worker",
operation: "handler.failed",
severity: "error",
requestId: "req_handler",
jobId: "job_3",
retryCount: 1,
error: expect.objectContaining({ errorName: "Error" }),
})
);
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("fixture-secret");
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("private@example.com");
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("fixture-secret");
expect(JSON.stringify(vi.mocked(reporter.report).mock.calls)).not.toContain("private@example.com");
});
test("shutdown closes workers and reports close rejections or success as structured records", async () => {
const logger = createLogger();
const closeError = new Error("close failed");
const reporter: ErrorReporter = { report: vi.fn() };
const emailWorker = createMockWorker(queueNames.email);
const notificationWorker = createMockWorker(
queueNames.notifications,
vi.fn(async () => {
throw closeError;
})
);
await expect(shutdownWorkerRuntime({ workers: [emailWorker, notificationWorker] }, { logger, reason: "SIGTERM", reporter })).rejects.toThrow(
"Failed to close workers: hivegrid.notifications."
);
expect(logger.error).toHaveBeenCalledWith(
{ error: expect.objectContaining({ message: "close failed" }), reason: "SIGTERM", workerName: queueNames.notifications },
"Failed to close worker."
);
expect(reporter.report).toHaveBeenCalledWith(
expect.objectContaining({
component: "worker",
operation: "shutdown.close_failed",
severity: "error",
queueName: queueNames.notifications,
workerName: queueNames.notifications,
reason: "SIGTERM",
error: expect.objectContaining({ errorName: "Error" }),
})
);
await expect(shutdownWorkerRuntime({ workers: [emailWorker] }, { logger, reason: "SIGINT", reporter })).resolves.toEqual({
closedWorkers: [queueNames.email],
});
expect(logger.info).toHaveBeenCalledWith({ closedWorkers: [queueNames.email], reason: "SIGINT" }, "Worker shutdown complete.");
});
test("swallows reporter failures without changing retry or shutdown behavior", async () => {
const logger = createLogger();
const reporter: ErrorReporter = {
report: vi.fn(async () => {
throw new Error("reporter failed");
}),
};
const recordFailedJob = vi.fn(async () => undefined);
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
expect(() =>
worker.emitFailed(
{
attemptsMade: 1,
data: { metadata: { requestId: "req_reporter_failure" } },
id: "job-report-fail",
name: "deliver-webhook",
opts: { attempts: 3 },
queueName: queueNames.email,
},
new Error("retry failure")
)
).not.toThrow();
await flushMicrotasks();
expect(recordFailedJob).not.toHaveBeenCalled();
const closingWorker = createMockWorker(
queueNames.email,
vi.fn(async () => {
throw new Error("close failed");
})
);
await expect(shutdownWorkerRuntime({ workers: [closingWorker] }, { logger, reporter })).rejects.toThrow("Failed to close workers: hivegrid.email.");
});
test("wraps each job in a Sentry queue.process span with queue/job/tenant tags", async () => {
type Span = { args: { op: string; name: string; attributes?: Record<string, unknown> }; status?: { code: number; message?: string } };
const spans: Span[] = [];
const captureException = vi.fn();
const sentry = {
captureException,
async startSpan<T>(
args: { op: string; name: string; attributes?: Record<string, unknown> },
callback: (s: { setAttribute(k: string, v: unknown): void; setStatus(s: { code: number; message?: string }): void }) => Promise<T> | T
): Promise<T> {
const record: Span = { args, status: undefined };
spans.push(record);
return callback({
setAttribute() {
/* no-op for assertion brevity */
},
setStatus(s) {
record.status = s;
},
});
},
};
const logger = createLogger();
const processor = vi.fn(async () => ({ ok: true }));
const { registeredProcessor } = startOneWorker({
logger,
sentry,
createProcessor: () => processor as unknown as TypedProcessor<QueueName>,
});
expect(registeredProcessor).toBeDefined();
await registeredProcessor?.({
attemptsMade: 0,
data: { tenantId: "tnt_42" },
id: "job-abc",
name: "deliver-webhook",
opts: { attempts: 3 },
queueName: queueNames.email,
} as never);
expect(spans).toHaveLength(1);
expect(spans[0].args.op).toBe("queue.process");
expect(spans[0].args.name).toBe(`${queueNames.email}/deliver-webhook`);
expect(spans[0].args.attributes?.["messaging.destination.name"]).toBe(queueNames.email);
expect(spans[0].args.attributes?.["messaging.bullmq.job.name"]).toBe("deliver-webhook");
expect(spans[0].args.attributes?.["messaging.bullmq.attempts_made"]).toBe(0);
expect(spans[0].args.attributes?.["messaging.bullmq.attempts_limit"]).toBe(3);
expect(spans[0].args.attributes?.["hivegrid.tenant_id"]).toBe("tnt_42");
expect(spans[0].status?.code).toBe(1);
expect(captureException).not.toHaveBeenCalled();
});
test("captures handler exceptions via Sentry.captureException with queue/job/tenant tags", async () => {
const spans: Array<{ status?: { code: number; message?: string } }> = [];
const captureException = vi.fn();
const sentry = {
captureException,
async startSpan<T>(
_args: unknown,
callback: (s: { setAttribute(k: string, v: unknown): void; setStatus(s: { code: number; message?: string }): void }) => Promise<T> | T
): Promise<T> {
const record: { status?: { code: number; message?: string } } = {};
spans.push(record);
return callback({
setAttribute() {
/* no-op */
},
setStatus(s) {
record.status = s;
},
});
},
};
const logger = createLogger();
const failing = vi.fn(async () => {
throw new Error("kaboom");
});
const { registeredProcessor } = startOneWorker({
logger,
sentry,
createProcessor: () => failing as unknown as TypedProcessor<QueueName>,
});
await expect(
registeredProcessor?.({
attemptsMade: 1,
data: { metadata: { tenantId: "tnt_meta" } },
id: 7,
name: "send-email",
opts: { attempts: 2 },
queueName: queueNames.email,
} as never)
).rejects.toThrow("kaboom");
expect(spans).toHaveLength(1);
expect(spans[0].status?.code).toBe(2);
expect(captureException).toHaveBeenCalledTimes(1);
const [thrown, ctx] = captureException.mock.calls[0] ?? [];
expect((thrown as Error).message).toBe("kaboom");
const tags = (ctx as { tags: Record<string, string> } | undefined)?.tags;
expect(tags?.component).toBe("worker");
expect(tags?.queue).toBe(queueNames.email);
expect(tags?.job_name).toBe("send-email");
expect(tags?.job_id).toBe("7");
expect(tags?.tenant_id).toBe("tnt_meta");
});
test("does not invoke sentry when options.sentry is omitted (default pass-through)", async () => {
const logger = createLogger();
const processor = vi.fn(async () => ({ ok: true }));
const { registeredProcessor } = startOneWorker({
logger,
createProcessor: () => processor as unknown as TypedProcessor<QueueName>,
});
await registeredProcessor?.({
attemptsMade: 0,
name: "ping",
queueName: queueNames.email,
} as never);
expect(processor).toHaveBeenCalledTimes(1);
});
describe("metrics instrumentation", () => {
function createMetricsStub() {
const samples = {
completed: [] as Array<Record<string, string>>,
failed: [] as Array<Record<string, string>>,
duration: [] as Array<{ labels: Record<string, string>; value: number }>,
};
const metrics = {
queueJobsCompletedTotal: {
labels: (labels: Record<string, string>) => ({
inc: () => {
samples.completed.push(labels);
},
}),
},
queueJobsFailedTotal: {
labels: (labels: Record<string, string>) => ({
inc: () => {
samples.failed.push(labels);
},
}),
},
queueJobDurationSeconds: {
labels: (labels: Record<string, string>) => ({
observe: (value: number) => {
samples.duration.push({ labels, value });
},
}),
},
};
return { metrics, samples };
}
test("records completion and duration while leaving queue-depth gauges to the scheduler", async () => {
const logger = createLogger();
const { metrics, samples } = createMetricsStub();
let registered: TypedProcessor<QueueName> | undefined;
const captureProcessor = vi.fn(<Name extends QueueName>(queueName: Name, processor: TypedProcessor<Name>) => {
registered = processor as TypedProcessor<QueueName>;
const worker = createMockWorker(queueName);
return worker;
}) as TestCreateWorker;
let tick = 0;
startWorkerRuntime({
createProcessor: () =>
vi.fn(async () => {
return undefined;
}) as TypedProcessor<QueueName>,
createWorkerImpl: captureProcessor,
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
logger,
queueNames: [queueNames.email],
metrics: metrics as never,
now: () => {
const next = tick;
tick += 250;
return next;
},
});
await registered?.({
attemptsMade: 0,
name: "send-mail",
queueName: queueNames.email,
} as never);
expect("queueJobsActive" in metrics).toBe(false);
expect(samples.completed).toEqual([{ queue: queueNames.email, job_name: "send-mail" }]);
expect(samples.failed).toEqual([]);
expect(samples.duration).toEqual([{ labels: { queue: queueNames.email, job_name: "send-mail" }, value: 0.25 }]);
});
test("records failed counter and duration on throw", async () => {
const logger = createLogger();
const { metrics, samples } = createMetricsStub();
let registered: TypedProcessor<QueueName> | undefined;
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name, processor: TypedProcessor<Name>) => {
registered = processor as TypedProcessor<QueueName>;
return createMockWorker(queueName);
}) as TestCreateWorker;
class TimeoutError extends Error {
override name = "TimeoutError";
}
startWorkerRuntime({
createProcessor: () =>
vi.fn(async () => {
throw new TimeoutError("oh no");
}) as TypedProcessor<QueueName>,
createWorkerImpl,
env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", ...databaseEnv, WORKER_ENABLED: "true", ...workerGroupEnv },
logger,
queueNames: [queueNames.email],
metrics: metrics as never,
});
await expect(
registered?.({
attemptsMade: 0,
name: "send-mail",
queueName: queueNames.email,
} as never)
).rejects.toBeInstanceOf(TimeoutError);
expect(samples.completed).toEqual([]);
expect(samples.failed).toEqual([{ queue: queueNames.email, job_name: "send-mail", error_type: "TimeoutError" }]);
expect(samples.duration.length).toBe(1);
});
});
});