Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
@@ -1,4 +1,14 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { WorkerConfigError } from "@hivegrid/config";
|
||||
import {
|
||||
bindEncryptedPortalSetupMailToOutbox,
|
||||
hashPortalSetupMailContent,
|
||||
materializePortalSetupMailContent,
|
||||
PORTAL_SETUP_MAIL_SOURCE,
|
||||
PORTAL_SETUP_TOKEN_PLACEHOLDER,
|
||||
} from "@hivegrid/db/portal-setup-mail-contract.server";
|
||||
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
|
||||
import type { ErrorReporter } from "@hivegrid/error-reporter";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { queueNames, type QueueName, type TypedProcessor } from "@hivegrid/queue";
|
||||
@@ -20,14 +30,79 @@ const mailjetEnv = {
|
||||
const databaseEnv = {
|
||||
DATABASE_URL: "postgresql://localhost/app",
|
||||
};
|
||||
const adminConnectionEnv = {
|
||||
ADMIN_CONNECTION_MASTER_KEY: "admin-connection-key-with-at-least-32-characters",
|
||||
};
|
||||
const tenantHardDeleteStorageEnv = {
|
||||
STORAGE_ACCESS_KEY_ID: "access-key",
|
||||
STORAGE_BUCKET: "exports",
|
||||
STORAGE_ENDPOINT: "https://storage.example.com",
|
||||
STORAGE_REGION: "eu-central-1",
|
||||
STORAGE_SECRET_ACCESS_KEY: "secret-key",
|
||||
};
|
||||
const portalGdprExportStorageEnv = {
|
||||
...tenantHardDeleteStorageEnv,
|
||||
GDPR_EXPORT_SSE_C_KEY_BASE64: "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=",
|
||||
};
|
||||
const workerGroupEnv = {
|
||||
WORKER_QUEUE_GROUP: "general",
|
||||
};
|
||||
const tenantHardDeleteSearchEnv = {
|
||||
MEILI_API_KEY: "test-meili-key",
|
||||
MEILI_URL: "http://meili.internal.test:7700",
|
||||
};
|
||||
|
||||
function createPortalSetupJobData(token: string, tokenId: string) {
|
||||
const subject = "Kundenportal-Zugang einrichten";
|
||||
const text = `https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}`;
|
||||
const html = `<a href="https://portal.example.test/acme/setup/${PORTAL_SETUP_TOKEN_PLACEHOLDER}">Einrichten</a>`;
|
||||
const encrypted = {
|
||||
kind: "mail.transactional",
|
||||
tenantId: "tenant_1",
|
||||
to: ["customer@example.test"],
|
||||
subject,
|
||||
text,
|
||||
html,
|
||||
metadata: {
|
||||
source: PORTAL_SETUP_MAIL_SOURCE,
|
||||
invitationSource: "portal_admin_invite",
|
||||
portalId: "portal_1",
|
||||
portalUserId: "portal_user_1",
|
||||
portalTokenId: tokenId,
|
||||
setupTokenHash: createHash("sha256").update(token).digest("hex"),
|
||||
expiresAt: "2026-07-25T12:00:00.000Z",
|
||||
mailContentHash: hashPortalSetupMailContent({ recipient: "customer@example.test", subject, text, html }),
|
||||
encryptedCapability: "v1.encrypted-capability",
|
||||
},
|
||||
};
|
||||
return materializePortalSetupMailContent(
|
||||
bindEncryptedPortalSetupMailToOutbox(encrypted, {
|
||||
outboxEventId: `outbox_${tokenId}`,
|
||||
dispatchKey: `portal-setup:${tokenId}`,
|
||||
}),
|
||||
token
|
||||
);
|
||||
}
|
||||
|
||||
function createLogger() {
|
||||
return { error: vi.fn(), info: vi.fn(), log: vi.fn(), warn: vi.fn() };
|
||||
}
|
||||
|
||||
function createKwpHandlerDependencies() {
|
||||
return {
|
||||
tenantMaintenanceLeaseStore: {} as never,
|
||||
syncKwpProjectBinding: vi.fn(),
|
||||
syncKwpProjectCatalog: vi.fn(),
|
||||
enqueueKwpProjectCatalogContinuation: vi.fn(),
|
||||
syncKwpProjectInbound: vi.fn(),
|
||||
syncKwpArchiveDiscovery: vi.fn(),
|
||||
enqueueKwpArchiveDiscovery: vi.fn(),
|
||||
enqueueKwpArchiveFileJobs: vi.fn(),
|
||||
syncKwpArchiveFile: vi.fn(),
|
||||
runKwpContractSentinel: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createChildLogger() {
|
||||
const childLoggers: Array<ReturnType<typeof createLogger>> = [];
|
||||
const logger = {
|
||||
@@ -48,6 +123,7 @@ type MockWorkerJob = {
|
||||
name: string;
|
||||
opts?: { attempts?: number; backoff?: number | { delay?: number; type?: string } };
|
||||
queueName?: string;
|
||||
updateData?: (data: never) => Promise<void>;
|
||||
};
|
||||
|
||||
type MockWorker = WorkerRuntimeWorker & {
|
||||
@@ -105,6 +181,59 @@ async function flushMicrotasks() {
|
||||
}
|
||||
|
||||
describe("worker runtime", () => {
|
||||
test("wires handler diagnostics into the structured runtime logger", () => {
|
||||
const logger = createLogger();
|
||||
let handlerLogger: { info: (fields: Record<string, unknown>, message: string) => void } | undefined;
|
||||
|
||||
startOneWorker({
|
||||
logger,
|
||||
createProcessor: (queueName, dependencies) => {
|
||||
handlerLogger = dependencies.logger;
|
||||
return vi.fn(async () => ({ queueName })) as TypedProcessor<typeof queueName>;
|
||||
},
|
||||
});
|
||||
|
||||
handlerLogger?.info({ reason: "provider_outage", connectionId: "kwp_1" }, "integrations.token-refresh.provider-failed");
|
||||
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
{ reason: "provider_outage", connectionId: "kwp_1" },
|
||||
"integrations.token-refresh.provider-failed",
|
||||
);
|
||||
});
|
||||
|
||||
test("binds each registered processor to its queue payload tenant", async () => {
|
||||
const contexts = new Map<string, ReturnType<typeof getRlsExecutionContext>>();
|
||||
const { registeredProcessor } = startOneWorker({
|
||||
logger: createLogger(),
|
||||
createProcessor: () =>
|
||||
(async (job) => {
|
||||
await Promise.resolve();
|
||||
const tenantId = (job.data as { tenantId?: string }).tenantId ?? "system";
|
||||
contexts.set(tenantId, getRlsExecutionContext());
|
||||
}) as TypedProcessor<QueueName>,
|
||||
});
|
||||
|
||||
await Promise.all([
|
||||
registeredProcessor?.({
|
||||
name: "send-transactional-email",
|
||||
data: { kind: "mail.transactional", tenantId: "tenant-a", userId: "user-a", to: ["a@example.test"] },
|
||||
}),
|
||||
registeredProcessor?.({
|
||||
name: "send-transactional-email",
|
||||
data: { kind: "mail.transactional", tenantId: "tenant-b", userId: "user-b", to: ["b@example.test"] },
|
||||
}),
|
||||
registeredProcessor?.({
|
||||
name: "deliver-webhook",
|
||||
data: { kind: "webhook.deliver", url: "https://example.test/hook", eventName: "system.event" },
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(contexts.get("tenant-a")).toEqual({ tenantId: "tenant-a" });
|
||||
expect(contexts.get("tenant-b")).toEqual({ tenantId: "tenant-b" });
|
||||
expect(contexts.get("system")).toBeNull();
|
||||
expect(getRlsExecutionContext()).toBeNull();
|
||||
});
|
||||
|
||||
test("returns disabled status without resolving Redis or creating workers", () => {
|
||||
const logger = createLogger();
|
||||
const createWorkerImpl = vi.fn();
|
||||
@@ -194,16 +323,93 @@ describe("worker runtime", () => {
|
||||
expect(vi.mocked(createWorkerImpl).mock.calls.map(([queueName]) => queueName)).toEqual(runtime.queueNames);
|
||||
});
|
||||
|
||||
test("keeps the import worker group idle until import runtime gates are enabled", () => {
|
||||
expect(getWorkerQueueNames({ WORKER_QUEUE_GROUP: "import" })).toEqual([]);
|
||||
test("keeps KWP synchronization active while generic import runtime gates are disabled", () => {
|
||||
expect(getWorkerQueueNames({ WORKER_QUEUE_GROUP: "import" })).toEqual([queueNames.kwpImports]);
|
||||
expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]);
|
||||
expect(getWorkerQueueNames({ IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([]);
|
||||
expect(getWorkerQueueNames({ IMPORT_QUEUE_ENABLED: "true", IMPORT_QUEUE_RUNTIME_READY: "true", WORKER_QUEUE_GROUP: "import" })).toEqual([
|
||||
queueNames.kwpImports,
|
||||
queueNames.imports,
|
||||
queueNames.exports,
|
||||
]);
|
||||
});
|
||||
|
||||
test("starts KWP plus exports with portal and privacy GDPR services in dedicated mode", () => {
|
||||
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
|
||||
|
||||
const runtime = startWorkerRuntime({
|
||||
createWorkerImpl,
|
||||
env: {
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
...databaseEnv,
|
||||
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
|
||||
...portalGdprExportStorageEnv,
|
||||
WORKER_ENABLED: "true",
|
||||
WORKER_QUEUE_GROUP: "import",
|
||||
},
|
||||
extraHandlerDependencies: {
|
||||
...createKwpHandlerDependencies(),
|
||||
portalGdprExportService: { runExport: vi.fn() },
|
||||
privacyGdprExportService: {
|
||||
withTenantBarrier: async (_payload, writer) => writer(async () => undefined),
|
||||
resolveReady: vi.fn(),
|
||||
resolveReclaimableProcessing: vi.fn(),
|
||||
createAttempt: vi.fn(),
|
||||
runExport: vi.fn(),
|
||||
deleteArtifact: vi.fn(),
|
||||
terminalizeReadyAfterQuiescence: vi.fn(),
|
||||
},
|
||||
},
|
||||
logger: createLogger(),
|
||||
});
|
||||
|
||||
expect(runtime.queueNames).toEqual([queueNames.kwpImports, queueNames.exports]);
|
||||
expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([queueNames.kwpImports, queueNames.exports]);
|
||||
});
|
||||
|
||||
test("fails closed when dedicated GDPR export services are not wired", () => {
|
||||
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
|
||||
expect(() =>
|
||||
startWorkerRuntime({
|
||||
createWorkerImpl: createWorkerImplMock as TestCreateWorker,
|
||||
env: {
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
...databaseEnv,
|
||||
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
|
||||
...portalGdprExportStorageEnv,
|
||||
WORKER_ENABLED: "true",
|
||||
WORKER_QUEUE_GROUP: "import",
|
||||
},
|
||||
extraHandlerDependencies: createKwpHandlerDependencies(),
|
||||
logger: createLogger(),
|
||||
})
|
||||
).toThrow(/KWP and portal\/privacy GDPR queues requires wired production services: missing portalGdprExportService, privacyGdprExportService/u);
|
||||
expect(createWorkerImplMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("fails closed when the dedicated export gate is assigned to another worker group", () => {
|
||||
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
|
||||
expect(() =>
|
||||
startWorkerRuntime({
|
||||
createWorkerImpl: createWorkerImplMock as TestCreateWorker,
|
||||
env: {
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
...databaseEnv,
|
||||
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
|
||||
...portalGdprExportStorageEnv,
|
||||
WORKER_ENABLED: "true",
|
||||
WORKER_QUEUE_GROUP: "general",
|
||||
},
|
||||
extraHandlerDependencies: createKwpHandlerDependencies(),
|
||||
logger: createLogger(),
|
||||
})
|
||||
).toThrow(/PORTAL_GDPR_EXPORT_QUEUE_ENABLED:invalid-worker-queue-group/u);
|
||||
expect(createWorkerImplMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("rejects import worker startup without wired import/export services", () => {
|
||||
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
const createWorkerImpl = createWorkerImplMock as TestCreateWorker;
|
||||
@@ -216,13 +422,56 @@ describe("worker runtime", () => {
|
||||
...databaseEnv,
|
||||
IMPORT_QUEUE_ENABLED: "true",
|
||||
IMPORT_QUEUE_RUNTIME_READY: "true",
|
||||
...portalGdprExportStorageEnv,
|
||||
...mailjetEnv,
|
||||
WORKER_ENABLED: "true",
|
||||
WORKER_QUEUE_GROUP: "import",
|
||||
},
|
||||
extraHandlerDependencies: createKwpHandlerDependencies(),
|
||||
logger: createLogger(),
|
||||
})
|
||||
).toThrow(/missing importService, scanService, exportService/u);
|
||||
).toThrow(
|
||||
/missing importService, scanService, adminSalesDataService, exportService/u
|
||||
);
|
||||
expect(createWorkerImplMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("rejects full import startup without the GDPR SSE-C key before queue ownership", () => {
|
||||
const createWorkerImplMock = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
const { GDPR_EXPORT_SSE_C_KEY_BASE64: _key, ...storageWithoutKey } = portalGdprExportStorageEnv;
|
||||
|
||||
expect(() =>
|
||||
startWorkerRuntime({
|
||||
createWorkerImpl: createWorkerImplMock as TestCreateWorker,
|
||||
env: {
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
...databaseEnv,
|
||||
IMPORT_QUEUE_ENABLED: "true",
|
||||
IMPORT_QUEUE_RUNTIME_READY: "true",
|
||||
...storageWithoutKey,
|
||||
...mailjetEnv,
|
||||
WORKER_ENABLED: "true",
|
||||
WORKER_QUEUE_GROUP: "import",
|
||||
},
|
||||
extraHandlerDependencies: {
|
||||
...createKwpHandlerDependencies(),
|
||||
exportService: { runExport: vi.fn() },
|
||||
importService: { runImport: vi.fn() },
|
||||
portalGdprExportService: { runExport: vi.fn() },
|
||||
privacyGdprExportService: {
|
||||
withTenantBarrier: async (_payload, writer) => writer(async () => undefined),
|
||||
resolveReady: vi.fn(),
|
||||
resolveReclaimableProcessing: vi.fn(),
|
||||
createAttempt: vi.fn(),
|
||||
runExport: vi.fn(),
|
||||
deleteArtifact: vi.fn(),
|
||||
terminalizeReadyAfterQuiescence: vi.fn(),
|
||||
},
|
||||
scanService: { scanObject: vi.fn() },
|
||||
},
|
||||
logger: createLogger(),
|
||||
})
|
||||
).toThrow(/GDPR_EXPORT_SSE_C_KEY_BASE64:missing-required/u);
|
||||
expect(createWorkerImplMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -237,20 +486,40 @@ describe("worker runtime", () => {
|
||||
...databaseEnv,
|
||||
IMPORT_QUEUE_ENABLED: "true",
|
||||
IMPORT_QUEUE_RUNTIME_READY: "true",
|
||||
...portalGdprExportStorageEnv,
|
||||
...mailjetEnv,
|
||||
WORKER_ENABLED: "true",
|
||||
WORKER_QUEUE_GROUP: "import",
|
||||
},
|
||||
extraHandlerDependencies: {
|
||||
...createKwpHandlerDependencies(),
|
||||
adminSalesDataService: {
|
||||
runImport: vi.fn(),
|
||||
runExport: vi.fn(),
|
||||
},
|
||||
exportService: { runExport: vi.fn() },
|
||||
importService: { runImport: vi.fn() },
|
||||
portalGdprExportService: { runExport: vi.fn() },
|
||||
privacyGdprExportService: {
|
||||
withTenantBarrier: async (_payload, writer) => writer(async () => undefined),
|
||||
resolveReady: vi.fn(),
|
||||
resolveReclaimableProcessing: vi.fn(),
|
||||
createAttempt: vi.fn(),
|
||||
runExport: vi.fn(),
|
||||
deleteArtifact: vi.fn(),
|
||||
terminalizeReadyAfterQuiescence: vi.fn(),
|
||||
},
|
||||
scanService: { scanObject: vi.fn() },
|
||||
},
|
||||
logger: createLogger(),
|
||||
});
|
||||
|
||||
expect(runtime.status).toBe("started");
|
||||
expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([queueNames.imports, queueNames.exports]);
|
||||
expect(createWorkerImplMock.mock.calls.map(([queueName]) => queueName)).toEqual([
|
||||
queueNames.kwpImports,
|
||||
queueNames.imports,
|
||||
queueNames.exports,
|
||||
]);
|
||||
});
|
||||
|
||||
test("rejects export retention startup without storage delete configuration", () => {
|
||||
@@ -267,9 +536,92 @@ describe("worker runtime", () => {
|
||||
},
|
||||
logger: createLogger(),
|
||||
})
|
||||
).toThrow("Export retention storage delete is missing required env");
|
||||
).toThrow(/Invalid worker configuration \(STORAGE_ENDPOINT:missing-required/u);
|
||||
});
|
||||
|
||||
test("rejects production general-worker startup before queue ownership without tenant hard-delete storage", () => {
|
||||
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
|
||||
expect(() =>
|
||||
startWorkerRuntime({
|
||||
createWorkerImpl: createWorkerImpl as TestCreateWorker,
|
||||
env: {
|
||||
NODE_ENV: "production",
|
||||
APP_BASE_DOMAIN: "example.test",
|
||||
AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes",
|
||||
...adminConnectionEnv,
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
...databaseEnv,
|
||||
...mailjetEnv,
|
||||
WORKER_ENABLED: "true",
|
||||
...workerGroupEnv,
|
||||
},
|
||||
logger: createLogger(),
|
||||
})
|
||||
).toThrow(/STORAGE_BUCKET:missing-required/u);
|
||||
expect(createWorkerImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("rejects production general-worker queue ownership without the tenant hard-delete service", () => {
|
||||
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
|
||||
expect(() =>
|
||||
startWorkerRuntime({
|
||||
createWorkerImpl: createWorkerImpl as TestCreateWorker,
|
||||
env: {
|
||||
NODE_ENV: "production",
|
||||
APP_BASE_DOMAIN: "example.test",
|
||||
AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes",
|
||||
...adminConnectionEnv,
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
...databaseEnv,
|
||||
...tenantHardDeleteStorageEnv,
|
||||
...tenantHardDeleteSearchEnv,
|
||||
...mailjetEnv,
|
||||
EXPORT_RETENTION_QUEUE_ENABLED: "true",
|
||||
WORKER_ENABLED: "true",
|
||||
...workerGroupEnv,
|
||||
},
|
||||
logger: createLogger(),
|
||||
queueNames: [queueNames.maintenance],
|
||||
})
|
||||
).toThrow(/missing privacyTenantDeletionService/u);
|
||||
expect(createWorkerImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test.each(["MEILI_URL", "MEILI_API_KEY"] as const)(
|
||||
"rejects production general-worker queue ownership without %s for tenant search cleanup",
|
||||
(missingName) => {
|
||||
const createWorkerImpl = vi.fn(<Name extends QueueName>(queueName: Name) => createMockWorker(queueName));
|
||||
const env = {
|
||||
NODE_ENV: "production",
|
||||
APP_BASE_DOMAIN: "example.test",
|
||||
AUDIT_CHAIN_SECRET: "audit-chain-secret-with-at-least-32-bytes",
|
||||
...adminConnectionEnv,
|
||||
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
|
||||
...databaseEnv,
|
||||
...tenantHardDeleteStorageEnv,
|
||||
...tenantHardDeleteSearchEnv,
|
||||
...mailjetEnv,
|
||||
EXPORT_RETENTION_QUEUE_ENABLED: "true",
|
||||
WORKER_ENABLED: "true",
|
||||
...workerGroupEnv,
|
||||
};
|
||||
delete env[missingName];
|
||||
|
||||
expect(() =>
|
||||
startWorkerRuntime({
|
||||
createWorkerImpl: createWorkerImpl as TestCreateWorker,
|
||||
env,
|
||||
extraHandlerDependencies: { privacyTenantDeletionService: { runScan: vi.fn() } },
|
||||
logger: createLogger(),
|
||||
queueNames: [queueNames.maintenance],
|
||||
})
|
||||
).toThrow(new RegExp(`missing ${missingName}`, "u"));
|
||||
expect(createWorkerImpl).not.toHaveBeenCalled();
|
||||
}
|
||||
);
|
||||
|
||||
test("binds requestId and retry fields through child loggers for completed and failed jobs", async () => {
|
||||
const { childLoggers, logger } = createChildLogger();
|
||||
const recordFailedJob = vi.fn(async () => undefined);
|
||||
@@ -326,6 +678,135 @@ describe("worker runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("redacts completed OWNER mail payloads while failed attempts retain the token for retry", async () => {
|
||||
const token = "owner-capability-token-fixture";
|
||||
const data = {
|
||||
kind: "mail.transactional",
|
||||
tenantId: "tenant_1",
|
||||
to: ["owner@example.test"],
|
||||
html: `<a href="https://app.example.test/invitation/${token}">Accept</a>`,
|
||||
metadata: {
|
||||
source: "workspace-owner-invitation",
|
||||
invitationId: "invite_1",
|
||||
invitationTokenHash: "a".repeat(64),
|
||||
issuanceKeyHash: "b".repeat(64),
|
||||
},
|
||||
};
|
||||
const completedUpdateData = vi.fn(async (_data: unknown) => undefined);
|
||||
const failedUpdateData = vi.fn(async (_data: unknown) => undefined);
|
||||
const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob: vi.fn(async () => undefined) });
|
||||
|
||||
worker.emitFailed(
|
||||
{
|
||||
attemptsMade: 1,
|
||||
data,
|
||||
id: "owner-mail-retry",
|
||||
name: "send-transactional-email",
|
||||
opts: { attempts: 3 },
|
||||
queueName: queueNames.email,
|
||||
updateData: failedUpdateData,
|
||||
},
|
||||
new Error("mail provider unavailable")
|
||||
);
|
||||
worker.emitCompleted({
|
||||
attemptsMade: 1,
|
||||
data,
|
||||
id: "owner-mail-complete",
|
||||
name: "send-transactional-email",
|
||||
opts: { attempts: 3 },
|
||||
queueName: queueNames.email,
|
||||
updateData: completedUpdateData,
|
||||
});
|
||||
await vi.waitFor(() => expect(completedUpdateData).toHaveBeenCalledOnce());
|
||||
|
||||
expect(failedUpdateData).not.toHaveBeenCalled();
|
||||
expect(JSON.stringify(data)).toContain(token);
|
||||
expect(JSON.stringify(completedUpdateData.mock.calls[0]?.[0])).not.toContain(token);
|
||||
expect(completedUpdateData.mock.calls[0]?.[0]).toMatchObject({
|
||||
metadata: expect.objectContaining({ source: "workspace-owner-invitation", secretBodyRedacted: true }),
|
||||
});
|
||||
});
|
||||
|
||||
test("redacts a terminal failed OWNER job so the Redis DLQ cannot retain its capability", async () => {
|
||||
const token = "terminal-owner-capability-token-fixture";
|
||||
const data = {
|
||||
kind: "mail.transactional",
|
||||
tenantId: "tenant_1",
|
||||
to: ["owner@example.test"],
|
||||
html: `<a href="https://app.example.test/invitation/${token}">Accept</a>`,
|
||||
metadata: {
|
||||
source: "workspace-owner-invitation",
|
||||
invitationId: "invite_terminal",
|
||||
invitationTokenHash: "a".repeat(64),
|
||||
issuanceKeyHash: "b".repeat(64),
|
||||
},
|
||||
};
|
||||
const updateData = vi.fn(async (_data: unknown) => undefined);
|
||||
const recordFailedJob = vi.fn(async () => undefined);
|
||||
const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob });
|
||||
|
||||
worker.emitFailed(
|
||||
{
|
||||
attemptsMade: 3,
|
||||
data,
|
||||
id: "owner-mail-terminal",
|
||||
name: "send-transactional-email",
|
||||
opts: { attempts: 3 },
|
||||
queueName: queueNames.email,
|
||||
updateData,
|
||||
},
|
||||
new Error("mail provider permanently unavailable")
|
||||
);
|
||||
|
||||
await vi.waitFor(() => expect(updateData).toHaveBeenCalledOnce());
|
||||
expect(JSON.stringify(updateData.mock.calls[0]?.[0])).not.toContain(token);
|
||||
expect(updateData.mock.calls[0]?.[0]).toMatchObject({
|
||||
metadata: expect.objectContaining({ secretBodyRedacted: true, ownerInvitationDeliveryStatus: "rotation_required" }),
|
||||
});
|
||||
expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ jobId: "owner-mail-terminal" }));
|
||||
});
|
||||
|
||||
test("redacts completed and terminal portal setup jobs so Redis retains no setup capability", async () => {
|
||||
const completedToken = "c".repeat(43);
|
||||
const failedToken = "f".repeat(43);
|
||||
const completedUpdateData = vi.fn(async (_data: unknown) => undefined);
|
||||
const failedUpdateData = vi.fn(async (_data: unknown) => undefined);
|
||||
const { worker } = startOneWorker({ logger: createLogger(), recordFailedJob: vi.fn(async () => undefined) });
|
||||
|
||||
worker.emitCompleted({
|
||||
attemptsMade: 0,
|
||||
data: createPortalSetupJobData(completedToken, "portal_token_completed"),
|
||||
id: "portal-setup-completed",
|
||||
name: "send-transactional-email",
|
||||
opts: { attempts: 3 },
|
||||
queueName: queueNames.email,
|
||||
updateData: completedUpdateData,
|
||||
});
|
||||
worker.emitFailed(
|
||||
{
|
||||
attemptsMade: 3,
|
||||
data: createPortalSetupJobData(failedToken, "portal_token_failed"),
|
||||
id: "portal-setup-failed",
|
||||
name: "send-transactional-email",
|
||||
opts: { attempts: 3 },
|
||||
queueName: queueNames.email,
|
||||
updateData: failedUpdateData,
|
||||
},
|
||||
new Error("provider unavailable")
|
||||
);
|
||||
|
||||
await vi.waitFor(() => expect(completedUpdateData).toHaveBeenCalledOnce());
|
||||
await vi.waitFor(() => expect(failedUpdateData).toHaveBeenCalledOnce());
|
||||
expect(JSON.stringify(completedUpdateData.mock.calls)).not.toContain(completedToken);
|
||||
expect(JSON.stringify(failedUpdateData.mock.calls)).not.toContain(failedToken);
|
||||
expect(completedUpdateData.mock.calls[0]?.[0]).toMatchObject({
|
||||
metadata: expect.objectContaining({ secretBodyRedacted: true }),
|
||||
});
|
||||
expect(failedUpdateData.mock.calls[0]?.[0]).toMatchObject({
|
||||
metadata: expect.objectContaining({ secretBodyRedacted: true, portalSetupDeliveryStatus: "rotation_required" }),
|
||||
});
|
||||
});
|
||||
|
||||
test("logs retry diagnostics without leaking payload data or persisting non-terminal failures", () => {
|
||||
const logger = createLogger();
|
||||
const recordFailedJob = vi.fn(async () => undefined);
|
||||
@@ -342,7 +823,10 @@ describe("worker runtime", () => {
|
||||
},
|
||||
failure
|
||||
);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ error: expect.objectContaining({ message: "remote service unavailable" }), requestId: "req_retry", attemptsLimit: 5 }), "Worker job failed.");
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ error: expect.objectContaining({ message: "remote service unavailable" }), requestId: "req_retry", attemptsLimit: 5 }),
|
||||
"Worker job failed."
|
||||
);
|
||||
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("fixture-body-secret");
|
||||
expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain("token-secret");
|
||||
expect(recordFailedJob).not.toHaveBeenCalled();
|
||||
@@ -416,13 +900,25 @@ describe("worker runtime", () => {
|
||||
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
|
||||
expect(() =>
|
||||
worker.emitFailed(
|
||||
{ attemptsMade: 1, data: { token: "persist-secret" }, id: "job-persist-error", name: "deliver-webhook", opts: { attempts: 1 }, queueName: queueNames.email },
|
||||
{
|
||||
attemptsMade: 1,
|
||||
data: { token: "persist-secret" },
|
||||
id: "job-persist-error",
|
||||
name: "deliver-webhook",
|
||||
opts: { attempts: 1 },
|
||||
queueName: queueNames.email,
|
||||
},
|
||||
new TypeError("terminal failure")
|
||||
)
|
||||
).not.toThrow();
|
||||
await flushMicrotasks();
|
||||
expect(recordFailedJob).toHaveBeenCalledWith(expect.objectContaining({ attemptsLimit: 1, errorName: "TypeError", errorMessage: "terminal failure", jobId: "job-persist-error" }));
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ error: expect.objectContaining({ message: "database unavailable" }), jobId: "job-persist-error" }), "Failed to persist terminal worker failure.");
|
||||
expect(recordFailedJob).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ attemptsLimit: 1, errorName: "TypeError", errorMessage: "terminal failure", jobId: "job-persist-error" })
|
||||
);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ error: expect.objectContaining({ message: "database unavailable" }), jobId: "job-persist-error" }),
|
||||
"Failed to persist terminal worker failure."
|
||||
);
|
||||
expect(reporter.report).toHaveBeenCalledWith(expect.objectContaining({ operation: "job.failed", jobId: "job-persist-error", requestId: undefined }));
|
||||
expect(reporter.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
@@ -442,9 +938,10 @@ describe("worker runtime", () => {
|
||||
const handlerError = new Error("handler exploded");
|
||||
const reporter: ErrorReporter = { report: vi.fn() };
|
||||
const { registeredProcessor } = startOneWorker({
|
||||
createProcessor: () => vi.fn(async () => {
|
||||
throw handlerError;
|
||||
}) as TypedProcessor<QueueName>,
|
||||
createProcessor: () =>
|
||||
vi.fn(async () => {
|
||||
throw handlerError;
|
||||
}) as TypedProcessor<QueueName>,
|
||||
logger,
|
||||
reporter,
|
||||
});
|
||||
@@ -457,7 +954,10 @@ describe("worker runtime", () => {
|
||||
opts: { attempts: 3, backoff: 750 },
|
||||
} as unknown as Parameters<TypedProcessor<QueueName>>[0])
|
||||
).rejects.toThrow(handlerError);
|
||||
expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ error: expect.objectContaining({ message: "handler exploded" }), requestId: "req_handler", backoffType: "fixed" }), "Worker handler failed.");
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ error: expect.objectContaining({ message: "handler exploded" }), requestId: "req_handler", backoffType: "fixed" }),
|
||||
"Worker handler failed."
|
||||
);
|
||||
expect(reporter.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
component: "worker",
|
||||
@@ -480,9 +980,19 @@ describe("worker runtime", () => {
|
||||
const closeError = new Error("close failed");
|
||||
const reporter: ErrorReporter = { report: vi.fn() };
|
||||
const emailWorker = createMockWorker(queueNames.email);
|
||||
const notificationWorker = createMockWorker(queueNames.notifications, vi.fn(async () => { throw closeError; }));
|
||||
await expect(shutdownWorkerRuntime({ workers: [emailWorker, notificationWorker] }, { logger, reason: "SIGTERM", reporter })).rejects.toThrow("Failed to close workers: hivegrid.notifications.");
|
||||
expect(logger.error).toHaveBeenCalledWith({ error: expect.objectContaining({ message: "close failed" }), reason: "SIGTERM", workerName: queueNames.notifications }, "Failed to close worker.");
|
||||
const notificationWorker = createMockWorker(
|
||||
queueNames.notifications,
|
||||
vi.fn(async () => {
|
||||
throw closeError;
|
||||
})
|
||||
);
|
||||
await expect(shutdownWorkerRuntime({ workers: [emailWorker, notificationWorker] }, { logger, reason: "SIGTERM", reporter })).rejects.toThrow(
|
||||
"Failed to close workers: hivegrid.notifications."
|
||||
);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
{ error: expect.objectContaining({ message: "close failed" }), reason: "SIGTERM", workerName: queueNames.notifications },
|
||||
"Failed to close worker."
|
||||
);
|
||||
expect(reporter.report).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
component: "worker",
|
||||
@@ -494,26 +1004,44 @@ describe("worker runtime", () => {
|
||||
error: expect.objectContaining({ errorName: "Error" }),
|
||||
})
|
||||
);
|
||||
await expect(shutdownWorkerRuntime({ workers: [emailWorker] }, { logger, reason: "SIGINT", reporter })).resolves.toEqual({ closedWorkers: [queueNames.email] });
|
||||
await expect(shutdownWorkerRuntime({ workers: [emailWorker] }, { logger, reason: "SIGINT", reporter })).resolves.toEqual({
|
||||
closedWorkers: [queueNames.email],
|
||||
});
|
||||
expect(logger.info).toHaveBeenCalledWith({ closedWorkers: [queueNames.email], reason: "SIGINT" }, "Worker shutdown complete.");
|
||||
});
|
||||
|
||||
test("swallows reporter failures without changing retry or shutdown behavior", async () => {
|
||||
const logger = createLogger();
|
||||
const reporter: ErrorReporter = { report: vi.fn(async () => { throw new Error("reporter failed"); }) };
|
||||
const reporter: ErrorReporter = {
|
||||
report: vi.fn(async () => {
|
||||
throw new Error("reporter failed");
|
||||
}),
|
||||
};
|
||||
const recordFailedJob = vi.fn(async () => undefined);
|
||||
const { worker } = startOneWorker({ logger, recordFailedJob, reporter });
|
||||
|
||||
expect(() =>
|
||||
worker.emitFailed(
|
||||
{ attemptsMade: 1, data: { metadata: { requestId: "req_reporter_failure" } }, id: "job-report-fail", name: "deliver-webhook", opts: { attempts: 3 }, queueName: queueNames.email },
|
||||
{
|
||||
attemptsMade: 1,
|
||||
data: { metadata: { requestId: "req_reporter_failure" } },
|
||||
id: "job-report-fail",
|
||||
name: "deliver-webhook",
|
||||
opts: { attempts: 3 },
|
||||
queueName: queueNames.email,
|
||||
},
|
||||
new Error("retry failure")
|
||||
)
|
||||
).not.toThrow();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(recordFailedJob).not.toHaveBeenCalled();
|
||||
const closingWorker = createMockWorker(queueNames.email, vi.fn(async () => { throw new Error("close failed"); }));
|
||||
const closingWorker = createMockWorker(
|
||||
queueNames.email,
|
||||
vi.fn(async () => {
|
||||
throw new Error("close failed");
|
||||
})
|
||||
);
|
||||
await expect(shutdownWorkerRuntime({ workers: [closingWorker] }, { logger, reporter })).rejects.toThrow("Failed to close workers: hivegrid.email.");
|
||||
});
|
||||
|
||||
@@ -530,8 +1058,12 @@ describe("worker runtime", () => {
|
||||
const record: Span = { args, status: undefined };
|
||||
spans.push(record);
|
||||
return callback({
|
||||
setAttribute() { /* no-op for assertion brevity */ },
|
||||
setStatus(s) { record.status = s; },
|
||||
setAttribute() {
|
||||
/* no-op for assertion brevity */
|
||||
},
|
||||
setStatus(s) {
|
||||
record.status = s;
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -578,8 +1110,12 @@ describe("worker runtime", () => {
|
||||
const record: { status?: { code: number; message?: string } } = {};
|
||||
spans.push(record);
|
||||
return callback({
|
||||
setAttribute() { /* no-op */ },
|
||||
setStatus(s) { record.status = s; },
|
||||
setAttribute() {
|
||||
/* no-op */
|
||||
},
|
||||
setStatus(s) {
|
||||
record.status = s;
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -738,7 +1274,7 @@ describe("worker runtime", () => {
|
||||
attemptsMade: 0,
|
||||
name: "send-mail",
|
||||
queueName: queueNames.email,
|
||||
} as never),
|
||||
} as never)
|
||||
).rejects.toBeInstanceOf(TimeoutError);
|
||||
|
||||
expect(samples.completed).toEqual([]);
|
||||
|
||||
Reference in new issue
Block a user