Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -40,7 +40,7 @@ Manual E2E note: real queued job processing requires a running Redis/container s
## Current processors
The worker has concrete handler paths for mail, webhooks, notifications, maintenance, storage scans, exports, imports, and guarded browser job descriptors. Some queues remain disabled unless feature flags and provider services are configured.
The worker has concrete handler paths for mail, webhooks, notifications, maintenance, storage scans, exports, generic imports, isolated KWP provider synchronization, and guarded browser job descriptors. Some queues remain disabled unless feature flags and provider services are configured.
- `hivegrid.email`
- `hivegrid.notifications`
@@ -49,4 +49,5 @@ The worker has concrete handler paths for mail, webhooks, notifications, mainten
- `hivegrid.file-scan`
- `hivegrid.exports`
- `hivegrid.imports`
- `hivegrid.kwp-imports`
- `hivegrid.plugin-invoke`
@@ -14,8 +14,12 @@
"typecheck": "tsc -p tsconfig.json --noEmit"
},
"dependencies": {
"@aws-sdk/client-s3": "3.1045.0",
"@aws-sdk/lib-storage": "3.1045.0",
"@aws-sdk/client-s3": "3.1048.0",
"@aws-sdk/lib-storage": "3.1048.0",
"@hivegrid/activity": "workspace:*",
"@hivegrid/billing": "workspace:*",
"@hivegrid/chat": "workspace:*",
"@hivegrid/cli": "workspace:*",
"@hivegrid/config": "workspace:*",
"@hivegrid/db": "workspace:*",
"@hivegrid/email-sequences": "workspace:*",
@@ -27,38 +31,48 @@
"@hivegrid/foundation": "workspace:*",
"@hivegrid/imports": "workspace:*",
"@hivegrid/integrations": "workspace:*",
"@hivegrid/i18n": "workspace:*",
"@hivegrid/logger": "workspace:*",
"@hivegrid/mail": "workspace:*",
"@hivegrid/marketing-outbound": "workspace:*",
"@hivegrid/metrics": "workspace:*",
"@hivegrid/observability": "workspace:*",
"@hivegrid/plugin-runtime": "workspace:*",
"@hivegrid/portal-domain-controller": "workspace:*",
"@hivegrid/queue": "workspace:*",
"@hivegrid/realtime": "workspace:*",
"@hivegrid/search": "workspace:*",
"@hivegrid/shared-store": "workspace:*",
"@hivegrid/storage": "workspace:*",
"@hivegrid/url": "workspace:*",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.76.0",
"@opentelemetry/core": "^2.7.1",
"@opentelemetry/exporter-trace-otlp-http": "^0.218.0",
"@opentelemetry/resources": "^2.7.1",
"@opentelemetry/sdk-node": "^0.218.0",
"@opentelemetry/sdk-trace-base": "^2.7.1",
"@opentelemetry/exporter-trace-otlp-http": "^0.220.0",
"@opentelemetry/instrumentation-aws-sdk": "^0.75.0",
"@opentelemetry/instrumentation-http": "^0.220.0",
"@opentelemetry/instrumentation-ioredis": "^0.68.0",
"@opentelemetry/instrumentation-pg": "^0.72.0",
"@opentelemetry/instrumentation-pino": "^0.66.0",
"@opentelemetry/instrumentation-undici": "^0.30.0",
"@opentelemetry/resources": "^2.9.0",
"@opentelemetry/sdk-node": "^0.220.0",
"@opentelemetry/sdk-trace-base": "^2.9.0",
"@opentelemetry/semantic-conventions": "^1.40.0",
"@sentry/node": "^10.53.1",
"bullmq": "^5.76.5",
"clamscan": "^2.4.0",
"heic-convert": "^2.1.0",
"ioredis": "5.10.1",
"ioredis": "5.11.1",
"meilisearch": "0.45.0",
"pdf-to-png-converter": "^4.1.1",
"pino": "^9.0.0",
"prom-client": "^15.1.3",
"semver": "^7.7.3",
"sharp": "^0.35.2"
"sharp": "^0.35.2",
"web-push": "^3.6.7"
},
"devDependencies": {
"@types/node": "^25.0.0",
"@types/web-push": "^3.6.4",
"tsx": "^4.6.2",
"typescript": "^5.3.2"
}
@@ -20,12 +20,12 @@ const externalBarePackagesPlugin = {
};
await build({
entryPoints: ["src/index.ts", "src/health.ts", "src/jobs/plugin-sandbox-worker.ts"],
entryPoints: ["src/index.ts", "src/health.ts", "src/deploy-health.ts", "src/jobs/plugin-sandbox-worker.ts", "src/jobs/file-preview-decoder-process.ts"],
outdir: "dist",
bundle: true,
platform: "node",
format: "esm",
target: "node20",
target: "node24",
sourcemap,
banner: {
js: 'import { createRequire as __hivegridCreateRequire } from "node:module"; const require = __hivegridCreateRequire(import.meta.url);',
@@ -0,0 +1,134 @@
import type { EnvRecord } from "@hivegrid/config";
import {
claimAdminCallRecordings,
claimExpiredAdminCallRecordings,
createPrismaAdminTelephonyRecordingRepository,
markAdminCallRecordingDeleted,
} from "@hivegrid/db/admin-telephony-recordings.server";
import type { DbClient } from "@hivegrid/db/client.server";
import { ADMIN_CONNECTION_MASTER_KEY_ENV } from "@hivegrid/db/crypto/admin-connection-envelope.server";
import { EnvKeyWrapper } from "@hivegrid/foundation/crypto";
import {
createAdminCallRecordingProcessor,
type AdminCallRecordingStorage,
type AdminCallRecordingScanner,
} from "@hivegrid/integrations/admin-call-recordings.server";
import { createClamAVBufferScanner } from "@hivegrid/storage";
import { resolveWorkerClamdConfig } from "./scanService.js";
import {
createWorkerStorageDeleteService,
getMissingWorkerStorageEnv,
} from "./storage.js";
type AdminCallRecordingWorkerEnv = EnvRecord & {
CLAMD_HOST?: string;
CLAMD_PORT?: string;
CLAMD_TIMEOUT_MS?: string;
CLAMAV_HOST?: string;
CLAMAV_PORT?: string;
CLAMAV_TIMEOUT_MS?: string;
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_BUCKET?: string;
STORAGE_ENDPOINT?: string;
STORAGE_FORCE_PATH_STYLE?: string;
STORAGE_PUBLIC_BASE_URL?: string;
STORAGE_REGION?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
};
export async function runAdminCallRecordingProcessing(input: {
readonly db: DbClient;
readonly env: AdminCallRecordingWorkerEnv;
readonly take?: number;
readonly scanner?: AdminCallRecordingScanner;
readonly storage?: AdminCallRecordingStorage;
}): Promise<{
readonly claimed: number;
readonly processed: number;
}> {
const recordings = await claimAdminCallRecordings(
input.db,
input.take ?? 10
);
if (recordings.length === 0) {
return { claimed: 0, processed: 0 };
}
const storage =
input.storage ?? requireRecordingStorage(input.env);
const scanner =
input.scanner ??
createClamAVBufferScanner(resolveWorkerClamdConfig(input.env));
const repository =
createPrismaAdminTelephonyRecordingRepository(input.db);
const processor = createAdminCallRecordingProcessor({
repository,
scanner,
storage,
keyWrapper: new EnvKeyWrapper(ADMIN_CONNECTION_MASTER_KEY_ENV),
});
for (const recording of recordings) {
await processor.process(recording);
}
return {
claimed: recordings.length,
processed: recordings.length,
};
}
export async function runAdminCallRecordingRetention(input: {
readonly db: DbClient;
readonly env: AdminCallRecordingWorkerEnv;
readonly now?: Date;
readonly take?: number;
readonly storage?: AdminCallRecordingStorage;
}): Promise<{
readonly claimed: number;
readonly deleted: number;
readonly failed: number;
}> {
const now = input.now ?? new Date();
const recordings = await claimExpiredAdminCallRecordings(
input.db,
now,
input.take ?? 50
);
if (recordings.length === 0) {
return { claimed: 0, deleted: 0, failed: 0 };
}
const storage =
input.storage ?? requireRecordingStorage(input.env);
let deleted = 0;
let failed = 0;
for (const recording of recordings) {
try {
if (recording.storageKey) {
await storage.deleteObject({
bucket: recording.storageBucket ?? undefined,
key: recording.storageKey,
});
}
await markAdminCallRecordingDeleted(input.db, {
id: recording.id,
now,
});
deleted += 1;
} catch {
failed += 1;
}
}
return { claimed: recordings.length, deleted, failed };
}
function requireRecordingStorage(
env: AdminCallRecordingWorkerEnv
): AdminCallRecordingStorage {
const missing = getMissingWorkerStorageEnv(env);
if (missing.length > 0) {
throw new Error(
`Admin recording storage is missing required env: ${missing.join(", ")}`
);
}
return createWorkerStorageDeleteService(env);
}
@@ -0,0 +1,37 @@
import type { EnvRecord } from "@hivegrid/config";
import { createAdminEmailAttachmentProcessor } from "@hivegrid/integrations/admin-email-attachments.server";
import { createClamAVBufferScanner } from "@hivegrid/storage";
import { resolveWorkerClamdConfig } from "./scanService.js";
import {
createWorkerStorageDeleteService,
getMissingWorkerStorageEnv,
} from "./storage.js";
type AdminEmailAttachmentEnv = EnvRecord & {
CLAMD_HOST?: string;
CLAMD_PORT?: string;
CLAMD_TIMEOUT_MS?: string;
CLAMAV_HOST?: string;
CLAMAV_PORT?: string;
CLAMAV_TIMEOUT_MS?: string;
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_BUCKET?: string;
STORAGE_ENDPOINT?: string;
STORAGE_FORCE_PATH_STYLE?: string;
STORAGE_PUBLIC_BASE_URL?: string;
STORAGE_REGION?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
};
export function createWorkerAdminEmailAttachmentProcessor(
env: AdminEmailAttachmentEnv
) {
const storage =
getMissingWorkerStorageEnv(env).length === 0
? createWorkerStorageDeleteService(env)
: undefined;
const clamd = resolveWorkerClamdConfig(env);
const scanner = createClamAVBufferScanner(clamd);
return createAdminEmailAttachmentProcessor({ scanner, storage });
}
@@ -0,0 +1,9 @@
import type { DbClient } from "@hivegrid/db/client.server";
import { processAdminOutboundCallRequests } from "@hivegrid/integrations/admin-outbound-calls.server";
export async function runAdminOutboundCalls(input: {
readonly db: DbClient;
readonly take?: number;
}) {
return processAdminOutboundCallRequests(input);
}
@@ -25,7 +25,9 @@ export type WorkerStatusReport = {
workflowTrigger?: WorkerStatusCheckOutcome;
pluginHookFanout?: WorkerStatusCheckOutcome;
transactionalMailOutbox?: WorkerStatusCheckOutcome;
portalStatusMail?: WorkerStatusCheckOutcome;
emailSequences?: WorkerStatusCheckOutcome;
chatLinkPreview?: WorkerStatusCheckOutcome;
};
// Resolved WORKFLOW_TRIGGER_EXECUTOR_MODE for this worker. Only present when the
// workflow-trigger subscriber runs on this worker (i.e. the check is not "disabled"),
@@ -0,0 +1,13 @@
import { readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
const source = readFileSync(path.join(path.dirname(fileURLToPath(import.meta.url)), "handlers.ts"), "utf8");
describe("billing activity summary i18n contract", () => {
it("persists a semantic summary key when dunning freezes a tenant", () => {
expect(source).toContain('summaryText: "activity.runtime.billing.statusChanged"');
expect(source).not.toContain('summaryText: "Billing-Status auf frozen gesetzt"');
});
});
@@ -0,0 +1,184 @@
import { describe, expect, test, vi } from "vitest";
import { runWorkerDeployHealthCli } from "./deploy-health.js";
function createIo() {
let stderr = "";
let stdout = "";
return {
io: {
stderr: {
write(chunk: string) {
stderr += chunk;
return true;
},
},
stdout: {
write(chunk: string) {
stdout += chunk;
return true;
},
},
},
readStderr: () => stderr,
readStdout: () => stdout,
};
}
function readyResponse(group = "general", overrides: Record<string, unknown> = {}) {
return {
status: 200,
json: vi.fn(async () => ({
status: "ready",
workerGroup: group,
checks: { redis: "ok", database: "ok" },
...overrides,
})),
};
}
function enabledEnv(revision: string, group = "general") {
return {
APP_REVISION: revision,
WORKER_ENABLED: "true",
WORKER_HEALTH_PORT: "9101",
WORKER_QUEUE_GROUP: group,
};
}
describe("worker deployment health", () => {
test("probes the live worker readiness endpoint for the exact baked revision and group", async () => {
const revision = "a".repeat(40);
const fetchReadiness = vi.fn(async () => readyResponse());
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "general", enabledEnv(revision), io, {
fetchReadiness,
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(0);
expect(JSON.parse(readStdout())).toEqual({ status: "ready", revision, workerGroup: "general" });
expect(readStderr()).toBe("");
expect(fetchReadiness).toHaveBeenCalledWith(
"http://127.0.0.1:9101/readyz",
expect.objectContaining({ headers: { Accept: "application/json" }, signal: expect.any(AbortSignal) })
);
});
test("rejects an environment override that disagrees with the baked image revision", async () => {
const expectedRevision = "b".repeat(40);
const fetchReadiness = vi.fn();
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(expectedRevision, "general", enabledEnv(expectedRevision), io, {
fetchReadiness,
readImageRevision: () => "a".repeat(40),
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/revision does not match/);
expect(readStderr()).not.toContain(expectedRevision);
expect(fetchReadiness).not.toHaveBeenCalled();
});
test("rejects a disabled worker before probing the runtime", async () => {
const revision = "a".repeat(40);
const fetchReadiness = vi.fn();
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(
revision,
"general",
{ ...enabledEnv(revision), WORKER_ENABLED: "false" },
io,
{ fetchReadiness, readImageRevision: () => revision }
);
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/processing to be enabled/);
expect(fetchReadiness).not.toHaveBeenCalled();
});
test("rejects a resource assigned to the wrong queue group", async () => {
const revision = "a".repeat(40);
const fetchReadiness = vi.fn();
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "mail", enabledEnv(revision, "general"), io, {
fetchReadiness,
readImageRevision: () => revision,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/group does not match/);
expect(readStderr()).not.toContain("mail");
expect(readStderr()).not.toContain("general");
expect(fetchReadiness).not.toHaveBeenCalled();
});
test("rejects a readiness response for a different runtime group", async () => {
const revision = "a".repeat(40);
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "general", enabledEnv(revision), io, {
fetchReadiness: vi.fn(async () => readyResponse("mail")),
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/response does not match/);
});
test("rejects unhealthy, malformed, or unreachable readiness responses", async () => {
const revision = "a".repeat(40);
const scenarios = [
vi.fn(async () => ({ status: 503, json: vi.fn() })),
vi.fn(async () => readyResponse("general", { checks: { redis: "ok", database: "error" } })),
vi.fn(async () => readyResponse("general", { checks: { first: "ok", second: "ok" } })),
vi.fn(async () => readyResponse("general", { checks: ["ok", "ok"] })),
vi.fn(async () => {
throw new Error("connection failed with secret details");
}),
];
for (const fetchReadiness of scenarios) {
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "general", enabledEnv(revision), io, {
fetchReadiness,
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).not.toContain("secret details");
}
});
test("rejects malformed revision, group, and health port arguments", async () => {
const revision = "a".repeat(40);
const cases = [
["not-a-sha", "general", enabledEnv(revision)],
[revision, "unknown-group", enabledEnv(revision)],
[revision, "general", { ...enabledEnv(revision), WORKER_HEALTH_PORT: "70000" }],
] as const;
for (const [expectedRevision, expectedGroup, env] of cases) {
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(expectedRevision, expectedGroup, env, io, {
readImageRevision: () => revision,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).not.toContain("not-a-sha");
expect(readStderr()).not.toContain("unknown-group");
}
});
});
@@ -0,0 +1,102 @@
import { readFileSync } from "node:fs";
import { isWorkerQueueGroup } from "@hivegrid/queue";
const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/;
const IMAGE_REVISION_PATH = "/myapp/.image-revision";
const DEFAULT_HEALTH_PORT = "9100";
const HEALTH_TIMEOUT_MS = 9_000;
type WorkerDeployHealthIo = {
stderr?: Pick<NodeJS.WriteStream, "write">;
stdout: Pick<NodeJS.WriteStream, "write">;
};
type WorkerReadinessResponse = {
status?: unknown;
workerGroup?: unknown;
checks?: unknown;
};
type WorkerDeployHealthDependencies = {
fetchReadiness?: (url: string, init: RequestInit) => Promise<Pick<Response, "status" | "json">>;
readImageRevision?: () => string;
timeoutSignal?: () => AbortSignal;
};
function fail(io: WorkerDeployHealthIo, message: string): number {
io.stderr?.write(`${message}\n`);
return 1;
}
export async function runWorkerDeployHealthCli(
expectedRevision: string,
expectedGroup: string,
env: NodeJS.ProcessEnv = process.env,
io: WorkerDeployHealthIo = process,
dependencies: WorkerDeployHealthDependencies = {}
): Promise<number> {
if (!FULL_GIT_SHA_PATTERN.test(expectedRevision)) {
return fail(io, "Worker deployment health requires a full lowercase Git SHA.");
}
if (!isWorkerQueueGroup(expectedGroup)) {
return fail(io, "Worker deployment health requires a valid worker group.");
}
let imageRevision = "";
try {
imageRevision = (dependencies.readImageRevision ?? (() => readFileSync(IMAGE_REVISION_PATH, "utf8")))().trim();
} catch {
return fail(io, "Worker deployment image revision is unavailable.");
}
if (imageRevision !== expectedRevision || env.APP_REVISION !== expectedRevision) {
return fail(io, "Worker deployment revision does not match the expected image.");
}
if (env.WORKER_ENABLED !== "true") {
return fail(io, "Worker deployment health requires worker processing to be enabled.");
}
if (env.WORKER_QUEUE_GROUP !== expectedGroup) {
return fail(io, "Worker deployment group does not match the expected resource.");
}
const port = env.WORKER_HEALTH_PORT || DEFAULT_HEALTH_PORT;
if (!/^\d{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65_535) {
return fail(io, "Worker deployment health port is invalid.");
}
try {
const fetchReadiness = dependencies.fetchReadiness ?? fetch;
const response = await fetchReadiness(`http://127.0.0.1:${port}/readyz`, {
headers: { Accept: "application/json" },
signal: (dependencies.timeoutSignal ?? (() => AbortSignal.timeout(HEALTH_TIMEOUT_MS)))(),
});
if (response.status !== 200) {
return fail(io, "Worker deployment readiness is not healthy.");
}
const payload = (await response.json()) as WorkerReadinessResponse;
const checks = payload.checks;
const readinessChecks = typeof checks === "object" && checks !== null && !Array.isArray(checks) ? (checks as Record<string, unknown>) : null;
const allChecksReady =
readinessChecks !== null &&
readinessChecks.database === "ok" &&
readinessChecks.redis === "ok" &&
Object.values(readinessChecks).every((status) => status === "ok");
if (payload.status !== "ready" || payload.workerGroup !== expectedGroup || !allChecksReady) {
return fail(io, "Worker deployment readiness response does not match the expected runtime.");
}
} catch {
return fail(io, "Worker deployment readiness probe failed.");
}
io.stdout.write(`${JSON.stringify({ status: "ready", revision: expectedRevision, workerGroup: expectedGroup })}\n`);
return 0;
}
if (import.meta.url === `file://${process.argv[1]}`) {
runWorkerDeployHealthCli(process.argv[2] || "", process.argv[3] || "")
.then((exitCode) => {
process.exitCode = exitCode;
})
.catch(() => {
process.exitCode = 1;
});
}
@@ -46,8 +46,8 @@ describe("resolveQueueNamesForGroup", () => {
expect(resolveQueueNamesForGroup("general")).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
});
test("import group includes imports + exports", () => {
expect(resolveQueueNamesForGroup("import")).toEqual([queueNames.imports, queueNames.exports]);
test("import group includes dedicated KWP, generic imports, and exports", () => {
expect(resolveQueueNamesForGroup("import")).toEqual([queueNames.kwpImports, queueNames.imports, queueNames.exports]);
});
test("webhook group covers browser automation", () => {
@@ -90,7 +90,7 @@ describe("bootstrapWorkerGroup", () => {
bootstrapWorkerGroup({ WORKER_QUEUE_GROUP: "import", WORKER_CONCURRENCY_IMPORT: "2" })
).toEqual({
group: "import",
queueNames: [queueNames.imports, queueNames.exports],
queueNames: [queueNames.kwpImports, queueNames.imports, queueNames.exports],
concurrency: 2,
concurrencyEnvVar: "WORKER_CONCURRENCY_IMPORT",
});
@@ -0,0 +1,124 @@
// @vitest-environment node
import { beforeEach, describe, expect, test, vi } from "vitest";
const integrationMocks = vi.hoisted(() => ({
cleanupExpiredKwpPendingAttemptsForTenant: vi.fn(),
executeKwpHealthProbe: vi.fn(),
listKwpHealthConnectionIdsForTenant: vi.fn(),
persistKwpHealthProbe: vi.fn(),
prepareKwpHealthProbeForConnection: vi.fn(),
refreshExpiringTokensForTenant: vi.fn(),
sweepConnectionHealthForTenant: vi.fn(),
}));
const kwpResourceMocks = vi.hoisted(() => ({
ensureKwpAccessTokenForConnection: vi.fn(),
}));
const dbMocks = vi.hoisted(() => ({ getDbClient: vi.fn() }));
vi.mock("@hivegrid/integrations/worker-tasks.server", () => integrationMocks);
vi.mock("@hivegrid/integrations/kwp-resource-client.server", () => kwpResourceMocks);
vi.mock("@hivegrid/db/client.server", () => dbMocks);
import { buildCleanupJob, queueNames } from "@hivegrid/queue";
import { createSharedStore, MemoryAdapter } from "@hivegrid/shared-store";
import { buildHandlerRegistry } from "./handlers.js";
describe("KWP demand-driven refresh maintenance", () => {
beforeEach(() => {
vi.clearAllMocks();
integrationMocks.cleanupExpiredKwpPendingAttemptsForTenant.mockResolvedValue({ restored: 0, purged: 0, failed: 0 });
integrationMocks.refreshExpiringTokensForTenant.mockResolvedValue({ refreshed: 2, failed: 0 });
integrationMocks.sweepConnectionHealthForTenant.mockResolvedValue({ checked: 0, unhealthy: 0 });
integrationMocks.listKwpHealthConnectionIdsForTenant.mockResolvedValue([]);
kwpResourceMocks.ensureKwpAccessTokenForConnection.mockResolvedValue(undefined);
});
test("keeps KWP out of the minute refresh sweep while retaining generic refresh and pending cleanup", async () => {
const tx = { $executeRaw: vi.fn(async () => 1) };
const db = {
tenant: { findMany: vi.fn(async () => [{ id: "tenant_1" }]) },
$transaction: vi.fn(async <T>(callback: (transaction: typeof tx) => Promise<T>) => callback(tx)),
};
dbMocks.getDbClient.mockResolvedValue(db);
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
env: { KWP_HIVEGRID_TENANT_ID: "tenant_1" },
logger,
tenantMaintenanceLeaseStore: createSharedStore({ namespace: "kwp-demand-refresh-handler", adapter: new MemoryAdapter() }),
});
await expect(
registry[queueNames.maintenance](buildCleanupJob({ task: "integrations-token-refresh", olderThanMinutes: 10 }).payload, {}),
).resolves.toEqual({
handled: true,
action: "integrations.token-refresh.completed",
details: {
task: "integrations-token-refresh",
tenants: 1,
refreshed: 2,
failed: 0,
retryScheduled: 0,
pendingRestored: 0,
pendingPurged: 0,
pendingCleanupFailed: 0,
},
});
expect(integrationMocks.refreshExpiringTokensForTenant).toHaveBeenCalledWith(
tx,
"tenant_1",
{ now: expect.any(Date), windowMs: 10 * 60_000 },
);
expect(integrationMocks.cleanupExpiredKwpPendingAttemptsForTenant).toHaveBeenCalledWith(
tx,
"tenant_1",
{ now: expect.any(Date) },
);
expect(kwpResourceMocks.ensureKwpAccessTokenForConnection).not.toHaveBeenCalled();
});
test("ensures one demand-refreshed token before the scheduled KWP health probe", async () => {
const tx = { $executeRaw: vi.fn(async () => 1) };
const db = {
tenant: { findMany: vi.fn(async () => [{ id: "tenant_1" }]) },
$transaction: vi.fn(async <T>(callback: (transaction: typeof tx) => Promise<T>) => callback(tx)),
};
dbMocks.getDbClient.mockResolvedValue(db);
integrationMocks.listKwpHealthConnectionIdsForTenant.mockResolvedValue(["kwp_connection_1"]);
const snapshot = { tenantId: "tenant_1", connectionId: "kwp_connection_1" };
integrationMocks.prepareKwpHealthProbeForConnection.mockResolvedValue({ state: "ready", snapshot });
integrationMocks.executeKwpHealthProbe.mockResolvedValue({ health: "active", messageKey: "integrations.kwp.health.active" });
integrationMocks.persistKwpHealthProbe.mockResolvedValue({ checked: 1, unhealthy: 0 });
const registry = buildHandlerRegistry({
env: { KWP_HIVEGRID_TENANT_ID: "tenant_1" },
logger: { info: vi.fn(), error: vi.fn() },
tenantMaintenanceLeaseStore: createSharedStore({ namespace: "kwp-demand-health-handler", adapter: new MemoryAdapter() }),
});
await expect(
registry[queueNames.maintenance](buildCleanupJob({ task: "integrations-health-sweep" }).payload, {}),
).resolves.toMatchObject({
action: "integrations.health-sweep.completed",
details: { checked: 1, unhealthy: 0 },
});
expect(kwpResourceMocks.ensureKwpAccessTokenForConnection).toHaveBeenCalledWith({
connectionId: "kwp_connection_1",
allowPaused: true,
});
expect(kwpResourceMocks.ensureKwpAccessTokenForConnection.mock.invocationCallOrder[0]).toBeLessThan(
integrationMocks.prepareKwpHealthProbeForConnection.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER,
);
expect(integrationMocks.prepareKwpHealthProbeForConnection).toHaveBeenCalledWith(
tx,
"tenant_1",
"kwp_connection_1",
{ now: expect.any(Date), kwpHiveGridTenantId: "tenant_1" },
);
expect(integrationMocks.listKwpHealthConnectionIdsForTenant).toHaveBeenCalledWith(
tx,
"tenant_1",
{ now: expect.any(Date) },
);
});
});
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -53,8 +53,8 @@ describe("worker health handler", () => {
test("/readyz returns 200 when both probes succeed", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
workerGroup: "mail",
});
const { res, captured } = createMockResponse();
@@ -68,10 +68,27 @@ describe("worker health handler", () => {
expect(body.checks).toEqual({ redis: "ok", database: "ok" });
});
test("/readyz clears timeout timers after fast probes complete", async () => {
vi.useFakeTimers();
try {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res } = createMockResponse();
await handler(createMockReq("/readyz"), res);
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
}
});
test("/readyz returns 503 when redis probe fails", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: false, error: "ECONNREFUSED" } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: false, error: "ECONNREFUSED" }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res, captured } = createMockResponse();
@@ -86,7 +103,7 @@ describe("worker health handler", () => {
test("/readyz returns 503 when database probe throws", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => {
throw new Error("pg disconnected");
},
@@ -101,11 +118,11 @@ describe("worker health handler", () => {
test("/readyz includes required runtime probes in readiness status", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
readinessProbes: {
outboxDispatcher: async () => ({ ok: true } as const),
workflowTrigger: async () => ({ ok: false, error: "subscriber disabled" } as const),
outboxDispatcher: async () => ({ ok: true }) as const,
workflowTrigger: async () => ({ ok: false, error: "subscriber disabled" }) as const,
},
workerGroup: "general",
});
@@ -129,7 +146,7 @@ describe("worker health handler", () => {
test("/readyz times out a hanging probe", async () => {
const handler = createWorkerHealthHandler({
redisProbe: () => new Promise(() => undefined),
databaseProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true }) as const,
probeTimeoutMs: 20,
});
const { res, captured } = createMockResponse();
@@ -142,8 +159,8 @@ describe("worker health handler", () => {
test("unknown routes return 404 JSON", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res, captured } = createMockResponse();
@@ -155,8 +172,8 @@ describe("worker health handler", () => {
test("/healthz omits workerGroup when none is configured", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res, captured } = createMockResponse();
@@ -134,16 +134,21 @@ export function startWorkerHealthServer(options: WorkerHealthServerOptions): Wor
}
async function runWithTimeout(probe: HealthProbe, timeoutMs: number, label: string): Promise<HealthProbeResult> {
let timeout: NodeJS.Timeout | undefined;
try {
const result = await Promise.race([
Promise.resolve().then(() => probe()),
new Promise<HealthProbeResult>((resolve) =>
setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs)
),
new Promise<HealthProbeResult>((resolve) => {
timeout = setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs);
}),
]);
return result;
} catch (error) {
return { ok: false, error: serialiseError(error).message };
} finally {
if (timeout) {
clearTimeout(timeout);
}
}
}
@@ -11,6 +11,16 @@ const mailjetEnv = {
MAILJET_SECRET_KEY: "fixture-mailjet-secret",
MAILJET_FROM_EMAIL: "sender@example.com",
};
const cryptoSecret = "shared-tenant-envelope-secret-value";
const gdprExportSseCKey = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
const storageEnv = {
STORAGE_ACCESS_KEY_ID: "access-key",
STORAGE_BUCKET: "exports",
STORAGE_ENDPOINT: "https://storage.example.com",
STORAGE_REGION: "eu-central-1",
STORAGE_SECRET_ACCESS_KEY: "secret-key",
GDPR_EXPORT_SSE_C_KEY_BASE64: gdprExportSseCKey,
};
function readPackageScripts(): Record<string, string> {
return JSON.parse(readFileSync(packageJsonPath, "utf8")).scripts;
@@ -171,6 +181,7 @@ describe("worker health", () => {
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
CRYPTO_SECRET: cryptoSecret,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "mail",
...mailjetEnv,
@@ -455,6 +466,27 @@ describe("worker health", () => {
);
});
test("reports mail-worker readiness misconfigured when the shared envelope key is absent", async () => {
const databaseProbe = vi.fn();
const redisProbe = vi.fn();
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
...mailjetEnv,
TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true",
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "mail",
},
{ databaseProbe, redisProbe }
);
expect(report.status).toBe("misconfigured");
expect(report.errors).toContain("CRYPTO_SECRET:missing-required:CRYPTO_SECRET is required.");
expect(databaseProbe).not.toHaveBeenCalled();
expect(redisProbe).not.toHaveBeenCalled();
});
test("reports misconfigured and skips DB probe when enabled worker lacks DATABASE_URL", async () => {
const databaseProbe = vi.fn();
const report = await buildWorkerHealthReport(
@@ -518,7 +550,7 @@ describe("worker health", () => {
expect(report.storageStatus).toBe("ready");
});
test("keeps the import worker group ready but idle until import runtime gates are enabled", async () => {
test("keeps the import worker ready with only KWP while generic import gates are disabled", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
@@ -531,11 +563,140 @@ describe("worker health", () => {
);
expect(report.status).toBe("ready");
expect(report.queues).toEqual([]);
expect(report.queues).toEqual([queueNames.kwpImports]);
expect(report.importStatus).toBe("skipped");
expect(report.errors).not.toEqual(expect.arrayContaining([expect.stringMatching(/missing-import/u)]));
});
test("reports the dedicated portal GDPR export mode ready with KWP and exports ownership", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report).toMatchObject({
status: "ready",
importConfigured: true,
importStatus: "ready",
storageConfigured: true,
storageStatus: "ready",
workerGroup: "import",
});
expect(report.queues).toEqual([queueNames.kwpImports, queueNames.exports]);
expect(report.errors).toEqual([]);
});
test("fails dedicated portal GDPR export readiness without complete storage", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.queues).toEqual([queueNames.kwpImports, queueNames.exports]);
expect(report.storageStatus).toBe("misconfigured");
expect(report.errors).toEqual(
expect.arrayContaining([
"STORAGE_ENDPOINT:missing-storage-config:Required when PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true.",
"GDPR_EXPORT_SSE_C_KEY_BASE64:missing-storage-config:Required when PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true.",
])
);
});
test("fails full import readiness without the GDPR SSE-C key", async () => {
const { GDPR_EXPORT_SSE_C_KEY_BASE64: _key, ...storageWithoutKey } = storageEnv;
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
IMPORT_QUEUE_ENABLED: "true",
IMPORT_QUEUE_RUNTIME_READY: "true",
...storageWithoutKey,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.storageStatus).toBe("misconfigured");
expect(report.errors).toEqual(
expect.arrayContaining(["GDPR_EXPORT_SSE_C_KEY_BASE64:missing-storage-config:Required when the full import runtime owns GDPR export jobs."])
);
});
test("reports the full import runtime ready only with encrypted GDPR storage", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
IMPORT_QUEUE_ENABLED: "true",
IMPORT_QUEUE_RUNTIME_READY: "true",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report).toMatchObject({ status: "ready", importStatus: "ready", storageStatus: "ready" });
expect(report.queues).toEqual([queueNames.kwpImports, queueNames.imports, queueNames.exports]);
});
test("fails dedicated portal GDPR export readiness on the wrong worker group", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "general",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.errors).toEqual(
expect.arrayContaining([
"PORTAL_GDPR_EXPORT_QUEUE_ENABLED:invalid-worker-queue-group:PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true requires WORKER_QUEUE_GROUP=import.",
])
);
});
test("fails closed instead of becoming healthy-idle for a whitespace-padded dedicated gate", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: " true ",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.queues).toEqual([]);
expect(report.errors).toEqual(
expect.arrayContaining(["PORTAL_GDPR_EXPORT_QUEUE_ENABLED:invalid-boolean:PORTAL_GDPR_EXPORT_QUEUE_ENABLED must be exactly true or false."])
);
});
test("reports misconfigured when import worker group is partially enabled", async () => {
const report = await buildWorkerHealthReport(
{
@@ -2,8 +2,8 @@ import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type Wor
import { getDbClient } from "@hivegrid/db/client.server";
import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@hivegrid/queue";
import { Queue } from "bullmq";
import { getWorkerQueueNames } from "./queues.js";
import { getMissingWorkerStorageEnv } from "./storage.js";
import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js";
import { getMissingWorkerGdprExportStorageEnv, getMissingWorkerStorageEnv } from "./storage.js";
export type WorkerHealthEnv = EnvRecord;
@@ -69,7 +69,10 @@ export async function buildWorkerHealthReport(
checkRedisReadiness(shouldProbeRedis, config.redis.url, dependencies.redisProbe),
checkDatabaseReadiness(shouldProbeDatabase, dependencies.databaseProbe),
]);
const storageReadiness = checkStorageReadiness(config.enabled && env.EXPORT_RETENTION_QUEUE_ENABLED === "true", env);
const storageReadiness = checkStorageReadiness(
config.enabled && (env.EXPORT_RETENTION_QUEUE_ENABLED === "true" || ["full", "portal-gdpr-export"].includes(resolveImportQueueRuntimeMode(env))),
env
);
const mailReadiness = checkMailReadiness(config.enabled, config.mailjet.configured, isMailProviderRequired(env), env);
const importReadiness = checkImportReadiness(config.enabled && result.ok && getWorkerGroup(env) === "import", env);
const pluginRuntimeReadiness = checkPluginRuntimeReadiness(config.enabled && result.ok, getWorkerGroup(env), env);
@@ -309,10 +312,15 @@ function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { con
const importEnabled = env.IMPORT_QUEUE_ENABLED === "true";
const runtimeReady = env.IMPORT_QUEUE_RUNTIME_READY === "true";
if (!importEnabled && !runtimeReady) {
const mode = resolveImportQueueRuntimeMode(env);
if (mode === "idle") {
return { configured: false, errors: [], status: "skipped" };
}
if (mode === "portal-gdpr-export" || mode === "full") {
return { configured: true, errors: [], status: "ready" };
}
if (!importEnabled || !runtimeReady) {
const errors: string[] = [];
if (!importEnabled) {
@@ -332,7 +340,11 @@ function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { con
};
}
return { configured: true, errors: [], status: "ready" };
return {
configured: false,
errors: ["WORKER_QUEUE_GROUP:invalid-import-runtime-mode:Import worker queue gates do not resolve to an allowed runtime mode."],
status: "misconfigured",
};
}
function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthStorageStatus } {
@@ -340,11 +352,18 @@ function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { co
return { configured: false, errors: [], status: "skipped" };
}
const missing = getMissingWorkerStorageEnv(env);
const importMode = resolveImportQueueRuntimeMode(env);
const ownsGdprExports = importMode === "full" || importMode === "portal-gdpr-export";
const missing = ownsGdprExports ? getMissingWorkerGdprExportStorageEnv(env) : getMissingWorkerStorageEnv(env);
if (missing.length > 0) {
const requiredBy = [
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? ["EXPORT_RETENTION_QUEUE_ENABLED=true"] : []),
...(importMode === "portal-gdpr-export" ? ["PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true"] : []),
...(importMode === "full" ? ["the full import runtime owns GDPR export jobs"] : []),
].join(" or ");
return {
configured: false,
errors: missing.map((name) => `${name}:missing-storage-config:Required when EXPORT_RETENTION_QUEUE_ENABLED=true.`),
errors: missing.map((name) => `${name}:missing-storage-config:Required when ${requiredBy}.`),
status: "misconfigured",
};
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,109 @@
import { describe, expect, test, vi } from "vitest";
import {
ACTIVE_TENANT_PAGE_SIZE,
runActiveTenantSweep,
TenantSweepCoordinator,
throwOnActiveTenantSweepFailures,
type ActiveTenantReader,
} from "./active-tenant-sweep.js";
function createReader(ids: string[]) {
const findMany = vi.fn(async (args: { take: number; cursor?: { id: string }; skip?: 1 }) => {
const start = args.cursor ? ids.indexOf(args.cursor.id) + (args.skip ?? 0) : 0;
return ids.slice(start, start + args.take).map((id) => ({ id }));
});
return { reader: { tenant: { findMany } } as ActiveTenantReader, findMany };
}
describe("active tenant sweep", () => {
test("keeps active-only as the default and can include deactivated tenants for legal cleanup", async () => {
const activeOnly = createReader(["tenant-active"]);
await runActiveTenantSweep(activeOnly.reader, {
initialState: 0,
worker: async () => 1,
merge: (total, value) => total + value,
});
expect(activeOnly.findMany.mock.calls[0]?.[0]).toMatchObject({ where: { active: true } });
const allTenants = createReader(["tenant-inactive"]);
await runActiveTenantSweep(allTenants.reader, {
initialState: 0,
includeInactive: true,
worker: async () => 1,
merge: (total, value) => total + value,
});
expect(allTenants.findMany.mock.calls[0]?.[0]).toMatchObject({ where: {} });
});
test("uses stable keyset pages and reduces results without retaining the full tenant list", async () => {
const ids = Array.from({ length: ACTIVE_TENANT_PAGE_SIZE * 2 + 7 }, (_, index) => `tenant-${String(index).padStart(4, "0")}`);
const { reader, findMany } = createReader(ids);
const result = await runActiveTenantSweep(reader, {
initialState: 0,
concurrency: 1,
worker: async () => 1,
merge: (total, value) => total + value,
});
expect(result).toEqual({ state: ids.length, inspected: ids.length, succeeded: ids.length, failures: [], pages: 3 });
expect(findMany).toHaveBeenCalledTimes(3);
expect(findMany.mock.calls[0]?.[0]).not.toHaveProperty("cursor");
expect(findMany.mock.calls[1]?.[0]).toMatchObject({ cursor: { id: ids[ACTIVE_TENANT_PAGE_SIZE - 1] }, skip: 1 });
expect(findMany.mock.calls[2]?.[0]).toMatchObject({ cursor: { id: ids[ACTIVE_TENANT_PAGE_SIZE * 2 - 1] }, skip: 1 });
});
test("shares a strict concurrency budget across overlapping cron sweeps", async () => {
const coordinator = new TenantSweepCoordinator(2);
const first = createReader(["a", "b", "c"]).reader;
const second = createReader(["d", "e", "f"]).reader;
let active = 0;
let peak = 0;
let started = 0;
const releases: Array<() => void> = [];
const worker = async () => {
started += 1;
active += 1;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => releases.push(resolve));
active -= 1;
return 1;
};
const runs = [first, second].map((reader) =>
runActiveTenantSweep(reader, { initialState: 0, coordinator, worker, merge: (total, value) => total + value })
);
await vi.waitFor(() => expect(releases).toHaveLength(2));
while (started < 6) {
releases.splice(0).forEach((release) => release());
await vi.waitFor(() => expect(started).toBeGreaterThan(0));
if (started < 6) await vi.waitFor(() => expect(releases.length).toBeGreaterThan(0));
}
releases.splice(0).forEach((release) => release());
await Promise.all(runs);
expect(peak).toBe(2);
});
test("continues after tenant failures, redacts diagnostics, and reports every failure for retry", async () => {
const { reader } = createReader(["good-a", "bad-token", "good-b", "bad-password"]);
const result = await runActiveTenantSweep(reader, {
initialState: [] as string[],
worker: async (tenantId) => {
if (tenantId.startsWith("bad")) throw new Error(`${tenantId} token=raw-secret password=hunter2`);
return tenantId;
},
merge: (completed, tenantId) => [...completed, tenantId],
});
expect(result.state).toEqual(["good-a", "good-b"]);
expect(result.inspected).toBe(4);
expect(result.succeeded).toBe(2);
expect(result.failures).toEqual([
{ tenantId: "bad-token", error: "bad-token token=[redacted] password=[redacted]" },
{ tenantId: "bad-password", error: "bad-password token=[redacted] password=[redacted]" },
]);
expect(() => throwOnActiveTenantSweepFailures("maintenance", result.failures)).toThrow("maintenance failed for 2 tenant(s)");
});
});
@@ -0,0 +1,171 @@
import { mapWithConcurrency } from "@hivegrid/foundation";
export const ACTIVE_TENANT_PAGE_SIZE = 100;
export const DEFAULT_ACTIVE_TENANT_CONCURRENCY = 2;
export const MAX_ACTIVE_TENANT_CONCURRENCY = 4;
type ActiveTenantRow = { id: string };
export type ActiveTenantReader = {
tenant: {
findMany(args: {
where: { active?: true };
select: { id: true };
orderBy: { id: "asc" };
take: number;
cursor?: { id: string };
skip?: 1;
}): Promise<ActiveTenantRow[]>;
};
};
export type ActiveTenantSweepFailure = {
tenantId: string;
error: string;
};
export type ActiveTenantSweepResult<State> = {
state: State;
inspected: number;
succeeded: number;
failures: ActiveTenantSweepFailure[];
pages: number;
};
type ActiveTenantSweepOptions<State, Value> = {
initialState: State;
worker: (tenantId: string) => Promise<Value>;
merge: (state: State, value: Value) => State;
pageSize?: number;
concurrency?: number;
coordinator?: TenantSweepCoordinator;
/** Include deactivated tenants when legal/financial cleanup must outlive tenant access. */
includeInactive?: boolean;
};
function resolveBoundedInteger(raw: string | undefined, fallback: number, max: number): number {
const parsed = raw === undefined ? Number.NaN : Number(raw);
if (!Number.isSafeInteger(parsed) || parsed < 1) return fallback;
return Math.min(parsed, max);
}
/**
* A process-wide semaphore keeps nested fan-out below the database pool even
* when several BullMQ maintenance jobs happen to run at the same time.
*/
export class TenantSweepCoordinator {
readonly concurrency: number;
private active = 0;
private readonly waiters: Array<() => void> = [];
constructor(concurrency: number = DEFAULT_ACTIVE_TENANT_CONCURRENCY) {
this.concurrency = Math.max(1, Math.min(Math.floor(concurrency), MAX_ACTIVE_TENANT_CONCURRENCY));
}
async run<Value>(worker: () => Promise<Value>): Promise<Value> {
await this.acquire();
try {
return await worker();
} finally {
this.release();
}
}
private async acquire(): Promise<void> {
if (this.active < this.concurrency) {
this.active += 1;
return;
}
await new Promise<void>((resolve) => this.waiters.push(resolve));
}
private release(): void {
const next = this.waiters.shift();
if (next) {
next();
return;
}
this.active -= 1;
}
}
const defaultCoordinator = new TenantSweepCoordinator(
resolveBoundedInteger(process.env.WORKER_TENANT_SWEEP_CONCURRENCY, DEFAULT_ACTIVE_TENANT_CONCURRENCY, MAX_ACTIVE_TENANT_CONCURRENCY)
);
/**
* Walk active tenants (or every tenant for cleanup obligations) with a stable
* id cursor and bounded concurrency. Only one page and its outcomes are retained
* at a time, so tenant growth cannot turn a cron tick into an unbounded read or
* promise fan-out.
*/
export async function runActiveTenantSweep<State, Value>(
client: ActiveTenantReader,
options: ActiveTenantSweepOptions<State, Value>
): Promise<ActiveTenantSweepResult<State>> {
const pageSize = Math.max(1, Math.min(Math.floor(options.pageSize ?? ACTIVE_TENANT_PAGE_SIZE), 1_000));
const coordinator = options.coordinator ?? defaultCoordinator;
const concurrency = Math.max(1, Math.min(Math.floor(options.concurrency ?? coordinator.concurrency), coordinator.concurrency));
const failures: ActiveTenantSweepFailure[] = [];
let state = options.initialState;
let inspected = 0;
let succeeded = 0;
let pages = 0;
let cursor: string | undefined;
while (true) {
const rows = await client.tenant.findMany({
where: options.includeInactive ? {} : { active: true },
select: { id: true },
orderBy: { id: "asc" },
take: pageSize,
...(cursor ? { cursor: { id: cursor }, skip: 1 as const } : {}),
});
if (rows.length === 0) break;
pages += 1;
inspected += rows.length;
const outcomes = new Array<{ ok: true; value: Value } | { ok: false; failure: ActiveTenantSweepFailure }>(rows.length);
await mapWithConcurrency(rows, concurrency, async (row, index) => {
try {
const value = await coordinator.run(() => options.worker(row.id));
outcomes[index] = { ok: true, value };
} catch (error) {
outcomes[index] = {
ok: false,
failure: { tenantId: row.id, error: redactTenantSweepError(error) },
};
}
});
for (const outcome of outcomes) {
if (outcome.ok) {
state = options.merge(state, outcome.value);
succeeded += 1;
} else {
failures.push(outcome.failure);
}
}
cursor = rows.at(-1)?.id;
if (rows.length < pageSize || !cursor) break;
}
return { state, inspected, succeeded, failures, pages };
}
export function throwOnActiveTenantSweepFailures(task: string, failures: readonly ActiveTenantSweepFailure[]): void {
if (failures.length === 0) return;
throw new AggregateError(
failures.map((failure) => new Error(`${failure.tenantId}: ${failure.error}`)),
`${task} failed for ${failures.length} tenant(s)`
);
}
function redactTenantSweepError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
.replace(/(token|secret|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
.slice(0, 500);
}
@@ -0,0 +1,71 @@
import { describe, expect, it, vi } from "vitest";
import {
bootstrapAdminBookingOutbox,
type AdminBookingOutboxWorkerHandle,
} from "./admin-booking-outbox-bootstrap.js";
describe("bootstrapAdminBookingOutbox", () => {
it("only subscribes on the general worker group", () => {
expect(
bootstrapAdminBookingOutbox(
{ WORKER_QUEUE_GROUP: "mail" },
{}
)
).toEqual({
status: "disabled",
reason: "non-general worker group",
});
});
it("binds the admin identity before processing the provider command", async () => {
let processor:
| ((job: { readonly data: unknown }) => Promise<unknown>)
| undefined;
const handle: AdminBookingOutboxWorkerHandle = {
name: "outbox.booking",
close: vi.fn().mockResolvedValue(undefined),
};
const loadEvent = vi.fn().mockResolvedValue({
bookingId: "booking_1",
adminUserId: "admin_1",
calendarConnectionId: "calendar_1",
provider: "google_calendar",
accessToken: "access",
calendarId: "primary",
subject: "Demo",
startsAt: new Date("2026-08-01T10:00:00.000Z"),
endsAt: new Date("2026-08-01T10:30:00.000Z"),
timeZone: "Europe/Berlin",
attendee: { email: "lead@example.test", name: "Ada Lead" },
});
const result = bootstrapAdminBookingOutbox(
{ WORKER_QUEUE_GROUP: "general" },
{
dependencies: {
loadEvent,
createEvent: vi
.fn()
.mockResolvedValue({ externalEventId: "provider_1" }),
updateEvent: vi.fn(),
deleteEvent: vi.fn(),
confirm: vi.fn(),
markFailed: vi.fn(),
},
createWorker: (_queue, next) => {
processor = next;
return handle;
},
}
);
expect(result).toEqual({ status: "started", handle });
await processor?.({
data: {
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
});
expect(loadEvent).toHaveBeenCalledWith("booking_1", "admin_1");
});
});
@@ -0,0 +1,88 @@
import type { EnvRecord } from "@hivegrid/config";
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
import {
ADMIN_BOOKING_OUTBOX_QUEUE,
parseAdminBookingOutboxEnvelope,
runAdminBookingOutboxJob,
type AdminBookingOutboxDependencies,
} from "./admin-booking-outbox.js";
export type AdminBookingOutboxBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
ADMIN_BOOKING_OUTBOX_ENABLED?: string;
ADMIN_BOOKING_OUTBOX_CONCURRENCY?: string;
};
export type AdminBookingOutboxWorkerHandle = {
readonly name: string;
close(): Promise<void>;
};
export type AdminBookingOutboxBootstrapResult =
| { readonly status: "disabled"; readonly reason: string }
| {
readonly status: "started";
readonly handle: AdminBookingOutboxWorkerHandle;
};
export function bootstrapAdminBookingOutbox(
env: AdminBookingOutboxBootstrapEnv,
input: {
readonly dependencies?: AdminBookingOutboxDependencies;
readonly createWorker?: (
queueName: string,
processor: (job: { readonly data: unknown }) => Promise<unknown>,
options: { readonly concurrency: number }
) => AdminBookingOutboxWorkerHandle;
readonly logger?: {
info?: (
record: Record<string, unknown>,
message?: string
) => void;
};
}
): AdminBookingOutboxBootstrapResult {
if (env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.ADMIN_BOOKING_OUTBOX_ENABLED === "false") {
return {
status: "disabled",
reason: "ADMIN_BOOKING_OUTBOX_ENABLED=false",
};
}
if (!input.dependencies || !input.createWorker) {
return { status: "disabled", reason: "runtime dependencies unavailable" };
}
const dependencies = input.dependencies;
const concurrency = positiveInteger(
env.ADMIN_BOOKING_OUTBOX_CONCURRENCY,
5
);
const handle = input.createWorker(
ADMIN_BOOKING_OUTBOX_QUEUE,
(job) => {
const envelope = parseAdminBookingOutboxEnvelope(job.data);
return runWithRlsContext(
{ userId: envelope.payload.adminUserId },
() => runAdminBookingOutboxJob(job.data, dependencies)
);
},
{ concurrency }
);
input.logger?.info?.(
{
component: "admin-booking-outbox",
queue: ADMIN_BOOKING_OUTBOX_QUEUE,
concurrency,
},
"admin booking provider worker started"
);
return { status: "started", handle };
}
function positiveInteger(value: string | undefined, fallback: number): number {
const parsed = Number(value);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
}
@@ -0,0 +1,162 @@
import { describe, expect, it, vi } from "vitest";
import {
parseAdminBookingOutboxEnvelope,
runAdminBookingOutboxJob,
type AdminBookingOutboxDependencies,
type AdminBookingProviderEvent,
} from "./admin-booking-outbox.js";
const event: AdminBookingProviderEvent = {
bookingId: "booking_1",
adminUserId: "admin_1",
calendarConnectionId: "calendar_1",
provider: "google_calendar",
accessToken: "access",
calendarId: "primary",
subject: "Demo",
startsAt: new Date("2026-08-01T10:00:00.000Z"),
endsAt: new Date("2026-08-01T10:30:00.000Z"),
timeZone: "Europe/Berlin",
attendee: { email: "lead@example.test", name: "Ada Lead" },
};
function dependencies(
overrides: Partial<AdminBookingOutboxDependencies> = {}
): AdminBookingOutboxDependencies {
return {
loadEvent: vi.fn().mockResolvedValue(event),
createEvent: vi
.fn()
.mockResolvedValue({ externalEventId: "provider_1", opaqueVersion: "v1" }),
updateEvent: vi
.fn()
.mockResolvedValue({ externalEventId: "provider_1", opaqueVersion: "v2" }),
deleteEvent: vi.fn().mockResolvedValue(undefined),
confirm: vi.fn().mockResolvedValue(undefined),
markFailed: vi.fn().mockResolvedValue(undefined),
now: () => new Date("2026-07-25T12:00:00.000Z"),
...overrides,
};
}
describe("admin booking outbox", () => {
it("creates and confirms a provider event idempotently", async () => {
const deps = dependencies();
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
deps
)
).resolves.toEqual({ status: "created", bookingId: "booking_1" });
expect(deps.createEvent).toHaveBeenCalledWith(event);
expect(deps.confirm).toHaveBeenCalledWith(
expect.objectContaining({
providerEventId: "provider_1",
providerEventVersion: "v1",
})
);
});
it("deduplicates a replay after provider confirmation", async () => {
const deps = dependencies({
loadEvent: vi
.fn()
.mockResolvedValue({ ...event, externalEventId: "provider_1" }),
});
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
deps
)
).resolves.toEqual({
status: "deduplicated",
bookingId: "booking_1",
});
expect(deps.createEvent).not.toHaveBeenCalled();
});
it("updates the original provider event during rescheduling", async () => {
const deps = dependencies({
loadEvent: vi
.fn()
.mockResolvedValueOnce({
...event,
externalEventId: "provider_1",
externalEventVersion: "v1",
})
.mockResolvedValueOnce({ ...event, bookingId: "booking_2" }),
});
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.reschedule",
payload: {
previousBookingId: "booking_1",
replacementBookingId: "booking_2",
adminUserId: "admin_1",
},
},
deps
)
).resolves.toEqual({
status: "rescheduled",
previousBookingId: "booking_1",
replacementBookingId: "booking_2",
});
expect(deps.updateEvent).toHaveBeenCalledWith(
expect.objectContaining({ bookingId: "booking_2" }),
"provider_1",
"v1"
);
});
it("records a redacted provider failure and preserves the retry", async () => {
const deps = dependencies({
createEvent: vi
.fn()
.mockRejectedValue(Object.assign(new Error("secret"), { code: "provider_outage" })),
});
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
deps
)
).rejects.toThrow("secret");
expect(deps.markFailed).toHaveBeenCalledWith({
bookingId: "booking_1",
adminUserId: "admin_1",
errorCode: "provider_outage",
});
});
it("rejects tenant or payload confusion before touching a provider", () => {
expect(() =>
parseAdminBookingOutboxEnvelope({
tenantId: "tenant_1",
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
})
).toThrow("invalid_envelope");
expect(() =>
parseAdminBookingOutboxEnvelope({
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "../booking", adminUserId: "admin_1" },
})
).toThrow("invalid_bookingId");
});
});
@@ -0,0 +1,270 @@
export const ADMIN_BOOKING_OUTBOX_QUEUE = "outbox.booking" as const;
export type AdminBookingProviderEvent = {
readonly bookingId: string;
readonly adminUserId: string;
readonly calendarConnectionId: string;
readonly provider: "google_calendar" | "microsoft_calendar";
readonly accessToken: string;
readonly externalEventId?: string | null;
readonly externalEventVersion?: string | null;
readonly calendarId: string;
readonly subject: string;
readonly bodyText?: string | null;
readonly startsAt: Date;
readonly endsAt: Date;
readonly timeZone: string;
readonly attendee: {
readonly email: string;
readonly name: string;
};
};
type CreateEnvelope = {
readonly eventType: "booking.provider.create";
readonly payload: {
readonly bookingId: string;
readonly adminUserId: string;
};
};
type CancelEnvelope = {
readonly eventType: "booking.provider.cancel";
readonly payload: {
readonly bookingId: string;
readonly adminUserId: string;
};
};
type RescheduleEnvelope = {
readonly eventType: "booking.provider.reschedule";
readonly payload: {
readonly previousBookingId: string;
readonly replacementBookingId: string;
readonly adminUserId: string;
};
};
export type AdminBookingOutboxEnvelope =
| CreateEnvelope
| CancelEnvelope
| RescheduleEnvelope;
export type AdminBookingOutboxDependencies = {
loadEvent(
bookingId: string,
adminUserId: string
): Promise<AdminBookingProviderEvent>;
createEvent(
event: AdminBookingProviderEvent
): Promise<{
readonly externalEventId: string;
readonly opaqueVersion?: string | null;
}>;
updateEvent(
event: AdminBookingProviderEvent,
externalEventId: string,
opaqueVersion?: string | null
): Promise<{
readonly externalEventId: string;
readonly opaqueVersion?: string | null;
}>;
deleteEvent(event: AdminBookingProviderEvent): Promise<void>;
confirm(input: {
readonly bookingId: string;
readonly adminUserId: string;
readonly calendarConnectionId: string;
readonly providerEventId: string;
readonly providerEventVersion?: string | null;
readonly syncedAt: Date;
}): Promise<void>;
markFailed(input: {
readonly bookingId: string;
readonly adminUserId: string;
readonly errorCode: string;
}): Promise<void>;
readonly now?: () => Date;
};
export async function runAdminBookingOutboxJob(
data: unknown,
dependencies: AdminBookingOutboxDependencies
): Promise<
| { readonly status: "created"; readonly bookingId: string }
| { readonly status: "cancelled"; readonly bookingId: string }
| {
readonly status: "rescheduled";
readonly previousBookingId: string;
readonly replacementBookingId: string;
}
| { readonly status: "deduplicated"; readonly bookingId: string }
> {
const envelope = parseAdminBookingOutboxEnvelope(data);
const now = dependencies.now ?? (() => new Date());
if (envelope.eventType === "booking.provider.create") {
const event = await dependencies.loadEvent(
envelope.payload.bookingId,
envelope.payload.adminUserId
);
if (event.externalEventId) {
return {
status: "deduplicated",
bookingId: envelope.payload.bookingId,
};
}
try {
const result = await dependencies.createEvent(event);
await dependencies.confirm({
bookingId: event.bookingId,
adminUserId: event.adminUserId,
calendarConnectionId: event.calendarConnectionId,
providerEventId: result.externalEventId,
providerEventVersion: result.opaqueVersion,
syncedAt: now(),
});
return { status: "created", bookingId: event.bookingId };
} catch (error) {
await markFailedSafely(dependencies, event, error);
throw error;
}
}
if (envelope.eventType === "booking.provider.cancel") {
const event = await dependencies.loadEvent(
envelope.payload.bookingId,
envelope.payload.adminUserId
);
if (!event.externalEventId) {
return {
status: "deduplicated",
bookingId: envelope.payload.bookingId,
};
}
await dependencies.deleteEvent(event);
return { status: "cancelled", bookingId: event.bookingId };
}
const previous = await dependencies.loadEvent(
envelope.payload.previousBookingId,
envelope.payload.adminUserId
);
const replacement = await dependencies.loadEvent(
envelope.payload.replacementBookingId,
envelope.payload.adminUserId
);
try {
const result = previous.externalEventId
? await dependencies.updateEvent(
replacement,
previous.externalEventId,
previous.externalEventVersion
)
: await dependencies.createEvent(replacement);
await dependencies.confirm({
bookingId: replacement.bookingId,
adminUserId: replacement.adminUserId,
calendarConnectionId: replacement.calendarConnectionId,
providerEventId: result.externalEventId,
providerEventVersion: result.opaqueVersion,
syncedAt: now(),
});
return {
status: "rescheduled",
previousBookingId: previous.bookingId,
replacementBookingId: replacement.bookingId,
};
} catch (error) {
await markFailedSafely(dependencies, replacement, error);
throw error;
}
}
export function parseAdminBookingOutboxEnvelope(
data: unknown
): AdminBookingOutboxEnvelope {
const envelope = record(data, "envelope");
if (
envelope.tenantId !== null ||
![
"booking.provider.create",
"booking.provider.cancel",
"booking.provider.reschedule",
].includes(String(envelope.eventType))
) {
throw new AdminBookingOutboxError("invalid_envelope");
}
const payload = record(envelope.payload, "payload");
const adminUserId = identifier(payload.adminUserId, "adminUserId");
if (envelope.eventType === "booking.provider.reschedule") {
return {
eventType: "booking.provider.reschedule",
payload: {
previousBookingId: identifier(
payload.previousBookingId,
"previousBookingId"
),
replacementBookingId: identifier(
payload.replacementBookingId,
"replacementBookingId"
),
adminUserId,
},
};
}
const bookingId = identifier(payload.bookingId, "bookingId");
return {
eventType: envelope.eventType as
| "booking.provider.create"
| "booking.provider.cancel",
payload: { bookingId, adminUserId },
};
}
async function markFailedSafely(
dependencies: AdminBookingOutboxDependencies,
event: AdminBookingProviderEvent,
error: unknown
): Promise<void> {
await dependencies
.markFailed({
bookingId: event.bookingId,
adminUserId: event.adminUserId,
errorCode: providerErrorCode(error),
})
.catch(() => undefined);
}
function providerErrorCode(error: unknown): string {
if (
error &&
typeof error === "object" &&
"code" in error &&
typeof error.code === "string"
) {
return error.code.slice(0, 100);
}
return error instanceof Error ? error.name.slice(0, 100) : "provider_error";
}
function record(value: unknown, field: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new AdminBookingOutboxError(`invalid_${field}`);
}
return value as Record<string, unknown>;
}
function identifier(value: unknown, field: string): string {
const normalized = typeof value === "string" ? value.trim() : "";
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,190}$/u.test(normalized)) {
throw new AdminBookingOutboxError(`invalid_${field}`);
}
return normalized;
}
export class AdminBookingOutboxError extends Error {
constructor(readonly code: string) {
super(code);
this.name = "AdminBookingOutboxError";
}
}
@@ -0,0 +1,29 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS,
ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
registerAdminBookingReminderSweepCron,
} from "./admin-booking-reminder-cron.js";
describe("registerAdminBookingReminderSweepCron", () => {
it("registers a bounded one-minute reminder sweep", async () => {
const upsertJobScheduler = vi.fn().mockResolvedValue(undefined);
await registerAdminBookingReminderSweepCron(
{ upsertJobScheduler },
{ ADMIN_BOOKING_REMINDER_SWEEP_BATCH_SIZE: "5000" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
take: 1000,
},
}
);
});
});
@@ -0,0 +1,32 @@
export const ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID =
"admin-booking-reminder-sweep" as const;
export const ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS = 60_000;
export async function registerAdminBookingReminderSweepCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const parsed = Number(env.ADMIN_BOOKING_REMINDER_SWEEP_BATCH_SIZE);
const take =
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 1_000)
: 250;
await queue.upsertJobScheduler(
ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
take,
},
}
);
}
@@ -0,0 +1,23 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_CALENDAR_SYNC_EVERY_MS,
ADMIN_CALENDAR_SYNC_JOB_ID,
registerAdminCalendarSyncCron,
} from "./admin-calendar-sync-cron.js";
describe("registerAdminCalendarSyncCron", () => {
it("registers a bounded five-minute sweep", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminCalendarSyncCron(
{ upsertJobScheduler },
{ ADMIN_CALENDAR_SYNC_BATCH_SIZE: "500" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_CALENDAR_SYNC_JOB_ID,
{ every: ADMIN_CALENDAR_SYNC_EVERY_MS },
expect.objectContaining({
data: expect.objectContaining({ take: 100 }),
})
);
});
});
@@ -0,0 +1,31 @@
export const ADMIN_CALENDAR_SYNC_JOB_ID = "admin-calendar-sync" as const;
export const ADMIN_CALENDAR_SYNC_EVERY_MS = 5 * 60 * 1_000;
export async function registerAdminCalendarSyncCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
) {
const configured = Number(env.ADMIN_CALENDAR_SYNC_BATCH_SIZE);
const take =
Number.isFinite(configured) && configured > 0
? Math.min(Math.floor(configured), 100)
: 25;
await queue.upsertJobScheduler(
ADMIN_CALENDAR_SYNC_JOB_ID,
{ every: ADMIN_CALENDAR_SYNC_EVERY_MS },
{
name: ADMIN_CALENDAR_SYNC_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALENDAR_SYNC_JOB_ID,
take,
},
}
);
}
@@ -0,0 +1,42 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_CALL_RECORDING_PROCESS_EVERY_MS,
ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
ADMIN_CALL_RECORDING_RETENTION_EVERY_MS,
ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
registerAdminCallRecordingCrons,
} from "./admin-call-recording-cron.js";
describe("admin call recording schedulers", () => {
it("registers prompt processing and hourly retention as stable jobs", async () => {
const upsertJobScheduler = vi.fn().mockResolvedValue(undefined);
await registerAdminCallRecordingCrons({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
{ every: ADMIN_CALL_RECORDING_PROCESS_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
take: 10,
},
}
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
{ every: ADMIN_CALL_RECORDING_RETENTION_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
take: 50,
},
}
);
});
});
@@ -0,0 +1,45 @@
export const ADMIN_CALL_RECORDING_PROCESS_JOB_ID =
"admin-call-recording-process" as const;
export const ADMIN_CALL_RECORDING_RETENTION_JOB_ID =
"admin-call-recording-retention" as const;
export const ADMIN_CALL_RECORDING_PROCESS_EVERY_MS = 60 * 1_000;
export const ADMIN_CALL_RECORDING_RETENTION_EVERY_MS = 60 * 60 * 1_000;
type RecordingScheduler = {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
};
export async function registerAdminCallRecordingCrons(
queue: RecordingScheduler
): Promise<void> {
await Promise.all([
queue.upsertJobScheduler(
ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
{ every: ADMIN_CALL_RECORDING_PROCESS_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
take: 10,
},
}
),
queue.upsertJobScheduler(
ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
{ every: ADMIN_CALL_RECORDING_RETENTION_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
take: 50,
},
}
),
]);
}
@@ -0,0 +1,41 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE,
ADMIN_EMAIL_DELIVERY_EVERY_MS,
ADMIN_EMAIL_DELIVERY_JOB_ID,
registerAdminEmailDeliveryCron,
resolveAdminEmailDeliveryConfig,
} from "./admin-email-delivery-cron.js";
describe("admin email delivery cron", () => {
it("keeps the delivery batch bounded", () => {
expect(resolveAdminEmailDeliveryConfig({})).toEqual({
take: ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE,
});
expect(
resolveAdminEmailDeliveryConfig({
ADMIN_EMAIL_DELIVERY_BATCH_SIZE: "500",
})
).toEqual({ take: 250 });
});
it("registers one idempotent minute scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailDeliveryCron(
{ upsertJobScheduler },
{ ADMIN_EMAIL_DELIVERY_BATCH_SIZE: "25" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_DELIVERY_JOB_ID,
{ every: ADMIN_EMAIL_DELIVERY_EVERY_MS },
{
name: ADMIN_EMAIL_DELIVERY_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DELIVERY_JOB_ID,
take: 25,
},
}
);
});
});
@@ -0,0 +1,40 @@
export const ADMIN_EMAIL_DELIVERY_JOB_ID = "admin-email-delivery" as const;
export const ADMIN_EMAIL_DELIVERY_EVERY_MS = 60 * 1_000;
export const ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE = 50;
export function resolveAdminEmailDeliveryConfig(
env: NodeJS.ProcessEnv = process.env
): { readonly take: number } {
const parsed = Number(env.ADMIN_EMAIL_DELIVERY_BATCH_SIZE);
return {
take:
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 250)
: ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE,
};
}
export async function registerAdminEmailDeliveryCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const config = resolveAdminEmailDeliveryConfig(env);
await queue.upsertJobScheduler(
ADMIN_EMAIL_DELIVERY_JOB_ID,
{ every: ADMIN_EMAIL_DELIVERY_EVERY_MS },
{
name: ADMIN_EMAIL_DELIVERY_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DELIVERY_JOB_ID,
take: config.take,
},
}
);
}
@@ -0,0 +1,28 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_DOMAIN_EVERY_MS,
ADMIN_EMAIL_DOMAIN_JOB_ID,
registerAdminEmailDomainCron,
} from "./admin-email-domain-cron.js";
describe("admin email sending-domain cron", () => {
it("registers one bounded idempotent DNS scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailDomainCron(
{ upsertJobScheduler },
{ ADMIN_EMAIL_DOMAIN_BATCH_SIZE: "500" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_DOMAIN_JOB_ID,
{ every: ADMIN_EMAIL_DOMAIN_EVERY_MS },
{
name: ADMIN_EMAIL_DOMAIN_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DOMAIN_JOB_ID,
take: 200,
},
}
);
});
});
@@ -0,0 +1,32 @@
export const ADMIN_EMAIL_DOMAIN_JOB_ID = "admin-email-domain-check" as const;
export const ADMIN_EMAIL_DOMAIN_EVERY_MS = 15 * 60 * 1_000;
export const ADMIN_EMAIL_DOMAIN_DEFAULT_BATCH_SIZE = 50;
export async function registerAdminEmailDomainCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const parsed = Number(env.ADMIN_EMAIL_DOMAIN_BATCH_SIZE);
const take =
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 200)
: ADMIN_EMAIL_DOMAIN_DEFAULT_BATCH_SIZE;
await queue.upsertJobScheduler(
ADMIN_EMAIL_DOMAIN_JOB_ID,
{ every: ADMIN_EMAIL_DOMAIN_EVERY_MS },
{
name: ADMIN_EMAIL_DOMAIN_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DOMAIN_JOB_ID,
take,
},
}
);
}
@@ -0,0 +1,27 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE,
ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS,
ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
registerAdminEmailEventRetentionCron,
} from "./admin-email-event-retention-cron.js";
describe("admin email event retention cron", () => {
it("registers one idempotent daily redaction scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailEventRetentionCron({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
{ every: ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS },
{
name: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
take: ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE,
},
}
);
});
});
@@ -0,0 +1,27 @@
export const ADMIN_EMAIL_EVENT_RETENTION_JOB_ID =
"admin-email-event-retention" as const;
export const ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS = 24 * 60 * 60 * 1_000;
export const ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE = 500;
export async function registerAdminEmailEventRetentionCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
}
): Promise<void> {
await queue.upsertJobScheduler(
ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
{ every: ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS },
{
name: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
take: ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE,
},
}
);
}
@@ -0,0 +1,39 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE,
ADMIN_EMAIL_SYNC_EVERY_MS,
ADMIN_EMAIL_SYNC_JOB_ID,
registerAdminEmailSyncCron,
resolveAdminEmailSyncConfig,
} from "./admin-email-sync-cron.js";
describe("admin email sync cron", () => {
it("uses a bounded mailbox cursor batch", () => {
expect(resolveAdminEmailSyncConfig({})).toEqual({
take: ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE,
});
expect(
resolveAdminEmailSyncConfig({ ADMIN_EMAIL_SYNC_BATCH_SIZE: "500" })
).toEqual({ take: 100 });
});
it("registers one idempotent two-minute scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailSyncCron(
{ upsertJobScheduler },
{ ADMIN_EMAIL_SYNC_BATCH_SIZE: "20" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_SYNC_JOB_ID,
{ every: ADMIN_EMAIL_SYNC_EVERY_MS },
{
name: ADMIN_EMAIL_SYNC_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_SYNC_JOB_ID,
take: 20,
},
}
);
});
});
@@ -0,0 +1,40 @@
export const ADMIN_EMAIL_SYNC_JOB_ID = "admin-email-sync" as const;
export const ADMIN_EMAIL_SYNC_EVERY_MS = 2 * 60 * 1_000;
export const ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE = 25;
export function resolveAdminEmailSyncConfig(
env: NodeJS.ProcessEnv = process.env
): { readonly take: number } {
const parsed = Number(env.ADMIN_EMAIL_SYNC_BATCH_SIZE);
return {
take:
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 100)
: ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE,
};
}
export async function registerAdminEmailSyncCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const config = resolveAdminEmailSyncConfig(env);
await queue.upsertJobScheduler(
ADMIN_EMAIL_SYNC_JOB_ID,
{ every: ADMIN_EMAIL_SYNC_EVERY_MS },
{
name: ADMIN_EMAIL_SYNC_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_SYNC_JOB_ID,
take: config.take,
},
}
);
}
@@ -0,0 +1,26 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_OUTBOUND_CALL_EVERY_MS,
ADMIN_OUTBOUND_CALL_JOB_ID,
registerAdminOutboundCallCron,
} from "./admin-outbound-call-cron.js";
describe("admin outbound call scheduler", () => {
it("registers one deterministic minute scheduler", async () => {
const queue = { upsertJobScheduler: vi.fn(async () => undefined) };
await registerAdminOutboundCallCron(queue);
expect(queue.upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_OUTBOUND_CALL_JOB_ID,
{ every: ADMIN_OUTBOUND_CALL_EVERY_MS },
{
name: ADMIN_OUTBOUND_CALL_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_OUTBOUND_CALL_JOB_ID,
take: 25,
},
}
);
});
});
@@ -0,0 +1,28 @@
export const ADMIN_OUTBOUND_CALL_JOB_ID =
"admin-outbound-call-process" as const;
export const ADMIN_OUTBOUND_CALL_EVERY_MS = 60 * 1_000;
type OutboundCallScheduler = {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
};
export async function registerAdminOutboundCallCron(
queue: OutboundCallScheduler
): Promise<void> {
await queue.upsertJobScheduler(
ADMIN_OUTBOUND_CALL_JOB_ID,
{ every: ADMIN_OUTBOUND_CALL_EVERY_MS },
{
name: ADMIN_OUTBOUND_CALL_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_OUTBOUND_CALL_JOB_ID,
take: 25,
},
}
);
}
@@ -0,0 +1,180 @@
import {
GetObjectCommand,
PutObjectCommand,
} from "@aws-sdk/client-s3";
import {
processAdminSalesContactsExport,
processAdminSalesContactsImport,
} from "@hivegrid/db/admin-sales-data.server";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createAdminSalesDataWorkerService } from "./admin-sales-data.js";
vi.mock("@hivegrid/db/admin-sales-data.server", () => ({
processAdminSalesContactsExport: vi.fn(),
processAdminSalesContactsImport: vi.fn(),
}));
const prisma = {
adminImportJob: { findFirst: vi.fn() },
adminExportJob: { findFirst: vi.fn() },
};
describe("admin sales data worker service", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("fails closed when the shared storage bucket is not configured", () => {
expect(
createAdminSalesDataWorkerService(
{},
{ prisma: prisma as never, s3Client: { send: vi.fn() } as never }
)
).toBeUndefined();
});
it("loads a bounded reviewed CSV and stores its error report encrypted", async () => {
prisma.adminImportJob.findFirst.mockResolvedValue({
sourceBucket: "hivegrid-files",
sourceKey: "admin-sales/imports/admin_1/import_1.csv",
});
const send = vi.fn(async (command: unknown) => {
if (command instanceof GetObjectCommand) {
return {
Body: {
transformToByteArray: async () =>
new TextEncoder().encode(
"Vorname,Nachname\nAda,Lovelace\n"
),
},
};
}
return {};
});
vi.mocked(processAdminSalesContactsImport).mockImplementation(
async (_client, input) => {
const artifact = await input.storeErrorReport({
importJobId: input.importJobId,
csvText: "Zeile,Fehler\n2,Dublette\n",
});
expect(artifact).toEqual({
bucket: "hivegrid-files",
key: "admin-sales/import-errors/import_1.csv",
});
return {
totalRows: 1,
acceptedRows: 0,
rejectedRows: 1,
errorReport: artifact,
};
}
);
const service = createAdminSalesDataWorkerService(
{ STORAGE_BUCKET: "hivegrid-files" },
{
prisma: prisma as never,
s3Client: { send } as never,
now: () => new Date("2026-07-24T12:00:00Z"),
}
);
await expect(
service!.runImport({
kind: "admin-sales.contacts-import",
importJobId: "import_1",
requestedByAdminUserId: "admin_1",
})
).resolves.toEqual({
totalRows: 1,
acceptedRows: 0,
rejectedRows: 1,
});
expect(processAdminSalesContactsImport).toHaveBeenCalledWith(
prisma,
expect.objectContaining({
importJobId: "import_1",
csvText: "Vorname,Nachname\nAda,Lovelace\n",
now: new Date("2026-07-24T12:00:00Z"),
})
);
const put = send.mock.calls
.map(([command]) => command)
.find((command) => command instanceof PutObjectCommand);
expect((put as PutObjectCommand).input).toEqual(
expect.objectContaining({
Bucket: "hivegrid-files",
Key: "admin-sales/import-errors/import_1.csv",
ContentType: "text/csv; charset=utf-8",
ServerSideEncryption: "AES256",
})
);
});
it("stores exports with a stable encrypted download artifact", async () => {
prisma.adminExportJob.findFirst.mockResolvedValue({ id: "export_1" });
const send = vi.fn(async (_command: unknown) => ({}));
vi.mocked(processAdminSalesContactsExport).mockImplementation(
async (_client, input) => {
const artifact = await input.storeArtifact({
exportJobId: input.exportJobId,
csvText: "\uFEFFVorname,Nachname\nAda,Lovelace\n",
});
return {
rowCount: 1,
artifact,
expiresAt: new Date("2026-07-31T12:00:00Z"),
};
}
);
const service = createAdminSalesDataWorkerService(
{ STORAGE_BUCKET: "hivegrid-files" },
{ prisma: prisma as never, s3Client: { send } as never }
);
await expect(
service!.runExport({
kind: "admin-sales.contacts-export",
exportJobId: "export_1",
requestedByAdminUserId: "admin_1",
})
).resolves.toEqual({
rowCount: 1,
artifactBucket: "hivegrid-files",
artifactKey: "admin-sales/exports/export_1.csv",
expiresAt: new Date("2026-07-31T12:00:00Z"),
});
const put = send.mock.calls[0]?.[0];
expect(put).toBeInstanceOf(PutObjectCommand);
expect((put as PutObjectCommand).input).toEqual(
expect.objectContaining({
Bucket: "hivegrid-files",
Key: "admin-sales/exports/export_1.csv",
ContentDisposition:
'attachment; filename="admin-sales-contacts.csv"',
ServerSideEncryption: "AES256",
})
);
});
it("rejects an import object outside the configured bucket", async () => {
prisma.adminImportJob.findFirst.mockResolvedValue({
sourceBucket: "other-bucket",
sourceKey: "contacts.csv",
});
const service = createAdminSalesDataWorkerService(
{ STORAGE_BUCKET: "hivegrid-files" },
{
prisma: prisma as never,
s3Client: { send: vi.fn() } as never,
}
);
await expect(
service!.runImport({
kind: "admin-sales.contacts-import",
importJobId: "import_1",
requestedByAdminUserId: "admin_1",
})
).rejects.toThrow("source bucket is not allowed");
});
});
@@ -0,0 +1,172 @@
import {
GetObjectCommand,
PutObjectCommand,
S3Client,
} from "@aws-sdk/client-s3";
import {
processAdminSalesContactsExport,
processAdminSalesContactsImport,
} from "@hivegrid/db/admin-sales-data.server";
import type {
AdminSalesContactsExportJob,
AdminSalesContactsImportJob,
} from "@hivegrid/queue";
const MAX_SOURCE_BYTES = 5 * 1024 * 1024;
export type AdminSalesDataWorkerEnv = {
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_BUCKET?: string;
STORAGE_ENDPOINT?: string;
STORAGE_FORCE_PATH_STYLE?: string;
STORAGE_REGION?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
};
export type AdminSalesDataWorkerService = {
runImport(payload: AdminSalesContactsImportJob): Promise<{
readonly totalRows: number;
readonly acceptedRows: number;
readonly rejectedRows: number;
}>;
runExport(payload: AdminSalesContactsExportJob): Promise<{
readonly rowCount: number;
readonly artifactBucket: string;
readonly artifactKey: string;
readonly expiresAt: Date;
}>;
};
type AdminSalesDataPrisma = Parameters<
typeof processAdminSalesContactsImport
>[0];
export function createAdminSalesDataWorkerService(
env: AdminSalesDataWorkerEnv,
dependencies: {
readonly prisma: AdminSalesDataPrisma;
readonly s3Client?: Pick<S3Client, "send">;
readonly now?: () => Date;
}
): AdminSalesDataWorkerService | undefined {
const bucket = env.STORAGE_BUCKET?.trim();
const s3 = dependencies.s3Client ?? createS3Client(env);
if (!bucket || !s3) return undefined;
const now = dependencies.now ?? (() => new Date());
return {
async runImport(payload) {
const job = await dependencies.prisma.adminImportJob.findFirst({
where: {
id: payload.importJobId,
requestedByAdminUserId: payload.requestedByAdminUserId,
},
select: { sourceBucket: true, sourceKey: true },
});
if (!job) throw new Error("Admin sales import job was not found.");
if (job.sourceBucket !== bucket) {
throw new Error("Admin sales import source bucket is not allowed.");
}
const response = await s3.send(
new GetObjectCommand({ Bucket: bucket, Key: job.sourceKey })
);
const bytes = await response.Body?.transformToByteArray();
if (!bytes) throw new Error("Admin sales import source is empty.");
if (bytes.byteLength > MAX_SOURCE_BYTES) {
throw new Error("Admin sales import source exceeds the 5 MB limit.");
}
const result = await processAdminSalesContactsImport(
dependencies.prisma,
{
importJobId: payload.importJobId,
csvText: new TextDecoder("utf-8", { fatal: true }).decode(bytes),
now: now(),
storeErrorReport: async ({ importJobId, csvText }) => {
const key = `admin-sales/import-errors/${importJobId}.csv`;
await s3.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: Buffer.from(csvText, "utf8"),
ContentType: "text/csv; charset=utf-8",
ServerSideEncryption: "AES256",
Metadata: { importJobId },
})
);
return { bucket, key };
},
}
);
return {
totalRows: result.totalRows,
acceptedRows: result.acceptedRows,
rejectedRows: result.rejectedRows,
};
},
async runExport(payload) {
const job = await dependencies.prisma.adminExportJob.findFirst({
where: {
id: payload.exportJobId,
requestedByAdminUserId: payload.requestedByAdminUserId,
},
select: { id: true },
});
if (!job) throw new Error("Admin sales export job was not found.");
const result = await processAdminSalesContactsExport(
dependencies.prisma,
{
exportJobId: payload.exportJobId,
now: now(),
storeArtifact: async ({ exportJobId, csvText }) => {
const key = `admin-sales/exports/${exportJobId}.csv`;
await s3.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: Buffer.from(csvText, "utf8"),
ContentType: "text/csv; charset=utf-8",
ContentDisposition:
'attachment; filename="admin-sales-contacts.csv"',
ServerSideEncryption: "AES256",
Metadata: { exportJobId },
})
);
return { bucket, key };
},
}
);
return {
rowCount: result.rowCount,
artifactBucket: result.artifact.bucket,
artifactKey: result.artifact.key,
expiresAt: result.expiresAt,
};
},
};
}
function createS3Client(
env: AdminSalesDataWorkerEnv
): S3Client | undefined {
if (
!env.STORAGE_REGION ||
!env.STORAGE_ACCESS_KEY_ID ||
!env.STORAGE_SECRET_ACCESS_KEY
) {
return undefined;
}
return new S3Client({
endpoint: env.STORAGE_ENDPOINT,
region: env.STORAGE_REGION,
forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === "true",
credentials: {
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY,
...(env.STORAGE_SESSION_TOKEN
? { sessionToken: env.STORAGE_SESSION_TOKEN }
: {}),
},
});
}
@@ -0,0 +1,49 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS,
ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
registerAdminSalesTaskReminderSweepCron,
resolveAdminSalesTaskReminderSweepConfig,
} from "./admin-sales-task-reminder-cron.js";
describe("admin sales task reminder cron", () => {
it("uses a bounded explicit batch size", () => {
expect(resolveAdminSalesTaskReminderSweepConfig({})).toEqual({
take: ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
});
expect(
resolveAdminSalesTaskReminderSweepConfig({
ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE: "75",
})
).toEqual({ take: 75 });
expect(
resolveAdminSalesTaskReminderSweepConfig({
ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE: "invalid",
})
).toEqual({ take: ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE });
});
it("registers the idempotent 15-minute maintenance scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminSalesTaskReminderSweepCron(
{ upsertJobScheduler },
{ ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE: "25" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
take: 25,
},
}
);
});
});
@@ -0,0 +1,41 @@
export const ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID =
"admin-sales-task-reminder-sweep" as const;
export const ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS = 15 * 60 * 1000;
export const ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE = 500;
export function resolveAdminSalesTaskReminderSweepConfig(
env: NodeJS.ProcessEnv = process.env
): { readonly take: number } {
const parsed = Number(env.ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE);
return {
take:
Number.isFinite(parsed) && parsed > 0
? Math.floor(parsed)
: ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
};
}
export async function registerAdminSalesTaskReminderSweepCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const config = resolveAdminSalesTaskReminderSweepConfig(env);
await queue.upsertJobScheduler(
ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
take: config.take,
},
}
);
}
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
import {
bootstrapAuditEventPersist,
@@ -66,14 +67,43 @@ describe("bootstrapAuditEventPersist", () => {
return fakeWorker();
});
const createWorkerImpl = inner as unknown as WorkerImpl;
const runInTransaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) =>
fn({
const payload = { tenantId: "tenant_1", action: "x.do", actorType: "system" };
const lockedUntil = "2099-07-17T10:05:00.000Z";
let observedTenantId: string | undefined;
const runInTransaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => {
observedTenantId = getRlsExecutionContext()?.tenantId;
return fn({
$queryRaw: async () => [],
auditEvent: {
findFirst: async () => null,
create: async () => ({ id: "audit_1", eventHash: "h1" }),
},
})
);
auditChainHead: {
findUnique: async () => null,
upsert: async () => undefined,
},
auditChainCheckpointHead: {
findUnique: async () => null,
},
auditChainCheckpoint: {
findUnique: async () => null,
findFirst: async () => null,
},
outboxEvent: {
findFirst: async () => ({
id: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload,
status: "PROCESSING",
deliveryGeneration: 0,
dispatchKey: null,
lockedUntil: new Date(lockedUntil),
}),
updateMany: async () => ({ count: 1 }),
},
});
});
bootstrapAuditEventPersist({ WORKER_QUEUE_GROUP: "general" }, { runInTransaction: runInTransaction as never, createWorkerImpl });
expect(captured).not.toBeNull();
@@ -83,10 +113,14 @@ describe("bootstrapAuditEventPersist", () => {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload: { action: "x.do", actorType: "system" },
payload,
dispatchKey: null,
deliveryGeneration: 0,
lockedUntil,
},
});
}
expect(runInTransaction).toHaveBeenCalledTimes(1);
expect(observedTenantId).toBe("tenant_1");
});
});
@@ -25,6 +25,7 @@ import {
type AuditEventPersistLogger,
type AuditEventPersistTransactionRunner,
} from "./audit-event-persist.js";
import { runWorkerPayloadWithRlsContext } from "../rls-context.js";
export type AuditEventPersistBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
@@ -85,7 +86,7 @@ export function bootstrapAuditEventPersist(
runInTransaction,
...(logger ? { logger } : {}),
};
return runAuditEventPersistJob(job.data, handlerDeps);
return runWorkerPayloadWithRlsContext(job.data, () => runAuditEventPersistJob(job.data, handlerDeps));
};
const createWorkerImpl = deps.createWorkerImpl ?? defaultCreateWorker;
@@ -1,11 +1,13 @@
import { describe, expect, it, vi } from "vitest";
import { computeAuditHash } from "@hivegrid/db/audit-hash";
import type { AuditChainHeadRecord } from "@hivegrid/db/audit-chain-checkpoint.server";
import {
AUDIT_EVENT_OUTBOX_QUEUE,
AuditEventPersistJobError,
parseAuditEventPersistJob,
runAuditEventPersistJob,
type AuditEventPersistDependencies,
type AuditEventPersistTransactionRunner,
type AuditEventPersistTxClient,
} from "./audit-event-persist.js";
@@ -19,64 +21,219 @@ type PersistedRow = {
createdAt: Date;
};
function makeStore(): {
const TEST_NOW = new Date("2026-07-17T10:00:00.000Z");
const TEST_LEASE = new Date("2026-07-17T10:05:00.000Z");
const basePayload = {
tenantId: "tenant_1",
actorId: "user_1",
actorType: "user",
actorEmail: null,
actorIp: null,
actorUserAgent: null,
action: "widget.created",
targetType: "widget",
targetId: "widget_1",
requestId: null,
sourceModule: "crm",
outcome: "success",
dataSensitivity: "internal",
regulatoryScope: ["soc2"],
metadata: null,
};
function auditJob(overrides: Record<string, unknown> = {}) {
return {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload: basePayload,
dispatchKey: null,
deliveryGeneration: 0,
lockedUntil: TEST_LEASE.toISOString(),
...overrides,
};
}
const baseJob = auditJob();
type DurableOutbox = {
id: string;
tenantId: string | null;
eventType: string;
payload: unknown;
status: "PENDING" | "PROCESSING" | "DONE" | "FAILED" | "CANCELLED";
deliveryGeneration: number;
dispatchKey: string | null;
lockedUntil: Date | null;
processedAt: Date | null;
lastError: string | null;
};
function makeStore(
jobs: ReturnType<typeof auditJob>[] = [baseJob],
options: { zeroAtUpdate?: number } = {},
): {
rows: PersistedRow[];
runInTransaction: AuditEventPersistTransactionRunner;
register(job: ReturnType<typeof auditJob>, overrides?: Partial<DurableOutbox>): void;
outbox: Map<string, DurableOutbox>;
updateMany: ReturnType<typeof vi.fn>;
} {
const rows: PersistedRow[] = [];
let idSeq = 0;
let chainSeq = 0;
const heads = new Map<string, AuditChainHeadRecord>();
const outbox = new Map<string, DurableOutbox>();
const register = (job: ReturnType<typeof auditJob>, overrides: Partial<DurableOutbox> = {}) => {
outbox.set(job.outboxEventId, {
id: job.outboxEventId,
tenantId: job.tenantId as string | null,
eventType: job.eventType,
payload: job.payload,
status: "PROCESSING",
deliveryGeneration: job.deliveryGeneration as number,
dispatchKey: job.dispatchKey as string | null,
lockedUntil: new Date(job.lockedUntil as string),
processedAt: null,
lastError: null,
...overrides,
});
};
jobs.forEach((job) => register(job));
let updateAttempt = 0;
const findFirst = vi.fn(
async (args: Parameters<AuditEventPersistTxClient["outboxEvent"]["findFirst"]>[0]) => {
const row = outbox.get(args.where.id);
if (
!row ||
row.tenantId !== args.where.tenantId ||
row.eventType !== args.where.eventType ||
row.deliveryGeneration !== args.where.deliveryGeneration
) {
return null;
}
return { ...row };
},
);
const updateMany = vi.fn(
async (args: Parameters<AuditEventPersistTxClient["outboxEvent"]["updateMany"]>[0]) => {
updateAttempt += 1;
if (options.zeroAtUpdate === updateAttempt) return { count: 0 };
const row = outbox.get(args.where.id);
if (
!row ||
row.tenantId !== args.where.tenantId ||
row.eventType !== args.where.eventType ||
row.deliveryGeneration !== args.where.deliveryGeneration ||
row.dispatchKey !== args.where.dispatchKey ||
row.status !== args.where.status ||
row.lockedUntil?.getTime() !== args.where.lockedUntil.getTime() ||
!sameJson(row.payload, args.where.payload.equals)
) {
return { count: 0 };
}
if ("status" in args.data) {
row.status = args.data.status;
row.processedAt = args.data.processedAt;
row.lastError = args.data.lastError;
row.lockedUntil = args.data.lockedUntil;
} else {
row.lockedUntil = args.data.lockedUntil;
}
return { count: 1 };
},
);
const tx: AuditEventPersistTxClient = {
$queryRaw: async <T = unknown>() => [] as T,
auditEvent: {
async findFirst({ where }) {
const tenantRows = rows.filter((r) => r.tenantId === where.tenantId && r.eventHash !== null);
if (tenantRows.length === 0) return null;
// newest first
tenantRows.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
return { eventHash: tenantRows[0]?.eventHash ?? null };
return tenantRows[0] ? { eventHash: tenantRows[0].eventHash, createdAt: tenantRows[0].createdAt } : null;
},
async create({ data }) {
idSeq += 1;
chainSeq += 1;
const row: PersistedRow = {
id: `audit_${idSeq}`,
id: data.id,
tenantId: data.tenantId,
action: data.action,
eventHash: data.eventHash,
previousHash: data.previousHash,
createdAt: new Date(2026, 4, 17, 0, 0, 0, chainSeq),
createdAt: data.createdAt,
};
rows.push(row);
return { id: row.id, eventHash: row.eventHash };
},
},
auditChainCheckpoint: {
async findFirst() {
return null;
},
async findUnique() {
return null;
},
},
auditChainCheckpointHead: {
async findUnique() {
return null;
},
},
auditChainHead: {
async findUnique({ where }) {
return heads.get(where.chainKey) ?? null;
},
async upsert({ where, create, update }) {
const value = heads.has(where.chainKey) ? { chainKey: where.chainKey, ...update } : create;
heads.set(where.chainKey, value);
return undefined;
},
},
outboxEvent: { findFirst, updateMany },
};
const runInTransaction: AuditEventPersistTransactionRunner = async <T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T> => fn(tx);
return { rows, runInTransaction };
const runInTransaction: AuditEventPersistTransactionRunner = async <T>(
fn: (tx: AuditEventPersistTxClient) => Promise<T>,
): Promise<T> => {
const rowsSnapshot = rows.map((row) => ({ ...row }));
const headsSnapshot = new Map([...heads].map(([key, value]) => [key, { ...value }]));
const outboxSnapshot = new Map(
[...outbox].map(([key, value]) => [key, { ...value, lockedUntil: value.lockedUntil ? new Date(value.lockedUntil) : null }]),
);
try {
return await fn(tx);
} catch (error) {
rows.splice(0, rows.length, ...rowsSnapshot);
heads.clear();
headsSnapshot.forEach((value, key) => heads.set(key, value));
outbox.clear();
outboxSnapshot.forEach((value, key) => outbox.set(key, value));
throw error;
}
};
return { rows, runInTransaction, register, outbox, updateMany };
}
const baseJob = {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload: {
tenantId: "tenant_1",
actorId: "user_1",
actorType: "user",
actorEmail: null,
actorIp: null,
actorUserAgent: null,
action: "widget.created",
targetType: "widget",
targetId: "widget_1",
requestId: null,
sourceModule: "crm",
outcome: "success",
dataSensitivity: "internal",
regulatoryScope: ["soc2"],
metadata: null,
},
};
function sameJson(left: unknown, right: unknown): boolean {
const sort = (value: unknown): unknown => {
if (Array.isArray(value)) return value.map(sort);
if (value && typeof value === "object") {
return Object.fromEntries(
Object.keys(value as Record<string, unknown>)
.sort()
.map((key) => [key, sort((value as Record<string, unknown>)[key])]),
);
}
return value;
};
return JSON.stringify(sort(left)) === JSON.stringify(sort(right));
}
function auditDependencies(
runInTransaction: AuditEventPersistTransactionRunner,
overrides: Partial<AuditEventPersistDependencies> = {},
): AuditEventPersistDependencies {
return { runInTransaction, now: () => TEST_NOW, ...overrides };
}
describe("AUDIT_EVENT_OUTBOX_QUEUE", () => {
it("matches the outbox audit queue name", () => {
@@ -104,46 +261,164 @@ describe("parseAuditEventPersistJob", () => {
});
it("normalizes payload through buildAuditEventCreateData", () => {
const parsed = parseAuditEventPersistJob({
eventType: "audit.event.create",
tenantId: "tenant_1",
payload: { action: "x.do", actorType: " ", regulatoryScope: ["SOC2"] },
});
const parsed = parseAuditEventPersistJob(auditJob({
payload: { tenantId: "tenant_1", action: "x.do", actorType: " ", regulatoryScope: ["SOC2"] },
}));
expect(parsed.payload.actorType).toBe("system");
expect(parsed.payload.regulatoryScope).toEqual(["soc2"]);
expect(parsed.payload.outcome).toBe("success");
expect(parsed.payload.dataSensitivity).toBe("internal");
});
it("requires the full durable outbox identity including dispatch, generation, and lease", () => {
for (const key of ["outboxEventId", "dispatchKey", "deliveryGeneration", "lockedUntil"] as const) {
const invalid = { ...baseJob } as Record<string, unknown>;
delete invalid[key];
expect(() => parseAuditEventPersistJob(invalid)).toThrowError(
expect.objectContaining({ code: "INVALID_PAYLOAD" }),
);
}
});
it("rejects a non-global payload whose tenant differs from the outbox envelope", () => {
expect(() =>
parseAuditEventPersistJob(auditJob({
tenantId: "tenant_2",
payload: { ...basePayload, tenantId: "tenant_1" },
})),
).toThrowError(expect.objectContaining({ code: "INVALID_PAYLOAD" }));
});
});
describe("runAuditEventPersistJob", () => {
it("persists the event with prevHash=null on chain root", async () => {
const { rows, runInTransaction } = makeStore();
const result = await runAuditEventPersistJob(baseJob, { runInTransaction });
expect(result).toEqual({ persisted: true, auditEventId: "audit_1", eventHash: expect.stringMatching(/^[0-9a-f]{64}$/) });
const { rows, runInTransaction, outbox, updateMany } = makeStore();
const result = await runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction));
expect(result).toEqual({ persisted: true, auditEventId: expect.any(String), eventHash: expect.stringMatching(/^v2:[0-9a-f]{64}$/) });
expect(rows).toHaveLength(1);
expect(rows[0]?.previousHash).toBeNull();
expect(outbox.get("outbox_1")).toMatchObject({ status: "DONE", lockedUntil: null, lastError: null });
expect(updateMany).toHaveBeenCalledTimes(2);
});
it("rejects a forged Redis job with no durable outbox row", async () => {
const { rows, runInTransaction } = makeStore([]);
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(rows).toEqual([]);
});
it("rejects tenant, generation, dispatch-key, and canonical payload mismatches", async () => {
const cases = [
auditJob({ tenantId: "tenant_2", payload: { ...basePayload, tenantId: "tenant_2" } }),
auditJob({ deliveryGeneration: 1 }),
auditJob({ dispatchKey: "audit:forged" }),
auditJob({ payload: { ...basePayload, action: "widget.forged" } }),
];
for (const forged of cases) {
const { rows, runInTransaction } = makeStore([baseJob]);
await expect(
runAuditEventPersistJob(forged, auditDependencies(runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(rows).toEqual([]);
}
});
it("rejects PENDING, DONE, and stale terminal generations", async () => {
for (const status of ["PENDING", "DONE", "FAILED", "CANCELLED"] as const) {
const state = makeStore([baseJob]);
state.outbox.get("outbox_1")!.status = status;
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toEqual([]);
}
});
it("rejects an expired or mismatched dispatcher lease", async () => {
const expired = makeStore([baseJob]);
await expect(
runAuditEventPersistJob(
baseJob,
auditDependencies(expired.runInTransaction, { now: () => TEST_LEASE }),
),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
const mismatchedJob = auditJob({ lockedUntil: "2026-07-17T10:06:00.000Z" });
const mismatched = makeStore([baseJob]);
await expect(
runAuditEventPersistJob(mismatchedJob, auditDependencies(mismatched.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
});
it("fails closed when the exact-generation claim CAS loses its race", async () => {
const state = makeStore([baseJob], { zeroAtUpdate: 1 });
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toEqual([]);
expect(state.outbox.get("outbox_1")).toMatchObject({ status: "PROCESSING", lockedUntil: TEST_LEASE });
});
it("rolls back the AuditEvent and claim when the atomic DONE receipt matches zero rows", async () => {
const state = makeStore([baseJob], { zeroAtUpdate: 2 });
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toEqual([]);
expect(state.outbox.get("outbox_1")).toMatchObject({ status: "PROCESSING", lockedUntil: TEST_LEASE });
});
it("fails closed on a duplicate delivery after the consumer-owned DONE receipt", async () => {
const state = makeStore([baseJob]);
await runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction));
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toHaveLength(1);
});
it("accepts an exact global audit payload carried by a tenant-scoped durable envelope", async () => {
const globalJob = auditJob({
payload: { ...basePayload, tenantId: null, action: "admin.portalDomain.killSwitchEnabled" },
});
const state = makeStore([globalJob]);
await expect(
runAuditEventPersistJob(globalJob, auditDependencies(state.runInTransaction)),
).resolves.toMatchObject({ persisted: true });
expect(state.rows).toHaveLength(1);
expect(state.rows[0]?.tenantId).toBeNull();
});
it("links the chain when a prior event exists for the same tenant", async () => {
const { rows, runInTransaction } = makeStore();
await runAuditEventPersistJob(baseJob, { runInTransaction });
const second = await runAuditEventPersistJob(
{ ...baseJob, payload: { ...baseJob.payload, action: "widget.updated" } },
{ runInTransaction }
);
const secondJob = auditJob({
outboxEventId: "outbox_2",
payload: { ...basePayload, action: "widget.updated" },
});
const { rows, runInTransaction } = makeStore([baseJob, secondJob]);
await runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction));
const second = await runAuditEventPersistJob(secondJob, auditDependencies(runInTransaction));
expect(rows).toHaveLength(2);
expect(rows[1]?.previousHash).toBe(rows[0]?.eventHash);
expect(second.eventHash).toBe(rows[1]?.eventHash);
});
it("isolates chains per tenant", async () => {
const { rows, runInTransaction } = makeStore();
await runAuditEventPersistJob(baseJob, { runInTransaction });
await runAuditEventPersistJob(
{ ...baseJob, tenantId: "tenant_2", payload: { ...baseJob.payload, tenantId: "tenant_2", action: "x.do" } },
{ runInTransaction }
);
const tenantTwoJob = auditJob({
outboxEventId: "outbox_2",
tenantId: "tenant_2",
payload: { ...basePayload, tenantId: "tenant_2", action: "x.do" },
});
const { rows, runInTransaction } = makeStore([baseJob, tenantTwoJob]);
await runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction));
await runAuditEventPersistJob(tenantTwoJob, auditDependencies(runInTransaction));
expect(rows[0]?.tenantId).toBe("tenant_1");
expect(rows[1]?.tenantId).toBe("tenant_2");
expect(rows[1]?.previousHash).toBeNull();
@@ -151,12 +426,15 @@ describe("runAuditEventPersistJob", () => {
it("computes an eventHash consistent with computeAuditHash", async () => {
const { rows, runInTransaction } = makeStore();
await runAuditEventPersistJob(baseJob, { runInTransaction, secret: "test-secret" });
await runAuditEventPersistJob(
baseJob,
auditDependencies(runInTransaction, { secret: "test-secret" }),
);
const recomputed = computeAuditHash({
previousHash: null,
payload: {
id: "",
createdAt: "",
id: rows[0]!.id,
createdAt: rows[0]!.createdAt.toISOString(),
tenantId: "tenant_1",
actorType: "user",
actorId: "user_1",
@@ -174,6 +452,7 @@ describe("runAuditEventPersistJob", () => {
metadata: null,
},
secret: "test-secret",
version: "hmac-v2",
});
expect(rows[0]?.eventHash).toBe(recomputed);
});
@@ -184,7 +463,15 @@ describe("runAuditEventPersistJob", () => {
// through a single chained Promise. Each call sees the previously
// committed row, so the chain stays consistent under parallel
// dispatch.
const { rows, runInTransaction: serialRunner } = makeStore();
const updatedJob = auditJob({
outboxEventId: "outbox_2",
payload: { ...basePayload, action: "widget.updated" },
});
const deletedJob = auditJob({
outboxEventId: "outbox_3",
payload: { ...basePayload, action: "widget.deleted" },
});
const { rows, runInTransaction: serialRunner } = makeStore([baseJob, updatedJob, deletedJob]);
let gate: Promise<void> = Promise.resolve();
const queuedRunner: AuditEventPersistTransactionRunner = async <T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T> => {
const prev = gate;
@@ -199,20 +486,16 @@ describe("runAuditEventPersistJob", () => {
};
const logger = { error: vi.fn() };
await Promise.all([
runAuditEventPersistJob(baseJob, { runInTransaction: queuedRunner, logger }),
runAuditEventPersistJob(
{ ...baseJob, payload: { ...baseJob.payload, action: "widget.updated" } },
{ runInTransaction: queuedRunner, logger }
),
runAuditEventPersistJob(
{ ...baseJob, payload: { ...baseJob.payload, action: "widget.deleted" } },
{ runInTransaction: queuedRunner, logger }
),
runAuditEventPersistJob(baseJob, auditDependencies(queuedRunner, { logger })),
runAuditEventPersistJob(updatedJob, auditDependencies(queuedRunner, { logger })),
runAuditEventPersistJob(deletedJob, auditDependencies(queuedRunner, { logger })),
]);
expect(rows).toHaveLength(3);
expect(rows[0]?.previousHash).toBeNull();
expect(rows[1]?.previousHash).toBe(rows[0]?.eventHash);
expect(rows[2]?.previousHash).toBe(rows[1]?.eventHash);
expect(rows[1]!.createdAt.getTime()).toBeGreaterThan(rows[0]!.createdAt.getTime());
expect(rows[2]!.createdAt.getTime()).toBeGreaterThan(rows[1]!.createdAt.getTime());
expect(logger.error).not.toHaveBeenCalled();
});
});
@@ -6,26 +6,32 @@
* inserts the AuditEvent row into Postgres.
*
* The hash chain is computed inside a short Prisma transaction:
* - SELECT the most-recent eventHash for the tenant (FOR UPDATE
* equivalent via serializable isolation)
* - acquire a tenant/platform PostgreSQL advisory transaction lock
* - authenticate and read the O(1) persistent chain head
* - compute the new event's eventHash
* - INSERT the new row
* - insert the event and atomically advance the authenticated head
*
* Running this under serializable isolation ensures concurrent persists
* for the same tenant either commit in some order with a coherent
* chain, or one of them retries on serialization-failure. The handler
* is therefore safe to invoke from N parallel BullMQ workers without
* corrupting the chain.
* The advisory lock is shared with direct writers and retention, so N
* parallel BullMQ workers cannot fork a tenant or platform chain.
*
* Idempotency: the producer-side `publishAuditEventViaOutbox` writes
* one outbox row per business-write. The dispatcher delivers each row
* exactly once (claim+ack); on retry we may re-process the same
* payload but the audit insert is a plain CREATE — duplicates simply
* extend the chain by one extra row. The acceptable outcome here is
* "at-least-once": a duplicated audit row is harmless evidence.
* Authority and idempotency: Redis is transport only. Before inserting an
* AuditEvent the consumer proves the exact tenant/generation/payload/dispatch
* key/lease against the durable OutboxEvent, claims that generation with a
* CAS, then commits the AuditEvent and terminal DONE receipt atomically.
*/
import { randomUUID } from "node:crypto";
import { computeAuditHash, type AuditHashPayload } from "@hivegrid/db/audit-hash";
import {
acquireAuditChainLock,
auditChainKey,
computeAuditChainHeadMac,
verifyAuditChainHead,
verifyAuditCheckpoint,
type AuditChainHeadRecord,
type AuditChainCheckpointRecord,
} from "@hivegrid/db/audit-chain-checkpoint.server";
import {
buildAuditEventCreateData,
type AuditEventCreateData,
@@ -33,6 +39,7 @@ import {
} from "@hivegrid/db/audit-events.server";
export const AUDIT_EVENT_OUTBOX_QUEUE = "outbox.audit" as const;
const AUDIT_OUTBOX_CONSUMER_LEASE_MS = 5 * 60 * 1_000;
export interface AuditEventPersistLogger {
info?: (record: Record<string, unknown>, message?: string) => void;
@@ -41,23 +48,89 @@ export interface AuditEventPersistLogger {
}
export interface AuditEventPersistTxClient {
$queryRaw<T = unknown>(query: TemplateStringsArray, ...values: unknown[]): Promise<T>;
auditEvent: {
findFirst(args: {
where: { tenantId: string | null; eventHash: { not: null } };
orderBy: { createdAt: "desc" };
select: { eventHash: true };
}): Promise<{ eventHash: string | null } | null>;
orderBy: Array<{ createdAt: "desc" } | { id: "desc" }>;
select: { eventHash: true; createdAt: true };
}): Promise<{ eventHash: string | null; createdAt: Date } | null>;
create(args: {
data: AuditEventCreateData & { previousHash: string | null; eventHash: string };
data: AuditEventCreateData & { id: string; createdAt: Date; previousHash: string | null; eventHash: string };
}): Promise<{ id: string; eventHash: string }>;
};
auditChainCheckpoint: {
findFirst(args: {
where: { chainKey: string };
orderBy: Array<{ createdAt: "desc" } | { id: "desc" }>;
}): Promise<AuditChainCheckpointRecord | null>;
findUnique(args: { where: { id: string } }): Promise<AuditChainCheckpointRecord | null>;
};
auditChainCheckpointHead: {
findUnique(args: {
where: { chainKey: string };
select: { checkpointId: true };
}): Promise<{ checkpointId: string } | null>;
};
auditChainHead: {
findUnique(args: { where: { chainKey: string } }): Promise<AuditChainHeadRecord | null>;
upsert(args: {
where: { chainKey: string };
create: AuditChainHeadRecord;
update: Omit<AuditChainHeadRecord, "chainKey">;
}): Promise<unknown>;
};
outboxEvent: {
findFirst(args: {
where: {
id: string;
tenantId: string | null;
eventType: "audit.event.create";
deliveryGeneration: number;
};
select: {
id: true;
tenantId: true;
eventType: true;
payload: true;
status: true;
deliveryGeneration: true;
dispatchKey: true;
lockedUntil: true;
};
}): Promise<{
id: string;
tenantId: string | null;
eventType: string;
payload: unknown;
status: "PENDING" | "PROCESSING" | "DONE" | "FAILED" | "CANCELLED";
deliveryGeneration: number;
dispatchKey: string | null;
lockedUntil: Date | null;
} | null>;
updateMany(args: {
where: {
id: string;
tenantId: string | null;
eventType: "audit.event.create";
deliveryGeneration: number;
dispatchKey: string | null;
status: "PROCESSING";
lockedUntil: Date;
payload: { equals: unknown };
};
data:
| { lockedUntil: Date }
| { status: "DONE"; processedAt: Date; lastError: null; lockedUntil: null };
}): Promise<{ count: number }>;
};
}
export interface AuditEventPersistTransactionRunner {
/**
* Run `fn` inside a Prisma transaction. Production wiring passes
* `prisma.$transaction` with isolationLevel="Serializable" so that
* the prevHash lookup + insert form an atomic chain link.
* Run `fn` inside a Prisma transaction. Production additionally uses
* serializable isolation; the per-chain advisory lock is the primary
* cross-writer serialization primitive.
*/
<T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T>;
}
@@ -70,11 +143,13 @@ export interface AuditEventPersistDependencies {
* leaves this undefined so the helper reads `process.env.AUDIT_CHAIN_SECRET`.
*/
readonly secret?: string | null;
/** Clock override for durable lease checks and deterministic tests. */
readonly now?: () => Date;
}
export class AuditEventPersistJobError extends Error {
override readonly name = "AuditEventPersistJobError";
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "MISSING_ACTION";
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "MISSING_ACTION" | "OUTBOX_PROOF_INVALID";
constructor(message: string, code: AuditEventPersistJobError["code"]) {
super(message);
this.code = code;
@@ -82,9 +157,13 @@ export class AuditEventPersistJobError extends Error {
}
export interface AuditEventPersistJobData {
readonly outboxEventId?: string;
readonly outboxEventId: string;
readonly tenantId: string | null;
readonly eventType: string;
readonly eventType: "audit.event.create";
readonly dispatchKey: string | null;
readonly deliveryGeneration: number;
readonly lockedUntil: Date;
readonly rawPayload: Readonly<Record<string, unknown>>;
readonly payload: AuditEventCreateData;
}
@@ -95,7 +174,7 @@ export interface AuditEventPersistResult {
}
export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobData {
if (!data || typeof data !== "object") {
if (!data || typeof data !== "object" || Array.isArray(data)) {
throw new AuditEventPersistJobError("audit-event persist job missing data envelope", "INVALID_PAYLOAD");
}
const envelope = data as Record<string, unknown>;
@@ -110,6 +189,9 @@ export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobDa
if (!rawPayload || typeof rawPayload !== "object") {
throw new AuditEventPersistJobError("audit-event persist job missing payload", "INVALID_PAYLOAD");
}
if (Array.isArray(rawPayload)) {
throw new AuditEventPersistJobError("audit-event persist job payload must be an object", "INVALID_PAYLOAD");
}
const inner = rawPayload as Record<string, unknown>;
const action = typeof inner.action === "string" ? inner.action.trim() : "";
if (action.length === 0) {
@@ -118,18 +200,67 @@ export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobDa
// Re-run normalization in case the producer emitted an older payload
// shape — defensive. `buildAuditEventCreateData` is idempotent.
const normalized = buildAuditEventCreateData(inner as unknown as AuditEventInput);
const outboxEventId = parseIdentifier(envelope.outboxEventId);
const tenantId = parseNullableIdentifier(envelope, "tenantId");
const dispatchKey = parseNullableDispatchKey(envelope);
const deliveryGeneration = envelope.deliveryGeneration;
const lockedUntil = parseLease(envelope.lockedUntil);
if (
!outboxEventId ||
tenantId === undefined ||
dispatchKey === undefined ||
!Number.isSafeInteger(deliveryGeneration) ||
(deliveryGeneration as number) < 0 ||
!lockedUntil
) {
throw new AuditEventPersistJobError("audit-event persist job has no exact outbox identity", "INVALID_PAYLOAD");
}
if (normalized.tenantId !== null && normalized.tenantId !== tenantId) {
throw new AuditEventPersistJobError("audit-event tenant does not match its outbox envelope", "INVALID_PAYLOAD");
}
return {
outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
tenantId: typeof envelope.tenantId === "string" ? envelope.tenantId : null,
eventType,
outboxEventId,
tenantId,
eventType: "audit.event.create",
dispatchKey,
deliveryGeneration: deliveryGeneration as number,
lockedUntil,
rawPayload: inner,
payload: normalized,
};
}
function toHashPayload(data: AuditEventCreateData): AuditHashPayload {
function parseIdentifier(value: unknown): string | null {
if (typeof value !== "string") return null;
const normalized = value.trim();
return normalized.length > 0 && normalized.length <= 191 && !normalized.includes("\u0000") ? normalized : null;
}
function parseNullableIdentifier(
envelope: Readonly<Record<string, unknown>>,
key: "tenantId",
): string | null | undefined {
if (!Object.hasOwn(envelope, key)) return undefined;
if (envelope[key] === null) return null;
return parseIdentifier(envelope[key]) ?? undefined;
}
function parseNullableDispatchKey(envelope: Readonly<Record<string, unknown>>): string | null | undefined {
if (!Object.hasOwn(envelope, "dispatchKey")) return undefined;
if (envelope.dispatchKey === null) return null;
if (typeof envelope.dispatchKey !== "string") return undefined;
return /^[A-Za-z0-9._:-]{1,256}$/u.test(envelope.dispatchKey) ? envelope.dispatchKey : undefined;
}
function parseLease(value: unknown): Date | null {
const parsed = value instanceof Date ? new Date(value.getTime()) : typeof value === "string" ? new Date(value) : null;
return parsed && Number.isFinite(parsed.getTime()) ? parsed : null;
}
function toHashPayload(data: AuditEventCreateData, id: string, createdAt: Date): AuditHashPayload {
return {
id: "",
createdAt: "",
id,
createdAt: createdAt.toISOString(),
tenantId: data.tenantId,
actorType: data.actorType,
actorId: data.actorId,
@@ -156,20 +287,139 @@ export async function runAuditEventPersistJob(
const payload = job.payload;
const result = await deps.runInTransaction(async (tx) => {
const prior = await tx.auditEvent.findFirst({
where: { tenantId: payload.tenantId, eventHash: { not: null } },
orderBy: { createdAt: "desc" },
select: { eventHash: true },
const durable = await tx.outboxEvent.findFirst({
where: {
id: job.outboxEventId,
tenantId: job.tenantId,
eventType: job.eventType,
deliveryGeneration: job.deliveryGeneration,
},
select: {
id: true,
tenantId: true,
eventType: true,
payload: true,
status: true,
deliveryGeneration: true,
dispatchKey: true,
lockedUntil: true,
},
});
const previousHash = prior?.eventHash ?? null;
const observedAt = deps.now?.() ?? new Date();
if (
!durable ||
durable.status !== "PROCESSING" ||
durable.dispatchKey !== job.dispatchKey ||
!durable.lockedUntil ||
durable.lockedUntil.getTime() !== job.lockedUntil.getTime() ||
durable.lockedUntil.getTime() <= observedAt.getTime() ||
!sameCanonicalJson(durable.payload, job.rawPayload)
) {
throw invalidOutboxProof();
}
const claimedLease = new Date(
Math.max(
observedAt.getTime() + AUDIT_OUTBOX_CONSUMER_LEASE_MS,
durable.lockedUntil.getTime() + 1,
),
);
const claimed = await tx.outboxEvent.updateMany({
where: {
id: job.outboxEventId,
tenantId: job.tenantId,
eventType: job.eventType,
deliveryGeneration: job.deliveryGeneration,
dispatchKey: job.dispatchKey,
status: "PROCESSING",
lockedUntil: durable.lockedUntil,
payload: { equals: job.rawPayload },
},
data: { lockedUntil: claimedLease },
});
if (claimed.count !== 1) throw invalidOutboxProof();
await acquireAuditChainLock(tx, payload.tenantId);
const key = auditChainKey(payload.tenantId);
const head = await tx.auditChainHead.findUnique({ where: { chainKey: key } });
if (head && !verifyAuditChainHead(head, deps.secret)) {
throw new Error(`Audit chain-head authentication failed for ${key}.`);
}
let previousHash = head?.eventHash ?? null;
let previousCreatedAt = head?.eventCreatedAt ?? null;
if (!previousHash) {
const prior = await tx.auditEvent.findFirst({
where: { tenantId: payload.tenantId, eventHash: { not: null } },
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
select: { eventHash: true, createdAt: true },
});
previousHash = prior?.eventHash ?? null;
previousCreatedAt = prior?.createdAt ?? null;
}
if (!previousHash) {
const checkpointHead = await tx.auditChainCheckpointHead.findUnique({
where: { chainKey: key },
select: { checkpointId: true },
});
const checkpoint = checkpointHead
? await tx.auditChainCheckpoint.findUnique({ where: { id: checkpointHead.checkpointId } })
: await tx.auditChainCheckpoint.findFirst({
where: { chainKey: key },
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
});
if (checkpoint) {
if (!verifyAuditCheckpoint(checkpoint, deps.secret)) {
throw new Error(`Audit checkpoint authentication failed for ${checkpoint.chainKey}.`);
}
previousHash = checkpoint.lastDeletedEventHash;
previousCreatedAt = checkpoint.deletedThroughAt;
}
}
const id = randomUUID();
const now = deps.now?.() ?? new Date();
const createdAt = previousCreatedAt && now.getTime() <= previousCreatedAt.getTime()
? new Date(previousCreatedAt.getTime() + 1)
: now;
const eventHash = computeAuditHash({
previousHash,
payload: toHashPayload(payload),
payload: toHashPayload(payload, id, createdAt),
secret: deps.secret,
version: "hmac-v2",
});
const inserted = await tx.auditEvent.create({
data: { ...payload, previousHash, eventHash },
data: { ...payload, id, createdAt, previousHash, eventHash },
});
const headWithoutMac: Omit<AuditChainHeadRecord, "headMac"> = {
chainKey: key,
tenantId: payload.tenantId,
eventId: id,
eventHash,
eventCreatedAt: createdAt,
};
const headMac = computeAuditChainHeadMac(headWithoutMac, deps.secret);
await tx.auditChainHead.upsert({
where: { chainKey: key },
create: { ...headWithoutMac, headMac },
update: { tenantId: payload.tenantId, eventId: id, eventHash, eventCreatedAt: createdAt, headMac },
});
const receipt = await tx.outboxEvent.updateMany({
where: {
id: job.outboxEventId,
tenantId: job.tenantId,
eventType: job.eventType,
deliveryGeneration: job.deliveryGeneration,
dispatchKey: job.dispatchKey,
status: "PROCESSING",
lockedUntil: claimedLease,
payload: { equals: job.rawPayload },
},
data: {
status: "DONE",
processedAt: createdAt,
lastError: null,
lockedUntil: null,
},
});
if (receipt.count !== 1) throw invalidOutboxProof();
return { auditEventId: inserted.id, eventHash };
});
@@ -186,3 +436,30 @@ export async function runAuditEventPersistJob(
return { persisted: true, auditEventId: result.auditEventId, eventHash: result.eventHash };
}
function invalidOutboxProof(): AuditEventPersistJobError {
return new AuditEventPersistJobError(
"audit-event persist job does not match an active durable outbox generation",
"OUTBOX_PROOF_INVALID",
);
}
function sameCanonicalJson(left: unknown, right: unknown): boolean {
try {
return JSON.stringify(canonicalJson(left)) === JSON.stringify(canonicalJson(right));
} catch {
return false;
}
}
function canonicalJson(value: unknown): unknown {
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
if (typeof value === "number" && Number.isFinite(value)) return value;
if (Array.isArray(value)) return value.map(canonicalJson);
if (!value || typeof value !== "object") throw new TypeError("Value is not canonical JSON");
return Object.fromEntries(
Object.keys(value as Record<string, unknown>)
.sort()
.map((key) => [key, canonicalJson((value as Record<string, unknown>)[key])]),
);
}
@@ -3,16 +3,14 @@ import { describe, expect, it, vi } from "vitest";
import {
AUDIT_RETENTION_CRON_DEFAULT,
AUDIT_RETENTION_JOB_ID,
DEFAULT_AUDIT_RETENTION_DAYS,
GDPR_AUDIT_RETENTION_DAYS,
SOC2_AUDIT_RETENTION_DAYS,
AUDIT_RETENTION_YEARS,
registerAuditRetentionCron,
resolveAuditRetentionPolicy,
resolveAuditRetentionThreshold,
runAuditRetentionJob,
} from "./audit-retention.js";
describe("registerAuditRetentionCron", () => {
it("upserts the nightly scheduler with the default pattern (idempotent key)", async () => {
it("upserts the nightly scheduler with the default pattern", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
const { pattern } = await registerAuditRetentionCron({ upsertJobScheduler }, {});
expect(pattern).toBe(AUDIT_RETENTION_CRON_DEFAULT);
@@ -30,62 +28,68 @@ describe("registerAuditRetentionCron", () => {
});
});
describe("resolveAuditRetentionPolicy", () => {
it("returns 7y for SOC2 + HIPAA tenants (HIPAA wins on label)", () => {
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["soc2"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "soc2" });
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["hipaa"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "hipaa" });
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["soc2", "hipaa"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "hipaa" });
});
it("returns 6y for GDPR-only tenants", () => {
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["gdpr"] })).toEqual({ retentionDays: GDPR_AUDIT_RETENTION_DAYS, source: "gdpr" });
});
it("returns 1y default for tenants with no regulatory scope", () => {
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: [] })).toEqual({ retentionDays: DEFAULT_AUDIT_RETENTION_DAYS, source: "default" });
describe("resolveAuditRetentionThreshold", () => {
it("uses an exact ten-calendar-year threshold, including leap years", () => {
expect(AUDIT_RETENTION_YEARS).toBe(10);
expect(resolveAuditRetentionThreshold(new Date("2028-02-29T12:00:00.000Z")).toISOString()).toBe(
"2018-03-01T12:00:00.000Z",
);
});
});
describe("runAuditRetentionJob", () => {
it("walks tenants, deletes rows, and emits a meta-audit for each non-empty sweep", async () => {
it("includes the platform chain and rotates bounded batches fairly", async () => {
const now = new Date("2026-05-17T00:00:00.000Z");
const tenants = [
{ tenantId: "t1", regulatoryScopes: ["soc2"] },
{ tenantId: "t2", regulatoryScopes: [] },
];
const eventDeleter = vi.fn(async (tenantId: string, threshold: Date) => {
void threshold;
return tenantId === "t1" ? 42 : 0;
const calls = new Map<string | null, number>();
const batchPruner = vi.fn(async (tenantId: string | null) => {
const call = (calls.get(tenantId) ?? 0) + 1;
calls.set(tenantId, call);
return { tenantId, deletedCount: 1, hasMore: call < 2, checkpointMac: "cp1:test" };
});
const auditWriter = vi.fn(async () => {});
const auditWriter = vi.fn(async () => undefined);
const result = await runAuditRetentionJob({
now,
tenantLister: async () => tenants,
eventDeleter,
chainLister: async () => [null, "tenant-1"],
batchPruner,
auditWriter,
maxBatches: 4,
});
expect(result).toEqual({ inspectedTenants: 2, deletedRows: 42, failures: [] });
expect(eventDeleter).toHaveBeenCalledTimes(2);
// SOC2 -> 7y threshold
const soc2Threshold = eventDeleter.mock.calls[0]?.[1];
expect(soc2Threshold).toBeInstanceOf(Date);
expect(soc2Threshold.toISOString()).toBe(new Date(now.getTime() - SOC2_AUDIT_RETENTION_DAYS * 86_400_000).toISOString());
// Only t1 had deletions -> only one meta-audit
expect(auditWriter).toHaveBeenCalledTimes(1);
expect(batchPruner.mock.calls.map((call) => call[0])).toEqual([null, "tenant-1", null, "tenant-1"]);
expect(result).toEqual({
inspectedChains: 2,
batches: 4,
deletedRows: 4,
backlogChains: [],
failures: [],
});
expect(auditWriter).toHaveBeenCalledWith(
expect.objectContaining({ tenantId: null, deletedCount: 2, retentionYears: 10 }),
);
expect(auditWriter).toHaveBeenCalledWith(
expect.objectContaining({ tenantId: "tenant-1", deletedCount: 2, retentionYears: 10 }),
);
});
it("records failures without aborting other tenants", async () => {
const now = new Date("2026-05-17T00:00:00.000Z");
const tenants = [
{ tenantId: "t-fail", regulatoryScopes: [] },
{ tenantId: "t-ok", regulatoryScopes: [] },
];
const eventDeleter = vi.fn(async (tenantId: string) => {
if (tenantId === "t-fail") throw new Error("boom");
return 5;
it("reports backlog when the global batch budget is exhausted", async () => {
const result = await runAuditRetentionJob({
chainLister: async () => ["t1", "t2"],
batchPruner: async (tenantId) => ({ tenantId, deletedCount: 1, hasMore: true, checkpointMac: "cp1:test" }),
maxBatches: 1,
});
expect(result.batches).toBe(1);
expect(result.backlogChains).toEqual(["t2", "t1"]);
});
it("records a chain failure without aborting other chains", async () => {
const result = await runAuditRetentionJob({
chainLister: async () => ["t-fail", "t-ok"],
batchPruner: async (tenantId) => {
if (tenantId === "t-fail") throw new Error("broken checkpoint");
return { tenantId, deletedCount: 5, hasMore: false, checkpointMac: "cp1:test" };
},
});
const result = await runAuditRetentionJob({ now, tenantLister: async () => tenants, eventDeleter });
expect(result.deletedRows).toBe(5);
expect(result.failures).toEqual([{ tenantId: "t-fail", error: "boom" }]);
expect(result.failures).toEqual([{ tenantId: "t-fail", error: "broken checkpoint" }]);
});
});
@@ -1,24 +1,13 @@
/**
* Phase 10 Task #67 — Audit-Event retention worker.
*
* Walks every tenant on a daily cron, looks up the tenant's retention
* window (plan-based default; configurable per-tenant via an override
* JSON in a future task), and deletes AuditEvent rows older than the
* window. Deletions are themselves audited as a system-action event so
* the cleanup itself is traceable — see
* docs/features/compliance-audit.md.
*
* Deliberately host-agnostic: the worker bootstrap (apps/worker/src/index.ts)
* supplies the seams (tenantLister, eventDeleter, auditWriter) so this
* file is trivially unit-testable without a real database.
* AuditEvent retention is a fixed ten-year policy. Each bounded batch is
* checkpointed by the database adapter before deletion, so a later verifier
* can authenticate the retained chain prefix.
*/
export const DEFAULT_AUDIT_RETENTION_DAYS = 365; // baseline / "none" scope
export const SOC2_AUDIT_RETENTION_DAYS = 7 * 365; // SOC2 + HIPAA: 7y
export const GDPR_AUDIT_RETENTION_DAYS = 6 * 365; // GDPR: 6y
export const AUDIT_RETENTION_YEARS = 10;
export const AUDIT_RETENTION_BATCH_SIZE = 1_000;
export const AUDIT_RETENTION_MAX_BATCHES = 100;
// AK-P3 AP-2 — cron wiring. Mirrors plugin-audit-retention so the worker
// bootstrap can install a nightly scheduler for the AuditEvent sweep.
export const AUDIT_RETENTION_QUEUE = "audit-retention" as const;
export const AUDIT_RETENTION_JOB_ID = "audit-retention" as const;
/** Nightly at 03:30 UTC (offset from plugin-audit-retention's 03:00). */
@@ -26,11 +15,6 @@ export const AUDIT_RETENTION_CRON_DEFAULT = "30 3 * * *" as const;
export type AuditRetentionEnv = Readonly<Record<string, string | undefined>>;
/**
* Install (or replace) the nightly AuditEvent-retention schedule on a
* BullMQ-like queue. Idempotent via a stable scheduler key. Cron pattern
* overridable via `AUDIT_RETENTION_CRON`.
*/
export async function registerAuditRetentionCron(
queue: {
upsertJobScheduler: (
@@ -50,78 +34,106 @@ export async function registerAuditRetentionCron(
return { pattern };
}
export type AuditRetentionPolicy = {
/** Hard cap in days. Events older than this are pruned. */
retentionDays: number;
/** Free-form label for log lines (`gdpr` | `soc2` | `plan:enterprise` | …). */
source: string;
};
export function resolveAuditRetentionThreshold(now: Date): Date {
const threshold = new Date(now);
threshold.setUTCFullYear(threshold.getUTCFullYear() - AUDIT_RETENTION_YEARS);
return threshold;
}
export type AuditRetentionTenant = {
tenantId: string;
/** Highest regulatoryScope used by events for this tenant. */
regulatoryScopes: ReadonlyArray<string>;
export type AuditRetentionBatchResult = {
tenantId: string | null;
deletedCount: number;
hasMore: boolean;
checkpointMac: string | null;
};
export type AuditRetentionInput = {
/** Reference "now" — defaults to `new Date()`. Injected for tests. */
now?: Date;
/** List tenants (each with the regulatoryScopes ever used by their events). */
tenantLister: () => Promise<ReadonlyArray<AuditRetentionTenant>>;
/** Delete events with createdAt < threshold for tenantId. Returns row count. */
eventDeleter: (tenantId: string, threshold: Date) => Promise<number>;
/** Optional audit writer for the meta-audit (cleanup itself). */
auditWriter?: (input: { tenantId: string; deletedCount: number; threshold: Date; policy: AuditRetentionPolicy }) => Promise<void>;
/** Logger seam. */
logger?: { info: (fields: Record<string, unknown>, msg: string) => void; error?: (fields: Record<string, unknown>, msg: string) => void };
/** Optional override for plan-resolved policy per tenant. */
policyResolver?: (tenant: AuditRetentionTenant) => AuditRetentionPolicy;
/** One aggregate query returns tenant chains plus the platform chain (null). */
chainLister: (threshold: Date) => Promise<ReadonlyArray<string | null>>;
/** Deletes one keyset-ordered, checkpointed transaction batch. */
batchPruner: (tenantId: string | null, threshold: Date, batchSize: number) => Promise<AuditRetentionBatchResult>;
auditWriter?: (input: {
tenantId: string | null;
deletedCount: number;
threshold: Date;
retentionYears: number;
}) => Promise<void>;
batchSize?: number;
maxBatches?: number;
logger?: {
info: (fields: Record<string, unknown>, msg: string) => void;
error?: (fields: Record<string, unknown>, msg: string) => void;
};
};
export function resolveAuditRetentionPolicy(tenant: AuditRetentionTenant): AuditRetentionPolicy {
const scopes = new Set(tenant.regulatoryScopes.map((s) => s.toLowerCase()));
if (scopes.has("soc2") || scopes.has("hipaa")) {
return { retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: scopes.has("hipaa") ? "hipaa" : "soc2" };
}
if (scopes.has("gdpr")) {
return { retentionDays: GDPR_AUDIT_RETENTION_DAYS, source: "gdpr" };
}
return { retentionDays: DEFAULT_AUDIT_RETENTION_DAYS, source: "default" };
}
export type AuditRetentionResult = {
inspectedTenants: number;
inspectedChains: number;
batches: number;
deletedRows: number;
failures: Array<{ tenantId: string; error: string }>;
backlogChains: Array<string | null>;
failures: Array<{ tenantId: string | null; error: string }>;
};
/**
* Run the retention sweep across every tenant. Caller (cron bootstrap)
* decides scheduling — this function is a single pass.
*/
export async function runAuditRetentionJob(input: AuditRetentionInput): Promise<AuditRetentionResult> {
const now = input.now ?? new Date();
const tenants = await input.tenantLister();
let deletedRows = 0;
const threshold = resolveAuditRetentionThreshold(input.now ?? new Date());
const batchSize = Math.max(1, Math.min(input.batchSize ?? AUDIT_RETENTION_BATCH_SIZE, 5_000));
const maxBatches = Math.max(1, input.maxBatches ?? AUDIT_RETENTION_MAX_BATCHES);
const chains = [...new Set(await input.chainLister(threshold))];
const pending = [...chains];
const deletedByChain = new Map<string | null, number>();
const failures: AuditRetentionResult["failures"] = [];
let batches = 0;
let deletedRows = 0;
for (const tenant of tenants) {
while (pending.length > 0 && batches < maxBatches) {
const tenantId = pending.shift() ?? null;
try {
const policy = (input.policyResolver ?? resolveAuditRetentionPolicy)(tenant);
const threshold = new Date(now.getTime() - policy.retentionDays * 86_400_000);
const count = await input.eventDeleter(tenant.tenantId, threshold);
deletedRows += count;
input.logger?.info({ tenantId: tenant.tenantId, deletedCount: count, retentionDays: policy.retentionDays, policySource: policy.source }, "audit.retention.tenant.swept");
if (count > 0 && input.auditWriter) {
await input.auditWriter({ tenantId: tenant.tenantId, deletedCount: count, threshold, policy });
}
const result = await input.batchPruner(tenantId, threshold, batchSize);
batches += 1;
deletedRows += result.deletedCount;
deletedByChain.set(tenantId, (deletedByChain.get(tenantId) ?? 0) + result.deletedCount);
if (result.hasMore) pending.push(tenantId);
input.logger?.info(
{
tenantId,
deletedCount: result.deletedCount,
retentionYears: AUDIT_RETENTION_YEARS,
hasMore: result.hasMore,
},
"audit.retention.chain.batch",
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
failures.push({ tenantId: tenant.tenantId, error: message });
input.logger?.error?.({ tenantId: tenant.tenantId, error: message }, "audit.retention.tenant.failed");
failures.push({ tenantId, error: message });
input.logger?.error?.({ tenantId, error: message }, "audit.retention.chain.failed");
}
}
input.logger?.info({ inspected: tenants.length, deletedRows, failures: failures.length }, "audit.retention.complete");
return { inspectedTenants: tenants.length, deletedRows, failures };
if (input.auditWriter) {
for (const [tenantId, deletedCount] of deletedByChain) {
if (deletedCount > 0) {
await input.auditWriter({
tenantId,
deletedCount,
threshold,
retentionYears: AUDIT_RETENTION_YEARS,
});
}
}
}
const backlogChains = [...new Set(pending)];
input.logger?.info(
{
inspected: chains.length,
batches,
deletedRows,
backlogChains: backlogChains.length,
failures: failures.length,
retentionYears: AUDIT_RETENTION_YEARS,
},
"audit.retention.complete",
);
return { inspectedChains: chains.length, batches, deletedRows, backlogChains, failures };
}
@@ -0,0 +1,31 @@
// R-P12c — worker seam tests: envelope parsing (tenant mismatch fail-closed)
// and companion-consumer skip behaviour for non-automation events.
import { describe, expect, test } from "vitest";
import { AutomationExecutorJobError, parseAutomationExecutorOutboxJob } from "./automation-forward.js";
describe("parseAutomationExecutorOutboxJob", () => {
test("accepts a well-formed automation.execution.requested envelope", () => {
expect(
parseAutomationExecutorOutboxJob({
eventType: "automation.execution.requested",
tenantId: "tenant-a",
payload: { executionId: "exec-1", tenantId: "tenant-a" },
})
).toEqual({ tenantId: "tenant-a", executionId: "exec-1" });
});
test("rejects foreign event types, missing ids and tenant mismatches fail-closed", () => {
expect(() => parseAutomationExecutorOutboxJob({ eventType: "workflow.trigger.event", tenantId: "t", payload: {} })).toThrow(AutomationExecutorJobError);
expect(() => parseAutomationExecutorOutboxJob({ eventType: "automation.execution.requested", tenantId: "", payload: { executionId: "e" } })).toThrow(
AutomationExecutorJobError
);
expect(() =>
parseAutomationExecutorOutboxJob({
eventType: "automation.execution.requested",
tenantId: "tenant-a",
payload: { executionId: "exec-1", tenantId: "tenant-b" },
})
).toThrow(/tenantId mismatch/);
});
});
@@ -0,0 +1,122 @@
// R-P12c — worker wiring for the generic automation forward path.
//
// Two seams:
// 1. `createRunAutomationForwardTrigger` — companion consumer on the durable
// `outbox.workflow` queue (next to runPortalAutomationTriggerJob). Maps
// domain events onto automation trigger keys and creates idempotent,
// guarded executions (packages/db/src/automation-forward.server.ts).
// 2. `createAutomationExecutorProcessor` — BullMQ worker processor for the
// `outbox.automation-executor` queue (routed via event-router for
// `automation.execution.*` outbox events). Runs the typed forward
// executor (packages/db/src/automation-executor.server.ts) with real
// side-effect adapters bound: notify(), logActivity(), SSRF-guarded HTTP
// and the permission-aware team-chat repository.
import { logActivity } from "@hivegrid/activity";
import { sendTopLevelChatMessage } from "@hivegrid/chat/repository.server";
import { queueAutomationExecutionsForDomainEventWithTx, AUTOMATION_DOMAIN_EVENT_TRIGGER_MAP } from "@hivegrid/db/automation-forward.server";
import { runAutomationExecutionJob, type AutomationExecutionJobResult } from "@hivegrid/db/automation-executor.server";
import type { DbClient } from "@hivegrid/db/client.server";
import { notify, type NotificationClient } from "@hivegrid/foundation";
import { fetchOutboundHttpWithRedirects, validateOutboundHttpUrl } from "@hivegrid/foundation/outbound-http";
import { withTenantRLS } from "../db/tenantRls.js";
import { parseWorkflowTriggerJob, type WorkflowTriggerLogger } from "./workflow-trigger.js";
export const AUTOMATION_EXECUTOR_OUTBOX_QUEUE = "outbox.automation-executor" as const;
export type AutomationForwardDeps = {
readonly prisma: DbClient;
readonly logger?: WorkflowTriggerLogger;
};
/** Companion consumer for `workflow.trigger.event` jobs (generic, non-portal). */
export function createRunAutomationForwardTrigger(deps: AutomationForwardDeps): (data: unknown) => Promise<unknown> {
return async (data: unknown) => {
const job = parseWorkflowTriggerJob(data);
if (!job.tenantId) return { skipped: "missing_tenant" };
if (!AUTOMATION_DOMAIN_EVENT_TRIGGER_MAP[job.eventName]) return { skipped: "not_an_automation_event" };
const payload = job.payload && typeof job.payload === "object" ? (job.payload as Record<string, unknown>) : {};
const eventData = payload.data && typeof payload.data === "object" && !Array.isArray(payload.data) ? (payload.data as Record<string, unknown>) : {};
const userId = typeof payload.userId === "string" ? payload.userId : null;
const sourceEventId = job.outboxEventId ?? job.dispatchKey ?? `${job.eventType}:${job.eventName}`;
const tenantId = job.tenantId;
const result = await withTenantRLS(deps.prisma, tenantId, (tx) =>
queueAutomationExecutionsForDomainEventWithTx(tx as never, {
tenantId,
domainEvent: job.eventName,
payload: eventData,
sourceEventId,
userId,
})
);
if (result.matched > 0) {
deps.logger?.info?.(
{ component: "automation-forward", tenantId, eventName: job.eventName, ...result, rejected: result.rejected.length },
"automation trigger matched"
);
}
return result;
};
}
export class AutomationExecutorJobError extends Error {
override readonly name = "AutomationExecutorJobError";
}
export function parseAutomationExecutorOutboxJob(data: unknown): { tenantId: string; executionId: string } {
if (!data || typeof data !== "object") {
throw new AutomationExecutorJobError("automation executor job missing envelope");
}
const envelope = data as Record<string, unknown>;
if (envelope.eventType !== "automation.execution.requested") {
throw new AutomationExecutorJobError(`unsupported automation executor eventType ${JSON.stringify(envelope.eventType)}`);
}
const tenantId = typeof envelope.tenantId === "string" ? envelope.tenantId : "";
const payload = envelope.payload && typeof envelope.payload === "object" ? (envelope.payload as Record<string, unknown>) : {};
const executionId = typeof payload.executionId === "string" ? payload.executionId : "";
if (!tenantId || !executionId) {
throw new AutomationExecutorJobError("automation executor job missing tenantId/executionId");
}
const payloadTenantId = typeof payload.tenantId === "string" ? payload.tenantId : tenantId;
if (payloadTenantId !== tenantId) {
throw new AutomationExecutorJobError("automation executor job tenantId mismatch");
}
return { tenantId, executionId };
}
export function createAutomationExecutorProcessor(deps: AutomationForwardDeps): (data: unknown) => Promise<AutomationExecutionJobResult> {
return async (data: unknown) => {
const { tenantId, executionId } = parseAutomationExecutorOutboxJob(data);
return runAutomationExecutionJob(
{ tenantId, executionId },
{
runInTenantTx: (fn) => withTenantRLS(deps.prisma, tenantId, (tx) => fn(tx as never)),
notify: (input, txClient) => notify(input as never, txClient as NotificationClient),
logActivity: (input, opts) => logActivity(input as never, opts as never),
fetchOutboundHttp: async ({ url, method, body }) => {
const validated = validateOutboundHttpUrl(url);
const response = await fetchOutboundHttpWithRedirects(validated, {
method,
...(body ? { body, headers: { "content-type": "application/json" } } : {}),
});
return { status: response.status, ok: response.ok };
},
sendChatMessage: async (input) => {
const result = await sendTopLevelChatMessage({
tenantId: input.tenantId,
userId: input.userId,
permissions: [],
chatId: input.chatId,
requestId: input.requestId,
clientMessageId: input.clientMessageId,
body: input.body,
});
return { messageId: result.message.id, replayed: result.replayed };
},
logger: deps.logger,
}
);
};
}
@@ -0,0 +1,18 @@
import { describe, expect, it, vi } from "vitest";
import { AUTOMATION_SCHEDULER_SWEEP_EVERY_MS, AUTOMATION_SCHEDULER_SWEEP_JOB_ID, registerAutomationSchedulerCron } from "./automation-scheduler-cron.js";
describe("automation scheduler cron", () => {
it("registers one singleton minute sweep", async () => {
const upsertJobScheduler = vi.fn().mockResolvedValue(undefined);
await registerAutomationSchedulerCron({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
{ every: AUTOMATION_SCHEDULER_SWEEP_EVERY_MS },
{
name: AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: AUTOMATION_SCHEDULER_SWEEP_JOB_ID },
}
);
});
});
@@ -0,0 +1,15 @@
export const AUTOMATION_SCHEDULER_SWEEP_JOB_ID = "automation-scheduler-sweep" as const;
export const AUTOMATION_SCHEDULER_SWEEP_EVERY_MS = 60_000;
export async function registerAutomationSchedulerCron(queue: {
upsertJobScheduler(schedulerId: string, options: { every: number }, template: { name: string; data: Record<string, unknown> }): Promise<unknown>;
}): Promise<void> {
await queue.upsertJobScheduler(
AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
{ every: AUTOMATION_SCHEDULER_SWEEP_EVERY_MS },
{
name: AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: AUTOMATION_SCHEDULER_SWEEP_JOB_ID },
}
);
}
@@ -0,0 +1,23 @@
import { describe, expect, test, vi } from "vitest";
import { BILLING_DUNNING_SWEEP_EVERY_MS, BILLING_DUNNING_SWEEP_JOB_ID, registerBillingDunningSweepCron } from "./billing-dunning-cron.js";
describe("billing-dunning cron", () => {
test("uses a 24h cleanup.run scheduler payload", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerBillingDunningSweepCron({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
BILLING_DUNNING_SWEEP_JOB_ID,
{ every: BILLING_DUNNING_SWEEP_EVERY_MS },
{
name: BILLING_DUNNING_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: BILLING_DUNNING_SWEEP_JOB_ID },
}
);
});
test("registers a daily cadence", () => {
expect(BILLING_DUNNING_SWEEP_EVERY_MS).toBe(24 * 60 * 60 * 1000);
});
});
@@ -0,0 +1,31 @@
// AK-P20 DoD 2 — billing dunning sweep cron.
//
// Daily maintenance job that drives the past_due dunning sequence (day 1/3/7
// notices, then freeze) for every tenant. The decision logic is the pure
// evaluateBillingDunning() from @hivegrid/db/billing-domain.server; the
// execution lives in handlers.ts, which sweeps per tenant from the global
// Tenant registry inside withTenantRLS() and sends notices through
// notifyWithTenantRlsContext() (NotificationDelivery is FORCE RLS). This file
// only carries the config + the idempotent scheduler registration (BullMQ
// `upsertJobScheduler`, like the other maintenance crons).
export const BILLING_DUNNING_SWEEP_JOB_ID = "billing-dunning-sweep" as const;
export const BILLING_DUNNING_SWEEP_EVERY_MS = 24 * 60 * 60 * 1000;
export async function registerBillingDunningSweepCron(
queue: {
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
}
): Promise<void> {
await queue.upsertJobScheduler(
BILLING_DUNNING_SWEEP_JOB_ID,
{ every: BILLING_DUNNING_SWEEP_EVERY_MS },
{
name: BILLING_DUNNING_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: BILLING_DUNNING_SWEEP_JOB_ID,
},
}
);
}
@@ -0,0 +1,146 @@
import { describe, expect, it, vi } from "vitest";
import {
cancelCampaignScheduleWithPrisma,
isCampaignDeliveryAllowedWithPrisma,
type CampaignScheduleCancellationPrisma,
} from "./campaign-schedule-cancellation.js";
function billingSubscription(status = "active", emailEnabled = true) {
return {
tenantId: "tenant-1",
status,
planCode: "starter",
seatQuantity: 2,
lagoCustomerId: null,
mollieCustomerId: null,
lastInvoiceExternalId: null,
lastPaymentExternalId: null,
trialStartsAt: null,
trialEndsAt: null,
pastDueSince: null,
frozenAt: status === "frozen" ? new Date("2026-07-25T12:00:00.000Z") : null,
cancelledAt: null,
products: emailEnabled
? [{
id: "email-product",
createdAt: new Date("2026-07-01T00:00:00.000Z"),
cancelledAt: null,
endsAt: null,
planCode: "email_marketing_s",
seatQuantity: 1,
quantity: 1,
currentPeriodStart: null,
currentPeriodEnd: null,
subscriptionProduct: {
id: "email-s",
code: "email_marketing_s",
lagoPlanCode: "email_marketing_s",
kind: "recurring_add_on",
features: [{
code: null,
name: "email-marketing",
title: "Email marketing",
type: 2,
value: 10_000,
quotaValue: null,
accumulate: true,
}],
},
}]
: [],
};
}
function harness(status = "cancelling", billingStatus = "active", emailEnabled = true) {
const tx = {
$executeRaw: vi.fn(async () => 1),
tenantSubscription: {
findUnique: vi.fn(async () => billingSubscription(billingStatus, emailEnabled)),
findMany: vi.fn(async () => []),
},
campaign: {
findFirst: vi.fn(async () => ({ status })),
updateMany: vi.fn(async () => ({ count: 1 })),
},
outboxEvent: {
updateMany: vi.fn(async () => ({ count: 3 })),
},
outboundEmail: {
deleteMany: vi.fn(async () => ({ count: 3 })),
},
};
const prisma = {
$transaction: vi.fn(async (callback: (value: typeof tx) => Promise<unknown>) => callback(tx)),
} as unknown as CampaignScheduleCancellationPrisma;
return { prisma, tx };
}
describe("campaign schedule cancellation", () => {
it("cancels only pending campaign events and resets the claimed schedule", async () => {
const now = new Date("2026-07-25T12:00:00.000Z");
const { prisma, tx } = harness();
await expect(
cancelCampaignScheduleWithPrisma(prisma, { tenantId: "tenant-1", campaignId: "campaign-1" }, now),
).resolves.toEqual({ cancelledOutboxEvents: 3, removedRecipients: 3, resetCampaigns: 1 });
expect(tx.outboxEvent.updateMany).toHaveBeenCalledWith({
where: {
tenantId: "tenant-1",
eventType: "mail.transactional.send",
dispatchKey: { startsWith: "campaign-send:campaign-1:" },
status: "PENDING",
},
data: {
status: "CANCELLED",
processedAt: now,
lockedUntil: null,
lastError: null,
},
});
expect(tx.outboundEmail.deleteMany).toHaveBeenCalledWith({
where: {
tenantId: "tenant-1",
campaignId: "campaign-1",
deliveredAt: null,
bouncedAt: null,
spamComplainedAt: null,
},
});
expect(tx.campaign.updateMany).toHaveBeenCalledWith({
where: { id: "campaign-1", tenantId: "tenant-1", status: "cancelling" },
data: { status: "draft", sentAt: null, scheduledAt: null },
});
});
it("rejects a command outside the cancellation claim", async () => {
const { prisma, tx } = harness("scheduled");
await expect(cancelCampaignScheduleWithPrisma(prisma, { tenantId: "tenant-1", campaignId: "campaign-1" })).rejects.toThrow(
/invalid lifecycle state/iu,
);
expect(tx.outboxEvent.updateMany).not.toHaveBeenCalled();
});
it.each([
["scheduled", true],
["sending", true],
["cancelling", false],
["draft", false],
])("allows delivery only in an active send state (%s)", async (status, expected) => {
const { prisma } = harness(status);
await expect(isCampaignDeliveryAllowedWithPrisma(prisma, { tenantId: "tenant-1", campaignId: "campaign-1" })).resolves.toBe(expected);
});
it("blocks queued recipients after the paid feature is frozen or revoked", async () => {
await expect(isCampaignDeliveryAllowedWithPrisma(harness("scheduled", "frozen").prisma, {
tenantId: "tenant-1",
campaignId: "campaign-1",
})).resolves.toBe(false);
await expect(isCampaignDeliveryAllowedWithPrisma(harness("scheduled", "active", false).prisma, {
tenantId: "tenant-1",
campaignId: "campaign-1",
})).resolves.toBe(false);
});
});
@@ -0,0 +1,136 @@
import { withTenantRLS, type RlsTransactionExecutor, type TransactionCapableClient } from "../db/tenantRls.js";
import { resolveTenantBillingState, type TenantBillingStateClient } from "@hivegrid/db/billing-domain.server";
import { featureKeys } from "@hivegrid/entitlements";
export type CampaignScheduleCancellationInput = {
tenantId: string;
campaignId: string;
};
export type CampaignScheduleCancellationResult = {
cancelledOutboxEvents: number;
removedRecipients: number;
resetCampaigns: number;
};
type CampaignScheduleCancellationTx = RlsTransactionExecutor & {
tenantSubscription: TenantBillingStateClient["tenantSubscription"];
campaign: {
findFirst(args: {
where: { id: string; tenantId: string };
select: { status: true };
}): Promise<{ status: string } | null>;
updateMany(args: {
where: { id: string; tenantId: string; status: string };
data: { status: string; sentAt: null; scheduledAt: null };
}): Promise<{ count: number }>;
};
outboxEvent: {
updateMany(args: {
where: {
tenantId: string;
eventType: string;
dispatchKey: { startsWith: string };
status: "PENDING";
};
data: {
status: "CANCELLED";
processedAt: Date;
lockedUntil: null;
lastError: null;
};
}): Promise<{ count: number }>;
};
outboundEmail: {
deleteMany(args: {
where: {
tenantId: string;
campaignId: string;
deliveredAt: null;
bouncedAt: null;
spamComplainedAt: null;
};
}): Promise<{ count: number }>;
};
};
export type CampaignScheduleCancellationPrisma = TransactionCapableClient<CampaignScheduleCancellationTx>;
export async function cancelCampaignScheduleWithPrisma(
prisma: CampaignScheduleCancellationPrisma,
input: CampaignScheduleCancellationInput,
now: Date = new Date(),
): Promise<CampaignScheduleCancellationResult> {
return withTenantRLS(prisma, input.tenantId, async (tx) => {
const campaign = await tx.campaign.findFirst({
where: { id: input.campaignId, tenantId: input.tenantId },
select: { status: true },
});
if (!campaign) {
return { cancelledOutboxEvents: 0, removedRecipients: 0, resetCampaigns: 0 };
}
if (campaign.status !== "cancelling") {
throw new Error("Campaign schedule cancellation rejected an invalid lifecycle state.");
}
const cancelled = await tx.outboxEvent.updateMany({
where: {
tenantId: input.tenantId,
eventType: "mail.transactional.send",
dispatchKey: { startsWith: `campaign-send:${input.campaignId}:` },
status: "PENDING",
},
data: {
status: "CANCELLED",
processedAt: now,
lockedUntil: null,
lastError: null,
},
});
const removed = await tx.outboundEmail.deleteMany({
where: {
tenantId: input.tenantId,
campaignId: input.campaignId,
deliveredAt: null,
bouncedAt: null,
spamComplainedAt: null,
},
});
const reset = await tx.campaign.updateMany({
where: { id: input.campaignId, tenantId: input.tenantId, status: "cancelling" },
data: { status: "draft", sentAt: null, scheduledAt: null },
});
if (reset.count !== 1) {
throw new Error("Campaign schedule cancellation lost its lifecycle claim.");
}
return {
cancelledOutboxEvents: cancelled.count,
removedRecipients: removed.count,
resetCampaigns: reset.count,
};
});
}
export async function isCampaignDeliveryAllowedWithPrisma(
prisma: CampaignScheduleCancellationPrisma,
input: CampaignScheduleCancellationInput,
): Promise<boolean> {
return withTenantRLS(prisma, input.tenantId, async (tx) => {
const [campaign, billing] = await Promise.all([
tx.campaign.findFirst({
where: { id: input.campaignId, tenantId: input.tenantId },
select: { status: true },
}),
resolveTenantBillingState(input.tenantId, {
client: tx as unknown as TenantBillingStateClient,
}),
]);
const emailMarketing = billing.entitlements.find((entitlement) => entitlement.featureKey === featureKeys.emailMarketing);
return (
(campaign?.status === "scheduled" || campaign?.status === "sending") &&
!billing.readOnly &&
emailMarketing?.enabled === true
);
});
}
@@ -0,0 +1,70 @@
import { CHAT_LINK_PREVIEW_REQUESTED_OUTBOX_EVENT_TYPE, hashChatLinkPreviewBody } from "@hivegrid/chat/link-preview.server";
import { describe, expect, it, vi } from "vitest";
import { bootstrapChatLinkPreview, type ChatLinkPreviewBootstrapDeps, type ChatLinkPreviewWorkerHandle } from "./chat-link-preview-bootstrap.js";
import { CHAT_LINK_PREVIEW_QUEUE } from "./chat-link-preview.js";
const tenantBudget = { consume: vi.fn(async () => ({ allowed: true, retryAfterMs: 0 })) };
function capturingWorker() {
const captured: { queueName?: string; processor?: (job: { data: unknown }) => Promise<unknown>; concurrency?: number } = {};
const handle: ChatLinkPreviewWorkerHandle = { name: "chat-link-preview", close: vi.fn(async () => undefined) };
const createWorkerImpl: NonNullable<ChatLinkPreviewBootstrapDeps["createWorkerImpl"]> = (queueName, processor, options) => {
captured.queueName = queueName;
captured.processor = processor;
captured.concurrency = options.concurrency;
return handle;
};
return { captured, createWorkerImpl, handle };
}
describe("bootstrapChatLinkPreview", () => {
it("starts only on the general worker group and honors its escape hatch", () => {
expect(bootstrapChatLinkPreview({ WORKER_QUEUE_GROUP: "mail" })).toEqual({ status: "disabled", reason: "non-general worker group" });
expect(bootstrapChatLinkPreview({ WORKER_QUEUE_GROUP: "general", CHAT_LINK_PREVIEW_ENABLED: "false" })).toEqual({
status: "disabled",
reason: "CHAT_LINK_PREVIEW_ENABLED=false",
});
});
it("binds the dedicated outbox queue with bounded concurrency", () => {
const { captured, createWorkerImpl, handle } = capturingWorker();
expect(bootstrapChatLinkPreview({ CHAT_LINK_PREVIEW_CONCURRENCY: "3" }, { createWorkerImpl, tenantBudget })).toEqual({ status: "started", handle });
expect(captured).toMatchObject({ queueName: CHAT_LINK_PREVIEW_QUEUE, concurrency: 3 });
});
it.each(["0", "-1", "bad"])('falls back to concurrency five for "%s"', (value) => {
const { captured, createWorkerImpl } = capturingWorker();
bootstrapChatLinkPreview({ CHAT_LINK_PREVIEW_CONCURRENCY: value }, { createWorkerImpl, tenantBudget });
expect(captured.concurrency).toBe(5);
});
it("fails closed when the host does not provide a worker factory", () => {
expect(bootstrapChatLinkPreview({})).toEqual({ status: "disabled", reason: "no createWorkerImpl wired" });
});
it("fails closed when tenant fairness cannot be enforced", () => {
const { createWorkerImpl } = capturingWorker();
expect(bootstrapChatLinkPreview({}, { createWorkerImpl })).toEqual({ status: "disabled", reason: "no tenant budget store wired" });
});
it("passes the original transactional outbox envelope to the processor", async () => {
const { captured, createWorkerImpl } = capturingWorker();
const runJobImpl = vi.fn(async () => ({
status: "skipped" as const,
messageId: "missing_message",
transientFailures: 0,
scheduledRetries: 0,
deferredByBudget: 0,
}));
bootstrapChatLinkPreview({}, { createWorkerImpl, runJobImpl, tenantBudget, now: () => 1_752_662_400_000 });
const envelope = {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_LINK_PREVIEW_REQUESTED_OUTBOX_EVENT_TYPE,
payload: { chatId: "chat_1", messageId: "missing_message", bodyHash: hashChatLinkPreviewBody("body") },
};
await expect(captured.processor?.({ data: envelope })).resolves.toMatchObject({ status: "skipped", messageId: "missing_message" });
expect(runJobImpl).toHaveBeenCalledWith(envelope, expect.objectContaining({ tenantBudget, now: expect.any(Function) }));
});
});
@@ -0,0 +1,98 @@
import type { EnvRecord } from "@hivegrid/config";
import { createSlidingWindowRateLimiter, type SharedStore } from "@hivegrid/shared-store";
import { CHAT_LINK_PREVIEW_QUEUE, runChatLinkPreviewOutboxJob } from "./chat-link-preview.js";
import type { ChatLinkPreviewLogger, ChatLinkPreviewTenantBudget } from "@hivegrid/chat/link-preview.server";
export type ChatLinkPreviewBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
CHAT_LINK_PREVIEW_ENABLED?: string;
CHAT_LINK_PREVIEW_CONCURRENCY?: string;
CHAT_LINK_PREVIEW_TENANT_REQUESTS_PER_MINUTE?: string;
};
export interface ChatLinkPreviewWorkerHandle {
readonly name: string;
close(): Promise<void>;
}
export interface ChatLinkPreviewBootstrapDeps {
readonly logger?: ChatLinkPreviewLogger;
readonly baseStore?: SharedStore;
readonly tenantBudget?: ChatLinkPreviewTenantBudget;
readonly now?: () => number;
readonly runJobImpl?: typeof runChatLinkPreviewOutboxJob;
readonly createWorkerImpl?: (
queueName: string,
processor: (job: { data: unknown }) => Promise<unknown>,
options: { concurrency?: number }
) => ChatLinkPreviewWorkerHandle;
}
export type ChatLinkPreviewBootstrapResult =
| { readonly status: "disabled"; readonly reason: string }
| { readonly status: "started"; readonly handle: ChatLinkPreviewWorkerHandle };
const DEFAULT_CONCURRENCY = 5;
const DEFAULT_TENANT_REQUESTS_PER_MINUTE = 30;
const TENANT_BUDGET_WINDOW_MS = 60_000;
export function bootstrapChatLinkPreview(env: ChatLinkPreviewBootstrapEnv, deps: ChatLinkPreviewBootstrapDeps = {}): ChatLinkPreviewBootstrapResult {
if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.CHAT_LINK_PREVIEW_ENABLED === "false") {
return { status: "disabled", reason: "CHAT_LINK_PREVIEW_ENABLED=false" };
}
if (!deps.createWorkerImpl) {
return { status: "disabled", reason: "no createWorkerImpl wired" };
}
const concurrency = parsePositiveInt(env.CHAT_LINK_PREVIEW_CONCURRENCY, DEFAULT_CONCURRENCY);
const tenantRequestsPerMinute = parsePositiveInt(env.CHAT_LINK_PREVIEW_TENANT_REQUESTS_PER_MINUTE, DEFAULT_TENANT_REQUESTS_PER_MINUTE);
const now = deps.now ?? Date.now;
const tenantBudget = deps.tenantBudget ?? createTenantBudget(deps.baseStore, tenantRequestsPerMinute, now);
if (!tenantBudget) {
return { status: "disabled", reason: "no tenant budget store wired" };
}
const processor = (job: { data: unknown }) =>
(deps.runJobImpl ?? runChatLinkPreviewOutboxJob)(job.data, {
tenantBudget,
now: () => new Date(now()),
...(deps.logger ? { logger: deps.logger } : {}),
});
const handle = deps.createWorkerImpl(CHAT_LINK_PREVIEW_QUEUE, processor, { concurrency });
deps.logger?.info?.(
{ component: "chat-link-preview", queue: CHAT_LINK_PREVIEW_QUEUE, concurrency, tenantRequestsPerMinute },
"chat link preview worker started"
);
return { status: "started", handle };
}
function createTenantBudget(baseStore: SharedStore | undefined, maxRequests: number, now: () => number): ChatLinkPreviewTenantBudget | null {
if (!baseStore) return null;
const limiter = createSlidingWindowRateLimiter({
store: baseStore,
prefix: "chat-link-preview-tenant",
windowSeconds: TENANT_BUDGET_WINDOW_MS / 1_000,
max: maxRequests,
now,
});
return {
async consume(tenantId) {
const decision = await limiter.consume(tenantId);
if (decision.allowed) return { allowed: true, retryAfterMs: 0 };
const oldestAt = decision.oldestAt?.getTime() ?? now();
return {
allowed: false,
retryAfterMs: Math.max(1_000, oldestAt + TENANT_BUDGET_WINDOW_MS - now()),
};
},
};
}
function parsePositiveInt(value: unknown, fallback: number): number {
if (typeof value !== "string" || value.length === 0) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
@@ -0,0 +1,15 @@
import {
CHAT_LINK_PREVIEW_OUTBOX_QUEUE,
runChatLinkPreviewJob,
type ChatLinkPreviewJobDependencies,
type ChatLinkPreviewJobResult,
} from "@hivegrid/chat/link-preview.server";
export const CHAT_LINK_PREVIEW_QUEUE = CHAT_LINK_PREVIEW_OUTBOX_QUEUE;
export function runChatLinkPreviewOutboxJob(
data: unknown,
dependencies: ChatLinkPreviewJobDependencies = {}
): Promise<ChatLinkPreviewJobResult> {
return runChatLinkPreviewJob(data, dependencies);
}
@@ -0,0 +1,121 @@
import { CHAT_EVENT_NAMES } from "@hivegrid/realtime";
import { MemoryAdapter, createSharedStore } from "@hivegrid/shared-store";
import { describe, expect, it, vi } from "vitest";
import { bootstrapChatRealtimeWakeup, type ChatRealtimeWakeupBootstrapDeps, type ChatRealtimeWakeupWorkerHandle } from "./chat-realtime-wakeup-bootstrap.js";
import { CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE, CHAT_REALTIME_SHARED_STORE_NAMESPACE, CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE } from "./chat-realtime-wakeup.js";
function capturingWorker() {
const captured: {
queueName?: string;
processor?: (job: { data: unknown }) => Promise<unknown>;
concurrency?: number;
} = {};
const handle: ChatRealtimeWakeupWorkerHandle = {
name: "chat-realtime-wakeup",
close: vi.fn(async () => undefined),
};
const createWorkerImpl: NonNullable<ChatRealtimeWakeupBootstrapDeps["createWorkerImpl"]> = (queueName, processor, options) => {
captured.queueName = queueName;
captured.processor = processor;
captured.concurrency = options.concurrency;
return handle;
};
return { captured, createWorkerImpl, handle };
}
function validJob() {
return {
data: {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
payload: {
event: {
event: CHAT_EVENT_NAMES.readReceipt,
cursor: "stream_1:7",
requestId: "request_1",
tenantId: "tenant_1",
chatId: "chat_1",
occurredAt: "2026-07-16T08:00:00.000Z",
data: {
messageId: "message_1",
userId: "user_1",
readAt: "2026-07-16T08:00:00.000Z",
},
},
},
},
};
}
describe("bootstrapChatRealtimeWakeup", () => {
it("disables outside the general worker group or via its escape hatch", () => {
expect(bootstrapChatRealtimeWakeup({ WORKER_QUEUE_GROUP: "mail" })).toEqual({
status: "disabled",
reason: "non-general worker group",
});
expect(
bootstrapChatRealtimeWakeup({
WORKER_QUEUE_GROUP: "general",
CHAT_REALTIME_WAKEUP_ENABLED: "false",
})
).toEqual({ status: "disabled", reason: "CHAT_REALTIME_WAKEUP_ENABLED=false" });
});
it("disables when required host seams are absent", () => {
const { createWorkerImpl } = capturingWorker();
expect(bootstrapChatRealtimeWakeup({}, { createWorkerImpl })).toMatchObject({
status: "disabled",
reason: "no shared-store adapter wired",
});
const baseStore = createSharedStore({ namespace: "worker", adapter: new MemoryAdapter() });
expect(bootstrapChatRealtimeWakeup({}, { baseStore })).toMatchObject({
status: "disabled",
reason: "no createWorkerImpl wired",
});
});
it("starts the dedicated outbox consumer with the configured concurrency", () => {
const { captured, createWorkerImpl, handle } = capturingWorker();
const baseStore = createSharedStore({ namespace: "worker", adapter: new MemoryAdapter() });
const result = bootstrapChatRealtimeWakeup({ WORKER_QUEUE_GROUP: "general", CHAT_REALTIME_WAKEUP_CONCURRENCY: "4" }, { baseStore, createWorkerImpl });
expect(result).toEqual({ status: "started", handle });
expect(captured.queueName).toBe(CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE);
expect(captured.concurrency).toBe(4);
});
it("rebinds the worker store to chat-realtime and publishes through the captured processor", async () => {
const { captured, createWorkerImpl } = capturingWorker();
const adapter = new MemoryAdapter();
const baseStore = createSharedStore({ namespace: "worker", adapter });
const realtimeStore = createSharedStore({ namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE, adapter });
const received: string[] = [];
const unsubscribe = await realtimeStore.subscribe("persistent:tenant_1:chat_1", (raw) => {
received.push(raw);
});
bootstrapChatRealtimeWakeup({}, { baseStore, createWorkerImpl });
await captured.processor?.(validJob());
expect(received).toHaveLength(1);
expect(JSON.parse(received[0] ?? "null")).toMatchObject({
event: CHAT_EVENT_NAMES.readReceipt,
tenantId: "tenant_1",
chatId: "chat_1",
cursor: "stream_1:7",
});
await unsubscribe();
});
it.each(["0", "-2", "nope"])("uses default concurrency for %s", (raw) => {
const { captured, createWorkerImpl } = capturingWorker();
const baseStore = createSharedStore({ namespace: "worker", adapter: new MemoryAdapter() });
bootstrapChatRealtimeWakeup({ CHAT_REALTIME_WAKEUP_CONCURRENCY: raw }, { baseStore, createWorkerImpl });
expect(captured.concurrency).toBe(10);
});
});
@@ -0,0 +1,83 @@
import type { EnvRecord } from "@hivegrid/config";
import { createSharedStore, type SharedStore } from "@hivegrid/shared-store";
import {
CHAT_REALTIME_SHARED_STORE_NAMESPACE,
CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE,
runChatRealtimeWakeupJob,
type ChatRealtimeWakeupLogger,
} from "./chat-realtime-wakeup.js";
export type ChatRealtimeWakeupBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
CHAT_REALTIME_WAKEUP_ENABLED?: string;
CHAT_REALTIME_WAKEUP_CONCURRENCY?: string;
};
export interface ChatRealtimeWakeupWorkerHandle {
readonly name: string;
close(): Promise<void>;
}
export interface ChatRealtimeWakeupBootstrapDeps {
readonly logger?: ChatRealtimeWakeupLogger;
readonly baseStore?: Pick<SharedStore, "adapter">;
readonly createWorkerImpl?: (
queueName: string,
processor: (job: { data: unknown }) => Promise<unknown>,
options: { concurrency?: number }
) => ChatRealtimeWakeupWorkerHandle;
}
export type ChatRealtimeWakeupBootstrapResult =
| { readonly status: "disabled"; readonly reason: string }
| { readonly status: "started"; readonly handle: ChatRealtimeWakeupWorkerHandle };
const DEFAULT_CONCURRENCY = 10;
export function bootstrapChatRealtimeWakeup(
env: ChatRealtimeWakeupBootstrapEnv,
deps: ChatRealtimeWakeupBootstrapDeps = {}
): ChatRealtimeWakeupBootstrapResult {
if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.CHAT_REALTIME_WAKEUP_ENABLED === "false") {
return { status: "disabled", reason: "CHAT_REALTIME_WAKEUP_ENABLED=false" };
}
if (!deps.baseStore) {
return { status: "disabled", reason: "no shared-store adapter wired" };
}
if (!deps.createWorkerImpl) {
return { status: "disabled", reason: "no createWorkerImpl wired" };
}
const store = createSharedStore({
namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE,
adapter: deps.baseStore.adapter,
});
const concurrency = parsePositiveInt(env.CHAT_REALTIME_WAKEUP_CONCURRENCY, DEFAULT_CONCURRENCY);
const processor = (job: { data: unknown }) =>
runChatRealtimeWakeupJob(job.data, {
store,
...(deps.logger ? { logger: deps.logger } : {}),
});
const handle = deps.createWorkerImpl(CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE, processor, { concurrency });
deps.logger?.info?.(
{
component: "chat-realtime-wakeup",
queue: CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE,
namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE,
concurrency,
},
"chat realtime wakeup worker started"
);
return { status: "started", handle };
}
function parsePositiveInt(value: unknown, fallback: number): number {
if (typeof value !== "string" || value.length === 0) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
@@ -0,0 +1,126 @@
import { CHAT_EVENT_NAMES } from "@hivegrid/realtime";
import { MemoryAdapter, createSharedStore } from "@hivegrid/shared-store";
import { describe, expect, it, vi } from "vitest";
import {
CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
CHAT_REALTIME_SHARED_STORE_NAMESPACE,
CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE,
ChatRealtimeWakeupJobError,
parseChatRealtimeWakeupJob,
runChatRealtimeWakeupJob,
} from "./chat-realtime-wakeup.js";
function persistentEvent() {
return {
event: CHAT_EVENT_NAMES.readReceipt,
cursor: "stream_1:7",
requestId: "request_1",
tenantId: "tenant_1",
chatId: "chat_1",
occurredAt: "2026-07-16T08:00:00.000Z",
data: {
messageId: "message_1",
userId: "user_1",
readAt: "2026-07-16T08:00:00.000Z",
},
};
}
function validJob() {
return {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
payload: { event: persistentEvent() },
dispatchKey: "chat-realtime:ignored-by-consumer",
retries: 0,
};
}
describe("chat realtime wakeup job", () => {
it("uses the dedicated queue, event type and shared-store namespace", () => {
expect(CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE).toBe("outbox.chat-realtime");
expect(CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE).toBe("chat.realtime.persistent");
expect(CHAT_REALTIME_SHARED_STORE_NAMESPACE).toBe("chat-realtime");
});
it("parses a dispatcher envelope and normalizes its persistent event", () => {
expect(parseChatRealtimeWakeupJob(validJob())).toEqual({
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
event: persistentEvent(),
});
});
it.each([
["missing envelope", null],
["unsupported event type", { ...validJob(), eventType: "chat.realtime.ephemeral" }],
[
"cross-tenant payload",
{
...validJob(),
payload: { event: { ...persistentEvent(), tenantId: "tenant_2" } },
},
],
[
"ephemeral event",
{
...validJob(),
payload: {
event: {
event: CHAT_EVENT_NAMES.typingStarted,
cursor: null,
requestId: "request_1",
tenantId: "tenant_1",
chatId: "chat_1",
occurredAt: "2026-07-16T08:00:00.000Z",
data: { userId: "user_1" },
},
},
},
],
])("rejects %s", (_label, input) => {
expect(() => parseChatRealtimeWakeupJob(input)).toThrow(ChatRealtimeWakeupJobError);
});
it("publishes the full event to the gateway's persistent room channel", async () => {
const adapter = new MemoryAdapter();
const store = createSharedStore({ namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE, adapter });
const received: string[] = [];
const unsubscribe = await store.subscribe("persistent:tenant_1:chat_1", (raw) => {
received.push(raw);
});
const result = await runChatRealtimeWakeupJob(validJob(), { store });
expect(result).toEqual({
published: true,
channel: "persistent:tenant_1:chat_1",
cursor: "stream_1:7",
subscriberCount: 1,
});
expect(received).toEqual([JSON.stringify(persistentEvent())]);
await unsubscribe();
});
it("propagates publish failures so BullMQ can retry the outbox delivery", async () => {
const publish = vi.fn(async () => {
throw new Error("redis unavailable");
});
await expect(
runChatRealtimeWakeupJob(validJob(), {
store: { namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE, publish },
})
).rejects.toThrow("redis unavailable");
});
it("rejects an incorrectly namespaced store before publishing", async () => {
const publish = vi.fn(async () => 0);
await expect(runChatRealtimeWakeupJob(validJob(), { store: { namespace: "worker", publish } })).rejects.toMatchObject({ code: "INVALID_STORE_NAMESPACE" });
expect(publish).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,121 @@
import { CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE } from "@hivegrid/chat/realtime-wakeup.server";
import { isPersistentChatEvent, validateChatServerEvent, type ChatPersistentEvent } from "@hivegrid/realtime";
import type { SharedStore } from "@hivegrid/shared-store";
export const CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE = "outbox.chat-realtime" as const;
export { CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE };
export const CHAT_REALTIME_SHARED_STORE_NAMESPACE = "chat-realtime" as const;
const SCOPE_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
const OUTBOX_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,190}$/;
export interface ChatRealtimeWakeupLogger {
info?: (record: Record<string, unknown>, message?: string) => void;
error?: (record: Record<string, unknown>, message?: string) => void;
}
export type ChatRealtimeWakeupStore = Pick<SharedStore, "namespace" | "publish">;
export interface ChatRealtimeWakeupDependencies {
readonly store: ChatRealtimeWakeupStore;
readonly logger?: ChatRealtimeWakeupLogger;
}
export interface ChatRealtimeWakeupJobData {
readonly outboxEventId: string;
readonly tenantId: string;
readonly eventType: typeof CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE;
readonly event: ChatPersistentEvent;
}
export interface ChatRealtimeWakeupResult {
readonly published: true;
readonly channel: string;
readonly cursor: string;
readonly subscriberCount: number;
}
export class ChatRealtimeWakeupJobError extends Error {
override readonly name = "ChatRealtimeWakeupJobError";
constructor(
message: string,
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "TENANT_MISMATCH" | "INVALID_STORE_NAMESPACE"
) {
super(message);
}
}
export function parseChatRealtimeWakeupJob(data: unknown): ChatRealtimeWakeupJobData {
const envelope = requireRecord(data, "chat realtime wakeup job envelope");
const outboxEventId = requireIdentifier(envelope.outboxEventId, "outboxEventId", OUTBOX_IDENTIFIER_PATTERN);
const tenantId = requireIdentifier(envelope.tenantId, "tenantId", SCOPE_IDENTIFIER_PATTERN);
if (envelope.eventType !== CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE) {
throw new ChatRealtimeWakeupJobError(`chat realtime wakeup rejected eventType ${JSON.stringify(envelope.eventType)}`, "INVALID_EVENT_TYPE");
}
const payload = requireRecord(envelope.payload, "chat realtime wakeup payload");
let event;
try {
event = validateChatServerEvent(payload.event);
} catch (error) {
throw new ChatRealtimeWakeupJobError(
`chat realtime wakeup rejected event payload: ${error instanceof Error ? error.message : "invalid event"}`,
"INVALID_PAYLOAD"
);
}
if (!isPersistentChatEvent(event)) {
throw new ChatRealtimeWakeupJobError("chat realtime wakeup only accepts persistent chat events", "INVALID_PAYLOAD");
}
requireIdentifier(event.chatId, "event.chatId", SCOPE_IDENTIFIER_PATTERN);
requireIdentifier(event.tenantId, "event.tenantId", SCOPE_IDENTIFIER_PATTERN);
if (event.tenantId !== tenantId) {
throw new ChatRealtimeWakeupJobError("chat realtime wakeup tenant does not match its outbox envelope", "TENANT_MISMATCH");
}
return {
outboxEventId,
tenantId,
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
event,
};
}
export async function runChatRealtimeWakeupJob(data: unknown, deps: ChatRealtimeWakeupDependencies): Promise<ChatRealtimeWakeupResult> {
if (deps.store.namespace !== CHAT_REALTIME_SHARED_STORE_NAMESPACE) {
throw new ChatRealtimeWakeupJobError(`chat realtime wakeup store must use namespace ${CHAT_REALTIME_SHARED_STORE_NAMESPACE}`, "INVALID_STORE_NAMESPACE");
}
const job = parseChatRealtimeWakeupJob(data);
const channel = `persistent:${job.tenantId}:${job.event.chatId}`;
// BullMQ may replay after a publish/ack crash. Repeating this wakeup is safe:
// gateways always gap-fill the durable TeamChatEvent ledger by cursor.
const subscriberCount = await deps.store.publish(channel, JSON.stringify(job.event));
deps.logger?.info?.(
{
component: "chat-realtime-wakeup",
outboxEventId: job.outboxEventId,
tenantId: job.tenantId,
chatId: job.event.chatId,
cursor: job.event.cursor,
subscriberCount,
},
"chat persistent realtime wakeup published"
);
return { published: true, channel, cursor: job.event.cursor, subscriberCount };
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new ChatRealtimeWakeupJobError(`${label} must be an object`, "INVALID_PAYLOAD");
}
return value as Record<string, unknown>;
}
function requireIdentifier(value: unknown, label: string, pattern: RegExp): string {
if (typeof value !== "string" || !pattern.test(value)) {
throw new ChatRealtimeWakeupJobError(`${label} is invalid`, "INVALID_PAYLOAD");
}
return value;
}
@@ -0,0 +1,61 @@
import { describe, expect, test, vi } from "vitest";
import { createSharedStore, MemoryAdapter } from "@hivegrid/shared-store";
import { withDistributedJobLease } from "./distributed-job-lease.js";
describe("distributed job lease", () => {
test("lets only one replica execute a cron key and releases it for the next tick", async () => {
const adapter = new MemoryAdapter();
const replicaA = createSharedStore({ namespace: "worker-lease-test", adapter });
const replicaB = createSharedStore({ namespace: "worker-lease-test", adapter });
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const worker = vi.fn(async () => {
await gate;
return "done";
});
const first = withDistributedJobLease(replicaA, "email-inbox-webhook-drain", worker, { ownerToken: "owner-a" });
await vi.waitFor(() => expect(worker).toHaveBeenCalledTimes(1));
await expect(withDistributedJobLease(replicaB, "email-inbox-webhook-drain", async () => "overlap", { ownerToken: "owner-b" })).resolves.toEqual({
acquired: false,
});
release();
await expect(first).resolves.toEqual({ acquired: true, value: "done" });
await expect(withDistributedJobLease(replicaB, "email-inbox-webhook-drain", async () => "next", { ownerToken: "owner-b" })).resolves.toEqual({
acquired: true,
value: "next",
});
});
test("never deletes a successor's lease when an old owner releases late", async () => {
let now = 0;
const adapter = new MemoryAdapter({ now: () => now });
const store = createSharedStore({ namespace: "worker-lease-expiry-test", adapter });
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const first = withDistributedJobLease(
store,
"task",
async () => {
await gate;
return "old";
},
{ ownerToken: "old-owner", leaseSeconds: 10 }
);
await vi.waitFor(() => expect(adapter.get("shared-store:worker-lease-expiry-test:job-lease:task")).resolves.toBe("old-owner"));
now = 11_000;
await expect(store.set("job-lease:task", "successor", { nx: true, ttlSeconds: 120 })).resolves.toBe(true);
release();
await expect(first).resolves.toEqual({ acquired: true, value: "old" });
await expect(store.get("job-lease:task")).resolves.toBe("successor");
});
});
@@ -0,0 +1,90 @@
import { randomUUID } from "node:crypto";
import { MemoryAdapter, type SharedStore } from "@hivegrid/shared-store";
export const DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS = 120;
const RENEW_LEASE_SCRIPT = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("expire", KEYS[1], ARGV[2])
end
return 0
`;
const RELEASE_LEASE_SCRIPT = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
end
return 0
`;
MemoryAdapter.registerScript(RENEW_LEASE_SCRIPT, async (adapter, keys, args) => {
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
return (await adapter.expire(keys[0]!, Number(args[1]))) ? 1 : 0;
});
MemoryAdapter.registerScript(RELEASE_LEASE_SCRIPT, async (adapter, keys, args) => {
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
return adapter.del(keys[0]!);
});
export class DistributedJobLeaseLostError extends Error {
constructor(jobKey: string) {
super(`Distributed lease was lost while ${jobKey} was running`);
this.name = "DistributedJobLeaseLostError";
}
}
export type DistributedJobLeaseResult<Value> = { acquired: true; value: Value } | { acquired: false };
/**
* Run a cron body under a replica-wide Redis lease. SET NX prevents overlap;
* token-checked Lua renewal/release prevents an expired owner from touching a
* successor's lease. Store failures fail closed instead of running unlocked.
*/
export async function withDistributedJobLease<Value>(
store: SharedStore,
jobKey: string,
worker: () => Promise<Value>,
options: { leaseSeconds?: number; ownerToken?: string } = {}
): Promise<DistributedJobLeaseResult<Value>> {
const leaseSeconds = Math.max(10, Math.min(Math.floor(options.leaseSeconds ?? DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS), 3_600));
const ownerToken = options.ownerToken ?? randomUUID();
const key = `job-lease:${jobKey}`;
const acquired = await store.set(key, ownerToken, { nx: true, ttlSeconds: leaseSeconds });
if (!acquired) return { acquired: false };
let stopped = false;
let lost = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const heartbeatMs = Math.max(1_000, Math.floor((leaseSeconds * 1_000) / 3));
const scheduleHeartbeat = () => {
timer = setTimeout(async () => {
if (stopped) return;
try {
const renewed = await store.eval(RENEW_LEASE_SCRIPT, [key], [ownerToken, String(leaseSeconds)]);
if (Number(renewed) !== 1) lost = true;
} catch {
lost = true;
}
if (!stopped && !lost) scheduleHeartbeat();
}, heartbeatMs);
timer.unref?.();
};
scheduleHeartbeat();
try {
const value = await worker();
if (lost) throw new DistributedJobLeaseLostError(jobKey);
return { acquired: true, value };
} finally {
stopped = true;
if (timer) clearTimeout(timer);
try {
await store.eval(RELEASE_LEASE_SCRIPT, [key], [ownerToken]);
} catch {
// The bounded TTL is the final safety net when Redis disappears during
// release. Never issue an unconditional DEL that could remove a new owner.
}
}
}
@@ -2,12 +2,7 @@ import { PassThrough } from "node:stream";
import { describe, expect, it, vi } from "vitest";
import {
runDsgvoExport,
type DsgvoExportDependencies,
type DsgvoExportJobInput,
type DsgvoExportSink,
} from "./dsgvo-export.js";
import { runDsgvoExport, type DsgvoExportDependencies, type DsgvoExportJobInput, type DsgvoExportSink } from "./dsgvo-export.js";
function makeSink(): { sink: DsgvoExportSink; collected: () => string } {
const writeStream = new PassThrough();
@@ -107,6 +102,37 @@ describe("runDsgvoExport", () => {
expect(mails[0]?.signedUrl).toBe("https://s3/signed");
});
it("honours writable backpressure before advancing the row source", async () => {
const { sink } = makeSink();
const originalWrite = sink.writeStream.write.bind(sink.writeStream);
let drains = 0;
sink.writeStream.write = ((chunk: string | Uint8Array) => {
originalWrite(chunk);
queueMicrotask(() => {
drains += 1;
sink.writeStream.emit("drain");
});
return false;
}) as typeof sink.writeStream.write;
let drainsBeforeFirstRow = -1;
const { deps } = makeDeps({
openSink: () => sink,
tableSource: {
async *iterate(_userId, entry) {
if (entry.model === "Log") {
drainsBeforeFirstRow = drains;
yield { id: "log1" };
}
},
},
});
await runDsgvoExport(deps, baseJob);
expect(drainsBeforeFirstRow).toBeGreaterThanOrEqual(5);
expect(drains).toBeGreaterThan(drainsBeforeFirstRow);
});
it("audits a failure when the user cannot be loaded", async () => {
const { deps, audits } = makeDeps({
loadUser: vi.fn().mockResolvedValue(null),
@@ -108,14 +108,7 @@ export type DsgvoExportDependencies = {
metadata: Record<string, unknown>;
}) => Promise<void>;
/** Notify the user that the export is ready. */
sendExportReadyEmail: (input: {
userId: string;
toEmail: string;
signedUrl: string;
expiresAt: Date;
sizeBytes: number;
rowCount: number;
}) => Promise<void>;
sendExportReadyEmail: (input: { userId: string; toEmail: string; signedUrl: string; expiresAt: Date; sizeBytes: number; rowCount: number }) => Promise<void>;
/** Object-key builder — caller controls bucket + path layout. */
buildObjectKey: (input: { userId: string; requestId: string }) => { bucket: string; key: string };
/** Logger seam — no console.* in production code. */
@@ -153,31 +146,31 @@ export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoEx
});
try {
sink.writeStream.write(`{\n "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
sink.writeStream.write(` "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
sink.writeStream.write(` "requestId": ${JSON.stringify(job.requestId)},\n`);
sink.writeStream.write(` "user": ${JSON.stringify(user)},\n`);
sink.writeStream.write(` "sections": {\n`);
await writeChunk(sink.writeStream, `{\n "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
await writeChunk(sink.writeStream, ` "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
await writeChunk(sink.writeStream, ` "requestId": ${JSON.stringify(job.requestId)},\n`);
await writeChunk(sink.writeStream, ` "user": ${JSON.stringify(user)},\n`);
await writeChunk(sink.writeStream, ` "sections": {\n`);
let firstSection = true;
for (const entry of exportEntries) {
if (!firstSection) sink.writeStream.write(",\n");
if (!firstSection) await writeChunk(sink.writeStream, ",\n");
firstSection = false;
sink.writeStream.write(` ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
await writeChunk(sink.writeStream, ` ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
let rowCount = 0;
let firstRow = true;
for await (const row of deps.tableSource.iterate(job.userId, entry)) {
if (!firstRow) sink.writeStream.write(",");
if (!firstRow) await writeChunk(sink.writeStream, ",");
firstRow = false;
sink.writeStream.write(JSON.stringify(row));
await writeChunk(sink.writeStream, JSON.stringify(row));
rowCount += 1;
}
sink.writeStream.write("]");
await writeChunk(sink.writeStream, "]");
sections.push({ model: entry.model, column: entry.column, rowCount });
totalRows += rowCount;
}
sink.writeStream.write("\n }\n}\n");
await writeChunk(sink.writeStream, "\n }\n}\n");
sink.writeStream.end();
const completion = await sink.completionPromise;
@@ -242,3 +235,24 @@ export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoEx
throw error;
}
}
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
@@ -0,0 +1,180 @@
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
import { USER_FK_MAP } from "@hivegrid/db/dsgvo/user-fk-map";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
createWorkerDsgvoGraceCronService,
DSGVO_WORKER_SKIPPED_MODELS,
listWorkerProcessableFkEntries,
type DsgvoGracePrisma,
} from "./dsgvo-grace-bootstrap.js";
const createAuditEventMock = vi.fn().mockResolvedValue(undefined);
vi.mock("@hivegrid/db/audit-events.server", () => ({
createAuditEvent: (input: unknown) => createAuditEventMock(input),
}));
function makeLogger() {
return { info: vi.fn(), error: vi.fn() };
}
function lowerFirst(value: string): string {
return value.charAt(0).toLowerCase() + value.slice(1);
}
/**
* Fake prisma that mimics the FORCE-RLS behaviour instead of answering
* every query (AGENTS §RLS: a mock that always answers hides exactly the
* unbound-context bug this wiring fixes). Every delegate records the
* ambient RLS context and returns zero rows when no context is bound.
*/
function makeFakePrisma(options: { memberships: Record<string, string[]>; overdueByTenant: Record<string, string[]> }) {
const calls: Array<{ model: string; op: string; ctx: { userId?: string; tenantId?: string } | null }> = [];
const ctx = () => {
const current = getRlsExecutionContext();
return current ? { userId: current.userId, tenantId: current.tenantId } : null;
};
const base: Record<string, unknown> = {
tenant: {
findMany: async () => Object.keys(options.overdueByTenant).map((id) => ({ id })),
},
user: {
findMany: async () => {
const current = ctx();
calls.push({ model: "User", op: "findMany", ctx: current });
// RLS: without a bound tenant the membership branch never matches.
if (!current?.tenantId) return [];
return (options.overdueByTenant[current.tenantId] ?? []).map((id) => ({
id,
scheduledDeletionAt: new Date("2026-06-01T00:00:00Z"),
deletionRequestId: `req-${id}`,
}));
},
update: async (args: { where: { id: string }; data: Record<string, unknown> }) => {
const current = ctx();
calls.push({ model: "User", op: "update", ctx: current });
if (current?.userId !== args.where.id) {
throw new Error("RLS: User update requires the user context branch.");
}
return {};
},
},
tenantUser: {
findMany: async (args: { where: { userId: string } }) => {
const current = ctx();
calls.push({ model: "TenantUser", op: "findMany", ctx: current });
if (current?.userId !== args.where.userId) return [];
return (options.memberships[args.where.userId] ?? []).map((tenantId) => ({ tenantId }));
},
},
};
// Generic delegates for every worker-processable FK-map model. Tenant
// passes count one row, the trailing user-only pass counts zero (the
// tenant pass already consumed the row) — mirroring policy visibility.
for (const entry of listWorkerProcessableFkEntries()) {
const key = lowerFirst(entry.model);
if (base[key] && (base[key] as Record<string, unknown>).updateMany) continue;
const record = (op: string) => {
const current = ctx();
calls.push({ model: entry.model, op, ctx: current });
return { count: current?.tenantId ? 1 : 0 };
};
base[key] = {
...(base[key] as Record<string, unknown> | undefined),
updateMany: async () => record("updateMany"),
deleteMany: async () => record("deleteMany"),
};
}
return { prisma: base as unknown as DsgvoGracePrisma, calls };
}
beforeEach(() => {
createAuditEventMock.mockClear();
});
describe("createWorkerDsgvoGraceCronService", () => {
it("scans overdue users per tenant under a bound RLS context and dedupes across tenants", async () => {
const { prisma, calls } = makeFakePrisma({
memberships: {},
overdueByTenant: { t1: ["u1", "u2"], t2: ["u1"] },
});
const service = createWorkerDsgvoGraceCronService({ prisma, logger: makeLogger() });
const candidates = await service.listOverdueUsers(new Date("2026-07-17T00:00:00Z"));
expect(candidates.map((candidate) => candidate.userId).sort()).toEqual(["u1", "u2"]);
expect(candidates).toHaveLength(2);
const scans = calls.filter((call) => call.model === "User" && call.op === "findMany");
expect(scans).toHaveLength(2);
for (const scan of scans) {
expect(scan.ctx?.tenantId).toBeTruthy();
}
});
it("runs the cascade once per membership tenant plus a user-only pass, all context-bound", async () => {
const { prisma, calls } = makeFakePrisma({
memberships: { u1: ["t1", "t2"] },
overdueByTenant: { t1: [], t2: [] },
});
const logger = makeLogger();
const service = createWorkerDsgvoGraceCronService({ prisma, logger });
const result = await service.enqueueDeletion({
userId: "u1",
tenantId: "t1",
requestId: "req-u1",
actorType: "system",
actorId: "dsgvo-grace-cron",
mode: "pseudonymize",
reason: "dsgvo-grace-period-elapsed",
});
expect(result.jobId).toBe("inline-req-u1");
// Every processable FK-map entry ran in 3 contexts (t1, t2, user-only).
const groupUserCalls = calls.filter((call) => call.model === "GroupUser");
expect(groupUserCalls).toHaveLength(3);
expect(groupUserCalls.map((call) => call.ctx?.tenantId)).toEqual(["t1", "t2", undefined]);
for (const call of groupUserCalls) {
expect(call.ctx?.userId).toBe("u1");
}
// Pseudonymize mode keeps the User row: exactly one context-bound update.
const userUpdates = calls.filter((call) => call.model === "User" && call.op === "update");
expect(userUpdates).toHaveLength(1);
expect(userUpdates[0]?.ctx?.userId).toBe("u1");
// Grant-blocked models were never touched (no delegate exists for them,
// so a missed skip would have thrown), and the skip is audited.
const audit = createAuditEventMock.mock.calls.at(-1)?.[0] as { metadata: { skippedModels: string[] } };
expect(audit.metadata.skippedModels).toContain("UserRole.userId");
expect(logger.info).toHaveBeenCalledWith(
expect.objectContaining({ model: "UserRole" }),
"dsgvo-deletion.table-skipped-no-worker-grant"
);
});
it("refuses the hard-delete mode the worker grant set cannot satisfy", async () => {
const { prisma } = makeFakePrisma({ memberships: { u1: [] }, overdueByTenant: {} });
const service = createWorkerDsgvoGraceCronService({ prisma, logger: makeLogger() });
await expect(
service.enqueueDeletion({
userId: "u1",
tenantId: null,
requestId: "req-u1",
actorType: "system",
actorId: "dsgvo-grace-cron",
mode: "delete",
reason: "dsgvo-grace-period-elapsed",
})
).rejects.toThrow(/hard delete is not available/i);
});
it("classifies every USER_FK_MAP model so new admin-surface tables cannot slip through untriaged", () => {
const processable = new Set(listWorkerProcessableFkEntries().map((entry) => entry.model));
for (const entry of USER_FK_MAP) {
expect(processable.has(entry.model) || DSGVO_WORKER_SKIPPED_MODELS.has(entry.model)).toBe(true);
}
});
});
@@ -0,0 +1,249 @@
/**
* R-P2b Restarbeit (2026-07-17) — production wiring for the DSGVO
* grace-period cron. Until now `dsgvoGraceCronService` was never
* constructed, so the daily scan acked every tick with
* "dsgvoGraceCronService not configured" and a confirmed deletion
* request was never executed.
*
* Design (mirrors privacy-tenant-deletion-bootstrap / AK-P21):
*
* listOverdueUsers → per-tenant sweep (runActiveTenantSweep) with the
* tenant context bound via runWithRlsContext(); the
* `User` select policy exposes a user to the worker
* only through a bound membership tenant, so a
* global unscoped scan would silently return zero
* rows under FORCE RLS. Candidates are deduped by
* userId across tenants.
* enqueueDeletion → runs the tested runDsgvoDeletion cascade inline
* (serialized, like the AK-P21 hard-delete scan —
* no extra queue hop; the cron core already
* isolates failures per candidate and the daily
* re-scan is the retry, idempotent because the
* cascade flips User.pendingDeletion=false).
*
* RLS strategy per cascade table walk: every USER_FK_MAP operation runs
* once per membership tenant under { userId, tenantId } and once under
* { userId } alone. Tenant-scoped tables (TenantUser, GroupUser, …) bind
* through the tenant branch, user-owned tables (UserSession,
* TrustedDevice, MagicLinkToken, …) through the user branch. Re-running
* an operation in a second context is idempotent (the first pass already
* rewrote/deleted the rows), so the summed counts stay accurate.
*
* Deliberate skips: the RLS contract (20260713190000_appwide_rls_cutover)
* grants the worker no write privileges on the platform-admin/support
* tables and only SELECT on UserRole. A write attempt would abort the
* cascade with a permission error, so those entries are skipped loudly
* (log + audit metadata). The rows keep referencing the pseudonymized
* User skeleton — the identity link is anonymized even though the rows
* survive. Extending the grants is a tracked follow-up on the RLS
* contract, not something this bootstrap guesses at.
*
* Mode is always "pseudonymize" (R-P2b: PII-Anonymisierung, kein
* Hard-Delete — audit hash chain and retention FKs stay intact).
*/
import { createAuditEvent } from "@hivegrid/db/audit-events.server";
import { USER_FK_MAP, type UserFkMapEntry } from "@hivegrid/db/dsgvo/user-fk-map";
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
import { runActiveTenantSweep, throwOnActiveTenantSweepFailures, type ActiveTenantReader } from "./active-tenant-sweep.js";
import { runDsgvoDeletion, type DsgvoDeletionJobInput } from "./dsgvo-deletion.js";
import type { DsgvoGraceCandidate } from "./dsgvo-grace-cron.js";
/**
* USER_FK_MAP models the worker must not touch: no write GRANT (platform
* admin/support surface) or SELECT-only (UserRole). Kept in sync with the
* grant matrix now frozen in prisma/migrations/00000000000000_hivegrid_baseline.
*/
export const DSGVO_WORKER_SKIPPED_MODELS: ReadonlySet<string> = new Set([
"AdminTenantNote",
"AdminUser",
"Feedback",
"ImpersonationAudit",
"SupportTicket",
"SupportTicketMessage",
"UserRole",
]);
/** Upper bound of overdue users fetched per tenant per daily pass. */
export const DSGVO_GRACE_SCAN_PAGE_SIZE = 100;
type UpdateManyDelegate = { updateMany(args: { where: Record<string, unknown>; data: Record<string, unknown> }): Promise<{ count: number }> };
type DeleteManyDelegate = { deleteMany(args: { where: Record<string, unknown> }): Promise<{ count: number }> };
export type DsgvoGracePrisma = Record<string, unknown> &
ActiveTenantReader & {
user: {
findMany(args: {
where: Record<string, unknown>;
select: { id: true; scheduledDeletionAt: true; deletionRequestId: true };
orderBy: { id: "asc" };
take: number;
}): Promise<Array<{ id: string; scheduledDeletionAt: Date | null; deletionRequestId: string | null }>>;
update(args: { where: { id: string }; data: Record<string, unknown> }): Promise<unknown>;
};
tenantUser: {
findMany(args: { where: { userId: string }; select: { tenantId: true } }): Promise<Array<{ tenantId: string }>>;
};
};
export type WorkerDsgvoGraceDependencies = {
prisma: DsgvoGracePrisma;
now?: () => Date;
logger?: {
info(fields: Record<string, unknown>, message: string): void;
error(fields: Record<string, unknown>, message: string): void;
};
};
export type WorkerDsgvoGraceCronService = {
listOverdueUsers(now: Date): Promise<DsgvoGraceCandidate[]>;
enqueueDeletion(input: DsgvoDeletionJobInput): Promise<{ jobId?: string }>;
};
const noopLogger = {
info: () => undefined,
error: () => undefined,
};
export function createWorkerDsgvoGraceCronService(deps: WorkerDsgvoGraceDependencies): WorkerDsgvoGraceCronService {
const logger = deps.logger ?? noopLogger;
return {
async listOverdueUsers(now: Date): Promise<DsgvoGraceCandidate[]> {
const sweep = await runActiveTenantSweep(deps.prisma, {
initialState: new Map<string, DsgvoGraceCandidate>(),
worker: (tenantId) =>
runWithRlsContext({ tenantId }, async () => {
const rows = await deps.prisma.user.findMany({
where: {
pendingDeletion: true,
scheduledDeletionAt: { lte: now },
// Membership filter doubles as the RLS visibility branch:
// the bound tenant only ever sees its own members.
tenants: { some: { tenantId } },
},
select: { id: true, scheduledDeletionAt: true, deletionRequestId: true },
orderBy: { id: "asc" },
take: DSGVO_GRACE_SCAN_PAGE_SIZE,
});
return rows.map(
(row): DsgvoGraceCandidate => ({
userId: row.id,
tenantId,
// The where clause guarantees a non-null overdue timestamp.
scheduledDeletionAt: row.scheduledDeletionAt ?? now,
deletionRequestId: row.deletionRequestId,
})
);
}),
merge: (state, candidates) => {
for (const candidate of candidates) {
// A user in several workspaces is dispatched once; the cascade
// itself walks every membership tenant.
if (!state.has(candidate.userId)) state.set(candidate.userId, candidate);
}
return state;
},
});
throwOnActiveTenantSweepFailures("dsgvo-grace-period-scan", sweep.failures);
return [...sweep.state.values()];
},
async enqueueDeletion(input: DsgvoDeletionJobInput): Promise<{ jobId?: string }> {
// Membership list is readable under the user branch of the
// TenantUser select policy — no tenant context needed yet.
const memberships = await runWithRlsContext({ userId: input.userId }, () =>
deps.prisma.tenantUser.findMany({ where: { userId: input.userId }, select: { tenantId: true } })
);
const contexts: Array<{ userId: string; tenantId?: string }> = [
...memberships.map((membership) => ({ userId: input.userId, tenantId: membership.tenantId })),
{ userId: input.userId },
];
const skippedModels: string[] = [];
const runForEntry = async (
entry: UserFkMapEntry,
operation: (delegate: UpdateManyDelegate & DeleteManyDelegate) => Promise<{ count: number }>
): Promise<number> => {
if (DSGVO_WORKER_SKIPPED_MODELS.has(entry.model)) {
skippedModels.push(`${entry.model}.${entry.column}`);
logger.info(
{ userId: input.userId, model: entry.model, column: entry.column, strategy: entry.strategy },
"dsgvo-deletion.table-skipped-no-worker-grant"
);
return 0;
}
const delegate = deps.prisma[lowerFirst(entry.model)] as (UpdateManyDelegate & DeleteManyDelegate) | undefined;
if (!delegate) {
throw new Error(`DSGVO cascade is missing the prisma delegate for model ${entry.model}.`);
}
let affected = 0;
// Sequential on purpose: contexts are AsyncLocalStorage-scoped and
// the second pass is an idempotent no-op for rows the first pass
// already rewrote, so summed counts stay accurate.
for (const context of contexts) {
const result = await runWithRlsContext(context, () => operation(delegate));
affected += result.count;
}
return affected;
};
const result = await runDsgvoDeletion(
{
tableClient: {
hardDelete: (entry, userId) => runForEntry(entry, (delegate) => delegate.deleteMany({ where: { [entry.column]: userId } })),
pseudonymize: (entry, userId, pseudonymUserId) =>
runForEntry(entry, (delegate) => delegate.updateMany({ where: { [entry.column]: userId }, data: { [entry.column]: pseudonymUserId } })),
retainTombstone: (entry, userId) =>
runForEntry(entry, (delegate) => delegate.updateMany({ where: { [entry.column]: userId }, data: { [entry.column]: null } })),
},
userClient: {
hardDeleteUser: () => {
// The worker's RLS grant set cannot satisfy a hard delete
// (see file header); the cron only ever dispatches
// pseudonymize. Fail loudly instead of half-deleting.
throw new Error("DSGVO hard delete is not available under the worker RLS contract; use mode=pseudonymize.");
},
pseudonymizeUser: async (userId, update) => {
await runWithRlsContext({ userId }, () => deps.prisma.user.update({ where: { id: userId }, data: update }));
},
},
writeAuditEvent: async (auditInput) => {
// AuditEvent/AuditChainHead policies carry an explicit
// hivegrid_worker branch, so the chained write needs no bound
// context here.
await createAuditEvent({
...auditInput,
metadata: { ...auditInput.metadata, skippedModels } as Parameters<typeof createAuditEvent>[0]["metadata"],
});
},
logger,
},
input
);
logger.info(
{
userId: input.userId,
requestId: input.requestId,
tenants: memberships.length,
tables: result.tables.length,
skippedModels,
},
"dsgvo-deletion.cascade-complete"
);
// Inline execution — the "job id" is synthetic and only used for logs.
return { jobId: `inline-${input.requestId}` };
},
};
}
/** Ensure the full FK map stays classifiable — new admin-surface models must be triaged. */
export function listWorkerProcessableFkEntries(): UserFkMapEntry[] {
return USER_FK_MAP.filter((entry) => !DSGVO_WORKER_SKIPPED_MODELS.has(entry.model));
}
function lowerFirst(value: string): string {
return value.charAt(0).toLowerCase() + value.slice(1);
}
@@ -32,7 +32,7 @@ describe("dsgvo-grace-cron", () => {
expect(result.failed).toBe(0);
expect(enqueue).toHaveBeenCalledTimes(2);
expect(enqueue.mock.calls[0][0].requestId).toBe("req-1");
expect(enqueue.mock.calls[0][0].mode).toBe("delete");
expect(enqueue.mock.calls[0][0].mode).toBe("pseudonymize");
expect(enqueue.mock.calls[0][0].actorType).toBe("system");
expect(enqueue.mock.calls[0][0].reason).toBe("dsgvo-grace-period-elapsed");
// u2 had no deletionRequestId — synthesize a grace-derived one.
@@ -49,7 +49,7 @@ describe("dsgvo-grace-cron", () => {
const enqueue = vi
.fn()
.mockResolvedValueOnce({ jobId: "j1" })
.mockRejectedValueOnce(new Error("redis down"))
.mockRejectedValueOnce(new Error("redis down for private@example.test with client_secret=secret-value"))
.mockResolvedValueOnce({ jobId: "j3" });
const logger = makeLogger();
const result = await runDsgvoGraceCron({
@@ -60,11 +60,16 @@ describe("dsgvo-grace-cron", () => {
expect(result.scanned).toBe(3);
expect(result.dispatched).toBe(2);
expect(result.failed).toBe(1);
expect(result.failures[0]).toEqual({ userId: "bad", error: "redis down" });
expect(result.failures[0]).toEqual({
userId: "bad",
error: "redis down for [redacted-email] with client_secret=[redacted]",
});
expect(logger.error).toHaveBeenCalledWith(
expect.objectContaining({ userId: "bad", err: "redis down" }),
expect.objectContaining({ userId: "bad", err: "redis down for [redacted-email] with client_secret=[redacted]" }),
"dsgvo-grace-cron.dispatch-failed",
);
expect(JSON.stringify(logger.error.mock.calls)).not.toContain("secret-value");
expect(JSON.stringify(logger.error.mock.calls)).not.toContain("private@example.test");
});
it("returns zero when no candidates", async () => {
@@ -50,7 +50,12 @@ export type DsgvoGraceCronDependencies = {
requestId: string;
actorType: "system";
actorId: string;
mode: "delete";
// R-P2b design: DSGVO deletion is PII pseudonymization, not a hard
// delete — the User row survives so the audit hash chain and
// retention-bound foreign keys stay intact. Hard delete additionally
// requires worker write grants the RLS contract deliberately withholds
// (AdminUser/UserRole/support tables), so the cron never dispatches it.
mode: "pseudonymize";
reason: string;
}) => Promise<{ jobId?: string }>;
/** Optional. Defaults to () => new Date(). */
@@ -96,7 +101,7 @@ export async function runDsgvoGraceCron(deps: DsgvoGraceCronDependencies): Promi
requestId,
actorType: "system",
actorId: systemActorId,
mode: "delete",
mode: "pseudonymize",
reason: "dsgvo-grace-period-elapsed",
});
dispatched += 1;
@@ -105,7 +110,7 @@ export async function runDsgvoGraceCron(deps: DsgvoGraceCronDependencies): Promi
"dsgvo-grace-cron.dispatched",
);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
const message = redactDsgvoGraceDiagnostic(err instanceof Error ? err.message : String(err));
failures.push({ userId: candidate.userId, error: message });
deps.logger.error({ userId: candidate.userId, err: message }, "dsgvo-grace-cron.dispatch-failed");
}
@@ -119,6 +124,18 @@ export async function runDsgvoGraceCron(deps: DsgvoGraceCronDependencies): Promi
};
}
function redactDsgvoGraceDiagnostic(message: string): string {
return message
.replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]")
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
.replace(
/(authorization|access[_-]?key(?:id)?|api[-_]?key|client[-_]?secret|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi,
"$1$2[redacted]",
)
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]")
.slice(0, 1_000);
}
/**
* Helper for the worker bootstrap. Given a BullMQ-like queue with the
* `upsertJobScheduler` API, install (or replace) the 24h repeat schedule.
@@ -91,7 +91,7 @@ describe("email sequence runtime deps", () => {
backoff: { type: "exponential", delay: 1_000 },
delay: 600_000,
jobId: "email-sequence-step:run_1:2",
removeOnComplete: false,
removeOnComplete: 10_000,
removeOnFail: 1_000,
}
);
@@ -78,6 +78,7 @@ export type EmailSequenceStepQueueOptions = {
const EMAIL_SEQUENCE_STEP_ATTEMPTS = 3;
const EMAIL_SEQUENCE_STEP_BACKOFF_MS = 1_000;
const EMAIL_SEQUENCE_STEP_REMOVE_ON_COMPLETE = 10_000;
const EMAIL_SEQUENCE_STEP_REMOVE_ON_FAIL = 1_000;
/**
@@ -266,11 +267,11 @@ function renderSequenceTemplate(
return { ...email, subject };
}
case "trial-reminder": {
const upgradeUrl = stringVariable(vars, trigger, "upgradeUrl", `${appUrl}/settings/subscription`);
const upgradeUrl = stringVariable(vars, trigger, "upgradeUrl", `${appUrl}/settings/billing`);
const email = renderEmail(TrialReminderEmail({
productName,
daysRemaining: numberVariable(vars, trigger, "daysRemaining", 3),
upgradeUrl: upgradeUrl ?? `${appUrl}/settings/subscription`,
upgradeUrl: upgradeUrl ?? `${appUrl}/settings/billing`,
unsubscribeUrl,
...(recipientName ? { recipientName } : {}),
}));
@@ -405,7 +406,7 @@ async function scheduleEmailSequenceStep(
backoff: { type: "exponential", delay: EMAIL_SEQUENCE_STEP_BACKOFF_MS },
delay,
jobId,
removeOnComplete: false,
removeOnComplete: EMAIL_SEQUENCE_STEP_REMOVE_ON_COMPLETE,
removeOnFail: EMAIL_SEQUENCE_STEP_REMOVE_ON_FAIL,
});
opts.logger.info?.(
@@ -0,0 +1,28 @@
import { describe, expect, test } from "vitest";
import {
assertPreviewOutputSize,
assertPreviewRasterMetadata,
assertPreviewSourceSize,
FILE_PREVIEW_MAX_DIMENSION,
FILE_PREVIEW_MAX_PAGES,
FILE_PREVIEW_MAX_PIXELS,
FILE_PREVIEW_MAX_SOURCE_BYTES,
} from "./file-preview-decoder-contract.js";
describe("file preview decoder limits", () => {
test("accepts a normal single-page image", () => {
expect(() => assertPreviewRasterMetadata({ width: 4_000, height: 3_000, pages: 1 })).not.toThrow();
});
test("rejects decompression-bomb dimensions, pixels and page counts", () => {
expect(() => assertPreviewRasterMetadata({ width: FILE_PREVIEW_MAX_DIMENSION + 1, height: 1 })).toThrow("dimension limit");
expect(() => assertPreviewRasterMetadata({ width: 10_000, height: Math.floor(FILE_PREVIEW_MAX_PIXELS / 10_000) + 1 })).toThrow("pixel limit");
expect(() => assertPreviewRasterMetadata({ width: 1, height: 1, pages: FILE_PREVIEW_MAX_PAGES + 1 })).toThrow("page limit");
});
test("rejects oversized source and generated output buffers", () => {
expect(() => assertPreviewSourceSize(FILE_PREVIEW_MAX_SOURCE_BYTES + 1)).toThrow("byte limit");
expect(() => assertPreviewOutputSize(11, 10, "fixture")).toThrow("output limit");
});
});
@@ -0,0 +1,57 @@
export const FILE_PREVIEW_MAX_SOURCE_BYTES = 100 * 1024 * 1024;
export const FILE_PREVIEW_MAX_PIXELS = 40_000_000;
export const FILE_PREVIEW_MAX_DIMENSION = 16_384;
export const FILE_PREVIEW_MAX_PAGES = 200;
export const FILE_PREVIEW_MAX_INTERMEDIATE_BYTES = 64 * 1024 * 1024;
export const FILE_PREVIEW_MAX_OUTPUT_BYTES = 20 * 1024 * 1024;
export const FILE_PREVIEW_MAX_THUMBNAIL_BYTES = 3 * 1024 * 1024;
export const FILE_PREVIEW_DECODE_TIMEOUT_MS = 20_000;
export const FILE_PREVIEW_DECODER_HEAP_MB = 256;
export const FILE_PREVIEW_MAX_CONCURRENT_DECODERS = 2;
export type FilePreviewDecoderMime = "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/heic" | "image/heif" | "application/pdf";
export type FilePreviewDecoderRequest = {
input: Uint8Array;
mime: FilePreviewDecoderMime;
};
export type FilePreviewDecoderSuccess = {
ok: true;
preview: Uint8Array;
thumbnail: Uint8Array;
previewExtension: "jpg" | "png" | "webp";
previewContentType: "image/jpeg" | "image/png" | "image/webp";
};
export type FilePreviewDecoderResponse = FilePreviewDecoderSuccess | { ok: false; error: string };
export function assertPreviewSourceSize(byteLength: number): void {
if (!Number.isSafeInteger(byteLength) || byteLength < 0 || byteLength > FILE_PREVIEW_MAX_SOURCE_BYTES) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_SOURCE_BYTES}-byte limit`);
}
}
export function assertPreviewRasterMetadata(metadata: { width?: number; height?: number; pages?: number }): void {
const width = metadata.width;
const height = metadata.height;
const pages = metadata.pages ?? 1;
if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width! <= 0 || height! <= 0) {
throw new Error("Preview source has invalid image dimensions");
}
if (width! > FILE_PREVIEW_MAX_DIMENSION || height! > FILE_PREVIEW_MAX_DIMENSION) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_DIMENSION}-pixel dimension limit`);
}
if (width! * height! > FILE_PREVIEW_MAX_PIXELS) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_PIXELS}-pixel limit`);
}
if (!Number.isSafeInteger(pages) || pages <= 0 || pages > FILE_PREVIEW_MAX_PAGES) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_PAGES}-page limit`);
}
}
export function assertPreviewOutputSize(byteLength: number, maxBytes: number, label: string): void {
if (!Number.isSafeInteger(byteLength) || byteLength <= 0 || byteLength > maxBytes) {
throw new Error(`${label} exceeds the ${maxBytes}-byte output limit`);
}
}
@@ -0,0 +1,107 @@
import { EventEmitter } from "node:events";
import type { ChildProcess, fork } from "node:child_process";
import { describe, expect, test, vi } from "vitest";
import { decodeFilePreviewIsolated } from "./file-preview-decoder-isolated.js";
function fakeChild() {
const child = Object.assign(new EventEmitter(), {
connected: true,
exitCode: null,
signalCode: null,
disconnect: vi.fn(),
kill: vi.fn(() => true),
send: vi.fn(),
stderr: null,
});
child.disconnect.mockImplementation(() => {
child.connected = false;
});
return child as unknown as ChildProcess;
}
describe("decodeFilePreviewIsolated", () => {
const response = {
ok: true,
preview: Uint8Array.from([1, 2]),
thumbnail: Uint8Array.from([3]),
previewExtension: "webp",
previewContentType: "image/webp",
} as const;
test("kills a decoder that exceeds its wall-clock budget", async () => {
const child = fakeChild();
const forkImpl = vi.fn(() => child) as unknown as typeof fork;
await expect(
decodeFilePreviewIsolated(Buffer.from("input"), "image/png", {
forkImpl,
processPath: "/fixture/decoder.js",
timeoutMs: 5,
})
).rejects.toThrow("5ms wall-clock limit");
expect(child.kill).toHaveBeenCalledWith("SIGKILL");
});
test("accepts bounded binary output and lets the disposable process exit cleanly", async () => {
const child = fakeChild();
const send = vi.fn(() => {
queueMicrotask(() => child.emit("message", response));
return true;
});
Object.defineProperty(child, "send", { configurable: true, value: send });
const forkImpl = vi.fn(() => child) as unknown as typeof fork;
await expect(decodeFilePreviewIsolated(Buffer.from("input"), "image/png", { forkImpl, processPath: "/fixture/decoder.js" })).resolves.toMatchObject({
previewExtension: "webp",
previewContentType: "image/webp",
});
expect(child.disconnect).toHaveBeenCalledOnce();
expect(child.kill).not.toHaveBeenCalled();
});
test("sends the exact input slice without copying its backing buffer", async () => {
const child = fakeChild();
let sentMessage: unknown;
const send = vi.fn((message: unknown) => {
sentMessage = message;
queueMicrotask(() => child.emit("message", response));
return true;
});
Object.defineProperty(child, "send", { configurable: true, value: send });
const forkImpl = vi.fn(() => child) as unknown as typeof fork;
const backing = Buffer.from([99, 98, 1, 2, 3, 97]);
const input = backing.subarray(2, 5);
await decodeFilePreviewIsolated(input, "image/png", { forkImpl, processPath: "/fixture/decoder.js" });
const message = sentMessage as { input: Uint8Array; mime: string };
expect(message.input.buffer).toBe(input.buffer);
expect(message.input.byteOffset).toBe(input.byteOffset);
expect(message.input.byteLength).toBe(input.byteLength);
expect(Array.from(message.input)).toEqual([1, 2, 3]);
expect(message.mime).toBe("image/png");
});
test("caps aggregate decoder CPU and memory pressure at two child processes", async () => {
const children = [fakeChild(), fakeChild(), fakeChild()];
let forkIndex = 0;
const forkMock = vi.fn(() => children[forkIndex++]!);
const forkImpl = forkMock as unknown as typeof fork;
const options = { forkImpl, processPath: "/fixture/decoder.js", timeoutMs: 1_000 };
const pending = [
decodeFilePreviewIsolated(Buffer.from("one"), "image/png", options),
decodeFilePreviewIsolated(Buffer.from("two"), "image/png", options),
decodeFilePreviewIsolated(Buffer.from("three"), "image/png", options),
];
await vi.waitFor(() => expect(forkMock).toHaveBeenCalledTimes(2));
children[0]!.emit("message", response);
await vi.waitFor(() => expect(forkMock).toHaveBeenCalledTimes(3));
children[1]!.emit("message", response);
children[2]!.emit("message", response);
await expect(Promise.all(pending)).resolves.toHaveLength(3);
});
});
@@ -0,0 +1,163 @@
import { fork } from "node:child_process";
import { existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
assertPreviewOutputSize,
assertPreviewSourceSize,
FILE_PREVIEW_DECODER_HEAP_MB,
FILE_PREVIEW_DECODE_TIMEOUT_MS,
FILE_PREVIEW_MAX_CONCURRENT_DECODERS,
FILE_PREVIEW_MAX_OUTPUT_BYTES,
FILE_PREVIEW_MAX_THUMBNAIL_BYTES,
type FilePreviewDecoderMime,
type FilePreviewDecoderResponse,
type FilePreviewDecoderSuccess,
} from "./file-preview-decoder-contract.js";
type ForkImplementation = typeof fork;
let activeDecoders = 0;
const decoderWaiters: Array<() => void> = [];
function childExecArgv(args: readonly string[]): string[] {
const retained: string[] = [];
for (let index = 0; index < args.length; index += 1) {
const value = args[index]!;
if (["--eval", "-e", "--print", "-p"].includes(value)) {
index += 1;
continue;
}
if (value === "--input-type") {
index += 1;
continue;
}
if (value.startsWith("--input-type=") || value.startsWith("--max-old-space-size=")) {
continue;
}
retained.push(value);
}
return retained.concat(`--max-old-space-size=${FILE_PREVIEW_DECODER_HEAP_MB}`);
}
async function acquireDecoderSlot(): Promise<() => void> {
if (activeDecoders < FILE_PREVIEW_MAX_CONCURRENT_DECODERS) {
activeDecoders += 1;
} else {
await new Promise<void>((resolve) => decoderWaiters.push(resolve));
}
let released = false;
return () => {
if (released) return;
released = true;
const next = decoderWaiters.shift();
if (next) next();
else activeDecoders -= 1;
};
}
export type IsolatedDecoderOptions = {
timeoutMs?: number;
processPath?: string;
forkImpl?: ForkImplementation;
isProduction?: boolean;
};
function compiledProcessCandidates(): string[] {
const currentDir = dirname(fileURLToPath(import.meta.url));
return [
join(currentDir, "file-preview-decoder-process.js"),
join(currentDir, "jobs", "file-preview-decoder-process.js"),
join(dirname(dirname(currentDir)), "dist", "jobs", "file-preview-decoder-process.js"),
];
}
export function resolveFilePreviewDecoderProcess(options: Pick<IsolatedDecoderOptions, "processPath" | "isProduction"> = {}): string {
if (options.processPath) return options.processPath;
for (const candidate of compiledProcessCandidates()) {
if (existsSync(candidate)) return candidate;
}
if (options.isProduction ?? process.env.NODE_ENV === "production") {
throw new Error("Compiled file preview decoder process is required in production");
}
return join(dirname(fileURLToPath(import.meta.url)), "file-preview-decoder-process.ts");
}
function validateResponse(message: unknown): FilePreviewDecoderSuccess {
if (!message || typeof message !== "object" || !("ok" in message)) {
throw new Error("Preview decoder returned an invalid response");
}
const response = message as FilePreviewDecoderResponse;
if (!response.ok) {
throw new Error(`Preview decoder rejected the source: ${response.error}`);
}
if (!(response.preview instanceof Uint8Array) || !(response.thumbnail instanceof Uint8Array)) {
throw new Error("Preview decoder returned invalid binary output");
}
assertPreviewOutputSize(response.preview.byteLength, FILE_PREVIEW_MAX_OUTPUT_BYTES, "Preview image");
assertPreviewOutputSize(response.thumbnail.byteLength, FILE_PREVIEW_MAX_THUMBNAIL_BYTES, "Preview thumbnail");
return response;
}
export async function decodeFilePreviewIsolated(
input: Buffer,
mime: FilePreviewDecoderMime,
options: IsolatedDecoderOptions = {}
): Promise<FilePreviewDecoderSuccess> {
assertPreviewSourceSize(input.byteLength);
const releaseDecoderSlot = await acquireDecoderSlot();
try {
return await decodeWithChildProcess(input, mime, options);
} finally {
releaseDecoderSlot();
}
}
async function decodeWithChildProcess(input: Buffer, mime: FilePreviewDecoderMime, options: IsolatedDecoderOptions): Promise<FilePreviewDecoderSuccess> {
const timeoutMs = Math.max(1, Math.min(Math.floor(options.timeoutMs ?? FILE_PREVIEW_DECODE_TIMEOUT_MS), 60_000));
const processPath = resolveFilePreviewDecoderProcess(options);
const forkImpl = options.forkImpl ?? fork;
const execArgv = childExecArgv(process.execArgv);
const child = forkImpl(processPath, [], {
execArgv,
serialization: "advanced",
stdio: ["ignore", "ignore", "pipe", "ipc"],
});
// Native libraries can emit diagnostics. Drain them so their pipe cannot
// backpressure the decoder; the parent logs only the bounded error message
// returned over IPC, never attacker-controlled binary input.
child.stderr?.resume();
return await new Promise<FilePreviewDecoderSuccess>((resolve, reject) => {
let settled = false;
const finish = (result: { ok: true; value: FilePreviewDecoderSuccess } | { ok: false; error: Error }, forceKill = false) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
child.removeAllListeners();
if (child.connected) child.disconnect();
if (forceKill && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
if (result.ok) resolve(result.value);
else reject(result.error);
};
const timeout = setTimeout(() => {
finish({ ok: false, error: new Error(`Preview decoder exceeded the ${timeoutMs}ms wall-clock limit`) }, true);
}, timeoutMs);
child.once("message", (message) => {
try {
finish({ ok: true, value: validateResponse(message) });
} catch (error) {
finish({ ok: false, error: error instanceof Error ? error : new Error(String(error)) }, true);
}
});
child.once("error", (error) => finish({ ok: false, error }, true));
child.once("exit", (code, signal) => {
finish({ ok: false, error: new Error(`Preview decoder exited before returning output (code=${code ?? "none"}, signal=${signal ?? "none"})`) });
});
const inputView = new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
child.send({ input: inputView, mime }, (error) => error && finish({ ok: false, error }, true));
});
}
@@ -0,0 +1,19 @@
import { describe, expect, test } from "vitest";
import sharp from "sharp";
import { decodeFilePreview } from "./file-preview-decoder-process.js";
describe("decodeFilePreview", () => {
test("renders a bounded raster preview and thumbnail", async () => {
const input = await sharp({
create: { width: 64, height: 48, channels: 3, background: "#f1b729" },
})
.png()
.toBuffer();
const result = await decodeFilePreview({ input, mime: "image/png" });
expect(result).toMatchObject({ ok: true, previewExtension: "webp", previewContentType: "image/webp" });
await expect(sharp(result.preview).metadata()).resolves.toMatchObject({ width: 64, height: 48, format: "webp" });
await expect(sharp(result.thumbnail).metadata()).resolves.toMatchObject({ width: 64, height: 48, format: "webp" });
});
});
@@ -0,0 +1,181 @@
import process from "node:process";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { pdfToPng } from "pdf-to-png-converter";
import sharp from "sharp";
import {
assertPreviewOutputSize,
assertPreviewRasterMetadata,
assertPreviewSourceSize,
FILE_PREVIEW_MAX_INTERMEDIATE_BYTES,
FILE_PREVIEW_MAX_OUTPUT_BYTES,
FILE_PREVIEW_MAX_PAGES,
FILE_PREVIEW_MAX_PIXELS,
FILE_PREVIEW_MAX_SOURCE_BYTES,
FILE_PREVIEW_MAX_THUMBNAIL_BYTES,
type FilePreviewDecoderRequest,
type FilePreviewDecoderResponse,
type FilePreviewDecoderSuccess,
} from "./file-preview-decoder-contract.js";
const PREVIEW_MAX_DIMENSION = 2_048;
const THUMBNAIL_MAX_DIMENSION = 480;
const SHARP_TIMEOUT_SECONDS = 15;
// This module runs in a disposable child process. Keeping libvips to one
// thread and disabling its cache makes peak native memory predictable; the
// parent enforces a separate hard wall-clock timeout and kills this process.
sharp.cache(false);
sharp.concurrency(1);
function asExactUint8Array(buffer: Buffer): Uint8Array {
return Uint8Array.from(buffer);
}
async function renderRaster(input: Buffer, format: "jpeg" | "png" | "webp", maxDimension: number): Promise<Buffer> {
const pipeline = sharp(input, {
failOn: "warning",
limitInputPixels: FILE_PREVIEW_MAX_PIXELS,
page: 0,
pages: 1,
sequentialRead: true,
});
const metadata = await pipeline.metadata();
assertPreviewRasterMetadata(metadata);
const resized = pipeline
.timeout({ seconds: SHARP_TIMEOUT_SECONDS })
.rotate()
.resize({ width: maxDimension, height: maxDimension, fit: "inside", withoutEnlargement: true });
if (format === "jpeg") return resized.jpeg({ quality: 82, mozjpeg: true }).toBuffer();
if (format === "png") return resized.png({ compressionLevel: 9 }).toBuffer();
return resized.webp({ quality: 80 }).toBuffer();
}
async function inspectRaster(input: Buffer): Promise<void> {
const metadata = await sharp(input, {
failOn: "warning",
limitInputPixels: FILE_PREVIEW_MAX_PIXELS,
page: 0,
pages: 1,
sequentialRead: true,
})
.timeout({ seconds: SHARP_TIMEOUT_SECONDS })
.metadata();
assertPreviewRasterMetadata(metadata);
}
async function convertHeicToJpeg(input: Buffer): Promise<Buffer> {
const moduleName = "heic-convert";
const loaded = (await import(/* @vite-ignore */ moduleName)) as
| { default: (options: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer> }
| ((options: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer>);
const heicConvert = typeof loaded === "function" ? loaded : loaded.default;
const converted = await heicConvert({ buffer: input, format: "JPEG", quality: 0.85 });
const output = Buffer.isBuffer(converted) ? Buffer.from(converted) : Buffer.from(new Uint8Array(converted));
assertPreviewOutputSize(output.byteLength, FILE_PREVIEW_MAX_INTERMEDIATE_BYTES, "HEIC intermediate image");
return output;
}
async function renderPdfFirstPage(input: Buffer): Promise<Buffer> {
const safeOptions = {
concurrencyLimit: 1,
disableFontFace: true,
enableXfa: false,
maxInputBytes: FILE_PREVIEW_MAX_SOURCE_BYTES,
processPagesInParallel: false,
useSystemFonts: false,
verbosityLevel: 0 as const,
viewportScale: 2,
};
// Inspect page 1 and the first forbidden page without creating a canvas.
// Presence of page MAX+1 proves the document exceeds the page limit.
const metadata = await pdfToPng(input, {
...safeOptions,
pagesToProcess: [1, FILE_PREVIEW_MAX_PAGES + 1],
returnMetadataOnly: true,
});
const firstPage = metadata.find((page) => page.pageNumber === 1);
if (!firstPage) {
throw new Error("PDF rendered no pages");
}
if (metadata.some((page) => page.pageNumber === FILE_PREVIEW_MAX_PAGES + 1)) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_PAGES}-page limit`);
}
assertPreviewRasterMetadata({ width: firstPage.width, height: firstPage.height, pages: 1 });
const pages = await pdfToPng(input, {
...safeOptions,
pagesToProcess: [1],
returnPageContent: true,
});
const content = pages[0]?.content;
if (!content) {
throw new Error("PDF rendered no page content");
}
const output = Buffer.from(content);
assertPreviewOutputSize(output.byteLength, FILE_PREVIEW_MAX_INTERMEDIATE_BYTES, "PDF intermediate image");
return output;
}
export async function decodeFilePreview(request: FilePreviewDecoderRequest): Promise<FilePreviewDecoderSuccess> {
assertPreviewSourceSize(request.input.byteLength);
const input = Buffer.from(request.input.buffer, request.input.byteOffset, request.input.byteLength);
let preview: Buffer;
let previewExtension: FilePreviewDecoderSuccess["previewExtension"];
let previewContentType: FilePreviewDecoderSuccess["previewContentType"];
if (request.mime === "image/heic" || request.mime === "image/heif") {
// Parse HEIF container dimensions through libvips before heic-convert can
// allocate the decoded bitmap. Failure is fail-closed even if the fallback
// decoder might otherwise accept the file.
await inspectRaster(input);
const jpeg = await convertHeicToJpeg(input);
preview = await renderRaster(jpeg, "jpeg", PREVIEW_MAX_DIMENSION);
previewExtension = "jpg";
previewContentType = "image/jpeg";
} else if (request.mime === "application/pdf") {
const png = await renderPdfFirstPage(input);
preview = await renderRaster(png, "png", PREVIEW_MAX_DIMENSION);
previewExtension = "png";
previewContentType = "image/png";
} else {
preview = await renderRaster(input, "webp", PREVIEW_MAX_DIMENSION);
previewExtension = "webp";
previewContentType = "image/webp";
}
assertPreviewOutputSize(preview.byteLength, FILE_PREVIEW_MAX_OUTPUT_BYTES, "Preview image");
const thumbnail = await renderRaster(preview, "webp", THUMBNAIL_MAX_DIMENSION);
assertPreviewOutputSize(thumbnail.byteLength, FILE_PREVIEW_MAX_THUMBNAIL_BYTES, "Preview thumbnail");
return {
ok: true,
preview: asExactUint8Array(preview),
thumbnail: asExactUint8Array(thumbnail),
previewExtension,
previewContentType,
};
}
function errorMessage(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message.slice(0, 1_024);
}
const isMainProcess = process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (isMainProcess && typeof process.send === "function") {
process.once("message", (message: FilePreviewDecoderRequest) => {
void decodeFilePreview(message)
.then((result) => {
process.send?.(result, () => process.exit(0));
})
.catch((error: unknown) => {
const result: FilePreviewDecoderResponse = { ok: false, error: errorMessage(error) };
process.send?.(result, () => process.exit(1));
});
});
}
@@ -0,0 +1,62 @@
import { Readable } from "node:stream";
import { describe, expect, test, vi } from "vitest";
import { assertPreviewSource, readPreviewBodyWithinLimit } from "./file-preview.js";
describe("readPreviewBodyWithinLimit", () => {
test("reads a streamed object within the configured limit", async () => {
const body = Readable.from([Buffer.from("hello "), Buffer.from("world")]);
await expect(readPreviewBodyWithinLimit(body, 11, 11)).resolves.toEqual(Buffer.from("hello world"));
});
test("rejects oversized metadata before consuming the body", async () => {
const destroy = vi.fn();
const body = { destroy, transformToByteArray: vi.fn() };
await expect(readPreviewBodyWithinLimit(body, 12, 11)).rejects.toThrow("exceeds the 11-byte limit");
expect(destroy).toHaveBeenCalledOnce();
expect(body.transformToByteArray).not.toHaveBeenCalled();
});
test("rejects a stream that exceeds the limit when metadata is missing", async () => {
const body = Readable.from([Buffer.alloc(6), Buffer.alloc(6)]);
await expect(readPreviewBodyWithinLimit(body, undefined, 11)).rejects.toThrow("exceeds the 11-byte limit");
});
test("checks fallback byte-array bodies as well", async () => {
const body = { transformToByteArray: vi.fn(async () => new Uint8Array(12)) };
await expect(readPreviewBodyWithinLimit(body, undefined, 11)).rejects.toThrow("exceeds the 11-byte limit");
});
test("destroys and rejects a stalled source stream at the wall-clock limit", async () => {
const destroy = vi.fn();
const body = { destroy, transformToByteArray: vi.fn(() => new Promise<Uint8Array>(() => {})) };
await expect(readPreviewBodyWithinLimit(body, undefined, 11, 5)).rejects.toThrow("5ms limit");
expect(destroy).toHaveBeenCalledOnce();
});
});
describe("assertPreviewSource", () => {
const payload = {
kind: "preview.file",
fileId: "file_1",
tenantId: "tenant_1",
bucket: "uploads",
storageKey: "tenants/tenant_1/files/file_1/original",
mimeType: "image/png",
} as const;
test("accepts the configured bucket and canonical tenant file key", () => {
expect(() => assertPreviewSource(payload, "uploads")).not.toThrow();
});
test("rejects cross-bucket and cross-tenant source targets", () => {
expect(() => assertPreviewSource({ ...payload, bucket: "backups" }, "uploads")).toThrow("bucket");
expect(() => assertPreviewSource({ ...payload, storageKey: "tenants/tenant_2/files/file_1/original" }, "uploads")).toThrow("canonical tenant file path");
});
});
@@ -0,0 +1,109 @@
import { beforeEach, describe, expect, test, vi } from "vitest";
const aws = vi.hoisted(() => ({ send: vi.fn() }));
vi.mock("@aws-sdk/client-s3", () => ({
GetObjectCommand: class GetObjectCommand {
constructor(readonly input: unknown) {}
},
PutObjectCommand: class PutObjectCommand {
constructor(readonly input: unknown) {}
},
S3Client: class S3Client {
send = aws.send;
},
}));
import { createWorkerFilePreviewService, type FilePreviewPrisma } from "./file-preview.js";
const payload = {
kind: "preview.file",
fileId: "file_1",
tenantId: "tenant_1",
bucket: "uploads",
storageKey: "tenants/tenant_1/files/file_1/original",
mimeType: "image/jpeg",
} as const;
function buildPrisma(scanStatus: string, overrides: Record<string, unknown> = {}) {
const findFirst = vi.fn(async () => ({
id: payload.fileId,
mimeType: "image/png",
previewStatus: "pending",
previewStorageKey: null,
scanStatus,
sizeBytes: 4,
storageKey: payload.storageKey,
thumbnailStorageKey: null,
...overrides,
}));
const update = vi.fn(async () => ({}));
const tx = { $executeRaw: vi.fn(async () => 1), file: { findFirst, update } };
const prisma = {
$transaction: vi.fn(async (callback: (transaction: typeof tx) => Promise<unknown>) => callback(tx)),
} as unknown as FilePreviewPrisma;
return { findFirst, prisma, update };
}
const env = {
STORAGE_ACCESS_KEY_ID: "access",
STORAGE_BUCKET: "uploads",
STORAGE_REGION: "eu-central-1",
STORAGE_SECRET_ACCESS_KEY: "secret",
};
describe("file preview malware-scan gate", () => {
beforeEach(() => aws.send.mockReset());
test.each(["pending", "infected", "skipped"])("never loads or decodes a %s file", async (scanStatus) => {
const { prisma, update } = buildPrisma(scanStatus);
const decodePreview = vi.fn();
const service = createWorkerFilePreviewService(env, {
prisma,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
decodePreview,
})!;
await expect(service.generatePreview(payload)).rejects.toThrow("malware scan is clean");
expect(aws.send).not.toHaveBeenCalled();
expect(decodePreview).not.toHaveBeenCalled();
expect(update).not.toHaveBeenCalled();
});
test("uses the persisted MIME and decodes only after a clean verdict", async () => {
const { prisma, update } = buildPrisma("clean");
aws.send.mockResolvedValueOnce({ Body: { transformToByteArray: async () => Uint8Array.from([1, 2, 3, 4]) }, ContentLength: 4 }).mockResolvedValue({});
const decodePreview = vi.fn(async () => ({
ok: true as const,
preview: Uint8Array.from([5, 6]),
thumbnail: Uint8Array.from([7]),
previewExtension: "webp" as const,
previewContentType: "image/webp" as const,
}));
const service = createWorkerFilePreviewService(env, {
prisma,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
decodePreview,
})!;
await expect(service.generatePreview(payload)).resolves.toMatchObject({ previewStatus: "ready" });
expect(decodePreview).toHaveBeenCalledWith(Buffer.from([1, 2, 3, 4]), "image/png");
expect(update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ previewStatus: "generating" }) }));
expect(update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ previewStatus: "ready" }) }));
});
test("marks a decoder failure and rethrows so BullMQ can retry", async () => {
const { prisma, update } = buildPrisma("clean");
aws.send.mockResolvedValueOnce({ Body: { transformToByteArray: async () => Uint8Array.from([1, 2, 3, 4]) }, ContentLength: 4 });
const service = createWorkerFilePreviewService(env, {
prisma,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
decodePreview: vi.fn(async () => {
throw new Error("decoder-timeout");
}),
})!;
await expect(service.generatePreview(payload)).rejects.toThrow("decoder-timeout");
expect(update).toHaveBeenCalledWith(expect.objectContaining({ data: { previewStatus: "failed" } }));
});
});
@@ -12,24 +12,25 @@
// text/plain | markdown -> ready (inline render from bytes, no file)
// anything else -> not_supported
//
// The native render libs (sharp, heic-convert, pdf-to-png-converter) are loaded
// via dynamic string import (same pattern as qdrant/sentry in index.ts) so
// typecheck/build stay green before they are installed.
// ACTIVATION: `pnpm add sharp heic-convert pdf-to-png-converter -F @hivegrid/worker`.
// If a lib is missing at runtime the service falls back cleanly to
// previewStatus=failed (no hard crash) and the user can regenerate from detail.
// Native rendering runs in a disposable child process. The child validates
// dimensions/pages/pixels and output sizes; the parent kills it at a hard
// wall-clock limit. A failure is rethrown so BullMQ retries before leaving the
// File row at previewStatus=failed for a user-triggered regeneration.
import { GetObjectCommand, PutObjectCommand, S3Client as AwsS3Client } from "@aws-sdk/client-s3";
import type { EnvRecord } from "@hivegrid/config";
import type { FilePreviewJob } from "@hivegrid/queue";
import { buildFilePreviewKey, buildFileStorageKeys, type FilePreviewExtension } from "@hivegrid/storage";
import { buildFilePreviewKey, buildFileStorageKeys } from "@hivegrid/storage";
import { withTenantRLS } from "../db/tenantRls.js";
import type { FilePreviewServiceResult } from "../handlers.js";
import { FILE_PREVIEW_MAX_SOURCE_BYTES, type FilePreviewDecoderMime, type FilePreviewDecoderSuccess } from "./file-preview-decoder-contract.js";
import { decodeFilePreviewIsolated } from "./file-preview-decoder-isolated.js";
export interface FilePreviewServiceEnv {
STORAGE_ENDPOINT?: string;
STORAGE_REGION?: string;
STORAGE_BUCKET?: string;
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
@@ -51,6 +52,28 @@ type PreviewLogger = {
interface FilePreviewTx {
$executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
file: {
findFirst(args: {
where: { id: string; tenantId: string };
select: {
id: true;
mimeType: true;
previewStatus: true;
previewStorageKey: true;
scanStatus: true;
sizeBytes: true;
storageKey: true;
thumbnailStorageKey: true;
};
}): Promise<{
id: string;
mimeType: string;
previewStatus: string;
previewStorageKey: string | null;
scanStatus: string;
sizeBytes: number;
storageKey: string;
thumbnailStorageKey: string | null;
} | null>;
update(args: {
where: { id: string };
data: {
@@ -71,8 +94,10 @@ export interface FilePreviewService {
generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult>;
}
const THUMBNAIL_MAX_DIMENSION = 480;
const PREVIEW_MAX_DIMENSION = 2048;
export { FILE_PREVIEW_MAX_SOURCE_BYTES } from "./file-preview-decoder-contract.js";
const FILE_PREVIEW_STORAGE_READ_TIMEOUT_MS = 15_000;
const FILE_PREVIEW_STORAGE_REQUEST_TIMEOUT_MS = 15_000;
const RASTER_IMAGE_TYPES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
const HEIC_TYPES = new Set(["image/heic", "image/heif"]);
@@ -82,6 +107,76 @@ function normalizeMime(mimeType: string): string {
return mimeType.split(";")[0]?.trim().toLowerCase() ?? "";
}
type PreviewObjectBody = {
transformToByteArray?: () => Promise<Uint8Array>;
destroy?: (error?: Error) => void;
[Symbol.asyncIterator]?: () => AsyncIterator<Uint8Array | string>;
};
/** Read an S3 object without letting missing or forged metadata bypass the cap. */
export async function readPreviewBodyWithinLimit(
body: PreviewObjectBody,
contentLength: number | undefined,
maxBytes = FILE_PREVIEW_MAX_SOURCE_BYTES,
timeoutMs = FILE_PREVIEW_STORAGE_READ_TIMEOUT_MS
): Promise<Buffer> {
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) {
throw new Error("Preview source limit must be a positive safe integer");
}
if (contentLength !== undefined && (!Number.isSafeInteger(contentLength) || contentLength < 0)) {
body.destroy?.();
throw new Error("S3 returned an invalid preview source length");
}
if (contentLength !== undefined && contentLength > maxBytes) {
body.destroy?.();
throw new Error(`Preview source exceeds the ${maxBytes}-byte limit`);
}
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
throw new Error("Preview source read timeout must be a positive safe integer");
}
const readBody = async (): Promise<Buffer> => {
if (body[Symbol.asyncIterator]) {
const chunks: Buffer[] = [];
let totalBytes = 0;
for await (const value of body as AsyncIterable<Uint8Array | string>) {
const chunk = typeof value === "string" ? Buffer.from(value) : Buffer.from(value.buffer, value.byteOffset, value.byteLength);
totalBytes += chunk.byteLength;
if (totalBytes > maxBytes) {
body.destroy?.();
throw new Error(`Preview source exceeds the ${maxBytes}-byte limit`);
}
chunks.push(chunk);
}
return Buffer.concat(chunks, totalBytes);
}
if (body.transformToByteArray) {
const bytes = await body.transformToByteArray();
if (bytes.byteLength > maxBytes) {
body.destroy?.();
throw new Error(`Preview source exceeds the ${maxBytes}-byte limit`);
}
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
}
throw new Error("S3 returned an unsupported preview source body");
};
let timeout: NodeJS.Timeout | undefined;
const timeoutFailure = new Promise<never>((_resolve, reject) => {
timeout = setTimeout(() => {
const error = new Error(`Preview source read exceeded the ${timeoutMs}ms limit`);
body.destroy?.(error);
reject(error);
}, timeoutMs);
});
try {
return await Promise.race([readBody(), timeoutFailure]);
} finally {
if (timeout) clearTimeout(timeout);
}
}
/**
* Builds the preview service from STORAGE_* env. Returns undefined when the S3
* credentials are missing (the bootstrap wiring then skips it and the handler
@@ -89,13 +184,19 @@ function normalizeMime(mimeType: string): string {
*/
export function createWorkerFilePreviewService(
env: EnvRecord & FilePreviewServiceEnv,
options: { prisma: FilePreviewPrisma; logger: PreviewLogger },
options: {
prisma: FilePreviewPrisma;
logger: PreviewLogger;
decodePreview?: (input: Buffer, mime: FilePreviewDecoderMime) => Promise<FilePreviewDecoderSuccess>;
}
): FilePreviewService | undefined {
if (!env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
const configuredBucket = env.STORAGE_BUCKET?.trim();
if (!configuredBucket || !env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
return undefined;
}
const { prisma, logger } = options;
const decodePreview = options.decodePreview ?? decodeFilePreviewIsolated;
const s3 = new AwsS3Client({
endpoint: env.STORAGE_ENDPOINT,
region: env.STORAGE_REGION,
@@ -107,35 +208,82 @@ export function createWorkerFilePreviewService(
},
});
async function loadOriginal(bucket: string, key: string): Promise<Buffer> {
const result = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
async function loadOriginal(bucket: string, key: string, expectedBytes: number): Promise<Buffer> {
const result = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }), {
abortSignal: AbortSignal.timeout(FILE_PREVIEW_STORAGE_REQUEST_TIMEOUT_MS),
});
const body = result.Body;
if (!body) {
throw new Error(`S3 GET ${bucket}/${key} returned an empty body`);
}
// The AWS SDK exposes transformToByteArray() on the Node stream body.
const bytes = await (body as { transformToByteArray(): Promise<Uint8Array> }).transformToByteArray();
return Buffer.from(bytes);
const bytes = await readPreviewBodyWithinLimit(body as PreviewObjectBody, result.ContentLength);
if (bytes.byteLength !== expectedBytes) {
throw new Error("Preview source byte length does not match its File record");
}
return bytes;
}
async function putRendered(bucket: string, key: string, body: Buffer, contentType: string): Promise<void> {
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, ContentType: contentType }));
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, ContentType: contentType }), {
abortSignal: AbortSignal.timeout(FILE_PREVIEW_STORAGE_REQUEST_TIMEOUT_MS),
});
}
async function writeStatus(
tenantId: string,
fileId: string,
data: { previewStatus: string; previewStorageKey?: string | null; thumbnailStorageKey?: string | null },
data: { previewStatus: string; previewStorageKey?: string | null; thumbnailStorageKey?: string | null }
): Promise<void> {
// Set the tenant GUC so the write satisfies the File RLS policy once it is
// activated (P5 cutover). Preview jobs always carry a tenantId.
await withTenantRLS(prisma, tenantId, (tx) => tx.file.update({ where: { id: fileId }, data }));
}
async function claimCleanSource(payload: FilePreviewJob) {
return withTenantRLS(prisma, payload.tenantId, async (tx) => {
const source = await tx.file.findFirst({
where: { id: payload.fileId, tenantId: payload.tenantId },
select: {
id: true,
mimeType: true,
previewStatus: true,
previewStorageKey: true,
scanStatus: true,
sizeBytes: true,
storageKey: true,
thumbnailStorageKey: true,
},
});
if (!source || source.scanStatus !== "clean") {
throw new Error("File preview is blocked until its malware scan is clean");
}
if (source.storageKey !== payload.storageKey) {
throw new Error("File preview payload does not match the persisted storage key");
}
if (!Number.isSafeInteger(source.sizeBytes) || source.sizeBytes < 0 || source.sizeBytes > FILE_PREVIEW_MAX_SOURCE_BYTES) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_SOURCE_BYTES}-byte limit`);
}
return source;
});
}
return {
async generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult> {
const mime = normalizeMime(payload.mimeType);
const keys = buildFileStorageKeys(payload.tenantId, payload.fileId);
assertPreviewSource(payload, configuredBucket, keys.original);
const source = await claimCleanSource(payload);
const mime = normalizeMime(source.mimeType);
if (source.previewStatus === "ready") {
return {
previewStatus: "ready",
...(source.previewStorageKey ? { previewStorageKey: source.previewStorageKey } : {}),
...(source.thumbnailStorageKey ? { thumbnailStorageKey: source.thumbnailStorageKey } : {}),
};
}
if (source.previewStatus === "not_supported") {
return { previewStatus: "not_supported" };
}
// Plain text / markdown: detail view renders inline from bytes, no
// rendered preview file needed.
@@ -156,35 +304,16 @@ export function createWorkerFilePreviewService(
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "generating" });
try {
const original = await loadOriginal(payload.bucket, payload.storageKey);
let previewBuffer: Buffer;
let previewExt: FilePreviewExtension;
let previewContentType: string;
const original = await loadOriginal(configuredBucket, payload.storageKey, source.sizeBytes);
const rendered = await decodePreview(original, mime as FilePreviewDecoderMime);
const previewBuffer = Buffer.from(rendered.preview);
const thumbnail = Buffer.from(rendered.thumbnail);
if (isHeic) {
// HEIC -> JPG (mandatory for iPhone field-worker photos).
const jpg = await heicToJpeg(original);
previewBuffer = await rasterToFormat(jpg, "jpeg", PREVIEW_MAX_DIMENSION);
previewExt = "jpg";
previewContentType = "image/jpeg";
} else if (isPdf) {
const png = await pdfFirstPageToPng(original);
previewBuffer = await rasterToFormat(png, "png", PREVIEW_MAX_DIMENSION);
previewExt = "png";
previewContentType = "image/png";
} else {
previewBuffer = await rasterToFormat(original, "webp", PREVIEW_MAX_DIMENSION);
previewExt = "webp";
previewContentType = "image/webp";
}
const thumbnail = await rasterToFormat(previewBuffer, "webp", THUMBNAIL_MAX_DIMENSION);
const previewKey = buildFilePreviewKey(payload.tenantId, payload.fileId, previewExt);
const previewKey = buildFilePreviewKey(payload.tenantId, payload.fileId, rendered.previewExtension);
const thumbnailKey = keys.thumbnail;
await putRendered(payload.bucket, previewKey, previewBuffer, previewContentType);
await putRendered(payload.bucket, thumbnailKey, thumbnail, "image/webp");
await putRendered(configuredBucket, previewKey, previewBuffer, rendered.previewContentType);
await putRendered(configuredBucket, thumbnailKey, thumbnail, "image/webp");
await writeStatus(payload.tenantId, payload.fileId, {
previewStatus: "ready",
@@ -197,61 +326,19 @@ export function createWorkerFilePreviewService(
const reason = error instanceof Error ? error.message : String(error);
logger.warn({ fileId: payload.fileId, tenantId: payload.tenantId, mime }, "file preview generation failed");
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "failed" });
return { previewStatus: "failed", reason };
throw new Error(reason, { cause: error });
}
},
};
}
// --- dynamic native-lib loaders (kept untyped so build stays green pre-install) ---
type SharpFactory = (input: Buffer) => {
rotate(): ReturnType<SharpFactory>;
resize(opts: { width: number; height: number; fit: "inside"; withoutEnlargement: boolean }): ReturnType<SharpFactory>;
jpeg(opts: { quality: number }): ReturnType<SharpFactory>;
png(): ReturnType<SharpFactory>;
webp(opts: { quality: number }): ReturnType<SharpFactory>;
toBuffer(): Promise<Buffer>;
};
let sharpModule: SharpFactory | undefined;
async function loadSharp(): Promise<SharpFactory> {
if (sharpModule) return sharpModule;
const moduleName = "sharp";
const loaded = (await import(/* @vite-ignore */ moduleName)) as { default?: SharpFactory } & SharpFactory;
sharpModule = (loaded.default ?? loaded) as SharpFactory;
return sharpModule;
}
async function rasterToFormat(input: Buffer, format: "jpeg" | "png" | "webp", maxDimension: number): Promise<Buffer> {
const sharp = await loadSharp();
const pipeline = sharp(input)
.rotate()
.resize({ width: maxDimension, height: maxDimension, fit: "inside", withoutEnlargement: true });
if (format === "jpeg") return pipeline.jpeg({ quality: 82 }).toBuffer();
if (format === "png") return pipeline.png().toBuffer();
return pipeline.webp({ quality: 80 }).toBuffer();
}
async function heicToJpeg(input: Buffer): Promise<Buffer> {
const moduleName = "heic-convert";
const loaded = (await import(/* @vite-ignore */ moduleName)) as
| { default: (opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer> }
| ((opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer>);
const convert = typeof loaded === "function" ? loaded : loaded.default;
const out = await convert({ buffer: input, format: "JPEG", quality: 0.85 });
return Buffer.from(out as ArrayBuffer);
}
async function pdfFirstPageToPng(input: Buffer): Promise<Buffer> {
const moduleName = "pdf-to-png-converter";
const loaded = (await import(/* @vite-ignore */ moduleName)) as {
pdfToPng: (input: Buffer, opts: Record<string, unknown>) => Promise<Array<{ content: Buffer }>>;
};
const pages = await loaded.pdfToPng(input, { pagesToProcess: [1], viewportScale: 2.0 });
const first = pages[0];
if (!first) {
throw new Error("PDF rendered no pages");
export function assertPreviewSource(payload: FilePreviewJob, configuredBucket: string, expectedStorageKey?: string): void {
const bucket = configuredBucket.trim();
if (!bucket || payload.bucket !== bucket) {
throw new Error("File preview source bucket does not match worker storage configuration");
}
const expected = expectedStorageKey ?? buildFileStorageKeys(payload.tenantId, payload.fileId).original;
if (payload.storageKey !== expected) {
throw new Error("File preview source key does not match the canonical tenant file path");
}
return Buffer.from(first.content);
}
@@ -1,9 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import {
createWorkerFileScanStatusWriter,
type FileScanStatusPrisma,
} from "./file-scan-status.js";
import { createWorkerFileScanStatusWriter, type FileScanStatusPrisma } from "./file-scan-status.js";
/**
* Build a Prisma stub whose `file.updateMany` reports `count` matched rows.
@@ -12,14 +9,23 @@ import {
*/
function createPrismaStub(count: number) {
const updateMany = vi.fn(async () => ({ count }));
const findFirst = vi.fn(
async (): Promise<{ id: string; createdByUserId: string | null; links: Array<{ id: string; entityId: string; createdAt: Date }> } | null> => null
);
const executeRaw = vi.fn(async () => 1);
const tx = { $executeRaw: executeRaw, file: { updateMany } };
const outboxCreate = vi.fn(async ({ data }: { data: Record<string, unknown> }) => ({ id: "outbox-1", ...data }));
const outboxFindUnique = vi.fn(async () => null);
const tx = {
$executeRaw: executeRaw,
file: { updateMany, findFirst },
outboxEvent: { create: outboxCreate, findUnique: outboxFindUnique },
};
const transaction = vi.fn(async (fn: (t: typeof tx) => Promise<unknown>) => fn(tx));
const prisma = {
$transaction: transaction,
file: { updateMany },
} as unknown as FileScanStatusPrisma;
return { prisma, updateMany, executeRaw, transaction };
return { prisma, updateMany, findFirst, outboxCreate, executeRaw, transaction };
}
function createLogger() {
@@ -32,18 +38,17 @@ describe("createWorkerFileScanStatusWriter — tenant-scoped (RLS) path", () =>
const logger = createLogger();
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await writer.recordVerdict({ fileId: "file-1", tenantId: "tenant-a", verdict: "infected" });
await expect(
writer.recordVerdict({ fileId: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a", verdict: "infected" })
).resolves.toBe(true);
expect(transaction).toHaveBeenCalledTimes(1);
expect(executeRaw).toHaveBeenCalledTimes(1);
expect(updateMany).toHaveBeenCalledWith({
where: { id: "file-1", tenantId: "tenant-a" },
where: { id: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a" },
data: { scanStatus: "infected" },
});
expect(logger.info).toHaveBeenCalledWith(
{ fileId: "file-1", verdict: "infected" },
"scan verdict written to File.scanStatus"
);
expect(logger.info).toHaveBeenCalledWith({ fileId: "file-1", verdict: "infected" }, "scan verdict written to File.scanStatus");
expect(logger.warn).not.toHaveBeenCalled();
});
@@ -53,16 +58,54 @@ describe("createWorkerFileScanStatusWriter — tenant-scoped (RLS) path", () =>
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await expect(
writer.recordVerdict({ fileId: "missing", tenantId: "tenant-a", verdict: "clean" })
).resolves.toBeUndefined();
writer.recordVerdict({ fileId: "missing", storageKey: "tenants/tenant-a/files/missing/original", tenantId: "tenant-a", verdict: "clean" })
).resolves.toBe(false);
expect(updateMany).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledWith(
{ fileId: "missing", tenantId: "tenant-a", verdict: "clean" },
"scan verdict write-back matched no File row"
);
expect(logger.warn).toHaveBeenCalledWith({ fileId: "missing", tenantId: "tenant-a", verdict: "clean" }, "scan verdict write-back matched no File row");
expect(logger.info).not.toHaveBeenCalled();
});
it("publishes one idempotent portal document-ready event per internal project link after a clean verdict", async () => {
const { prisma, findFirst, outboxCreate } = createPrismaStub(1);
const linkCreatedAt = new Date("2026-07-16T10:00:00.000Z");
findFirst.mockResolvedValue({
id: "file-1",
createdByUserId: "user-1",
links: [{ id: "link-1", entityId: "project-1", createdAt: linkCreatedAt }],
});
const writer = createWorkerFileScanStatusWriter({ prisma, logger: createLogger() });
await expect(
writer.recordVerdict({ fileId: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a", verdict: "clean" })
).resolves.toBe(true);
expect(findFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({ tenantId: "tenant-a", createdByPortalUserId: null, source: { not: "customer_portal" } }),
})
);
expect(outboxCreate).toHaveBeenCalledWith({
data: expect.objectContaining({
eventType: "workflow.trigger.event",
dispatchKey: `workflow-trigger:portal-document-ready:link-1:${linkCreatedAt.getTime()}`,
payload: expect.objectContaining({
event: "portal.document_ready",
data: { projectId: "project-1", fileId: "file-1", fileEntityLinkId: "link-1" },
}),
}),
});
});
it("does not publish a document-ready event for infected verdicts", async () => {
const { prisma, findFirst, outboxCreate } = createPrismaStub(1);
const writer = createWorkerFileScanStatusWriter({ prisma, logger: createLogger() });
await writer.recordVerdict({ fileId: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a", verdict: "infected" });
expect(findFirst).not.toHaveBeenCalled();
expect(outboxCreate).not.toHaveBeenCalled();
});
});
describe("createWorkerFileScanStatusWriter — legacy direct path (no tenantId)", () => {
@@ -71,17 +114,14 @@ describe("createWorkerFileScanStatusWriter — legacy direct path (no tenantId)"
const logger = createLogger();
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await writer.recordVerdict({ fileId: "file-2", verdict: "clean" });
await expect(writer.recordVerdict({ fileId: "file-2", storageKey: "legacy/file-2", verdict: "clean" })).resolves.toBe(true);
expect(transaction).not.toHaveBeenCalled();
expect(updateMany).toHaveBeenCalledWith({
where: { id: "file-2" },
where: { id: "file-2", storageKey: "legacy/file-2" },
data: { scanStatus: "clean" },
});
expect(logger.info).toHaveBeenCalledWith(
{ fileId: "file-2", verdict: "clean" },
"scan verdict written to File.scanStatus"
);
expect(logger.info).toHaveBeenCalledWith({ fileId: "file-2", verdict: "clean" }, "scan verdict written to File.scanStatus");
});
it("warns when the direct write matches no row", async () => {
@@ -89,12 +129,9 @@ describe("createWorkerFileScanStatusWriter — legacy direct path (no tenantId)"
const logger = createLogger();
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await writer.recordVerdict({ fileId: "gone", verdict: "infected" });
await expect(writer.recordVerdict({ fileId: "gone", storageKey: "legacy/gone", verdict: "infected" })).resolves.toBe(false);
expect(transaction).not.toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalledWith(
{ fileId: "gone", tenantId: undefined, verdict: "infected" },
"scan verdict write-back matched no File row"
);
expect(logger.warn).toHaveBeenCalledWith({ fileId: "gone", tenantId: undefined, verdict: "infected" }, "scan verdict write-back matched no File row");
});
});
@@ -8,8 +8,11 @@
//
// Mirrors createWorkerFilePreviewService: a Prisma seam is injected at the
// general-group bootstrap (index.ts) so handlers.ts stays free of direct DB
// imports. When the seam is unset the handler simply skips the write-back
// (legacy import-path scans carry no fileId either).
// imports. Legacy import-path scans carry no fileId; a real File scan fails and
// retries if this seam is unavailable.
import { publishOutboxEvent, type OutboxPublisherTransaction } from "@hivegrid/db/outbox";
import { buildPortalDocumentReadyOutboxEvent } from "@hivegrid/queue";
import { withTenantRLS } from "../db/tenantRls.js";
@@ -25,55 +28,95 @@ type ScanStatusLogger = {
interface FileScanStatusTx {
$executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
file: {
updateMany(args: {
where: { id: string; tenantId?: string };
data: { scanStatus: string };
}): Promise<{ count: number }>;
updateMany(args: { where: { id: string; storageKey: string; tenantId?: string }; data: { scanStatus: string } }): Promise<{ count: number }>;
findFirst(args: unknown): Promise<{
id: string;
createdByUserId: string | null;
links: Array<{ id: string; entityId: string; createdAt: Date }>;
} | null>;
};
outboxEvent: OutboxPublisherTransaction["outboxEvent"];
}
/** Minimal Prisma surface the write-back needs (direct write + tenant tx). */
export interface FileScanStatusPrisma {
$transaction<T>(fn: (tx: FileScanStatusTx) => Promise<T>): Promise<T>;
file: {
updateMany(args: {
where: { id: string; tenantId?: string };
data: { scanStatus: string };
}): Promise<{ count: number }>;
updateMany(args: { where: { id: string; storageKey: string; tenantId?: string }; data: { scanStatus: string } }): Promise<{ count: number }>;
};
}
export type FileScanVerdict = "clean" | "infected";
export interface FileScanStatusWriter {
recordVerdict(input: { fileId: string; tenantId?: string; verdict: FileScanVerdict }): Promise<void>;
recordVerdict(input: { fileId: string; storageKey: string; tenantId?: string; verdict: FileScanVerdict }): Promise<boolean>;
}
export function createWorkerFileScanStatusWriter(options: {
prisma: FileScanStatusPrisma;
logger: ScanStatusLogger;
}): FileScanStatusWriter {
export function createWorkerFileScanStatusWriter(options: { prisma: FileScanStatusPrisma; logger: ScanStatusLogger }): FileScanStatusWriter {
const { prisma, logger } = options;
return {
async recordVerdict({ fileId, tenantId, verdict }) {
async recordVerdict({ fileId, storageKey, tenantId, verdict }) {
// updateMany (not update) so a tenant-scoped/RLS-filtered or already
// deleted row is a logged no-op instead of a throw that dead-letters
// the job after the scan already ran.
const write = (db: Pick<FileScanStatusTx, "file">) =>
const write = (db: {
file: {
updateMany(args: { where: { id: string; storageKey: string; tenantId?: string }; data: { scanStatus: string } }): Promise<{ count: number }>;
};
}) =>
db.file.updateMany({
where: tenantId ? { id: fileId, tenantId } : { id: fileId },
where: tenantId ? { id: fileId, storageKey, tenantId } : { id: fileId, storageKey },
data: { scanStatus: verdict },
});
// Set the tenant GUC so the write satisfies the File RLS policy once it is
// activated (P5 cutover). Legacy import-path scans carry no tenantId and
// fall back to the direct write (no RLS context to set).
const result = tenantId
? await withTenantRLS(prisma, tenantId, (tx) => write(tx))
? await withTenantRLS(prisma, tenantId, async (tx) => {
const scopedResult = await write(tx);
if (verdict === "clean" && scopedResult.count > 0) {
const readyFile = await tx.file.findFirst({
where: {
id: fileId,
storageKey,
tenantId,
archivedAt: null,
scanStatus: "clean",
createdByPortalUserId: null,
source: { not: "customer_portal" },
},
select: {
id: true,
createdByUserId: true,
links: {
where: { tenantId, entityType: "project", createdByPortalUserId: null },
select: { id: true, entityId: true, createdAt: true },
},
},
});
for (const link of readyFile?.links ?? []) {
await publishOutboxEvent(
tx as unknown as OutboxPublisherTransaction,
buildPortalDocumentReadyOutboxEvent({
tenantId,
projectId: link.entityId,
fileId: readyFile!.id,
fileEntityLinkId: link.id,
linkVersion: link.createdAt,
userId: readyFile!.createdByUserId,
})
);
}
}
return scopedResult;
})
: await write(prisma);
if (result.count === 0) {
logger.warn({ fileId, tenantId, verdict }, "scan verdict write-back matched no File row");
return false;
} else {
logger.info({ fileId, verdict }, "scan verdict written to File.scanStatus");
return true;
}
},
};
@@ -0,0 +1,91 @@
import { describe, expect, it, vi } from "vitest";
import { verifyMarketingToken } from "@hivegrid/foundation/marketing-token";
import { runStartDoubleOptInFormAction, type StartDoubleOptInPrisma } from "./form-submission-actions.js";
type PublishedOptInEvent = Parameters<Parameters<typeof runStartDoubleOptInFormAction>[0]["publishOutboxEvent"]>[0];
const env = {
APP_URL: "https://app.example.test",
MARKETING_TOKEN_SECRET: "x".repeat(40),
};
function prismaStub(overrides: Partial<StartDoubleOptInPrisma> = {}): StartDoubleOptInPrisma {
return {
kontakt: {
findFirst: vi.fn(async () => ({ id: "kontakt_1", primaerEmail: "Ada@Example.com", vorname: "Ada", nachname: "Lovelace" })),
},
kontaktMarketingConsent: {
upsert: vi.fn(async () => ({})),
},
...overrides,
};
}
describe("runStartDoubleOptInFormAction", () => {
it("upserts single consent and queues the double-opt-in mail", async () => {
const prisma = prismaStub();
const publishOutboxEvent = vi.fn(async (event: PublishedOptInEvent) => {
void event;
});
const result = await runStartDoubleOptInFormAction({
prisma,
publishOutboxEvent,
tenantId: "tenant_1",
formId: "form_1",
submissionId: "sub_1",
actionId: "action_1",
mapped: { email: "ada@example.com" },
config: {},
env,
now: () => new Date("2026-07-09T10:00:00.000Z"),
});
expect(result).toEqual({ status: "started", kontaktId: "kontakt_1", email: "ada@example.com" });
expect(prisma.kontaktMarketingConsent.upsert).toHaveBeenCalledWith(
expect.objectContaining({
where: { tenantId_kontaktId: { tenantId: "tenant_1", kontaktId: "kontakt_1" } },
create: expect.objectContaining({ status: "single", optInSource: "form:form_1" }),
}),
);
const event = vi.mocked(publishOutboxEvent).mock.calls[0]?.[0];
expect(event).toMatchObject({
tenantId: "tenant_1",
type: "mail.transactional.send",
dispatchKey: "form-action-optin:sub_1:action_1",
payload: expect.objectContaining({
to: ["ada@example.com"],
subject: "Bitte bestätige deine Anmeldung",
}),
});
const link = String((event?.payload.html as string).match(/https:\/\/app\.example\.test\/optin\/([^"]+)/u)?.[1] ?? "");
expect(verifyMarketingToken(decodeURIComponent(link), { env, expectedPurpose: "optin" })).toMatchObject({
tenantId: "tenant_1",
kontaktId: "kontakt_1",
});
});
it("skips when no matching contact exists", async () => {
const prisma = prismaStub({
kontakt: {
findFirst: vi.fn(async () => null),
},
});
const publishOutboxEvent = vi.fn(async (event: PublishedOptInEvent) => {
void event;
});
await expect(runStartDoubleOptInFormAction({
prisma,
publishOutboxEvent,
tenantId: "tenant_1",
formId: "form_1",
submissionId: "sub_1",
actionId: "action_1",
mapped: { email: "nobody@example.com" },
config: {},
env,
})).resolves.toEqual({ status: "skipped", reason: "no_contact" });
expect(publishOutboxEvent).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,152 @@
import { signMarketingToken } from "@hivegrid/foundation/marketing-token";
import { renderSystemMessageEmail } from "@hivegrid/mail";
export type FormActionJsonValue = string | number | boolean | null | FormActionJsonValue[] | { [key: string]: FormActionJsonValue };
export interface StartDoubleOptInPrisma {
readonly kontakt: {
findFirst(args: Record<string, unknown>): Promise<{ id: string; primaerEmail: string | null; vorname: string | null; nachname: string | null } | null>;
};
readonly kontaktMarketingConsent: {
upsert(args: Record<string, unknown>): Promise<unknown>;
};
}
export interface StartDoubleOptInInput {
readonly prisma: StartDoubleOptInPrisma;
readonly publishOutboxEvent: (event: {
readonly tenantId: string;
readonly type: "mail.transactional.send";
readonly payload: Record<string, unknown>;
readonly dispatchKey: string;
}) => Promise<unknown>;
readonly tenantId: string;
readonly formId: string;
readonly submissionId: string;
readonly actionId: string;
readonly mapped: Record<string, unknown>;
readonly config: Record<string, unknown>;
readonly env?: Record<string, string | undefined>;
readonly now?: () => Date;
}
export type StartDoubleOptInResult =
| { readonly status: "started"; readonly kontaktId: string; readonly email: string }
| { readonly status: "skipped"; readonly reason: "no_contact" | "no_email" | "missing_app_url" };
function stringValue(values: Record<string, unknown>, keys: readonly string[]): string | null {
for (const key of keys) {
const value = values[key];
if (typeof value === "string" && value.trim()) return value.trim();
}
return null;
}
function readBaseUrl(config: Record<string, unknown>, env: Record<string, string | undefined>): string | null {
const configured = typeof config.confirmBaseUrl === "string" ? config.confirmBaseUrl.trim() : "";
const fallback = env.APP_URL?.trim() ?? "";
const raw = (configured || fallback).replace(/\/+$/u, "");
if (!raw) return null;
try {
const url = new URL(raw);
return url.origin + url.pathname.replace(/\/+$/u, "");
} catch {
return null;
}
}
function renderOptInMail(input: { link: string; name: string | null; subject?: string | null }) {
const subject = input.subject?.trim() || "Bitte bestätige deine Anmeldung";
const rendered = renderSystemMessageEmail({
locale: "de",
subject,
previewText: "Bestätige deine E-Mail-Adresse mit einem Klick.",
eyebrow: "E-Mail-Anmeldung",
title: input.name ? `${input.name}, bitte bestätige deine Anmeldung` : "Bitte bestätige deine Anmeldung",
intro: "Ein letzter Klick fehlt noch. Danach dürfen wir dir die gewünschten Informationen per E-Mail senden.",
actionUrl: input.link,
actionLabel: "Anmeldung bestätigen",
notice: "Wenn du dich nicht angemeldet hast, kannst du diese E-Mail einfach ignorieren.",
});
return { subject, text: rendered.text, html: rendered.html };
}
export async function runStartDoubleOptInFormAction(input: StartDoubleOptInInput): Promise<StartDoubleOptInResult> {
const configuredKontaktId = typeof input.config.kontaktId === "string" && input.config.kontaktId.trim() ? input.config.kontaktId.trim() : null;
const email = stringValue(input.mapped, ["email", "email_address", "primaerEmail"])?.toLowerCase() ?? null;
const phone = stringValue(input.mapped, ["phone", "telefon", "primaerTelefon"]);
if (!configuredKontaktId && !email && !phone) return { status: "skipped", reason: "no_contact" };
const contact = await input.prisma.kontakt.findFirst({
where: configuredKontaktId
? { id: configuredKontaktId, tenantId: input.tenantId, archivedAt: null }
: {
tenantId: input.tenantId,
archivedAt: null,
OR: [...(email ? [{ primaerEmail: email }] : []), ...(phone ? [{ primaerTelefon: phone }] : [])],
},
select: { id: true, primaerEmail: true, vorname: true, nachname: true },
});
if (!contact) return { status: "skipped", reason: "no_contact" };
const recipient = contact.primaerEmail?.trim().toLowerCase() || email;
if (!recipient) return { status: "skipped", reason: "no_email" };
const baseUrl = readBaseUrl(input.config, input.env ?? process.env);
if (!baseUrl) return { status: "skipped", reason: "missing_app_url" };
const token = signMarketingToken({
purpose: "optin",
tenantId: input.tenantId,
kontaktId: contact.id,
now: input.now,
env: input.env ?? process.env,
});
const now = input.now ? input.now() : new Date();
await input.prisma.kontaktMarketingConsent.upsert({
where: { tenantId_kontaktId: { tenantId: input.tenantId, kontaktId: contact.id } },
create: {
tenantId: input.tenantId,
kontaktId: contact.id,
status: "single",
optInSource: `form:${input.formId}`,
optInAt: now,
doubleOptInToken: token,
},
update: {
status: "single",
optInSource: `form:${input.formId}`,
optInAt: now,
doubleOptInToken: token,
unsubscribedAt: null,
},
});
const link = `${baseUrl}/optin/${encodeURIComponent(token)}`;
const displayName = [contact.vorname, contact.nachname].filter(Boolean).join(" ") || null;
const mail = renderOptInMail({
link,
name: displayName,
subject: typeof input.config.subject === "string" ? input.config.subject : null,
});
await input.publishOutboxEvent({
tenantId: input.tenantId,
type: "mail.transactional.send",
payload: {
tenantId: input.tenantId,
to: [recipient],
subject: mail.subject,
text: mail.text,
html: mail.html,
priorityOverride: "normal",
metadata: {
source: "form-double-opt-in",
formId: input.formId,
submissionId: input.submissionId,
actionId: input.actionId,
kontaktId: contact.id,
},
},
dispatchKey: `form-action-optin:${input.submissionId}:${input.actionId}`,
});
return { status: "started", kontaktId: contact.id, email: recipient };
}
@@ -0,0 +1,55 @@
import { describe, expect, it, vi } from "vitest";
import {
INTEGRATIONS_HEALTH_SWEEP_EVERY_MS,
INTEGRATIONS_HEALTH_SWEEP_JOB_ID,
INTEGRATIONS_TOKEN_REFRESH_EVERY_MS,
INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES,
KWP_PROJECT_MAINTENANCE_EVERY_MS,
KWP_PROJECT_MAINTENANCE_JOB_ID,
registerIntegrationsHealthSweepCron,
registerIntegrationsTokenRefreshCron,
registerKwpProjectMaintenanceCron,
} from "./integrations-maintenance-cron.js";
describe("integrations maintenance cron registration", () => {
it("sweeps token refresh every minute while retaining the generic ten-minute payload window", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
const queue = { upsertJobScheduler };
await registerIntegrationsTokenRefreshCron(queue);
await registerIntegrationsHealthSweepCron(queue);
await registerKwpProjectMaintenanceCron(queue);
expect(INTEGRATIONS_TOKEN_REFRESH_EVERY_MS).toBe(60_000);
expect(INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES).toBe(10);
expect(upsertJobScheduler).toHaveBeenCalledWith(
INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
{ every: 60_000 },
{
name: INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
data: {
kind: "cleanup.run",
task: INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
olderThanMinutes: 10,
},
},
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
INTEGRATIONS_HEALTH_SWEEP_JOB_ID,
{ every: INTEGRATIONS_HEALTH_SWEEP_EVERY_MS },
{
name: INTEGRATIONS_HEALTH_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: INTEGRATIONS_HEALTH_SWEEP_JOB_ID },
},
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
KWP_PROJECT_MAINTENANCE_JOB_ID,
{ every: KWP_PROJECT_MAINTENANCE_EVERY_MS },
{
name: KWP_PROJECT_MAINTENANCE_JOB_ID,
data: { kind: "cleanup.run", task: KWP_PROJECT_MAINTENANCE_JOB_ID },
},
);
});
});
@@ -5,10 +5,14 @@
export const INTEGRATIONS_TOKEN_REFRESH_JOB_ID = "integrations-token-refresh" as const;
export const INTEGRATIONS_HEALTH_SWEEP_JOB_ID = "integrations-health-sweep" as const;
export const KWP_PROJECT_MAINTENANCE_JOB_ID = "kwp-project-maintenance" as const;
// Token refresh runs often (tokens expire); health sweep less often.
export const INTEGRATIONS_TOKEN_REFRESH_EVERY_MS = 5 * 60 * 1000;
// KWP access tokens live for only five minutes. Sweep every minute so the
// KWP-specific two-minute refresh window has a real safety margin while the
// generic Apps-Engine refresh window remains unchanged.
export const INTEGRATIONS_TOKEN_REFRESH_EVERY_MS = 60 * 1000;
export const INTEGRATIONS_HEALTH_SWEEP_EVERY_MS = 15 * 60 * 1000;
export const KWP_PROJECT_MAINTENANCE_EVERY_MS = 15 * 60 * 1000;
/** Refresh tokens expiring within this window. Default 10 minutes. */
export const INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES = 10;
@@ -32,3 +36,11 @@ export async function registerIntegrationsHealthSweepCron(queue: SchedulerQueue)
{ name: INTEGRATIONS_HEALTH_SWEEP_JOB_ID, data: { kind: "cleanup.run", task: INTEGRATIONS_HEALTH_SWEEP_JOB_ID } },
);
}
export async function registerKwpProjectMaintenanceCron(queue: SchedulerQueue): Promise<void> {
await queue.upsertJobScheduler(
KWP_PROJECT_MAINTENANCE_JOB_ID,
{ every: KWP_PROJECT_MAINTENANCE_EVERY_MS },
{ name: KWP_PROJECT_MAINTENANCE_JOB_ID, data: { kind: "cleanup.run", task: KWP_PROJECT_MAINTENANCE_JOB_ID } },
);
}
@@ -0,0 +1,53 @@
import { describe, expect, it, vi } from "vitest";
import { runAllTenantKwpProjectMaintenance } from "./kwp-project-maintenance.js";
describe("KWP project maintenance", () => {
it("only schedules targeted provider work with stable cadence ids", async () => {
const add = vi.fn(async (_name: string, _payload: unknown, _options?: { jobId?: string }) => ({ id: "job" }));
const queue = { add, close: vi.fn(async () => undefined) };
const prisma = {
tenant: {
findMany: vi.fn(async (args: { cursor?: { id: string } }) =>
args.cursor ? [] : [{ id: "tenant_1" }],
),
},
appEntityBinding: {
findMany: vi.fn(async () => [{ id: "binding_1", dataConnectionId: "connection_1" }]),
},
dataConnection: {
findMany: vi.fn(async () => [{ id: "connection_1" }]),
},
};
const result = await runAllTenantKwpProjectMaintenance({
prisma,
queue,
now: new Date("2026-07-23T12:00:00.000Z"),
});
expect(result).toEqual({ inspectedTenants: 1, projectJobs: 1, catalogJobs: 1, contractJobs: 1, failures: 0 });
expect(add).toHaveBeenCalledTimes(3);
expect(add.mock.calls.map(([jobName]) => jobName)).toEqual([
"run-kwp-import",
"run-kwp-import",
"run-kwp-import",
]);
expect(add.mock.calls[0]?.[1]).toMatchObject({
kind: "kwp.project-sync",
tenantId: "tenant_1",
bindingId: "binding_1",
});
expect(add.mock.calls[1]?.[1]).toMatchObject({
kind: "kwp.project-catalog-sync",
tenantId: "tenant_1",
dataConnectionId: "connection_1",
});
expect(add.mock.calls[2]?.[1]).toMatchObject({
kind: "kwp.contract-sentinel",
tenantId: "tenant_1",
dataConnectionId: "connection_1",
});
expect(add.mock.calls[0]?.[2]?.jobId).toContain("kwp-project-sync%3Atenant_1%3Abinding_1%3A");
});
});
@@ -0,0 +1,152 @@
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
import {
buildKwpContractSentinelJob,
buildKwpProjectCatalogSyncJob,
buildKwpProjectSyncJob,
enqueueJob,
type TypedQueue,
queueNames,
} from "@hivegrid/queue";
import { runActiveTenantSweep, type ActiveTenantReader } from "./active-tenant-sweep.js";
const PROJECT_INTERVAL_MS = 15 * 60_000;
const CONTRACT_INTERVAL_MS = 24 * 60 * 60_000;
const BATCH_SIZE = 100;
type KwpMaintenancePrisma = ActiveTenantReader & {
appEntityBinding: {
findMany(args: {
where: {
tenantId: string;
externalEntityType: "kwp_project";
localEntityType: "project";
state: { in: readonly ["active", "missing"] };
OR: readonly [
{ lastSyncedAt: null },
{ lastSyncedAt: { lte: Date } },
];
};
select: { id: true; dataConnectionId: true };
orderBy: { id: "asc" };
take: number;
}): Promise<Array<{ id: string; dataConnectionId: string }>>;
};
dataConnection: {
findMany(args: {
where: {
tenantId: string;
appId: "kwp";
archivedAt: null;
connectionStatus: { in: readonly ["active", "degraded"] };
};
select: { id: true };
orderBy: { id: "asc" };
take: number;
}): Promise<Array<{ id: string }>>;
};
};
export type KwpMaintenanceResult = {
inspectedTenants: number;
projectJobs: number;
catalogJobs: number;
contractJobs: number;
failures: number;
};
function cadenceBucket(now: Date, intervalMs: number): string {
return String(Math.floor(now.getTime() / intervalMs));
}
export async function runAllTenantKwpProjectMaintenance(input: {
prisma: KwpMaintenancePrisma;
queue: TypedQueue<typeof queueNames.kwpImports>;
now?: Date;
}): Promise<KwpMaintenanceResult> {
const now = input.now ?? new Date();
const projectCutoff = new Date(now.getTime() - PROJECT_INTERVAL_MS);
const projectBucket = cadenceBucket(now, PROJECT_INTERVAL_MS);
const contractBucket = cadenceBucket(now, CONTRACT_INTERVAL_MS);
const result = await runActiveTenantSweep(input.prisma, {
initialState: { projectJobs: 0, catalogJobs: 0, contractJobs: 0 },
worker: (tenantId) => runWithRlsContext({ tenantId }, async () => {
const [bindings, connections] = await Promise.all([
input.prisma.appEntityBinding.findMany({
where: {
tenantId,
externalEntityType: "kwp_project",
localEntityType: "project",
state: { in: ["active", "missing"] },
OR: [{ lastSyncedAt: null }, { lastSyncedAt: { lte: projectCutoff } }],
},
select: { id: true, dataConnectionId: true },
orderBy: { id: "asc" },
take: BATCH_SIZE,
}),
input.prisma.dataConnection.findMany({
where: {
tenantId,
appId: "kwp",
archivedAt: null,
connectionStatus: { in: ["active", "degraded"] },
},
select: { id: true },
orderBy: { id: "asc" },
take: BATCH_SIZE,
}),
]);
for (const binding of bindings) {
const descriptor = buildKwpProjectSyncJob({
tenantId,
bindingId: binding.id,
dataConnectionId: binding.dataConnectionId,
cadenceBucket: projectBucket,
});
await enqueueJob(input.queue, descriptor, {
idempotencyKey: `kwp-project-sync:${tenantId}:${binding.id}:${projectBucket}`,
tenantId,
attempts: 5,
backoff: { type: "exponential", delay: 30_000 + Math.floor(Math.random() * 10_000) },
});
}
for (const connection of connections) {
const catalogDescriptor = buildKwpProjectCatalogSyncJob({
tenantId,
dataConnectionId: connection.id,
cadenceBucket: projectBucket,
});
await enqueueJob(input.queue, catalogDescriptor, {
idempotencyKey: `kwp-project-catalog:${tenantId}:${connection.id}:${projectBucket}`,
tenantId,
attempts: 5,
backoff: { type: "exponential", delay: 30_000 + Math.floor(Math.random() * 10_000) },
});
const descriptor = buildKwpContractSentinelJob({
tenantId,
dataConnectionId: connection.id,
cadenceBucket: contractBucket,
});
await enqueueJob(input.queue, descriptor, {
idempotencyKey: `kwp-contract-sentinel:${tenantId}:${connection.id}:${contractBucket}`,
tenantId,
attempts: 3,
backoff: { type: "exponential", delay: 60_000 + Math.floor(Math.random() * 20_000) },
});
}
return { projectJobs: bindings.length, catalogJobs: connections.length, contractJobs: connections.length };
}),
merge: (state, value) => ({
projectJobs: state.projectJobs + value.projectJobs,
catalogJobs: state.catalogJobs + value.catalogJobs,
contractJobs: state.contractJobs + value.contractJobs,
}),
});
return {
inspectedTenants: result.inspected,
projectJobs: result.state.projectJobs,
catalogJobs: result.state.catalogJobs,
contractJobs: result.state.contractJobs,
failures: result.failures.length,
};
}
@@ -5,10 +5,7 @@ import {
type LegacyEventReceiverBootstrapDeps,
type LegacyEventReceiverWorkerHandle,
} from "./legacy-event-receiver-bootstrap.js";
import {
LEGACY_EVENT_OUTBOX_QUEUE,
type LegacyEventReceiverStore,
} from "./legacy-event-receiver.js";
import { LEGACY_EVENT_OUTBOX_QUEUE, type LegacyEventReceiverStore } from "./legacy-event-receiver.js";
import type { OutboxReceiverIdempotencyEntry } from "@hivegrid/db/outbox-receiver";
function createStore(): LegacyEventReceiverStore & { calls: number; last: unknown } {
@@ -42,11 +39,7 @@ function createCapturingWorkerImpl() {
name: "legacy-event-receiver-worker",
close: vi.fn(async () => undefined),
};
const createWorkerImpl: NonNullable<LegacyEventReceiverBootstrapDeps["createWorkerImpl"]> = (
queueName,
processor,
options
) => {
const createWorkerImpl: NonNullable<LegacyEventReceiverBootstrapDeps["createWorkerImpl"]> = (queueName, processor, options) => {
captured.queueName = queueName;
captured.processor = processor;
captured.concurrency = options.concurrency;
@@ -117,10 +110,7 @@ describe("bootstrapLegacyEventReceiver", () => {
it("parses a positive concurrency override", () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
bootstrapLegacyEventReceiver(
{ LEGACY_EVENT_RECEIVER_CONCURRENCY: "12" },
{ store: createStore(), createWorkerImpl }
);
bootstrapLegacyEventReceiver({ LEGACY_EVENT_RECEIVER_CONCURRENCY: "12" }, { store: createStore(), createWorkerImpl });
expect(captured.concurrency).toBe(12);
});
@@ -131,10 +121,7 @@ describe("bootstrapLegacyEventReceiver", () => {
["", 5],
])("falls back to the default concurrency for %s", (raw, expected) => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
bootstrapLegacyEventReceiver(
{ LEGACY_EVENT_RECEIVER_CONCURRENCY: raw },
{ store: createStore(), createWorkerImpl }
);
bootstrapLegacyEventReceiver({ LEGACY_EVENT_RECEIVER_CONCURRENCY: raw }, { store: createStore(), createWorkerImpl });
expect(captured.concurrency).toBe(expected);
});
@@ -151,6 +138,114 @@ describe("bootstrapLegacyEventReceiver", () => {
expect(store.calls).toBe(1);
});
it("multiplexes platform notification events on the existing outbox.notification subscriber", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const store = createStore();
const enqueue = vi.fn(async () => undefined);
bootstrapLegacyEventReceiver(
{},
{
store,
createWorkerImpl,
platformNotification: { enqueue },
}
);
const outcome = await captured.processor!({
data: {
outboxEventId: "outbox_1",
tenantId: null,
eventType: "notification.platform.requested.v1",
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1", sourceId: "hivegrid-website" },
},
},
});
expect(outcome).toMatchObject({ status: "enqueued", deliveryId: "delivery_1" });
expect(enqueue).toHaveBeenCalledTimes(1);
expect(store.calls).toBe(0);
});
it("acknowledges the platform lead capture marker without duplicating its durable notification requests", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const store = createStore();
const enqueue = vi.fn(async () => undefined);
bootstrapLegacyEventReceiver(
{},
{
store,
createWorkerImpl,
platformNotification: { enqueue },
}
);
const marker = {
data: {
outboxEventId: "activity_1:lead-outbox",
tenantId: null,
eventType: "platform.admin_lead.captured.v1",
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
},
dispatchKey: "platform-lead-captured:lead_1",
retries: 0,
deliveryGeneration: 0,
lockedUntil: "2026-07-16T10:21:30.123Z",
},
};
await expect(captured.processor!(marker)).resolves.toEqual({
status: "acknowledged",
outboxEventId: "activity_1:lead-outbox",
leadId: "lead_1",
});
await expect(captured.processor!(marker)).resolves.toMatchObject({ status: "acknowledged" });
expect(enqueue).not.toHaveBeenCalled();
expect(store.calls).toBe(0);
});
it("keeps the platform notification subscriber active when the legacy receiver is disabled", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const enqueue = vi.fn(async () => undefined);
const result = bootstrapLegacyEventReceiver(
{ LEGACY_EVENT_RECEIVER_ENABLED: "false" },
{
createWorkerImpl,
platformNotification: { enqueue },
}
);
expect(result.status).toBe("started");
await captured.processor!({
data: {
outboxEventId: "outbox_1",
tenantId: null,
eventType: "notification.platform.requested.v1",
payload: {
deliveryId: "delivery_1",
userId: "user_1",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
},
},
});
expect(enqueue).toHaveBeenCalledTimes(1);
});
it("uses idempotency to skip a re-delivered outbox event", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const store = createStore();
@@ -1,9 +1,11 @@
/**
* Bootstrap helper for the legacy-event outbox receiver.
* Bootstrap helper for the shared notification outbox receiver.
*
* Subscribes a BullMQ worker to the `outbox.notification` queue (which
* the dispatcher routes `billing.subscription.*` events into) and feeds
* each job through {@link runLegacyEventReceiverJob}.
* billing jobs through {@link runLegacyEventReceiverJob} and platform
* notification jobs through their strict tenantless bridge. The platform
* branch remains active even when the legacy branch is disabled.
*
* Gating:
* - WORKER_QUEUE_GROUP === "general" (co-located with the outbox
@@ -20,6 +22,12 @@ import {
type LegacyEventReceiverLogger,
type LegacyEventReceiverStore,
} from "./legacy-event-receiver.js";
import {
isPlatformNotificationOutboxJob,
runPlatformNotificationOutboxJob,
type PlatformNotificationOutboxDependencies,
} from "./platform-notification-outbox.js";
import { isPlatformLeadCapturedOutboxJob, runPlatformLeadCapturedOutboxJob } from "./platform-lead-captured-outbox.js";
export type LegacyEventReceiverBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
@@ -36,6 +44,7 @@ export interface LegacyEventReceiverBootstrapDeps {
readonly logger?: LegacyEventReceiverLogger;
readonly store?: LegacyEventReceiverStore;
readonly idempotency?: LegacyEventReceiverDependencies["idempotency"];
readonly platformNotification?: PlatformNotificationOutboxDependencies;
readonly createWorkerImpl?: (
queueName: string,
processor: (job: { data: unknown }) => Promise<unknown>,
@@ -60,14 +69,16 @@ export function bootstrapLegacyEventReceiver(
deps: LegacyEventReceiverBootstrapDeps = {}
): LegacyEventReceiverBootstrapResult {
const logger = deps.logger;
const legacyEnabled = env.LEGACY_EVENT_RECEIVER_ENABLED !== "false";
const platformEnabled = deps.platformNotification !== undefined;
if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.LEGACY_EVENT_RECEIVER_ENABLED === "false") {
if (!legacyEnabled && !platformEnabled) {
return { status: "disabled", reason: "LEGACY_EVENT_RECEIVER_ENABLED=false" };
}
if (!deps.store) {
if (legacyEnabled && !deps.store && !platformEnabled) {
return { status: "disabled", reason: "no store wired (worker host must supply createLegacyEvent)" };
}
if (!deps.createWorkerImpl) {
@@ -79,6 +90,24 @@ export function bootstrapLegacyEventReceiver(
const idempotency = deps.idempotency;
const processor = async (job: { data: unknown }): Promise<unknown> => {
if (isPlatformLeadCapturedOutboxJob(job.data)) {
return runPlatformLeadCapturedOutboxJob(job.data, { ...(logger ? { logger } : {}) });
}
if (isPlatformNotificationOutboxJob(job.data)) {
if (!deps.platformNotification) {
throw new Error("platform notification outbox receiver is not wired");
}
return runPlatformNotificationOutboxJob(job.data, deps.platformNotification);
}
if (!legacyEnabled) {
throw new Error("legacy-event outbox receiver is disabled");
}
if (!store) {
throw new Error("legacy-event outbox receiver store is not wired");
}
const handlerDeps: LegacyEventReceiverDependencies = {
store,
...(idempotency ? { idempotency } : {}),
@@ -89,9 +118,6 @@ export function bootstrapLegacyEventReceiver(
const handle = deps.createWorkerImpl(LEGACY_EVENT_OUTBOX_QUEUE, processor, { concurrency });
logger?.info?.(
{ component: "legacy-event-receiver", queue: LEGACY_EVENT_OUTBOX_QUEUE, concurrency },
"legacy-event receiver worker started"
);
logger?.info?.({ component: "legacy-event-receiver", queue: LEGACY_EVENT_OUTBOX_QUEUE, concurrency }, "legacy-event receiver worker started");
return { status: "started", handle };
}
@@ -0,0 +1,104 @@
import { describe, expect, it, vi } from "vitest";
import {
PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
isPlatformLeadCapturedOutboxJob,
parsePlatformLeadCapturedOutboxJob,
runPlatformLeadCapturedOutboxJob,
} from "./platform-lead-captured-outbox.js";
function buildJob(overrides: Record<string, unknown> = {}): unknown {
return {
outboxEventId: "activity_1:lead-outbox",
tenantId: null,
eventType: PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
},
dispatchKey: "platform-lead-captured:lead_1",
retries: 0,
deliveryGeneration: 0,
lockedUntil: "2026-07-16T10:21:30.123Z",
...overrides,
};
}
describe("platform lead captured outbox marker", () => {
it("parses the exact tenantless PII-minimized producer envelope", () => {
expect(parsePlatformLeadCapturedOutboxJob(buildJob())).toEqual({
outboxEventId: "activity_1:lead-outbox",
tenantId: null,
eventType: PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
},
});
expect(isPlatformLeadCapturedOutboxJob(buildJob())).toBe(true);
expect(isPlatformLeadCapturedOutboxJob({ eventType: "notification.platform.requested.v1" })).toBe(false);
});
it("rejects tenant scope, PII fields, malformed timestamps and invalid routing", () => {
expect(() => parsePlatformLeadCapturedOutboxJob(buildJob({ tenantId: "tenant_escape" }))).toThrow("require tenantId=null");
expect(() => parsePlatformLeadCapturedOutboxJob(buildJob({ lockedUntil: null }))).toThrow("canonical UTC timestamp");
expect(() => parsePlatformLeadCapturedOutboxJob({ ...(buildJob() as object), email: "lead@example.test" })).toThrow(
"non-allowlisted fields"
);
expect(() =>
parsePlatformLeadCapturedOutboxJob(
buildJob({
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
message: "private lead message",
},
})
)
).toThrow("non-allowlisted fields");
expect(() =>
parsePlatformLeadCapturedOutboxJob(
buildJob({
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16 10:20:30",
routingUserIds: ["user_haris"],
},
})
)
).toThrow("canonical UTC timestamp");
expect(() =>
parsePlatformLeadCapturedOutboxJob(
buildJob({
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris", "user_haris"],
},
})
)
).toThrow("must not contain duplicates");
});
it("acknowledges replay idempotently and never logs routing identifiers or PII", async () => {
const logger = { info: vi.fn() };
const first = await runPlatformLeadCapturedOutboxJob(buildJob(), { logger });
const replay = await runPlatformLeadCapturedOutboxJob(buildJob(), { logger });
expect(first).toEqual({ status: "acknowledged", outboxEventId: "activity_1:lead-outbox", leadId: "lead_1" });
expect(replay).toEqual(first);
expect(logger.info).toHaveBeenCalledTimes(2);
const serializedLogs = JSON.stringify(logger.info.mock.calls);
expect(serializedLogs).not.toContain("user_haris");
expect(serializedLogs).not.toContain("lead@example.test");
});
});
@@ -0,0 +1,151 @@
/**
* Consumer for the PII-minimized platform-lead capture domain marker.
*
* The capture transaction already persists one NotificationDelivery and one
* `notification.platform.requested.v1` outbox event per configured recipient
* and channel. This handler therefore only validates and acknowledges the
* companion domain event. Re-delivery is inherently idempotent because it has
* no downstream side effect.
*/
export const PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE = "platform.admin_lead.captured.v1" as const;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/u;
const UTC_TIMESTAMP_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u;
const ALLOWED_ENVELOPE_KEYS = new Set([
"outboxEventId",
"tenantId",
"eventType",
"payload",
"dispatchKey",
"retries",
"deliveryGeneration",
"lockedUntil",
]);
const ALLOWED_PAYLOAD_KEYS = new Set(["leadId", "sourceId", "eventTime", "routingUserIds"]);
export type PlatformLeadCapturedOutboxJob = {
readonly outboxEventId: string;
readonly tenantId: null;
readonly eventType: typeof PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE;
readonly payload: {
readonly leadId: string;
readonly sourceId: string;
readonly eventTime: string;
readonly routingUserIds: readonly string[];
};
};
export type PlatformLeadCapturedOutboxDependencies = {
readonly logger?: {
info?: (record: Record<string, unknown>, message?: string) => void;
};
};
export class PlatformLeadCapturedOutboxError extends Error {
override readonly name = "PlatformLeadCapturedOutboxError";
readonly code = "INVALID_PAYLOAD" as const;
}
export function isPlatformLeadCapturedOutboxJob(data: unknown): boolean {
return Boolean(
data &&
typeof data === "object" &&
!Array.isArray(data) &&
(data as Record<string, unknown>).eventType === PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE
);
}
export function parsePlatformLeadCapturedOutboxJob(data: unknown): PlatformLeadCapturedOutboxJob {
const envelope = requireRecord(data, "platform lead captured outbox envelope");
assertAllowlistedKeys(envelope, ALLOWED_ENVELOPE_KEYS, "platform lead captured outbox envelope");
if (envelope.tenantId !== null) {
throw new PlatformLeadCapturedOutboxError("Platform lead captured outbox events require tenantId=null.");
}
if (envelope.eventType !== PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE) {
throw new PlatformLeadCapturedOutboxError("Unsupported platform lead captured outbox event type.");
}
requireCanonicalTimestamp(envelope.lockedUntil, "lockedUntil");
const payload = requireRecord(envelope.payload, "platform lead captured outbox payload");
assertAllowlistedKeys(payload, ALLOWED_PAYLOAD_KEYS, "platform lead captured outbox payload");
const eventTime = typeof payload.eventTime === "string" ? payload.eventTime : "";
if (!UTC_TIMESTAMP_PATTERN.test(eventTime) || Number.isNaN(Date.parse(eventTime)) || new Date(eventTime).toISOString() !== eventTime) {
throw new PlatformLeadCapturedOutboxError("eventTime must be a canonical UTC timestamp.");
}
if (!Array.isArray(payload.routingUserIds) || payload.routingUserIds.length === 0 || payload.routingUserIds.length > 20) {
throw new PlatformLeadCapturedOutboxError("routingUserIds must contain between one and twenty operator identifiers.");
}
const routingUserIds = payload.routingUserIds.map((value) => requireIdentifier(value, "routingUserId"));
if (new Set(routingUserIds).size !== routingUserIds.length) {
throw new PlatformLeadCapturedOutboxError("routingUserIds must not contain duplicates.");
}
return {
outboxEventId: requireIdentifier(envelope.outboxEventId, "outboxEventId"),
tenantId: null,
eventType: PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
payload: {
leadId: requireIdentifier(payload.leadId, "leadId"),
sourceId: requireIdentifier(payload.sourceId, "sourceId"),
eventTime,
routingUserIds,
},
};
}
export async function runPlatformLeadCapturedOutboxJob(
data: unknown,
deps: PlatformLeadCapturedOutboxDependencies = {}
): Promise<{ status: "acknowledged"; outboxEventId: string; leadId: string }> {
const job = parsePlatformLeadCapturedOutboxJob(data);
deps.logger?.info?.(
{
component: "platform-lead-captured-outbox",
outboxEventId: job.outboxEventId,
eventType: job.eventType,
leadId: job.payload.leadId,
sourceId: job.payload.sourceId,
routingRecipientCount: job.payload.routingUserIds.length,
},
"platform lead capture marker acknowledged"
);
return { status: "acknowledged", outboxEventId: job.outboxEventId, leadId: job.payload.leadId };
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new PlatformLeadCapturedOutboxError(`${label} must be an object.`);
}
return value as Record<string, unknown>;
}
function requireIdentifier(value: unknown, field: string): string {
const normalized = typeof value === "string" ? value.trim() : "";
if (!IDENTIFIER_PATTERN.test(normalized)) {
throw new PlatformLeadCapturedOutboxError(`${field} must be a stable non-secret identifier.`);
}
return normalized;
}
function requireCanonicalTimestamp(value: unknown, field: string): string {
if (typeof value !== "string") {
throw new PlatformLeadCapturedOutboxError(`${field} must be a canonical UTC timestamp.`);
}
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime()) || parsed.toISOString() !== value) {
throw new PlatformLeadCapturedOutboxError(`${field} must be a canonical UTC timestamp.`);
}
return value;
}
function assertAllowlistedKeys(record: Record<string, unknown>, allowlist: ReadonlySet<string>, label: string): void {
if (Object.keys(record).some((key) => !allowlist.has(key))) {
throw new PlatformLeadCapturedOutboxError(`${label} contains non-allowlisted fields.`);
}
}
@@ -0,0 +1,283 @@
import { describe, expect, it, vi } from "vitest";
import { buildPlatformNotificationJob, buildRetryJobOptions, type NotificationJobDescriptor, type QueueEnqueueOptions } from "@hivegrid/queue";
import type { MailProvider } from "@hivegrid/mail";
import { buildHandlerRegistry } from "../handlers.js";
import {
PLATFORM_NOTIFICATION_OUTBOX_TYPE,
isPlatformNotificationOutboxJob,
parsePlatformNotificationOutboxJob,
runPlatformNotificationOutboxJob,
} from "./platform-notification-outbox.js";
function buildJob(overrides: Record<string, unknown> = {}): unknown {
return {
outboxEventId: "outbox_1",
tenantId: null,
eventType: PLATFORM_NOTIFICATION_OUTBOX_TYPE,
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
metadata: {
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
},
},
...overrides,
};
}
describe("platform notification outbox parser", () => {
it("parses the strict tenantless PII-free envelope", () => {
const parsed = parsePlatformNotificationOutboxJob(buildJob());
expect(parsed).toEqual({
outboxEventId: "outbox_1",
tenantId: null,
eventType: PLATFORM_NOTIFICATION_OUTBOX_TYPE,
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
metadata: {
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
},
},
});
expect(isPlatformNotificationOutboxJob(buildJob())).toBe(true);
expect(isPlatformNotificationOutboxJob({ eventType: "billing.subscription.ended" })).toBe(false);
});
it("rejects tenant scope and non-allowlisted PII fields", () => {
expect(() => parsePlatformNotificationOutboxJob(buildJob({ tenantId: "tenant_escape" }))).toThrow("require tenantId=null");
expect(() =>
parsePlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
email: "lead@example.test",
},
})
)
).toThrow("non-allowlisted fields");
expect(() =>
parsePlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1", message: "private lead text" },
},
})
)
).toThrow("non-allowlisted fields");
});
});
describe("platform notification outbox bridge", () => {
it("enqueues a deterministic retryable worker job without recipient PII", async () => {
const enqueue = vi.fn(async () => undefined);
const logger = { info: vi.fn() };
const result = await runPlatformNotificationOutboxJob(buildJob(), { enqueue, logger });
expect(result).toEqual({ status: "enqueued", outboxEventId: "outbox_1", deliveryId: "delivery_1", channel: "email" });
expect(enqueue).toHaveBeenCalledWith(
expect.objectContaining({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
subject: "Neuer Lead in HiveGrid",
text: expect.stringContaining("Ein neuer Lead ist eingegangen"),
html: expect.stringContaining("https://hivegrid.de/hivegrid-logo.png"),
metadata: {
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
deliveryId: "delivery_1",
eventId: "outbox_1",
},
}),
{ tenantId: null, idempotencyKey: "platform-notification-outbox:outbox_1" }
);
expect(JSON.stringify(enqueue.mock.calls)).not.toContain("@example.test");
expect(JSON.stringify(logger.info.mock.calls)).not.toContain("haris@example.test");
});
it("keeps in-app delivery PII-free as well", async () => {
const enqueue = vi.fn(async () => undefined);
const job = buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
},
});
await runPlatformNotificationOutboxJob(job, { enqueue });
expect(enqueue).toHaveBeenCalledWith(expect.not.objectContaining({ email: expect.anything() }), expect.objectContaining({ tenantId: null }));
});
it("bridges an admin sales task reminder without requiring lead metadata", async () => {
const enqueue = vi.fn(async () => undefined);
const job = buildJob({
payload: {
deliveryId: "delivery_task_1",
adminUserId: "admin_1",
sourceKey: "admin-sales:task-reminder:task_1:2026-07-25",
channel: "in-app",
messageKey: "admin.sales.notifications.taskreminder",
category: "sales",
metadata: {
taskId: "task_1",
deeplink: "/sales/tasks/today",
},
},
});
await runPlatformNotificationOutboxJob(job, { enqueue });
expect(enqueue).toHaveBeenCalledWith(
expect.objectContaining({
adminUserId: "admin_1",
messageKey: "admin.sales.notifications.taskreminder",
subject: "Aufgabe ist bald fällig",
metadata: expect.objectContaining({ taskId: "task_1" }),
}),
{ tenantId: null, idempotencyKey: "platform-notification-outbox:outbox_1" }
);
});
it("bridges an opted-in live-open event to the separated admin identity", async () => {
const enqueue = vi.fn(async () => undefined);
const job = buildJob({
payload: {
deliveryId: "delivery_open_1",
adminUserId: "admin_1",
sourceKey: "admin-email-open:event_1",
channel: "in-app",
messageKey: "admin.sales.email.notifications.opened",
category: "admin-sales-email",
metadata: {
emailMessageId: "message_1",
emailEventId: "event_1",
deeplink: "/sales/email",
},
},
});
await runPlatformNotificationOutboxJob(job, { enqueue });
expect(enqueue).toHaveBeenCalledWith(
expect.objectContaining({
adminUserId: "admin_1",
messageKey: "admin.sales.email.notifications.opened",
metadata: expect.objectContaining({
emailMessageId: "message_1",
emailEventId: "event_1",
}),
}),
{
tenantId: null,
idempotencyKey: "platform-notification-outbox:outbox_1",
}
);
});
it("fails closed when a task reminder has no task id", async () => {
const enqueue = vi.fn(async () => undefined);
await expect(
runPlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_task_1",
userId: "admin_1",
sourceKey: "admin-sales:task-reminder:task_1:2026-07-25",
channel: "in-app",
messageKey: "admin.sales.notifications.taskreminder",
},
}),
{ enqueue }
)
).rejects.toMatchObject({ code: "INVALID_PAYLOAD" });
expect(enqueue).not.toHaveBeenCalled();
});
it("fails closed when a platform notification has no lead id", async () => {
const enqueue = vi.fn(async () => undefined);
await expect(
runPlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
},
}),
{ enqueue }
)
).rejects.toMatchObject({ code: "INVALID_PAYLOAD" });
expect(enqueue).not.toHaveBeenCalled();
});
it("hands the durable outbox request to the existing retryable notification worker", async () => {
let queued: { descriptor: NotificationJobDescriptor; options: QueueEnqueueOptions } | undefined;
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "mailjet_1" })),
};
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
await runPlatformNotificationOutboxJob(buildJob(), {
enqueue: async (input, options) => {
queued = { descriptor: buildPlatformNotificationJob(input), options };
},
});
expect(queued).toBeDefined();
expect(buildRetryJobOptions(queued!.descriptor)).toMatchObject({ attempts: 3, backoff: { type: "exponential", delay: 1_000 } });
expect(queued!.options).toEqual({ tenantId: null, idempotencyKey: "platform-notification-outbox:outbox_1" });
const registry = buildHandlerRegistry({
mailProvider,
createPlatformNotificationDelivery: vi.fn(async () => ({ id: "delivery_1", status: "PENDING" })),
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient: vi.fn(async () => ({ email: "operator@example.test" })),
});
await expect(registry[queued!.descriptor.queueName](queued!.descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toMatchObject({
action: "notification.platform.email.sent",
details: { deliveryId: "delivery_1" },
});
expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith(
expect.objectContaining({ id: "delivery_1", userId: "user_haris", status: "SENT", attemptsMade: 1 })
);
expect(JSON.stringify(markPlatformNotificationDeliveryStatus.mock.calls)).not.toContain("operator@example.test");
});
});
@@ -0,0 +1,262 @@
import type { PlatformNotificationJobInput, PlatformNotificationSafeMetadata, QueueEnqueueOptions } from "@hivegrid/queue";
import { renderSystemMessageEmail } from "@hivegrid/mail";
export const PLATFORM_NOTIFICATION_OUTBOX_TYPE = "notification.platform.requested.v1" as const;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/u;
const MESSAGE_KEY_PATTERN = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/u;
const DEEPLINK_PATTERN = /^\/[A-Za-z0-9/_-]{1,239}$/u;
const ALLOWED_PAYLOAD_KEYS = new Set(["deliveryId", "userId", "adminUserId", "sourceKey", "channel", "messageKey", "category", "metadata"]);
const ALLOWED_METADATA_KEYS = new Set([
"eventId",
"eventType",
"leadId",
"taskId",
"emailMessageId",
"emailEventId",
"requestId",
"sourceId",
"deeplink",
"replay",
]);
export type PlatformNotificationOutboxJob = {
outboxEventId: string;
tenantId: null;
eventType: typeof PLATFORM_NOTIFICATION_OUTBOX_TYPE;
payload: {
deliveryId: string;
userId?: string;
adminUserId?: string;
sourceKey: string;
channel: "email" | "in-app";
messageKey: string;
category?: string;
metadata?: PlatformNotificationSafeMetadata;
};
};
export type PlatformNotificationOutboxDependencies = {
enqueue(input: PlatformNotificationJobInput, options: QueueEnqueueOptions): Promise<void>;
logger?: {
info?: (record: Record<string, unknown>, message?: string) => void;
};
};
export class PlatformNotificationOutboxError extends Error {
override readonly name = "PlatformNotificationOutboxError";
constructor(
message: string,
readonly code: "INVALID_PAYLOAD" | "UNSUPPORTED_MESSAGE"
) {
super(message);
}
}
export function isPlatformNotificationOutboxJob(data: unknown): boolean {
return Boolean(data && typeof data === "object" && !Array.isArray(data) && (data as Record<string, unknown>).eventType === PLATFORM_NOTIFICATION_OUTBOX_TYPE);
}
export function parsePlatformNotificationOutboxJob(data: unknown): PlatformNotificationOutboxJob {
const envelope = requireRecord(data, "platform notification outbox envelope");
const outboxEventId = requireIdentifier(envelope.outboxEventId, "outboxEventId");
if (envelope.tenantId !== null) {
throw new PlatformNotificationOutboxError("Platform notification outbox events require tenantId=null.", "INVALID_PAYLOAD");
}
if (envelope.eventType !== PLATFORM_NOTIFICATION_OUTBOX_TYPE) {
throw new PlatformNotificationOutboxError("Unsupported platform notification outbox event type.", "INVALID_PAYLOAD");
}
const payload = requireRecord(envelope.payload, "platform notification outbox payload");
assertAllowlistedKeys(payload, ALLOWED_PAYLOAD_KEYS, "platform notification outbox payload");
const channel = payload.channel;
if (channel !== "email" && channel !== "in-app") {
throw new PlatformNotificationOutboxError("Platform notification outbox payload has an unsupported channel.", "INVALID_PAYLOAD");
}
const messageKey = typeof payload.messageKey === "string" ? payload.messageKey.trim() : "";
if (!MESSAGE_KEY_PATTERN.test(messageKey)) {
throw new PlatformNotificationOutboxError("Platform notification outbox payload has an invalid messageKey.", "INVALID_PAYLOAD");
}
const category = typeof payload.category === "string" ? payload.category.trim().toLowerCase() : undefined;
if (category !== undefined && !/^[a-z][a-z0-9._-]{0,63}$/u.test(category)) {
throw new PlatformNotificationOutboxError("Platform notification outbox payload has an invalid category.", "INVALID_PAYLOAD");
}
const userId = typeof payload.userId === "string" ? requireIdentifier(payload.userId, "userId") : undefined;
const adminUserId = typeof payload.adminUserId === "string" ? requireIdentifier(payload.adminUserId, "adminUserId") : undefined;
if (Boolean(userId) === Boolean(adminUserId)) {
throw new PlatformNotificationOutboxError("Platform notification outbox payload requires exactly one recipient identity.", "INVALID_PAYLOAD");
}
return {
outboxEventId,
tenantId: null,
eventType: PLATFORM_NOTIFICATION_OUTBOX_TYPE,
payload: {
deliveryId: requireIdentifier(payload.deliveryId, "deliveryId"),
...(userId ? { userId } : { adminUserId }),
sourceKey: requireIdentifier(payload.sourceKey, "sourceKey"),
channel,
messageKey,
...(category ? { category } : {}),
...(payload.metadata === undefined ? {} : { metadata: parseSafeMetadata(payload.metadata) }),
},
};
}
export async function runPlatformNotificationOutboxJob(
data: unknown,
deps: PlatformNotificationOutboxDependencies
): Promise<{ status: "enqueued"; outboxEventId: string; deliveryId: string; channel: "email" | "in-app" }> {
const job = parsePlatformNotificationOutboxJob(data);
const copy = resolveMessageCopy(job.payload.messageKey);
const rendered =
job.payload.channel === "email" && copy.subject && copy.text
? renderSystemMessageEmail({
locale: "de",
subject: copy.subject,
previewText: copy.text,
eyebrow: "HiveGrid-Benachrichtigung",
title: copy.subject,
intro: copy.text,
notice: "Öffne HiveGrid, um alle Details zu sehen und direkt weiterzuarbeiten.",
})
: undefined;
if (
(job.payload.messageKey === "platform.leads.captured" || job.payload.messageKey === "admin.sales.notifications.leadassigned") &&
!job.payload.metadata?.leadId
) {
throw new PlatformNotificationOutboxError("Platform lead notifications require a leadId.", "INVALID_PAYLOAD");
}
if (job.payload.messageKey === "admin.sales.notifications.taskreminder" && !job.payload.metadata?.taskId) {
throw new PlatformNotificationOutboxError("Admin sales task notifications require a taskId.", "INVALID_PAYLOAD");
}
if (job.payload.messageKey === "admin.sales.email.notifications.opened" && !job.payload.metadata?.emailMessageId) {
throw new PlatformNotificationOutboxError("Admin email open notifications require an emailMessageId.", "INVALID_PAYLOAD");
}
await deps.enqueue(
{
...(job.payload.userId ? { userId: job.payload.userId } : { adminUserId: job.payload.adminUserId }),
sourceKey: job.payload.sourceKey,
channel: job.payload.channel,
messageKey: job.payload.messageKey,
...(job.payload.category ? { category: job.payload.category } : {}),
...(copy.subject ? { subject: copy.subject } : {}),
...(rendered ? { text: rendered.text, html: rendered.html } : copy.text ? { text: copy.text } : {}),
metadata: {
...job.payload.metadata,
deliveryId: job.payload.deliveryId,
eventId: job.outboxEventId,
},
},
{
tenantId: null,
idempotencyKey: `platform-notification-outbox:${job.outboxEventId}`,
}
);
deps.logger?.info?.(
{
component: "platform-notification-outbox",
outboxEventId: job.outboxEventId,
deliveryId: job.payload.deliveryId,
...(job.payload.userId ? { userId: job.payload.userId } : { adminUserId: job.payload.adminUserId }),
channel: job.payload.channel,
messageKey: job.payload.messageKey,
},
"platform notification enqueued from outbox"
);
return {
status: "enqueued",
outboxEventId: job.outboxEventId,
deliveryId: job.payload.deliveryId,
channel: job.payload.channel,
};
}
function resolveMessageCopy(messageKey: string): { subject?: string; text?: string } {
if (messageKey === "platform.leads.captured") {
return {
subject: "Neuer Lead in HiveGrid",
text: "Ein neuer Lead ist eingegangen. Bitte öffne die Lead-Inbox in HiveGrid.",
};
}
if (messageKey === "admin.sales.notifications.taskreminder") {
return {
subject: "Aufgabe ist bald fällig",
text: "Eine Aufgabe in deinem HiveGrid Sales Workspace ist bald fällig.",
};
}
if (messageKey === "admin.sales.notifications.leadassigned") {
return {
subject: "Lead wurde dir zugewiesen",
text: "Ein Lead wurde dir im HiveGrid Sales Workspace zugewiesen.",
};
}
if (messageKey === "admin.sales.email.notifications.opened") {
return {
subject: "E-Mail wurde geöffnet",
text: "Ein Interessent hat gerade eine nachverfolgbare E-Mail geöffnet.",
};
}
throw new PlatformNotificationOutboxError(`Unsupported platform notification messageKey ${messageKey}.`, "UNSUPPORTED_MESSAGE");
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new PlatformNotificationOutboxError(`${label} must be an object.`, "INVALID_PAYLOAD");
}
return value as Record<string, unknown>;
}
function requireIdentifier(value: unknown, field: string): string {
const normalized = typeof value === "string" ? value.trim() : "";
if (!IDENTIFIER_PATTERN.test(normalized)) {
throw new PlatformNotificationOutboxError(`${field} must be a stable non-secret identifier.`, "INVALID_PAYLOAD");
}
return normalized;
}
function assertAllowlistedKeys(record: Record<string, unknown>, allowlist: ReadonlySet<string>, label: string): void {
const unexpected = Object.keys(record).filter((key) => !allowlist.has(key));
if (unexpected.length > 0) {
throw new PlatformNotificationOutboxError(`${label} contains non-allowlisted fields.`, "INVALID_PAYLOAD");
}
}
function parseSafeMetadata(value: unknown): PlatformNotificationSafeMetadata {
const metadata = requireRecord(value, "platform notification outbox metadata");
assertAllowlistedKeys(metadata, ALLOWED_METADATA_KEYS, "platform notification outbox metadata");
const parsed: PlatformNotificationSafeMetadata = {};
for (const key of ["eventId", "leadId", "taskId", "emailMessageId", "emailEventId", "requestId", "sourceId"] as const) {
if (metadata[key] !== undefined) {
parsed[key] = requireIdentifier(metadata[key], key);
}
}
if (metadata.eventType !== undefined) {
const eventType = typeof metadata.eventType === "string" ? metadata.eventType.trim() : "";
if (!/^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$/u.test(eventType)) {
throw new PlatformNotificationOutboxError("eventType must be a safe event identifier.", "INVALID_PAYLOAD");
}
parsed.eventType = eventType;
}
if (metadata.deeplink !== undefined) {
const deeplink = typeof metadata.deeplink === "string" ? metadata.deeplink.trim() : "";
if (!DEEPLINK_PATTERN.test(deeplink)) {
throw new PlatformNotificationOutboxError("deeplink must be an app-relative PII-free path.", "INVALID_PAYLOAD");
}
parsed.deeplink = deeplink;
}
if (metadata.replay !== undefined) {
if (typeof metadata.replay !== "boolean") {
throw new PlatformNotificationOutboxError("replay must be boolean.", "INVALID_PAYLOAD");
}
parsed.replay = metadata.replay;
}
return parsed;
}
Loaded 100 of 5955 files, more files were not shown because too many files have changed in this diff. Show more