Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -0,0 +1,134 @@
import type { EnvRecord } from "@hivegrid/config";
import {
claimAdminCallRecordings,
claimExpiredAdminCallRecordings,
createPrismaAdminTelephonyRecordingRepository,
markAdminCallRecordingDeleted,
} from "@hivegrid/db/admin-telephony-recordings.server";
import type { DbClient } from "@hivegrid/db/client.server";
import { ADMIN_CONNECTION_MASTER_KEY_ENV } from "@hivegrid/db/crypto/admin-connection-envelope.server";
import { EnvKeyWrapper } from "@hivegrid/foundation/crypto";
import {
createAdminCallRecordingProcessor,
type AdminCallRecordingStorage,
type AdminCallRecordingScanner,
} from "@hivegrid/integrations/admin-call-recordings.server";
import { createClamAVBufferScanner } from "@hivegrid/storage";
import { resolveWorkerClamdConfig } from "./scanService.js";
import {
createWorkerStorageDeleteService,
getMissingWorkerStorageEnv,
} from "./storage.js";
type AdminCallRecordingWorkerEnv = EnvRecord & {
CLAMD_HOST?: string;
CLAMD_PORT?: string;
CLAMD_TIMEOUT_MS?: string;
CLAMAV_HOST?: string;
CLAMAV_PORT?: string;
CLAMAV_TIMEOUT_MS?: string;
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_BUCKET?: string;
STORAGE_ENDPOINT?: string;
STORAGE_FORCE_PATH_STYLE?: string;
STORAGE_PUBLIC_BASE_URL?: string;
STORAGE_REGION?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
};
export async function runAdminCallRecordingProcessing(input: {
readonly db: DbClient;
readonly env: AdminCallRecordingWorkerEnv;
readonly take?: number;
readonly scanner?: AdminCallRecordingScanner;
readonly storage?: AdminCallRecordingStorage;
}): Promise<{
readonly claimed: number;
readonly processed: number;
}> {
const recordings = await claimAdminCallRecordings(
input.db,
input.take ?? 10
);
if (recordings.length === 0) {
return { claimed: 0, processed: 0 };
}
const storage =
input.storage ?? requireRecordingStorage(input.env);
const scanner =
input.scanner ??
createClamAVBufferScanner(resolveWorkerClamdConfig(input.env));
const repository =
createPrismaAdminTelephonyRecordingRepository(input.db);
const processor = createAdminCallRecordingProcessor({
repository,
scanner,
storage,
keyWrapper: new EnvKeyWrapper(ADMIN_CONNECTION_MASTER_KEY_ENV),
});
for (const recording of recordings) {
await processor.process(recording);
}
return {
claimed: recordings.length,
processed: recordings.length,
};
}
export async function runAdminCallRecordingRetention(input: {
readonly db: DbClient;
readonly env: AdminCallRecordingWorkerEnv;
readonly now?: Date;
readonly take?: number;
readonly storage?: AdminCallRecordingStorage;
}): Promise<{
readonly claimed: number;
readonly deleted: number;
readonly failed: number;
}> {
const now = input.now ?? new Date();
const recordings = await claimExpiredAdminCallRecordings(
input.db,
now,
input.take ?? 50
);
if (recordings.length === 0) {
return { claimed: 0, deleted: 0, failed: 0 };
}
const storage =
input.storage ?? requireRecordingStorage(input.env);
let deleted = 0;
let failed = 0;
for (const recording of recordings) {
try {
if (recording.storageKey) {
await storage.deleteObject({
bucket: recording.storageBucket ?? undefined,
key: recording.storageKey,
});
}
await markAdminCallRecordingDeleted(input.db, {
id: recording.id,
now,
});
deleted += 1;
} catch {
failed += 1;
}
}
return { claimed: recordings.length, deleted, failed };
}
function requireRecordingStorage(
env: AdminCallRecordingWorkerEnv
): AdminCallRecordingStorage {
const missing = getMissingWorkerStorageEnv(env);
if (missing.length > 0) {
throw new Error(
`Admin recording storage is missing required env: ${missing.join(", ")}`
);
}
return createWorkerStorageDeleteService(env);
}
@@ -0,0 +1,37 @@
import type { EnvRecord } from "@hivegrid/config";
import { createAdminEmailAttachmentProcessor } from "@hivegrid/integrations/admin-email-attachments.server";
import { createClamAVBufferScanner } from "@hivegrid/storage";
import { resolveWorkerClamdConfig } from "./scanService.js";
import {
createWorkerStorageDeleteService,
getMissingWorkerStorageEnv,
} from "./storage.js";
type AdminEmailAttachmentEnv = EnvRecord & {
CLAMD_HOST?: string;
CLAMD_PORT?: string;
CLAMD_TIMEOUT_MS?: string;
CLAMAV_HOST?: string;
CLAMAV_PORT?: string;
CLAMAV_TIMEOUT_MS?: string;
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_BUCKET?: string;
STORAGE_ENDPOINT?: string;
STORAGE_FORCE_PATH_STYLE?: string;
STORAGE_PUBLIC_BASE_URL?: string;
STORAGE_REGION?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
};
export function createWorkerAdminEmailAttachmentProcessor(
env: AdminEmailAttachmentEnv
) {
const storage =
getMissingWorkerStorageEnv(env).length === 0
? createWorkerStorageDeleteService(env)
: undefined;
const clamd = resolveWorkerClamdConfig(env);
const scanner = createClamAVBufferScanner(clamd);
return createAdminEmailAttachmentProcessor({ scanner, storage });
}
@@ -0,0 +1,9 @@
import type { DbClient } from "@hivegrid/db/client.server";
import { processAdminOutboundCallRequests } from "@hivegrid/integrations/admin-outbound-calls.server";
export async function runAdminOutboundCalls(input: {
readonly db: DbClient;
readonly take?: number;
}) {
return processAdminOutboundCallRequests(input);
}
@@ -25,7 +25,9 @@ export type WorkerStatusReport = {
workflowTrigger?: WorkerStatusCheckOutcome;
pluginHookFanout?: WorkerStatusCheckOutcome;
transactionalMailOutbox?: WorkerStatusCheckOutcome;
portalStatusMail?: WorkerStatusCheckOutcome;
emailSequences?: WorkerStatusCheckOutcome;
chatLinkPreview?: WorkerStatusCheckOutcome;
};
// Resolved WORKFLOW_TRIGGER_EXECUTOR_MODE for this worker. Only present when the
// workflow-trigger subscriber runs on this worker (i.e. the check is not "disabled"),
@@ -0,0 +1,13 @@
import { readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
const source = readFileSync(path.join(path.dirname(fileURLToPath(import.meta.url)), "handlers.ts"), "utf8");
describe("billing activity summary i18n contract", () => {
it("persists a semantic summary key when dunning freezes a tenant", () => {
expect(source).toContain('summaryText: "activity.runtime.billing.statusChanged"');
expect(source).not.toContain('summaryText: "Billing-Status auf frozen gesetzt"');
});
});
@@ -0,0 +1,184 @@
import { describe, expect, test, vi } from "vitest";
import { runWorkerDeployHealthCli } from "./deploy-health.js";
function createIo() {
let stderr = "";
let stdout = "";
return {
io: {
stderr: {
write(chunk: string) {
stderr += chunk;
return true;
},
},
stdout: {
write(chunk: string) {
stdout += chunk;
return true;
},
},
},
readStderr: () => stderr,
readStdout: () => stdout,
};
}
function readyResponse(group = "general", overrides: Record<string, unknown> = {}) {
return {
status: 200,
json: vi.fn(async () => ({
status: "ready",
workerGroup: group,
checks: { redis: "ok", database: "ok" },
...overrides,
})),
};
}
function enabledEnv(revision: string, group = "general") {
return {
APP_REVISION: revision,
WORKER_ENABLED: "true",
WORKER_HEALTH_PORT: "9101",
WORKER_QUEUE_GROUP: group,
};
}
describe("worker deployment health", () => {
test("probes the live worker readiness endpoint for the exact baked revision and group", async () => {
const revision = "a".repeat(40);
const fetchReadiness = vi.fn(async () => readyResponse());
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "general", enabledEnv(revision), io, {
fetchReadiness,
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(0);
expect(JSON.parse(readStdout())).toEqual({ status: "ready", revision, workerGroup: "general" });
expect(readStderr()).toBe("");
expect(fetchReadiness).toHaveBeenCalledWith(
"http://127.0.0.1:9101/readyz",
expect.objectContaining({ headers: { Accept: "application/json" }, signal: expect.any(AbortSignal) })
);
});
test("rejects an environment override that disagrees with the baked image revision", async () => {
const expectedRevision = "b".repeat(40);
const fetchReadiness = vi.fn();
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(expectedRevision, "general", enabledEnv(expectedRevision), io, {
fetchReadiness,
readImageRevision: () => "a".repeat(40),
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/revision does not match/);
expect(readStderr()).not.toContain(expectedRevision);
expect(fetchReadiness).not.toHaveBeenCalled();
});
test("rejects a disabled worker before probing the runtime", async () => {
const revision = "a".repeat(40);
const fetchReadiness = vi.fn();
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(
revision,
"general",
{ ...enabledEnv(revision), WORKER_ENABLED: "false" },
io,
{ fetchReadiness, readImageRevision: () => revision }
);
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/processing to be enabled/);
expect(fetchReadiness).not.toHaveBeenCalled();
});
test("rejects a resource assigned to the wrong queue group", async () => {
const revision = "a".repeat(40);
const fetchReadiness = vi.fn();
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "mail", enabledEnv(revision, "general"), io, {
fetchReadiness,
readImageRevision: () => revision,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/group does not match/);
expect(readStderr()).not.toContain("mail");
expect(readStderr()).not.toContain("general");
expect(fetchReadiness).not.toHaveBeenCalled();
});
test("rejects a readiness response for a different runtime group", async () => {
const revision = "a".repeat(40);
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "general", enabledEnv(revision), io, {
fetchReadiness: vi.fn(async () => readyResponse("mail")),
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/response does not match/);
});
test("rejects unhealthy, malformed, or unreachable readiness responses", async () => {
const revision = "a".repeat(40);
const scenarios = [
vi.fn(async () => ({ status: 503, json: vi.fn() })),
vi.fn(async () => readyResponse("general", { checks: { redis: "ok", database: "error" } })),
vi.fn(async () => readyResponse("general", { checks: { first: "ok", second: "ok" } })),
vi.fn(async () => readyResponse("general", { checks: ["ok", "ok"] })),
vi.fn(async () => {
throw new Error("connection failed with secret details");
}),
];
for (const fetchReadiness of scenarios) {
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(revision, "general", enabledEnv(revision), io, {
fetchReadiness,
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).not.toContain("secret details");
}
});
test("rejects malformed revision, group, and health port arguments", async () => {
const revision = "a".repeat(40);
const cases = [
["not-a-sha", "general", enabledEnv(revision)],
[revision, "unknown-group", enabledEnv(revision)],
[revision, "general", { ...enabledEnv(revision), WORKER_HEALTH_PORT: "70000" }],
] as const;
for (const [expectedRevision, expectedGroup, env] of cases) {
const { io, readStderr, readStdout } = createIo();
const exitCode = await runWorkerDeployHealthCli(expectedRevision, expectedGroup, env, io, {
readImageRevision: () => revision,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).not.toContain("not-a-sha");
expect(readStderr()).not.toContain("unknown-group");
}
});
});
@@ -0,0 +1,102 @@
import { readFileSync } from "node:fs";
import { isWorkerQueueGroup } from "@hivegrid/queue";
const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/;
const IMAGE_REVISION_PATH = "/myapp/.image-revision";
const DEFAULT_HEALTH_PORT = "9100";
const HEALTH_TIMEOUT_MS = 9_000;
type WorkerDeployHealthIo = {
stderr?: Pick<NodeJS.WriteStream, "write">;
stdout: Pick<NodeJS.WriteStream, "write">;
};
type WorkerReadinessResponse = {
status?: unknown;
workerGroup?: unknown;
checks?: unknown;
};
type WorkerDeployHealthDependencies = {
fetchReadiness?: (url: string, init: RequestInit) => Promise<Pick<Response, "status" | "json">>;
readImageRevision?: () => string;
timeoutSignal?: () => AbortSignal;
};
function fail(io: WorkerDeployHealthIo, message: string): number {
io.stderr?.write(`${message}\n`);
return 1;
}
export async function runWorkerDeployHealthCli(
expectedRevision: string,
expectedGroup: string,
env: NodeJS.ProcessEnv = process.env,
io: WorkerDeployHealthIo = process,
dependencies: WorkerDeployHealthDependencies = {}
): Promise<number> {
if (!FULL_GIT_SHA_PATTERN.test(expectedRevision)) {
return fail(io, "Worker deployment health requires a full lowercase Git SHA.");
}
if (!isWorkerQueueGroup(expectedGroup)) {
return fail(io, "Worker deployment health requires a valid worker group.");
}
let imageRevision = "";
try {
imageRevision = (dependencies.readImageRevision ?? (() => readFileSync(IMAGE_REVISION_PATH, "utf8")))().trim();
} catch {
return fail(io, "Worker deployment image revision is unavailable.");
}
if (imageRevision !== expectedRevision || env.APP_REVISION !== expectedRevision) {
return fail(io, "Worker deployment revision does not match the expected image.");
}
if (env.WORKER_ENABLED !== "true") {
return fail(io, "Worker deployment health requires worker processing to be enabled.");
}
if (env.WORKER_QUEUE_GROUP !== expectedGroup) {
return fail(io, "Worker deployment group does not match the expected resource.");
}
const port = env.WORKER_HEALTH_PORT || DEFAULT_HEALTH_PORT;
if (!/^\d{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65_535) {
return fail(io, "Worker deployment health port is invalid.");
}
try {
const fetchReadiness = dependencies.fetchReadiness ?? fetch;
const response = await fetchReadiness(`http://127.0.0.1:${port}/readyz`, {
headers: { Accept: "application/json" },
signal: (dependencies.timeoutSignal ?? (() => AbortSignal.timeout(HEALTH_TIMEOUT_MS)))(),
});
if (response.status !== 200) {
return fail(io, "Worker deployment readiness is not healthy.");
}
const payload = (await response.json()) as WorkerReadinessResponse;
const checks = payload.checks;
const readinessChecks = typeof checks === "object" && checks !== null && !Array.isArray(checks) ? (checks as Record<string, unknown>) : null;
const allChecksReady =
readinessChecks !== null &&
readinessChecks.database === "ok" &&
readinessChecks.redis === "ok" &&
Object.values(readinessChecks).every((status) => status === "ok");
if (payload.status !== "ready" || payload.workerGroup !== expectedGroup || !allChecksReady) {
return fail(io, "Worker deployment readiness response does not match the expected runtime.");
}
} catch {
return fail(io, "Worker deployment readiness probe failed.");
}
io.stdout.write(`${JSON.stringify({ status: "ready", revision: expectedRevision, workerGroup: expectedGroup })}\n`);
return 0;
}
if (import.meta.url === `file://${process.argv[1]}`) {
runWorkerDeployHealthCli(process.argv[2] || "", process.argv[3] || "")
.then((exitCode) => {
process.exitCode = exitCode;
})
.catch(() => {
process.exitCode = 1;
});
}
@@ -46,8 +46,8 @@ describe("resolveQueueNamesForGroup", () => {
expect(resolveQueueNamesForGroup("general")).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
});
test("import group includes imports + exports", () => {
expect(resolveQueueNamesForGroup("import")).toEqual([queueNames.imports, queueNames.exports]);
test("import group includes dedicated KWP, generic imports, and exports", () => {
expect(resolveQueueNamesForGroup("import")).toEqual([queueNames.kwpImports, queueNames.imports, queueNames.exports]);
});
test("webhook group covers browser automation", () => {
@@ -90,7 +90,7 @@ describe("bootstrapWorkerGroup", () => {
bootstrapWorkerGroup({ WORKER_QUEUE_GROUP: "import", WORKER_CONCURRENCY_IMPORT: "2" })
).toEqual({
group: "import",
queueNames: [queueNames.imports, queueNames.exports],
queueNames: [queueNames.kwpImports, queueNames.imports, queueNames.exports],
concurrency: 2,
concurrencyEnvVar: "WORKER_CONCURRENCY_IMPORT",
});
@@ -0,0 +1,124 @@
// @vitest-environment node
import { beforeEach, describe, expect, test, vi } from "vitest";
const integrationMocks = vi.hoisted(() => ({
cleanupExpiredKwpPendingAttemptsForTenant: vi.fn(),
executeKwpHealthProbe: vi.fn(),
listKwpHealthConnectionIdsForTenant: vi.fn(),
persistKwpHealthProbe: vi.fn(),
prepareKwpHealthProbeForConnection: vi.fn(),
refreshExpiringTokensForTenant: vi.fn(),
sweepConnectionHealthForTenant: vi.fn(),
}));
const kwpResourceMocks = vi.hoisted(() => ({
ensureKwpAccessTokenForConnection: vi.fn(),
}));
const dbMocks = vi.hoisted(() => ({ getDbClient: vi.fn() }));
vi.mock("@hivegrid/integrations/worker-tasks.server", () => integrationMocks);
vi.mock("@hivegrid/integrations/kwp-resource-client.server", () => kwpResourceMocks);
vi.mock("@hivegrid/db/client.server", () => dbMocks);
import { buildCleanupJob, queueNames } from "@hivegrid/queue";
import { createSharedStore, MemoryAdapter } from "@hivegrid/shared-store";
import { buildHandlerRegistry } from "./handlers.js";
describe("KWP demand-driven refresh maintenance", () => {
beforeEach(() => {
vi.clearAllMocks();
integrationMocks.cleanupExpiredKwpPendingAttemptsForTenant.mockResolvedValue({ restored: 0, purged: 0, failed: 0 });
integrationMocks.refreshExpiringTokensForTenant.mockResolvedValue({ refreshed: 2, failed: 0 });
integrationMocks.sweepConnectionHealthForTenant.mockResolvedValue({ checked: 0, unhealthy: 0 });
integrationMocks.listKwpHealthConnectionIdsForTenant.mockResolvedValue([]);
kwpResourceMocks.ensureKwpAccessTokenForConnection.mockResolvedValue(undefined);
});
test("keeps KWP out of the minute refresh sweep while retaining generic refresh and pending cleanup", async () => {
const tx = { $executeRaw: vi.fn(async () => 1) };
const db = {
tenant: { findMany: vi.fn(async () => [{ id: "tenant_1" }]) },
$transaction: vi.fn(async <T>(callback: (transaction: typeof tx) => Promise<T>) => callback(tx)),
};
dbMocks.getDbClient.mockResolvedValue(db);
const logger = { info: vi.fn(), error: vi.fn() };
const registry = buildHandlerRegistry({
env: { KWP_HIVEGRID_TENANT_ID: "tenant_1" },
logger,
tenantMaintenanceLeaseStore: createSharedStore({ namespace: "kwp-demand-refresh-handler", adapter: new MemoryAdapter() }),
});
await expect(
registry[queueNames.maintenance](buildCleanupJob({ task: "integrations-token-refresh", olderThanMinutes: 10 }).payload, {}),
).resolves.toEqual({
handled: true,
action: "integrations.token-refresh.completed",
details: {
task: "integrations-token-refresh",
tenants: 1,
refreshed: 2,
failed: 0,
retryScheduled: 0,
pendingRestored: 0,
pendingPurged: 0,
pendingCleanupFailed: 0,
},
});
expect(integrationMocks.refreshExpiringTokensForTenant).toHaveBeenCalledWith(
tx,
"tenant_1",
{ now: expect.any(Date), windowMs: 10 * 60_000 },
);
expect(integrationMocks.cleanupExpiredKwpPendingAttemptsForTenant).toHaveBeenCalledWith(
tx,
"tenant_1",
{ now: expect.any(Date) },
);
expect(kwpResourceMocks.ensureKwpAccessTokenForConnection).not.toHaveBeenCalled();
});
test("ensures one demand-refreshed token before the scheduled KWP health probe", async () => {
const tx = { $executeRaw: vi.fn(async () => 1) };
const db = {
tenant: { findMany: vi.fn(async () => [{ id: "tenant_1" }]) },
$transaction: vi.fn(async <T>(callback: (transaction: typeof tx) => Promise<T>) => callback(tx)),
};
dbMocks.getDbClient.mockResolvedValue(db);
integrationMocks.listKwpHealthConnectionIdsForTenant.mockResolvedValue(["kwp_connection_1"]);
const snapshot = { tenantId: "tenant_1", connectionId: "kwp_connection_1" };
integrationMocks.prepareKwpHealthProbeForConnection.mockResolvedValue({ state: "ready", snapshot });
integrationMocks.executeKwpHealthProbe.mockResolvedValue({ health: "active", messageKey: "integrations.kwp.health.active" });
integrationMocks.persistKwpHealthProbe.mockResolvedValue({ checked: 1, unhealthy: 0 });
const registry = buildHandlerRegistry({
env: { KWP_HIVEGRID_TENANT_ID: "tenant_1" },
logger: { info: vi.fn(), error: vi.fn() },
tenantMaintenanceLeaseStore: createSharedStore({ namespace: "kwp-demand-health-handler", adapter: new MemoryAdapter() }),
});
await expect(
registry[queueNames.maintenance](buildCleanupJob({ task: "integrations-health-sweep" }).payload, {}),
).resolves.toMatchObject({
action: "integrations.health-sweep.completed",
details: { checked: 1, unhealthy: 0 },
});
expect(kwpResourceMocks.ensureKwpAccessTokenForConnection).toHaveBeenCalledWith({
connectionId: "kwp_connection_1",
allowPaused: true,
});
expect(kwpResourceMocks.ensureKwpAccessTokenForConnection.mock.invocationCallOrder[0]).toBeLessThan(
integrationMocks.prepareKwpHealthProbeForConnection.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER,
);
expect(integrationMocks.prepareKwpHealthProbeForConnection).toHaveBeenCalledWith(
tx,
"tenant_1",
"kwp_connection_1",
{ now: expect.any(Date), kwpHiveGridTenantId: "tenant_1" },
);
expect(integrationMocks.listKwpHealthConnectionIdsForTenant).toHaveBeenCalledWith(
tx,
"tenant_1",
{ now: expect.any(Date) },
);
});
});
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -53,8 +53,8 @@ describe("worker health handler", () => {
test("/readyz returns 200 when both probes succeed", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
workerGroup: "mail",
});
const { res, captured } = createMockResponse();
@@ -68,10 +68,27 @@ describe("worker health handler", () => {
expect(body.checks).toEqual({ redis: "ok", database: "ok" });
});
test("/readyz clears timeout timers after fast probes complete", async () => {
vi.useFakeTimers();
try {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res } = createMockResponse();
await handler(createMockReq("/readyz"), res);
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
}
});
test("/readyz returns 503 when redis probe fails", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: false, error: "ECONNREFUSED" } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: false, error: "ECONNREFUSED" }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res, captured } = createMockResponse();
@@ -86,7 +103,7 @@ describe("worker health handler", () => {
test("/readyz returns 503 when database probe throws", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => {
throw new Error("pg disconnected");
},
@@ -101,11 +118,11 @@ describe("worker health handler", () => {
test("/readyz includes required runtime probes in readiness status", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
readinessProbes: {
outboxDispatcher: async () => ({ ok: true } as const),
workflowTrigger: async () => ({ ok: false, error: "subscriber disabled" } as const),
outboxDispatcher: async () => ({ ok: true }) as const,
workflowTrigger: async () => ({ ok: false, error: "subscriber disabled" }) as const,
},
workerGroup: "general",
});
@@ -129,7 +146,7 @@ describe("worker health handler", () => {
test("/readyz times out a hanging probe", async () => {
const handler = createWorkerHealthHandler({
redisProbe: () => new Promise(() => undefined),
databaseProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true }) as const,
probeTimeoutMs: 20,
});
const { res, captured } = createMockResponse();
@@ -142,8 +159,8 @@ describe("worker health handler", () => {
test("unknown routes return 404 JSON", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res, captured } = createMockResponse();
@@ -155,8 +172,8 @@ describe("worker health handler", () => {
test("/healthz omits workerGroup when none is configured", async () => {
const handler = createWorkerHealthHandler({
redisProbe: async () => ({ ok: true } as const),
databaseProbe: async () => ({ ok: true } as const),
redisProbe: async () => ({ ok: true }) as const,
databaseProbe: async () => ({ ok: true }) as const,
});
const { res, captured } = createMockResponse();
@@ -134,16 +134,21 @@ export function startWorkerHealthServer(options: WorkerHealthServerOptions): Wor
}
async function runWithTimeout(probe: HealthProbe, timeoutMs: number, label: string): Promise<HealthProbeResult> {
let timeout: NodeJS.Timeout | undefined;
try {
const result = await Promise.race([
Promise.resolve().then(() => probe()),
new Promise<HealthProbeResult>((resolve) =>
setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs)
),
new Promise<HealthProbeResult>((resolve) => {
timeout = setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs);
}),
]);
return result;
} catch (error) {
return { ok: false, error: serialiseError(error).message };
} finally {
if (timeout) {
clearTimeout(timeout);
}
}
}
@@ -11,6 +11,16 @@ const mailjetEnv = {
MAILJET_SECRET_KEY: "fixture-mailjet-secret",
MAILJET_FROM_EMAIL: "sender@example.com",
};
const cryptoSecret = "shared-tenant-envelope-secret-value";
const gdprExportSseCKey = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
const storageEnv = {
STORAGE_ACCESS_KEY_ID: "access-key",
STORAGE_BUCKET: "exports",
STORAGE_ENDPOINT: "https://storage.example.com",
STORAGE_REGION: "eu-central-1",
STORAGE_SECRET_ACCESS_KEY: "secret-key",
GDPR_EXPORT_SSE_C_KEY_BASE64: gdprExportSseCKey,
};
function readPackageScripts(): Record<string, string> {
return JSON.parse(readFileSync(packageJsonPath, "utf8")).scripts;
@@ -171,6 +181,7 @@ describe("worker health", () => {
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
CRYPTO_SECRET: cryptoSecret,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "mail",
...mailjetEnv,
@@ -455,6 +466,27 @@ describe("worker health", () => {
);
});
test("reports mail-worker readiness misconfigured when the shared envelope key is absent", async () => {
const databaseProbe = vi.fn();
const redisProbe = vi.fn();
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
...mailjetEnv,
TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true",
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "mail",
},
{ databaseProbe, redisProbe }
);
expect(report.status).toBe("misconfigured");
expect(report.errors).toContain("CRYPTO_SECRET:missing-required:CRYPTO_SECRET is required.");
expect(databaseProbe).not.toHaveBeenCalled();
expect(redisProbe).not.toHaveBeenCalled();
});
test("reports misconfigured and skips DB probe when enabled worker lacks DATABASE_URL", async () => {
const databaseProbe = vi.fn();
const report = await buildWorkerHealthReport(
@@ -518,7 +550,7 @@ describe("worker health", () => {
expect(report.storageStatus).toBe("ready");
});
test("keeps the import worker group ready but idle until import runtime gates are enabled", async () => {
test("keeps the import worker ready with only KWP while generic import gates are disabled", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
@@ -531,11 +563,140 @@ describe("worker health", () => {
);
expect(report.status).toBe("ready");
expect(report.queues).toEqual([]);
expect(report.queues).toEqual([queueNames.kwpImports]);
expect(report.importStatus).toBe("skipped");
expect(report.errors).not.toEqual(expect.arrayContaining([expect.stringMatching(/missing-import/u)]));
});
test("reports the dedicated portal GDPR export mode ready with KWP and exports ownership", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report).toMatchObject({
status: "ready",
importConfigured: true,
importStatus: "ready",
storageConfigured: true,
storageStatus: "ready",
workerGroup: "import",
});
expect(report.queues).toEqual([queueNames.kwpImports, queueNames.exports]);
expect(report.errors).toEqual([]);
});
test("fails dedicated portal GDPR export readiness without complete storage", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.queues).toEqual([queueNames.kwpImports, queueNames.exports]);
expect(report.storageStatus).toBe("misconfigured");
expect(report.errors).toEqual(
expect.arrayContaining([
"STORAGE_ENDPOINT:missing-storage-config:Required when PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true.",
"GDPR_EXPORT_SSE_C_KEY_BASE64:missing-storage-config:Required when PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true.",
])
);
});
test("fails full import readiness without the GDPR SSE-C key", async () => {
const { GDPR_EXPORT_SSE_C_KEY_BASE64: _key, ...storageWithoutKey } = storageEnv;
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
IMPORT_QUEUE_ENABLED: "true",
IMPORT_QUEUE_RUNTIME_READY: "true",
...storageWithoutKey,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.storageStatus).toBe("misconfigured");
expect(report.errors).toEqual(
expect.arrayContaining(["GDPR_EXPORT_SSE_C_KEY_BASE64:missing-storage-config:Required when the full import runtime owns GDPR export jobs."])
);
});
test("reports the full import runtime ready only with encrypted GDPR storage", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
IMPORT_QUEUE_ENABLED: "true",
IMPORT_QUEUE_RUNTIME_READY: "true",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report).toMatchObject({ status: "ready", importStatus: "ready", storageStatus: "ready" });
expect(report.queues).toEqual([queueNames.kwpImports, queueNames.imports, queueNames.exports]);
});
test("fails dedicated portal GDPR export readiness on the wrong worker group", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "general",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.errors).toEqual(
expect.arrayContaining([
"PORTAL_GDPR_EXPORT_QUEUE_ENABLED:invalid-worker-queue-group:PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true requires WORKER_QUEUE_GROUP=import.",
])
);
});
test("fails closed instead of becoming healthy-idle for a whitespace-padded dedicated gate", async () => {
const report = await buildWorkerHealthReport(
{
BULLMQ_REDIS_URL: "redis://localhost:6379/0",
DATABASE_URL: "postgresql://localhost/app",
PORTAL_GDPR_EXPORT_QUEUE_ENABLED: " true ",
...storageEnv,
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "import",
},
{ databaseProbe: vi.fn(), redisProbe: vi.fn() }
);
expect(report.status).toBe("misconfigured");
expect(report.queues).toEqual([]);
expect(report.errors).toEqual(
expect.arrayContaining(["PORTAL_GDPR_EXPORT_QUEUE_ENABLED:invalid-boolean:PORTAL_GDPR_EXPORT_QUEUE_ENABLED must be exactly true or false."])
);
});
test("reports misconfigured when import worker group is partially enabled", async () => {
const report = await buildWorkerHealthReport(
{
@@ -2,8 +2,8 @@ import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type Wor
import { getDbClient } from "@hivegrid/db/client.server";
import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@hivegrid/queue";
import { Queue } from "bullmq";
import { getWorkerQueueNames } from "./queues.js";
import { getMissingWorkerStorageEnv } from "./storage.js";
import { getWorkerQueueNames, resolveImportQueueRuntimeMode } from "./queues.js";
import { getMissingWorkerGdprExportStorageEnv, getMissingWorkerStorageEnv } from "./storage.js";
export type WorkerHealthEnv = EnvRecord;
@@ -69,7 +69,10 @@ export async function buildWorkerHealthReport(
checkRedisReadiness(shouldProbeRedis, config.redis.url, dependencies.redisProbe),
checkDatabaseReadiness(shouldProbeDatabase, dependencies.databaseProbe),
]);
const storageReadiness = checkStorageReadiness(config.enabled && env.EXPORT_RETENTION_QUEUE_ENABLED === "true", env);
const storageReadiness = checkStorageReadiness(
config.enabled && (env.EXPORT_RETENTION_QUEUE_ENABLED === "true" || ["full", "portal-gdpr-export"].includes(resolveImportQueueRuntimeMode(env))),
env
);
const mailReadiness = checkMailReadiness(config.enabled, config.mailjet.configured, isMailProviderRequired(env), env);
const importReadiness = checkImportReadiness(config.enabled && result.ok && getWorkerGroup(env) === "import", env);
const pluginRuntimeReadiness = checkPluginRuntimeReadiness(config.enabled && result.ok, getWorkerGroup(env), env);
@@ -309,10 +312,15 @@ function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { con
const importEnabled = env.IMPORT_QUEUE_ENABLED === "true";
const runtimeReady = env.IMPORT_QUEUE_RUNTIME_READY === "true";
if (!importEnabled && !runtimeReady) {
const mode = resolveImportQueueRuntimeMode(env);
if (mode === "idle") {
return { configured: false, errors: [], status: "skipped" };
}
if (mode === "portal-gdpr-export" || mode === "full") {
return { configured: true, errors: [], status: "ready" };
}
if (!importEnabled || !runtimeReady) {
const errors: string[] = [];
if (!importEnabled) {
@@ -332,7 +340,11 @@ function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { con
};
}
return { configured: true, errors: [], status: "ready" };
return {
configured: false,
errors: ["WORKER_QUEUE_GROUP:invalid-import-runtime-mode:Import worker queue gates do not resolve to an allowed runtime mode."],
status: "misconfigured",
};
}
function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthStorageStatus } {
@@ -340,11 +352,18 @@ function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { co
return { configured: false, errors: [], status: "skipped" };
}
const missing = getMissingWorkerStorageEnv(env);
const importMode = resolveImportQueueRuntimeMode(env);
const ownsGdprExports = importMode === "full" || importMode === "portal-gdpr-export";
const missing = ownsGdprExports ? getMissingWorkerGdprExportStorageEnv(env) : getMissingWorkerStorageEnv(env);
if (missing.length > 0) {
const requiredBy = [
...(env.EXPORT_RETENTION_QUEUE_ENABLED === "true" ? ["EXPORT_RETENTION_QUEUE_ENABLED=true"] : []),
...(importMode === "portal-gdpr-export" ? ["PORTAL_GDPR_EXPORT_QUEUE_ENABLED=true"] : []),
...(importMode === "full" ? ["the full import runtime owns GDPR export jobs"] : []),
].join(" or ");
return {
configured: false,
errors: missing.map((name) => `${name}:missing-storage-config:Required when EXPORT_RETENTION_QUEUE_ENABLED=true.`),
errors: missing.map((name) => `${name}:missing-storage-config:Required when ${requiredBy}.`),
status: "misconfigured",
};
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,109 @@
import { describe, expect, test, vi } from "vitest";
import {
ACTIVE_TENANT_PAGE_SIZE,
runActiveTenantSweep,
TenantSweepCoordinator,
throwOnActiveTenantSweepFailures,
type ActiveTenantReader,
} from "./active-tenant-sweep.js";
function createReader(ids: string[]) {
const findMany = vi.fn(async (args: { take: number; cursor?: { id: string }; skip?: 1 }) => {
const start = args.cursor ? ids.indexOf(args.cursor.id) + (args.skip ?? 0) : 0;
return ids.slice(start, start + args.take).map((id) => ({ id }));
});
return { reader: { tenant: { findMany } } as ActiveTenantReader, findMany };
}
describe("active tenant sweep", () => {
test("keeps active-only as the default and can include deactivated tenants for legal cleanup", async () => {
const activeOnly = createReader(["tenant-active"]);
await runActiveTenantSweep(activeOnly.reader, {
initialState: 0,
worker: async () => 1,
merge: (total, value) => total + value,
});
expect(activeOnly.findMany.mock.calls[0]?.[0]).toMatchObject({ where: { active: true } });
const allTenants = createReader(["tenant-inactive"]);
await runActiveTenantSweep(allTenants.reader, {
initialState: 0,
includeInactive: true,
worker: async () => 1,
merge: (total, value) => total + value,
});
expect(allTenants.findMany.mock.calls[0]?.[0]).toMatchObject({ where: {} });
});
test("uses stable keyset pages and reduces results without retaining the full tenant list", async () => {
const ids = Array.from({ length: ACTIVE_TENANT_PAGE_SIZE * 2 + 7 }, (_, index) => `tenant-${String(index).padStart(4, "0")}`);
const { reader, findMany } = createReader(ids);
const result = await runActiveTenantSweep(reader, {
initialState: 0,
concurrency: 1,
worker: async () => 1,
merge: (total, value) => total + value,
});
expect(result).toEqual({ state: ids.length, inspected: ids.length, succeeded: ids.length, failures: [], pages: 3 });
expect(findMany).toHaveBeenCalledTimes(3);
expect(findMany.mock.calls[0]?.[0]).not.toHaveProperty("cursor");
expect(findMany.mock.calls[1]?.[0]).toMatchObject({ cursor: { id: ids[ACTIVE_TENANT_PAGE_SIZE - 1] }, skip: 1 });
expect(findMany.mock.calls[2]?.[0]).toMatchObject({ cursor: { id: ids[ACTIVE_TENANT_PAGE_SIZE * 2 - 1] }, skip: 1 });
});
test("shares a strict concurrency budget across overlapping cron sweeps", async () => {
const coordinator = new TenantSweepCoordinator(2);
const first = createReader(["a", "b", "c"]).reader;
const second = createReader(["d", "e", "f"]).reader;
let active = 0;
let peak = 0;
let started = 0;
const releases: Array<() => void> = [];
const worker = async () => {
started += 1;
active += 1;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => releases.push(resolve));
active -= 1;
return 1;
};
const runs = [first, second].map((reader) =>
runActiveTenantSweep(reader, { initialState: 0, coordinator, worker, merge: (total, value) => total + value })
);
await vi.waitFor(() => expect(releases).toHaveLength(2));
while (started < 6) {
releases.splice(0).forEach((release) => release());
await vi.waitFor(() => expect(started).toBeGreaterThan(0));
if (started < 6) await vi.waitFor(() => expect(releases.length).toBeGreaterThan(0));
}
releases.splice(0).forEach((release) => release());
await Promise.all(runs);
expect(peak).toBe(2);
});
test("continues after tenant failures, redacts diagnostics, and reports every failure for retry", async () => {
const { reader } = createReader(["good-a", "bad-token", "good-b", "bad-password"]);
const result = await runActiveTenantSweep(reader, {
initialState: [] as string[],
worker: async (tenantId) => {
if (tenantId.startsWith("bad")) throw new Error(`${tenantId} token=raw-secret password=hunter2`);
return tenantId;
},
merge: (completed, tenantId) => [...completed, tenantId],
});
expect(result.state).toEqual(["good-a", "good-b"]);
expect(result.inspected).toBe(4);
expect(result.succeeded).toBe(2);
expect(result.failures).toEqual([
{ tenantId: "bad-token", error: "bad-token token=[redacted] password=[redacted]" },
{ tenantId: "bad-password", error: "bad-password token=[redacted] password=[redacted]" },
]);
expect(() => throwOnActiveTenantSweepFailures("maintenance", result.failures)).toThrow("maintenance failed for 2 tenant(s)");
});
});
@@ -0,0 +1,171 @@
import { mapWithConcurrency } from "@hivegrid/foundation";
export const ACTIVE_TENANT_PAGE_SIZE = 100;
export const DEFAULT_ACTIVE_TENANT_CONCURRENCY = 2;
export const MAX_ACTIVE_TENANT_CONCURRENCY = 4;
type ActiveTenantRow = { id: string };
export type ActiveTenantReader = {
tenant: {
findMany(args: {
where: { active?: true };
select: { id: true };
orderBy: { id: "asc" };
take: number;
cursor?: { id: string };
skip?: 1;
}): Promise<ActiveTenantRow[]>;
};
};
export type ActiveTenantSweepFailure = {
tenantId: string;
error: string;
};
export type ActiveTenantSweepResult<State> = {
state: State;
inspected: number;
succeeded: number;
failures: ActiveTenantSweepFailure[];
pages: number;
};
type ActiveTenantSweepOptions<State, Value> = {
initialState: State;
worker: (tenantId: string) => Promise<Value>;
merge: (state: State, value: Value) => State;
pageSize?: number;
concurrency?: number;
coordinator?: TenantSweepCoordinator;
/** Include deactivated tenants when legal/financial cleanup must outlive tenant access. */
includeInactive?: boolean;
};
function resolveBoundedInteger(raw: string | undefined, fallback: number, max: number): number {
const parsed = raw === undefined ? Number.NaN : Number(raw);
if (!Number.isSafeInteger(parsed) || parsed < 1) return fallback;
return Math.min(parsed, max);
}
/**
* A process-wide semaphore keeps nested fan-out below the database pool even
* when several BullMQ maintenance jobs happen to run at the same time.
*/
export class TenantSweepCoordinator {
readonly concurrency: number;
private active = 0;
private readonly waiters: Array<() => void> = [];
constructor(concurrency: number = DEFAULT_ACTIVE_TENANT_CONCURRENCY) {
this.concurrency = Math.max(1, Math.min(Math.floor(concurrency), MAX_ACTIVE_TENANT_CONCURRENCY));
}
async run<Value>(worker: () => Promise<Value>): Promise<Value> {
await this.acquire();
try {
return await worker();
} finally {
this.release();
}
}
private async acquire(): Promise<void> {
if (this.active < this.concurrency) {
this.active += 1;
return;
}
await new Promise<void>((resolve) => this.waiters.push(resolve));
}
private release(): void {
const next = this.waiters.shift();
if (next) {
next();
return;
}
this.active -= 1;
}
}
const defaultCoordinator = new TenantSweepCoordinator(
resolveBoundedInteger(process.env.WORKER_TENANT_SWEEP_CONCURRENCY, DEFAULT_ACTIVE_TENANT_CONCURRENCY, MAX_ACTIVE_TENANT_CONCURRENCY)
);
/**
* Walk active tenants (or every tenant for cleanup obligations) with a stable
* id cursor and bounded concurrency. Only one page and its outcomes are retained
* at a time, so tenant growth cannot turn a cron tick into an unbounded read or
* promise fan-out.
*/
export async function runActiveTenantSweep<State, Value>(
client: ActiveTenantReader,
options: ActiveTenantSweepOptions<State, Value>
): Promise<ActiveTenantSweepResult<State>> {
const pageSize = Math.max(1, Math.min(Math.floor(options.pageSize ?? ACTIVE_TENANT_PAGE_SIZE), 1_000));
const coordinator = options.coordinator ?? defaultCoordinator;
const concurrency = Math.max(1, Math.min(Math.floor(options.concurrency ?? coordinator.concurrency), coordinator.concurrency));
const failures: ActiveTenantSweepFailure[] = [];
let state = options.initialState;
let inspected = 0;
let succeeded = 0;
let pages = 0;
let cursor: string | undefined;
while (true) {
const rows = await client.tenant.findMany({
where: options.includeInactive ? {} : { active: true },
select: { id: true },
orderBy: { id: "asc" },
take: pageSize,
...(cursor ? { cursor: { id: cursor }, skip: 1 as const } : {}),
});
if (rows.length === 0) break;
pages += 1;
inspected += rows.length;
const outcomes = new Array<{ ok: true; value: Value } | { ok: false; failure: ActiveTenantSweepFailure }>(rows.length);
await mapWithConcurrency(rows, concurrency, async (row, index) => {
try {
const value = await coordinator.run(() => options.worker(row.id));
outcomes[index] = { ok: true, value };
} catch (error) {
outcomes[index] = {
ok: false,
failure: { tenantId: row.id, error: redactTenantSweepError(error) },
};
}
});
for (const outcome of outcomes) {
if (outcome.ok) {
state = options.merge(state, outcome.value);
succeeded += 1;
} else {
failures.push(outcome.failure);
}
}
cursor = rows.at(-1)?.id;
if (rows.length < pageSize || !cursor) break;
}
return { state, inspected, succeeded, failures, pages };
}
export function throwOnActiveTenantSweepFailures(task: string, failures: readonly ActiveTenantSweepFailure[]): void {
if (failures.length === 0) return;
throw new AggregateError(
failures.map((failure) => new Error(`${failure.tenantId}: ${failure.error}`)),
`${task} failed for ${failures.length} tenant(s)`
);
}
function redactTenantSweepError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
.replace(/(token|secret|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
.slice(0, 500);
}
@@ -0,0 +1,71 @@
import { describe, expect, it, vi } from "vitest";
import {
bootstrapAdminBookingOutbox,
type AdminBookingOutboxWorkerHandle,
} from "./admin-booking-outbox-bootstrap.js";
describe("bootstrapAdminBookingOutbox", () => {
it("only subscribes on the general worker group", () => {
expect(
bootstrapAdminBookingOutbox(
{ WORKER_QUEUE_GROUP: "mail" },
{}
)
).toEqual({
status: "disabled",
reason: "non-general worker group",
});
});
it("binds the admin identity before processing the provider command", async () => {
let processor:
| ((job: { readonly data: unknown }) => Promise<unknown>)
| undefined;
const handle: AdminBookingOutboxWorkerHandle = {
name: "outbox.booking",
close: vi.fn().mockResolvedValue(undefined),
};
const loadEvent = vi.fn().mockResolvedValue({
bookingId: "booking_1",
adminUserId: "admin_1",
calendarConnectionId: "calendar_1",
provider: "google_calendar",
accessToken: "access",
calendarId: "primary",
subject: "Demo",
startsAt: new Date("2026-08-01T10:00:00.000Z"),
endsAt: new Date("2026-08-01T10:30:00.000Z"),
timeZone: "Europe/Berlin",
attendee: { email: "lead@example.test", name: "Ada Lead" },
});
const result = bootstrapAdminBookingOutbox(
{ WORKER_QUEUE_GROUP: "general" },
{
dependencies: {
loadEvent,
createEvent: vi
.fn()
.mockResolvedValue({ externalEventId: "provider_1" }),
updateEvent: vi.fn(),
deleteEvent: vi.fn(),
confirm: vi.fn(),
markFailed: vi.fn(),
},
createWorker: (_queue, next) => {
processor = next;
return handle;
},
}
);
expect(result).toEqual({ status: "started", handle });
await processor?.({
data: {
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
});
expect(loadEvent).toHaveBeenCalledWith("booking_1", "admin_1");
});
});
@@ -0,0 +1,88 @@
import type { EnvRecord } from "@hivegrid/config";
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
import {
ADMIN_BOOKING_OUTBOX_QUEUE,
parseAdminBookingOutboxEnvelope,
runAdminBookingOutboxJob,
type AdminBookingOutboxDependencies,
} from "./admin-booking-outbox.js";
export type AdminBookingOutboxBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
ADMIN_BOOKING_OUTBOX_ENABLED?: string;
ADMIN_BOOKING_OUTBOX_CONCURRENCY?: string;
};
export type AdminBookingOutboxWorkerHandle = {
readonly name: string;
close(): Promise<void>;
};
export type AdminBookingOutboxBootstrapResult =
| { readonly status: "disabled"; readonly reason: string }
| {
readonly status: "started";
readonly handle: AdminBookingOutboxWorkerHandle;
};
export function bootstrapAdminBookingOutbox(
env: AdminBookingOutboxBootstrapEnv,
input: {
readonly dependencies?: AdminBookingOutboxDependencies;
readonly createWorker?: (
queueName: string,
processor: (job: { readonly data: unknown }) => Promise<unknown>,
options: { readonly concurrency: number }
) => AdminBookingOutboxWorkerHandle;
readonly logger?: {
info?: (
record: Record<string, unknown>,
message?: string
) => void;
};
}
): AdminBookingOutboxBootstrapResult {
if (env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.ADMIN_BOOKING_OUTBOX_ENABLED === "false") {
return {
status: "disabled",
reason: "ADMIN_BOOKING_OUTBOX_ENABLED=false",
};
}
if (!input.dependencies || !input.createWorker) {
return { status: "disabled", reason: "runtime dependencies unavailable" };
}
const dependencies = input.dependencies;
const concurrency = positiveInteger(
env.ADMIN_BOOKING_OUTBOX_CONCURRENCY,
5
);
const handle = input.createWorker(
ADMIN_BOOKING_OUTBOX_QUEUE,
(job) => {
const envelope = parseAdminBookingOutboxEnvelope(job.data);
return runWithRlsContext(
{ userId: envelope.payload.adminUserId },
() => runAdminBookingOutboxJob(job.data, dependencies)
);
},
{ concurrency }
);
input.logger?.info?.(
{
component: "admin-booking-outbox",
queue: ADMIN_BOOKING_OUTBOX_QUEUE,
concurrency,
},
"admin booking provider worker started"
);
return { status: "started", handle };
}
function positiveInteger(value: string | undefined, fallback: number): number {
const parsed = Number(value);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
}
@@ -0,0 +1,162 @@
import { describe, expect, it, vi } from "vitest";
import {
parseAdminBookingOutboxEnvelope,
runAdminBookingOutboxJob,
type AdminBookingOutboxDependencies,
type AdminBookingProviderEvent,
} from "./admin-booking-outbox.js";
const event: AdminBookingProviderEvent = {
bookingId: "booking_1",
adminUserId: "admin_1",
calendarConnectionId: "calendar_1",
provider: "google_calendar",
accessToken: "access",
calendarId: "primary",
subject: "Demo",
startsAt: new Date("2026-08-01T10:00:00.000Z"),
endsAt: new Date("2026-08-01T10:30:00.000Z"),
timeZone: "Europe/Berlin",
attendee: { email: "lead@example.test", name: "Ada Lead" },
};
function dependencies(
overrides: Partial<AdminBookingOutboxDependencies> = {}
): AdminBookingOutboxDependencies {
return {
loadEvent: vi.fn().mockResolvedValue(event),
createEvent: vi
.fn()
.mockResolvedValue({ externalEventId: "provider_1", opaqueVersion: "v1" }),
updateEvent: vi
.fn()
.mockResolvedValue({ externalEventId: "provider_1", opaqueVersion: "v2" }),
deleteEvent: vi.fn().mockResolvedValue(undefined),
confirm: vi.fn().mockResolvedValue(undefined),
markFailed: vi.fn().mockResolvedValue(undefined),
now: () => new Date("2026-07-25T12:00:00.000Z"),
...overrides,
};
}
describe("admin booking outbox", () => {
it("creates and confirms a provider event idempotently", async () => {
const deps = dependencies();
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
deps
)
).resolves.toEqual({ status: "created", bookingId: "booking_1" });
expect(deps.createEvent).toHaveBeenCalledWith(event);
expect(deps.confirm).toHaveBeenCalledWith(
expect.objectContaining({
providerEventId: "provider_1",
providerEventVersion: "v1",
})
);
});
it("deduplicates a replay after provider confirmation", async () => {
const deps = dependencies({
loadEvent: vi
.fn()
.mockResolvedValue({ ...event, externalEventId: "provider_1" }),
});
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
deps
)
).resolves.toEqual({
status: "deduplicated",
bookingId: "booking_1",
});
expect(deps.createEvent).not.toHaveBeenCalled();
});
it("updates the original provider event during rescheduling", async () => {
const deps = dependencies({
loadEvent: vi
.fn()
.mockResolvedValueOnce({
...event,
externalEventId: "provider_1",
externalEventVersion: "v1",
})
.mockResolvedValueOnce({ ...event, bookingId: "booking_2" }),
});
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.reschedule",
payload: {
previousBookingId: "booking_1",
replacementBookingId: "booking_2",
adminUserId: "admin_1",
},
},
deps
)
).resolves.toEqual({
status: "rescheduled",
previousBookingId: "booking_1",
replacementBookingId: "booking_2",
});
expect(deps.updateEvent).toHaveBeenCalledWith(
expect.objectContaining({ bookingId: "booking_2" }),
"provider_1",
"v1"
);
});
it("records a redacted provider failure and preserves the retry", async () => {
const deps = dependencies({
createEvent: vi
.fn()
.mockRejectedValue(Object.assign(new Error("secret"), { code: "provider_outage" })),
});
await expect(
runAdminBookingOutboxJob(
{
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
},
deps
)
).rejects.toThrow("secret");
expect(deps.markFailed).toHaveBeenCalledWith({
bookingId: "booking_1",
adminUserId: "admin_1",
errorCode: "provider_outage",
});
});
it("rejects tenant or payload confusion before touching a provider", () => {
expect(() =>
parseAdminBookingOutboxEnvelope({
tenantId: "tenant_1",
eventType: "booking.provider.create",
payload: { bookingId: "booking_1", adminUserId: "admin_1" },
})
).toThrow("invalid_envelope");
expect(() =>
parseAdminBookingOutboxEnvelope({
tenantId: null,
eventType: "booking.provider.create",
payload: { bookingId: "../booking", adminUserId: "admin_1" },
})
).toThrow("invalid_bookingId");
});
});
@@ -0,0 +1,270 @@
export const ADMIN_BOOKING_OUTBOX_QUEUE = "outbox.booking" as const;
export type AdminBookingProviderEvent = {
readonly bookingId: string;
readonly adminUserId: string;
readonly calendarConnectionId: string;
readonly provider: "google_calendar" | "microsoft_calendar";
readonly accessToken: string;
readonly externalEventId?: string | null;
readonly externalEventVersion?: string | null;
readonly calendarId: string;
readonly subject: string;
readonly bodyText?: string | null;
readonly startsAt: Date;
readonly endsAt: Date;
readonly timeZone: string;
readonly attendee: {
readonly email: string;
readonly name: string;
};
};
type CreateEnvelope = {
readonly eventType: "booking.provider.create";
readonly payload: {
readonly bookingId: string;
readonly adminUserId: string;
};
};
type CancelEnvelope = {
readonly eventType: "booking.provider.cancel";
readonly payload: {
readonly bookingId: string;
readonly adminUserId: string;
};
};
type RescheduleEnvelope = {
readonly eventType: "booking.provider.reschedule";
readonly payload: {
readonly previousBookingId: string;
readonly replacementBookingId: string;
readonly adminUserId: string;
};
};
export type AdminBookingOutboxEnvelope =
| CreateEnvelope
| CancelEnvelope
| RescheduleEnvelope;
export type AdminBookingOutboxDependencies = {
loadEvent(
bookingId: string,
adminUserId: string
): Promise<AdminBookingProviderEvent>;
createEvent(
event: AdminBookingProviderEvent
): Promise<{
readonly externalEventId: string;
readonly opaqueVersion?: string | null;
}>;
updateEvent(
event: AdminBookingProviderEvent,
externalEventId: string,
opaqueVersion?: string | null
): Promise<{
readonly externalEventId: string;
readonly opaqueVersion?: string | null;
}>;
deleteEvent(event: AdminBookingProviderEvent): Promise<void>;
confirm(input: {
readonly bookingId: string;
readonly adminUserId: string;
readonly calendarConnectionId: string;
readonly providerEventId: string;
readonly providerEventVersion?: string | null;
readonly syncedAt: Date;
}): Promise<void>;
markFailed(input: {
readonly bookingId: string;
readonly adminUserId: string;
readonly errorCode: string;
}): Promise<void>;
readonly now?: () => Date;
};
export async function runAdminBookingOutboxJob(
data: unknown,
dependencies: AdminBookingOutboxDependencies
): Promise<
| { readonly status: "created"; readonly bookingId: string }
| { readonly status: "cancelled"; readonly bookingId: string }
| {
readonly status: "rescheduled";
readonly previousBookingId: string;
readonly replacementBookingId: string;
}
| { readonly status: "deduplicated"; readonly bookingId: string }
> {
const envelope = parseAdminBookingOutboxEnvelope(data);
const now = dependencies.now ?? (() => new Date());
if (envelope.eventType === "booking.provider.create") {
const event = await dependencies.loadEvent(
envelope.payload.bookingId,
envelope.payload.adminUserId
);
if (event.externalEventId) {
return {
status: "deduplicated",
bookingId: envelope.payload.bookingId,
};
}
try {
const result = await dependencies.createEvent(event);
await dependencies.confirm({
bookingId: event.bookingId,
adminUserId: event.adminUserId,
calendarConnectionId: event.calendarConnectionId,
providerEventId: result.externalEventId,
providerEventVersion: result.opaqueVersion,
syncedAt: now(),
});
return { status: "created", bookingId: event.bookingId };
} catch (error) {
await markFailedSafely(dependencies, event, error);
throw error;
}
}
if (envelope.eventType === "booking.provider.cancel") {
const event = await dependencies.loadEvent(
envelope.payload.bookingId,
envelope.payload.adminUserId
);
if (!event.externalEventId) {
return {
status: "deduplicated",
bookingId: envelope.payload.bookingId,
};
}
await dependencies.deleteEvent(event);
return { status: "cancelled", bookingId: event.bookingId };
}
const previous = await dependencies.loadEvent(
envelope.payload.previousBookingId,
envelope.payload.adminUserId
);
const replacement = await dependencies.loadEvent(
envelope.payload.replacementBookingId,
envelope.payload.adminUserId
);
try {
const result = previous.externalEventId
? await dependencies.updateEvent(
replacement,
previous.externalEventId,
previous.externalEventVersion
)
: await dependencies.createEvent(replacement);
await dependencies.confirm({
bookingId: replacement.bookingId,
adminUserId: replacement.adminUserId,
calendarConnectionId: replacement.calendarConnectionId,
providerEventId: result.externalEventId,
providerEventVersion: result.opaqueVersion,
syncedAt: now(),
});
return {
status: "rescheduled",
previousBookingId: previous.bookingId,
replacementBookingId: replacement.bookingId,
};
} catch (error) {
await markFailedSafely(dependencies, replacement, error);
throw error;
}
}
export function parseAdminBookingOutboxEnvelope(
data: unknown
): AdminBookingOutboxEnvelope {
const envelope = record(data, "envelope");
if (
envelope.tenantId !== null ||
![
"booking.provider.create",
"booking.provider.cancel",
"booking.provider.reschedule",
].includes(String(envelope.eventType))
) {
throw new AdminBookingOutboxError("invalid_envelope");
}
const payload = record(envelope.payload, "payload");
const adminUserId = identifier(payload.adminUserId, "adminUserId");
if (envelope.eventType === "booking.provider.reschedule") {
return {
eventType: "booking.provider.reschedule",
payload: {
previousBookingId: identifier(
payload.previousBookingId,
"previousBookingId"
),
replacementBookingId: identifier(
payload.replacementBookingId,
"replacementBookingId"
),
adminUserId,
},
};
}
const bookingId = identifier(payload.bookingId, "bookingId");
return {
eventType: envelope.eventType as
| "booking.provider.create"
| "booking.provider.cancel",
payload: { bookingId, adminUserId },
};
}
async function markFailedSafely(
dependencies: AdminBookingOutboxDependencies,
event: AdminBookingProviderEvent,
error: unknown
): Promise<void> {
await dependencies
.markFailed({
bookingId: event.bookingId,
adminUserId: event.adminUserId,
errorCode: providerErrorCode(error),
})
.catch(() => undefined);
}
function providerErrorCode(error: unknown): string {
if (
error &&
typeof error === "object" &&
"code" in error &&
typeof error.code === "string"
) {
return error.code.slice(0, 100);
}
return error instanceof Error ? error.name.slice(0, 100) : "provider_error";
}
function record(value: unknown, field: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new AdminBookingOutboxError(`invalid_${field}`);
}
return value as Record<string, unknown>;
}
function identifier(value: unknown, field: string): string {
const normalized = typeof value === "string" ? value.trim() : "";
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,190}$/u.test(normalized)) {
throw new AdminBookingOutboxError(`invalid_${field}`);
}
return normalized;
}
export class AdminBookingOutboxError extends Error {
constructor(readonly code: string) {
super(code);
this.name = "AdminBookingOutboxError";
}
}
@@ -0,0 +1,29 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS,
ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
registerAdminBookingReminderSweepCron,
} from "./admin-booking-reminder-cron.js";
describe("registerAdminBookingReminderSweepCron", () => {
it("registers a bounded one-minute reminder sweep", async () => {
const upsertJobScheduler = vi.fn().mockResolvedValue(undefined);
await registerAdminBookingReminderSweepCron(
{ upsertJobScheduler },
{ ADMIN_BOOKING_REMINDER_SWEEP_BATCH_SIZE: "5000" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
take: 1000,
},
}
);
});
});
@@ -0,0 +1,32 @@
export const ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID =
"admin-booking-reminder-sweep" as const;
export const ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS = 60_000;
export async function registerAdminBookingReminderSweepCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const parsed = Number(env.ADMIN_BOOKING_REMINDER_SWEEP_BATCH_SIZE);
const take =
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 1_000)
: 250;
await queue.upsertJobScheduler(
ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_BOOKING_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_BOOKING_REMINDER_SWEEP_JOB_ID,
take,
},
}
);
}
@@ -0,0 +1,23 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_CALENDAR_SYNC_EVERY_MS,
ADMIN_CALENDAR_SYNC_JOB_ID,
registerAdminCalendarSyncCron,
} from "./admin-calendar-sync-cron.js";
describe("registerAdminCalendarSyncCron", () => {
it("registers a bounded five-minute sweep", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminCalendarSyncCron(
{ upsertJobScheduler },
{ ADMIN_CALENDAR_SYNC_BATCH_SIZE: "500" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_CALENDAR_SYNC_JOB_ID,
{ every: ADMIN_CALENDAR_SYNC_EVERY_MS },
expect.objectContaining({
data: expect.objectContaining({ take: 100 }),
})
);
});
});
@@ -0,0 +1,31 @@
export const ADMIN_CALENDAR_SYNC_JOB_ID = "admin-calendar-sync" as const;
export const ADMIN_CALENDAR_SYNC_EVERY_MS = 5 * 60 * 1_000;
export async function registerAdminCalendarSyncCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
) {
const configured = Number(env.ADMIN_CALENDAR_SYNC_BATCH_SIZE);
const take =
Number.isFinite(configured) && configured > 0
? Math.min(Math.floor(configured), 100)
: 25;
await queue.upsertJobScheduler(
ADMIN_CALENDAR_SYNC_JOB_ID,
{ every: ADMIN_CALENDAR_SYNC_EVERY_MS },
{
name: ADMIN_CALENDAR_SYNC_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALENDAR_SYNC_JOB_ID,
take,
},
}
);
}
@@ -0,0 +1,42 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_CALL_RECORDING_PROCESS_EVERY_MS,
ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
ADMIN_CALL_RECORDING_RETENTION_EVERY_MS,
ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
registerAdminCallRecordingCrons,
} from "./admin-call-recording-cron.js";
describe("admin call recording schedulers", () => {
it("registers prompt processing and hourly retention as stable jobs", async () => {
const upsertJobScheduler = vi.fn().mockResolvedValue(undefined);
await registerAdminCallRecordingCrons({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
{ every: ADMIN_CALL_RECORDING_PROCESS_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
take: 10,
},
}
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
{ every: ADMIN_CALL_RECORDING_RETENTION_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
take: 50,
},
}
);
});
});
@@ -0,0 +1,45 @@
export const ADMIN_CALL_RECORDING_PROCESS_JOB_ID =
"admin-call-recording-process" as const;
export const ADMIN_CALL_RECORDING_RETENTION_JOB_ID =
"admin-call-recording-retention" as const;
export const ADMIN_CALL_RECORDING_PROCESS_EVERY_MS = 60 * 1_000;
export const ADMIN_CALL_RECORDING_RETENTION_EVERY_MS = 60 * 60 * 1_000;
type RecordingScheduler = {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
};
export async function registerAdminCallRecordingCrons(
queue: RecordingScheduler
): Promise<void> {
await Promise.all([
queue.upsertJobScheduler(
ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
{ every: ADMIN_CALL_RECORDING_PROCESS_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_PROCESS_JOB_ID,
take: 10,
},
}
),
queue.upsertJobScheduler(
ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
{ every: ADMIN_CALL_RECORDING_RETENTION_EVERY_MS },
{
name: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_CALL_RECORDING_RETENTION_JOB_ID,
take: 50,
},
}
),
]);
}
@@ -0,0 +1,41 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE,
ADMIN_EMAIL_DELIVERY_EVERY_MS,
ADMIN_EMAIL_DELIVERY_JOB_ID,
registerAdminEmailDeliveryCron,
resolveAdminEmailDeliveryConfig,
} from "./admin-email-delivery-cron.js";
describe("admin email delivery cron", () => {
it("keeps the delivery batch bounded", () => {
expect(resolveAdminEmailDeliveryConfig({})).toEqual({
take: ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE,
});
expect(
resolveAdminEmailDeliveryConfig({
ADMIN_EMAIL_DELIVERY_BATCH_SIZE: "500",
})
).toEqual({ take: 250 });
});
it("registers one idempotent minute scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailDeliveryCron(
{ upsertJobScheduler },
{ ADMIN_EMAIL_DELIVERY_BATCH_SIZE: "25" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_DELIVERY_JOB_ID,
{ every: ADMIN_EMAIL_DELIVERY_EVERY_MS },
{
name: ADMIN_EMAIL_DELIVERY_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DELIVERY_JOB_ID,
take: 25,
},
}
);
});
});
@@ -0,0 +1,40 @@
export const ADMIN_EMAIL_DELIVERY_JOB_ID = "admin-email-delivery" as const;
export const ADMIN_EMAIL_DELIVERY_EVERY_MS = 60 * 1_000;
export const ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE = 50;
export function resolveAdminEmailDeliveryConfig(
env: NodeJS.ProcessEnv = process.env
): { readonly take: number } {
const parsed = Number(env.ADMIN_EMAIL_DELIVERY_BATCH_SIZE);
return {
take:
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 250)
: ADMIN_EMAIL_DELIVERY_DEFAULT_BATCH_SIZE,
};
}
export async function registerAdminEmailDeliveryCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const config = resolveAdminEmailDeliveryConfig(env);
await queue.upsertJobScheduler(
ADMIN_EMAIL_DELIVERY_JOB_ID,
{ every: ADMIN_EMAIL_DELIVERY_EVERY_MS },
{
name: ADMIN_EMAIL_DELIVERY_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DELIVERY_JOB_ID,
take: config.take,
},
}
);
}
@@ -0,0 +1,28 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_DOMAIN_EVERY_MS,
ADMIN_EMAIL_DOMAIN_JOB_ID,
registerAdminEmailDomainCron,
} from "./admin-email-domain-cron.js";
describe("admin email sending-domain cron", () => {
it("registers one bounded idempotent DNS scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailDomainCron(
{ upsertJobScheduler },
{ ADMIN_EMAIL_DOMAIN_BATCH_SIZE: "500" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_DOMAIN_JOB_ID,
{ every: ADMIN_EMAIL_DOMAIN_EVERY_MS },
{
name: ADMIN_EMAIL_DOMAIN_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DOMAIN_JOB_ID,
take: 200,
},
}
);
});
});
@@ -0,0 +1,32 @@
export const ADMIN_EMAIL_DOMAIN_JOB_ID = "admin-email-domain-check" as const;
export const ADMIN_EMAIL_DOMAIN_EVERY_MS = 15 * 60 * 1_000;
export const ADMIN_EMAIL_DOMAIN_DEFAULT_BATCH_SIZE = 50;
export async function registerAdminEmailDomainCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const parsed = Number(env.ADMIN_EMAIL_DOMAIN_BATCH_SIZE);
const take =
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 200)
: ADMIN_EMAIL_DOMAIN_DEFAULT_BATCH_SIZE;
await queue.upsertJobScheduler(
ADMIN_EMAIL_DOMAIN_JOB_ID,
{ every: ADMIN_EMAIL_DOMAIN_EVERY_MS },
{
name: ADMIN_EMAIL_DOMAIN_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_DOMAIN_JOB_ID,
take,
},
}
);
}
@@ -0,0 +1,27 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE,
ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS,
ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
registerAdminEmailEventRetentionCron,
} from "./admin-email-event-retention-cron.js";
describe("admin email event retention cron", () => {
it("registers one idempotent daily redaction scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailEventRetentionCron({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
{ every: ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS },
{
name: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
take: ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE,
},
}
);
});
});
@@ -0,0 +1,27 @@
export const ADMIN_EMAIL_EVENT_RETENTION_JOB_ID =
"admin-email-event-retention" as const;
export const ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS = 24 * 60 * 60 * 1_000;
export const ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE = 500;
export async function registerAdminEmailEventRetentionCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
}
): Promise<void> {
await queue.upsertJobScheduler(
ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
{ every: ADMIN_EMAIL_EVENT_RETENTION_EVERY_MS },
{
name: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_EVENT_RETENTION_JOB_ID,
take: ADMIN_EMAIL_EVENT_RETENTION_DEFAULT_BATCH_SIZE,
},
}
);
}
@@ -0,0 +1,39 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE,
ADMIN_EMAIL_SYNC_EVERY_MS,
ADMIN_EMAIL_SYNC_JOB_ID,
registerAdminEmailSyncCron,
resolveAdminEmailSyncConfig,
} from "./admin-email-sync-cron.js";
describe("admin email sync cron", () => {
it("uses a bounded mailbox cursor batch", () => {
expect(resolveAdminEmailSyncConfig({})).toEqual({
take: ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE,
});
expect(
resolveAdminEmailSyncConfig({ ADMIN_EMAIL_SYNC_BATCH_SIZE: "500" })
).toEqual({ take: 100 });
});
it("registers one idempotent two-minute scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminEmailSyncCron(
{ upsertJobScheduler },
{ ADMIN_EMAIL_SYNC_BATCH_SIZE: "20" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_EMAIL_SYNC_JOB_ID,
{ every: ADMIN_EMAIL_SYNC_EVERY_MS },
{
name: ADMIN_EMAIL_SYNC_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_SYNC_JOB_ID,
take: 20,
},
}
);
});
});
@@ -0,0 +1,40 @@
export const ADMIN_EMAIL_SYNC_JOB_ID = "admin-email-sync" as const;
export const ADMIN_EMAIL_SYNC_EVERY_MS = 2 * 60 * 1_000;
export const ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE = 25;
export function resolveAdminEmailSyncConfig(
env: NodeJS.ProcessEnv = process.env
): { readonly take: number } {
const parsed = Number(env.ADMIN_EMAIL_SYNC_BATCH_SIZE);
return {
take:
Number.isFinite(parsed) && parsed > 0
? Math.min(Math.floor(parsed), 100)
: ADMIN_EMAIL_SYNC_DEFAULT_BATCH_SIZE,
};
}
export async function registerAdminEmailSyncCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const config = resolveAdminEmailSyncConfig(env);
await queue.upsertJobScheduler(
ADMIN_EMAIL_SYNC_JOB_ID,
{ every: ADMIN_EMAIL_SYNC_EVERY_MS },
{
name: ADMIN_EMAIL_SYNC_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_EMAIL_SYNC_JOB_ID,
take: config.take,
},
}
);
}
@@ -0,0 +1,26 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_OUTBOUND_CALL_EVERY_MS,
ADMIN_OUTBOUND_CALL_JOB_ID,
registerAdminOutboundCallCron,
} from "./admin-outbound-call-cron.js";
describe("admin outbound call scheduler", () => {
it("registers one deterministic minute scheduler", async () => {
const queue = { upsertJobScheduler: vi.fn(async () => undefined) };
await registerAdminOutboundCallCron(queue);
expect(queue.upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_OUTBOUND_CALL_JOB_ID,
{ every: ADMIN_OUTBOUND_CALL_EVERY_MS },
{
name: ADMIN_OUTBOUND_CALL_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_OUTBOUND_CALL_JOB_ID,
take: 25,
},
}
);
});
});
@@ -0,0 +1,28 @@
export const ADMIN_OUTBOUND_CALL_JOB_ID =
"admin-outbound-call-process" as const;
export const ADMIN_OUTBOUND_CALL_EVERY_MS = 60 * 1_000;
type OutboundCallScheduler = {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data: Record<string, unknown> }
): Promise<unknown>;
};
export async function registerAdminOutboundCallCron(
queue: OutboundCallScheduler
): Promise<void> {
await queue.upsertJobScheduler(
ADMIN_OUTBOUND_CALL_JOB_ID,
{ every: ADMIN_OUTBOUND_CALL_EVERY_MS },
{
name: ADMIN_OUTBOUND_CALL_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_OUTBOUND_CALL_JOB_ID,
take: 25,
},
}
);
}
@@ -0,0 +1,180 @@
import {
GetObjectCommand,
PutObjectCommand,
} from "@aws-sdk/client-s3";
import {
processAdminSalesContactsExport,
processAdminSalesContactsImport,
} from "@hivegrid/db/admin-sales-data.server";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createAdminSalesDataWorkerService } from "./admin-sales-data.js";
vi.mock("@hivegrid/db/admin-sales-data.server", () => ({
processAdminSalesContactsExport: vi.fn(),
processAdminSalesContactsImport: vi.fn(),
}));
const prisma = {
adminImportJob: { findFirst: vi.fn() },
adminExportJob: { findFirst: vi.fn() },
};
describe("admin sales data worker service", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("fails closed when the shared storage bucket is not configured", () => {
expect(
createAdminSalesDataWorkerService(
{},
{ prisma: prisma as never, s3Client: { send: vi.fn() } as never }
)
).toBeUndefined();
});
it("loads a bounded reviewed CSV and stores its error report encrypted", async () => {
prisma.adminImportJob.findFirst.mockResolvedValue({
sourceBucket: "hivegrid-files",
sourceKey: "admin-sales/imports/admin_1/import_1.csv",
});
const send = vi.fn(async (command: unknown) => {
if (command instanceof GetObjectCommand) {
return {
Body: {
transformToByteArray: async () =>
new TextEncoder().encode(
"Vorname,Nachname\nAda,Lovelace\n"
),
},
};
}
return {};
});
vi.mocked(processAdminSalesContactsImport).mockImplementation(
async (_client, input) => {
const artifact = await input.storeErrorReport({
importJobId: input.importJobId,
csvText: "Zeile,Fehler\n2,Dublette\n",
});
expect(artifact).toEqual({
bucket: "hivegrid-files",
key: "admin-sales/import-errors/import_1.csv",
});
return {
totalRows: 1,
acceptedRows: 0,
rejectedRows: 1,
errorReport: artifact,
};
}
);
const service = createAdminSalesDataWorkerService(
{ STORAGE_BUCKET: "hivegrid-files" },
{
prisma: prisma as never,
s3Client: { send } as never,
now: () => new Date("2026-07-24T12:00:00Z"),
}
);
await expect(
service!.runImport({
kind: "admin-sales.contacts-import",
importJobId: "import_1",
requestedByAdminUserId: "admin_1",
})
).resolves.toEqual({
totalRows: 1,
acceptedRows: 0,
rejectedRows: 1,
});
expect(processAdminSalesContactsImport).toHaveBeenCalledWith(
prisma,
expect.objectContaining({
importJobId: "import_1",
csvText: "Vorname,Nachname\nAda,Lovelace\n",
now: new Date("2026-07-24T12:00:00Z"),
})
);
const put = send.mock.calls
.map(([command]) => command)
.find((command) => command instanceof PutObjectCommand);
expect((put as PutObjectCommand).input).toEqual(
expect.objectContaining({
Bucket: "hivegrid-files",
Key: "admin-sales/import-errors/import_1.csv",
ContentType: "text/csv; charset=utf-8",
ServerSideEncryption: "AES256",
})
);
});
it("stores exports with a stable encrypted download artifact", async () => {
prisma.adminExportJob.findFirst.mockResolvedValue({ id: "export_1" });
const send = vi.fn(async (_command: unknown) => ({}));
vi.mocked(processAdminSalesContactsExport).mockImplementation(
async (_client, input) => {
const artifact = await input.storeArtifact({
exportJobId: input.exportJobId,
csvText: "\uFEFFVorname,Nachname\nAda,Lovelace\n",
});
return {
rowCount: 1,
artifact,
expiresAt: new Date("2026-07-31T12:00:00Z"),
};
}
);
const service = createAdminSalesDataWorkerService(
{ STORAGE_BUCKET: "hivegrid-files" },
{ prisma: prisma as never, s3Client: { send } as never }
);
await expect(
service!.runExport({
kind: "admin-sales.contacts-export",
exportJobId: "export_1",
requestedByAdminUserId: "admin_1",
})
).resolves.toEqual({
rowCount: 1,
artifactBucket: "hivegrid-files",
artifactKey: "admin-sales/exports/export_1.csv",
expiresAt: new Date("2026-07-31T12:00:00Z"),
});
const put = send.mock.calls[0]?.[0];
expect(put).toBeInstanceOf(PutObjectCommand);
expect((put as PutObjectCommand).input).toEqual(
expect.objectContaining({
Bucket: "hivegrid-files",
Key: "admin-sales/exports/export_1.csv",
ContentDisposition:
'attachment; filename="admin-sales-contacts.csv"',
ServerSideEncryption: "AES256",
})
);
});
it("rejects an import object outside the configured bucket", async () => {
prisma.adminImportJob.findFirst.mockResolvedValue({
sourceBucket: "other-bucket",
sourceKey: "contacts.csv",
});
const service = createAdminSalesDataWorkerService(
{ STORAGE_BUCKET: "hivegrid-files" },
{
prisma: prisma as never,
s3Client: { send: vi.fn() } as never,
}
);
await expect(
service!.runImport({
kind: "admin-sales.contacts-import",
importJobId: "import_1",
requestedByAdminUserId: "admin_1",
})
).rejects.toThrow("source bucket is not allowed");
});
});
@@ -0,0 +1,172 @@
import {
GetObjectCommand,
PutObjectCommand,
S3Client,
} from "@aws-sdk/client-s3";
import {
processAdminSalesContactsExport,
processAdminSalesContactsImport,
} from "@hivegrid/db/admin-sales-data.server";
import type {
AdminSalesContactsExportJob,
AdminSalesContactsImportJob,
} from "@hivegrid/queue";
const MAX_SOURCE_BYTES = 5 * 1024 * 1024;
export type AdminSalesDataWorkerEnv = {
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_BUCKET?: string;
STORAGE_ENDPOINT?: string;
STORAGE_FORCE_PATH_STYLE?: string;
STORAGE_REGION?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
};
export type AdminSalesDataWorkerService = {
runImport(payload: AdminSalesContactsImportJob): Promise<{
readonly totalRows: number;
readonly acceptedRows: number;
readonly rejectedRows: number;
}>;
runExport(payload: AdminSalesContactsExportJob): Promise<{
readonly rowCount: number;
readonly artifactBucket: string;
readonly artifactKey: string;
readonly expiresAt: Date;
}>;
};
type AdminSalesDataPrisma = Parameters<
typeof processAdminSalesContactsImport
>[0];
export function createAdminSalesDataWorkerService(
env: AdminSalesDataWorkerEnv,
dependencies: {
readonly prisma: AdminSalesDataPrisma;
readonly s3Client?: Pick<S3Client, "send">;
readonly now?: () => Date;
}
): AdminSalesDataWorkerService | undefined {
const bucket = env.STORAGE_BUCKET?.trim();
const s3 = dependencies.s3Client ?? createS3Client(env);
if (!bucket || !s3) return undefined;
const now = dependencies.now ?? (() => new Date());
return {
async runImport(payload) {
const job = await dependencies.prisma.adminImportJob.findFirst({
where: {
id: payload.importJobId,
requestedByAdminUserId: payload.requestedByAdminUserId,
},
select: { sourceBucket: true, sourceKey: true },
});
if (!job) throw new Error("Admin sales import job was not found.");
if (job.sourceBucket !== bucket) {
throw new Error("Admin sales import source bucket is not allowed.");
}
const response = await s3.send(
new GetObjectCommand({ Bucket: bucket, Key: job.sourceKey })
);
const bytes = await response.Body?.transformToByteArray();
if (!bytes) throw new Error("Admin sales import source is empty.");
if (bytes.byteLength > MAX_SOURCE_BYTES) {
throw new Error("Admin sales import source exceeds the 5 MB limit.");
}
const result = await processAdminSalesContactsImport(
dependencies.prisma,
{
importJobId: payload.importJobId,
csvText: new TextDecoder("utf-8", { fatal: true }).decode(bytes),
now: now(),
storeErrorReport: async ({ importJobId, csvText }) => {
const key = `admin-sales/import-errors/${importJobId}.csv`;
await s3.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: Buffer.from(csvText, "utf8"),
ContentType: "text/csv; charset=utf-8",
ServerSideEncryption: "AES256",
Metadata: { importJobId },
})
);
return { bucket, key };
},
}
);
return {
totalRows: result.totalRows,
acceptedRows: result.acceptedRows,
rejectedRows: result.rejectedRows,
};
},
async runExport(payload) {
const job = await dependencies.prisma.adminExportJob.findFirst({
where: {
id: payload.exportJobId,
requestedByAdminUserId: payload.requestedByAdminUserId,
},
select: { id: true },
});
if (!job) throw new Error("Admin sales export job was not found.");
const result = await processAdminSalesContactsExport(
dependencies.prisma,
{
exportJobId: payload.exportJobId,
now: now(),
storeArtifact: async ({ exportJobId, csvText }) => {
const key = `admin-sales/exports/${exportJobId}.csv`;
await s3.send(
new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: Buffer.from(csvText, "utf8"),
ContentType: "text/csv; charset=utf-8",
ContentDisposition:
'attachment; filename="admin-sales-contacts.csv"',
ServerSideEncryption: "AES256",
Metadata: { exportJobId },
})
);
return { bucket, key };
},
}
);
return {
rowCount: result.rowCount,
artifactBucket: result.artifact.bucket,
artifactKey: result.artifact.key,
expiresAt: result.expiresAt,
};
},
};
}
function createS3Client(
env: AdminSalesDataWorkerEnv
): S3Client | undefined {
if (
!env.STORAGE_REGION ||
!env.STORAGE_ACCESS_KEY_ID ||
!env.STORAGE_SECRET_ACCESS_KEY
) {
return undefined;
}
return new S3Client({
endpoint: env.STORAGE_ENDPOINT,
region: env.STORAGE_REGION,
forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === "true",
credentials: {
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY,
...(env.STORAGE_SESSION_TOKEN
? { sessionToken: env.STORAGE_SESSION_TOKEN }
: {}),
},
});
}
@@ -0,0 +1,49 @@
import { describe, expect, it, vi } from "vitest";
import {
ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS,
ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
registerAdminSalesTaskReminderSweepCron,
resolveAdminSalesTaskReminderSweepConfig,
} from "./admin-sales-task-reminder-cron.js";
describe("admin sales task reminder cron", () => {
it("uses a bounded explicit batch size", () => {
expect(resolveAdminSalesTaskReminderSweepConfig({})).toEqual({
take: ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
});
expect(
resolveAdminSalesTaskReminderSweepConfig({
ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE: "75",
})
).toEqual({ take: 75 });
expect(
resolveAdminSalesTaskReminderSweepConfig({
ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE: "invalid",
})
).toEqual({ take: ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE });
});
it("registers the idempotent 15-minute maintenance scheduler", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerAdminSalesTaskReminderSweepCron(
{ upsertJobScheduler },
{ ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE: "25" }
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
take: 25,
},
}
);
});
});
@@ -0,0 +1,41 @@
export const ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID =
"admin-sales-task-reminder-sweep" as const;
export const ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS = 15 * 60 * 1000;
export const ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE = 500;
export function resolveAdminSalesTaskReminderSweepConfig(
env: NodeJS.ProcessEnv = process.env
): { readonly take: number } {
const parsed = Number(env.ADMIN_SALES_TASK_REMINDER_SWEEP_BATCH_SIZE);
return {
take:
Number.isFinite(parsed) && parsed > 0
? Math.floor(parsed)
: ADMIN_SALES_TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
};
}
export async function registerAdminSalesTaskReminderSweepCron(
queue: {
upsertJobScheduler(
schedulerId: string,
options: { every: number },
template: { name: string; data?: Record<string, unknown> }
): Promise<unknown>;
},
env: NodeJS.ProcessEnv = process.env
): Promise<void> {
const config = resolveAdminSalesTaskReminderSweepConfig(env);
await queue.upsertJobScheduler(
ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
{ every: ADMIN_SALES_TASK_REMINDER_SWEEP_EVERY_MS },
{
name: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: ADMIN_SALES_TASK_REMINDER_SWEEP_JOB_ID,
take: config.take,
},
}
);
}
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
import {
bootstrapAuditEventPersist,
@@ -66,14 +67,43 @@ describe("bootstrapAuditEventPersist", () => {
return fakeWorker();
});
const createWorkerImpl = inner as unknown as WorkerImpl;
const runInTransaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) =>
fn({
const payload = { tenantId: "tenant_1", action: "x.do", actorType: "system" };
const lockedUntil = "2099-07-17T10:05:00.000Z";
let observedTenantId: string | undefined;
const runInTransaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => {
observedTenantId = getRlsExecutionContext()?.tenantId;
return fn({
$queryRaw: async () => [],
auditEvent: {
findFirst: async () => null,
create: async () => ({ id: "audit_1", eventHash: "h1" }),
},
})
);
auditChainHead: {
findUnique: async () => null,
upsert: async () => undefined,
},
auditChainCheckpointHead: {
findUnique: async () => null,
},
auditChainCheckpoint: {
findUnique: async () => null,
findFirst: async () => null,
},
outboxEvent: {
findFirst: async () => ({
id: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload,
status: "PROCESSING",
deliveryGeneration: 0,
dispatchKey: null,
lockedUntil: new Date(lockedUntil),
}),
updateMany: async () => ({ count: 1 }),
},
});
});
bootstrapAuditEventPersist({ WORKER_QUEUE_GROUP: "general" }, { runInTransaction: runInTransaction as never, createWorkerImpl });
expect(captured).not.toBeNull();
@@ -83,10 +113,14 @@ describe("bootstrapAuditEventPersist", () => {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload: { action: "x.do", actorType: "system" },
payload,
dispatchKey: null,
deliveryGeneration: 0,
lockedUntil,
},
});
}
expect(runInTransaction).toHaveBeenCalledTimes(1);
expect(observedTenantId).toBe("tenant_1");
});
});
@@ -25,6 +25,7 @@ import {
type AuditEventPersistLogger,
type AuditEventPersistTransactionRunner,
} from "./audit-event-persist.js";
import { runWorkerPayloadWithRlsContext } from "../rls-context.js";
export type AuditEventPersistBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
@@ -85,7 +86,7 @@ export function bootstrapAuditEventPersist(
runInTransaction,
...(logger ? { logger } : {}),
};
return runAuditEventPersistJob(job.data, handlerDeps);
return runWorkerPayloadWithRlsContext(job.data, () => runAuditEventPersistJob(job.data, handlerDeps));
};
const createWorkerImpl = deps.createWorkerImpl ?? defaultCreateWorker;
@@ -1,11 +1,13 @@
import { describe, expect, it, vi } from "vitest";
import { computeAuditHash } from "@hivegrid/db/audit-hash";
import type { AuditChainHeadRecord } from "@hivegrid/db/audit-chain-checkpoint.server";
import {
AUDIT_EVENT_OUTBOX_QUEUE,
AuditEventPersistJobError,
parseAuditEventPersistJob,
runAuditEventPersistJob,
type AuditEventPersistDependencies,
type AuditEventPersistTransactionRunner,
type AuditEventPersistTxClient,
} from "./audit-event-persist.js";
@@ -19,64 +21,219 @@ type PersistedRow = {
createdAt: Date;
};
function makeStore(): {
const TEST_NOW = new Date("2026-07-17T10:00:00.000Z");
const TEST_LEASE = new Date("2026-07-17T10:05:00.000Z");
const basePayload = {
tenantId: "tenant_1",
actorId: "user_1",
actorType: "user",
actorEmail: null,
actorIp: null,
actorUserAgent: null,
action: "widget.created",
targetType: "widget",
targetId: "widget_1",
requestId: null,
sourceModule: "crm",
outcome: "success",
dataSensitivity: "internal",
regulatoryScope: ["soc2"],
metadata: null,
};
function auditJob(overrides: Record<string, unknown> = {}) {
return {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload: basePayload,
dispatchKey: null,
deliveryGeneration: 0,
lockedUntil: TEST_LEASE.toISOString(),
...overrides,
};
}
const baseJob = auditJob();
type DurableOutbox = {
id: string;
tenantId: string | null;
eventType: string;
payload: unknown;
status: "PENDING" | "PROCESSING" | "DONE" | "FAILED" | "CANCELLED";
deliveryGeneration: number;
dispatchKey: string | null;
lockedUntil: Date | null;
processedAt: Date | null;
lastError: string | null;
};
function makeStore(
jobs: ReturnType<typeof auditJob>[] = [baseJob],
options: { zeroAtUpdate?: number } = {},
): {
rows: PersistedRow[];
runInTransaction: AuditEventPersistTransactionRunner;
register(job: ReturnType<typeof auditJob>, overrides?: Partial<DurableOutbox>): void;
outbox: Map<string, DurableOutbox>;
updateMany: ReturnType<typeof vi.fn>;
} {
const rows: PersistedRow[] = [];
let idSeq = 0;
let chainSeq = 0;
const heads = new Map<string, AuditChainHeadRecord>();
const outbox = new Map<string, DurableOutbox>();
const register = (job: ReturnType<typeof auditJob>, overrides: Partial<DurableOutbox> = {}) => {
outbox.set(job.outboxEventId, {
id: job.outboxEventId,
tenantId: job.tenantId as string | null,
eventType: job.eventType,
payload: job.payload,
status: "PROCESSING",
deliveryGeneration: job.deliveryGeneration as number,
dispatchKey: job.dispatchKey as string | null,
lockedUntil: new Date(job.lockedUntil as string),
processedAt: null,
lastError: null,
...overrides,
});
};
jobs.forEach((job) => register(job));
let updateAttempt = 0;
const findFirst = vi.fn(
async (args: Parameters<AuditEventPersistTxClient["outboxEvent"]["findFirst"]>[0]) => {
const row = outbox.get(args.where.id);
if (
!row ||
row.tenantId !== args.where.tenantId ||
row.eventType !== args.where.eventType ||
row.deliveryGeneration !== args.where.deliveryGeneration
) {
return null;
}
return { ...row };
},
);
const updateMany = vi.fn(
async (args: Parameters<AuditEventPersistTxClient["outboxEvent"]["updateMany"]>[0]) => {
updateAttempt += 1;
if (options.zeroAtUpdate === updateAttempt) return { count: 0 };
const row = outbox.get(args.where.id);
if (
!row ||
row.tenantId !== args.where.tenantId ||
row.eventType !== args.where.eventType ||
row.deliveryGeneration !== args.where.deliveryGeneration ||
row.dispatchKey !== args.where.dispatchKey ||
row.status !== args.where.status ||
row.lockedUntil?.getTime() !== args.where.lockedUntil.getTime() ||
!sameJson(row.payload, args.where.payload.equals)
) {
return { count: 0 };
}
if ("status" in args.data) {
row.status = args.data.status;
row.processedAt = args.data.processedAt;
row.lastError = args.data.lastError;
row.lockedUntil = args.data.lockedUntil;
} else {
row.lockedUntil = args.data.lockedUntil;
}
return { count: 1 };
},
);
const tx: AuditEventPersistTxClient = {
$queryRaw: async <T = unknown>() => [] as T,
auditEvent: {
async findFirst({ where }) {
const tenantRows = rows.filter((r) => r.tenantId === where.tenantId && r.eventHash !== null);
if (tenantRows.length === 0) return null;
// newest first
tenantRows.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
return { eventHash: tenantRows[0]?.eventHash ?? null };
return tenantRows[0] ? { eventHash: tenantRows[0].eventHash, createdAt: tenantRows[0].createdAt } : null;
},
async create({ data }) {
idSeq += 1;
chainSeq += 1;
const row: PersistedRow = {
id: `audit_${idSeq}`,
id: data.id,
tenantId: data.tenantId,
action: data.action,
eventHash: data.eventHash,
previousHash: data.previousHash,
createdAt: new Date(2026, 4, 17, 0, 0, 0, chainSeq),
createdAt: data.createdAt,
};
rows.push(row);
return { id: row.id, eventHash: row.eventHash };
},
},
auditChainCheckpoint: {
async findFirst() {
return null;
},
async findUnique() {
return null;
},
},
auditChainCheckpointHead: {
async findUnique() {
return null;
},
},
auditChainHead: {
async findUnique({ where }) {
return heads.get(where.chainKey) ?? null;
},
async upsert({ where, create, update }) {
const value = heads.has(where.chainKey) ? { chainKey: where.chainKey, ...update } : create;
heads.set(where.chainKey, value);
return undefined;
},
},
outboxEvent: { findFirst, updateMany },
};
const runInTransaction: AuditEventPersistTransactionRunner = async <T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T> => fn(tx);
return { rows, runInTransaction };
const runInTransaction: AuditEventPersistTransactionRunner = async <T>(
fn: (tx: AuditEventPersistTxClient) => Promise<T>,
): Promise<T> => {
const rowsSnapshot = rows.map((row) => ({ ...row }));
const headsSnapshot = new Map([...heads].map(([key, value]) => [key, { ...value }]));
const outboxSnapshot = new Map(
[...outbox].map(([key, value]) => [key, { ...value, lockedUntil: value.lockedUntil ? new Date(value.lockedUntil) : null }]),
);
try {
return await fn(tx);
} catch (error) {
rows.splice(0, rows.length, ...rowsSnapshot);
heads.clear();
headsSnapshot.forEach((value, key) => heads.set(key, value));
outbox.clear();
outboxSnapshot.forEach((value, key) => outbox.set(key, value));
throw error;
}
};
return { rows, runInTransaction, register, outbox, updateMany };
}
const baseJob = {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: "audit.event.create",
payload: {
tenantId: "tenant_1",
actorId: "user_1",
actorType: "user",
actorEmail: null,
actorIp: null,
actorUserAgent: null,
action: "widget.created",
targetType: "widget",
targetId: "widget_1",
requestId: null,
sourceModule: "crm",
outcome: "success",
dataSensitivity: "internal",
regulatoryScope: ["soc2"],
metadata: null,
},
};
function sameJson(left: unknown, right: unknown): boolean {
const sort = (value: unknown): unknown => {
if (Array.isArray(value)) return value.map(sort);
if (value && typeof value === "object") {
return Object.fromEntries(
Object.keys(value as Record<string, unknown>)
.sort()
.map((key) => [key, sort((value as Record<string, unknown>)[key])]),
);
}
return value;
};
return JSON.stringify(sort(left)) === JSON.stringify(sort(right));
}
function auditDependencies(
runInTransaction: AuditEventPersistTransactionRunner,
overrides: Partial<AuditEventPersistDependencies> = {},
): AuditEventPersistDependencies {
return { runInTransaction, now: () => TEST_NOW, ...overrides };
}
describe("AUDIT_EVENT_OUTBOX_QUEUE", () => {
it("matches the outbox audit queue name", () => {
@@ -104,46 +261,164 @@ describe("parseAuditEventPersistJob", () => {
});
it("normalizes payload through buildAuditEventCreateData", () => {
const parsed = parseAuditEventPersistJob({
eventType: "audit.event.create",
tenantId: "tenant_1",
payload: { action: "x.do", actorType: " ", regulatoryScope: ["SOC2"] },
});
const parsed = parseAuditEventPersistJob(auditJob({
payload: { tenantId: "tenant_1", action: "x.do", actorType: " ", regulatoryScope: ["SOC2"] },
}));
expect(parsed.payload.actorType).toBe("system");
expect(parsed.payload.regulatoryScope).toEqual(["soc2"]);
expect(parsed.payload.outcome).toBe("success");
expect(parsed.payload.dataSensitivity).toBe("internal");
});
it("requires the full durable outbox identity including dispatch, generation, and lease", () => {
for (const key of ["outboxEventId", "dispatchKey", "deliveryGeneration", "lockedUntil"] as const) {
const invalid = { ...baseJob } as Record<string, unknown>;
delete invalid[key];
expect(() => parseAuditEventPersistJob(invalid)).toThrowError(
expect.objectContaining({ code: "INVALID_PAYLOAD" }),
);
}
});
it("rejects a non-global payload whose tenant differs from the outbox envelope", () => {
expect(() =>
parseAuditEventPersistJob(auditJob({
tenantId: "tenant_2",
payload: { ...basePayload, tenantId: "tenant_1" },
})),
).toThrowError(expect.objectContaining({ code: "INVALID_PAYLOAD" }));
});
});
describe("runAuditEventPersistJob", () => {
it("persists the event with prevHash=null on chain root", async () => {
const { rows, runInTransaction } = makeStore();
const result = await runAuditEventPersistJob(baseJob, { runInTransaction });
expect(result).toEqual({ persisted: true, auditEventId: "audit_1", eventHash: expect.stringMatching(/^[0-9a-f]{64}$/) });
const { rows, runInTransaction, outbox, updateMany } = makeStore();
const result = await runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction));
expect(result).toEqual({ persisted: true, auditEventId: expect.any(String), eventHash: expect.stringMatching(/^v2:[0-9a-f]{64}$/) });
expect(rows).toHaveLength(1);
expect(rows[0]?.previousHash).toBeNull();
expect(outbox.get("outbox_1")).toMatchObject({ status: "DONE", lockedUntil: null, lastError: null });
expect(updateMany).toHaveBeenCalledTimes(2);
});
it("rejects a forged Redis job with no durable outbox row", async () => {
const { rows, runInTransaction } = makeStore([]);
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(rows).toEqual([]);
});
it("rejects tenant, generation, dispatch-key, and canonical payload mismatches", async () => {
const cases = [
auditJob({ tenantId: "tenant_2", payload: { ...basePayload, tenantId: "tenant_2" } }),
auditJob({ deliveryGeneration: 1 }),
auditJob({ dispatchKey: "audit:forged" }),
auditJob({ payload: { ...basePayload, action: "widget.forged" } }),
];
for (const forged of cases) {
const { rows, runInTransaction } = makeStore([baseJob]);
await expect(
runAuditEventPersistJob(forged, auditDependencies(runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(rows).toEqual([]);
}
});
it("rejects PENDING, DONE, and stale terminal generations", async () => {
for (const status of ["PENDING", "DONE", "FAILED", "CANCELLED"] as const) {
const state = makeStore([baseJob]);
state.outbox.get("outbox_1")!.status = status;
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toEqual([]);
}
});
it("rejects an expired or mismatched dispatcher lease", async () => {
const expired = makeStore([baseJob]);
await expect(
runAuditEventPersistJob(
baseJob,
auditDependencies(expired.runInTransaction, { now: () => TEST_LEASE }),
),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
const mismatchedJob = auditJob({ lockedUntil: "2026-07-17T10:06:00.000Z" });
const mismatched = makeStore([baseJob]);
await expect(
runAuditEventPersistJob(mismatchedJob, auditDependencies(mismatched.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
});
it("fails closed when the exact-generation claim CAS loses its race", async () => {
const state = makeStore([baseJob], { zeroAtUpdate: 1 });
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toEqual([]);
expect(state.outbox.get("outbox_1")).toMatchObject({ status: "PROCESSING", lockedUntil: TEST_LEASE });
});
it("rolls back the AuditEvent and claim when the atomic DONE receipt matches zero rows", async () => {
const state = makeStore([baseJob], { zeroAtUpdate: 2 });
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toEqual([]);
expect(state.outbox.get("outbox_1")).toMatchObject({ status: "PROCESSING", lockedUntil: TEST_LEASE });
});
it("fails closed on a duplicate delivery after the consumer-owned DONE receipt", async () => {
const state = makeStore([baseJob]);
await runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction));
await expect(
runAuditEventPersistJob(baseJob, auditDependencies(state.runInTransaction)),
).rejects.toMatchObject({ code: "OUTBOX_PROOF_INVALID" });
expect(state.rows).toHaveLength(1);
});
it("accepts an exact global audit payload carried by a tenant-scoped durable envelope", async () => {
const globalJob = auditJob({
payload: { ...basePayload, tenantId: null, action: "admin.portalDomain.killSwitchEnabled" },
});
const state = makeStore([globalJob]);
await expect(
runAuditEventPersistJob(globalJob, auditDependencies(state.runInTransaction)),
).resolves.toMatchObject({ persisted: true });
expect(state.rows).toHaveLength(1);
expect(state.rows[0]?.tenantId).toBeNull();
});
it("links the chain when a prior event exists for the same tenant", async () => {
const { rows, runInTransaction } = makeStore();
await runAuditEventPersistJob(baseJob, { runInTransaction });
const second = await runAuditEventPersistJob(
{ ...baseJob, payload: { ...baseJob.payload, action: "widget.updated" } },
{ runInTransaction }
);
const secondJob = auditJob({
outboxEventId: "outbox_2",
payload: { ...basePayload, action: "widget.updated" },
});
const { rows, runInTransaction } = makeStore([baseJob, secondJob]);
await runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction));
const second = await runAuditEventPersistJob(secondJob, auditDependencies(runInTransaction));
expect(rows).toHaveLength(2);
expect(rows[1]?.previousHash).toBe(rows[0]?.eventHash);
expect(second.eventHash).toBe(rows[1]?.eventHash);
});
it("isolates chains per tenant", async () => {
const { rows, runInTransaction } = makeStore();
await runAuditEventPersistJob(baseJob, { runInTransaction });
await runAuditEventPersistJob(
{ ...baseJob, tenantId: "tenant_2", payload: { ...baseJob.payload, tenantId: "tenant_2", action: "x.do" } },
{ runInTransaction }
);
const tenantTwoJob = auditJob({
outboxEventId: "outbox_2",
tenantId: "tenant_2",
payload: { ...basePayload, tenantId: "tenant_2", action: "x.do" },
});
const { rows, runInTransaction } = makeStore([baseJob, tenantTwoJob]);
await runAuditEventPersistJob(baseJob, auditDependencies(runInTransaction));
await runAuditEventPersistJob(tenantTwoJob, auditDependencies(runInTransaction));
expect(rows[0]?.tenantId).toBe("tenant_1");
expect(rows[1]?.tenantId).toBe("tenant_2");
expect(rows[1]?.previousHash).toBeNull();
@@ -151,12 +426,15 @@ describe("runAuditEventPersistJob", () => {
it("computes an eventHash consistent with computeAuditHash", async () => {
const { rows, runInTransaction } = makeStore();
await runAuditEventPersistJob(baseJob, { runInTransaction, secret: "test-secret" });
await runAuditEventPersistJob(
baseJob,
auditDependencies(runInTransaction, { secret: "test-secret" }),
);
const recomputed = computeAuditHash({
previousHash: null,
payload: {
id: "",
createdAt: "",
id: rows[0]!.id,
createdAt: rows[0]!.createdAt.toISOString(),
tenantId: "tenant_1",
actorType: "user",
actorId: "user_1",
@@ -174,6 +452,7 @@ describe("runAuditEventPersistJob", () => {
metadata: null,
},
secret: "test-secret",
version: "hmac-v2",
});
expect(rows[0]?.eventHash).toBe(recomputed);
});
@@ -184,7 +463,15 @@ describe("runAuditEventPersistJob", () => {
// through a single chained Promise. Each call sees the previously
// committed row, so the chain stays consistent under parallel
// dispatch.
const { rows, runInTransaction: serialRunner } = makeStore();
const updatedJob = auditJob({
outboxEventId: "outbox_2",
payload: { ...basePayload, action: "widget.updated" },
});
const deletedJob = auditJob({
outboxEventId: "outbox_3",
payload: { ...basePayload, action: "widget.deleted" },
});
const { rows, runInTransaction: serialRunner } = makeStore([baseJob, updatedJob, deletedJob]);
let gate: Promise<void> = Promise.resolve();
const queuedRunner: AuditEventPersistTransactionRunner = async <T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T> => {
const prev = gate;
@@ -199,20 +486,16 @@ describe("runAuditEventPersistJob", () => {
};
const logger = { error: vi.fn() };
await Promise.all([
runAuditEventPersistJob(baseJob, { runInTransaction: queuedRunner, logger }),
runAuditEventPersistJob(
{ ...baseJob, payload: { ...baseJob.payload, action: "widget.updated" } },
{ runInTransaction: queuedRunner, logger }
),
runAuditEventPersistJob(
{ ...baseJob, payload: { ...baseJob.payload, action: "widget.deleted" } },
{ runInTransaction: queuedRunner, logger }
),
runAuditEventPersistJob(baseJob, auditDependencies(queuedRunner, { logger })),
runAuditEventPersistJob(updatedJob, auditDependencies(queuedRunner, { logger })),
runAuditEventPersistJob(deletedJob, auditDependencies(queuedRunner, { logger })),
]);
expect(rows).toHaveLength(3);
expect(rows[0]?.previousHash).toBeNull();
expect(rows[1]?.previousHash).toBe(rows[0]?.eventHash);
expect(rows[2]?.previousHash).toBe(rows[1]?.eventHash);
expect(rows[1]!.createdAt.getTime()).toBeGreaterThan(rows[0]!.createdAt.getTime());
expect(rows[2]!.createdAt.getTime()).toBeGreaterThan(rows[1]!.createdAt.getTime());
expect(logger.error).not.toHaveBeenCalled();
});
});
@@ -6,26 +6,32 @@
* inserts the AuditEvent row into Postgres.
*
* The hash chain is computed inside a short Prisma transaction:
* - SELECT the most-recent eventHash for the tenant (FOR UPDATE
* equivalent via serializable isolation)
* - acquire a tenant/platform PostgreSQL advisory transaction lock
* - authenticate and read the O(1) persistent chain head
* - compute the new event's eventHash
* - INSERT the new row
* - insert the event and atomically advance the authenticated head
*
* Running this under serializable isolation ensures concurrent persists
* for the same tenant either commit in some order with a coherent
* chain, or one of them retries on serialization-failure. The handler
* is therefore safe to invoke from N parallel BullMQ workers without
* corrupting the chain.
* The advisory lock is shared with direct writers and retention, so N
* parallel BullMQ workers cannot fork a tenant or platform chain.
*
* Idempotency: the producer-side `publishAuditEventViaOutbox` writes
* one outbox row per business-write. The dispatcher delivers each row
* exactly once (claim+ack); on retry we may re-process the same
* payload but the audit insert is a plain CREATE — duplicates simply
* extend the chain by one extra row. The acceptable outcome here is
* "at-least-once": a duplicated audit row is harmless evidence.
* Authority and idempotency: Redis is transport only. Before inserting an
* AuditEvent the consumer proves the exact tenant/generation/payload/dispatch
* key/lease against the durable OutboxEvent, claims that generation with a
* CAS, then commits the AuditEvent and terminal DONE receipt atomically.
*/
import { randomUUID } from "node:crypto";
import { computeAuditHash, type AuditHashPayload } from "@hivegrid/db/audit-hash";
import {
acquireAuditChainLock,
auditChainKey,
computeAuditChainHeadMac,
verifyAuditChainHead,
verifyAuditCheckpoint,
type AuditChainHeadRecord,
type AuditChainCheckpointRecord,
} from "@hivegrid/db/audit-chain-checkpoint.server";
import {
buildAuditEventCreateData,
type AuditEventCreateData,
@@ -33,6 +39,7 @@ import {
} from "@hivegrid/db/audit-events.server";
export const AUDIT_EVENT_OUTBOX_QUEUE = "outbox.audit" as const;
const AUDIT_OUTBOX_CONSUMER_LEASE_MS = 5 * 60 * 1_000;
export interface AuditEventPersistLogger {
info?: (record: Record<string, unknown>, message?: string) => void;
@@ -41,23 +48,89 @@ export interface AuditEventPersistLogger {
}
export interface AuditEventPersistTxClient {
$queryRaw<T = unknown>(query: TemplateStringsArray, ...values: unknown[]): Promise<T>;
auditEvent: {
findFirst(args: {
where: { tenantId: string | null; eventHash: { not: null } };
orderBy: { createdAt: "desc" };
select: { eventHash: true };
}): Promise<{ eventHash: string | null } | null>;
orderBy: Array<{ createdAt: "desc" } | { id: "desc" }>;
select: { eventHash: true; createdAt: true };
}): Promise<{ eventHash: string | null; createdAt: Date } | null>;
create(args: {
data: AuditEventCreateData & { previousHash: string | null; eventHash: string };
data: AuditEventCreateData & { id: string; createdAt: Date; previousHash: string | null; eventHash: string };
}): Promise<{ id: string; eventHash: string }>;
};
auditChainCheckpoint: {
findFirst(args: {
where: { chainKey: string };
orderBy: Array<{ createdAt: "desc" } | { id: "desc" }>;
}): Promise<AuditChainCheckpointRecord | null>;
findUnique(args: { where: { id: string } }): Promise<AuditChainCheckpointRecord | null>;
};
auditChainCheckpointHead: {
findUnique(args: {
where: { chainKey: string };
select: { checkpointId: true };
}): Promise<{ checkpointId: string } | null>;
};
auditChainHead: {
findUnique(args: { where: { chainKey: string } }): Promise<AuditChainHeadRecord | null>;
upsert(args: {
where: { chainKey: string };
create: AuditChainHeadRecord;
update: Omit<AuditChainHeadRecord, "chainKey">;
}): Promise<unknown>;
};
outboxEvent: {
findFirst(args: {
where: {
id: string;
tenantId: string | null;
eventType: "audit.event.create";
deliveryGeneration: number;
};
select: {
id: true;
tenantId: true;
eventType: true;
payload: true;
status: true;
deliveryGeneration: true;
dispatchKey: true;
lockedUntil: true;
};
}): Promise<{
id: string;
tenantId: string | null;
eventType: string;
payload: unknown;
status: "PENDING" | "PROCESSING" | "DONE" | "FAILED" | "CANCELLED";
deliveryGeneration: number;
dispatchKey: string | null;
lockedUntil: Date | null;
} | null>;
updateMany(args: {
where: {
id: string;
tenantId: string | null;
eventType: "audit.event.create";
deliveryGeneration: number;
dispatchKey: string | null;
status: "PROCESSING";
lockedUntil: Date;
payload: { equals: unknown };
};
data:
| { lockedUntil: Date }
| { status: "DONE"; processedAt: Date; lastError: null; lockedUntil: null };
}): Promise<{ count: number }>;
};
}
export interface AuditEventPersistTransactionRunner {
/**
* Run `fn` inside a Prisma transaction. Production wiring passes
* `prisma.$transaction` with isolationLevel="Serializable" so that
* the prevHash lookup + insert form an atomic chain link.
* Run `fn` inside a Prisma transaction. Production additionally uses
* serializable isolation; the per-chain advisory lock is the primary
* cross-writer serialization primitive.
*/
<T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T>;
}
@@ -70,11 +143,13 @@ export interface AuditEventPersistDependencies {
* leaves this undefined so the helper reads `process.env.AUDIT_CHAIN_SECRET`.
*/
readonly secret?: string | null;
/** Clock override for durable lease checks and deterministic tests. */
readonly now?: () => Date;
}
export class AuditEventPersistJobError extends Error {
override readonly name = "AuditEventPersistJobError";
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "MISSING_ACTION";
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "MISSING_ACTION" | "OUTBOX_PROOF_INVALID";
constructor(message: string, code: AuditEventPersistJobError["code"]) {
super(message);
this.code = code;
@@ -82,9 +157,13 @@ export class AuditEventPersistJobError extends Error {
}
export interface AuditEventPersistJobData {
readonly outboxEventId?: string;
readonly outboxEventId: string;
readonly tenantId: string | null;
readonly eventType: string;
readonly eventType: "audit.event.create";
readonly dispatchKey: string | null;
readonly deliveryGeneration: number;
readonly lockedUntil: Date;
readonly rawPayload: Readonly<Record<string, unknown>>;
readonly payload: AuditEventCreateData;
}
@@ -95,7 +174,7 @@ export interface AuditEventPersistResult {
}
export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobData {
if (!data || typeof data !== "object") {
if (!data || typeof data !== "object" || Array.isArray(data)) {
throw new AuditEventPersistJobError("audit-event persist job missing data envelope", "INVALID_PAYLOAD");
}
const envelope = data as Record<string, unknown>;
@@ -110,6 +189,9 @@ export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobDa
if (!rawPayload || typeof rawPayload !== "object") {
throw new AuditEventPersistJobError("audit-event persist job missing payload", "INVALID_PAYLOAD");
}
if (Array.isArray(rawPayload)) {
throw new AuditEventPersistJobError("audit-event persist job payload must be an object", "INVALID_PAYLOAD");
}
const inner = rawPayload as Record<string, unknown>;
const action = typeof inner.action === "string" ? inner.action.trim() : "";
if (action.length === 0) {
@@ -118,18 +200,67 @@ export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobDa
// Re-run normalization in case the producer emitted an older payload
// shape — defensive. `buildAuditEventCreateData` is idempotent.
const normalized = buildAuditEventCreateData(inner as unknown as AuditEventInput);
const outboxEventId = parseIdentifier(envelope.outboxEventId);
const tenantId = parseNullableIdentifier(envelope, "tenantId");
const dispatchKey = parseNullableDispatchKey(envelope);
const deliveryGeneration = envelope.deliveryGeneration;
const lockedUntil = parseLease(envelope.lockedUntil);
if (
!outboxEventId ||
tenantId === undefined ||
dispatchKey === undefined ||
!Number.isSafeInteger(deliveryGeneration) ||
(deliveryGeneration as number) < 0 ||
!lockedUntil
) {
throw new AuditEventPersistJobError("audit-event persist job has no exact outbox identity", "INVALID_PAYLOAD");
}
if (normalized.tenantId !== null && normalized.tenantId !== tenantId) {
throw new AuditEventPersistJobError("audit-event tenant does not match its outbox envelope", "INVALID_PAYLOAD");
}
return {
outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
tenantId: typeof envelope.tenantId === "string" ? envelope.tenantId : null,
eventType,
outboxEventId,
tenantId,
eventType: "audit.event.create",
dispatchKey,
deliveryGeneration: deliveryGeneration as number,
lockedUntil,
rawPayload: inner,
payload: normalized,
};
}
function toHashPayload(data: AuditEventCreateData): AuditHashPayload {
function parseIdentifier(value: unknown): string | null {
if (typeof value !== "string") return null;
const normalized = value.trim();
return normalized.length > 0 && normalized.length <= 191 && !normalized.includes("\u0000") ? normalized : null;
}
function parseNullableIdentifier(
envelope: Readonly<Record<string, unknown>>,
key: "tenantId",
): string | null | undefined {
if (!Object.hasOwn(envelope, key)) return undefined;
if (envelope[key] === null) return null;
return parseIdentifier(envelope[key]) ?? undefined;
}
function parseNullableDispatchKey(envelope: Readonly<Record<string, unknown>>): string | null | undefined {
if (!Object.hasOwn(envelope, "dispatchKey")) return undefined;
if (envelope.dispatchKey === null) return null;
if (typeof envelope.dispatchKey !== "string") return undefined;
return /^[A-Za-z0-9._:-]{1,256}$/u.test(envelope.dispatchKey) ? envelope.dispatchKey : undefined;
}
function parseLease(value: unknown): Date | null {
const parsed = value instanceof Date ? new Date(value.getTime()) : typeof value === "string" ? new Date(value) : null;
return parsed && Number.isFinite(parsed.getTime()) ? parsed : null;
}
function toHashPayload(data: AuditEventCreateData, id: string, createdAt: Date): AuditHashPayload {
return {
id: "",
createdAt: "",
id,
createdAt: createdAt.toISOString(),
tenantId: data.tenantId,
actorType: data.actorType,
actorId: data.actorId,
@@ -156,20 +287,139 @@ export async function runAuditEventPersistJob(
const payload = job.payload;
const result = await deps.runInTransaction(async (tx) => {
const prior = await tx.auditEvent.findFirst({
where: { tenantId: payload.tenantId, eventHash: { not: null } },
orderBy: { createdAt: "desc" },
select: { eventHash: true },
const durable = await tx.outboxEvent.findFirst({
where: {
id: job.outboxEventId,
tenantId: job.tenantId,
eventType: job.eventType,
deliveryGeneration: job.deliveryGeneration,
},
select: {
id: true,
tenantId: true,
eventType: true,
payload: true,
status: true,
deliveryGeneration: true,
dispatchKey: true,
lockedUntil: true,
},
});
const previousHash = prior?.eventHash ?? null;
const observedAt = deps.now?.() ?? new Date();
if (
!durable ||
durable.status !== "PROCESSING" ||
durable.dispatchKey !== job.dispatchKey ||
!durable.lockedUntil ||
durable.lockedUntil.getTime() !== job.lockedUntil.getTime() ||
durable.lockedUntil.getTime() <= observedAt.getTime() ||
!sameCanonicalJson(durable.payload, job.rawPayload)
) {
throw invalidOutboxProof();
}
const claimedLease = new Date(
Math.max(
observedAt.getTime() + AUDIT_OUTBOX_CONSUMER_LEASE_MS,
durable.lockedUntil.getTime() + 1,
),
);
const claimed = await tx.outboxEvent.updateMany({
where: {
id: job.outboxEventId,
tenantId: job.tenantId,
eventType: job.eventType,
deliveryGeneration: job.deliveryGeneration,
dispatchKey: job.dispatchKey,
status: "PROCESSING",
lockedUntil: durable.lockedUntil,
payload: { equals: job.rawPayload },
},
data: { lockedUntil: claimedLease },
});
if (claimed.count !== 1) throw invalidOutboxProof();
await acquireAuditChainLock(tx, payload.tenantId);
const key = auditChainKey(payload.tenantId);
const head = await tx.auditChainHead.findUnique({ where: { chainKey: key } });
if (head && !verifyAuditChainHead(head, deps.secret)) {
throw new Error(`Audit chain-head authentication failed for ${key}.`);
}
let previousHash = head?.eventHash ?? null;
let previousCreatedAt = head?.eventCreatedAt ?? null;
if (!previousHash) {
const prior = await tx.auditEvent.findFirst({
where: { tenantId: payload.tenantId, eventHash: { not: null } },
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
select: { eventHash: true, createdAt: true },
});
previousHash = prior?.eventHash ?? null;
previousCreatedAt = prior?.createdAt ?? null;
}
if (!previousHash) {
const checkpointHead = await tx.auditChainCheckpointHead.findUnique({
where: { chainKey: key },
select: { checkpointId: true },
});
const checkpoint = checkpointHead
? await tx.auditChainCheckpoint.findUnique({ where: { id: checkpointHead.checkpointId } })
: await tx.auditChainCheckpoint.findFirst({
where: { chainKey: key },
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
});
if (checkpoint) {
if (!verifyAuditCheckpoint(checkpoint, deps.secret)) {
throw new Error(`Audit checkpoint authentication failed for ${checkpoint.chainKey}.`);
}
previousHash = checkpoint.lastDeletedEventHash;
previousCreatedAt = checkpoint.deletedThroughAt;
}
}
const id = randomUUID();
const now = deps.now?.() ?? new Date();
const createdAt = previousCreatedAt && now.getTime() <= previousCreatedAt.getTime()
? new Date(previousCreatedAt.getTime() + 1)
: now;
const eventHash = computeAuditHash({
previousHash,
payload: toHashPayload(payload),
payload: toHashPayload(payload, id, createdAt),
secret: deps.secret,
version: "hmac-v2",
});
const inserted = await tx.auditEvent.create({
data: { ...payload, previousHash, eventHash },
data: { ...payload, id, createdAt, previousHash, eventHash },
});
const headWithoutMac: Omit<AuditChainHeadRecord, "headMac"> = {
chainKey: key,
tenantId: payload.tenantId,
eventId: id,
eventHash,
eventCreatedAt: createdAt,
};
const headMac = computeAuditChainHeadMac(headWithoutMac, deps.secret);
await tx.auditChainHead.upsert({
where: { chainKey: key },
create: { ...headWithoutMac, headMac },
update: { tenantId: payload.tenantId, eventId: id, eventHash, eventCreatedAt: createdAt, headMac },
});
const receipt = await tx.outboxEvent.updateMany({
where: {
id: job.outboxEventId,
tenantId: job.tenantId,
eventType: job.eventType,
deliveryGeneration: job.deliveryGeneration,
dispatchKey: job.dispatchKey,
status: "PROCESSING",
lockedUntil: claimedLease,
payload: { equals: job.rawPayload },
},
data: {
status: "DONE",
processedAt: createdAt,
lastError: null,
lockedUntil: null,
},
});
if (receipt.count !== 1) throw invalidOutboxProof();
return { auditEventId: inserted.id, eventHash };
});
@@ -186,3 +436,30 @@ export async function runAuditEventPersistJob(
return { persisted: true, auditEventId: result.auditEventId, eventHash: result.eventHash };
}
function invalidOutboxProof(): AuditEventPersistJobError {
return new AuditEventPersistJobError(
"audit-event persist job does not match an active durable outbox generation",
"OUTBOX_PROOF_INVALID",
);
}
function sameCanonicalJson(left: unknown, right: unknown): boolean {
try {
return JSON.stringify(canonicalJson(left)) === JSON.stringify(canonicalJson(right));
} catch {
return false;
}
}
function canonicalJson(value: unknown): unknown {
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
if (typeof value === "number" && Number.isFinite(value)) return value;
if (Array.isArray(value)) return value.map(canonicalJson);
if (!value || typeof value !== "object") throw new TypeError("Value is not canonical JSON");
return Object.fromEntries(
Object.keys(value as Record<string, unknown>)
.sort()
.map((key) => [key, canonicalJson((value as Record<string, unknown>)[key])]),
);
}
@@ -3,16 +3,14 @@ import { describe, expect, it, vi } from "vitest";
import {
AUDIT_RETENTION_CRON_DEFAULT,
AUDIT_RETENTION_JOB_ID,
DEFAULT_AUDIT_RETENTION_DAYS,
GDPR_AUDIT_RETENTION_DAYS,
SOC2_AUDIT_RETENTION_DAYS,
AUDIT_RETENTION_YEARS,
registerAuditRetentionCron,
resolveAuditRetentionPolicy,
resolveAuditRetentionThreshold,
runAuditRetentionJob,
} from "./audit-retention.js";
describe("registerAuditRetentionCron", () => {
it("upserts the nightly scheduler with the default pattern (idempotent key)", async () => {
it("upserts the nightly scheduler with the default pattern", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
const { pattern } = await registerAuditRetentionCron({ upsertJobScheduler }, {});
expect(pattern).toBe(AUDIT_RETENTION_CRON_DEFAULT);
@@ -30,62 +28,68 @@ describe("registerAuditRetentionCron", () => {
});
});
describe("resolveAuditRetentionPolicy", () => {
it("returns 7y for SOC2 + HIPAA tenants (HIPAA wins on label)", () => {
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["soc2"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "soc2" });
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["hipaa"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "hipaa" });
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["soc2", "hipaa"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "hipaa" });
});
it("returns 6y for GDPR-only tenants", () => {
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["gdpr"] })).toEqual({ retentionDays: GDPR_AUDIT_RETENTION_DAYS, source: "gdpr" });
});
it("returns 1y default for tenants with no regulatory scope", () => {
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: [] })).toEqual({ retentionDays: DEFAULT_AUDIT_RETENTION_DAYS, source: "default" });
describe("resolveAuditRetentionThreshold", () => {
it("uses an exact ten-calendar-year threshold, including leap years", () => {
expect(AUDIT_RETENTION_YEARS).toBe(10);
expect(resolveAuditRetentionThreshold(new Date("2028-02-29T12:00:00.000Z")).toISOString()).toBe(
"2018-03-01T12:00:00.000Z",
);
});
});
describe("runAuditRetentionJob", () => {
it("walks tenants, deletes rows, and emits a meta-audit for each non-empty sweep", async () => {
it("includes the platform chain and rotates bounded batches fairly", async () => {
const now = new Date("2026-05-17T00:00:00.000Z");
const tenants = [
{ tenantId: "t1", regulatoryScopes: ["soc2"] },
{ tenantId: "t2", regulatoryScopes: [] },
];
const eventDeleter = vi.fn(async (tenantId: string, threshold: Date) => {
void threshold;
return tenantId === "t1" ? 42 : 0;
const calls = new Map<string | null, number>();
const batchPruner = vi.fn(async (tenantId: string | null) => {
const call = (calls.get(tenantId) ?? 0) + 1;
calls.set(tenantId, call);
return { tenantId, deletedCount: 1, hasMore: call < 2, checkpointMac: "cp1:test" };
});
const auditWriter = vi.fn(async () => {});
const auditWriter = vi.fn(async () => undefined);
const result = await runAuditRetentionJob({
now,
tenantLister: async () => tenants,
eventDeleter,
chainLister: async () => [null, "tenant-1"],
batchPruner,
auditWriter,
maxBatches: 4,
});
expect(result).toEqual({ inspectedTenants: 2, deletedRows: 42, failures: [] });
expect(eventDeleter).toHaveBeenCalledTimes(2);
// SOC2 -> 7y threshold
const soc2Threshold = eventDeleter.mock.calls[0]?.[1];
expect(soc2Threshold).toBeInstanceOf(Date);
expect(soc2Threshold.toISOString()).toBe(new Date(now.getTime() - SOC2_AUDIT_RETENTION_DAYS * 86_400_000).toISOString());
// Only t1 had deletions -> only one meta-audit
expect(auditWriter).toHaveBeenCalledTimes(1);
expect(batchPruner.mock.calls.map((call) => call[0])).toEqual([null, "tenant-1", null, "tenant-1"]);
expect(result).toEqual({
inspectedChains: 2,
batches: 4,
deletedRows: 4,
backlogChains: [],
failures: [],
});
expect(auditWriter).toHaveBeenCalledWith(
expect.objectContaining({ tenantId: null, deletedCount: 2, retentionYears: 10 }),
);
expect(auditWriter).toHaveBeenCalledWith(
expect.objectContaining({ tenantId: "tenant-1", deletedCount: 2, retentionYears: 10 }),
);
});
it("records failures without aborting other tenants", async () => {
const now = new Date("2026-05-17T00:00:00.000Z");
const tenants = [
{ tenantId: "t-fail", regulatoryScopes: [] },
{ tenantId: "t-ok", regulatoryScopes: [] },
];
const eventDeleter = vi.fn(async (tenantId: string) => {
if (tenantId === "t-fail") throw new Error("boom");
return 5;
it("reports backlog when the global batch budget is exhausted", async () => {
const result = await runAuditRetentionJob({
chainLister: async () => ["t1", "t2"],
batchPruner: async (tenantId) => ({ tenantId, deletedCount: 1, hasMore: true, checkpointMac: "cp1:test" }),
maxBatches: 1,
});
expect(result.batches).toBe(1);
expect(result.backlogChains).toEqual(["t2", "t1"]);
});
it("records a chain failure without aborting other chains", async () => {
const result = await runAuditRetentionJob({
chainLister: async () => ["t-fail", "t-ok"],
batchPruner: async (tenantId) => {
if (tenantId === "t-fail") throw new Error("broken checkpoint");
return { tenantId, deletedCount: 5, hasMore: false, checkpointMac: "cp1:test" };
},
});
const result = await runAuditRetentionJob({ now, tenantLister: async () => tenants, eventDeleter });
expect(result.deletedRows).toBe(5);
expect(result.failures).toEqual([{ tenantId: "t-fail", error: "boom" }]);
expect(result.failures).toEqual([{ tenantId: "t-fail", error: "broken checkpoint" }]);
});
});
@@ -1,24 +1,13 @@
/**
* Phase 10 Task #67 — Audit-Event retention worker.
*
* Walks every tenant on a daily cron, looks up the tenant's retention
* window (plan-based default; configurable per-tenant via an override
* JSON in a future task), and deletes AuditEvent rows older than the
* window. Deletions are themselves audited as a system-action event so
* the cleanup itself is traceable — see
* docs/features/compliance-audit.md.
*
* Deliberately host-agnostic: the worker bootstrap (apps/worker/src/index.ts)
* supplies the seams (tenantLister, eventDeleter, auditWriter) so this
* file is trivially unit-testable without a real database.
* AuditEvent retention is a fixed ten-year policy. Each bounded batch is
* checkpointed by the database adapter before deletion, so a later verifier
* can authenticate the retained chain prefix.
*/
export const DEFAULT_AUDIT_RETENTION_DAYS = 365; // baseline / "none" scope
export const SOC2_AUDIT_RETENTION_DAYS = 7 * 365; // SOC2 + HIPAA: 7y
export const GDPR_AUDIT_RETENTION_DAYS = 6 * 365; // GDPR: 6y
export const AUDIT_RETENTION_YEARS = 10;
export const AUDIT_RETENTION_BATCH_SIZE = 1_000;
export const AUDIT_RETENTION_MAX_BATCHES = 100;
// AK-P3 AP-2 — cron wiring. Mirrors plugin-audit-retention so the worker
// bootstrap can install a nightly scheduler for the AuditEvent sweep.
export const AUDIT_RETENTION_QUEUE = "audit-retention" as const;
export const AUDIT_RETENTION_JOB_ID = "audit-retention" as const;
/** Nightly at 03:30 UTC (offset from plugin-audit-retention's 03:00). */
@@ -26,11 +15,6 @@ export const AUDIT_RETENTION_CRON_DEFAULT = "30 3 * * *" as const;
export type AuditRetentionEnv = Readonly<Record<string, string | undefined>>;
/**
* Install (or replace) the nightly AuditEvent-retention schedule on a
* BullMQ-like queue. Idempotent via a stable scheduler key. Cron pattern
* overridable via `AUDIT_RETENTION_CRON`.
*/
export async function registerAuditRetentionCron(
queue: {
upsertJobScheduler: (
@@ -50,78 +34,106 @@ export async function registerAuditRetentionCron(
return { pattern };
}
export type AuditRetentionPolicy = {
/** Hard cap in days. Events older than this are pruned. */
retentionDays: number;
/** Free-form label for log lines (`gdpr` | `soc2` | `plan:enterprise` | …). */
source: string;
};
export function resolveAuditRetentionThreshold(now: Date): Date {
const threshold = new Date(now);
threshold.setUTCFullYear(threshold.getUTCFullYear() - AUDIT_RETENTION_YEARS);
return threshold;
}
export type AuditRetentionTenant = {
tenantId: string;
/** Highest regulatoryScope used by events for this tenant. */
regulatoryScopes: ReadonlyArray<string>;
export type AuditRetentionBatchResult = {
tenantId: string | null;
deletedCount: number;
hasMore: boolean;
checkpointMac: string | null;
};
export type AuditRetentionInput = {
/** Reference "now" — defaults to `new Date()`. Injected for tests. */
now?: Date;
/** List tenants (each with the regulatoryScopes ever used by their events). */
tenantLister: () => Promise<ReadonlyArray<AuditRetentionTenant>>;
/** Delete events with createdAt < threshold for tenantId. Returns row count. */
eventDeleter: (tenantId: string, threshold: Date) => Promise<number>;
/** Optional audit writer for the meta-audit (cleanup itself). */
auditWriter?: (input: { tenantId: string; deletedCount: number; threshold: Date; policy: AuditRetentionPolicy }) => Promise<void>;
/** Logger seam. */
logger?: { info: (fields: Record<string, unknown>, msg: string) => void; error?: (fields: Record<string, unknown>, msg: string) => void };
/** Optional override for plan-resolved policy per tenant. */
policyResolver?: (tenant: AuditRetentionTenant) => AuditRetentionPolicy;
/** One aggregate query returns tenant chains plus the platform chain (null). */
chainLister: (threshold: Date) => Promise<ReadonlyArray<string | null>>;
/** Deletes one keyset-ordered, checkpointed transaction batch. */
batchPruner: (tenantId: string | null, threshold: Date, batchSize: number) => Promise<AuditRetentionBatchResult>;
auditWriter?: (input: {
tenantId: string | null;
deletedCount: number;
threshold: Date;
retentionYears: number;
}) => Promise<void>;
batchSize?: number;
maxBatches?: number;
logger?: {
info: (fields: Record<string, unknown>, msg: string) => void;
error?: (fields: Record<string, unknown>, msg: string) => void;
};
};
export function resolveAuditRetentionPolicy(tenant: AuditRetentionTenant): AuditRetentionPolicy {
const scopes = new Set(tenant.regulatoryScopes.map((s) => s.toLowerCase()));
if (scopes.has("soc2") || scopes.has("hipaa")) {
return { retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: scopes.has("hipaa") ? "hipaa" : "soc2" };
}
if (scopes.has("gdpr")) {
return { retentionDays: GDPR_AUDIT_RETENTION_DAYS, source: "gdpr" };
}
return { retentionDays: DEFAULT_AUDIT_RETENTION_DAYS, source: "default" };
}
export type AuditRetentionResult = {
inspectedTenants: number;
inspectedChains: number;
batches: number;
deletedRows: number;
failures: Array<{ tenantId: string; error: string }>;
backlogChains: Array<string | null>;
failures: Array<{ tenantId: string | null; error: string }>;
};
/**
* Run the retention sweep across every tenant. Caller (cron bootstrap)
* decides scheduling — this function is a single pass.
*/
export async function runAuditRetentionJob(input: AuditRetentionInput): Promise<AuditRetentionResult> {
const now = input.now ?? new Date();
const tenants = await input.tenantLister();
let deletedRows = 0;
const threshold = resolveAuditRetentionThreshold(input.now ?? new Date());
const batchSize = Math.max(1, Math.min(input.batchSize ?? AUDIT_RETENTION_BATCH_SIZE, 5_000));
const maxBatches = Math.max(1, input.maxBatches ?? AUDIT_RETENTION_MAX_BATCHES);
const chains = [...new Set(await input.chainLister(threshold))];
const pending = [...chains];
const deletedByChain = new Map<string | null, number>();
const failures: AuditRetentionResult["failures"] = [];
let batches = 0;
let deletedRows = 0;
for (const tenant of tenants) {
while (pending.length > 0 && batches < maxBatches) {
const tenantId = pending.shift() ?? null;
try {
const policy = (input.policyResolver ?? resolveAuditRetentionPolicy)(tenant);
const threshold = new Date(now.getTime() - policy.retentionDays * 86_400_000);
const count = await input.eventDeleter(tenant.tenantId, threshold);
deletedRows += count;
input.logger?.info({ tenantId: tenant.tenantId, deletedCount: count, retentionDays: policy.retentionDays, policySource: policy.source }, "audit.retention.tenant.swept");
if (count > 0 && input.auditWriter) {
await input.auditWriter({ tenantId: tenant.tenantId, deletedCount: count, threshold, policy });
}
const result = await input.batchPruner(tenantId, threshold, batchSize);
batches += 1;
deletedRows += result.deletedCount;
deletedByChain.set(tenantId, (deletedByChain.get(tenantId) ?? 0) + result.deletedCount);
if (result.hasMore) pending.push(tenantId);
input.logger?.info(
{
tenantId,
deletedCount: result.deletedCount,
retentionYears: AUDIT_RETENTION_YEARS,
hasMore: result.hasMore,
},
"audit.retention.chain.batch",
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
failures.push({ tenantId: tenant.tenantId, error: message });
input.logger?.error?.({ tenantId: tenant.tenantId, error: message }, "audit.retention.tenant.failed");
failures.push({ tenantId, error: message });
input.logger?.error?.({ tenantId, error: message }, "audit.retention.chain.failed");
}
}
input.logger?.info({ inspected: tenants.length, deletedRows, failures: failures.length }, "audit.retention.complete");
return { inspectedTenants: tenants.length, deletedRows, failures };
if (input.auditWriter) {
for (const [tenantId, deletedCount] of deletedByChain) {
if (deletedCount > 0) {
await input.auditWriter({
tenantId,
deletedCount,
threshold,
retentionYears: AUDIT_RETENTION_YEARS,
});
}
}
}
const backlogChains = [...new Set(pending)];
input.logger?.info(
{
inspected: chains.length,
batches,
deletedRows,
backlogChains: backlogChains.length,
failures: failures.length,
retentionYears: AUDIT_RETENTION_YEARS,
},
"audit.retention.complete",
);
return { inspectedChains: chains.length, batches, deletedRows, backlogChains, failures };
}
@@ -0,0 +1,31 @@
// R-P12c — worker seam tests: envelope parsing (tenant mismatch fail-closed)
// and companion-consumer skip behaviour for non-automation events.
import { describe, expect, test } from "vitest";
import { AutomationExecutorJobError, parseAutomationExecutorOutboxJob } from "./automation-forward.js";
describe("parseAutomationExecutorOutboxJob", () => {
test("accepts a well-formed automation.execution.requested envelope", () => {
expect(
parseAutomationExecutorOutboxJob({
eventType: "automation.execution.requested",
tenantId: "tenant-a",
payload: { executionId: "exec-1", tenantId: "tenant-a" },
})
).toEqual({ tenantId: "tenant-a", executionId: "exec-1" });
});
test("rejects foreign event types, missing ids and tenant mismatches fail-closed", () => {
expect(() => parseAutomationExecutorOutboxJob({ eventType: "workflow.trigger.event", tenantId: "t", payload: {} })).toThrow(AutomationExecutorJobError);
expect(() => parseAutomationExecutorOutboxJob({ eventType: "automation.execution.requested", tenantId: "", payload: { executionId: "e" } })).toThrow(
AutomationExecutorJobError
);
expect(() =>
parseAutomationExecutorOutboxJob({
eventType: "automation.execution.requested",
tenantId: "tenant-a",
payload: { executionId: "exec-1", tenantId: "tenant-b" },
})
).toThrow(/tenantId mismatch/);
});
});
@@ -0,0 +1,122 @@
// R-P12c — worker wiring for the generic automation forward path.
//
// Two seams:
// 1. `createRunAutomationForwardTrigger` — companion consumer on the durable
// `outbox.workflow` queue (next to runPortalAutomationTriggerJob). Maps
// domain events onto automation trigger keys and creates idempotent,
// guarded executions (packages/db/src/automation-forward.server.ts).
// 2. `createAutomationExecutorProcessor` — BullMQ worker processor for the
// `outbox.automation-executor` queue (routed via event-router for
// `automation.execution.*` outbox events). Runs the typed forward
// executor (packages/db/src/automation-executor.server.ts) with real
// side-effect adapters bound: notify(), logActivity(), SSRF-guarded HTTP
// and the permission-aware team-chat repository.
import { logActivity } from "@hivegrid/activity";
import { sendTopLevelChatMessage } from "@hivegrid/chat/repository.server";
import { queueAutomationExecutionsForDomainEventWithTx, AUTOMATION_DOMAIN_EVENT_TRIGGER_MAP } from "@hivegrid/db/automation-forward.server";
import { runAutomationExecutionJob, type AutomationExecutionJobResult } from "@hivegrid/db/automation-executor.server";
import type { DbClient } from "@hivegrid/db/client.server";
import { notify, type NotificationClient } from "@hivegrid/foundation";
import { fetchOutboundHttpWithRedirects, validateOutboundHttpUrl } from "@hivegrid/foundation/outbound-http";
import { withTenantRLS } from "../db/tenantRls.js";
import { parseWorkflowTriggerJob, type WorkflowTriggerLogger } from "./workflow-trigger.js";
export const AUTOMATION_EXECUTOR_OUTBOX_QUEUE = "outbox.automation-executor" as const;
export type AutomationForwardDeps = {
readonly prisma: DbClient;
readonly logger?: WorkflowTriggerLogger;
};
/** Companion consumer for `workflow.trigger.event` jobs (generic, non-portal). */
export function createRunAutomationForwardTrigger(deps: AutomationForwardDeps): (data: unknown) => Promise<unknown> {
return async (data: unknown) => {
const job = parseWorkflowTriggerJob(data);
if (!job.tenantId) return { skipped: "missing_tenant" };
if (!AUTOMATION_DOMAIN_EVENT_TRIGGER_MAP[job.eventName]) return { skipped: "not_an_automation_event" };
const payload = job.payload && typeof job.payload === "object" ? (job.payload as Record<string, unknown>) : {};
const eventData = payload.data && typeof payload.data === "object" && !Array.isArray(payload.data) ? (payload.data as Record<string, unknown>) : {};
const userId = typeof payload.userId === "string" ? payload.userId : null;
const sourceEventId = job.outboxEventId ?? job.dispatchKey ?? `${job.eventType}:${job.eventName}`;
const tenantId = job.tenantId;
const result = await withTenantRLS(deps.prisma, tenantId, (tx) =>
queueAutomationExecutionsForDomainEventWithTx(tx as never, {
tenantId,
domainEvent: job.eventName,
payload: eventData,
sourceEventId,
userId,
})
);
if (result.matched > 0) {
deps.logger?.info?.(
{ component: "automation-forward", tenantId, eventName: job.eventName, ...result, rejected: result.rejected.length },
"automation trigger matched"
);
}
return result;
};
}
export class AutomationExecutorJobError extends Error {
override readonly name = "AutomationExecutorJobError";
}
export function parseAutomationExecutorOutboxJob(data: unknown): { tenantId: string; executionId: string } {
if (!data || typeof data !== "object") {
throw new AutomationExecutorJobError("automation executor job missing envelope");
}
const envelope = data as Record<string, unknown>;
if (envelope.eventType !== "automation.execution.requested") {
throw new AutomationExecutorJobError(`unsupported automation executor eventType ${JSON.stringify(envelope.eventType)}`);
}
const tenantId = typeof envelope.tenantId === "string" ? envelope.tenantId : "";
const payload = envelope.payload && typeof envelope.payload === "object" ? (envelope.payload as Record<string, unknown>) : {};
const executionId = typeof payload.executionId === "string" ? payload.executionId : "";
if (!tenantId || !executionId) {
throw new AutomationExecutorJobError("automation executor job missing tenantId/executionId");
}
const payloadTenantId = typeof payload.tenantId === "string" ? payload.tenantId : tenantId;
if (payloadTenantId !== tenantId) {
throw new AutomationExecutorJobError("automation executor job tenantId mismatch");
}
return { tenantId, executionId };
}
export function createAutomationExecutorProcessor(deps: AutomationForwardDeps): (data: unknown) => Promise<AutomationExecutionJobResult> {
return async (data: unknown) => {
const { tenantId, executionId } = parseAutomationExecutorOutboxJob(data);
return runAutomationExecutionJob(
{ tenantId, executionId },
{
runInTenantTx: (fn) => withTenantRLS(deps.prisma, tenantId, (tx) => fn(tx as never)),
notify: (input, txClient) => notify(input as never, txClient as NotificationClient),
logActivity: (input, opts) => logActivity(input as never, opts as never),
fetchOutboundHttp: async ({ url, method, body }) => {
const validated = validateOutboundHttpUrl(url);
const response = await fetchOutboundHttpWithRedirects(validated, {
method,
...(body ? { body, headers: { "content-type": "application/json" } } : {}),
});
return { status: response.status, ok: response.ok };
},
sendChatMessage: async (input) => {
const result = await sendTopLevelChatMessage({
tenantId: input.tenantId,
userId: input.userId,
permissions: [],
chatId: input.chatId,
requestId: input.requestId,
clientMessageId: input.clientMessageId,
body: input.body,
});
return { messageId: result.message.id, replayed: result.replayed };
},
logger: deps.logger,
}
);
};
}
@@ -0,0 +1,18 @@
import { describe, expect, it, vi } from "vitest";
import { AUTOMATION_SCHEDULER_SWEEP_EVERY_MS, AUTOMATION_SCHEDULER_SWEEP_JOB_ID, registerAutomationSchedulerCron } from "./automation-scheduler-cron.js";
describe("automation scheduler cron", () => {
it("registers one singleton minute sweep", async () => {
const upsertJobScheduler = vi.fn().mockResolvedValue(undefined);
await registerAutomationSchedulerCron({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
{ every: AUTOMATION_SCHEDULER_SWEEP_EVERY_MS },
{
name: AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: AUTOMATION_SCHEDULER_SWEEP_JOB_ID },
}
);
});
});
@@ -0,0 +1,15 @@
export const AUTOMATION_SCHEDULER_SWEEP_JOB_ID = "automation-scheduler-sweep" as const;
export const AUTOMATION_SCHEDULER_SWEEP_EVERY_MS = 60_000;
export async function registerAutomationSchedulerCron(queue: {
upsertJobScheduler(schedulerId: string, options: { every: number }, template: { name: string; data: Record<string, unknown> }): Promise<unknown>;
}): Promise<void> {
await queue.upsertJobScheduler(
AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
{ every: AUTOMATION_SCHEDULER_SWEEP_EVERY_MS },
{
name: AUTOMATION_SCHEDULER_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: AUTOMATION_SCHEDULER_SWEEP_JOB_ID },
}
);
}
@@ -0,0 +1,23 @@
import { describe, expect, test, vi } from "vitest";
import { BILLING_DUNNING_SWEEP_EVERY_MS, BILLING_DUNNING_SWEEP_JOB_ID, registerBillingDunningSweepCron } from "./billing-dunning-cron.js";
describe("billing-dunning cron", () => {
test("uses a 24h cleanup.run scheduler payload", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
await registerBillingDunningSweepCron({ upsertJobScheduler });
expect(upsertJobScheduler).toHaveBeenCalledWith(
BILLING_DUNNING_SWEEP_JOB_ID,
{ every: BILLING_DUNNING_SWEEP_EVERY_MS },
{
name: BILLING_DUNNING_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: BILLING_DUNNING_SWEEP_JOB_ID },
}
);
});
test("registers a daily cadence", () => {
expect(BILLING_DUNNING_SWEEP_EVERY_MS).toBe(24 * 60 * 60 * 1000);
});
});
@@ -0,0 +1,31 @@
// AK-P20 DoD 2 — billing dunning sweep cron.
//
// Daily maintenance job that drives the past_due dunning sequence (day 1/3/7
// notices, then freeze) for every tenant. The decision logic is the pure
// evaluateBillingDunning() from @hivegrid/db/billing-domain.server; the
// execution lives in handlers.ts, which sweeps per tenant from the global
// Tenant registry inside withTenantRLS() and sends notices through
// notifyWithTenantRlsContext() (NotificationDelivery is FORCE RLS). This file
// only carries the config + the idempotent scheduler registration (BullMQ
// `upsertJobScheduler`, like the other maintenance crons).
export const BILLING_DUNNING_SWEEP_JOB_ID = "billing-dunning-sweep" as const;
export const BILLING_DUNNING_SWEEP_EVERY_MS = 24 * 60 * 60 * 1000;
export async function registerBillingDunningSweepCron(
queue: {
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
}
): Promise<void> {
await queue.upsertJobScheduler(
BILLING_DUNNING_SWEEP_JOB_ID,
{ every: BILLING_DUNNING_SWEEP_EVERY_MS },
{
name: BILLING_DUNNING_SWEEP_JOB_ID,
data: {
kind: "cleanup.run",
task: BILLING_DUNNING_SWEEP_JOB_ID,
},
}
);
}
@@ -0,0 +1,146 @@
import { describe, expect, it, vi } from "vitest";
import {
cancelCampaignScheduleWithPrisma,
isCampaignDeliveryAllowedWithPrisma,
type CampaignScheduleCancellationPrisma,
} from "./campaign-schedule-cancellation.js";
function billingSubscription(status = "active", emailEnabled = true) {
return {
tenantId: "tenant-1",
status,
planCode: "starter",
seatQuantity: 2,
lagoCustomerId: null,
mollieCustomerId: null,
lastInvoiceExternalId: null,
lastPaymentExternalId: null,
trialStartsAt: null,
trialEndsAt: null,
pastDueSince: null,
frozenAt: status === "frozen" ? new Date("2026-07-25T12:00:00.000Z") : null,
cancelledAt: null,
products: emailEnabled
? [{
id: "email-product",
createdAt: new Date("2026-07-01T00:00:00.000Z"),
cancelledAt: null,
endsAt: null,
planCode: "email_marketing_s",
seatQuantity: 1,
quantity: 1,
currentPeriodStart: null,
currentPeriodEnd: null,
subscriptionProduct: {
id: "email-s",
code: "email_marketing_s",
lagoPlanCode: "email_marketing_s",
kind: "recurring_add_on",
features: [{
code: null,
name: "email-marketing",
title: "Email marketing",
type: 2,
value: 10_000,
quotaValue: null,
accumulate: true,
}],
},
}]
: [],
};
}
function harness(status = "cancelling", billingStatus = "active", emailEnabled = true) {
const tx = {
$executeRaw: vi.fn(async () => 1),
tenantSubscription: {
findUnique: vi.fn(async () => billingSubscription(billingStatus, emailEnabled)),
findMany: vi.fn(async () => []),
},
campaign: {
findFirst: vi.fn(async () => ({ status })),
updateMany: vi.fn(async () => ({ count: 1 })),
},
outboxEvent: {
updateMany: vi.fn(async () => ({ count: 3 })),
},
outboundEmail: {
deleteMany: vi.fn(async () => ({ count: 3 })),
},
};
const prisma = {
$transaction: vi.fn(async (callback: (value: typeof tx) => Promise<unknown>) => callback(tx)),
} as unknown as CampaignScheduleCancellationPrisma;
return { prisma, tx };
}
describe("campaign schedule cancellation", () => {
it("cancels only pending campaign events and resets the claimed schedule", async () => {
const now = new Date("2026-07-25T12:00:00.000Z");
const { prisma, tx } = harness();
await expect(
cancelCampaignScheduleWithPrisma(prisma, { tenantId: "tenant-1", campaignId: "campaign-1" }, now),
).resolves.toEqual({ cancelledOutboxEvents: 3, removedRecipients: 3, resetCampaigns: 1 });
expect(tx.outboxEvent.updateMany).toHaveBeenCalledWith({
where: {
tenantId: "tenant-1",
eventType: "mail.transactional.send",
dispatchKey: { startsWith: "campaign-send:campaign-1:" },
status: "PENDING",
},
data: {
status: "CANCELLED",
processedAt: now,
lockedUntil: null,
lastError: null,
},
});
expect(tx.outboundEmail.deleteMany).toHaveBeenCalledWith({
where: {
tenantId: "tenant-1",
campaignId: "campaign-1",
deliveredAt: null,
bouncedAt: null,
spamComplainedAt: null,
},
});
expect(tx.campaign.updateMany).toHaveBeenCalledWith({
where: { id: "campaign-1", tenantId: "tenant-1", status: "cancelling" },
data: { status: "draft", sentAt: null, scheduledAt: null },
});
});
it("rejects a command outside the cancellation claim", async () => {
const { prisma, tx } = harness("scheduled");
await expect(cancelCampaignScheduleWithPrisma(prisma, { tenantId: "tenant-1", campaignId: "campaign-1" })).rejects.toThrow(
/invalid lifecycle state/iu,
);
expect(tx.outboxEvent.updateMany).not.toHaveBeenCalled();
});
it.each([
["scheduled", true],
["sending", true],
["cancelling", false],
["draft", false],
])("allows delivery only in an active send state (%s)", async (status, expected) => {
const { prisma } = harness(status);
await expect(isCampaignDeliveryAllowedWithPrisma(prisma, { tenantId: "tenant-1", campaignId: "campaign-1" })).resolves.toBe(expected);
});
it("blocks queued recipients after the paid feature is frozen or revoked", async () => {
await expect(isCampaignDeliveryAllowedWithPrisma(harness("scheduled", "frozen").prisma, {
tenantId: "tenant-1",
campaignId: "campaign-1",
})).resolves.toBe(false);
await expect(isCampaignDeliveryAllowedWithPrisma(harness("scheduled", "active", false).prisma, {
tenantId: "tenant-1",
campaignId: "campaign-1",
})).resolves.toBe(false);
});
});
@@ -0,0 +1,136 @@
import { withTenantRLS, type RlsTransactionExecutor, type TransactionCapableClient } from "../db/tenantRls.js";
import { resolveTenantBillingState, type TenantBillingStateClient } from "@hivegrid/db/billing-domain.server";
import { featureKeys } from "@hivegrid/entitlements";
export type CampaignScheduleCancellationInput = {
tenantId: string;
campaignId: string;
};
export type CampaignScheduleCancellationResult = {
cancelledOutboxEvents: number;
removedRecipients: number;
resetCampaigns: number;
};
type CampaignScheduleCancellationTx = RlsTransactionExecutor & {
tenantSubscription: TenantBillingStateClient["tenantSubscription"];
campaign: {
findFirst(args: {
where: { id: string; tenantId: string };
select: { status: true };
}): Promise<{ status: string } | null>;
updateMany(args: {
where: { id: string; tenantId: string; status: string };
data: { status: string; sentAt: null; scheduledAt: null };
}): Promise<{ count: number }>;
};
outboxEvent: {
updateMany(args: {
where: {
tenantId: string;
eventType: string;
dispatchKey: { startsWith: string };
status: "PENDING";
};
data: {
status: "CANCELLED";
processedAt: Date;
lockedUntil: null;
lastError: null;
};
}): Promise<{ count: number }>;
};
outboundEmail: {
deleteMany(args: {
where: {
tenantId: string;
campaignId: string;
deliveredAt: null;
bouncedAt: null;
spamComplainedAt: null;
};
}): Promise<{ count: number }>;
};
};
export type CampaignScheduleCancellationPrisma = TransactionCapableClient<CampaignScheduleCancellationTx>;
export async function cancelCampaignScheduleWithPrisma(
prisma: CampaignScheduleCancellationPrisma,
input: CampaignScheduleCancellationInput,
now: Date = new Date(),
): Promise<CampaignScheduleCancellationResult> {
return withTenantRLS(prisma, input.tenantId, async (tx) => {
const campaign = await tx.campaign.findFirst({
where: { id: input.campaignId, tenantId: input.tenantId },
select: { status: true },
});
if (!campaign) {
return { cancelledOutboxEvents: 0, removedRecipients: 0, resetCampaigns: 0 };
}
if (campaign.status !== "cancelling") {
throw new Error("Campaign schedule cancellation rejected an invalid lifecycle state.");
}
const cancelled = await tx.outboxEvent.updateMany({
where: {
tenantId: input.tenantId,
eventType: "mail.transactional.send",
dispatchKey: { startsWith: `campaign-send:${input.campaignId}:` },
status: "PENDING",
},
data: {
status: "CANCELLED",
processedAt: now,
lockedUntil: null,
lastError: null,
},
});
const removed = await tx.outboundEmail.deleteMany({
where: {
tenantId: input.tenantId,
campaignId: input.campaignId,
deliveredAt: null,
bouncedAt: null,
spamComplainedAt: null,
},
});
const reset = await tx.campaign.updateMany({
where: { id: input.campaignId, tenantId: input.tenantId, status: "cancelling" },
data: { status: "draft", sentAt: null, scheduledAt: null },
});
if (reset.count !== 1) {
throw new Error("Campaign schedule cancellation lost its lifecycle claim.");
}
return {
cancelledOutboxEvents: cancelled.count,
removedRecipients: removed.count,
resetCampaigns: reset.count,
};
});
}
export async function isCampaignDeliveryAllowedWithPrisma(
prisma: CampaignScheduleCancellationPrisma,
input: CampaignScheduleCancellationInput,
): Promise<boolean> {
return withTenantRLS(prisma, input.tenantId, async (tx) => {
const [campaign, billing] = await Promise.all([
tx.campaign.findFirst({
where: { id: input.campaignId, tenantId: input.tenantId },
select: { status: true },
}),
resolveTenantBillingState(input.tenantId, {
client: tx as unknown as TenantBillingStateClient,
}),
]);
const emailMarketing = billing.entitlements.find((entitlement) => entitlement.featureKey === featureKeys.emailMarketing);
return (
(campaign?.status === "scheduled" || campaign?.status === "sending") &&
!billing.readOnly &&
emailMarketing?.enabled === true
);
});
}
@@ -0,0 +1,70 @@
import { CHAT_LINK_PREVIEW_REQUESTED_OUTBOX_EVENT_TYPE, hashChatLinkPreviewBody } from "@hivegrid/chat/link-preview.server";
import { describe, expect, it, vi } from "vitest";
import { bootstrapChatLinkPreview, type ChatLinkPreviewBootstrapDeps, type ChatLinkPreviewWorkerHandle } from "./chat-link-preview-bootstrap.js";
import { CHAT_LINK_PREVIEW_QUEUE } from "./chat-link-preview.js";
const tenantBudget = { consume: vi.fn(async () => ({ allowed: true, retryAfterMs: 0 })) };
function capturingWorker() {
const captured: { queueName?: string; processor?: (job: { data: unknown }) => Promise<unknown>; concurrency?: number } = {};
const handle: ChatLinkPreviewWorkerHandle = { name: "chat-link-preview", close: vi.fn(async () => undefined) };
const createWorkerImpl: NonNullable<ChatLinkPreviewBootstrapDeps["createWorkerImpl"]> = (queueName, processor, options) => {
captured.queueName = queueName;
captured.processor = processor;
captured.concurrency = options.concurrency;
return handle;
};
return { captured, createWorkerImpl, handle };
}
describe("bootstrapChatLinkPreview", () => {
it("starts only on the general worker group and honors its escape hatch", () => {
expect(bootstrapChatLinkPreview({ WORKER_QUEUE_GROUP: "mail" })).toEqual({ status: "disabled", reason: "non-general worker group" });
expect(bootstrapChatLinkPreview({ WORKER_QUEUE_GROUP: "general", CHAT_LINK_PREVIEW_ENABLED: "false" })).toEqual({
status: "disabled",
reason: "CHAT_LINK_PREVIEW_ENABLED=false",
});
});
it("binds the dedicated outbox queue with bounded concurrency", () => {
const { captured, createWorkerImpl, handle } = capturingWorker();
expect(bootstrapChatLinkPreview({ CHAT_LINK_PREVIEW_CONCURRENCY: "3" }, { createWorkerImpl, tenantBudget })).toEqual({ status: "started", handle });
expect(captured).toMatchObject({ queueName: CHAT_LINK_PREVIEW_QUEUE, concurrency: 3 });
});
it.each(["0", "-1", "bad"])('falls back to concurrency five for "%s"', (value) => {
const { captured, createWorkerImpl } = capturingWorker();
bootstrapChatLinkPreview({ CHAT_LINK_PREVIEW_CONCURRENCY: value }, { createWorkerImpl, tenantBudget });
expect(captured.concurrency).toBe(5);
});
it("fails closed when the host does not provide a worker factory", () => {
expect(bootstrapChatLinkPreview({})).toEqual({ status: "disabled", reason: "no createWorkerImpl wired" });
});
it("fails closed when tenant fairness cannot be enforced", () => {
const { createWorkerImpl } = capturingWorker();
expect(bootstrapChatLinkPreview({}, { createWorkerImpl })).toEqual({ status: "disabled", reason: "no tenant budget store wired" });
});
it("passes the original transactional outbox envelope to the processor", async () => {
const { captured, createWorkerImpl } = capturingWorker();
const runJobImpl = vi.fn(async () => ({
status: "skipped" as const,
messageId: "missing_message",
transientFailures: 0,
scheduledRetries: 0,
deferredByBudget: 0,
}));
bootstrapChatLinkPreview({}, { createWorkerImpl, runJobImpl, tenantBudget, now: () => 1_752_662_400_000 });
const envelope = {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_LINK_PREVIEW_REQUESTED_OUTBOX_EVENT_TYPE,
payload: { chatId: "chat_1", messageId: "missing_message", bodyHash: hashChatLinkPreviewBody("body") },
};
await expect(captured.processor?.({ data: envelope })).resolves.toMatchObject({ status: "skipped", messageId: "missing_message" });
expect(runJobImpl).toHaveBeenCalledWith(envelope, expect.objectContaining({ tenantBudget, now: expect.any(Function) }));
});
});
@@ -0,0 +1,98 @@
import type { EnvRecord } from "@hivegrid/config";
import { createSlidingWindowRateLimiter, type SharedStore } from "@hivegrid/shared-store";
import { CHAT_LINK_PREVIEW_QUEUE, runChatLinkPreviewOutboxJob } from "./chat-link-preview.js";
import type { ChatLinkPreviewLogger, ChatLinkPreviewTenantBudget } from "@hivegrid/chat/link-preview.server";
export type ChatLinkPreviewBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
CHAT_LINK_PREVIEW_ENABLED?: string;
CHAT_LINK_PREVIEW_CONCURRENCY?: string;
CHAT_LINK_PREVIEW_TENANT_REQUESTS_PER_MINUTE?: string;
};
export interface ChatLinkPreviewWorkerHandle {
readonly name: string;
close(): Promise<void>;
}
export interface ChatLinkPreviewBootstrapDeps {
readonly logger?: ChatLinkPreviewLogger;
readonly baseStore?: SharedStore;
readonly tenantBudget?: ChatLinkPreviewTenantBudget;
readonly now?: () => number;
readonly runJobImpl?: typeof runChatLinkPreviewOutboxJob;
readonly createWorkerImpl?: (
queueName: string,
processor: (job: { data: unknown }) => Promise<unknown>,
options: { concurrency?: number }
) => ChatLinkPreviewWorkerHandle;
}
export type ChatLinkPreviewBootstrapResult =
| { readonly status: "disabled"; readonly reason: string }
| { readonly status: "started"; readonly handle: ChatLinkPreviewWorkerHandle };
const DEFAULT_CONCURRENCY = 5;
const DEFAULT_TENANT_REQUESTS_PER_MINUTE = 30;
const TENANT_BUDGET_WINDOW_MS = 60_000;
export function bootstrapChatLinkPreview(env: ChatLinkPreviewBootstrapEnv, deps: ChatLinkPreviewBootstrapDeps = {}): ChatLinkPreviewBootstrapResult {
if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.CHAT_LINK_PREVIEW_ENABLED === "false") {
return { status: "disabled", reason: "CHAT_LINK_PREVIEW_ENABLED=false" };
}
if (!deps.createWorkerImpl) {
return { status: "disabled", reason: "no createWorkerImpl wired" };
}
const concurrency = parsePositiveInt(env.CHAT_LINK_PREVIEW_CONCURRENCY, DEFAULT_CONCURRENCY);
const tenantRequestsPerMinute = parsePositiveInt(env.CHAT_LINK_PREVIEW_TENANT_REQUESTS_PER_MINUTE, DEFAULT_TENANT_REQUESTS_PER_MINUTE);
const now = deps.now ?? Date.now;
const tenantBudget = deps.tenantBudget ?? createTenantBudget(deps.baseStore, tenantRequestsPerMinute, now);
if (!tenantBudget) {
return { status: "disabled", reason: "no tenant budget store wired" };
}
const processor = (job: { data: unknown }) =>
(deps.runJobImpl ?? runChatLinkPreviewOutboxJob)(job.data, {
tenantBudget,
now: () => new Date(now()),
...(deps.logger ? { logger: deps.logger } : {}),
});
const handle = deps.createWorkerImpl(CHAT_LINK_PREVIEW_QUEUE, processor, { concurrency });
deps.logger?.info?.(
{ component: "chat-link-preview", queue: CHAT_LINK_PREVIEW_QUEUE, concurrency, tenantRequestsPerMinute },
"chat link preview worker started"
);
return { status: "started", handle };
}
function createTenantBudget(baseStore: SharedStore | undefined, maxRequests: number, now: () => number): ChatLinkPreviewTenantBudget | null {
if (!baseStore) return null;
const limiter = createSlidingWindowRateLimiter({
store: baseStore,
prefix: "chat-link-preview-tenant",
windowSeconds: TENANT_BUDGET_WINDOW_MS / 1_000,
max: maxRequests,
now,
});
return {
async consume(tenantId) {
const decision = await limiter.consume(tenantId);
if (decision.allowed) return { allowed: true, retryAfterMs: 0 };
const oldestAt = decision.oldestAt?.getTime() ?? now();
return {
allowed: false,
retryAfterMs: Math.max(1_000, oldestAt + TENANT_BUDGET_WINDOW_MS - now()),
};
},
};
}
function parsePositiveInt(value: unknown, fallback: number): number {
if (typeof value !== "string" || value.length === 0) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
@@ -0,0 +1,15 @@
import {
CHAT_LINK_PREVIEW_OUTBOX_QUEUE,
runChatLinkPreviewJob,
type ChatLinkPreviewJobDependencies,
type ChatLinkPreviewJobResult,
} from "@hivegrid/chat/link-preview.server";
export const CHAT_LINK_PREVIEW_QUEUE = CHAT_LINK_PREVIEW_OUTBOX_QUEUE;
export function runChatLinkPreviewOutboxJob(
data: unknown,
dependencies: ChatLinkPreviewJobDependencies = {}
): Promise<ChatLinkPreviewJobResult> {
return runChatLinkPreviewJob(data, dependencies);
}
@@ -0,0 +1,121 @@
import { CHAT_EVENT_NAMES } from "@hivegrid/realtime";
import { MemoryAdapter, createSharedStore } from "@hivegrid/shared-store";
import { describe, expect, it, vi } from "vitest";
import { bootstrapChatRealtimeWakeup, type ChatRealtimeWakeupBootstrapDeps, type ChatRealtimeWakeupWorkerHandle } from "./chat-realtime-wakeup-bootstrap.js";
import { CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE, CHAT_REALTIME_SHARED_STORE_NAMESPACE, CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE } from "./chat-realtime-wakeup.js";
function capturingWorker() {
const captured: {
queueName?: string;
processor?: (job: { data: unknown }) => Promise<unknown>;
concurrency?: number;
} = {};
const handle: ChatRealtimeWakeupWorkerHandle = {
name: "chat-realtime-wakeup",
close: vi.fn(async () => undefined),
};
const createWorkerImpl: NonNullable<ChatRealtimeWakeupBootstrapDeps["createWorkerImpl"]> = (queueName, processor, options) => {
captured.queueName = queueName;
captured.processor = processor;
captured.concurrency = options.concurrency;
return handle;
};
return { captured, createWorkerImpl, handle };
}
function validJob() {
return {
data: {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
payload: {
event: {
event: CHAT_EVENT_NAMES.readReceipt,
cursor: "stream_1:7",
requestId: "request_1",
tenantId: "tenant_1",
chatId: "chat_1",
occurredAt: "2026-07-16T08:00:00.000Z",
data: {
messageId: "message_1",
userId: "user_1",
readAt: "2026-07-16T08:00:00.000Z",
},
},
},
},
};
}
describe("bootstrapChatRealtimeWakeup", () => {
it("disables outside the general worker group or via its escape hatch", () => {
expect(bootstrapChatRealtimeWakeup({ WORKER_QUEUE_GROUP: "mail" })).toEqual({
status: "disabled",
reason: "non-general worker group",
});
expect(
bootstrapChatRealtimeWakeup({
WORKER_QUEUE_GROUP: "general",
CHAT_REALTIME_WAKEUP_ENABLED: "false",
})
).toEqual({ status: "disabled", reason: "CHAT_REALTIME_WAKEUP_ENABLED=false" });
});
it("disables when required host seams are absent", () => {
const { createWorkerImpl } = capturingWorker();
expect(bootstrapChatRealtimeWakeup({}, { createWorkerImpl })).toMatchObject({
status: "disabled",
reason: "no shared-store adapter wired",
});
const baseStore = createSharedStore({ namespace: "worker", adapter: new MemoryAdapter() });
expect(bootstrapChatRealtimeWakeup({}, { baseStore })).toMatchObject({
status: "disabled",
reason: "no createWorkerImpl wired",
});
});
it("starts the dedicated outbox consumer with the configured concurrency", () => {
const { captured, createWorkerImpl, handle } = capturingWorker();
const baseStore = createSharedStore({ namespace: "worker", adapter: new MemoryAdapter() });
const result = bootstrapChatRealtimeWakeup({ WORKER_QUEUE_GROUP: "general", CHAT_REALTIME_WAKEUP_CONCURRENCY: "4" }, { baseStore, createWorkerImpl });
expect(result).toEqual({ status: "started", handle });
expect(captured.queueName).toBe(CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE);
expect(captured.concurrency).toBe(4);
});
it("rebinds the worker store to chat-realtime and publishes through the captured processor", async () => {
const { captured, createWorkerImpl } = capturingWorker();
const adapter = new MemoryAdapter();
const baseStore = createSharedStore({ namespace: "worker", adapter });
const realtimeStore = createSharedStore({ namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE, adapter });
const received: string[] = [];
const unsubscribe = await realtimeStore.subscribe("persistent:tenant_1:chat_1", (raw) => {
received.push(raw);
});
bootstrapChatRealtimeWakeup({}, { baseStore, createWorkerImpl });
await captured.processor?.(validJob());
expect(received).toHaveLength(1);
expect(JSON.parse(received[0] ?? "null")).toMatchObject({
event: CHAT_EVENT_NAMES.readReceipt,
tenantId: "tenant_1",
chatId: "chat_1",
cursor: "stream_1:7",
});
await unsubscribe();
});
it.each(["0", "-2", "nope"])("uses default concurrency for %s", (raw) => {
const { captured, createWorkerImpl } = capturingWorker();
const baseStore = createSharedStore({ namespace: "worker", adapter: new MemoryAdapter() });
bootstrapChatRealtimeWakeup({ CHAT_REALTIME_WAKEUP_CONCURRENCY: raw }, { baseStore, createWorkerImpl });
expect(captured.concurrency).toBe(10);
});
});
@@ -0,0 +1,83 @@
import type { EnvRecord } from "@hivegrid/config";
import { createSharedStore, type SharedStore } from "@hivegrid/shared-store";
import {
CHAT_REALTIME_SHARED_STORE_NAMESPACE,
CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE,
runChatRealtimeWakeupJob,
type ChatRealtimeWakeupLogger,
} from "./chat-realtime-wakeup.js";
export type ChatRealtimeWakeupBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
CHAT_REALTIME_WAKEUP_ENABLED?: string;
CHAT_REALTIME_WAKEUP_CONCURRENCY?: string;
};
export interface ChatRealtimeWakeupWorkerHandle {
readonly name: string;
close(): Promise<void>;
}
export interface ChatRealtimeWakeupBootstrapDeps {
readonly logger?: ChatRealtimeWakeupLogger;
readonly baseStore?: Pick<SharedStore, "adapter">;
readonly createWorkerImpl?: (
queueName: string,
processor: (job: { data: unknown }) => Promise<unknown>,
options: { concurrency?: number }
) => ChatRealtimeWakeupWorkerHandle;
}
export type ChatRealtimeWakeupBootstrapResult =
| { readonly status: "disabled"; readonly reason: string }
| { readonly status: "started"; readonly handle: ChatRealtimeWakeupWorkerHandle };
const DEFAULT_CONCURRENCY = 10;
export function bootstrapChatRealtimeWakeup(
env: ChatRealtimeWakeupBootstrapEnv,
deps: ChatRealtimeWakeupBootstrapDeps = {}
): ChatRealtimeWakeupBootstrapResult {
if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.CHAT_REALTIME_WAKEUP_ENABLED === "false") {
return { status: "disabled", reason: "CHAT_REALTIME_WAKEUP_ENABLED=false" };
}
if (!deps.baseStore) {
return { status: "disabled", reason: "no shared-store adapter wired" };
}
if (!deps.createWorkerImpl) {
return { status: "disabled", reason: "no createWorkerImpl wired" };
}
const store = createSharedStore({
namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE,
adapter: deps.baseStore.adapter,
});
const concurrency = parsePositiveInt(env.CHAT_REALTIME_WAKEUP_CONCURRENCY, DEFAULT_CONCURRENCY);
const processor = (job: { data: unknown }) =>
runChatRealtimeWakeupJob(job.data, {
store,
...(deps.logger ? { logger: deps.logger } : {}),
});
const handle = deps.createWorkerImpl(CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE, processor, { concurrency });
deps.logger?.info?.(
{
component: "chat-realtime-wakeup",
queue: CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE,
namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE,
concurrency,
},
"chat realtime wakeup worker started"
);
return { status: "started", handle };
}
function parsePositiveInt(value: unknown, fallback: number): number {
if (typeof value !== "string" || value.length === 0) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
@@ -0,0 +1,126 @@
import { CHAT_EVENT_NAMES } from "@hivegrid/realtime";
import { MemoryAdapter, createSharedStore } from "@hivegrid/shared-store";
import { describe, expect, it, vi } from "vitest";
import {
CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
CHAT_REALTIME_SHARED_STORE_NAMESPACE,
CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE,
ChatRealtimeWakeupJobError,
parseChatRealtimeWakeupJob,
runChatRealtimeWakeupJob,
} from "./chat-realtime-wakeup.js";
function persistentEvent() {
return {
event: CHAT_EVENT_NAMES.readReceipt,
cursor: "stream_1:7",
requestId: "request_1",
tenantId: "tenant_1",
chatId: "chat_1",
occurredAt: "2026-07-16T08:00:00.000Z",
data: {
messageId: "message_1",
userId: "user_1",
readAt: "2026-07-16T08:00:00.000Z",
},
};
}
function validJob() {
return {
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
payload: { event: persistentEvent() },
dispatchKey: "chat-realtime:ignored-by-consumer",
retries: 0,
};
}
describe("chat realtime wakeup job", () => {
it("uses the dedicated queue, event type and shared-store namespace", () => {
expect(CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE).toBe("outbox.chat-realtime");
expect(CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE).toBe("chat.realtime.persistent");
expect(CHAT_REALTIME_SHARED_STORE_NAMESPACE).toBe("chat-realtime");
});
it("parses a dispatcher envelope and normalizes its persistent event", () => {
expect(parseChatRealtimeWakeupJob(validJob())).toEqual({
outboxEventId: "outbox_1",
tenantId: "tenant_1",
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
event: persistentEvent(),
});
});
it.each([
["missing envelope", null],
["unsupported event type", { ...validJob(), eventType: "chat.realtime.ephemeral" }],
[
"cross-tenant payload",
{
...validJob(),
payload: { event: { ...persistentEvent(), tenantId: "tenant_2" } },
},
],
[
"ephemeral event",
{
...validJob(),
payload: {
event: {
event: CHAT_EVENT_NAMES.typingStarted,
cursor: null,
requestId: "request_1",
tenantId: "tenant_1",
chatId: "chat_1",
occurredAt: "2026-07-16T08:00:00.000Z",
data: { userId: "user_1" },
},
},
},
],
])("rejects %s", (_label, input) => {
expect(() => parseChatRealtimeWakeupJob(input)).toThrow(ChatRealtimeWakeupJobError);
});
it("publishes the full event to the gateway's persistent room channel", async () => {
const adapter = new MemoryAdapter();
const store = createSharedStore({ namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE, adapter });
const received: string[] = [];
const unsubscribe = await store.subscribe("persistent:tenant_1:chat_1", (raw) => {
received.push(raw);
});
const result = await runChatRealtimeWakeupJob(validJob(), { store });
expect(result).toEqual({
published: true,
channel: "persistent:tenant_1:chat_1",
cursor: "stream_1:7",
subscriberCount: 1,
});
expect(received).toEqual([JSON.stringify(persistentEvent())]);
await unsubscribe();
});
it("propagates publish failures so BullMQ can retry the outbox delivery", async () => {
const publish = vi.fn(async () => {
throw new Error("redis unavailable");
});
await expect(
runChatRealtimeWakeupJob(validJob(), {
store: { namespace: CHAT_REALTIME_SHARED_STORE_NAMESPACE, publish },
})
).rejects.toThrow("redis unavailable");
});
it("rejects an incorrectly namespaced store before publishing", async () => {
const publish = vi.fn(async () => 0);
await expect(runChatRealtimeWakeupJob(validJob(), { store: { namespace: "worker", publish } })).rejects.toMatchObject({ code: "INVALID_STORE_NAMESPACE" });
expect(publish).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,121 @@
import { CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE } from "@hivegrid/chat/realtime-wakeup.server";
import { isPersistentChatEvent, validateChatServerEvent, type ChatPersistentEvent } from "@hivegrid/realtime";
import type { SharedStore } from "@hivegrid/shared-store";
export const CHAT_REALTIME_WAKEUP_OUTBOX_QUEUE = "outbox.chat-realtime" as const;
export { CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE };
export const CHAT_REALTIME_SHARED_STORE_NAMESPACE = "chat-realtime" as const;
const SCOPE_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
const OUTBOX_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,190}$/;
export interface ChatRealtimeWakeupLogger {
info?: (record: Record<string, unknown>, message?: string) => void;
error?: (record: Record<string, unknown>, message?: string) => void;
}
export type ChatRealtimeWakeupStore = Pick<SharedStore, "namespace" | "publish">;
export interface ChatRealtimeWakeupDependencies {
readonly store: ChatRealtimeWakeupStore;
readonly logger?: ChatRealtimeWakeupLogger;
}
export interface ChatRealtimeWakeupJobData {
readonly outboxEventId: string;
readonly tenantId: string;
readonly eventType: typeof CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE;
readonly event: ChatPersistentEvent;
}
export interface ChatRealtimeWakeupResult {
readonly published: true;
readonly channel: string;
readonly cursor: string;
readonly subscriberCount: number;
}
export class ChatRealtimeWakeupJobError extends Error {
override readonly name = "ChatRealtimeWakeupJobError";
constructor(
message: string,
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "TENANT_MISMATCH" | "INVALID_STORE_NAMESPACE"
) {
super(message);
}
}
export function parseChatRealtimeWakeupJob(data: unknown): ChatRealtimeWakeupJobData {
const envelope = requireRecord(data, "chat realtime wakeup job envelope");
const outboxEventId = requireIdentifier(envelope.outboxEventId, "outboxEventId", OUTBOX_IDENTIFIER_PATTERN);
const tenantId = requireIdentifier(envelope.tenantId, "tenantId", SCOPE_IDENTIFIER_PATTERN);
if (envelope.eventType !== CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE) {
throw new ChatRealtimeWakeupJobError(`chat realtime wakeup rejected eventType ${JSON.stringify(envelope.eventType)}`, "INVALID_EVENT_TYPE");
}
const payload = requireRecord(envelope.payload, "chat realtime wakeup payload");
let event;
try {
event = validateChatServerEvent(payload.event);
} catch (error) {
throw new ChatRealtimeWakeupJobError(
`chat realtime wakeup rejected event payload: ${error instanceof Error ? error.message : "invalid event"}`,
"INVALID_PAYLOAD"
);
}
if (!isPersistentChatEvent(event)) {
throw new ChatRealtimeWakeupJobError("chat realtime wakeup only accepts persistent chat events", "INVALID_PAYLOAD");
}
requireIdentifier(event.chatId, "event.chatId", SCOPE_IDENTIFIER_PATTERN);
requireIdentifier(event.tenantId, "event.tenantId", SCOPE_IDENTIFIER_PATTERN);
if (event.tenantId !== tenantId) {
throw new ChatRealtimeWakeupJobError("chat realtime wakeup tenant does not match its outbox envelope", "TENANT_MISMATCH");
}
return {
outboxEventId,
tenantId,
eventType: CHAT_PERSISTENT_REALTIME_OUTBOX_EVENT_TYPE,
event,
};
}
export async function runChatRealtimeWakeupJob(data: unknown, deps: ChatRealtimeWakeupDependencies): Promise<ChatRealtimeWakeupResult> {
if (deps.store.namespace !== CHAT_REALTIME_SHARED_STORE_NAMESPACE) {
throw new ChatRealtimeWakeupJobError(`chat realtime wakeup store must use namespace ${CHAT_REALTIME_SHARED_STORE_NAMESPACE}`, "INVALID_STORE_NAMESPACE");
}
const job = parseChatRealtimeWakeupJob(data);
const channel = `persistent:${job.tenantId}:${job.event.chatId}`;
// BullMQ may replay after a publish/ack crash. Repeating this wakeup is safe:
// gateways always gap-fill the durable TeamChatEvent ledger by cursor.
const subscriberCount = await deps.store.publish(channel, JSON.stringify(job.event));
deps.logger?.info?.(
{
component: "chat-realtime-wakeup",
outboxEventId: job.outboxEventId,
tenantId: job.tenantId,
chatId: job.event.chatId,
cursor: job.event.cursor,
subscriberCount,
},
"chat persistent realtime wakeup published"
);
return { published: true, channel, cursor: job.event.cursor, subscriberCount };
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new ChatRealtimeWakeupJobError(`${label} must be an object`, "INVALID_PAYLOAD");
}
return value as Record<string, unknown>;
}
function requireIdentifier(value: unknown, label: string, pattern: RegExp): string {
if (typeof value !== "string" || !pattern.test(value)) {
throw new ChatRealtimeWakeupJobError(`${label} is invalid`, "INVALID_PAYLOAD");
}
return value;
}
@@ -0,0 +1,61 @@
import { describe, expect, test, vi } from "vitest";
import { createSharedStore, MemoryAdapter } from "@hivegrid/shared-store";
import { withDistributedJobLease } from "./distributed-job-lease.js";
describe("distributed job lease", () => {
test("lets only one replica execute a cron key and releases it for the next tick", async () => {
const adapter = new MemoryAdapter();
const replicaA = createSharedStore({ namespace: "worker-lease-test", adapter });
const replicaB = createSharedStore({ namespace: "worker-lease-test", adapter });
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const worker = vi.fn(async () => {
await gate;
return "done";
});
const first = withDistributedJobLease(replicaA, "email-inbox-webhook-drain", worker, { ownerToken: "owner-a" });
await vi.waitFor(() => expect(worker).toHaveBeenCalledTimes(1));
await expect(withDistributedJobLease(replicaB, "email-inbox-webhook-drain", async () => "overlap", { ownerToken: "owner-b" })).resolves.toEqual({
acquired: false,
});
release();
await expect(first).resolves.toEqual({ acquired: true, value: "done" });
await expect(withDistributedJobLease(replicaB, "email-inbox-webhook-drain", async () => "next", { ownerToken: "owner-b" })).resolves.toEqual({
acquired: true,
value: "next",
});
});
test("never deletes a successor's lease when an old owner releases late", async () => {
let now = 0;
const adapter = new MemoryAdapter({ now: () => now });
const store = createSharedStore({ namespace: "worker-lease-expiry-test", adapter });
let release!: () => void;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const first = withDistributedJobLease(
store,
"task",
async () => {
await gate;
return "old";
},
{ ownerToken: "old-owner", leaseSeconds: 10 }
);
await vi.waitFor(() => expect(adapter.get("shared-store:worker-lease-expiry-test:job-lease:task")).resolves.toBe("old-owner"));
now = 11_000;
await expect(store.set("job-lease:task", "successor", { nx: true, ttlSeconds: 120 })).resolves.toBe(true);
release();
await expect(first).resolves.toEqual({ acquired: true, value: "old" });
await expect(store.get("job-lease:task")).resolves.toBe("successor");
});
});
@@ -0,0 +1,90 @@
import { randomUUID } from "node:crypto";
import { MemoryAdapter, type SharedStore } from "@hivegrid/shared-store";
export const DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS = 120;
const RENEW_LEASE_SCRIPT = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("expire", KEYS[1], ARGV[2])
end
return 0
`;
const RELEASE_LEASE_SCRIPT = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
end
return 0
`;
MemoryAdapter.registerScript(RENEW_LEASE_SCRIPT, async (adapter, keys, args) => {
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
return (await adapter.expire(keys[0]!, Number(args[1]))) ? 1 : 0;
});
MemoryAdapter.registerScript(RELEASE_LEASE_SCRIPT, async (adapter, keys, args) => {
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
return adapter.del(keys[0]!);
});
export class DistributedJobLeaseLostError extends Error {
constructor(jobKey: string) {
super(`Distributed lease was lost while ${jobKey} was running`);
this.name = "DistributedJobLeaseLostError";
}
}
export type DistributedJobLeaseResult<Value> = { acquired: true; value: Value } | { acquired: false };
/**
* Run a cron body under a replica-wide Redis lease. SET NX prevents overlap;
* token-checked Lua renewal/release prevents an expired owner from touching a
* successor's lease. Store failures fail closed instead of running unlocked.
*/
export async function withDistributedJobLease<Value>(
store: SharedStore,
jobKey: string,
worker: () => Promise<Value>,
options: { leaseSeconds?: number; ownerToken?: string } = {}
): Promise<DistributedJobLeaseResult<Value>> {
const leaseSeconds = Math.max(10, Math.min(Math.floor(options.leaseSeconds ?? DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS), 3_600));
const ownerToken = options.ownerToken ?? randomUUID();
const key = `job-lease:${jobKey}`;
const acquired = await store.set(key, ownerToken, { nx: true, ttlSeconds: leaseSeconds });
if (!acquired) return { acquired: false };
let stopped = false;
let lost = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const heartbeatMs = Math.max(1_000, Math.floor((leaseSeconds * 1_000) / 3));
const scheduleHeartbeat = () => {
timer = setTimeout(async () => {
if (stopped) return;
try {
const renewed = await store.eval(RENEW_LEASE_SCRIPT, [key], [ownerToken, String(leaseSeconds)]);
if (Number(renewed) !== 1) lost = true;
} catch {
lost = true;
}
if (!stopped && !lost) scheduleHeartbeat();
}, heartbeatMs);
timer.unref?.();
};
scheduleHeartbeat();
try {
const value = await worker();
if (lost) throw new DistributedJobLeaseLostError(jobKey);
return { acquired: true, value };
} finally {
stopped = true;
if (timer) clearTimeout(timer);
try {
await store.eval(RELEASE_LEASE_SCRIPT, [key], [ownerToken]);
} catch {
// The bounded TTL is the final safety net when Redis disappears during
// release. Never issue an unconditional DEL that could remove a new owner.
}
}
}
@@ -2,12 +2,7 @@ import { PassThrough } from "node:stream";
import { describe, expect, it, vi } from "vitest";
import {
runDsgvoExport,
type DsgvoExportDependencies,
type DsgvoExportJobInput,
type DsgvoExportSink,
} from "./dsgvo-export.js";
import { runDsgvoExport, type DsgvoExportDependencies, type DsgvoExportJobInput, type DsgvoExportSink } from "./dsgvo-export.js";
function makeSink(): { sink: DsgvoExportSink; collected: () => string } {
const writeStream = new PassThrough();
@@ -107,6 +102,37 @@ describe("runDsgvoExport", () => {
expect(mails[0]?.signedUrl).toBe("https://s3/signed");
});
it("honours writable backpressure before advancing the row source", async () => {
const { sink } = makeSink();
const originalWrite = sink.writeStream.write.bind(sink.writeStream);
let drains = 0;
sink.writeStream.write = ((chunk: string | Uint8Array) => {
originalWrite(chunk);
queueMicrotask(() => {
drains += 1;
sink.writeStream.emit("drain");
});
return false;
}) as typeof sink.writeStream.write;
let drainsBeforeFirstRow = -1;
const { deps } = makeDeps({
openSink: () => sink,
tableSource: {
async *iterate(_userId, entry) {
if (entry.model === "Log") {
drainsBeforeFirstRow = drains;
yield { id: "log1" };
}
},
},
});
await runDsgvoExport(deps, baseJob);
expect(drainsBeforeFirstRow).toBeGreaterThanOrEqual(5);
expect(drains).toBeGreaterThan(drainsBeforeFirstRow);
});
it("audits a failure when the user cannot be loaded", async () => {
const { deps, audits } = makeDeps({
loadUser: vi.fn().mockResolvedValue(null),
@@ -108,14 +108,7 @@ export type DsgvoExportDependencies = {
metadata: Record<string, unknown>;
}) => Promise<void>;
/** Notify the user that the export is ready. */
sendExportReadyEmail: (input: {
userId: string;
toEmail: string;
signedUrl: string;
expiresAt: Date;
sizeBytes: number;
rowCount: number;
}) => Promise<void>;
sendExportReadyEmail: (input: { userId: string; toEmail: string; signedUrl: string; expiresAt: Date; sizeBytes: number; rowCount: number }) => Promise<void>;
/** Object-key builder — caller controls bucket + path layout. */
buildObjectKey: (input: { userId: string; requestId: string }) => { bucket: string; key: string };
/** Logger seam — no console.* in production code. */
@@ -153,31 +146,31 @@ export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoEx
});
try {
sink.writeStream.write(`{\n "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
sink.writeStream.write(` "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
sink.writeStream.write(` "requestId": ${JSON.stringify(job.requestId)},\n`);
sink.writeStream.write(` "user": ${JSON.stringify(user)},\n`);
sink.writeStream.write(` "sections": {\n`);
await writeChunk(sink.writeStream, `{\n "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
await writeChunk(sink.writeStream, ` "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
await writeChunk(sink.writeStream, ` "requestId": ${JSON.stringify(job.requestId)},\n`);
await writeChunk(sink.writeStream, ` "user": ${JSON.stringify(user)},\n`);
await writeChunk(sink.writeStream, ` "sections": {\n`);
let firstSection = true;
for (const entry of exportEntries) {
if (!firstSection) sink.writeStream.write(",\n");
if (!firstSection) await writeChunk(sink.writeStream, ",\n");
firstSection = false;
sink.writeStream.write(` ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
await writeChunk(sink.writeStream, ` ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
let rowCount = 0;
let firstRow = true;
for await (const row of deps.tableSource.iterate(job.userId, entry)) {
if (!firstRow) sink.writeStream.write(",");
if (!firstRow) await writeChunk(sink.writeStream, ",");
firstRow = false;
sink.writeStream.write(JSON.stringify(row));
await writeChunk(sink.writeStream, JSON.stringify(row));
rowCount += 1;
}
sink.writeStream.write("]");
await writeChunk(sink.writeStream, "]");
sections.push({ model: entry.model, column: entry.column, rowCount });
totalRows += rowCount;
}
sink.writeStream.write("\n }\n}\n");
await writeChunk(sink.writeStream, "\n }\n}\n");
sink.writeStream.end();
const completion = await sink.completionPromise;
@@ -242,3 +235,24 @@ export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoEx
throw error;
}
}
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
@@ -0,0 +1,180 @@
import { getRlsExecutionContext } from "@hivegrid/db/rls-context.server";
import { USER_FK_MAP } from "@hivegrid/db/dsgvo/user-fk-map";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
createWorkerDsgvoGraceCronService,
DSGVO_WORKER_SKIPPED_MODELS,
listWorkerProcessableFkEntries,
type DsgvoGracePrisma,
} from "./dsgvo-grace-bootstrap.js";
const createAuditEventMock = vi.fn().mockResolvedValue(undefined);
vi.mock("@hivegrid/db/audit-events.server", () => ({
createAuditEvent: (input: unknown) => createAuditEventMock(input),
}));
function makeLogger() {
return { info: vi.fn(), error: vi.fn() };
}
function lowerFirst(value: string): string {
return value.charAt(0).toLowerCase() + value.slice(1);
}
/**
* Fake prisma that mimics the FORCE-RLS behaviour instead of answering
* every query (AGENTS §RLS: a mock that always answers hides exactly the
* unbound-context bug this wiring fixes). Every delegate records the
* ambient RLS context and returns zero rows when no context is bound.
*/
function makeFakePrisma(options: { memberships: Record<string, string[]>; overdueByTenant: Record<string, string[]> }) {
const calls: Array<{ model: string; op: string; ctx: { userId?: string; tenantId?: string } | null }> = [];
const ctx = () => {
const current = getRlsExecutionContext();
return current ? { userId: current.userId, tenantId: current.tenantId } : null;
};
const base: Record<string, unknown> = {
tenant: {
findMany: async () => Object.keys(options.overdueByTenant).map((id) => ({ id })),
},
user: {
findMany: async () => {
const current = ctx();
calls.push({ model: "User", op: "findMany", ctx: current });
// RLS: without a bound tenant the membership branch never matches.
if (!current?.tenantId) return [];
return (options.overdueByTenant[current.tenantId] ?? []).map((id) => ({
id,
scheduledDeletionAt: new Date("2026-06-01T00:00:00Z"),
deletionRequestId: `req-${id}`,
}));
},
update: async (args: { where: { id: string }; data: Record<string, unknown> }) => {
const current = ctx();
calls.push({ model: "User", op: "update", ctx: current });
if (current?.userId !== args.where.id) {
throw new Error("RLS: User update requires the user context branch.");
}
return {};
},
},
tenantUser: {
findMany: async (args: { where: { userId: string } }) => {
const current = ctx();
calls.push({ model: "TenantUser", op: "findMany", ctx: current });
if (current?.userId !== args.where.userId) return [];
return (options.memberships[args.where.userId] ?? []).map((tenantId) => ({ tenantId }));
},
},
};
// Generic delegates for every worker-processable FK-map model. Tenant
// passes count one row, the trailing user-only pass counts zero (the
// tenant pass already consumed the row) — mirroring policy visibility.
for (const entry of listWorkerProcessableFkEntries()) {
const key = lowerFirst(entry.model);
if (base[key] && (base[key] as Record<string, unknown>).updateMany) continue;
const record = (op: string) => {
const current = ctx();
calls.push({ model: entry.model, op, ctx: current });
return { count: current?.tenantId ? 1 : 0 };
};
base[key] = {
...(base[key] as Record<string, unknown> | undefined),
updateMany: async () => record("updateMany"),
deleteMany: async () => record("deleteMany"),
};
}
return { prisma: base as unknown as DsgvoGracePrisma, calls };
}
beforeEach(() => {
createAuditEventMock.mockClear();
});
describe("createWorkerDsgvoGraceCronService", () => {
it("scans overdue users per tenant under a bound RLS context and dedupes across tenants", async () => {
const { prisma, calls } = makeFakePrisma({
memberships: {},
overdueByTenant: { t1: ["u1", "u2"], t2: ["u1"] },
});
const service = createWorkerDsgvoGraceCronService({ prisma, logger: makeLogger() });
const candidates = await service.listOverdueUsers(new Date("2026-07-17T00:00:00Z"));
expect(candidates.map((candidate) => candidate.userId).sort()).toEqual(["u1", "u2"]);
expect(candidates).toHaveLength(2);
const scans = calls.filter((call) => call.model === "User" && call.op === "findMany");
expect(scans).toHaveLength(2);
for (const scan of scans) {
expect(scan.ctx?.tenantId).toBeTruthy();
}
});
it("runs the cascade once per membership tenant plus a user-only pass, all context-bound", async () => {
const { prisma, calls } = makeFakePrisma({
memberships: { u1: ["t1", "t2"] },
overdueByTenant: { t1: [], t2: [] },
});
const logger = makeLogger();
const service = createWorkerDsgvoGraceCronService({ prisma, logger });
const result = await service.enqueueDeletion({
userId: "u1",
tenantId: "t1",
requestId: "req-u1",
actorType: "system",
actorId: "dsgvo-grace-cron",
mode: "pseudonymize",
reason: "dsgvo-grace-period-elapsed",
});
expect(result.jobId).toBe("inline-req-u1");
// Every processable FK-map entry ran in 3 contexts (t1, t2, user-only).
const groupUserCalls = calls.filter((call) => call.model === "GroupUser");
expect(groupUserCalls).toHaveLength(3);
expect(groupUserCalls.map((call) => call.ctx?.tenantId)).toEqual(["t1", "t2", undefined]);
for (const call of groupUserCalls) {
expect(call.ctx?.userId).toBe("u1");
}
// Pseudonymize mode keeps the User row: exactly one context-bound update.
const userUpdates = calls.filter((call) => call.model === "User" && call.op === "update");
expect(userUpdates).toHaveLength(1);
expect(userUpdates[0]?.ctx?.userId).toBe("u1");
// Grant-blocked models were never touched (no delegate exists for them,
// so a missed skip would have thrown), and the skip is audited.
const audit = createAuditEventMock.mock.calls.at(-1)?.[0] as { metadata: { skippedModels: string[] } };
expect(audit.metadata.skippedModels).toContain("UserRole.userId");
expect(logger.info).toHaveBeenCalledWith(
expect.objectContaining({ model: "UserRole" }),
"dsgvo-deletion.table-skipped-no-worker-grant"
);
});
it("refuses the hard-delete mode the worker grant set cannot satisfy", async () => {
const { prisma } = makeFakePrisma({ memberships: { u1: [] }, overdueByTenant: {} });
const service = createWorkerDsgvoGraceCronService({ prisma, logger: makeLogger() });
await expect(
service.enqueueDeletion({
userId: "u1",
tenantId: null,
requestId: "req-u1",
actorType: "system",
actorId: "dsgvo-grace-cron",
mode: "delete",
reason: "dsgvo-grace-period-elapsed",
})
).rejects.toThrow(/hard delete is not available/i);
});
it("classifies every USER_FK_MAP model so new admin-surface tables cannot slip through untriaged", () => {
const processable = new Set(listWorkerProcessableFkEntries().map((entry) => entry.model));
for (const entry of USER_FK_MAP) {
expect(processable.has(entry.model) || DSGVO_WORKER_SKIPPED_MODELS.has(entry.model)).toBe(true);
}
});
});
@@ -0,0 +1,249 @@
/**
* R-P2b Restarbeit (2026-07-17) — production wiring for the DSGVO
* grace-period cron. Until now `dsgvoGraceCronService` was never
* constructed, so the daily scan acked every tick with
* "dsgvoGraceCronService not configured" and a confirmed deletion
* request was never executed.
*
* Design (mirrors privacy-tenant-deletion-bootstrap / AK-P21):
*
* listOverdueUsers → per-tenant sweep (runActiveTenantSweep) with the
* tenant context bound via runWithRlsContext(); the
* `User` select policy exposes a user to the worker
* only through a bound membership tenant, so a
* global unscoped scan would silently return zero
* rows under FORCE RLS. Candidates are deduped by
* userId across tenants.
* enqueueDeletion → runs the tested runDsgvoDeletion cascade inline
* (serialized, like the AK-P21 hard-delete scan —
* no extra queue hop; the cron core already
* isolates failures per candidate and the daily
* re-scan is the retry, idempotent because the
* cascade flips User.pendingDeletion=false).
*
* RLS strategy per cascade table walk: every USER_FK_MAP operation runs
* once per membership tenant under { userId, tenantId } and once under
* { userId } alone. Tenant-scoped tables (TenantUser, GroupUser, …) bind
* through the tenant branch, user-owned tables (UserSession,
* TrustedDevice, MagicLinkToken, …) through the user branch. Re-running
* an operation in a second context is idempotent (the first pass already
* rewrote/deleted the rows), so the summed counts stay accurate.
*
* Deliberate skips: the RLS contract (20260713190000_appwide_rls_cutover)
* grants the worker no write privileges on the platform-admin/support
* tables and only SELECT on UserRole. A write attempt would abort the
* cascade with a permission error, so those entries are skipped loudly
* (log + audit metadata). The rows keep referencing the pseudonymized
* User skeleton — the identity link is anonymized even though the rows
* survive. Extending the grants is a tracked follow-up on the RLS
* contract, not something this bootstrap guesses at.
*
* Mode is always "pseudonymize" (R-P2b: PII-Anonymisierung, kein
* Hard-Delete — audit hash chain and retention FKs stay intact).
*/
import { createAuditEvent } from "@hivegrid/db/audit-events.server";
import { USER_FK_MAP, type UserFkMapEntry } from "@hivegrid/db/dsgvo/user-fk-map";
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
import { runActiveTenantSweep, throwOnActiveTenantSweepFailures, type ActiveTenantReader } from "./active-tenant-sweep.js";
import { runDsgvoDeletion, type DsgvoDeletionJobInput } from "./dsgvo-deletion.js";
import type { DsgvoGraceCandidate } from "./dsgvo-grace-cron.js";
/**
* USER_FK_MAP models the worker must not touch: no write GRANT (platform
* admin/support surface) or SELECT-only (UserRole). Kept in sync with the
* grant matrix now frozen in prisma/migrations/00000000000000_hivegrid_baseline.
*/
export const DSGVO_WORKER_SKIPPED_MODELS: ReadonlySet<string> = new Set([
"AdminTenantNote",
"AdminUser",
"Feedback",
"ImpersonationAudit",
"SupportTicket",
"SupportTicketMessage",
"UserRole",
]);
/** Upper bound of overdue users fetched per tenant per daily pass. */
export const DSGVO_GRACE_SCAN_PAGE_SIZE = 100;
type UpdateManyDelegate = { updateMany(args: { where: Record<string, unknown>; data: Record<string, unknown> }): Promise<{ count: number }> };
type DeleteManyDelegate = { deleteMany(args: { where: Record<string, unknown> }): Promise<{ count: number }> };
export type DsgvoGracePrisma = Record<string, unknown> &
ActiveTenantReader & {
user: {
findMany(args: {
where: Record<string, unknown>;
select: { id: true; scheduledDeletionAt: true; deletionRequestId: true };
orderBy: { id: "asc" };
take: number;
}): Promise<Array<{ id: string; scheduledDeletionAt: Date | null; deletionRequestId: string | null }>>;
update(args: { where: { id: string }; data: Record<string, unknown> }): Promise<unknown>;
};
tenantUser: {
findMany(args: { where: { userId: string }; select: { tenantId: true } }): Promise<Array<{ tenantId: string }>>;
};
};
export type WorkerDsgvoGraceDependencies = {
prisma: DsgvoGracePrisma;
now?: () => Date;
logger?: {
info(fields: Record<string, unknown>, message: string): void;
error(fields: Record<string, unknown>, message: string): void;
};
};
export type WorkerDsgvoGraceCronService = {
listOverdueUsers(now: Date): Promise<DsgvoGraceCandidate[]>;
enqueueDeletion(input: DsgvoDeletionJobInput): Promise<{ jobId?: string }>;
};
const noopLogger = {
info: () => undefined,
error: () => undefined,
};
export function createWorkerDsgvoGraceCronService(deps: WorkerDsgvoGraceDependencies): WorkerDsgvoGraceCronService {
const logger = deps.logger ?? noopLogger;
return {
async listOverdueUsers(now: Date): Promise<DsgvoGraceCandidate[]> {
const sweep = await runActiveTenantSweep(deps.prisma, {
initialState: new Map<string, DsgvoGraceCandidate>(),
worker: (tenantId) =>
runWithRlsContext({ tenantId }, async () => {
const rows = await deps.prisma.user.findMany({
where: {
pendingDeletion: true,
scheduledDeletionAt: { lte: now },
// Membership filter doubles as the RLS visibility branch:
// the bound tenant only ever sees its own members.
tenants: { some: { tenantId } },
},
select: { id: true, scheduledDeletionAt: true, deletionRequestId: true },
orderBy: { id: "asc" },
take: DSGVO_GRACE_SCAN_PAGE_SIZE,
});
return rows.map(
(row): DsgvoGraceCandidate => ({
userId: row.id,
tenantId,
// The where clause guarantees a non-null overdue timestamp.
scheduledDeletionAt: row.scheduledDeletionAt ?? now,
deletionRequestId: row.deletionRequestId,
})
);
}),
merge: (state, candidates) => {
for (const candidate of candidates) {
// A user in several workspaces is dispatched once; the cascade
// itself walks every membership tenant.
if (!state.has(candidate.userId)) state.set(candidate.userId, candidate);
}
return state;
},
});
throwOnActiveTenantSweepFailures("dsgvo-grace-period-scan", sweep.failures);
return [...sweep.state.values()];
},
async enqueueDeletion(input: DsgvoDeletionJobInput): Promise<{ jobId?: string }> {
// Membership list is readable under the user branch of the
// TenantUser select policy — no tenant context needed yet.
const memberships = await runWithRlsContext({ userId: input.userId }, () =>
deps.prisma.tenantUser.findMany({ where: { userId: input.userId }, select: { tenantId: true } })
);
const contexts: Array<{ userId: string; tenantId?: string }> = [
...memberships.map((membership) => ({ userId: input.userId, tenantId: membership.tenantId })),
{ userId: input.userId },
];
const skippedModels: string[] = [];
const runForEntry = async (
entry: UserFkMapEntry,
operation: (delegate: UpdateManyDelegate & DeleteManyDelegate) => Promise<{ count: number }>
): Promise<number> => {
if (DSGVO_WORKER_SKIPPED_MODELS.has(entry.model)) {
skippedModels.push(`${entry.model}.${entry.column}`);
logger.info(
{ userId: input.userId, model: entry.model, column: entry.column, strategy: entry.strategy },
"dsgvo-deletion.table-skipped-no-worker-grant"
);
return 0;
}
const delegate = deps.prisma[lowerFirst(entry.model)] as (UpdateManyDelegate & DeleteManyDelegate) | undefined;
if (!delegate) {
throw new Error(`DSGVO cascade is missing the prisma delegate for model ${entry.model}.`);
}
let affected = 0;
// Sequential on purpose: contexts are AsyncLocalStorage-scoped and
// the second pass is an idempotent no-op for rows the first pass
// already rewrote, so summed counts stay accurate.
for (const context of contexts) {
const result = await runWithRlsContext(context, () => operation(delegate));
affected += result.count;
}
return affected;
};
const result = await runDsgvoDeletion(
{
tableClient: {
hardDelete: (entry, userId) => runForEntry(entry, (delegate) => delegate.deleteMany({ where: { [entry.column]: userId } })),
pseudonymize: (entry, userId, pseudonymUserId) =>
runForEntry(entry, (delegate) => delegate.updateMany({ where: { [entry.column]: userId }, data: { [entry.column]: pseudonymUserId } })),
retainTombstone: (entry, userId) =>
runForEntry(entry, (delegate) => delegate.updateMany({ where: { [entry.column]: userId }, data: { [entry.column]: null } })),
},
userClient: {
hardDeleteUser: () => {
// The worker's RLS grant set cannot satisfy a hard delete
// (see file header); the cron only ever dispatches
// pseudonymize. Fail loudly instead of half-deleting.
throw new Error("DSGVO hard delete is not available under the worker RLS contract; use mode=pseudonymize.");
},
pseudonymizeUser: async (userId, update) => {
await runWithRlsContext({ userId }, () => deps.prisma.user.update({ where: { id: userId }, data: update }));
},
},
writeAuditEvent: async (auditInput) => {
// AuditEvent/AuditChainHead policies carry an explicit
// hivegrid_worker branch, so the chained write needs no bound
// context here.
await createAuditEvent({
...auditInput,
metadata: { ...auditInput.metadata, skippedModels } as Parameters<typeof createAuditEvent>[0]["metadata"],
});
},
logger,
},
input
);
logger.info(
{
userId: input.userId,
requestId: input.requestId,
tenants: memberships.length,
tables: result.tables.length,
skippedModels,
},
"dsgvo-deletion.cascade-complete"
);
// Inline execution — the "job id" is synthetic and only used for logs.
return { jobId: `inline-${input.requestId}` };
},
};
}
/** Ensure the full FK map stays classifiable — new admin-surface models must be triaged. */
export function listWorkerProcessableFkEntries(): UserFkMapEntry[] {
return USER_FK_MAP.filter((entry) => !DSGVO_WORKER_SKIPPED_MODELS.has(entry.model));
}
function lowerFirst(value: string): string {
return value.charAt(0).toLowerCase() + value.slice(1);
}
@@ -32,7 +32,7 @@ describe("dsgvo-grace-cron", () => {
expect(result.failed).toBe(0);
expect(enqueue).toHaveBeenCalledTimes(2);
expect(enqueue.mock.calls[0][0].requestId).toBe("req-1");
expect(enqueue.mock.calls[0][0].mode).toBe("delete");
expect(enqueue.mock.calls[0][0].mode).toBe("pseudonymize");
expect(enqueue.mock.calls[0][0].actorType).toBe("system");
expect(enqueue.mock.calls[0][0].reason).toBe("dsgvo-grace-period-elapsed");
// u2 had no deletionRequestId — synthesize a grace-derived one.
@@ -49,7 +49,7 @@ describe("dsgvo-grace-cron", () => {
const enqueue = vi
.fn()
.mockResolvedValueOnce({ jobId: "j1" })
.mockRejectedValueOnce(new Error("redis down"))
.mockRejectedValueOnce(new Error("redis down for private@example.test with client_secret=secret-value"))
.mockResolvedValueOnce({ jobId: "j3" });
const logger = makeLogger();
const result = await runDsgvoGraceCron({
@@ -60,11 +60,16 @@ describe("dsgvo-grace-cron", () => {
expect(result.scanned).toBe(3);
expect(result.dispatched).toBe(2);
expect(result.failed).toBe(1);
expect(result.failures[0]).toEqual({ userId: "bad", error: "redis down" });
expect(result.failures[0]).toEqual({
userId: "bad",
error: "redis down for [redacted-email] with client_secret=[redacted]",
});
expect(logger.error).toHaveBeenCalledWith(
expect.objectContaining({ userId: "bad", err: "redis down" }),
expect.objectContaining({ userId: "bad", err: "redis down for [redacted-email] with client_secret=[redacted]" }),
"dsgvo-grace-cron.dispatch-failed",
);
expect(JSON.stringify(logger.error.mock.calls)).not.toContain("secret-value");
expect(JSON.stringify(logger.error.mock.calls)).not.toContain("private@example.test");
});
it("returns zero when no candidates", async () => {
@@ -50,7 +50,12 @@ export type DsgvoGraceCronDependencies = {
requestId: string;
actorType: "system";
actorId: string;
mode: "delete";
// R-P2b design: DSGVO deletion is PII pseudonymization, not a hard
// delete — the User row survives so the audit hash chain and
// retention-bound foreign keys stay intact. Hard delete additionally
// requires worker write grants the RLS contract deliberately withholds
// (AdminUser/UserRole/support tables), so the cron never dispatches it.
mode: "pseudonymize";
reason: string;
}) => Promise<{ jobId?: string }>;
/** Optional. Defaults to () => new Date(). */
@@ -96,7 +101,7 @@ export async function runDsgvoGraceCron(deps: DsgvoGraceCronDependencies): Promi
requestId,
actorType: "system",
actorId: systemActorId,
mode: "delete",
mode: "pseudonymize",
reason: "dsgvo-grace-period-elapsed",
});
dispatched += 1;
@@ -105,7 +110,7 @@ export async function runDsgvoGraceCron(deps: DsgvoGraceCronDependencies): Promi
"dsgvo-grace-cron.dispatched",
);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
const message = redactDsgvoGraceDiagnostic(err instanceof Error ? err.message : String(err));
failures.push({ userId: candidate.userId, error: message });
deps.logger.error({ userId: candidate.userId, err: message }, "dsgvo-grace-cron.dispatch-failed");
}
@@ -119,6 +124,18 @@ export async function runDsgvoGraceCron(deps: DsgvoGraceCronDependencies): Promi
};
}
function redactDsgvoGraceDiagnostic(message: string): string {
return message
.replace(/([?&](?:access_token|api_key|client_secret|password|secret|token)=)[^&\s]+/gi, "$1[redacted]")
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
.replace(
/(authorization|access[_-]?key(?:id)?|api[-_]?key|client[-_]?secret|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password|body)(\s*[=:]\s*)\S+/gi,
"$1$2[redacted]",
)
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted-email]")
.slice(0, 1_000);
}
/**
* Helper for the worker bootstrap. Given a BullMQ-like queue with the
* `upsertJobScheduler` API, install (or replace) the 24h repeat schedule.
@@ -91,7 +91,7 @@ describe("email sequence runtime deps", () => {
backoff: { type: "exponential", delay: 1_000 },
delay: 600_000,
jobId: "email-sequence-step:run_1:2",
removeOnComplete: false,
removeOnComplete: 10_000,
removeOnFail: 1_000,
}
);
@@ -78,6 +78,7 @@ export type EmailSequenceStepQueueOptions = {
const EMAIL_SEQUENCE_STEP_ATTEMPTS = 3;
const EMAIL_SEQUENCE_STEP_BACKOFF_MS = 1_000;
const EMAIL_SEQUENCE_STEP_REMOVE_ON_COMPLETE = 10_000;
const EMAIL_SEQUENCE_STEP_REMOVE_ON_FAIL = 1_000;
/**
@@ -266,11 +267,11 @@ function renderSequenceTemplate(
return { ...email, subject };
}
case "trial-reminder": {
const upgradeUrl = stringVariable(vars, trigger, "upgradeUrl", `${appUrl}/settings/subscription`);
const upgradeUrl = stringVariable(vars, trigger, "upgradeUrl", `${appUrl}/settings/billing`);
const email = renderEmail(TrialReminderEmail({
productName,
daysRemaining: numberVariable(vars, trigger, "daysRemaining", 3),
upgradeUrl: upgradeUrl ?? `${appUrl}/settings/subscription`,
upgradeUrl: upgradeUrl ?? `${appUrl}/settings/billing`,
unsubscribeUrl,
...(recipientName ? { recipientName } : {}),
}));
@@ -405,7 +406,7 @@ async function scheduleEmailSequenceStep(
backoff: { type: "exponential", delay: EMAIL_SEQUENCE_STEP_BACKOFF_MS },
delay,
jobId,
removeOnComplete: false,
removeOnComplete: EMAIL_SEQUENCE_STEP_REMOVE_ON_COMPLETE,
removeOnFail: EMAIL_SEQUENCE_STEP_REMOVE_ON_FAIL,
});
opts.logger.info?.(
@@ -0,0 +1,28 @@
import { describe, expect, test } from "vitest";
import {
assertPreviewOutputSize,
assertPreviewRasterMetadata,
assertPreviewSourceSize,
FILE_PREVIEW_MAX_DIMENSION,
FILE_PREVIEW_MAX_PAGES,
FILE_PREVIEW_MAX_PIXELS,
FILE_PREVIEW_MAX_SOURCE_BYTES,
} from "./file-preview-decoder-contract.js";
describe("file preview decoder limits", () => {
test("accepts a normal single-page image", () => {
expect(() => assertPreviewRasterMetadata({ width: 4_000, height: 3_000, pages: 1 })).not.toThrow();
});
test("rejects decompression-bomb dimensions, pixels and page counts", () => {
expect(() => assertPreviewRasterMetadata({ width: FILE_PREVIEW_MAX_DIMENSION + 1, height: 1 })).toThrow("dimension limit");
expect(() => assertPreviewRasterMetadata({ width: 10_000, height: Math.floor(FILE_PREVIEW_MAX_PIXELS / 10_000) + 1 })).toThrow("pixel limit");
expect(() => assertPreviewRasterMetadata({ width: 1, height: 1, pages: FILE_PREVIEW_MAX_PAGES + 1 })).toThrow("page limit");
});
test("rejects oversized source and generated output buffers", () => {
expect(() => assertPreviewSourceSize(FILE_PREVIEW_MAX_SOURCE_BYTES + 1)).toThrow("byte limit");
expect(() => assertPreviewOutputSize(11, 10, "fixture")).toThrow("output limit");
});
});
@@ -0,0 +1,57 @@
export const FILE_PREVIEW_MAX_SOURCE_BYTES = 100 * 1024 * 1024;
export const FILE_PREVIEW_MAX_PIXELS = 40_000_000;
export const FILE_PREVIEW_MAX_DIMENSION = 16_384;
export const FILE_PREVIEW_MAX_PAGES = 200;
export const FILE_PREVIEW_MAX_INTERMEDIATE_BYTES = 64 * 1024 * 1024;
export const FILE_PREVIEW_MAX_OUTPUT_BYTES = 20 * 1024 * 1024;
export const FILE_PREVIEW_MAX_THUMBNAIL_BYTES = 3 * 1024 * 1024;
export const FILE_PREVIEW_DECODE_TIMEOUT_MS = 20_000;
export const FILE_PREVIEW_DECODER_HEAP_MB = 256;
export const FILE_PREVIEW_MAX_CONCURRENT_DECODERS = 2;
export type FilePreviewDecoderMime = "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/heic" | "image/heif" | "application/pdf";
export type FilePreviewDecoderRequest = {
input: Uint8Array;
mime: FilePreviewDecoderMime;
};
export type FilePreviewDecoderSuccess = {
ok: true;
preview: Uint8Array;
thumbnail: Uint8Array;
previewExtension: "jpg" | "png" | "webp";
previewContentType: "image/jpeg" | "image/png" | "image/webp";
};
export type FilePreviewDecoderResponse = FilePreviewDecoderSuccess | { ok: false; error: string };
export function assertPreviewSourceSize(byteLength: number): void {
if (!Number.isSafeInteger(byteLength) || byteLength < 0 || byteLength > FILE_PREVIEW_MAX_SOURCE_BYTES) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_SOURCE_BYTES}-byte limit`);
}
}
export function assertPreviewRasterMetadata(metadata: { width?: number; height?: number; pages?: number }): void {
const width = metadata.width;
const height = metadata.height;
const pages = metadata.pages ?? 1;
if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width! <= 0 || height! <= 0) {
throw new Error("Preview source has invalid image dimensions");
}
if (width! > FILE_PREVIEW_MAX_DIMENSION || height! > FILE_PREVIEW_MAX_DIMENSION) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_DIMENSION}-pixel dimension limit`);
}
if (width! * height! > FILE_PREVIEW_MAX_PIXELS) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_PIXELS}-pixel limit`);
}
if (!Number.isSafeInteger(pages) || pages <= 0 || pages > FILE_PREVIEW_MAX_PAGES) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_PAGES}-page limit`);
}
}
export function assertPreviewOutputSize(byteLength: number, maxBytes: number, label: string): void {
if (!Number.isSafeInteger(byteLength) || byteLength <= 0 || byteLength > maxBytes) {
throw new Error(`${label} exceeds the ${maxBytes}-byte output limit`);
}
}
@@ -0,0 +1,107 @@
import { EventEmitter } from "node:events";
import type { ChildProcess, fork } from "node:child_process";
import { describe, expect, test, vi } from "vitest";
import { decodeFilePreviewIsolated } from "./file-preview-decoder-isolated.js";
function fakeChild() {
const child = Object.assign(new EventEmitter(), {
connected: true,
exitCode: null,
signalCode: null,
disconnect: vi.fn(),
kill: vi.fn(() => true),
send: vi.fn(),
stderr: null,
});
child.disconnect.mockImplementation(() => {
child.connected = false;
});
return child as unknown as ChildProcess;
}
describe("decodeFilePreviewIsolated", () => {
const response = {
ok: true,
preview: Uint8Array.from([1, 2]),
thumbnail: Uint8Array.from([3]),
previewExtension: "webp",
previewContentType: "image/webp",
} as const;
test("kills a decoder that exceeds its wall-clock budget", async () => {
const child = fakeChild();
const forkImpl = vi.fn(() => child) as unknown as typeof fork;
await expect(
decodeFilePreviewIsolated(Buffer.from("input"), "image/png", {
forkImpl,
processPath: "/fixture/decoder.js",
timeoutMs: 5,
})
).rejects.toThrow("5ms wall-clock limit");
expect(child.kill).toHaveBeenCalledWith("SIGKILL");
});
test("accepts bounded binary output and lets the disposable process exit cleanly", async () => {
const child = fakeChild();
const send = vi.fn(() => {
queueMicrotask(() => child.emit("message", response));
return true;
});
Object.defineProperty(child, "send", { configurable: true, value: send });
const forkImpl = vi.fn(() => child) as unknown as typeof fork;
await expect(decodeFilePreviewIsolated(Buffer.from("input"), "image/png", { forkImpl, processPath: "/fixture/decoder.js" })).resolves.toMatchObject({
previewExtension: "webp",
previewContentType: "image/webp",
});
expect(child.disconnect).toHaveBeenCalledOnce();
expect(child.kill).not.toHaveBeenCalled();
});
test("sends the exact input slice without copying its backing buffer", async () => {
const child = fakeChild();
let sentMessage: unknown;
const send = vi.fn((message: unknown) => {
sentMessage = message;
queueMicrotask(() => child.emit("message", response));
return true;
});
Object.defineProperty(child, "send", { configurable: true, value: send });
const forkImpl = vi.fn(() => child) as unknown as typeof fork;
const backing = Buffer.from([99, 98, 1, 2, 3, 97]);
const input = backing.subarray(2, 5);
await decodeFilePreviewIsolated(input, "image/png", { forkImpl, processPath: "/fixture/decoder.js" });
const message = sentMessage as { input: Uint8Array; mime: string };
expect(message.input.buffer).toBe(input.buffer);
expect(message.input.byteOffset).toBe(input.byteOffset);
expect(message.input.byteLength).toBe(input.byteLength);
expect(Array.from(message.input)).toEqual([1, 2, 3]);
expect(message.mime).toBe("image/png");
});
test("caps aggregate decoder CPU and memory pressure at two child processes", async () => {
const children = [fakeChild(), fakeChild(), fakeChild()];
let forkIndex = 0;
const forkMock = vi.fn(() => children[forkIndex++]!);
const forkImpl = forkMock as unknown as typeof fork;
const options = { forkImpl, processPath: "/fixture/decoder.js", timeoutMs: 1_000 };
const pending = [
decodeFilePreviewIsolated(Buffer.from("one"), "image/png", options),
decodeFilePreviewIsolated(Buffer.from("two"), "image/png", options),
decodeFilePreviewIsolated(Buffer.from("three"), "image/png", options),
];
await vi.waitFor(() => expect(forkMock).toHaveBeenCalledTimes(2));
children[0]!.emit("message", response);
await vi.waitFor(() => expect(forkMock).toHaveBeenCalledTimes(3));
children[1]!.emit("message", response);
children[2]!.emit("message", response);
await expect(Promise.all(pending)).resolves.toHaveLength(3);
});
});
@@ -0,0 +1,163 @@
import { fork } from "node:child_process";
import { existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
assertPreviewOutputSize,
assertPreviewSourceSize,
FILE_PREVIEW_DECODER_HEAP_MB,
FILE_PREVIEW_DECODE_TIMEOUT_MS,
FILE_PREVIEW_MAX_CONCURRENT_DECODERS,
FILE_PREVIEW_MAX_OUTPUT_BYTES,
FILE_PREVIEW_MAX_THUMBNAIL_BYTES,
type FilePreviewDecoderMime,
type FilePreviewDecoderResponse,
type FilePreviewDecoderSuccess,
} from "./file-preview-decoder-contract.js";
type ForkImplementation = typeof fork;
let activeDecoders = 0;
const decoderWaiters: Array<() => void> = [];
function childExecArgv(args: readonly string[]): string[] {
const retained: string[] = [];
for (let index = 0; index < args.length; index += 1) {
const value = args[index]!;
if (["--eval", "-e", "--print", "-p"].includes(value)) {
index += 1;
continue;
}
if (value === "--input-type") {
index += 1;
continue;
}
if (value.startsWith("--input-type=") || value.startsWith("--max-old-space-size=")) {
continue;
}
retained.push(value);
}
return retained.concat(`--max-old-space-size=${FILE_PREVIEW_DECODER_HEAP_MB}`);
}
async function acquireDecoderSlot(): Promise<() => void> {
if (activeDecoders < FILE_PREVIEW_MAX_CONCURRENT_DECODERS) {
activeDecoders += 1;
} else {
await new Promise<void>((resolve) => decoderWaiters.push(resolve));
}
let released = false;
return () => {
if (released) return;
released = true;
const next = decoderWaiters.shift();
if (next) next();
else activeDecoders -= 1;
};
}
export type IsolatedDecoderOptions = {
timeoutMs?: number;
processPath?: string;
forkImpl?: ForkImplementation;
isProduction?: boolean;
};
function compiledProcessCandidates(): string[] {
const currentDir = dirname(fileURLToPath(import.meta.url));
return [
join(currentDir, "file-preview-decoder-process.js"),
join(currentDir, "jobs", "file-preview-decoder-process.js"),
join(dirname(dirname(currentDir)), "dist", "jobs", "file-preview-decoder-process.js"),
];
}
export function resolveFilePreviewDecoderProcess(options: Pick<IsolatedDecoderOptions, "processPath" | "isProduction"> = {}): string {
if (options.processPath) return options.processPath;
for (const candidate of compiledProcessCandidates()) {
if (existsSync(candidate)) return candidate;
}
if (options.isProduction ?? process.env.NODE_ENV === "production") {
throw new Error("Compiled file preview decoder process is required in production");
}
return join(dirname(fileURLToPath(import.meta.url)), "file-preview-decoder-process.ts");
}
function validateResponse(message: unknown): FilePreviewDecoderSuccess {
if (!message || typeof message !== "object" || !("ok" in message)) {
throw new Error("Preview decoder returned an invalid response");
}
const response = message as FilePreviewDecoderResponse;
if (!response.ok) {
throw new Error(`Preview decoder rejected the source: ${response.error}`);
}
if (!(response.preview instanceof Uint8Array) || !(response.thumbnail instanceof Uint8Array)) {
throw new Error("Preview decoder returned invalid binary output");
}
assertPreviewOutputSize(response.preview.byteLength, FILE_PREVIEW_MAX_OUTPUT_BYTES, "Preview image");
assertPreviewOutputSize(response.thumbnail.byteLength, FILE_PREVIEW_MAX_THUMBNAIL_BYTES, "Preview thumbnail");
return response;
}
export async function decodeFilePreviewIsolated(
input: Buffer,
mime: FilePreviewDecoderMime,
options: IsolatedDecoderOptions = {}
): Promise<FilePreviewDecoderSuccess> {
assertPreviewSourceSize(input.byteLength);
const releaseDecoderSlot = await acquireDecoderSlot();
try {
return await decodeWithChildProcess(input, mime, options);
} finally {
releaseDecoderSlot();
}
}
async function decodeWithChildProcess(input: Buffer, mime: FilePreviewDecoderMime, options: IsolatedDecoderOptions): Promise<FilePreviewDecoderSuccess> {
const timeoutMs = Math.max(1, Math.min(Math.floor(options.timeoutMs ?? FILE_PREVIEW_DECODE_TIMEOUT_MS), 60_000));
const processPath = resolveFilePreviewDecoderProcess(options);
const forkImpl = options.forkImpl ?? fork;
const execArgv = childExecArgv(process.execArgv);
const child = forkImpl(processPath, [], {
execArgv,
serialization: "advanced",
stdio: ["ignore", "ignore", "pipe", "ipc"],
});
// Native libraries can emit diagnostics. Drain them so their pipe cannot
// backpressure the decoder; the parent logs only the bounded error message
// returned over IPC, never attacker-controlled binary input.
child.stderr?.resume();
return await new Promise<FilePreviewDecoderSuccess>((resolve, reject) => {
let settled = false;
const finish = (result: { ok: true; value: FilePreviewDecoderSuccess } | { ok: false; error: Error }, forceKill = false) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
child.removeAllListeners();
if (child.connected) child.disconnect();
if (forceKill && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
if (result.ok) resolve(result.value);
else reject(result.error);
};
const timeout = setTimeout(() => {
finish({ ok: false, error: new Error(`Preview decoder exceeded the ${timeoutMs}ms wall-clock limit`) }, true);
}, timeoutMs);
child.once("message", (message) => {
try {
finish({ ok: true, value: validateResponse(message) });
} catch (error) {
finish({ ok: false, error: error instanceof Error ? error : new Error(String(error)) }, true);
}
});
child.once("error", (error) => finish({ ok: false, error }, true));
child.once("exit", (code, signal) => {
finish({ ok: false, error: new Error(`Preview decoder exited before returning output (code=${code ?? "none"}, signal=${signal ?? "none"})`) });
});
const inputView = new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
child.send({ input: inputView, mime }, (error) => error && finish({ ok: false, error }, true));
});
}
@@ -0,0 +1,19 @@
import { describe, expect, test } from "vitest";
import sharp from "sharp";
import { decodeFilePreview } from "./file-preview-decoder-process.js";
describe("decodeFilePreview", () => {
test("renders a bounded raster preview and thumbnail", async () => {
const input = await sharp({
create: { width: 64, height: 48, channels: 3, background: "#f1b729" },
})
.png()
.toBuffer();
const result = await decodeFilePreview({ input, mime: "image/png" });
expect(result).toMatchObject({ ok: true, previewExtension: "webp", previewContentType: "image/webp" });
await expect(sharp(result.preview).metadata()).resolves.toMatchObject({ width: 64, height: 48, format: "webp" });
await expect(sharp(result.thumbnail).metadata()).resolves.toMatchObject({ width: 64, height: 48, format: "webp" });
});
});
@@ -0,0 +1,181 @@
import process from "node:process";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { pdfToPng } from "pdf-to-png-converter";
import sharp from "sharp";
import {
assertPreviewOutputSize,
assertPreviewRasterMetadata,
assertPreviewSourceSize,
FILE_PREVIEW_MAX_INTERMEDIATE_BYTES,
FILE_PREVIEW_MAX_OUTPUT_BYTES,
FILE_PREVIEW_MAX_PAGES,
FILE_PREVIEW_MAX_PIXELS,
FILE_PREVIEW_MAX_SOURCE_BYTES,
FILE_PREVIEW_MAX_THUMBNAIL_BYTES,
type FilePreviewDecoderRequest,
type FilePreviewDecoderResponse,
type FilePreviewDecoderSuccess,
} from "./file-preview-decoder-contract.js";
const PREVIEW_MAX_DIMENSION = 2_048;
const THUMBNAIL_MAX_DIMENSION = 480;
const SHARP_TIMEOUT_SECONDS = 15;
// This module runs in a disposable child process. Keeping libvips to one
// thread and disabling its cache makes peak native memory predictable; the
// parent enforces a separate hard wall-clock timeout and kills this process.
sharp.cache(false);
sharp.concurrency(1);
function asExactUint8Array(buffer: Buffer): Uint8Array {
return Uint8Array.from(buffer);
}
async function renderRaster(input: Buffer, format: "jpeg" | "png" | "webp", maxDimension: number): Promise<Buffer> {
const pipeline = sharp(input, {
failOn: "warning",
limitInputPixels: FILE_PREVIEW_MAX_PIXELS,
page: 0,
pages: 1,
sequentialRead: true,
});
const metadata = await pipeline.metadata();
assertPreviewRasterMetadata(metadata);
const resized = pipeline
.timeout({ seconds: SHARP_TIMEOUT_SECONDS })
.rotate()
.resize({ width: maxDimension, height: maxDimension, fit: "inside", withoutEnlargement: true });
if (format === "jpeg") return resized.jpeg({ quality: 82, mozjpeg: true }).toBuffer();
if (format === "png") return resized.png({ compressionLevel: 9 }).toBuffer();
return resized.webp({ quality: 80 }).toBuffer();
}
async function inspectRaster(input: Buffer): Promise<void> {
const metadata = await sharp(input, {
failOn: "warning",
limitInputPixels: FILE_PREVIEW_MAX_PIXELS,
page: 0,
pages: 1,
sequentialRead: true,
})
.timeout({ seconds: SHARP_TIMEOUT_SECONDS })
.metadata();
assertPreviewRasterMetadata(metadata);
}
async function convertHeicToJpeg(input: Buffer): Promise<Buffer> {
const moduleName = "heic-convert";
const loaded = (await import(/* @vite-ignore */ moduleName)) as
| { default: (options: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer> }
| ((options: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer>);
const heicConvert = typeof loaded === "function" ? loaded : loaded.default;
const converted = await heicConvert({ buffer: input, format: "JPEG", quality: 0.85 });
const output = Buffer.isBuffer(converted) ? Buffer.from(converted) : Buffer.from(new Uint8Array(converted));
assertPreviewOutputSize(output.byteLength, FILE_PREVIEW_MAX_INTERMEDIATE_BYTES, "HEIC intermediate image");
return output;
}
async function renderPdfFirstPage(input: Buffer): Promise<Buffer> {
const safeOptions = {
concurrencyLimit: 1,
disableFontFace: true,
enableXfa: false,
maxInputBytes: FILE_PREVIEW_MAX_SOURCE_BYTES,
processPagesInParallel: false,
useSystemFonts: false,
verbosityLevel: 0 as const,
viewportScale: 2,
};
// Inspect page 1 and the first forbidden page without creating a canvas.
// Presence of page MAX+1 proves the document exceeds the page limit.
const metadata = await pdfToPng(input, {
...safeOptions,
pagesToProcess: [1, FILE_PREVIEW_MAX_PAGES + 1],
returnMetadataOnly: true,
});
const firstPage = metadata.find((page) => page.pageNumber === 1);
if (!firstPage) {
throw new Error("PDF rendered no pages");
}
if (metadata.some((page) => page.pageNumber === FILE_PREVIEW_MAX_PAGES + 1)) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_PAGES}-page limit`);
}
assertPreviewRasterMetadata({ width: firstPage.width, height: firstPage.height, pages: 1 });
const pages = await pdfToPng(input, {
...safeOptions,
pagesToProcess: [1],
returnPageContent: true,
});
const content = pages[0]?.content;
if (!content) {
throw new Error("PDF rendered no page content");
}
const output = Buffer.from(content);
assertPreviewOutputSize(output.byteLength, FILE_PREVIEW_MAX_INTERMEDIATE_BYTES, "PDF intermediate image");
return output;
}
export async function decodeFilePreview(request: FilePreviewDecoderRequest): Promise<FilePreviewDecoderSuccess> {
assertPreviewSourceSize(request.input.byteLength);
const input = Buffer.from(request.input.buffer, request.input.byteOffset, request.input.byteLength);
let preview: Buffer;
let previewExtension: FilePreviewDecoderSuccess["previewExtension"];
let previewContentType: FilePreviewDecoderSuccess["previewContentType"];
if (request.mime === "image/heic" || request.mime === "image/heif") {
// Parse HEIF container dimensions through libvips before heic-convert can
// allocate the decoded bitmap. Failure is fail-closed even if the fallback
// decoder might otherwise accept the file.
await inspectRaster(input);
const jpeg = await convertHeicToJpeg(input);
preview = await renderRaster(jpeg, "jpeg", PREVIEW_MAX_DIMENSION);
previewExtension = "jpg";
previewContentType = "image/jpeg";
} else if (request.mime === "application/pdf") {
const png = await renderPdfFirstPage(input);
preview = await renderRaster(png, "png", PREVIEW_MAX_DIMENSION);
previewExtension = "png";
previewContentType = "image/png";
} else {
preview = await renderRaster(input, "webp", PREVIEW_MAX_DIMENSION);
previewExtension = "webp";
previewContentType = "image/webp";
}
assertPreviewOutputSize(preview.byteLength, FILE_PREVIEW_MAX_OUTPUT_BYTES, "Preview image");
const thumbnail = await renderRaster(preview, "webp", THUMBNAIL_MAX_DIMENSION);
assertPreviewOutputSize(thumbnail.byteLength, FILE_PREVIEW_MAX_THUMBNAIL_BYTES, "Preview thumbnail");
return {
ok: true,
preview: asExactUint8Array(preview),
thumbnail: asExactUint8Array(thumbnail),
previewExtension,
previewContentType,
};
}
function errorMessage(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message.slice(0, 1_024);
}
const isMainProcess = process.argv[1] !== undefined && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (isMainProcess && typeof process.send === "function") {
process.once("message", (message: FilePreviewDecoderRequest) => {
void decodeFilePreview(message)
.then((result) => {
process.send?.(result, () => process.exit(0));
})
.catch((error: unknown) => {
const result: FilePreviewDecoderResponse = { ok: false, error: errorMessage(error) };
process.send?.(result, () => process.exit(1));
});
});
}
@@ -0,0 +1,62 @@
import { Readable } from "node:stream";
import { describe, expect, test, vi } from "vitest";
import { assertPreviewSource, readPreviewBodyWithinLimit } from "./file-preview.js";
describe("readPreviewBodyWithinLimit", () => {
test("reads a streamed object within the configured limit", async () => {
const body = Readable.from([Buffer.from("hello "), Buffer.from("world")]);
await expect(readPreviewBodyWithinLimit(body, 11, 11)).resolves.toEqual(Buffer.from("hello world"));
});
test("rejects oversized metadata before consuming the body", async () => {
const destroy = vi.fn();
const body = { destroy, transformToByteArray: vi.fn() };
await expect(readPreviewBodyWithinLimit(body, 12, 11)).rejects.toThrow("exceeds the 11-byte limit");
expect(destroy).toHaveBeenCalledOnce();
expect(body.transformToByteArray).not.toHaveBeenCalled();
});
test("rejects a stream that exceeds the limit when metadata is missing", async () => {
const body = Readable.from([Buffer.alloc(6), Buffer.alloc(6)]);
await expect(readPreviewBodyWithinLimit(body, undefined, 11)).rejects.toThrow("exceeds the 11-byte limit");
});
test("checks fallback byte-array bodies as well", async () => {
const body = { transformToByteArray: vi.fn(async () => new Uint8Array(12)) };
await expect(readPreviewBodyWithinLimit(body, undefined, 11)).rejects.toThrow("exceeds the 11-byte limit");
});
test("destroys and rejects a stalled source stream at the wall-clock limit", async () => {
const destroy = vi.fn();
const body = { destroy, transformToByteArray: vi.fn(() => new Promise<Uint8Array>(() => {})) };
await expect(readPreviewBodyWithinLimit(body, undefined, 11, 5)).rejects.toThrow("5ms limit");
expect(destroy).toHaveBeenCalledOnce();
});
});
describe("assertPreviewSource", () => {
const payload = {
kind: "preview.file",
fileId: "file_1",
tenantId: "tenant_1",
bucket: "uploads",
storageKey: "tenants/tenant_1/files/file_1/original",
mimeType: "image/png",
} as const;
test("accepts the configured bucket and canonical tenant file key", () => {
expect(() => assertPreviewSource(payload, "uploads")).not.toThrow();
});
test("rejects cross-bucket and cross-tenant source targets", () => {
expect(() => assertPreviewSource({ ...payload, bucket: "backups" }, "uploads")).toThrow("bucket");
expect(() => assertPreviewSource({ ...payload, storageKey: "tenants/tenant_2/files/file_1/original" }, "uploads")).toThrow("canonical tenant file path");
});
});
@@ -0,0 +1,109 @@
import { beforeEach, describe, expect, test, vi } from "vitest";
const aws = vi.hoisted(() => ({ send: vi.fn() }));
vi.mock("@aws-sdk/client-s3", () => ({
GetObjectCommand: class GetObjectCommand {
constructor(readonly input: unknown) {}
},
PutObjectCommand: class PutObjectCommand {
constructor(readonly input: unknown) {}
},
S3Client: class S3Client {
send = aws.send;
},
}));
import { createWorkerFilePreviewService, type FilePreviewPrisma } from "./file-preview.js";
const payload = {
kind: "preview.file",
fileId: "file_1",
tenantId: "tenant_1",
bucket: "uploads",
storageKey: "tenants/tenant_1/files/file_1/original",
mimeType: "image/jpeg",
} as const;
function buildPrisma(scanStatus: string, overrides: Record<string, unknown> = {}) {
const findFirst = vi.fn(async () => ({
id: payload.fileId,
mimeType: "image/png",
previewStatus: "pending",
previewStorageKey: null,
scanStatus,
sizeBytes: 4,
storageKey: payload.storageKey,
thumbnailStorageKey: null,
...overrides,
}));
const update = vi.fn(async () => ({}));
const tx = { $executeRaw: vi.fn(async () => 1), file: { findFirst, update } };
const prisma = {
$transaction: vi.fn(async (callback: (transaction: typeof tx) => Promise<unknown>) => callback(tx)),
} as unknown as FilePreviewPrisma;
return { findFirst, prisma, update };
}
const env = {
STORAGE_ACCESS_KEY_ID: "access",
STORAGE_BUCKET: "uploads",
STORAGE_REGION: "eu-central-1",
STORAGE_SECRET_ACCESS_KEY: "secret",
};
describe("file preview malware-scan gate", () => {
beforeEach(() => aws.send.mockReset());
test.each(["pending", "infected", "skipped"])("never loads or decodes a %s file", async (scanStatus) => {
const { prisma, update } = buildPrisma(scanStatus);
const decodePreview = vi.fn();
const service = createWorkerFilePreviewService(env, {
prisma,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
decodePreview,
})!;
await expect(service.generatePreview(payload)).rejects.toThrow("malware scan is clean");
expect(aws.send).not.toHaveBeenCalled();
expect(decodePreview).not.toHaveBeenCalled();
expect(update).not.toHaveBeenCalled();
});
test("uses the persisted MIME and decodes only after a clean verdict", async () => {
const { prisma, update } = buildPrisma("clean");
aws.send.mockResolvedValueOnce({ Body: { transformToByteArray: async () => Uint8Array.from([1, 2, 3, 4]) }, ContentLength: 4 }).mockResolvedValue({});
const decodePreview = vi.fn(async () => ({
ok: true as const,
preview: Uint8Array.from([5, 6]),
thumbnail: Uint8Array.from([7]),
previewExtension: "webp" as const,
previewContentType: "image/webp" as const,
}));
const service = createWorkerFilePreviewService(env, {
prisma,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
decodePreview,
})!;
await expect(service.generatePreview(payload)).resolves.toMatchObject({ previewStatus: "ready" });
expect(decodePreview).toHaveBeenCalledWith(Buffer.from([1, 2, 3, 4]), "image/png");
expect(update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ previewStatus: "generating" }) }));
expect(update).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining({ previewStatus: "ready" }) }));
});
test("marks a decoder failure and rethrows so BullMQ can retry", async () => {
const { prisma, update } = buildPrisma("clean");
aws.send.mockResolvedValueOnce({ Body: { transformToByteArray: async () => Uint8Array.from([1, 2, 3, 4]) }, ContentLength: 4 });
const service = createWorkerFilePreviewService(env, {
prisma,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
decodePreview: vi.fn(async () => {
throw new Error("decoder-timeout");
}),
})!;
await expect(service.generatePreview(payload)).rejects.toThrow("decoder-timeout");
expect(update).toHaveBeenCalledWith(expect.objectContaining({ data: { previewStatus: "failed" } }));
});
});
@@ -12,24 +12,25 @@
// text/plain | markdown -> ready (inline render from bytes, no file)
// anything else -> not_supported
//
// The native render libs (sharp, heic-convert, pdf-to-png-converter) are loaded
// via dynamic string import (same pattern as qdrant/sentry in index.ts) so
// typecheck/build stay green before they are installed.
// ACTIVATION: `pnpm add sharp heic-convert pdf-to-png-converter -F @hivegrid/worker`.
// If a lib is missing at runtime the service falls back cleanly to
// previewStatus=failed (no hard crash) and the user can regenerate from detail.
// Native rendering runs in a disposable child process. The child validates
// dimensions/pages/pixels and output sizes; the parent kills it at a hard
// wall-clock limit. A failure is rethrown so BullMQ retries before leaving the
// File row at previewStatus=failed for a user-triggered regeneration.
import { GetObjectCommand, PutObjectCommand, S3Client as AwsS3Client } from "@aws-sdk/client-s3";
import type { EnvRecord } from "@hivegrid/config";
import type { FilePreviewJob } from "@hivegrid/queue";
import { buildFilePreviewKey, buildFileStorageKeys, type FilePreviewExtension } from "@hivegrid/storage";
import { buildFilePreviewKey, buildFileStorageKeys } from "@hivegrid/storage";
import { withTenantRLS } from "../db/tenantRls.js";
import type { FilePreviewServiceResult } from "../handlers.js";
import { FILE_PREVIEW_MAX_SOURCE_BYTES, type FilePreviewDecoderMime, type FilePreviewDecoderSuccess } from "./file-preview-decoder-contract.js";
import { decodeFilePreviewIsolated } from "./file-preview-decoder-isolated.js";
export interface FilePreviewServiceEnv {
STORAGE_ENDPOINT?: string;
STORAGE_REGION?: string;
STORAGE_BUCKET?: string;
STORAGE_ACCESS_KEY_ID?: string;
STORAGE_SECRET_ACCESS_KEY?: string;
STORAGE_SESSION_TOKEN?: string;
@@ -51,6 +52,28 @@ type PreviewLogger = {
interface FilePreviewTx {
$executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
file: {
findFirst(args: {
where: { id: string; tenantId: string };
select: {
id: true;
mimeType: true;
previewStatus: true;
previewStorageKey: true;
scanStatus: true;
sizeBytes: true;
storageKey: true;
thumbnailStorageKey: true;
};
}): Promise<{
id: string;
mimeType: string;
previewStatus: string;
previewStorageKey: string | null;
scanStatus: string;
sizeBytes: number;
storageKey: string;
thumbnailStorageKey: string | null;
} | null>;
update(args: {
where: { id: string };
data: {
@@ -71,8 +94,10 @@ export interface FilePreviewService {
generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult>;
}
const THUMBNAIL_MAX_DIMENSION = 480;
const PREVIEW_MAX_DIMENSION = 2048;
export { FILE_PREVIEW_MAX_SOURCE_BYTES } from "./file-preview-decoder-contract.js";
const FILE_PREVIEW_STORAGE_READ_TIMEOUT_MS = 15_000;
const FILE_PREVIEW_STORAGE_REQUEST_TIMEOUT_MS = 15_000;
const RASTER_IMAGE_TYPES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
const HEIC_TYPES = new Set(["image/heic", "image/heif"]);
@@ -82,6 +107,76 @@ function normalizeMime(mimeType: string): string {
return mimeType.split(";")[0]?.trim().toLowerCase() ?? "";
}
type PreviewObjectBody = {
transformToByteArray?: () => Promise<Uint8Array>;
destroy?: (error?: Error) => void;
[Symbol.asyncIterator]?: () => AsyncIterator<Uint8Array | string>;
};
/** Read an S3 object without letting missing or forged metadata bypass the cap. */
export async function readPreviewBodyWithinLimit(
body: PreviewObjectBody,
contentLength: number | undefined,
maxBytes = FILE_PREVIEW_MAX_SOURCE_BYTES,
timeoutMs = FILE_PREVIEW_STORAGE_READ_TIMEOUT_MS
): Promise<Buffer> {
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) {
throw new Error("Preview source limit must be a positive safe integer");
}
if (contentLength !== undefined && (!Number.isSafeInteger(contentLength) || contentLength < 0)) {
body.destroy?.();
throw new Error("S3 returned an invalid preview source length");
}
if (contentLength !== undefined && contentLength > maxBytes) {
body.destroy?.();
throw new Error(`Preview source exceeds the ${maxBytes}-byte limit`);
}
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) {
throw new Error("Preview source read timeout must be a positive safe integer");
}
const readBody = async (): Promise<Buffer> => {
if (body[Symbol.asyncIterator]) {
const chunks: Buffer[] = [];
let totalBytes = 0;
for await (const value of body as AsyncIterable<Uint8Array | string>) {
const chunk = typeof value === "string" ? Buffer.from(value) : Buffer.from(value.buffer, value.byteOffset, value.byteLength);
totalBytes += chunk.byteLength;
if (totalBytes > maxBytes) {
body.destroy?.();
throw new Error(`Preview source exceeds the ${maxBytes}-byte limit`);
}
chunks.push(chunk);
}
return Buffer.concat(chunks, totalBytes);
}
if (body.transformToByteArray) {
const bytes = await body.transformToByteArray();
if (bytes.byteLength > maxBytes) {
body.destroy?.();
throw new Error(`Preview source exceeds the ${maxBytes}-byte limit`);
}
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
}
throw new Error("S3 returned an unsupported preview source body");
};
let timeout: NodeJS.Timeout | undefined;
const timeoutFailure = new Promise<never>((_resolve, reject) => {
timeout = setTimeout(() => {
const error = new Error(`Preview source read exceeded the ${timeoutMs}ms limit`);
body.destroy?.(error);
reject(error);
}, timeoutMs);
});
try {
return await Promise.race([readBody(), timeoutFailure]);
} finally {
if (timeout) clearTimeout(timeout);
}
}
/**
* Builds the preview service from STORAGE_* env. Returns undefined when the S3
* credentials are missing (the bootstrap wiring then skips it and the handler
@@ -89,13 +184,19 @@ function normalizeMime(mimeType: string): string {
*/
export function createWorkerFilePreviewService(
env: EnvRecord & FilePreviewServiceEnv,
options: { prisma: FilePreviewPrisma; logger: PreviewLogger },
options: {
prisma: FilePreviewPrisma;
logger: PreviewLogger;
decodePreview?: (input: Buffer, mime: FilePreviewDecoderMime) => Promise<FilePreviewDecoderSuccess>;
}
): FilePreviewService | undefined {
if (!env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
const configuredBucket = env.STORAGE_BUCKET?.trim();
if (!configuredBucket || !env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
return undefined;
}
const { prisma, logger } = options;
const decodePreview = options.decodePreview ?? decodeFilePreviewIsolated;
const s3 = new AwsS3Client({
endpoint: env.STORAGE_ENDPOINT,
region: env.STORAGE_REGION,
@@ -107,35 +208,82 @@ export function createWorkerFilePreviewService(
},
});
async function loadOriginal(bucket: string, key: string): Promise<Buffer> {
const result = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
async function loadOriginal(bucket: string, key: string, expectedBytes: number): Promise<Buffer> {
const result = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }), {
abortSignal: AbortSignal.timeout(FILE_PREVIEW_STORAGE_REQUEST_TIMEOUT_MS),
});
const body = result.Body;
if (!body) {
throw new Error(`S3 GET ${bucket}/${key} returned an empty body`);
}
// The AWS SDK exposes transformToByteArray() on the Node stream body.
const bytes = await (body as { transformToByteArray(): Promise<Uint8Array> }).transformToByteArray();
return Buffer.from(bytes);
const bytes = await readPreviewBodyWithinLimit(body as PreviewObjectBody, result.ContentLength);
if (bytes.byteLength !== expectedBytes) {
throw new Error("Preview source byte length does not match its File record");
}
return bytes;
}
async function putRendered(bucket: string, key: string, body: Buffer, contentType: string): Promise<void> {
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, ContentType: contentType }));
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, ContentType: contentType }), {
abortSignal: AbortSignal.timeout(FILE_PREVIEW_STORAGE_REQUEST_TIMEOUT_MS),
});
}
async function writeStatus(
tenantId: string,
fileId: string,
data: { previewStatus: string; previewStorageKey?: string | null; thumbnailStorageKey?: string | null },
data: { previewStatus: string; previewStorageKey?: string | null; thumbnailStorageKey?: string | null }
): Promise<void> {
// Set the tenant GUC so the write satisfies the File RLS policy once it is
// activated (P5 cutover). Preview jobs always carry a tenantId.
await withTenantRLS(prisma, tenantId, (tx) => tx.file.update({ where: { id: fileId }, data }));
}
async function claimCleanSource(payload: FilePreviewJob) {
return withTenantRLS(prisma, payload.tenantId, async (tx) => {
const source = await tx.file.findFirst({
where: { id: payload.fileId, tenantId: payload.tenantId },
select: {
id: true,
mimeType: true,
previewStatus: true,
previewStorageKey: true,
scanStatus: true,
sizeBytes: true,
storageKey: true,
thumbnailStorageKey: true,
},
});
if (!source || source.scanStatus !== "clean") {
throw new Error("File preview is blocked until its malware scan is clean");
}
if (source.storageKey !== payload.storageKey) {
throw new Error("File preview payload does not match the persisted storage key");
}
if (!Number.isSafeInteger(source.sizeBytes) || source.sizeBytes < 0 || source.sizeBytes > FILE_PREVIEW_MAX_SOURCE_BYTES) {
throw new Error(`Preview source exceeds the ${FILE_PREVIEW_MAX_SOURCE_BYTES}-byte limit`);
}
return source;
});
}
return {
async generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult> {
const mime = normalizeMime(payload.mimeType);
const keys = buildFileStorageKeys(payload.tenantId, payload.fileId);
assertPreviewSource(payload, configuredBucket, keys.original);
const source = await claimCleanSource(payload);
const mime = normalizeMime(source.mimeType);
if (source.previewStatus === "ready") {
return {
previewStatus: "ready",
...(source.previewStorageKey ? { previewStorageKey: source.previewStorageKey } : {}),
...(source.thumbnailStorageKey ? { thumbnailStorageKey: source.thumbnailStorageKey } : {}),
};
}
if (source.previewStatus === "not_supported") {
return { previewStatus: "not_supported" };
}
// Plain text / markdown: detail view renders inline from bytes, no
// rendered preview file needed.
@@ -156,35 +304,16 @@ export function createWorkerFilePreviewService(
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "generating" });
try {
const original = await loadOriginal(payload.bucket, payload.storageKey);
let previewBuffer: Buffer;
let previewExt: FilePreviewExtension;
let previewContentType: string;
const original = await loadOriginal(configuredBucket, payload.storageKey, source.sizeBytes);
const rendered = await decodePreview(original, mime as FilePreviewDecoderMime);
const previewBuffer = Buffer.from(rendered.preview);
const thumbnail = Buffer.from(rendered.thumbnail);
if (isHeic) {
// HEIC -> JPG (mandatory for iPhone field-worker photos).
const jpg = await heicToJpeg(original);
previewBuffer = await rasterToFormat(jpg, "jpeg", PREVIEW_MAX_DIMENSION);
previewExt = "jpg";
previewContentType = "image/jpeg";
} else if (isPdf) {
const png = await pdfFirstPageToPng(original);
previewBuffer = await rasterToFormat(png, "png", PREVIEW_MAX_DIMENSION);
previewExt = "png";
previewContentType = "image/png";
} else {
previewBuffer = await rasterToFormat(original, "webp", PREVIEW_MAX_DIMENSION);
previewExt = "webp";
previewContentType = "image/webp";
}
const thumbnail = await rasterToFormat(previewBuffer, "webp", THUMBNAIL_MAX_DIMENSION);
const previewKey = buildFilePreviewKey(payload.tenantId, payload.fileId, previewExt);
const previewKey = buildFilePreviewKey(payload.tenantId, payload.fileId, rendered.previewExtension);
const thumbnailKey = keys.thumbnail;
await putRendered(payload.bucket, previewKey, previewBuffer, previewContentType);
await putRendered(payload.bucket, thumbnailKey, thumbnail, "image/webp");
await putRendered(configuredBucket, previewKey, previewBuffer, rendered.previewContentType);
await putRendered(configuredBucket, thumbnailKey, thumbnail, "image/webp");
await writeStatus(payload.tenantId, payload.fileId, {
previewStatus: "ready",
@@ -197,61 +326,19 @@ export function createWorkerFilePreviewService(
const reason = error instanceof Error ? error.message : String(error);
logger.warn({ fileId: payload.fileId, tenantId: payload.tenantId, mime }, "file preview generation failed");
await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "failed" });
return { previewStatus: "failed", reason };
throw new Error(reason, { cause: error });
}
},
};
}
// --- dynamic native-lib loaders (kept untyped so build stays green pre-install) ---
type SharpFactory = (input: Buffer) => {
rotate(): ReturnType<SharpFactory>;
resize(opts: { width: number; height: number; fit: "inside"; withoutEnlargement: boolean }): ReturnType<SharpFactory>;
jpeg(opts: { quality: number }): ReturnType<SharpFactory>;
png(): ReturnType<SharpFactory>;
webp(opts: { quality: number }): ReturnType<SharpFactory>;
toBuffer(): Promise<Buffer>;
};
let sharpModule: SharpFactory | undefined;
async function loadSharp(): Promise<SharpFactory> {
if (sharpModule) return sharpModule;
const moduleName = "sharp";
const loaded = (await import(/* @vite-ignore */ moduleName)) as { default?: SharpFactory } & SharpFactory;
sharpModule = (loaded.default ?? loaded) as SharpFactory;
return sharpModule;
}
async function rasterToFormat(input: Buffer, format: "jpeg" | "png" | "webp", maxDimension: number): Promise<Buffer> {
const sharp = await loadSharp();
const pipeline = sharp(input)
.rotate()
.resize({ width: maxDimension, height: maxDimension, fit: "inside", withoutEnlargement: true });
if (format === "jpeg") return pipeline.jpeg({ quality: 82 }).toBuffer();
if (format === "png") return pipeline.png().toBuffer();
return pipeline.webp({ quality: 80 }).toBuffer();
}
async function heicToJpeg(input: Buffer): Promise<Buffer> {
const moduleName = "heic-convert";
const loaded = (await import(/* @vite-ignore */ moduleName)) as
| { default: (opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer> }
| ((opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer>);
const convert = typeof loaded === "function" ? loaded : loaded.default;
const out = await convert({ buffer: input, format: "JPEG", quality: 0.85 });
return Buffer.from(out as ArrayBuffer);
}
async function pdfFirstPageToPng(input: Buffer): Promise<Buffer> {
const moduleName = "pdf-to-png-converter";
const loaded = (await import(/* @vite-ignore */ moduleName)) as {
pdfToPng: (input: Buffer, opts: Record<string, unknown>) => Promise<Array<{ content: Buffer }>>;
};
const pages = await loaded.pdfToPng(input, { pagesToProcess: [1], viewportScale: 2.0 });
const first = pages[0];
if (!first) {
throw new Error("PDF rendered no pages");
export function assertPreviewSource(payload: FilePreviewJob, configuredBucket: string, expectedStorageKey?: string): void {
const bucket = configuredBucket.trim();
if (!bucket || payload.bucket !== bucket) {
throw new Error("File preview source bucket does not match worker storage configuration");
}
const expected = expectedStorageKey ?? buildFileStorageKeys(payload.tenantId, payload.fileId).original;
if (payload.storageKey !== expected) {
throw new Error("File preview source key does not match the canonical tenant file path");
}
return Buffer.from(first.content);
}
@@ -1,9 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import {
createWorkerFileScanStatusWriter,
type FileScanStatusPrisma,
} from "./file-scan-status.js";
import { createWorkerFileScanStatusWriter, type FileScanStatusPrisma } from "./file-scan-status.js";
/**
* Build a Prisma stub whose `file.updateMany` reports `count` matched rows.
@@ -12,14 +9,23 @@ import {
*/
function createPrismaStub(count: number) {
const updateMany = vi.fn(async () => ({ count }));
const findFirst = vi.fn(
async (): Promise<{ id: string; createdByUserId: string | null; links: Array<{ id: string; entityId: string; createdAt: Date }> } | null> => null
);
const executeRaw = vi.fn(async () => 1);
const tx = { $executeRaw: executeRaw, file: { updateMany } };
const outboxCreate = vi.fn(async ({ data }: { data: Record<string, unknown> }) => ({ id: "outbox-1", ...data }));
const outboxFindUnique = vi.fn(async () => null);
const tx = {
$executeRaw: executeRaw,
file: { updateMany, findFirst },
outboxEvent: { create: outboxCreate, findUnique: outboxFindUnique },
};
const transaction = vi.fn(async (fn: (t: typeof tx) => Promise<unknown>) => fn(tx));
const prisma = {
$transaction: transaction,
file: { updateMany },
} as unknown as FileScanStatusPrisma;
return { prisma, updateMany, executeRaw, transaction };
return { prisma, updateMany, findFirst, outboxCreate, executeRaw, transaction };
}
function createLogger() {
@@ -32,18 +38,17 @@ describe("createWorkerFileScanStatusWriter — tenant-scoped (RLS) path", () =>
const logger = createLogger();
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await writer.recordVerdict({ fileId: "file-1", tenantId: "tenant-a", verdict: "infected" });
await expect(
writer.recordVerdict({ fileId: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a", verdict: "infected" })
).resolves.toBe(true);
expect(transaction).toHaveBeenCalledTimes(1);
expect(executeRaw).toHaveBeenCalledTimes(1);
expect(updateMany).toHaveBeenCalledWith({
where: { id: "file-1", tenantId: "tenant-a" },
where: { id: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a" },
data: { scanStatus: "infected" },
});
expect(logger.info).toHaveBeenCalledWith(
{ fileId: "file-1", verdict: "infected" },
"scan verdict written to File.scanStatus"
);
expect(logger.info).toHaveBeenCalledWith({ fileId: "file-1", verdict: "infected" }, "scan verdict written to File.scanStatus");
expect(logger.warn).not.toHaveBeenCalled();
});
@@ -53,16 +58,54 @@ describe("createWorkerFileScanStatusWriter — tenant-scoped (RLS) path", () =>
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await expect(
writer.recordVerdict({ fileId: "missing", tenantId: "tenant-a", verdict: "clean" })
).resolves.toBeUndefined();
writer.recordVerdict({ fileId: "missing", storageKey: "tenants/tenant-a/files/missing/original", tenantId: "tenant-a", verdict: "clean" })
).resolves.toBe(false);
expect(updateMany).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledWith(
{ fileId: "missing", tenantId: "tenant-a", verdict: "clean" },
"scan verdict write-back matched no File row"
);
expect(logger.warn).toHaveBeenCalledWith({ fileId: "missing", tenantId: "tenant-a", verdict: "clean" }, "scan verdict write-back matched no File row");
expect(logger.info).not.toHaveBeenCalled();
});
it("publishes one idempotent portal document-ready event per internal project link after a clean verdict", async () => {
const { prisma, findFirst, outboxCreate } = createPrismaStub(1);
const linkCreatedAt = new Date("2026-07-16T10:00:00.000Z");
findFirst.mockResolvedValue({
id: "file-1",
createdByUserId: "user-1",
links: [{ id: "link-1", entityId: "project-1", createdAt: linkCreatedAt }],
});
const writer = createWorkerFileScanStatusWriter({ prisma, logger: createLogger() });
await expect(
writer.recordVerdict({ fileId: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a", verdict: "clean" })
).resolves.toBe(true);
expect(findFirst).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({ tenantId: "tenant-a", createdByPortalUserId: null, source: { not: "customer_portal" } }),
})
);
expect(outboxCreate).toHaveBeenCalledWith({
data: expect.objectContaining({
eventType: "workflow.trigger.event",
dispatchKey: `workflow-trigger:portal-document-ready:link-1:${linkCreatedAt.getTime()}`,
payload: expect.objectContaining({
event: "portal.document_ready",
data: { projectId: "project-1", fileId: "file-1", fileEntityLinkId: "link-1" },
}),
}),
});
});
it("does not publish a document-ready event for infected verdicts", async () => {
const { prisma, findFirst, outboxCreate } = createPrismaStub(1);
const writer = createWorkerFileScanStatusWriter({ prisma, logger: createLogger() });
await writer.recordVerdict({ fileId: "file-1", storageKey: "tenants/tenant-a/files/file-1/original", tenantId: "tenant-a", verdict: "infected" });
expect(findFirst).not.toHaveBeenCalled();
expect(outboxCreate).not.toHaveBeenCalled();
});
});
describe("createWorkerFileScanStatusWriter — legacy direct path (no tenantId)", () => {
@@ -71,17 +114,14 @@ describe("createWorkerFileScanStatusWriter — legacy direct path (no tenantId)"
const logger = createLogger();
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await writer.recordVerdict({ fileId: "file-2", verdict: "clean" });
await expect(writer.recordVerdict({ fileId: "file-2", storageKey: "legacy/file-2", verdict: "clean" })).resolves.toBe(true);
expect(transaction).not.toHaveBeenCalled();
expect(updateMany).toHaveBeenCalledWith({
where: { id: "file-2" },
where: { id: "file-2", storageKey: "legacy/file-2" },
data: { scanStatus: "clean" },
});
expect(logger.info).toHaveBeenCalledWith(
{ fileId: "file-2", verdict: "clean" },
"scan verdict written to File.scanStatus"
);
expect(logger.info).toHaveBeenCalledWith({ fileId: "file-2", verdict: "clean" }, "scan verdict written to File.scanStatus");
});
it("warns when the direct write matches no row", async () => {
@@ -89,12 +129,9 @@ describe("createWorkerFileScanStatusWriter — legacy direct path (no tenantId)"
const logger = createLogger();
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
await writer.recordVerdict({ fileId: "gone", verdict: "infected" });
await expect(writer.recordVerdict({ fileId: "gone", storageKey: "legacy/gone", verdict: "infected" })).resolves.toBe(false);
expect(transaction).not.toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalledWith(
{ fileId: "gone", tenantId: undefined, verdict: "infected" },
"scan verdict write-back matched no File row"
);
expect(logger.warn).toHaveBeenCalledWith({ fileId: "gone", tenantId: undefined, verdict: "infected" }, "scan verdict write-back matched no File row");
});
});
@@ -8,8 +8,11 @@
//
// Mirrors createWorkerFilePreviewService: a Prisma seam is injected at the
// general-group bootstrap (index.ts) so handlers.ts stays free of direct DB
// imports. When the seam is unset the handler simply skips the write-back
// (legacy import-path scans carry no fileId either).
// imports. Legacy import-path scans carry no fileId; a real File scan fails and
// retries if this seam is unavailable.
import { publishOutboxEvent, type OutboxPublisherTransaction } from "@hivegrid/db/outbox";
import { buildPortalDocumentReadyOutboxEvent } from "@hivegrid/queue";
import { withTenantRLS } from "../db/tenantRls.js";
@@ -25,55 +28,95 @@ type ScanStatusLogger = {
interface FileScanStatusTx {
$executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
file: {
updateMany(args: {
where: { id: string; tenantId?: string };
data: { scanStatus: string };
}): Promise<{ count: number }>;
updateMany(args: { where: { id: string; storageKey: string; tenantId?: string }; data: { scanStatus: string } }): Promise<{ count: number }>;
findFirst(args: unknown): Promise<{
id: string;
createdByUserId: string | null;
links: Array<{ id: string; entityId: string; createdAt: Date }>;
} | null>;
};
outboxEvent: OutboxPublisherTransaction["outboxEvent"];
}
/** Minimal Prisma surface the write-back needs (direct write + tenant tx). */
export interface FileScanStatusPrisma {
$transaction<T>(fn: (tx: FileScanStatusTx) => Promise<T>): Promise<T>;
file: {
updateMany(args: {
where: { id: string; tenantId?: string };
data: { scanStatus: string };
}): Promise<{ count: number }>;
updateMany(args: { where: { id: string; storageKey: string; tenantId?: string }; data: { scanStatus: string } }): Promise<{ count: number }>;
};
}
export type FileScanVerdict = "clean" | "infected";
export interface FileScanStatusWriter {
recordVerdict(input: { fileId: string; tenantId?: string; verdict: FileScanVerdict }): Promise<void>;
recordVerdict(input: { fileId: string; storageKey: string; tenantId?: string; verdict: FileScanVerdict }): Promise<boolean>;
}
export function createWorkerFileScanStatusWriter(options: {
prisma: FileScanStatusPrisma;
logger: ScanStatusLogger;
}): FileScanStatusWriter {
export function createWorkerFileScanStatusWriter(options: { prisma: FileScanStatusPrisma; logger: ScanStatusLogger }): FileScanStatusWriter {
const { prisma, logger } = options;
return {
async recordVerdict({ fileId, tenantId, verdict }) {
async recordVerdict({ fileId, storageKey, tenantId, verdict }) {
// updateMany (not update) so a tenant-scoped/RLS-filtered or already
// deleted row is a logged no-op instead of a throw that dead-letters
// the job after the scan already ran.
const write = (db: Pick<FileScanStatusTx, "file">) =>
const write = (db: {
file: {
updateMany(args: { where: { id: string; storageKey: string; tenantId?: string }; data: { scanStatus: string } }): Promise<{ count: number }>;
};
}) =>
db.file.updateMany({
where: tenantId ? { id: fileId, tenantId } : { id: fileId },
where: tenantId ? { id: fileId, storageKey, tenantId } : { id: fileId, storageKey },
data: { scanStatus: verdict },
});
// Set the tenant GUC so the write satisfies the File RLS policy once it is
// activated (P5 cutover). Legacy import-path scans carry no tenantId and
// fall back to the direct write (no RLS context to set).
const result = tenantId
? await withTenantRLS(prisma, tenantId, (tx) => write(tx))
? await withTenantRLS(prisma, tenantId, async (tx) => {
const scopedResult = await write(tx);
if (verdict === "clean" && scopedResult.count > 0) {
const readyFile = await tx.file.findFirst({
where: {
id: fileId,
storageKey,
tenantId,
archivedAt: null,
scanStatus: "clean",
createdByPortalUserId: null,
source: { not: "customer_portal" },
},
select: {
id: true,
createdByUserId: true,
links: {
where: { tenantId, entityType: "project", createdByPortalUserId: null },
select: { id: true, entityId: true, createdAt: true },
},
},
});
for (const link of readyFile?.links ?? []) {
await publishOutboxEvent(
tx as unknown as OutboxPublisherTransaction,
buildPortalDocumentReadyOutboxEvent({
tenantId,
projectId: link.entityId,
fileId: readyFile!.id,
fileEntityLinkId: link.id,
linkVersion: link.createdAt,
userId: readyFile!.createdByUserId,
})
);
}
}
return scopedResult;
})
: await write(prisma);
if (result.count === 0) {
logger.warn({ fileId, tenantId, verdict }, "scan verdict write-back matched no File row");
return false;
} else {
logger.info({ fileId, verdict }, "scan verdict written to File.scanStatus");
return true;
}
},
};
@@ -0,0 +1,91 @@
import { describe, expect, it, vi } from "vitest";
import { verifyMarketingToken } from "@hivegrid/foundation/marketing-token";
import { runStartDoubleOptInFormAction, type StartDoubleOptInPrisma } from "./form-submission-actions.js";
type PublishedOptInEvent = Parameters<Parameters<typeof runStartDoubleOptInFormAction>[0]["publishOutboxEvent"]>[0];
const env = {
APP_URL: "https://app.example.test",
MARKETING_TOKEN_SECRET: "x".repeat(40),
};
function prismaStub(overrides: Partial<StartDoubleOptInPrisma> = {}): StartDoubleOptInPrisma {
return {
kontakt: {
findFirst: vi.fn(async () => ({ id: "kontakt_1", primaerEmail: "Ada@Example.com", vorname: "Ada", nachname: "Lovelace" })),
},
kontaktMarketingConsent: {
upsert: vi.fn(async () => ({})),
},
...overrides,
};
}
describe("runStartDoubleOptInFormAction", () => {
it("upserts single consent and queues the double-opt-in mail", async () => {
const prisma = prismaStub();
const publishOutboxEvent = vi.fn(async (event: PublishedOptInEvent) => {
void event;
});
const result = await runStartDoubleOptInFormAction({
prisma,
publishOutboxEvent,
tenantId: "tenant_1",
formId: "form_1",
submissionId: "sub_1",
actionId: "action_1",
mapped: { email: "ada@example.com" },
config: {},
env,
now: () => new Date("2026-07-09T10:00:00.000Z"),
});
expect(result).toEqual({ status: "started", kontaktId: "kontakt_1", email: "ada@example.com" });
expect(prisma.kontaktMarketingConsent.upsert).toHaveBeenCalledWith(
expect.objectContaining({
where: { tenantId_kontaktId: { tenantId: "tenant_1", kontaktId: "kontakt_1" } },
create: expect.objectContaining({ status: "single", optInSource: "form:form_1" }),
}),
);
const event = vi.mocked(publishOutboxEvent).mock.calls[0]?.[0];
expect(event).toMatchObject({
tenantId: "tenant_1",
type: "mail.transactional.send",
dispatchKey: "form-action-optin:sub_1:action_1",
payload: expect.objectContaining({
to: ["ada@example.com"],
subject: "Bitte bestätige deine Anmeldung",
}),
});
const link = String((event?.payload.html as string).match(/https:\/\/app\.example\.test\/optin\/([^"]+)/u)?.[1] ?? "");
expect(verifyMarketingToken(decodeURIComponent(link), { env, expectedPurpose: "optin" })).toMatchObject({
tenantId: "tenant_1",
kontaktId: "kontakt_1",
});
});
it("skips when no matching contact exists", async () => {
const prisma = prismaStub({
kontakt: {
findFirst: vi.fn(async () => null),
},
});
const publishOutboxEvent = vi.fn(async (event: PublishedOptInEvent) => {
void event;
});
await expect(runStartDoubleOptInFormAction({
prisma,
publishOutboxEvent,
tenantId: "tenant_1",
formId: "form_1",
submissionId: "sub_1",
actionId: "action_1",
mapped: { email: "nobody@example.com" },
config: {},
env,
})).resolves.toEqual({ status: "skipped", reason: "no_contact" });
expect(publishOutboxEvent).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,152 @@
import { signMarketingToken } from "@hivegrid/foundation/marketing-token";
import { renderSystemMessageEmail } from "@hivegrid/mail";
export type FormActionJsonValue = string | number | boolean | null | FormActionJsonValue[] | { [key: string]: FormActionJsonValue };
export interface StartDoubleOptInPrisma {
readonly kontakt: {
findFirst(args: Record<string, unknown>): Promise<{ id: string; primaerEmail: string | null; vorname: string | null; nachname: string | null } | null>;
};
readonly kontaktMarketingConsent: {
upsert(args: Record<string, unknown>): Promise<unknown>;
};
}
export interface StartDoubleOptInInput {
readonly prisma: StartDoubleOptInPrisma;
readonly publishOutboxEvent: (event: {
readonly tenantId: string;
readonly type: "mail.transactional.send";
readonly payload: Record<string, unknown>;
readonly dispatchKey: string;
}) => Promise<unknown>;
readonly tenantId: string;
readonly formId: string;
readonly submissionId: string;
readonly actionId: string;
readonly mapped: Record<string, unknown>;
readonly config: Record<string, unknown>;
readonly env?: Record<string, string | undefined>;
readonly now?: () => Date;
}
export type StartDoubleOptInResult =
| { readonly status: "started"; readonly kontaktId: string; readonly email: string }
| { readonly status: "skipped"; readonly reason: "no_contact" | "no_email" | "missing_app_url" };
function stringValue(values: Record<string, unknown>, keys: readonly string[]): string | null {
for (const key of keys) {
const value = values[key];
if (typeof value === "string" && value.trim()) return value.trim();
}
return null;
}
function readBaseUrl(config: Record<string, unknown>, env: Record<string, string | undefined>): string | null {
const configured = typeof config.confirmBaseUrl === "string" ? config.confirmBaseUrl.trim() : "";
const fallback = env.APP_URL?.trim() ?? "";
const raw = (configured || fallback).replace(/\/+$/u, "");
if (!raw) return null;
try {
const url = new URL(raw);
return url.origin + url.pathname.replace(/\/+$/u, "");
} catch {
return null;
}
}
function renderOptInMail(input: { link: string; name: string | null; subject?: string | null }) {
const subject = input.subject?.trim() || "Bitte bestätige deine Anmeldung";
const rendered = renderSystemMessageEmail({
locale: "de",
subject,
previewText: "Bestätige deine E-Mail-Adresse mit einem Klick.",
eyebrow: "E-Mail-Anmeldung",
title: input.name ? `${input.name}, bitte bestätige deine Anmeldung` : "Bitte bestätige deine Anmeldung",
intro: "Ein letzter Klick fehlt noch. Danach dürfen wir dir die gewünschten Informationen per E-Mail senden.",
actionUrl: input.link,
actionLabel: "Anmeldung bestätigen",
notice: "Wenn du dich nicht angemeldet hast, kannst du diese E-Mail einfach ignorieren.",
});
return { subject, text: rendered.text, html: rendered.html };
}
export async function runStartDoubleOptInFormAction(input: StartDoubleOptInInput): Promise<StartDoubleOptInResult> {
const configuredKontaktId = typeof input.config.kontaktId === "string" && input.config.kontaktId.trim() ? input.config.kontaktId.trim() : null;
const email = stringValue(input.mapped, ["email", "email_address", "primaerEmail"])?.toLowerCase() ?? null;
const phone = stringValue(input.mapped, ["phone", "telefon", "primaerTelefon"]);
if (!configuredKontaktId && !email && !phone) return { status: "skipped", reason: "no_contact" };
const contact = await input.prisma.kontakt.findFirst({
where: configuredKontaktId
? { id: configuredKontaktId, tenantId: input.tenantId, archivedAt: null }
: {
tenantId: input.tenantId,
archivedAt: null,
OR: [...(email ? [{ primaerEmail: email }] : []), ...(phone ? [{ primaerTelefon: phone }] : [])],
},
select: { id: true, primaerEmail: true, vorname: true, nachname: true },
});
if (!contact) return { status: "skipped", reason: "no_contact" };
const recipient = contact.primaerEmail?.trim().toLowerCase() || email;
if (!recipient) return { status: "skipped", reason: "no_email" };
const baseUrl = readBaseUrl(input.config, input.env ?? process.env);
if (!baseUrl) return { status: "skipped", reason: "missing_app_url" };
const token = signMarketingToken({
purpose: "optin",
tenantId: input.tenantId,
kontaktId: contact.id,
now: input.now,
env: input.env ?? process.env,
});
const now = input.now ? input.now() : new Date();
await input.prisma.kontaktMarketingConsent.upsert({
where: { tenantId_kontaktId: { tenantId: input.tenantId, kontaktId: contact.id } },
create: {
tenantId: input.tenantId,
kontaktId: contact.id,
status: "single",
optInSource: `form:${input.formId}`,
optInAt: now,
doubleOptInToken: token,
},
update: {
status: "single",
optInSource: `form:${input.formId}`,
optInAt: now,
doubleOptInToken: token,
unsubscribedAt: null,
},
});
const link = `${baseUrl}/optin/${encodeURIComponent(token)}`;
const displayName = [contact.vorname, contact.nachname].filter(Boolean).join(" ") || null;
const mail = renderOptInMail({
link,
name: displayName,
subject: typeof input.config.subject === "string" ? input.config.subject : null,
});
await input.publishOutboxEvent({
tenantId: input.tenantId,
type: "mail.transactional.send",
payload: {
tenantId: input.tenantId,
to: [recipient],
subject: mail.subject,
text: mail.text,
html: mail.html,
priorityOverride: "normal",
metadata: {
source: "form-double-opt-in",
formId: input.formId,
submissionId: input.submissionId,
actionId: input.actionId,
kontaktId: contact.id,
},
},
dispatchKey: `form-action-optin:${input.submissionId}:${input.actionId}`,
});
return { status: "started", kontaktId: contact.id, email: recipient };
}
@@ -0,0 +1,55 @@
import { describe, expect, it, vi } from "vitest";
import {
INTEGRATIONS_HEALTH_SWEEP_EVERY_MS,
INTEGRATIONS_HEALTH_SWEEP_JOB_ID,
INTEGRATIONS_TOKEN_REFRESH_EVERY_MS,
INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES,
KWP_PROJECT_MAINTENANCE_EVERY_MS,
KWP_PROJECT_MAINTENANCE_JOB_ID,
registerIntegrationsHealthSweepCron,
registerIntegrationsTokenRefreshCron,
registerKwpProjectMaintenanceCron,
} from "./integrations-maintenance-cron.js";
describe("integrations maintenance cron registration", () => {
it("sweeps token refresh every minute while retaining the generic ten-minute payload window", async () => {
const upsertJobScheduler = vi.fn(async () => undefined);
const queue = { upsertJobScheduler };
await registerIntegrationsTokenRefreshCron(queue);
await registerIntegrationsHealthSweepCron(queue);
await registerKwpProjectMaintenanceCron(queue);
expect(INTEGRATIONS_TOKEN_REFRESH_EVERY_MS).toBe(60_000);
expect(INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES).toBe(10);
expect(upsertJobScheduler).toHaveBeenCalledWith(
INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
{ every: 60_000 },
{
name: INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
data: {
kind: "cleanup.run",
task: INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
olderThanMinutes: 10,
},
},
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
INTEGRATIONS_HEALTH_SWEEP_JOB_ID,
{ every: INTEGRATIONS_HEALTH_SWEEP_EVERY_MS },
{
name: INTEGRATIONS_HEALTH_SWEEP_JOB_ID,
data: { kind: "cleanup.run", task: INTEGRATIONS_HEALTH_SWEEP_JOB_ID },
},
);
expect(upsertJobScheduler).toHaveBeenCalledWith(
KWP_PROJECT_MAINTENANCE_JOB_ID,
{ every: KWP_PROJECT_MAINTENANCE_EVERY_MS },
{
name: KWP_PROJECT_MAINTENANCE_JOB_ID,
data: { kind: "cleanup.run", task: KWP_PROJECT_MAINTENANCE_JOB_ID },
},
);
});
});
@@ -5,10 +5,14 @@
export const INTEGRATIONS_TOKEN_REFRESH_JOB_ID = "integrations-token-refresh" as const;
export const INTEGRATIONS_HEALTH_SWEEP_JOB_ID = "integrations-health-sweep" as const;
export const KWP_PROJECT_MAINTENANCE_JOB_ID = "kwp-project-maintenance" as const;
// Token refresh runs often (tokens expire); health sweep less often.
export const INTEGRATIONS_TOKEN_REFRESH_EVERY_MS = 5 * 60 * 1000;
// KWP access tokens live for only five minutes. Sweep every minute so the
// KWP-specific two-minute refresh window has a real safety margin while the
// generic Apps-Engine refresh window remains unchanged.
export const INTEGRATIONS_TOKEN_REFRESH_EVERY_MS = 60 * 1000;
export const INTEGRATIONS_HEALTH_SWEEP_EVERY_MS = 15 * 60 * 1000;
export const KWP_PROJECT_MAINTENANCE_EVERY_MS = 15 * 60 * 1000;
/** Refresh tokens expiring within this window. Default 10 minutes. */
export const INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES = 10;
@@ -32,3 +36,11 @@ export async function registerIntegrationsHealthSweepCron(queue: SchedulerQueue)
{ name: INTEGRATIONS_HEALTH_SWEEP_JOB_ID, data: { kind: "cleanup.run", task: INTEGRATIONS_HEALTH_SWEEP_JOB_ID } },
);
}
export async function registerKwpProjectMaintenanceCron(queue: SchedulerQueue): Promise<void> {
await queue.upsertJobScheduler(
KWP_PROJECT_MAINTENANCE_JOB_ID,
{ every: KWP_PROJECT_MAINTENANCE_EVERY_MS },
{ name: KWP_PROJECT_MAINTENANCE_JOB_ID, data: { kind: "cleanup.run", task: KWP_PROJECT_MAINTENANCE_JOB_ID } },
);
}
@@ -0,0 +1,53 @@
import { describe, expect, it, vi } from "vitest";
import { runAllTenantKwpProjectMaintenance } from "./kwp-project-maintenance.js";
describe("KWP project maintenance", () => {
it("only schedules targeted provider work with stable cadence ids", async () => {
const add = vi.fn(async (_name: string, _payload: unknown, _options?: { jobId?: string }) => ({ id: "job" }));
const queue = { add, close: vi.fn(async () => undefined) };
const prisma = {
tenant: {
findMany: vi.fn(async (args: { cursor?: { id: string } }) =>
args.cursor ? [] : [{ id: "tenant_1" }],
),
},
appEntityBinding: {
findMany: vi.fn(async () => [{ id: "binding_1", dataConnectionId: "connection_1" }]),
},
dataConnection: {
findMany: vi.fn(async () => [{ id: "connection_1" }]),
},
};
const result = await runAllTenantKwpProjectMaintenance({
prisma,
queue,
now: new Date("2026-07-23T12:00:00.000Z"),
});
expect(result).toEqual({ inspectedTenants: 1, projectJobs: 1, catalogJobs: 1, contractJobs: 1, failures: 0 });
expect(add).toHaveBeenCalledTimes(3);
expect(add.mock.calls.map(([jobName]) => jobName)).toEqual([
"run-kwp-import",
"run-kwp-import",
"run-kwp-import",
]);
expect(add.mock.calls[0]?.[1]).toMatchObject({
kind: "kwp.project-sync",
tenantId: "tenant_1",
bindingId: "binding_1",
});
expect(add.mock.calls[1]?.[1]).toMatchObject({
kind: "kwp.project-catalog-sync",
tenantId: "tenant_1",
dataConnectionId: "connection_1",
});
expect(add.mock.calls[2]?.[1]).toMatchObject({
kind: "kwp.contract-sentinel",
tenantId: "tenant_1",
dataConnectionId: "connection_1",
});
expect(add.mock.calls[0]?.[2]?.jobId).toContain("kwp-project-sync%3Atenant_1%3Abinding_1%3A");
});
});
@@ -0,0 +1,152 @@
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
import {
buildKwpContractSentinelJob,
buildKwpProjectCatalogSyncJob,
buildKwpProjectSyncJob,
enqueueJob,
type TypedQueue,
queueNames,
} from "@hivegrid/queue";
import { runActiveTenantSweep, type ActiveTenantReader } from "./active-tenant-sweep.js";
const PROJECT_INTERVAL_MS = 15 * 60_000;
const CONTRACT_INTERVAL_MS = 24 * 60 * 60_000;
const BATCH_SIZE = 100;
type KwpMaintenancePrisma = ActiveTenantReader & {
appEntityBinding: {
findMany(args: {
where: {
tenantId: string;
externalEntityType: "kwp_project";
localEntityType: "project";
state: { in: readonly ["active", "missing"] };
OR: readonly [
{ lastSyncedAt: null },
{ lastSyncedAt: { lte: Date } },
];
};
select: { id: true; dataConnectionId: true };
orderBy: { id: "asc" };
take: number;
}): Promise<Array<{ id: string; dataConnectionId: string }>>;
};
dataConnection: {
findMany(args: {
where: {
tenantId: string;
appId: "kwp";
archivedAt: null;
connectionStatus: { in: readonly ["active", "degraded"] };
};
select: { id: true };
orderBy: { id: "asc" };
take: number;
}): Promise<Array<{ id: string }>>;
};
};
export type KwpMaintenanceResult = {
inspectedTenants: number;
projectJobs: number;
catalogJobs: number;
contractJobs: number;
failures: number;
};
function cadenceBucket(now: Date, intervalMs: number): string {
return String(Math.floor(now.getTime() / intervalMs));
}
export async function runAllTenantKwpProjectMaintenance(input: {
prisma: KwpMaintenancePrisma;
queue: TypedQueue<typeof queueNames.kwpImports>;
now?: Date;
}): Promise<KwpMaintenanceResult> {
const now = input.now ?? new Date();
const projectCutoff = new Date(now.getTime() - PROJECT_INTERVAL_MS);
const projectBucket = cadenceBucket(now, PROJECT_INTERVAL_MS);
const contractBucket = cadenceBucket(now, CONTRACT_INTERVAL_MS);
const result = await runActiveTenantSweep(input.prisma, {
initialState: { projectJobs: 0, catalogJobs: 0, contractJobs: 0 },
worker: (tenantId) => runWithRlsContext({ tenantId }, async () => {
const [bindings, connections] = await Promise.all([
input.prisma.appEntityBinding.findMany({
where: {
tenantId,
externalEntityType: "kwp_project",
localEntityType: "project",
state: { in: ["active", "missing"] },
OR: [{ lastSyncedAt: null }, { lastSyncedAt: { lte: projectCutoff } }],
},
select: { id: true, dataConnectionId: true },
orderBy: { id: "asc" },
take: BATCH_SIZE,
}),
input.prisma.dataConnection.findMany({
where: {
tenantId,
appId: "kwp",
archivedAt: null,
connectionStatus: { in: ["active", "degraded"] },
},
select: { id: true },
orderBy: { id: "asc" },
take: BATCH_SIZE,
}),
]);
for (const binding of bindings) {
const descriptor = buildKwpProjectSyncJob({
tenantId,
bindingId: binding.id,
dataConnectionId: binding.dataConnectionId,
cadenceBucket: projectBucket,
});
await enqueueJob(input.queue, descriptor, {
idempotencyKey: `kwp-project-sync:${tenantId}:${binding.id}:${projectBucket}`,
tenantId,
attempts: 5,
backoff: { type: "exponential", delay: 30_000 + Math.floor(Math.random() * 10_000) },
});
}
for (const connection of connections) {
const catalogDescriptor = buildKwpProjectCatalogSyncJob({
tenantId,
dataConnectionId: connection.id,
cadenceBucket: projectBucket,
});
await enqueueJob(input.queue, catalogDescriptor, {
idempotencyKey: `kwp-project-catalog:${tenantId}:${connection.id}:${projectBucket}`,
tenantId,
attempts: 5,
backoff: { type: "exponential", delay: 30_000 + Math.floor(Math.random() * 10_000) },
});
const descriptor = buildKwpContractSentinelJob({
tenantId,
dataConnectionId: connection.id,
cadenceBucket: contractBucket,
});
await enqueueJob(input.queue, descriptor, {
idempotencyKey: `kwp-contract-sentinel:${tenantId}:${connection.id}:${contractBucket}`,
tenantId,
attempts: 3,
backoff: { type: "exponential", delay: 60_000 + Math.floor(Math.random() * 20_000) },
});
}
return { projectJobs: bindings.length, catalogJobs: connections.length, contractJobs: connections.length };
}),
merge: (state, value) => ({
projectJobs: state.projectJobs + value.projectJobs,
catalogJobs: state.catalogJobs + value.catalogJobs,
contractJobs: state.contractJobs + value.contractJobs,
}),
});
return {
inspectedTenants: result.inspected,
projectJobs: result.state.projectJobs,
catalogJobs: result.state.catalogJobs,
contractJobs: result.state.contractJobs,
failures: result.failures.length,
};
}
@@ -5,10 +5,7 @@ import {
type LegacyEventReceiverBootstrapDeps,
type LegacyEventReceiverWorkerHandle,
} from "./legacy-event-receiver-bootstrap.js";
import {
LEGACY_EVENT_OUTBOX_QUEUE,
type LegacyEventReceiverStore,
} from "./legacy-event-receiver.js";
import { LEGACY_EVENT_OUTBOX_QUEUE, type LegacyEventReceiverStore } from "./legacy-event-receiver.js";
import type { OutboxReceiverIdempotencyEntry } from "@hivegrid/db/outbox-receiver";
function createStore(): LegacyEventReceiverStore & { calls: number; last: unknown } {
@@ -42,11 +39,7 @@ function createCapturingWorkerImpl() {
name: "legacy-event-receiver-worker",
close: vi.fn(async () => undefined),
};
const createWorkerImpl: NonNullable<LegacyEventReceiverBootstrapDeps["createWorkerImpl"]> = (
queueName,
processor,
options
) => {
const createWorkerImpl: NonNullable<LegacyEventReceiverBootstrapDeps["createWorkerImpl"]> = (queueName, processor, options) => {
captured.queueName = queueName;
captured.processor = processor;
captured.concurrency = options.concurrency;
@@ -117,10 +110,7 @@ describe("bootstrapLegacyEventReceiver", () => {
it("parses a positive concurrency override", () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
bootstrapLegacyEventReceiver(
{ LEGACY_EVENT_RECEIVER_CONCURRENCY: "12" },
{ store: createStore(), createWorkerImpl }
);
bootstrapLegacyEventReceiver({ LEGACY_EVENT_RECEIVER_CONCURRENCY: "12" }, { store: createStore(), createWorkerImpl });
expect(captured.concurrency).toBe(12);
});
@@ -131,10 +121,7 @@ describe("bootstrapLegacyEventReceiver", () => {
["", 5],
])("falls back to the default concurrency for %s", (raw, expected) => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
bootstrapLegacyEventReceiver(
{ LEGACY_EVENT_RECEIVER_CONCURRENCY: raw },
{ store: createStore(), createWorkerImpl }
);
bootstrapLegacyEventReceiver({ LEGACY_EVENT_RECEIVER_CONCURRENCY: raw }, { store: createStore(), createWorkerImpl });
expect(captured.concurrency).toBe(expected);
});
@@ -151,6 +138,114 @@ describe("bootstrapLegacyEventReceiver", () => {
expect(store.calls).toBe(1);
});
it("multiplexes platform notification events on the existing outbox.notification subscriber", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const store = createStore();
const enqueue = vi.fn(async () => undefined);
bootstrapLegacyEventReceiver(
{},
{
store,
createWorkerImpl,
platformNotification: { enqueue },
}
);
const outcome = await captured.processor!({
data: {
outboxEventId: "outbox_1",
tenantId: null,
eventType: "notification.platform.requested.v1",
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1", sourceId: "hivegrid-website" },
},
},
});
expect(outcome).toMatchObject({ status: "enqueued", deliveryId: "delivery_1" });
expect(enqueue).toHaveBeenCalledTimes(1);
expect(store.calls).toBe(0);
});
it("acknowledges the platform lead capture marker without duplicating its durable notification requests", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const store = createStore();
const enqueue = vi.fn(async () => undefined);
bootstrapLegacyEventReceiver(
{},
{
store,
createWorkerImpl,
platformNotification: { enqueue },
}
);
const marker = {
data: {
outboxEventId: "activity_1:lead-outbox",
tenantId: null,
eventType: "platform.admin_lead.captured.v1",
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
},
dispatchKey: "platform-lead-captured:lead_1",
retries: 0,
deliveryGeneration: 0,
lockedUntil: "2026-07-16T10:21:30.123Z",
},
};
await expect(captured.processor!(marker)).resolves.toEqual({
status: "acknowledged",
outboxEventId: "activity_1:lead-outbox",
leadId: "lead_1",
});
await expect(captured.processor!(marker)).resolves.toMatchObject({ status: "acknowledged" });
expect(enqueue).not.toHaveBeenCalled();
expect(store.calls).toBe(0);
});
it("keeps the platform notification subscriber active when the legacy receiver is disabled", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const enqueue = vi.fn(async () => undefined);
const result = bootstrapLegacyEventReceiver(
{ LEGACY_EVENT_RECEIVER_ENABLED: "false" },
{
createWorkerImpl,
platformNotification: { enqueue },
}
);
expect(result.status).toBe("started");
await captured.processor!({
data: {
outboxEventId: "outbox_1",
tenantId: null,
eventType: "notification.platform.requested.v1",
payload: {
deliveryId: "delivery_1",
userId: "user_1",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
},
},
});
expect(enqueue).toHaveBeenCalledTimes(1);
});
it("uses idempotency to skip a re-delivered outbox event", async () => {
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
const store = createStore();
@@ -1,9 +1,11 @@
/**
* Bootstrap helper for the legacy-event outbox receiver.
* Bootstrap helper for the shared notification outbox receiver.
*
* Subscribes a BullMQ worker to the `outbox.notification` queue (which
* the dispatcher routes `billing.subscription.*` events into) and feeds
* each job through {@link runLegacyEventReceiverJob}.
* billing jobs through {@link runLegacyEventReceiverJob} and platform
* notification jobs through their strict tenantless bridge. The platform
* branch remains active even when the legacy branch is disabled.
*
* Gating:
* - WORKER_QUEUE_GROUP === "general" (co-located with the outbox
@@ -20,6 +22,12 @@ import {
type LegacyEventReceiverLogger,
type LegacyEventReceiverStore,
} from "./legacy-event-receiver.js";
import {
isPlatformNotificationOutboxJob,
runPlatformNotificationOutboxJob,
type PlatformNotificationOutboxDependencies,
} from "./platform-notification-outbox.js";
import { isPlatformLeadCapturedOutboxJob, runPlatformLeadCapturedOutboxJob } from "./platform-lead-captured-outbox.js";
export type LegacyEventReceiverBootstrapEnv = EnvRecord & {
WORKER_QUEUE_GROUP?: string;
@@ -36,6 +44,7 @@ export interface LegacyEventReceiverBootstrapDeps {
readonly logger?: LegacyEventReceiverLogger;
readonly store?: LegacyEventReceiverStore;
readonly idempotency?: LegacyEventReceiverDependencies["idempotency"];
readonly platformNotification?: PlatformNotificationOutboxDependencies;
readonly createWorkerImpl?: (
queueName: string,
processor: (job: { data: unknown }) => Promise<unknown>,
@@ -60,14 +69,16 @@ export function bootstrapLegacyEventReceiver(
deps: LegacyEventReceiverBootstrapDeps = {}
): LegacyEventReceiverBootstrapResult {
const logger = deps.logger;
const legacyEnabled = env.LEGACY_EVENT_RECEIVER_ENABLED !== "false";
const platformEnabled = deps.platformNotification !== undefined;
if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
return { status: "disabled", reason: "non-general worker group" };
}
if (env.LEGACY_EVENT_RECEIVER_ENABLED === "false") {
if (!legacyEnabled && !platformEnabled) {
return { status: "disabled", reason: "LEGACY_EVENT_RECEIVER_ENABLED=false" };
}
if (!deps.store) {
if (legacyEnabled && !deps.store && !platformEnabled) {
return { status: "disabled", reason: "no store wired (worker host must supply createLegacyEvent)" };
}
if (!deps.createWorkerImpl) {
@@ -79,6 +90,24 @@ export function bootstrapLegacyEventReceiver(
const idempotency = deps.idempotency;
const processor = async (job: { data: unknown }): Promise<unknown> => {
if (isPlatformLeadCapturedOutboxJob(job.data)) {
return runPlatformLeadCapturedOutboxJob(job.data, { ...(logger ? { logger } : {}) });
}
if (isPlatformNotificationOutboxJob(job.data)) {
if (!deps.platformNotification) {
throw new Error("platform notification outbox receiver is not wired");
}
return runPlatformNotificationOutboxJob(job.data, deps.platformNotification);
}
if (!legacyEnabled) {
throw new Error("legacy-event outbox receiver is disabled");
}
if (!store) {
throw new Error("legacy-event outbox receiver store is not wired");
}
const handlerDeps: LegacyEventReceiverDependencies = {
store,
...(idempotency ? { idempotency } : {}),
@@ -89,9 +118,6 @@ export function bootstrapLegacyEventReceiver(
const handle = deps.createWorkerImpl(LEGACY_EVENT_OUTBOX_QUEUE, processor, { concurrency });
logger?.info?.(
{ component: "legacy-event-receiver", queue: LEGACY_EVENT_OUTBOX_QUEUE, concurrency },
"legacy-event receiver worker started"
);
logger?.info?.({ component: "legacy-event-receiver", queue: LEGACY_EVENT_OUTBOX_QUEUE, concurrency }, "legacy-event receiver worker started");
return { status: "started", handle };
}
@@ -0,0 +1,104 @@
import { describe, expect, it, vi } from "vitest";
import {
PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
isPlatformLeadCapturedOutboxJob,
parsePlatformLeadCapturedOutboxJob,
runPlatformLeadCapturedOutboxJob,
} from "./platform-lead-captured-outbox.js";
function buildJob(overrides: Record<string, unknown> = {}): unknown {
return {
outboxEventId: "activity_1:lead-outbox",
tenantId: null,
eventType: PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
},
dispatchKey: "platform-lead-captured:lead_1",
retries: 0,
deliveryGeneration: 0,
lockedUntil: "2026-07-16T10:21:30.123Z",
...overrides,
};
}
describe("platform lead captured outbox marker", () => {
it("parses the exact tenantless PII-minimized producer envelope", () => {
expect(parsePlatformLeadCapturedOutboxJob(buildJob())).toEqual({
outboxEventId: "activity_1:lead-outbox",
tenantId: null,
eventType: PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
},
});
expect(isPlatformLeadCapturedOutboxJob(buildJob())).toBe(true);
expect(isPlatformLeadCapturedOutboxJob({ eventType: "notification.platform.requested.v1" })).toBe(false);
});
it("rejects tenant scope, PII fields, malformed timestamps and invalid routing", () => {
expect(() => parsePlatformLeadCapturedOutboxJob(buildJob({ tenantId: "tenant_escape" }))).toThrow("require tenantId=null");
expect(() => parsePlatformLeadCapturedOutboxJob(buildJob({ lockedUntil: null }))).toThrow("canonical UTC timestamp");
expect(() => parsePlatformLeadCapturedOutboxJob({ ...(buildJob() as object), email: "lead@example.test" })).toThrow(
"non-allowlisted fields"
);
expect(() =>
parsePlatformLeadCapturedOutboxJob(
buildJob({
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris"],
message: "private lead message",
},
})
)
).toThrow("non-allowlisted fields");
expect(() =>
parsePlatformLeadCapturedOutboxJob(
buildJob({
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16 10:20:30",
routingUserIds: ["user_haris"],
},
})
)
).toThrow("canonical UTC timestamp");
expect(() =>
parsePlatformLeadCapturedOutboxJob(
buildJob({
payload: {
leadId: "lead_1",
sourceId: "hivegrid-website",
eventTime: "2026-07-16T10:20:30.123Z",
routingUserIds: ["user_haris", "user_haris"],
},
})
)
).toThrow("must not contain duplicates");
});
it("acknowledges replay idempotently and never logs routing identifiers or PII", async () => {
const logger = { info: vi.fn() };
const first = await runPlatformLeadCapturedOutboxJob(buildJob(), { logger });
const replay = await runPlatformLeadCapturedOutboxJob(buildJob(), { logger });
expect(first).toEqual({ status: "acknowledged", outboxEventId: "activity_1:lead-outbox", leadId: "lead_1" });
expect(replay).toEqual(first);
expect(logger.info).toHaveBeenCalledTimes(2);
const serializedLogs = JSON.stringify(logger.info.mock.calls);
expect(serializedLogs).not.toContain("user_haris");
expect(serializedLogs).not.toContain("lead@example.test");
});
});
@@ -0,0 +1,151 @@
/**
* Consumer for the PII-minimized platform-lead capture domain marker.
*
* The capture transaction already persists one NotificationDelivery and one
* `notification.platform.requested.v1` outbox event per configured recipient
* and channel. This handler therefore only validates and acknowledges the
* companion domain event. Re-delivery is inherently idempotent because it has
* no downstream side effect.
*/
export const PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE = "platform.admin_lead.captured.v1" as const;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/u;
const UTC_TIMESTAMP_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u;
const ALLOWED_ENVELOPE_KEYS = new Set([
"outboxEventId",
"tenantId",
"eventType",
"payload",
"dispatchKey",
"retries",
"deliveryGeneration",
"lockedUntil",
]);
const ALLOWED_PAYLOAD_KEYS = new Set(["leadId", "sourceId", "eventTime", "routingUserIds"]);
export type PlatformLeadCapturedOutboxJob = {
readonly outboxEventId: string;
readonly tenantId: null;
readonly eventType: typeof PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE;
readonly payload: {
readonly leadId: string;
readonly sourceId: string;
readonly eventTime: string;
readonly routingUserIds: readonly string[];
};
};
export type PlatformLeadCapturedOutboxDependencies = {
readonly logger?: {
info?: (record: Record<string, unknown>, message?: string) => void;
};
};
export class PlatformLeadCapturedOutboxError extends Error {
override readonly name = "PlatformLeadCapturedOutboxError";
readonly code = "INVALID_PAYLOAD" as const;
}
export function isPlatformLeadCapturedOutboxJob(data: unknown): boolean {
return Boolean(
data &&
typeof data === "object" &&
!Array.isArray(data) &&
(data as Record<string, unknown>).eventType === PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE
);
}
export function parsePlatformLeadCapturedOutboxJob(data: unknown): PlatformLeadCapturedOutboxJob {
const envelope = requireRecord(data, "platform lead captured outbox envelope");
assertAllowlistedKeys(envelope, ALLOWED_ENVELOPE_KEYS, "platform lead captured outbox envelope");
if (envelope.tenantId !== null) {
throw new PlatformLeadCapturedOutboxError("Platform lead captured outbox events require tenantId=null.");
}
if (envelope.eventType !== PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE) {
throw new PlatformLeadCapturedOutboxError("Unsupported platform lead captured outbox event type.");
}
requireCanonicalTimestamp(envelope.lockedUntil, "lockedUntil");
const payload = requireRecord(envelope.payload, "platform lead captured outbox payload");
assertAllowlistedKeys(payload, ALLOWED_PAYLOAD_KEYS, "platform lead captured outbox payload");
const eventTime = typeof payload.eventTime === "string" ? payload.eventTime : "";
if (!UTC_TIMESTAMP_PATTERN.test(eventTime) || Number.isNaN(Date.parse(eventTime)) || new Date(eventTime).toISOString() !== eventTime) {
throw new PlatformLeadCapturedOutboxError("eventTime must be a canonical UTC timestamp.");
}
if (!Array.isArray(payload.routingUserIds) || payload.routingUserIds.length === 0 || payload.routingUserIds.length > 20) {
throw new PlatformLeadCapturedOutboxError("routingUserIds must contain between one and twenty operator identifiers.");
}
const routingUserIds = payload.routingUserIds.map((value) => requireIdentifier(value, "routingUserId"));
if (new Set(routingUserIds).size !== routingUserIds.length) {
throw new PlatformLeadCapturedOutboxError("routingUserIds must not contain duplicates.");
}
return {
outboxEventId: requireIdentifier(envelope.outboxEventId, "outboxEventId"),
tenantId: null,
eventType: PLATFORM_LEAD_CAPTURED_OUTBOX_TYPE,
payload: {
leadId: requireIdentifier(payload.leadId, "leadId"),
sourceId: requireIdentifier(payload.sourceId, "sourceId"),
eventTime,
routingUserIds,
},
};
}
export async function runPlatformLeadCapturedOutboxJob(
data: unknown,
deps: PlatformLeadCapturedOutboxDependencies = {}
): Promise<{ status: "acknowledged"; outboxEventId: string; leadId: string }> {
const job = parsePlatformLeadCapturedOutboxJob(data);
deps.logger?.info?.(
{
component: "platform-lead-captured-outbox",
outboxEventId: job.outboxEventId,
eventType: job.eventType,
leadId: job.payload.leadId,
sourceId: job.payload.sourceId,
routingRecipientCount: job.payload.routingUserIds.length,
},
"platform lead capture marker acknowledged"
);
return { status: "acknowledged", outboxEventId: job.outboxEventId, leadId: job.payload.leadId };
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new PlatformLeadCapturedOutboxError(`${label} must be an object.`);
}
return value as Record<string, unknown>;
}
function requireIdentifier(value: unknown, field: string): string {
const normalized = typeof value === "string" ? value.trim() : "";
if (!IDENTIFIER_PATTERN.test(normalized)) {
throw new PlatformLeadCapturedOutboxError(`${field} must be a stable non-secret identifier.`);
}
return normalized;
}
function requireCanonicalTimestamp(value: unknown, field: string): string {
if (typeof value !== "string") {
throw new PlatformLeadCapturedOutboxError(`${field} must be a canonical UTC timestamp.`);
}
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime()) || parsed.toISOString() !== value) {
throw new PlatformLeadCapturedOutboxError(`${field} must be a canonical UTC timestamp.`);
}
return value;
}
function assertAllowlistedKeys(record: Record<string, unknown>, allowlist: ReadonlySet<string>, label: string): void {
if (Object.keys(record).some((key) => !allowlist.has(key))) {
throw new PlatformLeadCapturedOutboxError(`${label} contains non-allowlisted fields.`);
}
}
@@ -0,0 +1,283 @@
import { describe, expect, it, vi } from "vitest";
import { buildPlatformNotificationJob, buildRetryJobOptions, type NotificationJobDescriptor, type QueueEnqueueOptions } from "@hivegrid/queue";
import type { MailProvider } from "@hivegrid/mail";
import { buildHandlerRegistry } from "../handlers.js";
import {
PLATFORM_NOTIFICATION_OUTBOX_TYPE,
isPlatformNotificationOutboxJob,
parsePlatformNotificationOutboxJob,
runPlatformNotificationOutboxJob,
} from "./platform-notification-outbox.js";
function buildJob(overrides: Record<string, unknown> = {}): unknown {
return {
outboxEventId: "outbox_1",
tenantId: null,
eventType: PLATFORM_NOTIFICATION_OUTBOX_TYPE,
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
metadata: {
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
},
},
...overrides,
};
}
describe("platform notification outbox parser", () => {
it("parses the strict tenantless PII-free envelope", () => {
const parsed = parsePlatformNotificationOutboxJob(buildJob());
expect(parsed).toEqual({
outboxEventId: "outbox_1",
tenantId: null,
eventType: PLATFORM_NOTIFICATION_OUTBOX_TYPE,
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
metadata: {
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
},
},
});
expect(isPlatformNotificationOutboxJob(buildJob())).toBe(true);
expect(isPlatformNotificationOutboxJob({ eventType: "billing.subscription.ended" })).toBe(false);
});
it("rejects tenant scope and non-allowlisted PII fields", () => {
expect(() => parsePlatformNotificationOutboxJob(buildJob({ tenantId: "tenant_escape" }))).toThrow("require tenantId=null");
expect(() =>
parsePlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
email: "lead@example.test",
},
})
)
).toThrow("non-allowlisted fields");
expect(() =>
parsePlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1", message: "private lead text" },
},
})
)
).toThrow("non-allowlisted fields");
});
});
describe("platform notification outbox bridge", () => {
it("enqueues a deterministic retryable worker job without recipient PII", async () => {
const enqueue = vi.fn(async () => undefined);
const logger = { info: vi.fn() };
const result = await runPlatformNotificationOutboxJob(buildJob(), { enqueue, logger });
expect(result).toEqual({ status: "enqueued", outboxEventId: "outbox_1", deliveryId: "delivery_1", channel: "email" });
expect(enqueue).toHaveBeenCalledWith(
expect.objectContaining({
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
subject: "Neuer Lead in HiveGrid",
text: expect.stringContaining("Ein neuer Lead ist eingegangen"),
html: expect.stringContaining("https://hivegrid.de/hivegrid-logo.png"),
metadata: {
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
deliveryId: "delivery_1",
eventId: "outbox_1",
},
}),
{ tenantId: null, idempotencyKey: "platform-notification-outbox:outbox_1" }
);
expect(JSON.stringify(enqueue.mock.calls)).not.toContain("@example.test");
expect(JSON.stringify(logger.info.mock.calls)).not.toContain("haris@example.test");
});
it("keeps in-app delivery PII-free as well", async () => {
const enqueue = vi.fn(async () => undefined);
const job = buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
metadata: { leadId: "lead_1" },
},
});
await runPlatformNotificationOutboxJob(job, { enqueue });
expect(enqueue).toHaveBeenCalledWith(expect.not.objectContaining({ email: expect.anything() }), expect.objectContaining({ tenantId: null }));
});
it("bridges an admin sales task reminder without requiring lead metadata", async () => {
const enqueue = vi.fn(async () => undefined);
const job = buildJob({
payload: {
deliveryId: "delivery_task_1",
adminUserId: "admin_1",
sourceKey: "admin-sales:task-reminder:task_1:2026-07-25",
channel: "in-app",
messageKey: "admin.sales.notifications.taskreminder",
category: "sales",
metadata: {
taskId: "task_1",
deeplink: "/sales/tasks/today",
},
},
});
await runPlatformNotificationOutboxJob(job, { enqueue });
expect(enqueue).toHaveBeenCalledWith(
expect.objectContaining({
adminUserId: "admin_1",
messageKey: "admin.sales.notifications.taskreminder",
subject: "Aufgabe ist bald fällig",
metadata: expect.objectContaining({ taskId: "task_1" }),
}),
{ tenantId: null, idempotencyKey: "platform-notification-outbox:outbox_1" }
);
});
it("bridges an opted-in live-open event to the separated admin identity", async () => {
const enqueue = vi.fn(async () => undefined);
const job = buildJob({
payload: {
deliveryId: "delivery_open_1",
adminUserId: "admin_1",
sourceKey: "admin-email-open:event_1",
channel: "in-app",
messageKey: "admin.sales.email.notifications.opened",
category: "admin-sales-email",
metadata: {
emailMessageId: "message_1",
emailEventId: "event_1",
deeplink: "/sales/email",
},
},
});
await runPlatformNotificationOutboxJob(job, { enqueue });
expect(enqueue).toHaveBeenCalledWith(
expect.objectContaining({
adminUserId: "admin_1",
messageKey: "admin.sales.email.notifications.opened",
metadata: expect.objectContaining({
emailMessageId: "message_1",
emailEventId: "event_1",
}),
}),
{
tenantId: null,
idempotencyKey: "platform-notification-outbox:outbox_1",
}
);
});
it("fails closed when a task reminder has no task id", async () => {
const enqueue = vi.fn(async () => undefined);
await expect(
runPlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_task_1",
userId: "admin_1",
sourceKey: "admin-sales:task-reminder:task_1:2026-07-25",
channel: "in-app",
messageKey: "admin.sales.notifications.taskreminder",
},
}),
{ enqueue }
)
).rejects.toMatchObject({ code: "INVALID_PAYLOAD" });
expect(enqueue).not.toHaveBeenCalled();
});
it("fails closed when a platform notification has no lead id", async () => {
const enqueue = vi.fn(async () => undefined);
await expect(
runPlatformNotificationOutboxJob(
buildJob({
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
},
}),
{ enqueue }
)
).rejects.toMatchObject({ code: "INVALID_PAYLOAD" });
expect(enqueue).not.toHaveBeenCalled();
});
it("hands the durable outbox request to the existing retryable notification worker", async () => {
let queued: { descriptor: NotificationJobDescriptor; options: QueueEnqueueOptions } | undefined;
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => ({ provider: "mailjet" as const, status: "queued" as const, messageId: "mailjet_1" })),
};
const markPlatformNotificationDeliveryStatus = vi.fn(async () => ({ count: 1 }));
await runPlatformNotificationOutboxJob(buildJob(), {
enqueue: async (input, options) => {
queued = { descriptor: buildPlatformNotificationJob(input), options };
},
});
expect(queued).toBeDefined();
expect(buildRetryJobOptions(queued!.descriptor)).toMatchObject({ attempts: 3, backoff: { type: "exponential", delay: 1_000 } });
expect(queued!.options).toEqual({ tenantId: null, idempotencyKey: "platform-notification-outbox:outbox_1" });
const registry = buildHandlerRegistry({
mailProvider,
createPlatformNotificationDelivery: vi.fn(async () => ({ id: "delivery_1", status: "PENDING" })),
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient: vi.fn(async () => ({ email: "operator@example.test" })),
});
await expect(registry[queued!.descriptor.queueName](queued!.descriptor.payload, {}, { attemptsMade: 0, attemptsLimit: 3 })).resolves.toMatchObject({
action: "notification.platform.email.sent",
details: { deliveryId: "delivery_1" },
});
expect(markPlatformNotificationDeliveryStatus).toHaveBeenCalledWith(
expect.objectContaining({ id: "delivery_1", userId: "user_haris", status: "SENT", attemptsMade: 1 })
);
expect(JSON.stringify(markPlatformNotificationDeliveryStatus.mock.calls)).not.toContain("operator@example.test");
});
});
@@ -0,0 +1,262 @@
import type { PlatformNotificationJobInput, PlatformNotificationSafeMetadata, QueueEnqueueOptions } from "@hivegrid/queue";
import { renderSystemMessageEmail } from "@hivegrid/mail";
export const PLATFORM_NOTIFICATION_OUTBOX_TYPE = "notification.platform.requested.v1" as const;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/u;
const MESSAGE_KEY_PATTERN = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/u;
const DEEPLINK_PATTERN = /^\/[A-Za-z0-9/_-]{1,239}$/u;
const ALLOWED_PAYLOAD_KEYS = new Set(["deliveryId", "userId", "adminUserId", "sourceKey", "channel", "messageKey", "category", "metadata"]);
const ALLOWED_METADATA_KEYS = new Set([
"eventId",
"eventType",
"leadId",
"taskId",
"emailMessageId",
"emailEventId",
"requestId",
"sourceId",
"deeplink",
"replay",
]);
export type PlatformNotificationOutboxJob = {
outboxEventId: string;
tenantId: null;
eventType: typeof PLATFORM_NOTIFICATION_OUTBOX_TYPE;
payload: {
deliveryId: string;
userId?: string;
adminUserId?: string;
sourceKey: string;
channel: "email" | "in-app";
messageKey: string;
category?: string;
metadata?: PlatformNotificationSafeMetadata;
};
};
export type PlatformNotificationOutboxDependencies = {
enqueue(input: PlatformNotificationJobInput, options: QueueEnqueueOptions): Promise<void>;
logger?: {
info?: (record: Record<string, unknown>, message?: string) => void;
};
};
export class PlatformNotificationOutboxError extends Error {
override readonly name = "PlatformNotificationOutboxError";
constructor(
message: string,
readonly code: "INVALID_PAYLOAD" | "UNSUPPORTED_MESSAGE"
) {
super(message);
}
}
export function isPlatformNotificationOutboxJob(data: unknown): boolean {
return Boolean(data && typeof data === "object" && !Array.isArray(data) && (data as Record<string, unknown>).eventType === PLATFORM_NOTIFICATION_OUTBOX_TYPE);
}
export function parsePlatformNotificationOutboxJob(data: unknown): PlatformNotificationOutboxJob {
const envelope = requireRecord(data, "platform notification outbox envelope");
const outboxEventId = requireIdentifier(envelope.outboxEventId, "outboxEventId");
if (envelope.tenantId !== null) {
throw new PlatformNotificationOutboxError("Platform notification outbox events require tenantId=null.", "INVALID_PAYLOAD");
}
if (envelope.eventType !== PLATFORM_NOTIFICATION_OUTBOX_TYPE) {
throw new PlatformNotificationOutboxError("Unsupported platform notification outbox event type.", "INVALID_PAYLOAD");
}
const payload = requireRecord(envelope.payload, "platform notification outbox payload");
assertAllowlistedKeys(payload, ALLOWED_PAYLOAD_KEYS, "platform notification outbox payload");
const channel = payload.channel;
if (channel !== "email" && channel !== "in-app") {
throw new PlatformNotificationOutboxError("Platform notification outbox payload has an unsupported channel.", "INVALID_PAYLOAD");
}
const messageKey = typeof payload.messageKey === "string" ? payload.messageKey.trim() : "";
if (!MESSAGE_KEY_PATTERN.test(messageKey)) {
throw new PlatformNotificationOutboxError("Platform notification outbox payload has an invalid messageKey.", "INVALID_PAYLOAD");
}
const category = typeof payload.category === "string" ? payload.category.trim().toLowerCase() : undefined;
if (category !== undefined && !/^[a-z][a-z0-9._-]{0,63}$/u.test(category)) {
throw new PlatformNotificationOutboxError("Platform notification outbox payload has an invalid category.", "INVALID_PAYLOAD");
}
const userId = typeof payload.userId === "string" ? requireIdentifier(payload.userId, "userId") : undefined;
const adminUserId = typeof payload.adminUserId === "string" ? requireIdentifier(payload.adminUserId, "adminUserId") : undefined;
if (Boolean(userId) === Boolean(adminUserId)) {
throw new PlatformNotificationOutboxError("Platform notification outbox payload requires exactly one recipient identity.", "INVALID_PAYLOAD");
}
return {
outboxEventId,
tenantId: null,
eventType: PLATFORM_NOTIFICATION_OUTBOX_TYPE,
payload: {
deliveryId: requireIdentifier(payload.deliveryId, "deliveryId"),
...(userId ? { userId } : { adminUserId }),
sourceKey: requireIdentifier(payload.sourceKey, "sourceKey"),
channel,
messageKey,
...(category ? { category } : {}),
...(payload.metadata === undefined ? {} : { metadata: parseSafeMetadata(payload.metadata) }),
},
};
}
export async function runPlatformNotificationOutboxJob(
data: unknown,
deps: PlatformNotificationOutboxDependencies
): Promise<{ status: "enqueued"; outboxEventId: string; deliveryId: string; channel: "email" | "in-app" }> {
const job = parsePlatformNotificationOutboxJob(data);
const copy = resolveMessageCopy(job.payload.messageKey);
const rendered =
job.payload.channel === "email" && copy.subject && copy.text
? renderSystemMessageEmail({
locale: "de",
subject: copy.subject,
previewText: copy.text,
eyebrow: "HiveGrid-Benachrichtigung",
title: copy.subject,
intro: copy.text,
notice: "Öffne HiveGrid, um alle Details zu sehen und direkt weiterzuarbeiten.",
})
: undefined;
if (
(job.payload.messageKey === "platform.leads.captured" || job.payload.messageKey === "admin.sales.notifications.leadassigned") &&
!job.payload.metadata?.leadId
) {
throw new PlatformNotificationOutboxError("Platform lead notifications require a leadId.", "INVALID_PAYLOAD");
}
if (job.payload.messageKey === "admin.sales.notifications.taskreminder" && !job.payload.metadata?.taskId) {
throw new PlatformNotificationOutboxError("Admin sales task notifications require a taskId.", "INVALID_PAYLOAD");
}
if (job.payload.messageKey === "admin.sales.email.notifications.opened" && !job.payload.metadata?.emailMessageId) {
throw new PlatformNotificationOutboxError("Admin email open notifications require an emailMessageId.", "INVALID_PAYLOAD");
}
await deps.enqueue(
{
...(job.payload.userId ? { userId: job.payload.userId } : { adminUserId: job.payload.adminUserId }),
sourceKey: job.payload.sourceKey,
channel: job.payload.channel,
messageKey: job.payload.messageKey,
...(job.payload.category ? { category: job.payload.category } : {}),
...(copy.subject ? { subject: copy.subject } : {}),
...(rendered ? { text: rendered.text, html: rendered.html } : copy.text ? { text: copy.text } : {}),
metadata: {
...job.payload.metadata,
deliveryId: job.payload.deliveryId,
eventId: job.outboxEventId,
},
},
{
tenantId: null,
idempotencyKey: `platform-notification-outbox:${job.outboxEventId}`,
}
);
deps.logger?.info?.(
{
component: "platform-notification-outbox",
outboxEventId: job.outboxEventId,
deliveryId: job.payload.deliveryId,
...(job.payload.userId ? { userId: job.payload.userId } : { adminUserId: job.payload.adminUserId }),
channel: job.payload.channel,
messageKey: job.payload.messageKey,
},
"platform notification enqueued from outbox"
);
return {
status: "enqueued",
outboxEventId: job.outboxEventId,
deliveryId: job.payload.deliveryId,
channel: job.payload.channel,
};
}
function resolveMessageCopy(messageKey: string): { subject?: string; text?: string } {
if (messageKey === "platform.leads.captured") {
return {
subject: "Neuer Lead in HiveGrid",
text: "Ein neuer Lead ist eingegangen. Bitte öffne die Lead-Inbox in HiveGrid.",
};
}
if (messageKey === "admin.sales.notifications.taskreminder") {
return {
subject: "Aufgabe ist bald fällig",
text: "Eine Aufgabe in deinem HiveGrid Sales Workspace ist bald fällig.",
};
}
if (messageKey === "admin.sales.notifications.leadassigned") {
return {
subject: "Lead wurde dir zugewiesen",
text: "Ein Lead wurde dir im HiveGrid Sales Workspace zugewiesen.",
};
}
if (messageKey === "admin.sales.email.notifications.opened") {
return {
subject: "E-Mail wurde geöffnet",
text: "Ein Interessent hat gerade eine nachverfolgbare E-Mail geöffnet.",
};
}
throw new PlatformNotificationOutboxError(`Unsupported platform notification messageKey ${messageKey}.`, "UNSUPPORTED_MESSAGE");
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new PlatformNotificationOutboxError(`${label} must be an object.`, "INVALID_PAYLOAD");
}
return value as Record<string, unknown>;
}
function requireIdentifier(value: unknown, field: string): string {
const normalized = typeof value === "string" ? value.trim() : "";
if (!IDENTIFIER_PATTERN.test(normalized)) {
throw new PlatformNotificationOutboxError(`${field} must be a stable non-secret identifier.`, "INVALID_PAYLOAD");
}
return normalized;
}
function assertAllowlistedKeys(record: Record<string, unknown>, allowlist: ReadonlySet<string>, label: string): void {
const unexpected = Object.keys(record).filter((key) => !allowlist.has(key));
if (unexpected.length > 0) {
throw new PlatformNotificationOutboxError(`${label} contains non-allowlisted fields.`, "INVALID_PAYLOAD");
}
}
function parseSafeMetadata(value: unknown): PlatformNotificationSafeMetadata {
const metadata = requireRecord(value, "platform notification outbox metadata");
assertAllowlistedKeys(metadata, ALLOWED_METADATA_KEYS, "platform notification outbox metadata");
const parsed: PlatformNotificationSafeMetadata = {};
for (const key of ["eventId", "leadId", "taskId", "emailMessageId", "emailEventId", "requestId", "sourceId"] as const) {
if (metadata[key] !== undefined) {
parsed[key] = requireIdentifier(metadata[key], key);
}
}
if (metadata.eventType !== undefined) {
const eventType = typeof metadata.eventType === "string" ? metadata.eventType.trim() : "";
if (!/^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$/u.test(eventType)) {
throw new PlatformNotificationOutboxError("eventType must be a safe event identifier.", "INVALID_PAYLOAD");
}
parsed.eventType = eventType;
}
if (metadata.deeplink !== undefined) {
const deeplink = typeof metadata.deeplink === "string" ? metadata.deeplink.trim() : "";
if (!DEEPLINK_PATTERN.test(deeplink)) {
throw new PlatformNotificationOutboxError("deeplink must be an app-relative PII-free path.", "INVALID_PAYLOAD");
}
parsed.deeplink = deeplink;
}
if (metadata.replay !== undefined) {
if (typeof metadata.replay !== "boolean") {
throw new PlatformNotificationOutboxError("replay must be boolean.", "INVALID_PAYLOAD");
}
parsed.replay = metadata.replay;
}
return parsed;
}
@@ -0,0 +1,163 @@
// @vitest-environment node
import { randomUUID } from "node:crypto";
import type { MailProvider } from "@hivegrid/mail";
import {
buildPlatformNotificationJob,
enqueueJob,
parseRedisUrl,
queueNames,
type QueuePayload,
type TypedQueue,
} from "@hivegrid/queue";
import { createQueue, createQueueEvents } from "@hivegrid/queue/runtime";
import { Queue, QueueEvents } from "bullmq";
import { describe, expect, it, vi } from "vitest";
import { shutdownWorkerRuntime, startWorkerRuntime } from "../runtime.js";
import { PLATFORM_NOTIFICATION_OUTBOX_TYPE, runPlatformNotificationOutboxJob } from "./platform-notification-outbox.js";
const REDIS_URL = process.env.PLATFORM_NOTIFICATION_REDIS_URL;
const describeIfRedis = REDIS_URL ? describe : describe.skip;
function diagnosticText(value: unknown): string {
if (value instanceof Error) return `${value.name}:${value.message}`;
if (Array.isArray(value)) return value.map(diagnosticText).join(" ");
if (value && typeof value === "object") {
return Object.entries(value)
.map(([key, entry]) => `${key}:${diagnosticText(entry)}`)
.join(" ");
}
return String(value ?? "");
}
describeIfRedis("platform notification Redis retry integration", () => {
it("moves a durable outbox request through BullMQ and retries the existing worker until delivery succeeds", async () => {
const suffix = randomUUID().replaceAll("-", "");
const outboxEventId = `outbox_${suffix}`;
const deliveryId = `delivery_${suffix}`;
const userId = `user_${suffix}`;
const leadId = `lead_${suffix}`;
const recipientEmail = "platform-operator@example.test";
const providerSecret = "super-secret-provider-token";
const connection = parseRedisUrl(REDIS_URL!);
const queue = createQueue(queueNames.notifications, { connection });
const queueEvents = createQueueEvents(queueNames.notifications, { connection }) as QueueEvents;
const bullQueue = queue as unknown as Queue<QueuePayload<typeof queueNames.notifications>, unknown, "send-notification">;
const deliveryUpdates: Array<Record<string, unknown>> = [];
let deliveryStatus = "PENDING";
const mailProvider: MailProvider = {
id: "mailjet",
send: vi.fn(async () => {
const attempt = vi.mocked(mailProvider.send).mock.calls.length;
if (attempt < 3) {
throw new Error(`provider token=${providerSecret} recipient ${recipientEmail}`);
}
return { provider: "mailjet" as const, status: "queued" as const, messageId: `mailjet_${suffix}` };
}),
};
const createPlatformNotificationDelivery = vi.fn(async () => ({ id: deliveryId, status: deliveryStatus }));
const resolvePlatformLeadNotificationRecipient = vi.fn(async () => ({ email: recipientEmail }));
const markPlatformNotificationDeliveryStatus = vi.fn(async (input: Record<string, unknown>) => {
deliveryUpdates.push(input);
deliveryStatus = String(input.status);
return { count: 1 };
});
const logger = { error: vi.fn(), info: vi.fn(), log: vi.fn(), warn: vi.fn() };
const runtime = startWorkerRuntime({
env: {
BULLMQ_REDIS_URL: REDIS_URL!,
DATABASE_URL: "postgresql://unused:unused@127.0.0.1:1/unused",
WORKER_ENABLED: "true",
WORKER_QUEUE_GROUP: "general",
},
extraHandlerDependencies: {
mailProvider,
createPlatformNotificationDelivery,
markPlatformNotificationDeliveryStatus,
resolvePlatformLeadNotificationRecipient,
},
logger,
queueNames: [queueNames.notifications],
recordFailedJob: vi.fn(async () => undefined),
});
try {
await queueEvents.waitUntilReady();
let queuedJobId: string | undefined;
await runPlatformNotificationOutboxJob(
{
outboxEventId,
tenantId: null,
eventType: PLATFORM_NOTIFICATION_OUTBOX_TYPE,
payload: {
deliveryId,
userId,
sourceKey: `platform-lead:${leadId}:email`,
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
metadata: { leadId, sourceId: "hivegrid-website", deeplink: `/leads/${leadId}` },
},
},
{
enqueue: async (input, options) => {
const job = await enqueueJob(
queue as TypedQueue<typeof queueNames.notifications>,
buildPlatformNotificationJob(input),
options
);
queuedJobId = job.id === undefined ? undefined : String(job.id);
},
logger,
}
);
expect(queuedJobId).toBe(`platform-notification-outbox%3A${outboxEventId}`);
const queuedJob = await bullQueue.getJob(queuedJobId!);
expect(queuedJob).toBeDefined();
const result = await queuedJob!.waitUntilFinished(queueEvents, 15_000);
expect(result).toMatchObject({
handled: true,
action: "notification.platform.email.sent",
details: { deliveryId, userId, messageKey: "platform.leads.captured" },
});
expect(mailProvider.send).toHaveBeenCalledTimes(3);
expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledTimes(3);
expect(resolvePlatformLeadNotificationRecipient).toHaveBeenCalledWith({ userId, leadId });
expect(createPlatformNotificationDelivery).toHaveBeenCalledTimes(3);
expect(deliveryUpdates).toEqual([
expect.objectContaining({ id: deliveryId, userId, status: "PENDING", attemptsMade: 1 }),
expect.objectContaining({ id: deliveryId, userId, status: "PENDING", attemptsMade: 2 }),
expect.objectContaining({ id: deliveryId, userId, status: "SENT", attemptsMade: 3, providerMessageId: `mailjet_${suffix}` }),
]);
expect(deliveryUpdates.slice(0, 2)).toEqual([
expect.objectContaining({ lastError: "provider token=[redacted] recipient [redacted-email]" }),
expect.objectContaining({ lastError: "provider token=[redacted] recipient [redacted-email]" }),
]);
for (const [input] of vi.mocked(mailProvider.send).mock.calls) {
expect(input.idempotencyKey).toBe(`notification-delivery:${deliveryId}`);
}
const completedJob = await bullQueue.getJob(queuedJobId!);
expect(completedJob?.opts.attempts).toBe(3);
expect(completedJob?.attemptsMade).toBe(3);
expect(completedJob?.failedReason).not.toContain(providerSecret);
expect(completedJob?.failedReason).not.toContain(recipientEmail);
expect(completedJob?.failedReason).toContain("[redacted]");
const logText = diagnosticText([logger.error.mock.calls, logger.info.mock.calls, logger.log.mock.calls, logger.warn.mock.calls]);
expect(logText).toContain("[redacted]");
expect(logText).not.toContain(providerSecret);
expect(logText).not.toContain(recipientEmail);
expect(JSON.stringify(deliveryUpdates)).not.toContain(providerSecret);
expect(JSON.stringify(deliveryUpdates)).not.toContain(recipientEmail);
} finally {
await shutdownWorkerRuntime(runtime, { logger, reason: "platform-notification-redis-integration-test" });
await queueEvents.close();
await queue.close();
}
}, 20_000);
});
@@ -1,10 +1,6 @@
import { describe, expect, test, vi } from "vitest";
import {
bootstrapPluginHookFanout,
isPluginHookFanoutRequired,
PluginHookFanoutRequiredBootstrapError,
} from "./plugin-hook-fanout-bootstrap.js";
import { bootstrapPluginHookFanout, isPluginHookFanoutRequired, PluginHookFanoutRequiredBootstrapError } from "./plugin-hook-fanout-bootstrap.js";
import { PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE } from "./plugin-hook-fanout.js";
type WorkerProcessor = (job: { data: unknown }) => Promise<unknown>;
@@ -70,7 +66,10 @@ describe("bootstrapPluginHookFanout", () => {
const { createWorkerImpl } = fakeWorker();
expect(() =>
bootstrapPluginHookFanout({ WORKER_QUEUE_GROUP: "general", PLUGIN_HOOK_FANOUT_REQUIRED: "true", PLUGIN_HOOK_FANOUT_ENABLED: "false" }, { createWorkerImpl })
bootstrapPluginHookFanout(
{ WORKER_QUEUE_GROUP: "general", PLUGIN_HOOK_FANOUT_REQUIRED: "true", PLUGIN_HOOK_FANOUT_ENABLED: "false" },
{ createWorkerImpl }
)
).toThrow(PluginHookFanoutRequiredBootstrapError);
expect(createWorkerImpl).not.toHaveBeenCalled();
});
@@ -135,11 +134,7 @@ describe("bootstrapPluginHookFanout", () => {
expect(result.status).toBe("started");
expect(result.status === "started" ? result.handle : null).toBe(handle);
expect(createWorkerImpl).toHaveBeenCalledWith(
PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE,
expect.any(Function),
expect.objectContaining({ concurrency: 3 })
);
expect(createWorkerImpl).toHaveBeenCalledWith(PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE, expect.any(Function), expect.objectContaining({ concurrency: 3 }));
const processor = calls[0]?.[1];
expect(processor).toBeDefined();
@@ -161,7 +156,7 @@ describe("bootstrapPluginHookFanout", () => {
expect(add).toHaveBeenCalledWith(
"invoke-plugin",
expect.objectContaining({ kind: "plugin.invoke", pluginId: "com.acme.audit" }),
expect.objectContaining({ jobId: "plugin-hook:evt_1:com.acme.audit:1.0.0" })
expect.objectContaining({ jobId: "plugin-hook%3Aevt_1%3Acom.acme.audit%3A1.0.0" })
);
expect(prisma.plugin.findMany).toHaveBeenCalledWith({
where: { tenantId: "tenant_1", enabled: true },
@@ -16,6 +16,10 @@
* `plugin-invoke.test.ts`; here we only assert the composition itself.
*/
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, test, vi } from "vitest";
import { bootstrapPluginInvokeDeps, type PluginInvokeBootstrapPrismaSurface } from "./plugin-invoke-bootstrap.js";
@@ -307,6 +311,31 @@ describe("bootstrapPluginInvokeDeps", () => {
);
});
test("loadBinary rejects oversized WASM before reading it into memory", async () => {
const basePath = await mkdtemp(path.join(tmpdir(), "hivegrid-plugin-binary-"));
const binaryDirectory = path.join(basePath, "de.hivegrid.large", "1.0.0");
await mkdir(binaryDirectory, { recursive: true });
await writeFile(path.join(binaryDirectory, "plugin.wasm"), Buffer.alloc(9));
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
try {
const deps = bootstrapPluginInvokeDeps({
env: { PLUGIN_BINARY_BASE_PATH: basePath, PLUGIN_MAX_BINARY_BYTES: "8" },
prisma: makeFakePrisma(),
store: makeFakeStore() as never,
logger,
});
await expect(deps.loadBinary({ pluginId: "de.hivegrid.large", version: "1.0.0" })).resolves.toBeNull();
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({ pluginId: "de.hivegrid.large", version: "1.0.0", sizeBytes: 9, maxBytes: 8 }),
"plugin.invoke.binary_size_rejected",
);
} finally {
await rm(basePath, { recursive: true, force: true });
}
});
test("consumeRateLimit allows the first N calls then rejects", async () => {
const store = makeFakeStore();
const deps = bootstrapPluginInvokeDeps({
Loaded 100 of 5955 files, more files were not shown because too many files have changed in this diff. Show more