Update hivegrid-cloud reference to latest main
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e98d38e716
commit
7b21524c86
3781 files changed
+373484
-225607
No files matched your search
@@ -1,4 +1,4 @@
|
||||
# @toolctrl/worker
|
||||
# @hivegrid/worker
|
||||
|
||||
Opt-in BullMQ worker runtime for queue-backed jobs.
|
||||
|
||||
@@ -9,19 +9,19 @@ The worker is disabled by default and is not imported by the web runtime. Runnin
|
||||
Check worker readiness and Redis configuration without starting processors:
|
||||
|
||||
```sh
|
||||
pnpm --filter @toolctrl/worker health
|
||||
pnpm --filter @hivegrid/worker health
|
||||
```
|
||||
|
||||
Run the disabled-by-default worker locally:
|
||||
|
||||
```sh
|
||||
pnpm --filter @toolctrl/worker dev
|
||||
pnpm --filter @hivegrid/worker dev
|
||||
```
|
||||
|
||||
Start the enabled worker against a local Redis instance and database:
|
||||
|
||||
```sh
|
||||
WORKER_ENABLED=true WORKER_QUEUE_GROUP=general BULLMQ_REDIS_URL=redis://localhost:6379/0 DATABASE_URL=postgresql://localhost/app pnpm --filter @toolctrl/worker dev
|
||||
WORKER_ENABLED=true WORKER_QUEUE_GROUP=general BULLMQ_REDIS_URL=redis://localhost:6379/0 DATABASE_URL=postgresql://localhost/app pnpm --filter @hivegrid/worker dev
|
||||
```
|
||||
|
||||
The worker stays disabled unless `WORKER_ENABLED=true`. When enabled, `DATABASE_URL` is required, and Redis is resolved from `BULLMQ_REDIS_URL` first; `REDIS_URL` is fallback only when `BULLMQ_REDIS_URL` is absent for deployments that already provide a generic Redis binding.
|
||||
@@ -40,14 +40,13 @@ Manual E2E note: real queued job processing requires a running Redis/container s
|
||||
|
||||
## Current processors
|
||||
|
||||
The worker has concrete handler paths for mail, webhooks, notifications, maintenance, storage scans, exports, imports, and guarded AI/browser job descriptors. Some queues remain disabled unless feature flags and provider services are configured.
|
||||
The worker has concrete handler paths for mail, webhooks, notifications, maintenance, storage scans, exports, imports, and guarded browser job descriptors. Some queues remain disabled unless feature flags and provider services are configured.
|
||||
|
||||
- `toolctrl.email`
|
||||
- `toolctrl.notifications`
|
||||
- `toolctrl.maintenance`
|
||||
- `toolctrl.ai-chatbot`
|
||||
- `toolctrl.browser-automation`
|
||||
- `toolctrl.file-scan`
|
||||
- `toolctrl.exports`
|
||||
- `toolctrl.imports`
|
||||
- `toolctrl.plugin-invoke`
|
||||
- `hivegrid.email`
|
||||
- `hivegrid.notifications`
|
||||
- `hivegrid.maintenance`
|
||||
- `hivegrid.browser-automation`
|
||||
- `hivegrid.file-scan`
|
||||
- `hivegrid.exports`
|
||||
- `hivegrid.imports`
|
||||
- `hivegrid.plugin-invoke`
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "@toolctrl/worker",
|
||||
"name": "@hivegrid/worker",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
@@ -16,6 +16,27 @@
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.1045.0",
|
||||
"@aws-sdk/lib-storage": "3.1045.0",
|
||||
"@hivegrid/config": "workspace:*",
|
||||
"@hivegrid/db": "workspace:*",
|
||||
"@hivegrid/email-sequences": "workspace:*",
|
||||
"@hivegrid/email-templates": "workspace:*",
|
||||
"@hivegrid/entitlements": "workspace:*",
|
||||
"@hivegrid/error-reporter": "workspace:*",
|
||||
"@hivegrid/exports": "workspace:*",
|
||||
"@hivegrid/form-widget": "workspace:*",
|
||||
"@hivegrid/foundation": "workspace:*",
|
||||
"@hivegrid/imports": "workspace:*",
|
||||
"@hivegrid/integrations": "workspace:*",
|
||||
"@hivegrid/logger": "workspace:*",
|
||||
"@hivegrid/mail": "workspace:*",
|
||||
"@hivegrid/marketing-outbound": "workspace:*",
|
||||
"@hivegrid/metrics": "workspace:*",
|
||||
"@hivegrid/observability": "workspace:*",
|
||||
"@hivegrid/plugin-runtime": "workspace:*",
|
||||
"@hivegrid/queue": "workspace:*",
|
||||
"@hivegrid/search": "workspace:*",
|
||||
"@hivegrid/shared-store": "workspace:*",
|
||||
"@hivegrid/storage": "workspace:*",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.76.0",
|
||||
"@opentelemetry/core": "^2.7.1",
|
||||
@@ -25,26 +46,6 @@
|
||||
"@opentelemetry/sdk-trace-base": "^2.7.1",
|
||||
"@opentelemetry/semantic-conventions": "^1.40.0",
|
||||
"@sentry/node": "^10.53.1",
|
||||
"@toolctrl/config": "workspace:*",
|
||||
"@toolctrl/db": "workspace:*",
|
||||
"@toolctrl/email-sequences": "workspace:*",
|
||||
"@toolctrl/email-templates": "workspace:*",
|
||||
"@toolctrl/entitlements": "workspace:*",
|
||||
"@toolctrl/error-reporter": "workspace:*",
|
||||
"@toolctrl/exports": "workspace:*",
|
||||
"@toolctrl/form-widget": "workspace:*",
|
||||
"@toolctrl/foundation": "workspace:*",
|
||||
"@toolctrl/imports": "workspace:*",
|
||||
"@toolctrl/logger": "workspace:*",
|
||||
"@toolctrl/mail": "workspace:*",
|
||||
"@toolctrl/marketing-outbound": "workspace:*",
|
||||
"@toolctrl/metrics": "workspace:*",
|
||||
"@toolctrl/observability": "workspace:*",
|
||||
"@toolctrl/plugin-runtime": "workspace:*",
|
||||
"@toolctrl/queue": "workspace:*",
|
||||
"@toolctrl/search": "workspace:*",
|
||||
"@toolctrl/shared-store": "workspace:*",
|
||||
"@toolctrl/storage": "workspace:*",
|
||||
"bullmq": "^5.76.5",
|
||||
"clamscan": "^2.4.0",
|
||||
"heic-convert": "^2.1.0",
|
||||
@@ -54,8 +55,7 @@
|
||||
"pino": "^9.0.0",
|
||||
"prom-client": "^15.1.3",
|
||||
"semver": "^7.7.3",
|
||||
"sharp": "^0.35.2",
|
||||
"stripe": "^19.1.0"
|
||||
"sharp": "^0.35.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^25.0.0",
|
||||
|
||||
@@ -6,11 +6,11 @@ const externalBarePackagesPlugin = {
|
||||
name: "external-bare-packages",
|
||||
setup(builder) {
|
||||
builder.onResolve({ filter: /^[^./]|^\.[^./]|^\.\.[^/]/ }, (args) => {
|
||||
if (args.path === "@toolctrl/queue/runtime" || args.path.startsWith("@toolctrl/queue/runtime/")) {
|
||||
if (args.path === "@hivegrid/queue/runtime" || args.path.startsWith("@hivegrid/queue/runtime/")) {
|
||||
return { path: args.path, external: true };
|
||||
}
|
||||
|
||||
if (args.path.startsWith("@toolctrl/")) {
|
||||
if (args.path.startsWith("@hivegrid/")) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ await build({
|
||||
target: "node20",
|
||||
sourcemap,
|
||||
banner: {
|
||||
js: 'import { createRequire as __toolctrlCreateRequire } from "node:module"; const require = __toolctrlCreateRequire(import.meta.url);',
|
||||
js: 'import { createRequire as __hivegridCreateRequire } from "node:module"; const require = __hivegridCreateRequire(import.meta.url);',
|
||||
},
|
||||
logLevel: "warning",
|
||||
plugins: [externalBarePackagesPlugin],
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { createMetricsRegistry, createDefaultMetrics, type MetricsRegistry, type DefaultMetrics } from "@toolctrl/metrics";
|
||||
import { createMetricsRegistry, createDefaultMetrics, type MetricsRegistry, type DefaultMetrics } from "@hivegrid/metrics";
|
||||
import { createWorkerAdminHandler, type WorkerStatusReport } from "./admin-server.js";
|
||||
|
||||
let registry: MetricsRegistry;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
* file descriptor budget.
|
||||
*/
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import { isMetricsEnabled, validateMetricsTokenAuth, type MetricsRegistry } from "@toolctrl/metrics";
|
||||
import { isMetricsEnabled, validateMetricsTokenAuth, type MetricsRegistry } from "@hivegrid/metrics";
|
||||
|
||||
export type WorkerStatusCheckOutcome = "ok" | "down" | "disabled";
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import { parseRedisUrl, type ConnectionOptions, type WorkerQueueGroup } from "@toolctrl/queue";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import { parseRedisUrl, type ConnectionOptions, type WorkerQueueGroup } from "@hivegrid/queue";
|
||||
|
||||
/**
|
||||
* Per-group default Redis connection pool size. BullMQ uses ioredis under
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { withTenantRLS, type RlsTransactionExecutor, type TransactionCapableClient } from "./tenantRls.js";
|
||||
|
||||
/**
|
||||
* Build a fake transaction-capable client whose `$transaction` immediately
|
||||
* yields a tx exposing a spied `$executeRaw`. Records the interpolated GUC
|
||||
* values so tests can assert the `app.tenant_id` set_config call.
|
||||
*/
|
||||
function createFakeClient() {
|
||||
const guc: unknown[] = [];
|
||||
const tx: RlsTransactionExecutor = {
|
||||
$executeRaw: vi.fn(async (_strings: TemplateStringsArray, ...values: unknown[]) => {
|
||||
guc.push(values[0]);
|
||||
return 1;
|
||||
}),
|
||||
};
|
||||
const client: TransactionCapableClient<RlsTransactionExecutor> = {
|
||||
// vi.fn cannot represent the generic `<T>(fn) => Promise<T>` call signature the
|
||||
// interface requires, so assert the mock to the method's type. Runtime behavior
|
||||
// (run the callback against the same tx) is unchanged.
|
||||
$transaction: vi.fn(async <T>(fn: (tx: RlsTransactionExecutor) => Promise<T>) => fn(tx)) as TransactionCapableClient<RlsTransactionExecutor>["$transaction"],
|
||||
};
|
||||
return { client, tx, guc };
|
||||
}
|
||||
|
||||
describe("withTenantRLS", () => {
|
||||
it("rejects an empty tenantId without opening a transaction", async () => {
|
||||
const { client } = createFakeClient();
|
||||
const callback = vi.fn(async () => "unreachable");
|
||||
|
||||
await expect(withTenantRLS(client, "", callback)).rejects.toThrow(
|
||||
"withTenantRLS requires a non-empty tenantId"
|
||||
);
|
||||
|
||||
expect(client.$transaction).not.toHaveBeenCalled();
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("sets app.tenant_id before running the callback and returns its value", async () => {
|
||||
const { client, tx, guc } = createFakeClient();
|
||||
const order: string[] = [];
|
||||
|
||||
const result = await withTenantRLS(client, "tenant-x", async (received) => {
|
||||
order.push("callback");
|
||||
// The callback receives the same tx the GUC was set on.
|
||||
expect(received).toBe(tx);
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
expect(result).toEqual({ ok: true });
|
||||
// GUC set with the tenant id, exactly once, before the callback ran.
|
||||
expect(guc).toEqual(["tenant-x"]);
|
||||
expect(tx.$executeRaw).toHaveBeenCalledTimes(1);
|
||||
expect(order).toEqual(["callback"]);
|
||||
});
|
||||
|
||||
it("propagates errors thrown inside the callback", async () => {
|
||||
const { client } = createFakeClient();
|
||||
const boom = new Error("callback exploded");
|
||||
|
||||
await expect(
|
||||
withTenantRLS(client, "tenant-y", async () => {
|
||||
throw boom;
|
||||
})
|
||||
).rejects.toBe(boom);
|
||||
|
||||
// The transaction was still opened (the throw happens inside it).
|
||||
expect(client.$transaction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("runs the GUC set and the callback inside the SAME transaction", async () => {
|
||||
const { client, tx } = createFakeClient();
|
||||
|
||||
await withTenantRLS(client, "tenant-z", async (received) => {
|
||||
// $executeRaw was already invoked on this very tx before the callback.
|
||||
expect(tx.$executeRaw).toHaveBeenCalledTimes(1);
|
||||
expect(received).toBe(tx);
|
||||
return null;
|
||||
});
|
||||
|
||||
expect(client.$transaction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@
|
||||
// (apps/web/app/utils/db/.server/tenantRls.server.ts). The worker cannot import
|
||||
// the web helper (it is bound to the web `db` singleton), so it gets its own thin
|
||||
// wrapper that takes the Prisma client as a parameter (the worker uses
|
||||
// getDbClient() from @toolctrl/db/client.server).
|
||||
// getDbClient() from @hivegrid/db/client.server).
|
||||
//
|
||||
// Sets the tenant context TRANSACTION-LOCALLY (set_config(..., true) == SET LOCAL,
|
||||
// PgBouncer transaction-pooling safe) and runs the queries inside that same
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { queueNames } from "@toolctrl/queue";
|
||||
import { queueNames } from "@hivegrid/queue";
|
||||
import {
|
||||
DEFAULT_GROUP_CONCURRENCY,
|
||||
InvalidWorkerQueueGroupError,
|
||||
@@ -42,16 +42,16 @@ describe("resolveQueueNamesForGroup", () => {
|
||||
expect(resolveQueueNamesForGroup("mail")).toEqual([queueNames.email]);
|
||||
});
|
||||
|
||||
test("general group includes notifications + maintenance + fileScan", () => {
|
||||
expect(resolveQueueNamesForGroup("general")).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan]);
|
||||
test("general group includes notifications + maintenance + fileScan + filePreview", () => {
|
||||
expect(resolveQueueNamesForGroup("general")).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
|
||||
});
|
||||
|
||||
test("import group includes imports + exports", () => {
|
||||
expect(resolveQueueNamesForGroup("import")).toEqual([queueNames.imports, queueNames.exports]);
|
||||
});
|
||||
|
||||
test("webhook group covers AI + browser automation", () => {
|
||||
expect(resolveQueueNamesForGroup("webhook")).toEqual([queueNames.aiChatbot, queueNames.browserAutomation]);
|
||||
test("webhook group covers browser automation", () => {
|
||||
expect(resolveQueueNamesForGroup("webhook")).toEqual([queueNames.browserAutomation]);
|
||||
});
|
||||
|
||||
test("plugin group resolves to plugin-invoke queue", () => {
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import {
|
||||
WORKER_QUEUE_GROUP_NAMES,
|
||||
getQueueNamesForGroup,
|
||||
isWorkerQueueGroup,
|
||||
type QueueName,
|
||||
type WorkerQueueGroup,
|
||||
} from "@toolctrl/queue";
|
||||
} from "@hivegrid/queue";
|
||||
|
||||
/**
|
||||
* Default per-group concurrency. These are deliberate — they reflect the
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import {
|
||||
buildAiBrowserTaskJob,
|
||||
buildAiChatCompletionJob,
|
||||
buildAiEmbeddingJob,
|
||||
buildBrowserSessionCleanupJob,
|
||||
buildBrowserSessionRunJob,
|
||||
buildBrowserUsageFlushJob,
|
||||
@@ -14,14 +11,16 @@ import {
|
||||
buildTransactionalMailJob,
|
||||
queueNames,
|
||||
type WebhookDeliveryJobDescriptor,
|
||||
} from "@toolctrl/queue";
|
||||
} from "@hivegrid/queue";
|
||||
import { buildHandlerRegistry, type HandlerDependencies } from "./handlers.js";
|
||||
import type { CreateNotificationDeliveryInput } from "@toolctrl/db/notifications.server";
|
||||
import { MailProviderError, type MailProvider } from "@toolctrl/mail";
|
||||
import type { StorageScanResult } from "@toolctrl/storage";
|
||||
import { PRIVACY_GDPR_USER_SELF_EXPORT_TYPE } from "./jobs/privacy-gdpr-export.js";
|
||||
import type { CreateNotificationDeliveryInput } from "@hivegrid/db/notifications.server";
|
||||
import { MailProviderError, type MailProvider } from "@hivegrid/mail";
|
||||
import type { StorageScanResult } from "@hivegrid/storage";
|
||||
|
||||
type TestScanService = NonNullable<HandlerDependencies["scanService"]>;
|
||||
type TestExportService = NonNullable<HandlerDependencies["exportService"]>;
|
||||
type TestPrivacyGdprExportService = NonNullable<HandlerDependencies["privacyGdprExportService"]>;
|
||||
type TestImportService = NonNullable<HandlerDependencies["importService"]>;
|
||||
type TestStorageDeleteService = NonNullable<HandlerDependencies["storageDeleteService"]>;
|
||||
type TestRetentionListInput = Parameters<NonNullable<HandlerDependencies["listSoftDeletedExportJobsForHardDelete"]>>[0];
|
||||
@@ -51,9 +50,9 @@ describe("worker handlers", () => {
|
||||
cc: [{ email: "cc@example.com", name: "CC" }],
|
||||
bcc: ["bcc@example.com"],
|
||||
replyTo: "reply@example.com",
|
||||
subject: "Welcome to ToolCTRL",
|
||||
text: "Hello from ToolCTRL",
|
||||
html: "<p>Hello from <strong>ToolCTRL</strong></p>",
|
||||
subject: "Welcome to HiveGrid",
|
||||
text: "Hello from HiveGrid",
|
||||
html: "<p>Hello from <strong>HiveGrid</strong></p>",
|
||||
templateId: "tmpl_welcome",
|
||||
attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }],
|
||||
tags: ["welcome"],
|
||||
@@ -87,9 +86,9 @@ describe("worker handlers", () => {
|
||||
cc: [{ email: "cc@example.com", name: "CC" }],
|
||||
bcc: ["bcc@example.com"],
|
||||
replyTo: "reply@example.com",
|
||||
subject: "Welcome to ToolCTRL",
|
||||
text: "Hello from ToolCTRL",
|
||||
html: "<p>Hello from <strong>ToolCTRL</strong></p>",
|
||||
subject: "Welcome to HiveGrid",
|
||||
text: "Hello from HiveGrid",
|
||||
html: "<p>Hello from <strong>HiveGrid</strong></p>",
|
||||
templateId: "tmpl_welcome",
|
||||
attachments: [{ content: "Zm9v", filename: "foo.txt", disposition: "attachment" }],
|
||||
tags: ["welcome"],
|
||||
@@ -115,7 +114,7 @@ describe("worker handlers", () => {
|
||||
const registry = buildHandlerRegistry({ mailProvider });
|
||||
const descriptor = buildTransactionalMailJob({
|
||||
to: ["user@example.com", "admin@example.com"],
|
||||
subject: "Welcome to ToolCTRL",
|
||||
subject: "Welcome to HiveGrid",
|
||||
templateId: "tmpl_welcome",
|
||||
templateData: {
|
||||
"user@example.com": { firstName: "Ada" },
|
||||
@@ -129,7 +128,7 @@ describe("worker handlers", () => {
|
||||
});
|
||||
expect(mailProvider.send).toHaveBeenCalledWith({
|
||||
to: ["user@example.com", "admin@example.com"],
|
||||
subject: "Welcome to ToolCTRL",
|
||||
subject: "Welcome to HiveGrid",
|
||||
text: undefined,
|
||||
html: undefined,
|
||||
templateId: "tmpl_welcome",
|
||||
@@ -1607,39 +1606,6 @@ describe("worker handlers", () => {
|
||||
expect(listSoftDeletedExportJobsForHardDelete).toHaveBeenCalledWith({ now: expect.any(Date), take: undefined });
|
||||
});
|
||||
|
||||
test("accepts descriptor-derived AI jobs through registry", async () => {
|
||||
const registry = buildHandlerRegistry();
|
||||
const descriptors = [
|
||||
buildAiChatCompletionJob({
|
||||
tenantId: "tenant_123",
|
||||
userId: "user_123",
|
||||
conversationId: "conversation_123",
|
||||
messages: [{ role: "user", content: "fixture prompt not surfaced" }],
|
||||
}),
|
||||
buildAiEmbeddingJob({
|
||||
tenantId: "tenant_123",
|
||||
userId: "user_123",
|
||||
model: "fixture-embedding-model",
|
||||
task: { text: "fixture embedding text not surfaced" },
|
||||
}),
|
||||
buildAiBrowserTaskJob({
|
||||
tenantId: "tenant_123",
|
||||
userId: "user_123",
|
||||
task: { instruction: "fixture browser task not surfaced" },
|
||||
}),
|
||||
];
|
||||
|
||||
await Promise.all(
|
||||
descriptors.map((descriptor) =>
|
||||
expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
||||
action: "worker.unsupported",
|
||||
message: "AI worker job is not configured.",
|
||||
details: { queueName: queueNames.aiChatbot, kind: descriptor.payload.kind, tenantId: "tenant_123", userId: "user_123" },
|
||||
})
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects descriptor-derived browser automation jobs through registry without global fetch", async () => {
|
||||
const fetchSpy = vi.spyOn(globalThis, "fetch");
|
||||
const registry = buildHandlerRegistry();
|
||||
@@ -1858,6 +1824,112 @@ describe("worker handlers", () => {
|
||||
expect(JSON.stringify(markExportJobSucceeded.mock.calls)).not.toContain("payload-secret");
|
||||
});
|
||||
|
||||
test("routes privacy GDPR export jobs through the GDPR service and status table", async () => {
|
||||
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
|
||||
const exportService: TestExportService = {
|
||||
runExport: vi.fn(async () => ({ artifactBucket: "unused", artifactKey: "unused", rowCount: 0, expiresAt })),
|
||||
};
|
||||
const privacyGdprExportService: TestPrivacyGdprExportService = {
|
||||
runExport: vi.fn(async () => ({
|
||||
artifactBucket: "exports",
|
||||
artifactKey: "tenants/tenant_1/gdpr/export_job_1.zip",
|
||||
rowCount: 7,
|
||||
sizeBytes: 4096,
|
||||
expiresAt,
|
||||
})),
|
||||
};
|
||||
const markGdprExportProcessing = vi.fn(async () => undefined);
|
||||
const markGdprExportReady = vi.fn(async () => undefined);
|
||||
const markGdprExportFailed = vi.fn(async () => undefined);
|
||||
const markExportJobRunning = vi.fn(async () => undefined);
|
||||
const registry = buildHandlerRegistry({
|
||||
exportService,
|
||||
privacyGdprExportService,
|
||||
markGdprExportProcessing,
|
||||
markGdprExportReady,
|
||||
markGdprExportFailed,
|
||||
markExportJobRunning,
|
||||
});
|
||||
const descriptor = buildExportJob({
|
||||
exportJobId: "gdpr_export_job_1",
|
||||
tenantId: "tenant_1",
|
||||
requestedByUserId: "user_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
format: "json",
|
||||
requestId: "req_gdpr_export_success",
|
||||
metadata: { attemptsMade: 1, token: "payload-secret" },
|
||||
});
|
||||
|
||||
await expect(registry[descriptor.queueName](descriptor.payload, {})).resolves.toEqual({
|
||||
handled: true,
|
||||
action: "privacy-gdpr-export.completed",
|
||||
details: {
|
||||
exportJobId: "gdpr_export_job_1",
|
||||
tenantId: "tenant_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
format: "json",
|
||||
requestId: "req_gdpr_export_success",
|
||||
artifactBucket: "exports",
|
||||
artifactKey: "tenants/tenant_1/gdpr/export_job_1.zip",
|
||||
attemptsMade: 1,
|
||||
expiresAt: "2026-05-09T00:00:00.000Z",
|
||||
rowCount: 7,
|
||||
sizeBytes: 4096,
|
||||
status: "ready",
|
||||
},
|
||||
});
|
||||
expect(markGdprExportProcessing).toHaveBeenCalledWith({ id: "gdpr_export_job_1", tenantId: "tenant_1" });
|
||||
expect(privacyGdprExportService.runExport).toHaveBeenCalledWith(descriptor.payload);
|
||||
expect(markGdprExportReady).toHaveBeenCalledWith({
|
||||
id: "gdpr_export_job_1",
|
||||
tenantId: "tenant_1",
|
||||
resultBucket: "exports",
|
||||
resultKey: "tenants/tenant_1/gdpr/export_job_1.zip",
|
||||
resultSizeBytes: 4096,
|
||||
resultExpiresAt: expiresAt,
|
||||
});
|
||||
expect(markGdprExportFailed).not.toHaveBeenCalled();
|
||||
expect(exportService.runExport).not.toHaveBeenCalled();
|
||||
expect(markExportJobRunning).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("fails privacy GDPR export jobs when the dedicated service is missing", async () => {
|
||||
const markGdprExportProcessing = vi.fn(async () => undefined);
|
||||
const markGdprExportFailed = vi.fn(async () => undefined);
|
||||
const markExportJobRunning = vi.fn(async () => undefined);
|
||||
const registry = buildHandlerRegistry({ markGdprExportProcessing, markGdprExportFailed, markExportJobRunning });
|
||||
const descriptor = buildExportJob({
|
||||
exportJobId: "gdpr_export_job_missing_service",
|
||||
tenantId: "tenant_1",
|
||||
requestedByUserId: "user_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
format: "json",
|
||||
requestId: "req_gdpr_export_missing_service",
|
||||
});
|
||||
|
||||
await expect(registry[descriptor.queueName](descriptor.payload, {})).rejects.toMatchObject({
|
||||
action: "export.error",
|
||||
message: "Privacy GDPR export service is not configured.",
|
||||
details: {
|
||||
exportJobId: "gdpr_export_job_missing_service",
|
||||
tenantId: "tenant_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
format: "json",
|
||||
requestId: "req_gdpr_export_missing_service",
|
||||
attemptsMade: 0,
|
||||
status: "failed",
|
||||
code: "privacy_gdpr_export_service_missing",
|
||||
},
|
||||
});
|
||||
expect(markGdprExportProcessing).toHaveBeenCalledWith({ id: "gdpr_export_job_missing_service", tenantId: "tenant_1" });
|
||||
expect(markGdprExportFailed).toHaveBeenCalledWith({
|
||||
id: "gdpr_export_job_missing_service",
|
||||
tenantId: "tenant_1",
|
||||
errorMessage: "Privacy GDPR export service is not configured.",
|
||||
});
|
||||
expect(markExportJobRunning).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("uses BullMQ attempt metadata for export job persistence when available", async () => {
|
||||
const expiresAt = new Date("2026-05-09T00:00:00.000Z");
|
||||
const exportService: TestExportService = {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {
|
||||
updateLegacyEventWebhookAttempt as updateLegacyEventWebhookAttemptInDb,
|
||||
type LegacyEventWebhookAttemptUpdate,
|
||||
} from "@toolctrl/db/event-webhook-attempts.server";
|
||||
} from "@hivegrid/db/event-webhook-attempts.server";
|
||||
import {
|
||||
listExpiredCompletedExportJobs as listExpiredCompletedExportJobsInDb,
|
||||
listSoftDeletedExportJobsForHardDelete as listSoftDeletedExportJobsForHardDeleteInDb,
|
||||
@@ -23,7 +23,15 @@ import {
|
||||
type ExportJobRetentionRecord,
|
||||
type ExportJobRunningInput,
|
||||
type ExportJobSucceededInput,
|
||||
} from "@toolctrl/db/export-jobs.server";
|
||||
} from "@hivegrid/db/export-jobs.server";
|
||||
import {
|
||||
markGdprExportFailed as markGdprExportFailedInDb,
|
||||
markGdprExportProcessing as markGdprExportProcessingInDb,
|
||||
markGdprExportReady as markGdprExportReadyInDb,
|
||||
type MarkGdprExportFailedInput,
|
||||
type MarkGdprExportProcessingInput,
|
||||
type MarkGdprExportReadyInput,
|
||||
} from "@hivegrid/db/privacy-gdpr.server";
|
||||
import {
|
||||
markImportJobFailed as markImportJobFailedInDb,
|
||||
markImportJobRunning as markImportJobRunningInDb,
|
||||
@@ -31,7 +39,7 @@ import {
|
||||
type ImportJobFailedInput,
|
||||
type ImportJobRunningInput,
|
||||
type ImportJobSucceededInput,
|
||||
} from "@toolctrl/db/import-jobs.server";
|
||||
} from "@hivegrid/db/import-jobs.server";
|
||||
import {
|
||||
createNotificationDelivery as createNotificationDeliveryInDb,
|
||||
getUserNotificationPreferences as getUserNotificationPreferencesInDb,
|
||||
@@ -39,15 +47,15 @@ import {
|
||||
redactNotificationDiagnosticText,
|
||||
type CreateNotificationDeliveryInput,
|
||||
type MarkNotificationDeliveryStatusInput,
|
||||
} from "@toolctrl/db/notifications.server";
|
||||
} from "@hivegrid/db/notifications.server";
|
||||
import {
|
||||
deleteDoneOutboxEvents as deleteDoneOutboxEventsInDb,
|
||||
type DeleteDoneOutboxEventsOptions,
|
||||
type DeleteDoneOutboxEventsResult,
|
||||
} from "@toolctrl/db/outbox-retention";
|
||||
import { type EnvRecord } from "@toolctrl/config";
|
||||
import type { ValidationReport } from "@toolctrl/imports";
|
||||
import { createMailjetProviderFromEnv, type MailMetricsHook, type MailProvider } from "@toolctrl/mail";
|
||||
} from "@hivegrid/db/outbox-retention";
|
||||
import { type EnvRecord } from "@hivegrid/config";
|
||||
import type { ValidationReport } from "@hivegrid/imports";
|
||||
import { createMailjetProviderFromEnv, type MailMetricsHook, type MailProvider } from "@hivegrid/mail";
|
||||
import {
|
||||
queueNames,
|
||||
type ExportJob,
|
||||
@@ -57,20 +65,38 @@ import {
|
||||
type QueueName,
|
||||
type QueuePayload,
|
||||
type TypedProcessor,
|
||||
} from "@toolctrl/queue";
|
||||
} from "@hivegrid/queue";
|
||||
import {
|
||||
authorizeStorageObjectAccess,
|
||||
type DeleteObjectInput,
|
||||
type DeleteObjectResult,
|
||||
type StorageScanDiagnostic,
|
||||
type StorageScanResult,
|
||||
} from "@toolctrl/storage";
|
||||
import { mapWithConcurrency, notify, type NotificationClient } from "@toolctrl/foundation";
|
||||
import { OutboundHttpUrlValidationError, fetchOutboundHttpWithRedirects, type OutboundHttpAddressResolver } from "@toolctrl/foundation/outbound-http";
|
||||
} from "@hivegrid/storage";
|
||||
import { mapWithConcurrency, notify, type NotificationClient } from "@hivegrid/foundation";
|
||||
import { OutboundHttpUrlValidationError, fetchOutboundHttpWithRedirects, type OutboundHttpAddressResolver } from "@hivegrid/foundation/outbound-http";
|
||||
|
||||
import {
|
||||
drainCalendarWebhookEventsForTenant,
|
||||
refreshCalendarTokensForTenant,
|
||||
renewCalendarWebhooksForTenant,
|
||||
syncInternalCalendarEventsForTenant,
|
||||
} from "@hivegrid/integrations/calendar-worker-tasks.server";
|
||||
import {
|
||||
cleanupEmailWebhookEventsForTenant,
|
||||
drainEmailWebhookEventsForTenant,
|
||||
pollStaleEmailMailboxesForTenant,
|
||||
refreshEmailTokensForTenant,
|
||||
renewEmailWebhooksForTenant,
|
||||
runEmailInitialSyncForTenant,
|
||||
} from "@hivegrid/integrations/email-worker-tasks.server";
|
||||
import { createEmailAutoMatchHook } from "@hivegrid/integrations/email-match-hook.server";
|
||||
import { refreshExpiringTokensForTenant, sweepConnectionHealthForTenant } from "@hivegrid/integrations/worker-tasks.server";
|
||||
import { withTenantRLS } from "./db/tenantRls.js";
|
||||
import { runDsgvoGraceCron, type DsgvoGraceCandidate } from "./jobs/dsgvo-grace-cron.js";
|
||||
import { reminderChannel, reminderMessageKey, reminderScanBounds, reminderSourceKey, selectReminderWindows } from "./jobs/task-reminder-cron.js";
|
||||
import { audienceMatches, nextReminderDue, HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS } from "./jobs/hr-upload-reminder-cron.js";
|
||||
import { createDnsDomainResolver, decideDomainProbeStatus, type DomainDnsResolver } from "./jobs/email-domain-probe-cron.js";
|
||||
import { handlePluginInvokeJob, type PluginInvokeDeps } from "./jobs/plugin-invoke.js";
|
||||
import {
|
||||
reconcileWorkflowExecutionStalls,
|
||||
@@ -78,6 +104,10 @@ import {
|
||||
type WorkflowExecutionStalledReconcileResult,
|
||||
type WorkflowExecutionStalledReconcilerPrisma,
|
||||
} from "./jobs/workflow-execution-stalled-reconciler.js";
|
||||
import {
|
||||
isPrivacyGdprExportType,
|
||||
type PrivacyGdprExportRunResult,
|
||||
} from "./jobs/privacy-gdpr-export.js";
|
||||
|
||||
const DEFAULT_WORKER_FETCH_TIMEOUT_MS = 10_000;
|
||||
const WEBHOOK_RESPONSE_BODY_LIMIT_BYTES = 4_096;
|
||||
@@ -141,6 +171,9 @@ export type HandlerDependencies = {
|
||||
exportService?: {
|
||||
runExport(payload: ExportJob): Promise<WorkerExportRunResult>;
|
||||
};
|
||||
privacyGdprExportService?: {
|
||||
runExport(payload: ExportJob): Promise<PrivacyGdprExportRunResult>;
|
||||
};
|
||||
importService?: {
|
||||
runImport(payload: ImportJob): Promise<WorkerImportRunResult>;
|
||||
};
|
||||
@@ -157,6 +190,9 @@ export type HandlerDependencies = {
|
||||
markExportJobRunning?: (data: ExportJobRunningInput) => Promise<unknown>;
|
||||
markExportJobSucceeded?: (data: ExportJobSucceededInput) => Promise<unknown>;
|
||||
markExportJobFailed?: (data: ExportJobFailedInput) => Promise<unknown>;
|
||||
markGdprExportProcessing?: (data: MarkGdprExportProcessingInput) => Promise<unknown>;
|
||||
markGdprExportReady?: (data: MarkGdprExportReadyInput) => Promise<unknown>;
|
||||
markGdprExportFailed?: (data: MarkGdprExportFailedInput) => Promise<unknown>;
|
||||
markImportJobRunning?: (data: ImportJobRunningInput) => Promise<unknown>;
|
||||
markImportJobSucceeded?: (data: ImportJobSucceededInput) => Promise<unknown>;
|
||||
markImportJobFailed?: (data: ImportJobFailedInput) => Promise<unknown>;
|
||||
@@ -490,7 +526,10 @@ export function buildHandlerRegistry(dependencies: HandlerDependencies = {}): Ha
|
||||
};
|
||||
}
|
||||
|
||||
const fetchImpl = deps.fetch ?? fetch;
|
||||
// Only an injected (test) fetch is passed through; in production we leave
|
||||
// it undefined so the outbound guard uses its own undici fetch with the
|
||||
// DNS-rebinding IP pin engaged (see fetchWebhookWithRedirects).
|
||||
const fetchImpl = deps.fetch;
|
||||
const updateLegacyEventWebhookAttempt = deps.updateLegacyEventWebhookAttempt ?? updateLegacyEventWebhookAttemptInDb;
|
||||
const legacyAttemptId = typeof payload.metadata?.legacyAttemptId === "string" ? payload.metadata.legacyAttemptId : undefined;
|
||||
const legacyAttemptTenantId = typeof payload.metadata?.legacyAttemptTenantId === "string" ? payload.metadata.legacyAttemptTenantId : null;
|
||||
@@ -787,10 +826,70 @@ export function buildHandlerRegistry(dependencies: HandlerDependencies = {}): Ha
|
||||
return runTaskArchiveRetentionHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "lead-retention-sweep") {
|
||||
return runLeadRetentionHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "task-reminder-sweep") {
|
||||
return runTaskReminderSweepHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "integrations-token-refresh") {
|
||||
return runIntegrationsTokenRefreshHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "integrations-health-sweep") {
|
||||
return runIntegrationsHealthSweepHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "calendar-token-refresh") {
|
||||
return runCalendarTokenRefreshHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "calendar-webhook-renewal") {
|
||||
return runCalendarWebhookRenewalHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "calendar-webhook-drain") {
|
||||
return runCalendarWebhookDrainHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "calendar-internal-write-sync") {
|
||||
return runCalendarInternalWriteSyncHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "email-inbox-token-refresh") {
|
||||
return runEmailInboxTokenRefreshHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "email-inbox-webhook-renewal") {
|
||||
return runEmailInboxWebhookRenewalHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "email-inbox-webhook-drain") {
|
||||
return runEmailInboxWebhookDrainHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "email-inbox-initial-sync") {
|
||||
return runEmailInboxInitialSyncHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "email-inbox-stale-poll") {
|
||||
return runEmailInboxStalePollHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "email-inbox-event-cleanup") {
|
||||
return runEmailInboxEventCleanupHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "hr-upload-reminder-dispatch") {
|
||||
return runHrUploadReminderDispatchHandler(payload, deps);
|
||||
}
|
||||
|
||||
if (payload.task === "email-domain-probe") {
|
||||
return runEmailDomainProbeHandler(payload, deps);
|
||||
}
|
||||
|
||||
throw new UnsupportedWorkerJobError("Unsupported maintenance task.", {
|
||||
queueName: queueNames.maintenance,
|
||||
task: payload.task,
|
||||
@@ -798,14 +897,6 @@ export function buildHandlerRegistry(dependencies: HandlerDependencies = {}): Ha
|
||||
olderThanMinutes: payload.olderThanMinutes,
|
||||
});
|
||||
},
|
||||
[queueNames.aiChatbot]: async (payload) => {
|
||||
throw new UnsupportedWorkerJobError("AI worker job is not configured.", {
|
||||
queueName: queueNames.aiChatbot,
|
||||
kind: payload.kind,
|
||||
tenantId: payload.tenantId,
|
||||
userId: payload.userId,
|
||||
});
|
||||
},
|
||||
[queueNames.browserAutomation]: async (payload) => {
|
||||
throw new UnsupportedWorkerJobError("Browser automation worker job is not configured.", {
|
||||
queueName: queueNames.browserAutomation,
|
||||
@@ -1147,7 +1238,7 @@ async function runWorkflowExecutionStalledReconcileHandler(
|
||||
}
|
||||
|
||||
async function getDefaultDbClient() {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
return getDbClient();
|
||||
}
|
||||
|
||||
@@ -1190,6 +1281,284 @@ async function runTaskArchiveRetentionHandler(payload: QueuePayload<typeof queue
|
||||
};
|
||||
}
|
||||
|
||||
async function runLeadRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let archived = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const dueIds = await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.inboxLead.findMany({
|
||||
where: { tenantId, archivedAt: null, autoDeleteAt: { lte: now } },
|
||||
select: { id: true },
|
||||
orderBy: { autoDeleteAt: "asc" },
|
||||
take,
|
||||
}),
|
||||
);
|
||||
if (dueIds.length === 0) continue;
|
||||
const result = await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.inboxLead.updateMany({
|
||||
where: { tenantId, id: { in: dueIds.map((row) => row.id) }, archivedAt: null },
|
||||
data: { archivedAt: now },
|
||||
}),
|
||||
);
|
||||
archived += result.count;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, archived }, "leads.retention.completed");
|
||||
return {
|
||||
handled: true,
|
||||
action: "leads.retention.completed",
|
||||
details: { task: payload.task, tenants: tenants.length, archived, take },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* R-P5 §P2d — refresh OAuth tokens nearing expiry across all tenants. Enumerates
|
||||
* active tenants from the global registry (not RLS-scoped) and runs the scoped
|
||||
* refresh per tenant inside withTenantRLS() (so the policy binds at cutover). The
|
||||
* per-connection logic (decrypt → adapter.refreshToken → re-encrypt) lives in
|
||||
* @hivegrid/integrations/worker-tasks.server, shared with the web side.
|
||||
*/
|
||||
async function runIntegrationsTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let refreshed = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => refreshExpiringTokensForTenant(tx as unknown as Parameters<typeof refreshExpiringTokensForTenant>[0], tenantId, { now, windowMs }));
|
||||
refreshed += result.refreshed;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "integrations.token-refresh.completed");
|
||||
return { handled: true, action: "integrations.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
|
||||
}
|
||||
|
||||
/**
|
||||
* R-P5 §P2d — probe every active connection's health across all tenants, persist
|
||||
* the status, and notify owners on degraded/broken. Same per-tenant RLS sweep
|
||||
* pattern; per-connection logic shared via worker-tasks.server.
|
||||
*/
|
||||
async function runIntegrationsHealthSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let checked = 0;
|
||||
let unhealthy = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => sweepConnectionHealthForTenant(tx as unknown as Parameters<typeof sweepConnectionHealthForTenant>[0], tenantId, { now }));
|
||||
checked += result.checked;
|
||||
unhealthy += result.unhealthy;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, checked, unhealthy }, "integrations.health-sweep.completed");
|
||||
return { handled: true, action: "integrations.health-sweep.completed", details: { task: payload.task, tenants: tenants.length, checked, unhealthy } };
|
||||
}
|
||||
|
||||
async function runCalendarTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let refreshed = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => refreshCalendarTokensForTenant(tx as never, tenantId, { now, windowMs }));
|
||||
refreshed += result.refreshed;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "calendar.token-refresh.completed");
|
||||
return { handled: true, action: "calendar.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
|
||||
}
|
||||
|
||||
async function runCalendarWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
|
||||
const webhookUrl = process.env.GOOGLE_CALENDAR_WEBHOOK_URL;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let checked = 0;
|
||||
let renewed = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => renewCalendarWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, webhookUrl }));
|
||||
checked += result.checked;
|
||||
renewed += result.renewed;
|
||||
failed += result.failed;
|
||||
skipped += result.skipped;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped }, "calendar.webhook-renewal.completed");
|
||||
return { handled: true, action: "calendar.webhook-renewal.completed", details: { task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped } };
|
||||
}
|
||||
|
||||
async function runCalendarWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let claimed = 0;
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => drainCalendarWebhookEventsForTenant(tx as never, tenantId, { now, take }));
|
||||
claimed += result.claimed;
|
||||
completed += result.completed;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, claimed, completed, failed }, "calendar.webhook-drain.completed");
|
||||
return { handled: true, action: "calendar.webhook-drain.completed", details: { task: payload.task, tenants: tenants.length, claimed, completed, failed } };
|
||||
}
|
||||
|
||||
async function runCalendarInternalWriteSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let scanned = 0;
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
let deleted = 0;
|
||||
let conflicts = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => syncInternalCalendarEventsForTenant(tx as never, tenantId, { now, take }));
|
||||
scanned += result.scanned;
|
||||
created += result.created;
|
||||
updated += result.updated;
|
||||
deleted += result.deleted;
|
||||
conflicts += result.conflicts;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, scanned, created, updated, deleted, conflicts, failed }, "calendar.internal-write-sync.completed");
|
||||
return {
|
||||
handled: true,
|
||||
action: "calendar.internal-write-sync.completed",
|
||||
details: { task: payload.task, tenants: tenants.length, scanned, created, updated, deleted, conflicts, failed },
|
||||
};
|
||||
}
|
||||
|
||||
async function runEmailInboxTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let refreshed = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => refreshEmailTokensForTenant(tx as never, tenantId, { now, windowMs }));
|
||||
refreshed += result.refreshed;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "email-inbox.token-refresh.completed");
|
||||
return { handled: true, action: "email-inbox.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
|
||||
}
|
||||
|
||||
async function runEmailInboxWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
|
||||
const graphWebhookUrl = process.env.MICROSOFT_MAIL_WEBHOOK_URL;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let checked = 0;
|
||||
let renewed = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => renewEmailWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, graphWebhookUrl }));
|
||||
checked += result.checked;
|
||||
renewed += result.renewed;
|
||||
failed += result.failed;
|
||||
skipped += result.skipped;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped }, "email-inbox.webhook-renewal.completed");
|
||||
return { handled: true, action: "email-inbox.webhook-renewal.completed", details: { task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped } };
|
||||
}
|
||||
|
||||
async function runEmailInboxWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let claimed = 0;
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => drainEmailWebhookEventsForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
|
||||
claimed += result.claimed;
|
||||
completed += result.completed;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, claimed, completed, failed }, "email-inbox.webhook-drain.completed");
|
||||
return { handled: true, action: "email-inbox.webhook-drain.completed", details: { task: payload.task, tenants: tenants.length, claimed, completed, failed } };
|
||||
}
|
||||
|
||||
async function runEmailInboxInitialSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 10;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let folders = 0;
|
||||
let fetched = 0;
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => runEmailInitialSyncForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
|
||||
folders += result.folders;
|
||||
fetched += result.fetched;
|
||||
completed += result.completed;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, folders, fetched, completed, failed }, "email-inbox.initial-sync.completed");
|
||||
return { handled: true, action: "email-inbox.initial-sync.completed", details: { task: payload.task, tenants: tenants.length, folders, fetched, completed, failed } };
|
||||
}
|
||||
|
||||
async function runEmailInboxStalePollHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let polled = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => pollStaleEmailMailboxesForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
|
||||
polled += result.polled;
|
||||
failed += result.failed;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, polled, failed }, "email-inbox.stale-poll.completed");
|
||||
return { handled: true, action: "email-inbox.stale-poll.completed", details: { task: payload.task, tenants: tenants.length, polled, failed } };
|
||||
}
|
||||
|
||||
async function runEmailInboxEventCleanupHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
let deleted = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const result = await withTenantRLS(db, tenantId, (tx) => cleanupEmailWebhookEventsForTenant(tx as never, tenantId, { now }));
|
||||
deleted += result.deleted;
|
||||
}
|
||||
logger.info({ task: payload.task, tenants: tenants.length, deleted }, "email-inbox.event-cleanup.completed");
|
||||
return { handled: true, action: "email-inbox.event-cleanup.completed", details: { task: payload.task, tenants: tenants.length, deleted } };
|
||||
}
|
||||
|
||||
// VA-P23 · auto-match + activity logging seam of the sync engine (M09 Block E/H).
|
||||
const emailInboxMessageHook = createEmailAutoMatchHook();
|
||||
|
||||
/**
|
||||
* P4 Tasks (AK-P4b) §7 — reminder sweep (DoD 4). For each open task with a
|
||||
* deadline in scan range, sends reminders to the assignee (or assigned group's
|
||||
@@ -1281,6 +1650,170 @@ async function runTaskReminderSweepHandler(payload: QueuePayload<typeof queueNam
|
||||
return { handled: true, action: "tasks.reminder-sweep.completed", details: { task: payload.task, tenants: tenants.length, scanned, sent } };
|
||||
}
|
||||
|
||||
/**
|
||||
* P17 HR-Hub (AK-P17b) — upload-reminder dispatch. For every active reminder
|
||||
* past its `dueAt`, fan out per-employee dispatch rows (idempotent via the unique
|
||||
* reminder+employee+dueAt), notify each employee, advance the reminder by its
|
||||
* cadence (or complete a one-off), then escalate overdue mandatory dispatches to
|
||||
* the employee's manager. Per-tenant from the global Tenant registry; all HR
|
||||
* reads/writes run inside withTenantRLS() so the policy binds once activated.
|
||||
* notify() runs outside the tenant transaction (slow I/O), deduped by sourceKey.
|
||||
*/
|
||||
async function runHrUploadReminderDispatchHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const db = await getDefaultDbClient();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
|
||||
let dispatched = 0;
|
||||
let escalated = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const due = await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.employeeUploadReminder.findMany({ where: { tenantId, status: "active", archivedAt: null, dueAt: { not: null, lte: now } } }),
|
||||
);
|
||||
const profiles =
|
||||
due.length > 0
|
||||
? await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.hrEmployeeProfile.findMany({ where: { tenantId }, select: { userId: true, team: true, locationId: true } }),
|
||||
)
|
||||
: [];
|
||||
|
||||
for (const reminder of due) {
|
||||
const dueAt = reminder.dueAt ?? now;
|
||||
const members = profiles.filter((profile) => audienceMatches(reminder.audienceKind, reminder.audienceRef, profile));
|
||||
|
||||
for (const member of members) {
|
||||
const created = await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.employeeUploadReminderDispatch.createMany({
|
||||
data: [{ tenantId, reminderId: reminder.id, employeeUserId: member.userId, dueAt, status: "sent", sentAt: now }],
|
||||
skipDuplicates: true,
|
||||
}),
|
||||
);
|
||||
if (created.count > 0) {
|
||||
dispatched += created.count;
|
||||
await notify(
|
||||
{
|
||||
tenantId,
|
||||
userId: member.userId,
|
||||
channel: "IN_APP",
|
||||
category: "hr",
|
||||
messageKey: "hr.notify.reminder.due",
|
||||
sourceKey: `hr-reminder:${reminder.id}:${dueAt.getTime()}:${member.userId}`,
|
||||
payloadMetadata: { reminderId: reminder.id },
|
||||
},
|
||||
db as unknown as NotificationClient,
|
||||
).catch((err) => logger.error({ task: payload.task, reminderId: reminder.id, userId: member.userId, err: err instanceof Error ? err.message : String(err) }, "hr.upload-reminder.notify-failed"));
|
||||
}
|
||||
}
|
||||
|
||||
const next = nextReminderDue({ cadenceKind: reminder.cadenceKind, intervalMonths: reminder.intervalMonths, from: dueAt });
|
||||
await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.employeeUploadReminder.update({ where: { id: reminder.id }, data: next ? { dueAt: next } : { dueAt: null, status: "completed" } }),
|
||||
);
|
||||
}
|
||||
|
||||
// Escalate still-unacknowledged mandatory dispatches past the grace window.
|
||||
const cutoff = new Date(now.getTime() - HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS);
|
||||
const stale = await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.employeeUploadReminderDispatch.findMany({ where: { tenantId, status: "sent", dueAt: { lt: cutoff }, reminder: { escalateToManager: true, mandatory: true } } }),
|
||||
);
|
||||
const staleEmployeeUserIds = [...new Set(stale.map((dispatch) => dispatch.employeeUserId))];
|
||||
const escalationProfiles =
|
||||
staleEmployeeUserIds.length > 0
|
||||
? await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.hrEmployeeProfile.findMany({
|
||||
where: { tenantId, userId: { in: staleEmployeeUserIds } },
|
||||
select: { userId: true, managerUserId: true },
|
||||
}),
|
||||
)
|
||||
: [];
|
||||
const managerUserIdsByEmployee = new Map(escalationProfiles.map((profile) => [profile.userId, profile.managerUserId]));
|
||||
for (const dispatch of stale) {
|
||||
await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminderDispatch.update({ where: { id: dispatch.id }, data: { status: "escalated", escalatedAt: now } }));
|
||||
escalated += 1;
|
||||
const managerUserId = managerUserIdsByEmployee.get(dispatch.employeeUserId);
|
||||
if (managerUserId) {
|
||||
await notify(
|
||||
{
|
||||
tenantId,
|
||||
userId: managerUserId,
|
||||
channel: "IN_APP",
|
||||
category: "hr",
|
||||
messageKey: "hr.notify.reminder.escalated",
|
||||
sourceKey: `hr-reminder-escalate:${dispatch.id}`,
|
||||
payloadMetadata: { dispatchId: dispatch.id },
|
||||
},
|
||||
db as unknown as NotificationClient,
|
||||
).catch((err) => logger.error({ task: payload.task, dispatchId: dispatch.id, err: err instanceof Error ? err.message : String(err) }, "hr.upload-reminder.escalate-notify-failed"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info({ task: payload.task, tenants: tenants.length, dispatched, escalated }, "hr.upload-reminder-dispatch.completed");
|
||||
return { handled: true, action: "hr.upload-reminder-dispatch.completed", details: { task: payload.task, tenants: tenants.length, dispatched, escalated } };
|
||||
}
|
||||
|
||||
/**
|
||||
* AK-P14a · P14 Email — sending-domain DNS auto-probe (5-minute cadence). For
|
||||
* each active tenant, resolves DKIM/SPF/DMARC for every not-yet-settled sending
|
||||
* domain and advances `verificationStatus` via the pure decideDomainProbeStatus()
|
||||
* helper. Reads/writes run inside withTenantRLS() so the policy binds once the
|
||||
* P5 cutover activates RLS; the domain UPDATE is auto-audited by the M001 Prisma
|
||||
* middleware. A tenant-level notification fires only on the verified/failed
|
||||
* transition (not on every probe). The DNS resolver is the node:dns binding; the
|
||||
* decision logic is unit-tested without DNS/DB.
|
||||
*/
|
||||
async function runEmailDomainProbeHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
||||
const logger = deps.logger ?? noopHandlerLogger;
|
||||
const now = new Date();
|
||||
const db = await getDefaultDbClient();
|
||||
const resolver: DomainDnsResolver = createDnsDomainResolver();
|
||||
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
||||
|
||||
let probed = 0;
|
||||
let verified = 0;
|
||||
let failed = 0;
|
||||
for (const { id: tenantId } of tenants) {
|
||||
const domains = await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.tenantSendingDomain.findMany({ where: { tenantId, archivedAt: null, verificationStatus: { in: ["pending", "verifying"] } } }),
|
||||
);
|
||||
|
||||
for (const domain of domains) {
|
||||
probed += 1;
|
||||
const dns = await resolver({ domain: domain.domain, dkimSelector: domain.dkimSelector });
|
||||
const firstProbeAt = domain.firstProbeAt ?? now;
|
||||
const decision = decideDomainProbeStatus({ dns, firstProbeAt, now });
|
||||
const changed = decision.status !== domain.verificationStatus;
|
||||
|
||||
await withTenantRLS(db, tenantId, (tx) =>
|
||||
tx.tenantSendingDomain.update({
|
||||
where: { id: domain.id },
|
||||
data: { verificationStatus: decision.status, lastError: decision.lastError, lastProbedAt: now, firstProbeAt },
|
||||
}),
|
||||
);
|
||||
|
||||
if (changed && (decision.status === "verified" || decision.status === "failed")) {
|
||||
if (decision.status === "verified") verified += 1;
|
||||
else failed += 1;
|
||||
await notify(
|
||||
{
|
||||
tenantId,
|
||||
channel: "IN_APP",
|
||||
category: "email",
|
||||
messageKey: decision.status === "verified" ? "email.domain.notify.verified" : "email.domain.notify.failed",
|
||||
sourceKey: `email-domain-${decision.status}:${domain.id}`,
|
||||
payloadMetadata: { domainId: domain.id, domain: domain.domain },
|
||||
},
|
||||
db as unknown as NotificationClient,
|
||||
).catch((err) => logger.error({ task: payload.task, domainId: domain.id, err: err instanceof Error ? err.message : String(err) }, "email.domain-probe.notify-failed"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info({ task: payload.task, tenants: tenants.length, probed, verified, failed }, "email.domain-probe.completed");
|
||||
return { handled: true, action: "email.domain-probe.completed", details: { task: payload.task, tenants: tenants.length, probed, verified, failed } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 10 #68 — maintenance-handler adapter for the DSGVO grace-period
|
||||
* scan. Wraps `runDsgvoGraceCron` (the pure unit-tested core) with the
|
||||
@@ -1352,6 +1885,10 @@ async function runExportHandler(
|
||||
});
|
||||
}
|
||||
|
||||
if (isPrivacyGdprExportType(payload.exportType)) {
|
||||
return runPrivacyGdprExportHandler(payload, deps, context);
|
||||
}
|
||||
|
||||
const markPaused = deps.markExportJobPaused ?? markExportJobPausedInDb;
|
||||
const markResumed = deps.markExportJobResumed ?? markExportJobResumedInDb;
|
||||
const markRunning = deps.markExportJobRunning ?? markExportJobRunningInDb;
|
||||
@@ -1434,6 +1971,66 @@ async function runExportHandler(
|
||||
}
|
||||
}
|
||||
|
||||
async function runPrivacyGdprExportHandler(
|
||||
payload: ExportJob,
|
||||
deps: HandlerDependencies,
|
||||
context?: HandlerExecutionContext<typeof queueNames.exports>
|
||||
): Promise<HandlerResult> {
|
||||
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
||||
const markProcessing = deps.markGdprExportProcessing ?? markGdprExportProcessingInDb;
|
||||
const markReady = deps.markGdprExportReady ?? markGdprExportReadyInDb;
|
||||
const markFailed = deps.markGdprExportFailed ?? markGdprExportFailedInDb;
|
||||
|
||||
await markProcessing({ id: payload.exportJobId, tenantId: payload.tenantId });
|
||||
|
||||
if (!deps.privacyGdprExportService) {
|
||||
const message = "Privacy GDPR export service is not configured.";
|
||||
await markFailed({
|
||||
id: payload.exportJobId,
|
||||
tenantId: payload.tenantId,
|
||||
errorMessage: message,
|
||||
});
|
||||
throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "privacy_gdpr_export_service_missing" }));
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await deps.privacyGdprExportService.runExport(payload);
|
||||
await markReady({
|
||||
id: payload.exportJobId,
|
||||
tenantId: payload.tenantId,
|
||||
resultBucket: result.artifactBucket,
|
||||
resultKey: result.artifactKey,
|
||||
resultSizeBytes: result.sizeBytes,
|
||||
resultExpiresAt: result.expiresAt,
|
||||
});
|
||||
|
||||
return {
|
||||
handled: true,
|
||||
action: "privacy-gdpr-export.completed",
|
||||
details: buildExportDetails(payload, {
|
||||
artifactBucket: result.artifactBucket,
|
||||
artifactKey: result.artifactKey,
|
||||
attemptsMade,
|
||||
expiresAt: result.expiresAt.toISOString(),
|
||||
rowCount: result.rowCount,
|
||||
sizeBytes: result.sizeBytes,
|
||||
status: "ready",
|
||||
}),
|
||||
};
|
||||
} catch (error) {
|
||||
const errorName = error instanceof Error ? error.name : "Error";
|
||||
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
||||
if (isTerminalJobAttempt(attemptsMade, context)) {
|
||||
await markFailed({
|
||||
id: payload.exportJobId,
|
||||
tenantId: payload.tenantId,
|
||||
errorMessage,
|
||||
});
|
||||
}
|
||||
throw new ExportHandlerError("Privacy GDPR export job failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }));
|
||||
}
|
||||
}
|
||||
|
||||
async function runImportHandler(
|
||||
payload: QueuePayload<typeof queueNames.imports>,
|
||||
deps: HandlerDependencies,
|
||||
@@ -1743,14 +2340,16 @@ function serializeWebhookResponseDiagnostics(response: Response, body: string) {
|
||||
}
|
||||
|
||||
async function fetchWebhookWithRedirects(
|
||||
fetchImpl: typeof fetch,
|
||||
fetchImpl: typeof fetch | undefined,
|
||||
input: string,
|
||||
init: RequestInit,
|
||||
addressResolver?: OutboundHttpAddressResolver
|
||||
): Promise<Response> {
|
||||
try {
|
||||
return await fetchOutboundHttpWithRedirects(input, init, {
|
||||
fetchImpl,
|
||||
// In production fetchImpl is undefined, so the guard uses its pinned
|
||||
// undici fetch; a test-injected fetchImpl disables pinning.
|
||||
...(fetchImpl ? { fetchImpl } : {}),
|
||||
...(addressResolver ? { addressResolver } : {}),
|
||||
timeoutMs: DEFAULT_WORKER_FETCH_TIMEOUT_MS,
|
||||
redirectLimit: WEBHOOK_REDIRECT_LIMIT,
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
* `http.createServer` keeps the bundle size and startup cost minimal.
|
||||
*/
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import type { WorkerQueueGroup } from "@toolctrl/queue";
|
||||
import type { WorkerQueueGroup } from "@hivegrid/queue";
|
||||
|
||||
export const DEFAULT_WORKER_HEALTH_PORT = 9100;
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { queueNames } from "@toolctrl/queue";
|
||||
import { queueNames } from "@hivegrid/queue";
|
||||
import { buildWorkerHealthReport, runWorkerHealthCli } from "./health.js";
|
||||
|
||||
const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), "../package.json");
|
||||
@@ -101,7 +101,7 @@ describe("worker health", () => {
|
||||
workerEnabled: true,
|
||||
workerGroup: "general",
|
||||
});
|
||||
expect(report.queues).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan]);
|
||||
expect(report.queues).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
|
||||
expect(redisProbe).toHaveBeenCalledWith("redis://localhost:6379/0");
|
||||
expect(databaseProbe).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
@@ -514,7 +514,7 @@ describe("worker health", () => {
|
||||
);
|
||||
|
||||
expect(report.status).toBe("ready");
|
||||
expect(report.queues).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan]);
|
||||
expect(report.queues).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview]);
|
||||
expect(report.storageStatus).toBe("ready");
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type WorkerConfigRedisParseStatus } from "@toolctrl/config";
|
||||
import { getDbClient } from "@toolctrl/db/client.server";
|
||||
import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@toolctrl/queue";
|
||||
import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type WorkerConfigRedisParseStatus } from "@hivegrid/config";
|
||||
import { getDbClient } from "@hivegrid/db/client.server";
|
||||
import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@hivegrid/queue";
|
||||
import { Queue } from "bullmq";
|
||||
import { getWorkerQueueNames } from "./queues.js";
|
||||
import { getMissingWorkerStorageEnv } from "./storage.js";
|
||||
@@ -317,7 +317,7 @@ function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { con
|
||||
const errors: string[] = [];
|
||||
if (!importEnabled) {
|
||||
errors.push(
|
||||
"IMPORT_QUEUE_ENABLED:missing-import-runtime-gate:Required with IMPORT_QUEUE_RUNTIME_READY=true before WORKER_QUEUE_GROUP=import can consume toolctrl.imports/toolctrl.exports."
|
||||
"IMPORT_QUEUE_ENABLED:missing-import-runtime-gate:Required with IMPORT_QUEUE_RUNTIME_READY=true before WORKER_QUEUE_GROUP=import can consume hivegrid.imports/hivegrid.exports."
|
||||
);
|
||||
}
|
||||
if (!runtimeReady) {
|
||||
@@ -396,7 +396,7 @@ async function defaultDatabaseProbe(): Promise<void> {
|
||||
}
|
||||
|
||||
async function defaultRedisProbe(redisUrl: string): Promise<void> {
|
||||
const queue = new Queue("__toolctrl_worker_health__", {
|
||||
const queue = new Queue("__hivegrid_worker_health__", {
|
||||
connection: parseRedisUrl(redisUrl),
|
||||
});
|
||||
|
||||
|
||||
@@ -76,6 +76,7 @@ const createWorkflowTriggerExecutorFromEnv = vi.fn().mockReturnValue(workflowTri
|
||||
const createPrismaWorkflowTriggerStore = vi.fn().mockReturnValue(workflowTriggerStore);
|
||||
const registerOutboxDoneRetentionCron = vi.fn(async () => undefined);
|
||||
const registerWorkflowExecutionStalledReconcilerCron = vi.fn(async () => undefined);
|
||||
const registerLeadRetentionCron = vi.fn(async () => undefined);
|
||||
const workflowExecutionStalledReconciler = vi.fn(async () => ({
|
||||
cutoff: new Date("2026-05-29T02:30:00.000Z"),
|
||||
take: 100,
|
||||
@@ -185,7 +186,7 @@ vi.mock("./jobs/workflow-trigger-store.js", () => ({
|
||||
createPrismaWorkflowTriggerStore,
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/shared-store", () => ({
|
||||
vi.mock("@hivegrid/shared-store", () => ({
|
||||
createIdempotencyStore,
|
||||
}));
|
||||
|
||||
@@ -193,25 +194,16 @@ vi.mock("./shared-store.js", () => ({
|
||||
getSharedStore: vi.fn(() => sharedStore),
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/db/client.server", () => ({
|
||||
vi.mock("@hivegrid/db/client.server", () => ({
|
||||
getDbClient,
|
||||
getReadDbClient: getDbClient,
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/db/events.server", () => ({
|
||||
vi.mock("@hivegrid/db/events.server", () => ({
|
||||
createLegacyEvent: vi.fn(async () => ({ id: "evt-1" })),
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/db/entities.server", () => ({
|
||||
findLegacyEntityByName: vi.fn(async () => ({ id: "contact_entity" })),
|
||||
findLegacyEntityByIdNameOrSlug: vi.fn(async () => ({ id: "target_entity" })),
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/db/rows.server", () => ({
|
||||
createLegacyEntityRow: vi.fn(async () => ({ id: "row_1" })),
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/db/outbox", () => ({
|
||||
vi.mock("@hivegrid/db/outbox", () => ({
|
||||
publishOutboxEvent: vi.fn(async () => ({ id: "outbox_1" })),
|
||||
}));
|
||||
|
||||
@@ -262,6 +254,10 @@ vi.mock("./jobs/workflow-execution-stalled-reconciler-cron.js", () => ({
|
||||
registerWorkflowExecutionStalledReconcilerCron,
|
||||
}));
|
||||
|
||||
vi.mock("./jobs/lead-retention-cron.js", () => ({
|
||||
registerLeadRetentionCron,
|
||||
}));
|
||||
|
||||
vi.mock("./jobs/workflow-execution-stalled-reconciler.js", () => ({
|
||||
createWorkflowExecutionStalledReconciler,
|
||||
}));
|
||||
@@ -359,7 +355,7 @@ describe("worker entrypoint", () => {
|
||||
});
|
||||
|
||||
test("wires SIGINT shutdown to the started runtime", async () => {
|
||||
const runtime = { status: "started" as const, workers: [{ name: "toolctrl.email" }] };
|
||||
const runtime = { status: "started" as const, workers: [{ name: "hivegrid.email" }] };
|
||||
startWorkerRuntime.mockReturnValue(runtime);
|
||||
shutdownWorkerRuntime.mockResolvedValue(undefined);
|
||||
|
||||
@@ -370,7 +366,7 @@ describe("worker entrypoint", () => {
|
||||
});
|
||||
|
||||
test("reuses one shutdown promise for repeated SIGTERM signals", async () => {
|
||||
const runtime = { status: "started" as const, workers: [{ name: "toolctrl.email" }] };
|
||||
const runtime = { status: "started" as const, workers: [{ name: "hivegrid.email" }] };
|
||||
let resolveShutdown: (() => void) | undefined;
|
||||
startWorkerRuntime.mockReturnValue(runtime);
|
||||
shutdownWorkerRuntime.mockReturnValue(
|
||||
@@ -1039,8 +1035,6 @@ describe("worker entrypoint", () => {
|
||||
logger: workerLogger,
|
||||
handlerDeps: expect.objectContaining({
|
||||
store: expect.objectContaining({ load: expect.any(Function), markRouted: expect.any(Function) }),
|
||||
crm: expect.objectContaining({ createContact: expect.any(Function) }),
|
||||
entity: expect.objectContaining({ createRow: expect.any(Function) }),
|
||||
emitFollowUps: expect.any(Function),
|
||||
}),
|
||||
createWorkerImpl: expect.any(Function),
|
||||
@@ -1354,7 +1348,7 @@ describe("worker entrypoint", () => {
|
||||
|
||||
expect(bootstrapPluginInvokeDeps).not.toHaveBeenCalled();
|
||||
expect(workerLogger.warn).toHaveBeenCalledWith(
|
||||
{ workerGroup: "plugin", queueNames: ["toolctrl.plugin-invoke"] },
|
||||
{ workerGroup: "plugin", queueNames: ["hivegrid.plugin-invoke"] },
|
||||
"plugin-invoke runtime disabled before production signoff; starting plugin worker idle without queue consumers."
|
||||
);
|
||||
expect(startWorkerRuntime).toHaveBeenCalledWith(
|
||||
@@ -1385,11 +1379,11 @@ describe("worker entrypoint", () => {
|
||||
|
||||
await import("./index.js");
|
||||
|
||||
// import owns toolctrl.imports/exports whose handlers need wired services;
|
||||
// import owns hivegrid.imports/exports whose handlers need wired services;
|
||||
// with the runtime gated off the worker must boot idle (no queue consumers)
|
||||
// instead of tripping assertQueueServicesWired.
|
||||
expect(workerLogger.warn).toHaveBeenCalledWith(
|
||||
{ workerGroup: "import", queueNames: ["toolctrl.imports", "toolctrl.exports"] },
|
||||
{ workerGroup: "import", queueNames: ["hivegrid.imports", "hivegrid.exports"] },
|
||||
"import/export queue runtime gated off (IMPORT_QUEUE_ENABLED/IMPORT_QUEUE_RUNTIME_READY); starting import worker idle without queue consumers."
|
||||
);
|
||||
expect(startWorkerRuntime).toHaveBeenCalledWith(expect.objectContaining({ queueNames: [] }));
|
||||
@@ -1656,7 +1650,7 @@ describe("worker entrypoint", () => {
|
||||
});
|
||||
|
||||
test("preserves the first signal reason across repeated shutdown signals", async () => {
|
||||
const runtime = { status: "started" as const, workers: [{ name: "toolctrl.email" }] };
|
||||
const runtime = { status: "started" as const, workers: [{ name: "hivegrid.email" }] };
|
||||
startWorkerRuntime.mockReturnValue(runtime);
|
||||
shutdownWorkerRuntime.mockResolvedValue(undefined);
|
||||
|
||||
@@ -1670,7 +1664,7 @@ describe("worker entrypoint", () => {
|
||||
});
|
||||
|
||||
test("logs redacted shutdown failure and exits non-zero", async () => {
|
||||
const runtime = { status: "started" as const, workers: [{ name: "toolctrl.email" }] };
|
||||
const runtime = { status: "started" as const, workers: [{ name: "hivegrid.email" }] };
|
||||
startWorkerRuntime.mockReturnValue(runtime);
|
||||
shutdownWorkerRuntime.mockRejectedValue(new Error("postgres://user:token@localhost/db"));
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
type WorkflowTriggerBootstrapResult,
|
||||
type WorkflowTriggerWorkerHandle,
|
||||
} from "./jobs/workflow-trigger-bootstrap.js";
|
||||
import type { QueueJsonValue } from "@hivegrid/queue";
|
||||
import {
|
||||
createWorkflowTriggerExecutorFromEnv,
|
||||
parseWorkflowTriggerExecutorMode,
|
||||
@@ -69,25 +70,50 @@ import { registerDsgvoGraceCron } from "./jobs/dsgvo-grace-cron.js";
|
||||
import { registerOutboxDoneRetentionCron } from "./jobs/outbox-done-retention-cron.js";
|
||||
import { registerWorkflowExecutionStalledReconcilerCron } from "./jobs/workflow-execution-stalled-reconciler-cron.js";
|
||||
import { registerTaskArchiveRetentionCron } from "./jobs/task-archive-retention-cron.js";
|
||||
import { registerLeadRetentionCron } from "./jobs/lead-retention-cron.js";
|
||||
import { registerIntegrationsHealthSweepCron, registerIntegrationsTokenRefreshCron } from "./jobs/integrations-maintenance-cron.js";
|
||||
import {
|
||||
registerCalendarInternalWriteSyncCron,
|
||||
registerCalendarTokenRefreshCron,
|
||||
registerCalendarWebhookDrainCron,
|
||||
registerCalendarWebhookRenewalCron,
|
||||
} from "./jobs/calendar-maintenance-cron.js";
|
||||
import {
|
||||
registerEmailInboxEventCleanupCron,
|
||||
registerEmailInboxInitialSyncCron,
|
||||
registerEmailInboxStalePollCron,
|
||||
registerEmailInboxTokenRefreshCron,
|
||||
registerEmailInboxWebhookDrainCron,
|
||||
registerEmailInboxWebhookRenewalCron,
|
||||
} from "./jobs/email-inbox-maintenance-cron.js";
|
||||
import { registerTaskReminderSweepCron } from "./jobs/task-reminder-cron.js";
|
||||
import { registerHrUploadReminderDispatchCron } from "./jobs/hr-upload-reminder-cron.js";
|
||||
import { registerEmailDomainProbeCron } from "./jobs/email-domain-probe-cron.js";
|
||||
import {
|
||||
PLUGIN_AUDIT_RETENTION_QUEUE,
|
||||
registerPluginAuditRetentionCron,
|
||||
resolvePluginAuditRetentionConfig,
|
||||
runPluginAuditRetention,
|
||||
} from "./jobs/plugin-audit-retention.js";
|
||||
import {
|
||||
AUDIT_RETENTION_QUEUE,
|
||||
registerAuditRetentionCron,
|
||||
resolveAuditRetentionPolicy,
|
||||
runAuditRetentionJob,
|
||||
type AuditRetentionTenant,
|
||||
} from "./jobs/audit-retention.js";
|
||||
import {
|
||||
createWorkflowExecutionStalledReconciler,
|
||||
type WorkflowExecutionStalledReconcilerPrisma,
|
||||
} from "./jobs/workflow-execution-stalled-reconciler.js";
|
||||
import type { HandlerDependencies } from "./handlers.js";
|
||||
import { Queue as BullMQQueue, Worker as BullMQWorker } from "bullmq";
|
||||
import { enqueueJob, parseRedisUrl, queueNames } from "@toolctrl/queue";
|
||||
import { createIdempotencyStore } from "@toolctrl/shared-store";
|
||||
import { enqueueJob, parseRedisUrl, queueNames } from "@hivegrid/queue";
|
||||
import { createIdempotencyStore } from "@hivegrid/shared-store";
|
||||
import type { SentryLikeForWorkerTracing } from "./runtime.js";
|
||||
import { shutdownWorkerRuntime, startWorkerRuntime, type WorkerRuntimeStatus } from "./runtime.js";
|
||||
import { createWorker } from "@toolctrl/queue/runtime";
|
||||
import type { Prisma } from "@toolctrl/db/prisma-browser";
|
||||
import { createWorker } from "@hivegrid/queue/runtime";
|
||||
import type { Prisma } from "@hivegrid/db/prisma-browser";
|
||||
import { workerLogger } from "./logger.js";
|
||||
import { getWorkerMetrics } from "./metrics.js";
|
||||
import { workerErrorReporter } from "./sentry.js";
|
||||
@@ -112,6 +138,7 @@ type ShutdownState = {
|
||||
pluginCapabilityUse?: PluginCapabilityUseBootstrapResult | undefined;
|
||||
pluginHookInvokeQueue?: BullMQQueue | undefined;
|
||||
pluginAuditRetentionWorker?: BullMQWorker | undefined;
|
||||
auditRetentionWorker?: BullMQWorker | undefined;
|
||||
};
|
||||
|
||||
function buildWorkerStatus(
|
||||
@@ -175,7 +202,7 @@ function createRedisHealthProbe(env: NodeJS.ProcessEnv): () => Promise<{ ok: tru
|
||||
let queue: import("bullmq").Queue | undefined;
|
||||
try {
|
||||
const { Queue } = await import("bullmq");
|
||||
queue = new Queue("__toolctrl_worker_health__", { connection: parseRedisUrl(redisUrl) });
|
||||
queue = new Queue("__hivegrid_worker_health__", { connection: parseRedisUrl(redisUrl) });
|
||||
await queue.waitUntilReady();
|
||||
return { ok: true };
|
||||
} catch (error) {
|
||||
@@ -189,7 +216,7 @@ function createRedisHealthProbe(env: NodeJS.ProcessEnv): () => Promise<{ ok: tru
|
||||
function createDatabaseHealthProbe(): () => Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
return async () => {
|
||||
try {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const db = getDbClient();
|
||||
await db.$queryRaw`SELECT 1`;
|
||||
return { ok: true };
|
||||
@@ -364,7 +391,7 @@ try {
|
||||
const extraHandlerDependencies: Partial<HandlerDependencies> = {};
|
||||
if (groupResolution?.group === "general") {
|
||||
try {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
extraHandlerDependencies.reconcileWorkflowExecutionStalls = createWorkflowExecutionStalledReconciler({
|
||||
prisma: getDbClient() as unknown as WorkflowExecutionStalledReconcilerPrisma,
|
||||
});
|
||||
@@ -375,13 +402,13 @@ try {
|
||||
}
|
||||
|
||||
// AK-P4a §4 — file-preview generator service on the `general` group
|
||||
// (the group that owns the toolctrl.file-preview queue). Composed with a
|
||||
// Prisma client + STORAGE_* S3 seams; injected via extraHandlerDependencies
|
||||
// exactly like aiCostStripeReporter above. When STORAGE_* is unset the
|
||||
// factory returns undefined and the handler dead-letters preview jobs.
|
||||
// (the group that owns the hivegrid.file-preview queue). Composed with a
|
||||
// Prisma client + STORAGE_* S3 seams; injected via extraHandlerDependencies.
|
||||
// When STORAGE_* is unset the factory returns undefined and the handler
|
||||
// dead-letters preview jobs.
|
||||
if (groupResolution?.group === "general") {
|
||||
try {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const { createWorkerFilePreviewService } = await import("./jobs/file-preview.js");
|
||||
const previewService = createWorkerFilePreviewService(process.env as Parameters<typeof createWorkerFilePreviewService>[0], {
|
||||
prisma: getDbClient() as unknown as Parameters<typeof createWorkerFilePreviewService>[1]["prisma"],
|
||||
@@ -395,7 +422,7 @@ try {
|
||||
}
|
||||
|
||||
// AK-P4a §5 — scan-verdict write-back. The general group owns the
|
||||
// toolctrl.file-scan queue, so wire the File.scanStatus writer here with
|
||||
// hivegrid.file-scan queue, so wire the File.scanStatus writer here with
|
||||
// the same Prisma seam the preview service uses.
|
||||
const { createWorkerFileScanStatusWriter } = await import("./jobs/file-scan-status.js");
|
||||
extraHandlerDependencies.fileScanStatusWriter = createWorkerFileScanStatusWriter({
|
||||
@@ -416,7 +443,7 @@ try {
|
||||
const pluginInvokeRuntimeEnabled = groupResolution?.group === "plugin" ? isPluginInvokeRuntimeEnabled(process.env) : true;
|
||||
if (groupResolution?.group === "plugin" && pluginInvokeRuntimeEnabled) {
|
||||
try {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const { bootstrapPluginInvokeDeps } = await import("./jobs/plugin-invoke-bootstrap.js");
|
||||
const { getSharedStore } = await import("./shared-store.js");
|
||||
const prisma = getDbClient() as unknown as Parameters<typeof bootstrapPluginInvokeDeps>[0]["prisma"];
|
||||
@@ -458,6 +485,24 @@ try {
|
||||
);
|
||||
}
|
||||
|
||||
if (groupResolution?.group === "import" && importQueueRuntimeEnabled) {
|
||||
try {
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const { createWorkerPrivacyGdprExportService } = await import("./jobs/privacy-gdpr-export-bootstrap.js");
|
||||
const privacyGdprExportService = createWorkerPrivacyGdprExportService(process.env, {
|
||||
prisma: getDbClient() as unknown as Parameters<typeof createWorkerPrivacyGdprExportService>[1]["prisma"],
|
||||
});
|
||||
if (privacyGdprExportService) {
|
||||
extraHandlerDependencies.privacyGdprExportService = privacyGdprExportService;
|
||||
workerLogger.info({}, "privacy GDPR export service bootstrapped (import group).");
|
||||
} else {
|
||||
workerLogger.info({}, "privacy GDPR export service skipped: STORAGE_* not configured.");
|
||||
}
|
||||
} catch (error) {
|
||||
workerLogger.error({ error }, "privacy GDPR export bootstrap failed; GDPR export jobs will dead-letter until fixed.");
|
||||
}
|
||||
}
|
||||
|
||||
const runtime = startWorkerRuntime({
|
||||
logger: workerLogger,
|
||||
reporter: workerErrorReporter,
|
||||
@@ -565,7 +610,7 @@ try {
|
||||
const outboxRequiredForGroup = shouldRunOutbox && (isOutboxDispatcherRequired(process.env) || isWorkflowTriggerOutboxRequired(process.env));
|
||||
if (shouldRunOutbox) {
|
||||
try {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const prisma = getDbClient();
|
||||
outbox = await bootstrapOutboxDispatcher(process.env, {
|
||||
prisma,
|
||||
@@ -602,7 +647,7 @@ try {
|
||||
workerLogger.info({ reason: workflowTrigger.reason }, "workflow-trigger worker disabled by configuration.");
|
||||
}
|
||||
} else if (redisUrl) {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const prisma = getDbClient();
|
||||
const executor = createWorkflowTriggerExecutorFromEnv(process.env, {
|
||||
logger: workerLogger,
|
||||
@@ -746,8 +791,38 @@ try {
|
||||
try {
|
||||
await registerTaskArchiveRetentionCron(taskCronQueue, process.env);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "task-archive-retention scheduler registered (24h cadence).");
|
||||
await registerLeadRetentionCron(taskCronQueue, process.env);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "lead-retention-sweep scheduler registered (24h cadence).");
|
||||
await registerTaskReminderSweepCron(taskCronQueue, process.env);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "task-reminder-sweep scheduler registered (1h cadence).");
|
||||
await registerIntegrationsTokenRefreshCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "integrations-token-refresh scheduler registered (5m cadence).");
|
||||
await registerIntegrationsHealthSweepCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "integrations-health-sweep scheduler registered (15m cadence).");
|
||||
await registerCalendarTokenRefreshCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "calendar-token-refresh scheduler registered (5m cadence).");
|
||||
await registerCalendarWebhookRenewalCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-renewal scheduler registered (6h cadence).");
|
||||
await registerCalendarWebhookDrainCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "calendar-webhook-drain scheduler registered (1m cadence).");
|
||||
await registerCalendarInternalWriteSyncCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "calendar-internal-write-sync scheduler registered (5m cadence).");
|
||||
await registerEmailInboxTokenRefreshCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-token-refresh scheduler registered (5m cadence).");
|
||||
await registerEmailInboxWebhookRenewalCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-renewal scheduler registered (6h cadence).");
|
||||
await registerEmailInboxWebhookDrainCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-webhook-drain scheduler registered (1m cadence).");
|
||||
await registerEmailInboxInitialSyncCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-initial-sync scheduler registered (2m cadence).");
|
||||
await registerEmailInboxStalePollCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-stale-poll scheduler registered (30m cadence).");
|
||||
await registerEmailInboxEventCleanupCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "email-inbox-event-cleanup scheduler registered (24h cadence).");
|
||||
await registerHrUploadReminderDispatchCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "hr-upload-reminder-dispatch scheduler registered (24h cadence).");
|
||||
await registerEmailDomainProbeCron(taskCronQueue);
|
||||
workerLogger.info({ queue: queueNames.maintenance }, "email-domain-probe scheduler registered (5m cadence).");
|
||||
} finally {
|
||||
await taskCronQueue.close();
|
||||
}
|
||||
@@ -784,7 +859,7 @@ try {
|
||||
pluginAuditRetentionWorker = new BullMQWorker(
|
||||
PLUGIN_AUDIT_RETENTION_QUEUE,
|
||||
async () => {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const prisma = getDbClient() as unknown as {
|
||||
pluginCapabilityUse: {
|
||||
deleteMany: (args: { where: { id: { in: string[] } } }) => Promise<{ count: number }>;
|
||||
@@ -834,11 +909,110 @@ try {
|
||||
}
|
||||
}
|
||||
|
||||
// AK-P3 AP-2 — AuditEvent retention cron. Nightly at 03:30 UTC
|
||||
// (AUDIT_RETENTION_CRON overridable). Single-instance scheduler + worker on
|
||||
// the `general` group, dedicated queue (AUDIT_RETENTION_QUEUE). Mirrors the
|
||||
// plugin-audit-retention pattern. Compliance-safe: per-tenant regulatory
|
||||
// scopes are resolved so the policy never prunes below the required window.
|
||||
let auditRetentionWorker: import("bullmq").Worker | undefined;
|
||||
if (groupResolution?.group === "general") {
|
||||
try {
|
||||
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
||||
if (redisUrl) {
|
||||
const connection = parseRedisUrl(redisUrl);
|
||||
const auditRetentionQueue = new BullMQQueue(AUDIT_RETENTION_QUEUE, { connection });
|
||||
try {
|
||||
const { pattern } = await registerAuditRetentionCron(auditRetentionQueue, process.env);
|
||||
workerLogger.info({ queue: AUDIT_RETENTION_QUEUE, pattern }, "audit-retention scheduler registered.");
|
||||
} finally {
|
||||
await auditRetentionQueue.close();
|
||||
}
|
||||
|
||||
auditRetentionWorker = new BullMQWorker(
|
||||
AUDIT_RETENTION_QUEUE,
|
||||
async () => {
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const { createAuditEvent } = await import("@hivegrid/db/audit-events.server");
|
||||
const prisma = getDbClient() as unknown as {
|
||||
auditEvent: {
|
||||
findMany: (args: Record<string, unknown>) => Promise<Array<{ id?: string; tenantId?: string | null }>>;
|
||||
deleteMany: (args: { where: { id: { in: string[] } } }) => Promise<{ count: number }>;
|
||||
count: (args: { where: Record<string, unknown> }) => Promise<number>;
|
||||
};
|
||||
};
|
||||
return runAuditRetentionJob({
|
||||
logger: workerLogger,
|
||||
// List tenants that have any audit events, with the regulatory
|
||||
// scopes actually used (cheap `has` counts → policy picks the max).
|
||||
tenantLister: async (): Promise<ReadonlyArray<AuditRetentionTenant>> => {
|
||||
const rows = await prisma.auditEvent.findMany({
|
||||
where: { tenantId: { not: null } },
|
||||
distinct: ["tenantId"],
|
||||
select: { tenantId: true },
|
||||
});
|
||||
const tenants: AuditRetentionTenant[] = [];
|
||||
for (const row of rows) {
|
||||
const tenantId = row.tenantId;
|
||||
if (!tenantId) continue;
|
||||
const scopes: string[] = [];
|
||||
for (const scope of ["soc2", "hipaa", "gdpr"]) {
|
||||
const hit = await prisma.auditEvent.count({ where: { tenantId, regulatoryScope: { has: scope } } });
|
||||
if (hit > 0) scopes.push(scope);
|
||||
}
|
||||
tenants.push({ tenantId, regulatoryScopes: scopes });
|
||||
}
|
||||
return tenants;
|
||||
},
|
||||
// Bounded batch delete (Postgres has no LIMIT on DELETE): pick a
|
||||
// capped id-set older than the threshold, then deleteMany by id.
|
||||
eventDeleter: async (tenantId: string, threshold: Date): Promise<number> => {
|
||||
let deleted = 0;
|
||||
for (let batch = 0; batch < 100; batch += 1) {
|
||||
const candidates = await prisma.auditEvent.findMany({
|
||||
where: { tenantId, createdAt: { lt: threshold } },
|
||||
select: { id: true },
|
||||
take: 1000,
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
if (candidates.length === 0) break;
|
||||
const ids = candidates.map((row) => row.id).filter((id): id is string => Boolean(id));
|
||||
const { count } = await prisma.auditEvent.deleteMany({ where: { id: { in: ids } } });
|
||||
deleted += count;
|
||||
if (candidates.length < 1000) break;
|
||||
}
|
||||
return deleted;
|
||||
},
|
||||
// The cleanup is itself audited as a system action (traceable).
|
||||
auditWriter: async ({ tenantId, deletedCount, threshold, policy }) => {
|
||||
await createAuditEvent({
|
||||
tenantId,
|
||||
actorType: "system",
|
||||
action: "audit.retention.pruned",
|
||||
targetType: "AuditEvent",
|
||||
sourceModule: "audit",
|
||||
dataSensitivity: "internal",
|
||||
metadata: { deletedCount, threshold: threshold.toISOString(), retentionDays: policy.retentionDays, policySource: policy.source },
|
||||
});
|
||||
},
|
||||
policyResolver: resolveAuditRetentionPolicy,
|
||||
});
|
||||
},
|
||||
{ connection: parseRedisUrl(redisUrl), concurrency: 1 },
|
||||
);
|
||||
workerLogger.info({ queue: AUDIT_RETENTION_QUEUE }, "audit-retention worker started.");
|
||||
} else {
|
||||
workerLogger.info({}, "audit-retention scheduler skipped: no Redis URL configured.");
|
||||
}
|
||||
} catch (error) {
|
||||
workerLogger.error({ error }, "audit-retention scheduler registration failed; continuing.");
|
||||
}
|
||||
}
|
||||
|
||||
let searchIndexer: SearchIndexerBootstrapResult | undefined;
|
||||
if (groupResolution?.group === "general") {
|
||||
try {
|
||||
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const { createSearchIndexerBulkPagers } = await import("./jobs/search-indexer-bulk-pagers.js");
|
||||
const prisma = getDbClient();
|
||||
searchIndexer = bootstrapSearchIndexer(process.env, {
|
||||
@@ -865,28 +1039,63 @@ try {
|
||||
try {
|
||||
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
||||
if (redisUrl) {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const prisma = getDbClient() as unknown as {
|
||||
formSubmission: {
|
||||
findUnique: (args: { where: { id: string }; select: { status: true; tenantId: true } }) => Promise<{ status: string; tenantId: string } | null>;
|
||||
updateMany: (args: {
|
||||
where: { id: string; tenantId: string };
|
||||
data: { status: string; result: Record<string, unknown> };
|
||||
data: Record<string, unknown>;
|
||||
}) => Promise<{ count: number }>;
|
||||
};
|
||||
formAction: {
|
||||
findMany: (args: Record<string, unknown>) => Promise<Array<{
|
||||
id: string;
|
||||
actionType: string;
|
||||
name: string;
|
||||
fieldMapping: unknown;
|
||||
actionConfig: unknown;
|
||||
}>>;
|
||||
};
|
||||
formActionLog: {
|
||||
create: (args: Record<string, unknown>) => Promise<{ id: string; attempt: number }>;
|
||||
update: (args: Record<string, unknown>) => Promise<unknown>;
|
||||
};
|
||||
inboxLead: {
|
||||
findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null>;
|
||||
create: (args: Record<string, unknown>) => Promise<{ id: string }>;
|
||||
};
|
||||
kontakt: {
|
||||
findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null>;
|
||||
create: (args: Record<string, unknown>) => Promise<{ id: string }>;
|
||||
};
|
||||
pipeline: { findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null> };
|
||||
pipelineStage: { findFirst: (args: Record<string, unknown>) => Promise<{ id: string } | null> };
|
||||
projekt: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
|
||||
task: { create: (args: Record<string, unknown>) => Promise<{ id: string }> };
|
||||
};
|
||||
const [
|
||||
{ findLegacyEntityByIdNameOrSlug, findLegacyEntityByName },
|
||||
{ createLegacyEntityRow },
|
||||
{ publishOutboxEvent },
|
||||
{ planFormSubmissionFollowUps },
|
||||
{ buildWorkflowTriggerOutboxEvent },
|
||||
{ applyFieldMapping },
|
||||
] = await Promise.all([
|
||||
import("@toolctrl/db/entities.server"),
|
||||
import("@toolctrl/db/rows.server"),
|
||||
import("@toolctrl/db/outbox"),
|
||||
import("@hivegrid/db/outbox"),
|
||||
import("./jobs/form-submission-followups.js"),
|
||||
import("@hivegrid/queue"),
|
||||
import("@hivegrid/form-widget"),
|
||||
]);
|
||||
const connection = parseRedisUrl(redisUrl);
|
||||
function stringValue(values: Record<string, unknown>, keys: readonly string[]): string | null {
|
||||
for (const key of keys) {
|
||||
const value = values[key];
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function objectValue(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
formSubmissionRoute = bootstrapFormSubmissionRoute(process.env, {
|
||||
logger: workerLogger,
|
||||
handlerDeps: {
|
||||
@@ -906,28 +1115,6 @@ try {
|
||||
}
|
||||
},
|
||||
},
|
||||
crm: {
|
||||
createContact: async (input) => {
|
||||
const entity = await findLegacyEntityByName({ tenantId: input.tenantId, name: "contact", active: true });
|
||||
const row = await createLegacyEntityRow({
|
||||
entity: entity as Parameters<typeof createLegacyEntityRow>[0]["entity"],
|
||||
tenantId: input.tenantId,
|
||||
data: input.values,
|
||||
});
|
||||
return { id: row.id };
|
||||
},
|
||||
},
|
||||
entity: {
|
||||
createRow: async (input) => {
|
||||
const entity = await findLegacyEntityByIdNameOrSlug({ tenantId: input.tenantId, idNameOrSlug: input.entityId });
|
||||
const row = await createLegacyEntityRow({
|
||||
entity: entity as Parameters<typeof createLegacyEntityRow>[0]["entity"],
|
||||
tenantId: input.tenantId,
|
||||
data: input.values,
|
||||
});
|
||||
return { id: row.id };
|
||||
},
|
||||
},
|
||||
emitFollowUps: async (followUp) => {
|
||||
const events = planFormSubmissionFollowUps(followUp);
|
||||
for (const event of events) {
|
||||
@@ -939,6 +1126,191 @@ try {
|
||||
});
|
||||
}
|
||||
},
|
||||
actionRunner: async ({ submissionId, formId, tenantId, values }) => {
|
||||
const actions = await prisma.formAction.findMany({
|
||||
where: { formId, tenantId, enabled: true },
|
||||
orderBy: { position: "asc" },
|
||||
select: { id: true, actionType: true, name: true, fieldMapping: true, actionConfig: true },
|
||||
});
|
||||
for (const action of actions) {
|
||||
const mapped = applyFieldMapping({
|
||||
payload: values,
|
||||
fieldMapping: objectValue(action.fieldMapping) as Record<string, string>,
|
||||
});
|
||||
const idempotencyKey = `form-action:${submissionId}:${action.id}`;
|
||||
const log = await prisma.formActionLog.create({
|
||||
data: {
|
||||
formActionId: action.id,
|
||||
submissionId,
|
||||
formId,
|
||||
tenantId,
|
||||
actionType: action.actionType,
|
||||
status: "pending",
|
||||
inputSnapshot: { values: mapped },
|
||||
idempotencyKey,
|
||||
},
|
||||
select: { id: true, attempt: true },
|
||||
});
|
||||
try {
|
||||
const config = objectValue(action.actionConfig);
|
||||
let target: { type: string; id: string | null } = { type: action.actionType, id: null };
|
||||
if (action.actionType === "auto_match") {
|
||||
const email = stringValue(mapped, ["email", "primaerEmail", "email_address"]);
|
||||
const phone = stringValue(mapped, ["phone", "telefon", "primaerTelefon"]);
|
||||
const contact = email || phone
|
||||
? await prisma.kontakt.findFirst({
|
||||
where: {
|
||||
tenantId,
|
||||
archivedAt: null,
|
||||
OR: [
|
||||
...(email ? [{ primaerEmail: email.toLowerCase() }] : []),
|
||||
...(phone ? [{ primaerTelefon: phone }] : []),
|
||||
],
|
||||
},
|
||||
select: { id: true },
|
||||
})
|
||||
: null;
|
||||
const lead = !contact && (email || phone)
|
||||
? await prisma.inboxLead.findFirst({
|
||||
where: { tenantId, archivedAt: null, OR: [...(email ? [{ email: email.toLowerCase() }] : []), ...(phone ? [{ telefon: phone }] : [])] },
|
||||
select: { id: true },
|
||||
})
|
||||
: null;
|
||||
target = contact ? { type: "contact", id: contact.id } : lead ? { type: "lead", id: lead.id } : { type: "none", id: null };
|
||||
if (target.id) {
|
||||
await prisma.formSubmission.updateMany({
|
||||
where: { id: submissionId, tenantId },
|
||||
data: { linkedEntityType: target.type, linkedEntityId: target.id },
|
||||
});
|
||||
}
|
||||
} else if (action.actionType === "create_lead") {
|
||||
const lead = await prisma.inboxLead.create({
|
||||
data: {
|
||||
tenantId,
|
||||
createdByUserId: typeof config.userId === "string" ? config.userId : null,
|
||||
quelle: "form",
|
||||
quelleDetail: { formId, submissionId },
|
||||
sourceOrigin: `/forms/${formId}`,
|
||||
vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]),
|
||||
nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]),
|
||||
email: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null,
|
||||
telefon: stringValue(mapped, ["telefon", "phone"]),
|
||||
message: stringValue(mapped, ["message", "nachricht", "details"]),
|
||||
rohdaten: { formId, submissionId, values: mapped },
|
||||
weiterverarbeitungFreigabe: "ja",
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
target = { type: "lead", id: lead.id };
|
||||
} else if (action.actionType === "create_contact") {
|
||||
const contact = await prisma.kontakt.create({
|
||||
data: {
|
||||
tenantId,
|
||||
createdByUserId: typeof config.userId === "string" ? config.userId : "system",
|
||||
kontaktTyp: "privatperson",
|
||||
vorname: stringValue(mapped, ["vorname", "firstName", "first_name"]),
|
||||
nachname: stringValue(mapped, ["nachname", "lastName", "last_name"]) ?? stringValue(mapped, ["name"]) ?? "Formular-Kontakt",
|
||||
primaerEmail: stringValue(mapped, ["email", "email_address"])?.toLowerCase() ?? null,
|
||||
primaerTelefon: stringValue(mapped, ["telefon", "phone"]),
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
target = { type: "contact", id: contact.id };
|
||||
} else if (action.actionType === "create_project") {
|
||||
const pipeline = await prisma.pipeline.findFirst({ where: { tenantId, archivedAt: null }, orderBy: { position: "asc" }, select: { id: true } });
|
||||
const stage = pipeline ? await prisma.pipelineStage.findFirst({ where: { tenantId, pipelineId: pipeline.id, archivedAt: null }, orderBy: { position: "asc" }, select: { id: true } }) : null;
|
||||
if (!pipeline || !stage) throw new Error("No project pipeline/stage available for form action");
|
||||
const project = await prisma.projekt.create({
|
||||
data: {
|
||||
tenantId,
|
||||
createdByUserId: typeof config.userId === "string" ? config.userId : null,
|
||||
projektNummer: `FORM-${Date.now()}`,
|
||||
projektname: stringValue(mapped, ["projektname", "projectName", "name"]) ?? "Projekt aus Formular",
|
||||
beschreibung: stringValue(mapped, ["message", "description", "details"]),
|
||||
quelle: "form",
|
||||
pipelineId: pipeline.id,
|
||||
stageId: stage.id,
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
target = { type: "project", id: project.id };
|
||||
} else if (action.actionType === "create_task") {
|
||||
const task = await prisma.task.create({
|
||||
data: {
|
||||
tenantId,
|
||||
createdByUserId: typeof config.userId === "string" ? config.userId : "system",
|
||||
title: stringValue(mapped, ["title", "taskTitle", "subject"]) ?? "Aufgabe aus Formular",
|
||||
description: stringValue(mapped, ["message", "description", "details"]),
|
||||
assignedToUserId: typeof config.assignedToUserId === "string" ? config.assignedToUserId : typeof config.userId === "string" ? config.userId : "system",
|
||||
source: "form_submission",
|
||||
sourceRef: submissionId,
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
target = { type: "task", id: task.id };
|
||||
} else if (action.actionType === "webhook") {
|
||||
const webhookUrl = typeof config.webhookUrl === "string" ? config.webhookUrl : null;
|
||||
if (!webhookUrl) throw new Error("Webhook action missing webhookUrl");
|
||||
const webhookPayload = { submissionId, formId, tenantId, webhookUrl, values: mapped } as unknown as Prisma.InputJsonValue;
|
||||
await publishOutboxEvent(prisma as never, {
|
||||
tenantId,
|
||||
type: "webhook.form-submission.deliver",
|
||||
payload: webhookPayload,
|
||||
dispatchKey: `form-action-webhook:${submissionId}:${action.id}`,
|
||||
});
|
||||
} else if (action.actionType === "send_email_to_team" || action.actionType === "send_email_to_submitter") {
|
||||
const recipients = action.actionType === "send_email_to_submitter"
|
||||
? [stringValue(mapped, ["email", "email_address"])].filter((email): email is string => Boolean(email))
|
||||
: Array.isArray(config.recipients) ? config.recipients.filter((email): email is string => typeof email === "string" && email.includes("@")) : [];
|
||||
if (recipients.length === 0) {
|
||||
await prisma.formActionLog.update({ where: { id: log.id }, data: { status: "skipped", outputSnapshot: { reason: "no_recipients" }, finishedAt: new Date() } });
|
||||
continue;
|
||||
}
|
||||
const mailPayload = { submissionId, formId, tenantId, recipients, values: mapped } as unknown as Prisma.InputJsonValue;
|
||||
await publishOutboxEvent(prisma as never, {
|
||||
tenantId,
|
||||
type: "mail.transactional.form-submission",
|
||||
payload: mailPayload,
|
||||
dispatchKey: `form-action-mail:${submissionId}:${action.id}`,
|
||||
});
|
||||
} else if (action.actionType === "trigger_automation") {
|
||||
const automationData = { formId, submissionId, actionId: action.id, values: mapped } as unknown as QueueJsonValue;
|
||||
await publishOutboxEvent(
|
||||
prisma as never,
|
||||
buildWorkflowTriggerOutboxEvent({
|
||||
tenantId,
|
||||
event: "form.submitted",
|
||||
data: automationData,
|
||||
userId: typeof config.userId === "string" ? config.userId : null,
|
||||
sourceId: `form:${formId}:submission:${submissionId}:action:${action.id}`,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
await prisma.formActionLog.update({ where: { id: log.id }, data: { status: "skipped", outputSnapshot: { reason: "unsupported_action" }, finishedAt: new Date() } });
|
||||
continue;
|
||||
}
|
||||
await prisma.formActionLog.update({
|
||||
where: { id: log.id },
|
||||
data: {
|
||||
status: "success",
|
||||
targetType: target.type,
|
||||
targetId: target.id,
|
||||
outputSnapshot: { target },
|
||||
finishedAt: new Date(),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
await prisma.formActionLog.update({
|
||||
where: { id: log.id },
|
||||
data: {
|
||||
status: "failed",
|
||||
errorMessage: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500),
|
||||
finishedAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
logger: workerLogger,
|
||||
},
|
||||
createWorkerImpl: (queueName, processor, options) => {
|
||||
@@ -1037,7 +1409,7 @@ try {
|
||||
import("./jobs/email-sequence-scheduler.js"),
|
||||
]);
|
||||
// Production deps (Prisma store + BullMQ scheduleStep producer +
|
||||
// template renderer + @toolctrl/mail Mailjet adapter +
|
||||
// template renderer + @hivegrid/mail Mailjet adapter +
|
||||
// unsubscribe-url builder) are loaded lazily inside the
|
||||
// processors so startup-time optional config issues do not
|
||||
// crash the mail worker. Missing Redis/Mailjet config still
|
||||
@@ -1080,7 +1452,7 @@ try {
|
||||
if (groupResolution?.group === "general") {
|
||||
try {
|
||||
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const prisma = getDbClient();
|
||||
auditEventPersist = bootstrapAuditEventPersist(process.env, {
|
||||
logger: workerLogger,
|
||||
@@ -1115,7 +1487,7 @@ try {
|
||||
try {
|
||||
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
||||
if (redisUrl) {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const prisma = getDbClient();
|
||||
const connection = parseRedisUrl(redisUrl);
|
||||
pluginHookInvokeQueue = new BullMQQueue(queueNames.pluginInvoke, { connection });
|
||||
@@ -1161,7 +1533,7 @@ try {
|
||||
try {
|
||||
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
||||
if (redisUrl) {
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { getDbClient } = await import("@hivegrid/db/client.server");
|
||||
const prisma = getDbClient();
|
||||
const connection = parseRedisUrl(redisUrl);
|
||||
pluginCapabilityUse = bootstrapPluginCapabilityUse(process.env, {
|
||||
@@ -1186,15 +1558,15 @@ try {
|
||||
// Phase 03 #42 — Legacy-event outbox receiver. Subscribes the
|
||||
// `outbox.notification` queue on the `general` group and writes
|
||||
// billing.subscription.* events into the legacy Event table on
|
||||
// behalf of the migrated Stripe webhook (apps/api/src/billing/
|
||||
// stripe-webhook-route.ts). Disabled when the group is non-general
|
||||
// behalf of the migrated Mollie webhook (apps/api/src/billing/
|
||||
// mollie-webhook-route.ts). Disabled when the group is non-general
|
||||
// or no Redis URL is configured.
|
||||
let legacyEventReceiver: LegacyEventReceiverBootstrapResult | undefined;
|
||||
if (groupResolution?.group === "general") {
|
||||
try {
|
||||
const redisUrl = process.env.BULLMQ_REDIS_URL ?? process.env.REDIS_URL;
|
||||
if (redisUrl) {
|
||||
const { createLegacyEvent } = await import("@toolctrl/db/events.server");
|
||||
const { createLegacyEvent } = await import("@hivegrid/db/events.server");
|
||||
legacyEventReceiver = bootstrapLegacyEventReceiver(process.env, {
|
||||
logger: workerLogger,
|
||||
store: {
|
||||
@@ -1237,6 +1609,7 @@ try {
|
||||
pluginCapabilityUse,
|
||||
pluginHookInvokeQueue,
|
||||
pluginAuditRetentionWorker,
|
||||
auditRetentionWorker,
|
||||
};
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
@@ -1272,7 +1645,8 @@ function shutdown(runtimeToClose: WorkerRuntimeStatus, state: ShutdownState, sig
|
||||
state.pluginHookFanout,
|
||||
state.pluginCapabilityUse,
|
||||
state.pluginHookInvokeQueue,
|
||||
state.pluginAuditRetentionWorker
|
||||
state.pluginAuditRetentionWorker,
|
||||
state.auditRetentionWorker
|
||||
);
|
||||
return state.promise;
|
||||
}
|
||||
@@ -1295,7 +1669,8 @@ async function closeRuntime(
|
||||
pluginHookFanout?: PluginHookFanoutBootstrapResult,
|
||||
pluginCapabilityUse?: PluginCapabilityUseBootstrapResult,
|
||||
pluginHookInvokeQueue?: BullMQQueue,
|
||||
pluginAuditRetentionWorker?: BullMQWorker
|
||||
pluginAuditRetentionWorker?: BullMQWorker,
|
||||
auditRetentionWorker?: BullMQWorker
|
||||
): Promise<void> {
|
||||
try {
|
||||
if (outbox?.handle) {
|
||||
@@ -1397,6 +1772,13 @@ async function closeRuntime(
|
||||
workerLogger.error({ error: retentionError, signal }, "plugin-audit-retention worker shutdown failed.");
|
||||
}
|
||||
}
|
||||
if (auditRetentionWorker) {
|
||||
try {
|
||||
await auditRetentionWorker.close();
|
||||
} catch (retentionError) {
|
||||
workerLogger.error({ error: retentionError, signal }, "audit-retention worker shutdown failed.");
|
||||
}
|
||||
}
|
||||
await shutdownWorkerRuntime(runtimeToClose, { logger: workerLogger, reason: signal, reporter: workerErrorReporter });
|
||||
if (adminServer) {
|
||||
await adminServer.close();
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@
|
||||
* semantics without spinning up BullMQ.
|
||||
*/
|
||||
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
|
||||
import {
|
||||
AUDIT_EVENT_OUTBOX_QUEUE,
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { computeAuditHash } from "@toolctrl/db/audit-hash";
|
||||
import { computeAuditHash } from "@hivegrid/db/audit-hash";
|
||||
import {
|
||||
AUDIT_EVENT_OUTBOX_QUEUE,
|
||||
AuditEventPersistJobError,
|
||||
|
||||
+2
-2
@@ -25,12 +25,12 @@
|
||||
* "at-least-once": a duplicated audit row is harmless evidence.
|
||||
*/
|
||||
|
||||
import { computeAuditHash, type AuditHashPayload } from "@toolctrl/db/audit-hash";
|
||||
import { computeAuditHash, type AuditHashPayload } from "@hivegrid/db/audit-hash";
|
||||
import {
|
||||
buildAuditEventCreateData,
|
||||
type AuditEventCreateData,
|
||||
type AuditEventInput,
|
||||
} from "@toolctrl/db/audit-events.server";
|
||||
} from "@hivegrid/db/audit-events.server";
|
||||
|
||||
export const AUDIT_EVENT_OUTBOX_QUEUE = "outbox.audit" as const;
|
||||
|
||||
|
||||
@@ -1,13 +1,35 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
AUDIT_RETENTION_CRON_DEFAULT,
|
||||
AUDIT_RETENTION_JOB_ID,
|
||||
DEFAULT_AUDIT_RETENTION_DAYS,
|
||||
GDPR_AUDIT_RETENTION_DAYS,
|
||||
SOC2_AUDIT_RETENTION_DAYS,
|
||||
registerAuditRetentionCron,
|
||||
resolveAuditRetentionPolicy,
|
||||
runAuditRetentionJob,
|
||||
} from "./audit-retention.js";
|
||||
|
||||
describe("registerAuditRetentionCron", () => {
|
||||
it("upserts the nightly scheduler with the default pattern (idempotent key)", async () => {
|
||||
const upsertJobScheduler = vi.fn(async () => undefined);
|
||||
const { pattern } = await registerAuditRetentionCron({ upsertJobScheduler }, {});
|
||||
expect(pattern).toBe(AUDIT_RETENTION_CRON_DEFAULT);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
AUDIT_RETENTION_JOB_ID,
|
||||
{ pattern: AUDIT_RETENTION_CRON_DEFAULT },
|
||||
{ name: AUDIT_RETENTION_JOB_ID, data: { task: AUDIT_RETENTION_JOB_ID } },
|
||||
);
|
||||
});
|
||||
|
||||
it("honours an AUDIT_RETENTION_CRON override", async () => {
|
||||
const upsertJobScheduler = vi.fn(async () => undefined);
|
||||
const { pattern } = await registerAuditRetentionCron({ upsertJobScheduler }, { AUDIT_RETENTION_CRON: "0 5 * * *" });
|
||||
expect(pattern).toBe("0 5 * * *");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveAuditRetentionPolicy", () => {
|
||||
it("returns 7y for SOC2 + HIPAA tenants (HIPAA wins on label)", () => {
|
||||
expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["soc2"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "soc2" });
|
||||
|
||||
@@ -17,6 +17,39 @@ export const DEFAULT_AUDIT_RETENTION_DAYS = 365; // baseline / "none" scope
|
||||
export const SOC2_AUDIT_RETENTION_DAYS = 7 * 365; // SOC2 + HIPAA: 7y
|
||||
export const GDPR_AUDIT_RETENTION_DAYS = 6 * 365; // GDPR: 6y
|
||||
|
||||
// AK-P3 AP-2 — cron wiring. Mirrors plugin-audit-retention so the worker
|
||||
// bootstrap can install a nightly scheduler for the AuditEvent sweep.
|
||||
export const AUDIT_RETENTION_QUEUE = "audit-retention" as const;
|
||||
export const AUDIT_RETENTION_JOB_ID = "audit-retention" as const;
|
||||
/** Nightly at 03:30 UTC (offset from plugin-audit-retention's 03:00). */
|
||||
export const AUDIT_RETENTION_CRON_DEFAULT = "30 3 * * *" as const;
|
||||
|
||||
export type AuditRetentionEnv = Readonly<Record<string, string | undefined>>;
|
||||
|
||||
/**
|
||||
* Install (or replace) the nightly AuditEvent-retention schedule on a
|
||||
* BullMQ-like queue. Idempotent via a stable scheduler key. Cron pattern
|
||||
* overridable via `AUDIT_RETENTION_CRON`.
|
||||
*/
|
||||
export async function registerAuditRetentionCron(
|
||||
queue: {
|
||||
upsertJobScheduler: (
|
||||
schedulerId: string,
|
||||
options: { pattern: string },
|
||||
template: { name: string; data?: Record<string, unknown> },
|
||||
) => Promise<unknown>;
|
||||
},
|
||||
env: AuditRetentionEnv | undefined = process.env,
|
||||
): Promise<{ pattern: string }> {
|
||||
const pattern = env?.AUDIT_RETENTION_CRON?.trim() || AUDIT_RETENTION_CRON_DEFAULT;
|
||||
await queue.upsertJobScheduler(
|
||||
AUDIT_RETENTION_JOB_ID,
|
||||
{ pattern },
|
||||
{ name: AUDIT_RETENTION_JOB_ID, data: { task: AUDIT_RETENTION_JOB_ID } },
|
||||
);
|
||||
return { pattern };
|
||||
}
|
||||
|
||||
export type AuditRetentionPolicy = {
|
||||
/** Hard cap in days. Events older than this are pruned. */
|
||||
retentionDays: number;
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID,
|
||||
CALENDAR_TOKEN_REFRESH_JOB_ID,
|
||||
CALENDAR_WEBHOOK_DRAIN_JOB_ID,
|
||||
CALENDAR_WEBHOOK_RENEWAL_JOB_ID,
|
||||
registerCalendarInternalWriteSyncCron,
|
||||
registerCalendarTokenRefreshCron,
|
||||
registerCalendarWebhookDrainCron,
|
||||
registerCalendarWebhookRenewalCron,
|
||||
} from "./calendar-maintenance-cron.js";
|
||||
|
||||
describe("calendar maintenance cron registration", () => {
|
||||
it("registers cleanup.run payloads for every calendar maintenance task", async () => {
|
||||
const upsertJobScheduler = vi.fn(async () => undefined);
|
||||
const queue = { upsertJobScheduler };
|
||||
|
||||
await registerCalendarTokenRefreshCron(queue);
|
||||
await registerCalendarWebhookRenewalCron(queue);
|
||||
await registerCalendarWebhookDrainCron(queue);
|
||||
await registerCalendarInternalWriteSyncCron(queue);
|
||||
|
||||
expect(upsertJobScheduler).toHaveBeenCalledTimes(4);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
CALENDAR_TOKEN_REFRESH_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: CALENDAR_TOKEN_REFRESH_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_TOKEN_REFRESH_JOB_ID, olderThanMinutes: 10 } },
|
||||
);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
CALENDAR_WEBHOOK_RENEWAL_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: CALENDAR_WEBHOOK_RENEWAL_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_WEBHOOK_RENEWAL_JOB_ID, olderThanMinutes: 1440 } },
|
||||
);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
CALENDAR_WEBHOOK_DRAIN_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: CALENDAR_WEBHOOK_DRAIN_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_WEBHOOK_DRAIN_JOB_ID, take: 50 } },
|
||||
);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID, take: 50 } },
|
||||
);
|
||||
});
|
||||
});
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
// AK-P16a (P16) Calendar maintenance crons. Registration only; execution lives
|
||||
// in handlers.ts so it can share the worker DB/RLS pattern.
|
||||
|
||||
export const CALENDAR_TOKEN_REFRESH_JOB_ID = "calendar-token-refresh" as const;
|
||||
export const CALENDAR_WEBHOOK_RENEWAL_JOB_ID = "calendar-webhook-renewal" as const;
|
||||
export const CALENDAR_WEBHOOK_DRAIN_JOB_ID = "calendar-webhook-drain" as const;
|
||||
export const CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID = "calendar-internal-write-sync" as const;
|
||||
|
||||
export const CALENDAR_TOKEN_REFRESH_EVERY_MS = 5 * 60 * 1000;
|
||||
export const CALENDAR_WEBHOOK_RENEWAL_EVERY_MS = 6 * 60 * 60 * 1000;
|
||||
export const CALENDAR_WEBHOOK_DRAIN_EVERY_MS = 60 * 1000;
|
||||
export const CALENDAR_INTERNAL_WRITE_SYNC_EVERY_MS = 5 * 60 * 1000;
|
||||
|
||||
export const CALENDAR_TOKEN_REFRESH_WINDOW_MINUTES = 10;
|
||||
export const CALENDAR_WEBHOOK_RENEWAL_THRESHOLD_MINUTES = 24 * 60;
|
||||
export const CALENDAR_WEBHOOK_DRAIN_TAKE = 50;
|
||||
export const CALENDAR_INTERNAL_WRITE_SYNC_TAKE = 50;
|
||||
|
||||
type SchedulerQueue = {
|
||||
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
|
||||
};
|
||||
|
||||
export async function registerCalendarTokenRefreshCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
CALENDAR_TOKEN_REFRESH_JOB_ID,
|
||||
{ every: CALENDAR_TOKEN_REFRESH_EVERY_MS },
|
||||
{ name: CALENDAR_TOKEN_REFRESH_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_TOKEN_REFRESH_JOB_ID, olderThanMinutes: CALENDAR_TOKEN_REFRESH_WINDOW_MINUTES } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerCalendarWebhookRenewalCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
CALENDAR_WEBHOOK_RENEWAL_JOB_ID,
|
||||
{ every: CALENDAR_WEBHOOK_RENEWAL_EVERY_MS },
|
||||
{ name: CALENDAR_WEBHOOK_RENEWAL_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_WEBHOOK_RENEWAL_JOB_ID, olderThanMinutes: CALENDAR_WEBHOOK_RENEWAL_THRESHOLD_MINUTES } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerCalendarWebhookDrainCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
CALENDAR_WEBHOOK_DRAIN_JOB_ID,
|
||||
{ every: CALENDAR_WEBHOOK_DRAIN_EVERY_MS },
|
||||
{ name: CALENDAR_WEBHOOK_DRAIN_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_WEBHOOK_DRAIN_JOB_ID, take: CALENDAR_WEBHOOK_DRAIN_TAKE } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerCalendarInternalWriteSyncCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID,
|
||||
{ every: CALENDAR_INTERNAL_WRITE_SYNC_EVERY_MS },
|
||||
{ name: CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID, data: { kind: "cleanup.run", task: CALENDAR_INTERNAL_WRITE_SYNC_JOB_ID, take: CALENDAR_INTERNAL_WRITE_SYNC_TAKE } },
|
||||
);
|
||||
}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { USER_FK_MAP } from "@toolctrl/db/dsgvo/user-fk-map";
|
||||
import { USER_FK_MAP } from "@hivegrid/db/dsgvo/user-fk-map";
|
||||
|
||||
import { runDsgvoDeletion, type DsgvoDeletionDependencies, type DsgvoDeletionJobInput } from "./dsgvo-deletion.js";
|
||||
|
||||
|
||||
@@ -23,8 +23,8 @@
|
||||
* client + audit writer so this file is fully unit-testable.
|
||||
*/
|
||||
|
||||
import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
|
||||
import { buildUserPseudonymizationUpdate, pseudonymize } from "@toolctrl/db/dsgvo/pseudonymize";
|
||||
import { USER_FK_MAP, type UserFkMapEntry } from "@hivegrid/db/dsgvo/user-fk-map";
|
||||
import { buildUserPseudonymizationUpdate, pseudonymize } from "@hivegrid/db/dsgvo/pseudonymize";
|
||||
|
||||
export type DsgvoDeletionMode = "delete" | "pseudonymize";
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
|
||||
import { USER_FK_MAP, type UserFkMapEntry } from "@hivegrid/db/dsgvo/user-fk-map";
|
||||
|
||||
export type DsgvoExportJobInput = {
|
||||
userId: string;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { USER_FK_MAP } from "@toolctrl/db/dsgvo/user-fk-map";
|
||||
import { USER_FK_MAP } from "@hivegrid/db/dsgvo/user-fk-map";
|
||||
|
||||
import { runDsgvoVerify, type DsgvoVerifyDependencies } from "./dsgvo-verify.js";
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
* operator can escalate the incident.
|
||||
*/
|
||||
|
||||
import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
|
||||
import { USER_FK_MAP, type UserFkMapEntry } from "@hivegrid/db/dsgvo/user-fk-map";
|
||||
|
||||
export type DsgvoVerifyJobInput = {
|
||||
userId: string;
|
||||
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
EMAIL_DOMAIN_PROBE_EVERY_MS,
|
||||
EMAIL_DOMAIN_PROBE_FAILURE_AFTER_MS,
|
||||
EMAIL_DOMAIN_PROBE_FAILURE_KEY,
|
||||
EMAIL_DOMAIN_PROBE_JOB_ID,
|
||||
createDnsDomainResolver,
|
||||
decideDomainProbeStatus,
|
||||
registerEmailDomainProbeCron,
|
||||
} from "./email-domain-probe-cron.js";
|
||||
|
||||
const T0 = new Date("2026-06-01T00:00:00.000Z");
|
||||
const within7Days = new Date(T0.getTime() + 3 * 24 * 60 * 60 * 1000);
|
||||
const past7Days = new Date(T0.getTime() + EMAIL_DOMAIN_PROBE_FAILURE_AFTER_MS + 1000);
|
||||
|
||||
describe("decideDomainProbeStatus", () => {
|
||||
it("verifies once all three records resolve", () => {
|
||||
expect(decideDomainProbeStatus({ dns: { dkim: true, spf: true, dmarc: true }, firstProbeAt: T0, now: within7Days })).toEqual({
|
||||
status: "verified",
|
||||
lastError: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("stays pending when nothing resolves yet inside the window", () => {
|
||||
expect(decideDomainProbeStatus({ dns: { dkim: false, spf: false, dmarc: false }, firstProbeAt: T0, now: within7Days })).toEqual({
|
||||
status: "pending",
|
||||
lastError: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("moves to verifying when at least one record resolves inside the window", () => {
|
||||
expect(decideDomainProbeStatus({ dns: { dkim: true, spf: false, dmarc: false }, firstProbeAt: T0, now: within7Days })).toEqual({
|
||||
status: "verifying",
|
||||
lastError: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("fails with a Klartext key once incomplete past the 7-day window", () => {
|
||||
expect(decideDomainProbeStatus({ dns: { dkim: true, spf: true, dmarc: false }, firstProbeAt: T0, now: past7Days })).toEqual({
|
||||
status: "failed",
|
||||
lastError: EMAIL_DOMAIN_PROBE_FAILURE_KEY,
|
||||
});
|
||||
});
|
||||
|
||||
it("treats a null firstProbeAt as the first probe (never instantly failed)", () => {
|
||||
expect(decideDomainProbeStatus({ dns: { dkim: false, spf: false, dmarc: false }, firstProbeAt: null, now: past7Days })).toEqual({
|
||||
status: "pending",
|
||||
lastError: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("createDnsDomainResolver", () => {
|
||||
it("maps TXT markers to the three record flags and joins chunked answers", async () => {
|
||||
const resolveTxt = vi.fn(async (hostname: string) => {
|
||||
if (hostname === "mailjet._domainkey.acme.test") return [["v=DKIM1; k=rsa; p=", "MIIBI..."]];
|
||||
if (hostname === "acme.test") return [["v=spf1 include:spf.mailjet.com ~all"]];
|
||||
if (hostname === "_dmarc.acme.test") return [["v=DMARC1; p=none"]];
|
||||
return [];
|
||||
});
|
||||
const resolver = createDnsDomainResolver({ resolveTxt });
|
||||
await expect(resolver({ domain: "acme.test", dkimSelector: "mailjet" })).resolves.toEqual({ dkim: true, spf: true, dmarc: true });
|
||||
});
|
||||
|
||||
it("treats lookup failures as 'not present' instead of throwing", async () => {
|
||||
const resolveTxt = vi.fn(async () => {
|
||||
throw new Error("ENOTFOUND");
|
||||
});
|
||||
const resolver = createDnsDomainResolver({ resolveTxt });
|
||||
await expect(resolver({ domain: "acme.test", dkimSelector: "mailjet" })).resolves.toEqual({ dkim: false, spf: false, dmarc: false });
|
||||
});
|
||||
|
||||
it("skips the DKIM lookup when no selector is known", async () => {
|
||||
const resolveTxt = vi.fn(async (hostname: string) => (hostname === "acme.test" ? [["v=spf1 ~all"]] : []));
|
||||
const resolver = createDnsDomainResolver({ resolveTxt });
|
||||
await expect(resolver({ domain: "acme.test", dkimSelector: null })).resolves.toEqual({ dkim: false, spf: true, dmarc: false });
|
||||
expect(resolveTxt).not.toHaveBeenCalledWith("._domainkey.acme.test");
|
||||
});
|
||||
});
|
||||
|
||||
describe("registerEmailDomainProbeCron", () => {
|
||||
it("registers a five-minute scheduler with the cleanup.run payload", async () => {
|
||||
const upsert = vi.fn().mockResolvedValue(undefined);
|
||||
await registerEmailDomainProbeCron({ upsertJobScheduler: upsert });
|
||||
|
||||
expect(upsert).toHaveBeenCalledTimes(1);
|
||||
expect(upsert.mock.calls[0]?.[0]).toBe(EMAIL_DOMAIN_PROBE_JOB_ID);
|
||||
expect(upsert.mock.calls[0]?.[1]).toEqual({ every: EMAIL_DOMAIN_PROBE_EVERY_MS });
|
||||
expect(upsert.mock.calls[0]?.[2]).toEqual({
|
||||
name: EMAIL_DOMAIN_PROBE_JOB_ID,
|
||||
data: { kind: "cleanup.run", task: EMAIL_DOMAIN_PROBE_JOB_ID },
|
||||
});
|
||||
});
|
||||
});
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
// AK-P14a · P14 Email — sending-domain DNS auto-probe cron.
|
||||
//
|
||||
// Every 5 minutes the worker walks each tenant's not-yet-settled sending domains
|
||||
// (status pending|verifying), resolves the published DKIM/SPF/DMARC records and
|
||||
// advances `verificationStatus`:
|
||||
//
|
||||
// pending → verifying (at least one record now resolves)
|
||||
// * → verified (all three resolve)
|
||||
// * → failed (still incomplete 7 days after the first probe)
|
||||
//
|
||||
// Mirrors the hr-upload-reminder-cron / task-reminder-sweep shape: this file
|
||||
// carries the config + scheduler registration + the PURE decision helper
|
||||
// (unit-tested without DNS or a DB). The execution logic (Prisma + notify())
|
||||
// lives in the maintenance handler which holds the DB client. The DNS lookup is
|
||||
// behind an injectable `DomainDnsResolver` seam so the handler binds the real
|
||||
// node:dns resolver while tests inject a deterministic fake.
|
||||
|
||||
import { resolveTxt } from "node:dns/promises";
|
||||
|
||||
export const EMAIL_DOMAIN_PROBE_JOB_ID = "email-domain-probe" as const;
|
||||
/** Five-minute cadence per the AK-P14a DoD (5-Min-Auto-Probe). */
|
||||
export const EMAIL_DOMAIN_PROBE_EVERY_MS = 5 * 60 * 1000;
|
||||
/** After this long without full verification a domain flips to `failed` (Klartext). */
|
||||
export const EMAIL_DOMAIN_PROBE_FAILURE_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
/** i18n (M002) key surfaced on `lastError` once the 7-day window elapses. */
|
||||
export const EMAIL_DOMAIN_PROBE_FAILURE_KEY = "email.domain.error.notVerifiedAfter7Days" as const;
|
||||
|
||||
export type SendingDomainProbeStatus = "pending" | "verifying" | "verified" | "failed";
|
||||
|
||||
/** Which of the three required records currently resolve in DNS. */
|
||||
export type DnsProbeResult = {
|
||||
readonly dkim: boolean;
|
||||
readonly spf: boolean;
|
||||
readonly dmarc: boolean;
|
||||
};
|
||||
|
||||
export type ProbeDecision = {
|
||||
readonly status: SendingDomainProbeStatus;
|
||||
/** i18n key for the Klartext failure detail, or null. */
|
||||
readonly lastError: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Pure: given which records resolve, when probing started and the current time,
|
||||
* decide the next verification status and Klartext error. No DNS, no DB — fully
|
||||
* unit-testable. `failureAfterMs` is injectable for tests; production uses the
|
||||
* 7-day default.
|
||||
*/
|
||||
export function decideDomainProbeStatus(input: {
|
||||
dns: DnsProbeResult;
|
||||
firstProbeAt: Date | null;
|
||||
now: Date;
|
||||
failureAfterMs?: number;
|
||||
}): ProbeDecision {
|
||||
if (input.dns.dkim && input.dns.spf && input.dns.dmarc) {
|
||||
return { status: "verified", lastError: null };
|
||||
}
|
||||
const failureAfterMs = input.failureAfterMs ?? EMAIL_DOMAIN_PROBE_FAILURE_AFTER_MS;
|
||||
const elapsed = input.firstProbeAt ? input.now.getTime() - input.firstProbeAt.getTime() : 0;
|
||||
if (input.firstProbeAt && elapsed >= failureAfterMs) {
|
||||
return { status: "failed", lastError: EMAIL_DOMAIN_PROBE_FAILURE_KEY };
|
||||
}
|
||||
const anyPresent = input.dns.dkim || input.dns.spf || input.dns.dmarc;
|
||||
return { status: anyPresent ? "verifying" : "pending", lastError: null };
|
||||
}
|
||||
|
||||
/** Lookup seam: resolves the three records for a domain. Injected for testing. */
|
||||
export type DomainDnsResolver = (input: { domain: string; dkimSelector: string | null }) => Promise<DnsProbeResult>;
|
||||
|
||||
/** Join a multi-string TXT answer (DNS chunks records at 255 chars) into one. */
|
||||
function joinTxt(records: string[][]): string[] {
|
||||
return records.map((chunks) => chunks.join(""));
|
||||
}
|
||||
|
||||
/**
|
||||
* Production DNS resolver (node:dns). A record counts as present when its TXT
|
||||
* marker resolves: DKIM at `<selector>._domainkey.<domain>`, SPF (`v=spf1`) at
|
||||
* the apex, DMARC (`v=DMARC1`) at `_dmarc.<domain>`. Lookups are best-effort —
|
||||
* a NXDOMAIN / SERVFAIL resolves to "not present", never throws.
|
||||
*/
|
||||
export function createDnsDomainResolver(deps: { resolveTxt: (hostname: string) => Promise<string[][]> } = { resolveTxt }): DomainDnsResolver {
|
||||
const lookup = async (hostname: string): Promise<string[]> => {
|
||||
try {
|
||||
return joinTxt(await deps.resolveTxt(hostname));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
return async ({ domain, dkimSelector }) => {
|
||||
const [dkimTxt, apexTxt, dmarcTxt] = await Promise.all([
|
||||
dkimSelector ? lookup(`${dkimSelector}._domainkey.${domain}`) : Promise.resolve([]),
|
||||
lookup(domain),
|
||||
lookup(`_dmarc.${domain}`),
|
||||
]);
|
||||
return {
|
||||
dkim: dkimTxt.some((value) => value.includes("v=DKIM1") || value.includes("k=rsa")),
|
||||
spf: apexTxt.some((value) => value.toLowerCase().includes("v=spf1")),
|
||||
dmarc: dmarcTxt.some((value) => value.toUpperCase().includes("V=DMARC1")),
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export async function registerEmailDomainProbeCron(queue: {
|
||||
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
|
||||
}): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
EMAIL_DOMAIN_PROBE_JOB_ID,
|
||||
{ every: EMAIL_DOMAIN_PROBE_EVERY_MS },
|
||||
{ name: EMAIL_DOMAIN_PROBE_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_DOMAIN_PROBE_JOB_ID } },
|
||||
);
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
EMAIL_INBOX_EVENT_CLEANUP_JOB_ID,
|
||||
EMAIL_INBOX_INITIAL_SYNC_JOB_ID,
|
||||
EMAIL_INBOX_STALE_POLL_JOB_ID,
|
||||
EMAIL_INBOX_TOKEN_REFRESH_JOB_ID,
|
||||
EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID,
|
||||
EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID,
|
||||
registerEmailInboxEventCleanupCron,
|
||||
registerEmailInboxInitialSyncCron,
|
||||
registerEmailInboxStalePollCron,
|
||||
registerEmailInboxTokenRefreshCron,
|
||||
registerEmailInboxWebhookDrainCron,
|
||||
registerEmailInboxWebhookRenewalCron,
|
||||
} from "./email-inbox-maintenance-cron.js";
|
||||
|
||||
describe("email-inbox maintenance cron registration", () => {
|
||||
it("registers cleanup.run payloads for every email-inbox maintenance task", async () => {
|
||||
const upsertJobScheduler = vi.fn(async () => undefined);
|
||||
const queue = { upsertJobScheduler };
|
||||
|
||||
await registerEmailInboxTokenRefreshCron(queue);
|
||||
await registerEmailInboxWebhookRenewalCron(queue);
|
||||
await registerEmailInboxWebhookDrainCron(queue);
|
||||
await registerEmailInboxInitialSyncCron(queue);
|
||||
await registerEmailInboxStalePollCron(queue);
|
||||
await registerEmailInboxEventCleanupCron(queue);
|
||||
|
||||
expect(upsertJobScheduler).toHaveBeenCalledTimes(6);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
EMAIL_INBOX_TOKEN_REFRESH_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: EMAIL_INBOX_TOKEN_REFRESH_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_TOKEN_REFRESH_JOB_ID, olderThanMinutes: 10 } },
|
||||
);
|
||||
// Renewal threshold: 24h before expiry covers Graph (~3d) AND Gmail (~7d).
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID, olderThanMinutes: 1440 } },
|
||||
);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID, take: 25 } },
|
||||
);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
EMAIL_INBOX_INITIAL_SYNC_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: EMAIL_INBOX_INITIAL_SYNC_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_INITIAL_SYNC_JOB_ID, take: 10 } },
|
||||
);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
EMAIL_INBOX_STALE_POLL_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: EMAIL_INBOX_STALE_POLL_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_STALE_POLL_JOB_ID, take: 25 } },
|
||||
);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
EMAIL_INBOX_EVENT_CLEANUP_JOB_ID,
|
||||
expect.any(Object),
|
||||
{ name: EMAIL_INBOX_EVENT_CLEANUP_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_EVENT_CLEANUP_JOB_ID } },
|
||||
);
|
||||
});
|
||||
});
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
// VA-P23 (P23) email-inbox maintenance crons. Registration only; execution lives
|
||||
// in handlers.ts so it can share the worker DB/RLS pattern (mirror of
|
||||
// calendar-maintenance-cron.ts).
|
||||
//
|
||||
// Renewal cadence: Graph subscriptions live ~3 days and Gmail watches ~7 days —
|
||||
// a 6h renewal sweep with a 24h expiry threshold keeps both alive with margin.
|
||||
// Drain runs every 60s (push events should surface fast); the 30min stale-poll
|
||||
// is the 4h-silence safety net (push stays primary, M09 Block D).
|
||||
|
||||
export const EMAIL_INBOX_TOKEN_REFRESH_JOB_ID = "email-inbox-token-refresh" as const;
|
||||
export const EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID = "email-inbox-webhook-renewal" as const;
|
||||
export const EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID = "email-inbox-webhook-drain" as const;
|
||||
export const EMAIL_INBOX_INITIAL_SYNC_JOB_ID = "email-inbox-initial-sync" as const;
|
||||
export const EMAIL_INBOX_STALE_POLL_JOB_ID = "email-inbox-stale-poll" as const;
|
||||
export const EMAIL_INBOX_EVENT_CLEANUP_JOB_ID = "email-inbox-event-cleanup" as const;
|
||||
|
||||
export const EMAIL_INBOX_TOKEN_REFRESH_EVERY_MS = 5 * 60 * 1000;
|
||||
export const EMAIL_INBOX_WEBHOOK_RENEWAL_EVERY_MS = 6 * 60 * 60 * 1000;
|
||||
export const EMAIL_INBOX_WEBHOOK_DRAIN_EVERY_MS = 60 * 1000;
|
||||
export const EMAIL_INBOX_INITIAL_SYNC_EVERY_MS = 2 * 60 * 1000;
|
||||
export const EMAIL_INBOX_STALE_POLL_EVERY_MS = 30 * 60 * 1000;
|
||||
export const EMAIL_INBOX_EVENT_CLEANUP_EVERY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
export const EMAIL_INBOX_TOKEN_REFRESH_WINDOW_MINUTES = 10;
|
||||
export const EMAIL_INBOX_WEBHOOK_RENEWAL_THRESHOLD_MINUTES = 24 * 60;
|
||||
export const EMAIL_INBOX_WEBHOOK_DRAIN_TAKE = 25;
|
||||
export const EMAIL_INBOX_INITIAL_SYNC_TAKE = 10;
|
||||
export const EMAIL_INBOX_STALE_POLL_TAKE = 25;
|
||||
|
||||
type SchedulerQueue = {
|
||||
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
|
||||
};
|
||||
|
||||
export async function registerEmailInboxTokenRefreshCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
EMAIL_INBOX_TOKEN_REFRESH_JOB_ID,
|
||||
{ every: EMAIL_INBOX_TOKEN_REFRESH_EVERY_MS },
|
||||
{ name: EMAIL_INBOX_TOKEN_REFRESH_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_TOKEN_REFRESH_JOB_ID, olderThanMinutes: EMAIL_INBOX_TOKEN_REFRESH_WINDOW_MINUTES } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerEmailInboxWebhookRenewalCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID,
|
||||
{ every: EMAIL_INBOX_WEBHOOK_RENEWAL_EVERY_MS },
|
||||
{ name: EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_WEBHOOK_RENEWAL_JOB_ID, olderThanMinutes: EMAIL_INBOX_WEBHOOK_RENEWAL_THRESHOLD_MINUTES } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerEmailInboxWebhookDrainCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID,
|
||||
{ every: EMAIL_INBOX_WEBHOOK_DRAIN_EVERY_MS },
|
||||
{ name: EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_WEBHOOK_DRAIN_JOB_ID, take: EMAIL_INBOX_WEBHOOK_DRAIN_TAKE } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerEmailInboxInitialSyncCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
EMAIL_INBOX_INITIAL_SYNC_JOB_ID,
|
||||
{ every: EMAIL_INBOX_INITIAL_SYNC_EVERY_MS },
|
||||
{ name: EMAIL_INBOX_INITIAL_SYNC_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_INITIAL_SYNC_JOB_ID, take: EMAIL_INBOX_INITIAL_SYNC_TAKE } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerEmailInboxStalePollCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
EMAIL_INBOX_STALE_POLL_JOB_ID,
|
||||
{ every: EMAIL_INBOX_STALE_POLL_EVERY_MS },
|
||||
{ name: EMAIL_INBOX_STALE_POLL_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_STALE_POLL_JOB_ID, take: EMAIL_INBOX_STALE_POLL_TAKE } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerEmailInboxEventCleanupCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
EMAIL_INBOX_EVENT_CLEANUP_JOB_ID,
|
||||
{ every: EMAIL_INBOX_EVENT_CLEANUP_EVERY_MS },
|
||||
{ name: EMAIL_INBOX_EVENT_CLEANUP_JOB_ID, data: { kind: "cleanup.run", task: EMAIL_INBOX_EVENT_CLEANUP_JOB_ID } },
|
||||
);
|
||||
}
|
||||
+1
-1
@@ -20,7 +20,7 @@
|
||||
* spinning up BullMQ.
|
||||
*/
|
||||
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import { EMAIL_SEQUENCE_TRIGGER_QUEUE } from "./email-sequence-trigger.js";
|
||||
import { EMAIL_SEQUENCE_QUEUE } from "./email-sequence-scheduler.js";
|
||||
|
||||
|
||||
+3
-3
@@ -2,7 +2,7 @@
|
||||
* Marketing-Funnel Phase 3 (Phase 05 #80) — Prisma-backed adapter for
|
||||
* the email-sequence engine + step-runner storage surface.
|
||||
*
|
||||
* The pure-logic package `@toolctrl/email-sequences` accepts an
|
||||
* The pure-logic package `@hivegrid/email-sequences` accepts an
|
||||
* injected `StepRunnerStore` so this module is the production wiring:
|
||||
* EmailSequence + EmailSequenceRun + EmailSequenceStepDelivery rows
|
||||
* mapped to/from the engine types.
|
||||
@@ -19,8 +19,8 @@ import type {
|
||||
SequenceRunStatus,
|
||||
StepDeliveryStatus,
|
||||
SequenceStep,
|
||||
} from "@toolctrl/email-sequences";
|
||||
import { parseSequenceSteps } from "@toolctrl/email-sequences";
|
||||
} from "@hivegrid/email-sequences";
|
||||
import { parseSequenceSteps } from "@hivegrid/email-sequences";
|
||||
|
||||
/**
|
||||
* Narrow Prisma surface used by the adapter. Kept structural so tests
|
||||
|
||||
+8
-8
@@ -5,8 +5,8 @@ import {
|
||||
createEmailSequenceTriggerDeps,
|
||||
} from "./email-sequence-runtime.js";
|
||||
import { EMAIL_SEQUENCE_STEP_JOB_NAME, runEmailSequenceStepJob } from "./email-sequence-scheduler.js";
|
||||
import type { SequenceRunRow, SequenceStep, SequenceTriggerContext } from "@toolctrl/email-sequences";
|
||||
import { verifyUnsubscribeToken } from "@toolctrl/email-sequences/unsubscribe-token";
|
||||
import type { SequenceRunRow, SequenceStep, SequenceTriggerContext } from "@hivegrid/email-sequences";
|
||||
import { verifyUnsubscribeToken } from "@hivegrid/email-sequences/unsubscribe-token";
|
||||
|
||||
const loader = { load: vi.fn() };
|
||||
const store = {
|
||||
@@ -30,11 +30,11 @@ const mailMocks = vi.hoisted(() => ({
|
||||
createMailjetProviderFromEnv: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/db/client.server", () => ({
|
||||
vi.mock("@hivegrid/db/client.server", () => ({
|
||||
getDbClient: vi.fn(() => prisma),
|
||||
}));
|
||||
|
||||
vi.mock("@toolctrl/mail", () => ({
|
||||
vi.mock("@hivegrid/mail", () => ({
|
||||
createMailjetProviderFromEnv: mailMocks.createMailjetProviderFromEnv,
|
||||
}));
|
||||
|
||||
@@ -148,7 +148,7 @@ describe("email sequence runtime deps", () => {
|
||||
delayHours: 0,
|
||||
template: "welcome",
|
||||
subject: "Welcome {{trigger.recipientName}}",
|
||||
variables: { ctaUrl: "https://app.example.test/start", productName: "ToolCTRL Test" },
|
||||
variables: { ctaUrl: "https://app.example.test/start", productName: "HiveGrid Test" },
|
||||
},
|
||||
{
|
||||
triggerEventId: "evt_1",
|
||||
@@ -163,7 +163,7 @@ describe("email sequence runtime deps", () => {
|
||||
expect(rendered).toMatchObject({
|
||||
subject: "Welcome User",
|
||||
});
|
||||
expect(rendered.html).toContain("ToolCTRL Test");
|
||||
expect(rendered.html).toContain("HiveGrid Test");
|
||||
expect(rendered.html).toContain("https://app.example.test/start");
|
||||
expect(rendered.html).toContain("/unsubscribe/u1.");
|
||||
await expect(
|
||||
@@ -202,7 +202,7 @@ describe("email sequence runtime deps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("sends sequence mail through @toolctrl/mail when Mailjet env is configured", async () => {
|
||||
it("sends sequence mail through @hivegrid/mail when Mailjet env is configured", async () => {
|
||||
const log = logger();
|
||||
const deps = await createEmailSequenceSchedulerDeps({
|
||||
logger: log,
|
||||
@@ -252,7 +252,7 @@ describe("email sequence runtime deps", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("maps @toolctrl/mail provider failures to stable sequence error tags", async () => {
|
||||
it("maps @hivegrid/mail provider failures to stable sequence error tags", async () => {
|
||||
const log = logger();
|
||||
mailMocks.provider.send.mockRejectedValue(Object.assign(new Error("provider rejected"), { statusCode: 429 }));
|
||||
const deps = await createEmailSequenceSchedulerDeps({
|
||||
|
||||
+9
-9
@@ -13,17 +13,17 @@
|
||||
* fix the upstream config.
|
||||
*/
|
||||
|
||||
import { parseRedisUrl, type RedisConnectionEnv } from "@toolctrl/queue";
|
||||
import { resolvePath, type StepRunnerDeps, type SequenceEngineQueue } from "@toolctrl/email-sequences";
|
||||
import { buildUnsubscribeUrl as buildSignedUnsubscribeUrl } from "@toolctrl/email-sequences/unsubscribe-token";
|
||||
import type { MailProvider } from "@toolctrl/mail";
|
||||
import { parseRedisUrl, type RedisConnectionEnv } from "@hivegrid/queue";
|
||||
import { resolvePath, type StepRunnerDeps, type SequenceEngineQueue } from "@hivegrid/email-sequences";
|
||||
import { buildUnsubscribeUrl as buildSignedUnsubscribeUrl } from "@hivegrid/email-sequences/unsubscribe-token";
|
||||
import type { MailProvider } from "@hivegrid/mail";
|
||||
import {
|
||||
GenericStepEmail,
|
||||
ReActivationEmail,
|
||||
renderEmail,
|
||||
TrialReminderEmail,
|
||||
WelcomeEmail,
|
||||
} from "@toolctrl/email-templates";
|
||||
} from "@hivegrid/email-templates";
|
||||
import {
|
||||
EMAIL_SEQUENCE_QUEUE,
|
||||
EMAIL_SEQUENCE_STEP_JOB_NAME,
|
||||
@@ -89,7 +89,7 @@ export async function createEmailSequenceTriggerDeps(
|
||||
): Promise<SequenceTriggerDeps | null> {
|
||||
try {
|
||||
const [{ getDbClient }, { createPrismaSequenceLoader, createPrismaSequenceStore }] = await Promise.all([
|
||||
import("@toolctrl/db/client.server"),
|
||||
import("@hivegrid/db/client.server"),
|
||||
import("./email-sequence-prisma-store.js"),
|
||||
]);
|
||||
const prisma = getDbClient() as unknown as Parameters<typeof createPrismaSequenceLoader>[0];
|
||||
@@ -118,7 +118,7 @@ export async function createEmailSequenceSchedulerDeps(
|
||||
): Promise<StepRunnerDeps | null> {
|
||||
try {
|
||||
const [{ getDbClient }, { createPrismaSequenceStore }] = await Promise.all([
|
||||
import("@toolctrl/db/client.server"),
|
||||
import("@hivegrid/db/client.server"),
|
||||
import("./email-sequence-prisma-store.js"),
|
||||
]);
|
||||
const prisma = getDbClient() as unknown as Parameters<typeof createPrismaSequenceStore>[0];
|
||||
@@ -206,7 +206,7 @@ async function createEmailSequenceMailProviderFromEnv(opts: EmailSequenceRuntime
|
||||
}
|
||||
|
||||
try {
|
||||
const { createMailjetProviderFromEnv } = await import("@toolctrl/mail");
|
||||
const { createMailjetProviderFromEnv } = await import("@hivegrid/mail");
|
||||
return createMailjetProviderFromEnv(env);
|
||||
} catch (error) {
|
||||
opts.logger.error(
|
||||
@@ -237,7 +237,7 @@ function renderSequenceTemplate(
|
||||
const vars = step.variables ?? {};
|
||||
const env = opts.env ?? process.env;
|
||||
const appUrl = (env.APP_URL?.trim() || "https://app.example.com").replace(/\/+$/, "");
|
||||
const defaultProductName = env.APP_NAME?.trim() || "ToolCTRL";
|
||||
const defaultProductName = env.APP_NAME?.trim() || "HiveGrid";
|
||||
const productName = stringVariable(vars, trigger, "productName", defaultProductName) ?? defaultProductName;
|
||||
const recipientName = stringVariable(vars, trigger, "recipientName", trigger.recipientName ?? undefined);
|
||||
const unsubscribeUrl = buildSignedUnsubscribeUrl({
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ import {
|
||||
parseEmailSequenceStepJob,
|
||||
runEmailSequenceStepJob,
|
||||
} from "./email-sequence-scheduler.js";
|
||||
import type { StepRunnerDeps } from "@toolctrl/email-sequences";
|
||||
import type { StepRunnerDeps } from "@hivegrid/email-sequences";
|
||||
|
||||
function buildDeps(): StepRunnerDeps {
|
||||
return {
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
* in production; tests pass in-memory fakes.
|
||||
*/
|
||||
|
||||
import { runStep, type StepRunnerDeps, type StepRunResult } from "@toolctrl/email-sequences";
|
||||
import { runStep, type StepRunnerDeps, type StepRunResult } from "@hivegrid/email-sequences";
|
||||
|
||||
export const EMAIL_SEQUENCE_QUEUE = "email-sequence.step" as const;
|
||||
export const EMAIL_SEQUENCE_STEP_JOB_NAME = "run-email-sequence-step" as const;
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import {
|
||||
runSequenceTriggerJob,
|
||||
type SequenceTriggerDeps,
|
||||
} from "./email-sequence-trigger.js";
|
||||
import type { SequenceDefinition } from "@toolctrl/email-sequences";
|
||||
import type { SequenceDefinition } from "@hivegrid/email-sequences";
|
||||
|
||||
function buildSequence(overrides: Partial<SequenceDefinition> = {}): SequenceDefinition {
|
||||
return {
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ import {
|
||||
type SequenceEngineQueue,
|
||||
type SequenceEngineStore,
|
||||
type SequenceTriggerContext,
|
||||
} from "@toolctrl/email-sequences";
|
||||
} from "@hivegrid/email-sequences";
|
||||
|
||||
export const EMAIL_SEQUENCE_TRIGGER_QUEUE = "outbox.marketing" as const;
|
||||
|
||||
|
||||
@@ -2,10 +2,10 @@ import { PassThrough } from "node:stream";
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { TenantPlanTier } from "@toolctrl/exports/limits";
|
||||
import type { ExportRecord } from "@toolctrl/exports/types";
|
||||
import type { ExportJob } from "@toolctrl/queue";
|
||||
import type { SharedStore } from "@toolctrl/shared-store";
|
||||
import type { TenantPlanTier } from "@hivegrid/exports/limits";
|
||||
import type { ExportRecord } from "@hivegrid/exports/types";
|
||||
import type { ExportJob } from "@hivegrid/queue";
|
||||
import type { SharedStore } from "@hivegrid/shared-store";
|
||||
|
||||
import { createExportJobRunner, type ExportRowSource, type ExportStreamerRegistry } from "./export.js";
|
||||
|
||||
|
||||
@@ -2,16 +2,16 @@ import type { Writable } from "node:stream";
|
||||
|
||||
import type { Upload } from "@aws-sdk/lib-storage";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import { serializeExportTableToCsvStream } from "@toolctrl/exports/csv";
|
||||
import { serializeExportTableToJsonStream } from "@toolctrl/exports/json";
|
||||
import { ExportRowCapExceededError, assertExportRowCap, resolveExportRowCap, type ExportFormat, type TenantPlanTier } from "@toolctrl/exports/limits";
|
||||
import { serializeExportTableToPdfStream } from "@toolctrl/exports/pdf";
|
||||
import { createS3MultipartUploadSink } from "@toolctrl/exports/s3-multipart";
|
||||
import type { ExportRecord } from "@toolctrl/exports/types";
|
||||
import { serializeExportTableToXlsxStream } from "@toolctrl/exports/xlsx";
|
||||
import type { SharedStore } from "@toolctrl/shared-store";
|
||||
import type { ExportJob } from "@toolctrl/queue";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import { serializeExportTableToCsvStream } from "@hivegrid/exports/csv";
|
||||
import { serializeExportTableToJsonStream } from "@hivegrid/exports/json";
|
||||
import { ExportRowCapExceededError, assertExportRowCap, resolveExportRowCap, type ExportFormat, type TenantPlanTier } from "@hivegrid/exports/limits";
|
||||
import { serializeExportTableToPdfStream } from "@hivegrid/exports/pdf";
|
||||
import { createS3MultipartUploadSink } from "@hivegrid/exports/s3-multipart";
|
||||
import type { ExportRecord } from "@hivegrid/exports/types";
|
||||
import { serializeExportTableToXlsxStream } from "@hivegrid/exports/xlsx";
|
||||
import type { SharedStore } from "@hivegrid/shared-store";
|
||||
import type { ExportJob } from "@hivegrid/queue";
|
||||
|
||||
import { workerLogger } from "../logger.js";
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// AK-P4a §4 — file-preview service (BullMQ `toolctrl.file-preview`).
|
||||
// AK-P4a §4 — file-preview service (BullMQ `hivegrid.file-preview`).
|
||||
//
|
||||
// Mirrors scanService.ts: a dedicated AWS S3 client built from STORAGE_* env
|
||||
// GETs the original, renders the preview and PUTs preview/thumb into the same
|
||||
@@ -15,14 +15,14 @@
|
||||
// The native render libs (sharp, heic-convert, pdf-to-png-converter) are loaded
|
||||
// via dynamic string import (same pattern as qdrant/sentry in index.ts) so
|
||||
// typecheck/build stay green before they are installed.
|
||||
// ACTIVATION: `pnpm add sharp heic-convert pdf-to-png-converter -F @toolctrl/worker`.
|
||||
// ACTIVATION: `pnpm add sharp heic-convert pdf-to-png-converter -F @hivegrid/worker`.
|
||||
// If a lib is missing at runtime the service falls back cleanly to
|
||||
// previewStatus=failed (no hard crash) and the user can regenerate from detail.
|
||||
|
||||
import { GetObjectCommand, PutObjectCommand, S3Client as AwsS3Client } from "@aws-sdk/client-s3";
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { FilePreviewJob } from "@toolctrl/queue";
|
||||
import { buildFilePreviewKey, buildFileStorageKeys, type FilePreviewExtension } from "@toolctrl/storage";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import type { FilePreviewJob } from "@hivegrid/queue";
|
||||
import { buildFilePreviewKey, buildFileStorageKeys, type FilePreviewExtension } from "@hivegrid/storage";
|
||||
|
||||
import { withTenantRLS } from "../db/tenantRls.js";
|
||||
import type { FilePreviewServiceResult } from "../handlers.js";
|
||||
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
createWorkerFileScanStatusWriter,
|
||||
type FileScanStatusPrisma,
|
||||
} from "./file-scan-status.js";
|
||||
|
||||
/**
|
||||
* Build a Prisma stub whose `file.updateMany` reports `count` matched rows.
|
||||
* The same `updateMany` spy backs both the direct path and the transactional
|
||||
* (RLS) path so a single assertion target works for both.
|
||||
*/
|
||||
function createPrismaStub(count: number) {
|
||||
const updateMany = vi.fn(async () => ({ count }));
|
||||
const executeRaw = vi.fn(async () => 1);
|
||||
const tx = { $executeRaw: executeRaw, file: { updateMany } };
|
||||
const transaction = vi.fn(async (fn: (t: typeof tx) => Promise<unknown>) => fn(tx));
|
||||
const prisma = {
|
||||
$transaction: transaction,
|
||||
file: { updateMany },
|
||||
} as unknown as FileScanStatusPrisma;
|
||||
return { prisma, updateMany, executeRaw, transaction };
|
||||
}
|
||||
|
||||
function createLogger() {
|
||||
return { info: vi.fn(), warn: vi.fn() };
|
||||
}
|
||||
|
||||
describe("createWorkerFileScanStatusWriter — tenant-scoped (RLS) path", () => {
|
||||
it("sets the tenant GUC and writes the verdict inside a transaction", async () => {
|
||||
const { prisma, updateMany, executeRaw, transaction } = createPrismaStub(1);
|
||||
const logger = createLogger();
|
||||
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
|
||||
|
||||
await writer.recordVerdict({ fileId: "file-1", tenantId: "tenant-a", verdict: "infected" });
|
||||
|
||||
expect(transaction).toHaveBeenCalledTimes(1);
|
||||
expect(executeRaw).toHaveBeenCalledTimes(1);
|
||||
expect(updateMany).toHaveBeenCalledWith({
|
||||
where: { id: "file-1", tenantId: "tenant-a" },
|
||||
data: { scanStatus: "infected" },
|
||||
});
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
{ fileId: "file-1", verdict: "infected" },
|
||||
"scan verdict written to File.scanStatus"
|
||||
);
|
||||
expect(logger.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("logs a warning (no throw) when the tenant-scoped write matches no row", async () => {
|
||||
const { prisma, updateMany } = createPrismaStub(0);
|
||||
const logger = createLogger();
|
||||
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
|
||||
|
||||
await expect(
|
||||
writer.recordVerdict({ fileId: "missing", tenantId: "tenant-a", verdict: "clean" })
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(updateMany).toHaveBeenCalledTimes(1);
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
{ fileId: "missing", tenantId: "tenant-a", verdict: "clean" },
|
||||
"scan verdict write-back matched no File row"
|
||||
);
|
||||
expect(logger.info).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("createWorkerFileScanStatusWriter — legacy direct path (no tenantId)", () => {
|
||||
it("writes directly without opening a transaction", async () => {
|
||||
const { prisma, updateMany, transaction } = createPrismaStub(1);
|
||||
const logger = createLogger();
|
||||
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
|
||||
|
||||
await writer.recordVerdict({ fileId: "file-2", verdict: "clean" });
|
||||
|
||||
expect(transaction).not.toHaveBeenCalled();
|
||||
expect(updateMany).toHaveBeenCalledWith({
|
||||
where: { id: "file-2" },
|
||||
data: { scanStatus: "clean" },
|
||||
});
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
{ fileId: "file-2", verdict: "clean" },
|
||||
"scan verdict written to File.scanStatus"
|
||||
);
|
||||
});
|
||||
|
||||
it("warns when the direct write matches no row", async () => {
|
||||
const { prisma, transaction } = createPrismaStub(0);
|
||||
const logger = createLogger();
|
||||
const writer = createWorkerFileScanStatusWriter({ prisma, logger });
|
||||
|
||||
await writer.recordVerdict({ fileId: "gone", verdict: "infected" });
|
||||
|
||||
expect(transaction).not.toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
{ fileId: "gone", tenantId: undefined, verdict: "infected" },
|
||||
"scan verdict write-back matched no File row"
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
// AK-P4a §5 — scan-verdict write-back for the BullMQ `toolctrl.file-scan` queue.
|
||||
// AK-P4a §5 — scan-verdict write-back for the BullMQ `hivegrid.file-scan` queue.
|
||||
//
|
||||
// The scan handler (handlers.ts) calls deps.scanService.scanObject() which runs
|
||||
// ClamAV INSTREAM and quarantines infected objects, but it does NOT touch the
|
||||
|
||||
+4
-4
@@ -86,11 +86,11 @@ describe("planFormSubmissionFollowUps", () => {
|
||||
const events = planFormSubmissionFollowUps(
|
||||
baseInput({
|
||||
webhookUrl: "https://example.com/hook",
|
||||
targetType: "entity-row",
|
||||
target: { id: "row_99" },
|
||||
targetType: "crm-contact",
|
||||
target: { id: "contact_99" },
|
||||
}),
|
||||
);
|
||||
expect(events[0]!.payload.targetType).toBe("entity-row");
|
||||
expect(events[0]!.payload.targetId).toBe("row_99");
|
||||
expect(events[0]!.payload.targetType).toBe("crm-contact");
|
||||
expect(events[0]!.payload.targetId).toBe("contact_99");
|
||||
});
|
||||
});
|
||||
+1
-1
@@ -25,7 +25,7 @@ export interface FormSubmissionFollowUpInput {
|
||||
readonly submissionId: string;
|
||||
readonly formId: string;
|
||||
readonly tenantId: string;
|
||||
readonly targetType: "generic" | "crm-contact" | "entity-row";
|
||||
readonly targetType: "generic" | "crm-contact";
|
||||
readonly target: { id: string | null };
|
||||
readonly webhookUrl: string | null;
|
||||
readonly notifyEmails: readonly string[];
|
||||
|
||||
+3
-3
@@ -4,7 +4,7 @@
|
||||
* Subscribes a BullMQ worker to the `outbox.form-submission-route`
|
||||
* queue inside the `general` worker group. Each delivered job carries
|
||||
* a `form.submission.process` outbox payload; the handler routes the
|
||||
* submission to CRM-contact / entity-row / generic targets and updates
|
||||
* submission to CRM-contact / generic targets and updates
|
||||
* `FormSubmission.status` with the outcome.
|
||||
*
|
||||
* Gating:
|
||||
@@ -16,7 +16,7 @@
|
||||
* semantics without spinning up BullMQ.
|
||||
*/
|
||||
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
|
||||
import {
|
||||
FORM_SUBMISSION_ROUTE_QUEUE,
|
||||
@@ -38,7 +38,7 @@ export interface FormSubmissionRouteWorkerHandle {
|
||||
|
||||
export interface FormSubmissionRouteBootstrapDeps {
|
||||
readonly logger?: FormSubmissionRouteLogger;
|
||||
/** Pre-built handler deps (store + crm + entity + reporter + follow-ups). */
|
||||
/** Pre-built handler deps (store + crm + reporter + follow-ups). */
|
||||
readonly handlerDeps?: FormSubmissionRouteDependencies;
|
||||
/** BullMQ worker factory. Production builds a real Worker; tests fake it. */
|
||||
readonly createWorkerImpl?: (
|
||||
|
||||
+34
-40
@@ -5,7 +5,6 @@ import {
|
||||
parseFormSubmissionRouteJob,
|
||||
runFormSubmissionRouteJob,
|
||||
type CrmContactCreator,
|
||||
type EntityRowCreator,
|
||||
type FormSubmissionStore,
|
||||
} from "./form-submission-route.js";
|
||||
|
||||
@@ -23,10 +22,6 @@ function makeCrm(): CrmContactCreator & { createContact: ReturnType<typeof vi.fn
|
||||
return { createContact: vi.fn(async () => ({ id: "contact_1" })) };
|
||||
}
|
||||
|
||||
function makeEntity(): EntityRowCreator & { createRow: ReturnType<typeof vi.fn> } {
|
||||
return { createRow: vi.fn(async () => ({ id: "row_1" })) };
|
||||
}
|
||||
|
||||
function buildJob(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
outboxEventId: "ob_1",
|
||||
@@ -121,27 +116,6 @@ describe("runFormSubmissionRouteJob", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("creates an entity row for targetType=entity-row", async () => {
|
||||
const store = makeStore();
|
||||
const entity = makeEntity();
|
||||
const result = await runFormSubmissionRouteJob(
|
||||
buildJob({
|
||||
targetType: "entity-row",
|
||||
targetEntityId: "ent_42",
|
||||
fieldMapping: { name: "title" },
|
||||
values: { name: "My Lead" },
|
||||
}),
|
||||
{ store, entity },
|
||||
);
|
||||
expect(result.status).toBe("routed-to-entity");
|
||||
expect(result.target.id).toBe("row_1");
|
||||
expect(entity.createRow).toHaveBeenCalledWith({
|
||||
tenantId: "t_1",
|
||||
entityId: "ent_42",
|
||||
values: { title: "My Lead" },
|
||||
});
|
||||
});
|
||||
|
||||
it("throws and marks route-failed when the CRM adapter is missing", async () => {
|
||||
const store = makeStore();
|
||||
await expect(
|
||||
@@ -152,20 +126,6 @@ describe("runFormSubmissionRouteJob", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("throws and marks route-failed when entity-row routing has no entityId", async () => {
|
||||
const store = makeStore();
|
||||
const entity = makeEntity();
|
||||
await expect(
|
||||
runFormSubmissionRouteJob(
|
||||
buildJob({ targetType: "entity-row", targetEntityId: null }),
|
||||
{ store, entity },
|
||||
),
|
||||
).rejects.toBeInstanceOf(FormSubmissionRouteJobError);
|
||||
expect(store.markRouted).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ status: "route-failed" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("propagates and marks route-failed when the CRM adapter throws", async () => {
|
||||
const store = makeStore();
|
||||
const crm: CrmContactCreator = {
|
||||
@@ -258,4 +218,38 @@ describe("runFormSubmissionRouteJob", () => {
|
||||
expect(result.status).toBe("routed-to-crm");
|
||||
expect(reporter.captureException).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("runs post-submit actions after a successful route", async () => {
|
||||
const store = makeStore();
|
||||
const actionRunner = vi.fn(async () => undefined);
|
||||
await runFormSubmissionRouteJob(buildJob({ values: { email: "a@b.co" } }), { store, actionRunner });
|
||||
expect(actionRunner).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
submissionId: "sub_1",
|
||||
formId: "form_1",
|
||||
tenantId: "t_1",
|
||||
values: { email: "a@b.co" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("logs action failures without failing the already-routed submission", async () => {
|
||||
const store = makeStore();
|
||||
const reporter = { captureException: vi.fn() };
|
||||
const logger = { warn: vi.fn(), error: vi.fn(), info: vi.fn() };
|
||||
const result = await runFormSubmissionRouteJob(buildJob(), {
|
||||
store,
|
||||
actionRunner: async () => {
|
||||
throw new Error("action failed");
|
||||
},
|
||||
reporter,
|
||||
logger,
|
||||
});
|
||||
expect(result.status).toBe("processed");
|
||||
expect(reporter.captureException).toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ op: "form-submission-route.actions-failed" }),
|
||||
"post-submit actions failed after submission routing",
|
||||
);
|
||||
});
|
||||
});
|
||||
+43
-40
@@ -12,11 +12,10 @@
|
||||
* - `generic` → no-op (the row is already stored)
|
||||
* - `crm-contact` → create a CRM contact via `CrmService.createContact`
|
||||
* using the form's `fieldMapping`
|
||||
* - `entity-row` → create a row in `targetEntityId` via `RowsApi.create`
|
||||
*
|
||||
* On success the worker:
|
||||
* - updates `FormSubmission.status` to `routed-to-crm` /
|
||||
* `routed-to-entity` / `processed` (for generic), and
|
||||
* `processed` (for generic), and
|
||||
* - writes the target reference (e.g. created row id) to
|
||||
* `FormSubmission.result`.
|
||||
*
|
||||
@@ -32,7 +31,7 @@
|
||||
* makes manual replays safe.
|
||||
*/
|
||||
|
||||
import { applyFieldMapping } from "@toolctrl/form-widget";
|
||||
import { applyFieldMapping } from "@hivegrid/form-widget";
|
||||
|
||||
export const FORM_SUBMISSION_ROUTE_QUEUE = "outbox.form-submission-route" as const;
|
||||
|
||||
@@ -41,7 +40,6 @@ export type FormSubmissionRoutingStatus =
|
||||
| "received"
|
||||
| "processed"
|
||||
| "routed-to-crm"
|
||||
| "routed-to-entity"
|
||||
| "route-failed";
|
||||
|
||||
export interface FormSubmissionRouteJobData {
|
||||
@@ -52,7 +50,7 @@ export interface FormSubmissionRouteJobData {
|
||||
readonly submissionId: string;
|
||||
readonly formId: string;
|
||||
readonly tenantId: string;
|
||||
readonly targetType: "generic" | "crm-contact" | "entity-row";
|
||||
readonly targetType: "generic" | "crm-contact";
|
||||
readonly targetEntityId: string | null;
|
||||
readonly fieldMapping: Record<string, string>;
|
||||
readonly webhookUrl: string | null;
|
||||
@@ -92,14 +90,6 @@ export interface CrmContactCreator {
|
||||
}): Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
export interface EntityRowCreator {
|
||||
createRow(input: {
|
||||
tenantId: string;
|
||||
entityId: string;
|
||||
values: Record<string, unknown>;
|
||||
}): Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
export interface FormSubmissionRouteLogger {
|
||||
info?: (record: Record<string, unknown>, message?: string) => void;
|
||||
warn?: (record: Record<string, unknown>, message?: string) => void;
|
||||
@@ -113,7 +103,6 @@ export interface FormSubmissionRouteReporter {
|
||||
export interface FormSubmissionRouteDependencies {
|
||||
readonly store: FormSubmissionStore;
|
||||
readonly crm?: CrmContactCreator;
|
||||
readonly entity?: EntityRowCreator;
|
||||
readonly logger?: FormSubmissionRouteLogger;
|
||||
readonly reporter?: FormSubmissionRouteReporter;
|
||||
/**
|
||||
@@ -131,6 +120,14 @@ export interface FormSubmissionRouteDependencies {
|
||||
notifyEmails: readonly string[];
|
||||
values: Record<string, unknown>;
|
||||
}) => Promise<void>;
|
||||
readonly actionRunner?: (input: {
|
||||
submissionId: string;
|
||||
formId: string;
|
||||
tenantId: string;
|
||||
values: Record<string, unknown>;
|
||||
sourceOrigin: string | null;
|
||||
target: { id: string | null; type: FormSubmissionRouteJobData["payload"]["targetType"] };
|
||||
}) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface FormSubmissionRouteJobResult {
|
||||
@@ -145,9 +142,7 @@ export class FormSubmissionRouteJobError extends Error {
|
||||
| "INVALID_PAYLOAD"
|
||||
| "INVALID_EVENT_TYPE"
|
||||
| "TENANT_MISMATCH"
|
||||
| "MISSING_CRM_ADAPTER"
|
||||
| "MISSING_ENTITY_ADAPTER"
|
||||
| "MISSING_TARGET_ENTITY";
|
||||
| "MISSING_CRM_ADAPTER";
|
||||
constructor(message: string, code: FormSubmissionRouteJobError["code"]) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
@@ -193,7 +188,7 @@ export function parseFormSubmissionRouteJob(data: unknown): FormSubmissionRouteJ
|
||||
"INVALID_PAYLOAD",
|
||||
);
|
||||
}
|
||||
if (targetType !== "generic" && targetType !== "crm-contact" && targetType !== "entity-row") {
|
||||
if (targetType !== "generic" && targetType !== "crm-contact") {
|
||||
throw new FormSubmissionRouteJobError(
|
||||
`form-submission-route payload has invalid targetType ${JSON.stringify(targetType)}`,
|
||||
"INVALID_PAYLOAD",
|
||||
@@ -246,7 +241,7 @@ export async function runFormSubmissionRouteJob(
|
||||
deps: FormSubmissionRouteDependencies,
|
||||
): Promise<FormSubmissionRouteJobResult> {
|
||||
const job = parseFormSubmissionRouteJob(data);
|
||||
const { submissionId, formId, tenantId, targetType, targetEntityId, fieldMapping, values } =
|
||||
const { submissionId, formId, tenantId, targetType, fieldMapping, values } =
|
||||
job.payload;
|
||||
|
||||
const existing = await deps.store.load(submissionId);
|
||||
@@ -292,27 +287,6 @@ export async function runFormSubmissionRouteJob(
|
||||
const contact = await deps.crm.createContact({ tenantId, values: mapped });
|
||||
routedTarget = { id: contact.id };
|
||||
newStatus = "routed-to-crm";
|
||||
} else if (targetType === "entity-row") {
|
||||
if (!targetEntityId) {
|
||||
throw new FormSubmissionRouteJobError(
|
||||
"entity-row routing requires targetEntityId",
|
||||
"MISSING_TARGET_ENTITY",
|
||||
);
|
||||
}
|
||||
if (!deps.entity) {
|
||||
throw new FormSubmissionRouteJobError(
|
||||
"Entity adapter not configured",
|
||||
"MISSING_ENTITY_ADAPTER",
|
||||
);
|
||||
}
|
||||
const mapped = applyFieldMapping({ payload: values, fieldMapping });
|
||||
const row = await deps.entity.createRow({
|
||||
tenantId,
|
||||
entityId: targetEntityId,
|
||||
values: mapped,
|
||||
});
|
||||
routedTarget = { id: row.id };
|
||||
newStatus = "routed-to-entity";
|
||||
}
|
||||
|
||||
await deps.store.markRouted({
|
||||
@@ -366,6 +340,35 @@ export async function runFormSubmissionRouteJob(
|
||||
}
|
||||
}
|
||||
|
||||
if (deps.actionRunner) {
|
||||
try {
|
||||
await deps.actionRunner({
|
||||
submissionId,
|
||||
formId,
|
||||
tenantId,
|
||||
values,
|
||||
sourceOrigin: typeof values.__sourceOrigin === "string" ? values.__sourceOrigin : null,
|
||||
target: { id: routedTarget.id, type: targetType },
|
||||
});
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
deps.logger?.warn?.(
|
||||
{
|
||||
op: "form-submission-route.actions-failed",
|
||||
submissionId,
|
||||
err: redactErrorMessage(msg),
|
||||
},
|
||||
"post-submit actions failed after submission routing",
|
||||
);
|
||||
deps.reporter?.captureException?.(err, {
|
||||
component: "form-submission-route",
|
||||
op: "post-submit-actions",
|
||||
submissionId,
|
||||
formId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { submissionId, status: newStatus, target: routedTarget };
|
||||
} catch (err) {
|
||||
const message = redactErrorMessage(err instanceof Error ? err.message : String(err));
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
|
||||
import {
|
||||
FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE,
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { queueNames } from "@toolctrl/queue";
|
||||
import { queueNames } from "@hivegrid/queue";
|
||||
import {
|
||||
FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE,
|
||||
FormSubmissionWebhookDeliveryJobError,
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
/**
|
||||
* Bridges form-submission webhook outbox events onto the existing
|
||||
* `toolctrl.email` webhook delivery processor.
|
||||
* `hivegrid.email` webhook delivery processor.
|
||||
*/
|
||||
|
||||
import {
|
||||
buildWebhookDeliveryJob,
|
||||
type QueueEnqueueOptions,
|
||||
type WebhookDeliveryJobDescriptor,
|
||||
} from "@toolctrl/queue";
|
||||
} from "@hivegrid/queue";
|
||||
|
||||
export const FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE = "outbox.webhook" as const;
|
||||
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { audienceMatches, nextReminderDue } from "./hr-upload-reminder-cron.js";
|
||||
|
||||
// AK-P17b — the cron's pure helpers (cadence + audience) are unit-tested without
|
||||
// a DB; the DB orchestration lives in the maintenance handler.
|
||||
|
||||
describe("nextReminderDue", () => {
|
||||
const from = new Date("2026-01-15T00:00:00.000Z");
|
||||
it("does not repeat a one-off reminder", () => {
|
||||
expect(nextReminderDue({ cadenceKind: "once", intervalMonths: null, from })).toBeNull();
|
||||
});
|
||||
it("advances yearly by 12 months", () => {
|
||||
expect(nextReminderDue({ cadenceKind: "yearly", intervalMonths: null, from })!.getFullYear()).toBe(2027);
|
||||
});
|
||||
it("advances every_n_months by the interval (default 1)", () => {
|
||||
expect(nextReminderDue({ cadenceKind: "every_n_months", intervalMonths: 3, from })!.getMonth()).toBe(3);
|
||||
expect(nextReminderDue({ cadenceKind: "every_n_months", intervalMonths: null, from })!.getMonth()).toBe(1);
|
||||
});
|
||||
it("uses the document expiry for on_expiry", () => {
|
||||
const expiry = new Date("2026-06-30T00:00:00.000Z");
|
||||
expect(nextReminderDue({ cadenceKind: "on_expiry", intervalMonths: null, from, validUntil: expiry })).toEqual(expiry);
|
||||
expect(nextReminderDue({ cadenceKind: "on_expiry", intervalMonths: null, from, validUntil: null })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("audienceMatches", () => {
|
||||
const profile = { team: "platform", locationId: "loc1" };
|
||||
it("matches everyone for 'all'", () => {
|
||||
expect(audienceMatches("all", null, profile)).toBe(true);
|
||||
});
|
||||
it("matches team only on the same team ref", () => {
|
||||
expect(audienceMatches("team", "platform", profile)).toBe(true);
|
||||
expect(audienceMatches("team", "sales", profile)).toBe(false);
|
||||
expect(audienceMatches("team", null, profile)).toBe(false);
|
||||
});
|
||||
it("matches location only on the same location ref", () => {
|
||||
expect(audienceMatches("location", "loc1", profile)).toBe(true);
|
||||
expect(audienceMatches("location", "loc2", profile)).toBe(false);
|
||||
});
|
||||
});
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
// P17 HR-Hub workflows (AK-P17b) — upload-reminder dispatch cron.
|
||||
//
|
||||
// Daily maintenance job. For every active reminder whose `dueAt` has elapsed it
|
||||
// creates per-employee dispatch rows (idempotent via the unique
|
||||
// reminder+employee+dueAt), notifies each employee, advances the reminder by its
|
||||
// cadence, and escalates overdue mandatory dispatches to the employee's manager.
|
||||
//
|
||||
// Mirrors the task-reminder-sweep shape: the execution logic (Prisma + notify())
|
||||
// lives in the maintenance handler which holds the DB client; this file carries
|
||||
// the config + scheduler registration + the PURE helpers (cadence + audience
|
||||
// matching), unit-tested without a DB. The cadence helper intentionally
|
||||
// duplicates @hivegrid/view-models' computeNextDue so the worker stays free of a
|
||||
// view-models dependency; both are unit-tested and must stay in sync.
|
||||
|
||||
export const HR_UPLOAD_REMINDER_DISPATCH_JOB_ID = "hr-upload-reminder-dispatch" as const;
|
||||
/** Daily cadence: reminder due dates are day-granular. */
|
||||
export const HR_UPLOAD_REMINDER_DISPATCH_EVERY_MS = 24 * 60 * 60 * 1000;
|
||||
/** Grace period before a still-unacknowledged mandatory dispatch escalates. */
|
||||
export const HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS = 3 * 24 * 60 * 60 * 1000;
|
||||
|
||||
export type HrReminderCadenceKind = "once" | "yearly" | "every_n_months" | "on_expiry";
|
||||
export type HrAudienceKind = "all" | "team" | "location";
|
||||
|
||||
/** Add `months` calendar months to `date` (clamps to month length). */
|
||||
function addMonths(date: Date, months: number): Date {
|
||||
const result = new Date(date.getTime());
|
||||
result.setMonth(result.getMonth() + months);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure: the NEXT due date for a reminder, or null when it does not repeat.
|
||||
* Mirrors @hivegrid/view-models computeNextDue (kept local to avoid a
|
||||
* view-models dependency in the worker).
|
||||
*/
|
||||
export function nextReminderDue(input: { cadenceKind: HrReminderCadenceKind; intervalMonths: number | null; from: Date; validUntil?: Date | null }): Date | null {
|
||||
switch (input.cadenceKind) {
|
||||
case "once":
|
||||
return null;
|
||||
case "yearly":
|
||||
return addMonths(input.from, 12);
|
||||
case "every_n_months": {
|
||||
const interval = input.intervalMonths && input.intervalMonths > 0 ? Math.floor(input.intervalMonths) : 1;
|
||||
return addMonths(input.from, interval);
|
||||
}
|
||||
case "on_expiry":
|
||||
return input.validUntil ?? null;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure: whether an employee profile falls into a reminder's audience. */
|
||||
export function audienceMatches(audienceKind: HrAudienceKind, audienceRef: string | null, profile: { team: string | null; locationId: string | null }): boolean {
|
||||
if (audienceKind === "all") return true;
|
||||
if (audienceKind === "team") return audienceRef != null && audienceRef === profile.team;
|
||||
if (audienceKind === "location") return audienceRef != null && audienceRef === profile.locationId;
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function registerHrUploadReminderDispatchCron(queue: {
|
||||
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
|
||||
}): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
HR_UPLOAD_REMINDER_DISPATCH_JOB_ID,
|
||||
{ every: HR_UPLOAD_REMINDER_DISPATCH_EVERY_MS },
|
||||
{ name: HR_UPLOAD_REMINDER_DISPATCH_JOB_ID, data: { kind: "cleanup.run", task: HR_UPLOAD_REMINDER_DISPATCH_JOB_ID } },
|
||||
);
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
// R-P5 (P5) §P2d — Apps-Engine maintenance crons: OAuth token refresh + health
|
||||
// sweep. Config + idempotent scheduler registration only; the execution logic
|
||||
// lives in handlers.ts (which holds the Prisma client), mirroring the task
|
||||
// retention/reminder crons.
|
||||
|
||||
export const INTEGRATIONS_TOKEN_REFRESH_JOB_ID = "integrations-token-refresh" as const;
|
||||
export const INTEGRATIONS_HEALTH_SWEEP_JOB_ID = "integrations-health-sweep" as const;
|
||||
|
||||
// Token refresh runs often (tokens expire); health sweep less often.
|
||||
export const INTEGRATIONS_TOKEN_REFRESH_EVERY_MS = 5 * 60 * 1000;
|
||||
export const INTEGRATIONS_HEALTH_SWEEP_EVERY_MS = 15 * 60 * 1000;
|
||||
|
||||
/** Refresh tokens expiring within this window. Default 10 minutes. */
|
||||
export const INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES = 10;
|
||||
|
||||
type SchedulerQueue = {
|
||||
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
|
||||
};
|
||||
|
||||
export async function registerIntegrationsTokenRefreshCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
INTEGRATIONS_TOKEN_REFRESH_JOB_ID,
|
||||
{ every: INTEGRATIONS_TOKEN_REFRESH_EVERY_MS },
|
||||
{ name: INTEGRATIONS_TOKEN_REFRESH_JOB_ID, data: { kind: "cleanup.run", task: INTEGRATIONS_TOKEN_REFRESH_JOB_ID, olderThanMinutes: INTEGRATIONS_TOKEN_REFRESH_WINDOW_MINUTES } },
|
||||
);
|
||||
}
|
||||
|
||||
export async function registerIntegrationsHealthSweepCron(queue: SchedulerQueue): Promise<void> {
|
||||
await queue.upsertJobScheduler(
|
||||
INTEGRATIONS_HEALTH_SWEEP_JOB_ID,
|
||||
{ every: INTEGRATIONS_HEALTH_SWEEP_EVERY_MS },
|
||||
{ name: INTEGRATIONS_HEALTH_SWEEP_JOB_ID, data: { kind: "cleanup.run", task: INTEGRATIONS_HEALTH_SWEEP_JOB_ID } },
|
||||
);
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { LEAD_RETENTION_EVERY_MS, LEAD_RETENTION_JOB_ID, registerLeadRetentionCron, resolveLeadRetentionCronConfig } from "./lead-retention-cron.js";
|
||||
|
||||
describe("lead-retention cron", () => {
|
||||
test("uses a 24h cleanup.run scheduler payload", async () => {
|
||||
const upsertJobScheduler = vi.fn(async () => undefined);
|
||||
|
||||
await registerLeadRetentionCron({ upsertJobScheduler }, { LEAD_RETENTION_BATCH_SIZE: "250" });
|
||||
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
LEAD_RETENTION_JOB_ID,
|
||||
{ every: LEAD_RETENTION_EVERY_MS },
|
||||
{
|
||||
name: LEAD_RETENTION_JOB_ID,
|
||||
data: { kind: "cleanup.run", task: LEAD_RETENTION_JOB_ID, take: 250 },
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("falls back to a positive batch size", () => {
|
||||
expect(resolveLeadRetentionCronConfig({ LEAD_RETENTION_BATCH_SIZE: "-1" })).toEqual({ take: 1000 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
// R-P7 Lead-Inbox — retention sweep scheduler.
|
||||
//
|
||||
// Spam leads are archived after 30 days; lost leads are archived when their
|
||||
// tenant-level anti-spam config sets lostAutoDeleteDays > 0. The web service
|
||||
// stamps InboxLead.autoDeleteAt on status changes, this daily maintenance task
|
||||
// only performs the due archive operation.
|
||||
|
||||
export const LEAD_RETENTION_JOB_ID = "lead-retention-sweep" as const;
|
||||
export const LEAD_RETENTION_EVERY_MS = 24 * 60 * 60 * 1000;
|
||||
export const LEAD_RETENTION_DEFAULT_BATCH_SIZE = 1000;
|
||||
|
||||
export function resolveLeadRetentionCronConfig(env: NodeJS.ProcessEnv = process.env): { take: number } {
|
||||
return {
|
||||
take: parsePositiveInteger(env.LEAD_RETENTION_BATCH_SIZE, LEAD_RETENTION_DEFAULT_BATCH_SIZE),
|
||||
};
|
||||
}
|
||||
|
||||
export async function registerLeadRetentionCron(
|
||||
queue: {
|
||||
upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
|
||||
},
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<void> {
|
||||
const config = resolveLeadRetentionCronConfig(env);
|
||||
await queue.upsertJobScheduler(
|
||||
LEAD_RETENTION_JOB_ID,
|
||||
{ every: LEAD_RETENTION_EVERY_MS },
|
||||
{
|
||||
name: LEAD_RETENTION_JOB_ID,
|
||||
data: {
|
||||
kind: "cleanup.run",
|
||||
task: LEAD_RETENTION_JOB_ID,
|
||||
take: config.take,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value: string | undefined, fallback: number): number {
|
||||
if (!value) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
||||
return Math.floor(parsed);
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mapWithConcurrency } from "@toolctrl/foundation";
|
||||
import { mapWithConcurrency } from "@hivegrid/foundation";
|
||||
|
||||
/**
|
||||
* Marketing-Funnel Phase 3 (Phase 05 #80) — stale-lead trigger cron.
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
bootstrapLegacyEventReceiver,
|
||||
type LegacyEventReceiverBootstrapDeps,
|
||||
type LegacyEventReceiverWorkerHandle,
|
||||
} from "./legacy-event-receiver-bootstrap.js";
|
||||
import {
|
||||
LEGACY_EVENT_OUTBOX_QUEUE,
|
||||
type LegacyEventReceiverStore,
|
||||
} from "./legacy-event-receiver.js";
|
||||
import type { OutboxReceiverIdempotencyEntry } from "@hivegrid/db/outbox-receiver";
|
||||
|
||||
function createStore(): LegacyEventReceiverStore & { calls: number; last: unknown } {
|
||||
const state = { calls: 0, last: null as unknown };
|
||||
return {
|
||||
get calls() {
|
||||
return state.calls;
|
||||
},
|
||||
get last() {
|
||||
return state.last;
|
||||
},
|
||||
async create(data) {
|
||||
state.calls += 1;
|
||||
state.last = data;
|
||||
return { id: `evt-row-${state.calls}` };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Capturing worker factory: records the queue name + processor it was handed
|
||||
* so the test can drive the processor directly (no real BullMQ).
|
||||
*/
|
||||
function createCapturingWorkerImpl() {
|
||||
const captured: {
|
||||
queueName?: string;
|
||||
processor?: (job: { data: unknown }) => Promise<unknown>;
|
||||
concurrency?: number;
|
||||
} = {};
|
||||
const handle: LegacyEventReceiverWorkerHandle = {
|
||||
name: "legacy-event-receiver-worker",
|
||||
close: vi.fn(async () => undefined),
|
||||
};
|
||||
const createWorkerImpl: NonNullable<LegacyEventReceiverBootstrapDeps["createWorkerImpl"]> = (
|
||||
queueName,
|
||||
processor,
|
||||
options
|
||||
) => {
|
||||
captured.queueName = queueName;
|
||||
captured.processor = processor;
|
||||
captured.concurrency = options.concurrency;
|
||||
return handle;
|
||||
};
|
||||
return { createWorkerImpl, captured, handle };
|
||||
}
|
||||
|
||||
function validJob() {
|
||||
return {
|
||||
data: {
|
||||
outboxEventId: "evt-1",
|
||||
tenantId: "tenant-a",
|
||||
eventType: "billing.subscription.ended",
|
||||
payload: {
|
||||
name: "subscription.ended",
|
||||
tenantId: "tenant-a",
|
||||
userId: null,
|
||||
data: JSON.stringify({ tenant: { id: "tenant-a" } }),
|
||||
description: "Acme subscription ended",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("bootstrapLegacyEventReceiver", () => {
|
||||
it("disables on a non-general worker group", () => {
|
||||
const result = bootstrapLegacyEventReceiver({ WORKER_QUEUE_GROUP: "mail" });
|
||||
expect(result).toEqual({ status: "disabled", reason: "non-general worker group" });
|
||||
});
|
||||
|
||||
it("disables when LEGACY_EVENT_RECEIVER_ENABLED=false (escape hatch)", () => {
|
||||
const { createWorkerImpl } = createCapturingWorkerImpl();
|
||||
const result = bootstrapLegacyEventReceiver(
|
||||
{ WORKER_QUEUE_GROUP: "general", LEGACY_EVENT_RECEIVER_ENABLED: "false" },
|
||||
{ store: createStore(), createWorkerImpl }
|
||||
);
|
||||
expect(result).toEqual({ status: "disabled", reason: "LEGACY_EVENT_RECEIVER_ENABLED=false" });
|
||||
});
|
||||
|
||||
it("disables when no store is wired", () => {
|
||||
const { createWorkerImpl } = createCapturingWorkerImpl();
|
||||
const result = bootstrapLegacyEventReceiver({}, { createWorkerImpl });
|
||||
expect(result.status).toBe("disabled");
|
||||
expect(result).toMatchObject({ reason: expect.stringContaining("no store wired") });
|
||||
});
|
||||
|
||||
it("disables when no createWorkerImpl is wired", () => {
|
||||
const result = bootstrapLegacyEventReceiver({}, { store: createStore() });
|
||||
expect(result.status).toBe("disabled");
|
||||
expect(result).toMatchObject({ reason: expect.stringContaining("no createWorkerImpl wired") });
|
||||
});
|
||||
|
||||
it("starts on the outbox.notification queue with the default concurrency and logs", () => {
|
||||
const { createWorkerImpl, captured, handle } = createCapturingWorkerImpl();
|
||||
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
|
||||
const result = bootstrapLegacyEventReceiver({}, { store: createStore(), createWorkerImpl, logger });
|
||||
|
||||
expect(result).toEqual({ status: "started", handle });
|
||||
expect(captured.queueName).toBe(LEGACY_EVENT_OUTBOX_QUEUE);
|
||||
expect(captured.concurrency).toBe(5);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ component: "legacy-event-receiver", concurrency: 5 }),
|
||||
"legacy-event receiver worker started"
|
||||
);
|
||||
});
|
||||
|
||||
it("parses a positive concurrency override", () => {
|
||||
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
|
||||
bootstrapLegacyEventReceiver(
|
||||
{ LEGACY_EVENT_RECEIVER_CONCURRENCY: "12" },
|
||||
{ store: createStore(), createWorkerImpl }
|
||||
);
|
||||
expect(captured.concurrency).toBe(12);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["0", 5],
|
||||
["-3", 5],
|
||||
["abc", 5],
|
||||
["", 5],
|
||||
])("falls back to the default concurrency for %s", (raw, expected) => {
|
||||
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
|
||||
bootstrapLegacyEventReceiver(
|
||||
{ LEGACY_EVENT_RECEIVER_CONCURRENCY: raw },
|
||||
{ store: createStore(), createWorkerImpl }
|
||||
);
|
||||
expect(captured.concurrency).toBe(expected);
|
||||
});
|
||||
|
||||
it("wires a processor that feeds job.data through the receiver and persists", async () => {
|
||||
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
|
||||
const store = createStore();
|
||||
|
||||
bootstrapLegacyEventReceiver({}, { store, createWorkerImpl });
|
||||
|
||||
expect(captured.processor).toBeTypeOf("function");
|
||||
const outcome = (await captured.processor!(validJob())) as { status: string };
|
||||
|
||||
expect(outcome.status).toBe("persisted");
|
||||
expect(store.calls).toBe(1);
|
||||
});
|
||||
|
||||
it("uses idempotency to skip a re-delivered outbox event", async () => {
|
||||
const { createWorkerImpl, captured } = createCapturingWorkerImpl();
|
||||
const store = createStore();
|
||||
// In-memory idempotency store implementing the full claim/complete/get
|
||||
// contract that withOutboxIdempotency depends on.
|
||||
const entries = new Map<string, OutboxReceiverIdempotencyEntry>();
|
||||
const idempotency = {
|
||||
claim: vi.fn(async (key: string) => {
|
||||
if (entries.has(key)) return false;
|
||||
entries.set(key, { state: "pending" });
|
||||
return true;
|
||||
}),
|
||||
complete: vi.fn(async (key: string, result: unknown) => {
|
||||
entries.set(key, { state: "complete", result });
|
||||
}),
|
||||
get: vi.fn(async (key: string) => entries.get(key) ?? null),
|
||||
};
|
||||
|
||||
bootstrapLegacyEventReceiver({}, { store, createWorkerImpl, idempotency });
|
||||
|
||||
const first = (await captured.processor!(validJob())) as { status: string };
|
||||
const second = (await captured.processor!(validJob())) as { status: string };
|
||||
|
||||
expect(first.status).toBe("persisted");
|
||||
expect(second.status).toBe("skipped");
|
||||
// The duplicate must not produce a second Event row.
|
||||
expect(store.calls).toBe(1);
|
||||
});
|
||||
});
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
* - LEGACY_EVENT_RECEIVER_ENABLED !== "false" (escape hatch)
|
||||
*/
|
||||
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
|
||||
import {
|
||||
LEGACY_EVENT_OUTBOX_QUEUE,
|
||||
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Outbox receiver for "legacy" Event-table inserts.
|
||||
*
|
||||
* Phase 03 #42 — Stripe-webhook migration. The Stripe webhook used to call
|
||||
* Phase 03 #42 — Mollie-webhook migration. The Mollie webhook used to call
|
||||
* `createLegacyEvent` directly after updating a TenantSubscriptionProduct,
|
||||
* meaning the DB write + the legacy-Event row were committed in two
|
||||
* separate transactions; a worker crash between the two would silently
|
||||
@@ -22,7 +22,7 @@
|
||||
import {
|
||||
withOutboxIdempotency,
|
||||
type OutboxReceiverIdempotencyStore,
|
||||
} from "@toolctrl/db/outbox-receiver";
|
||||
} from "@hivegrid/db/outbox-receiver";
|
||||
|
||||
export const LEGACY_EVENT_OUTBOX_QUEUE = "outbox.notification" as const;
|
||||
|
||||
@@ -74,7 +74,7 @@ export class LegacyEventReceiverJobError extends Error {
|
||||
export interface LegacyEventReceiverStore {
|
||||
/**
|
||||
* Insert the Event row. Production wires `createLegacyEvent` from
|
||||
* `@toolctrl/db/events.server`; tests pass an in-memory recorder.
|
||||
* `@hivegrid/db/events.server`; tests pass an in-memory recorder.
|
||||
*/
|
||||
create(data: LegacyEventReceiverPayload): Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@
|
||||
* the outbox row.
|
||||
*/
|
||||
|
||||
import { resolveAdapter, type MarketingLeadEventType, type MarketingLeadPayload } from "@toolctrl/marketing-outbound";
|
||||
import { withOutboxIdempotency, type OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
|
||||
import { resolveAdapter, type MarketingLeadEventType, type MarketingLeadPayload } from "@hivegrid/marketing-outbound";
|
||||
import { withOutboxIdempotency, type OutboxReceiverIdempotencyStore } from "@hivegrid/db/outbox-receiver";
|
||||
|
||||
export const MARKETING_OUTBOUND_ACTIVE_SUBSCRIBER = false as const;
|
||||
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import { describePluginRuntimeProductionGate } from "@toolctrl/plugin-runtime";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import { describePluginRuntimeProductionGate } from "@hivegrid/plugin-runtime";
|
||||
|
||||
import {
|
||||
PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE,
|
||||
|
||||
+1
-1
@@ -270,7 +270,7 @@ export async function handlePluginCapabilityUseJob(
|
||||
// per-tenant; the store optionally exposes getChainHead so the
|
||||
// chain survives across batches (otherwise each batch is a fresh
|
||||
// mini-chain — still useful for intra-batch integrity).
|
||||
const { stampPluginCapabilityUseChain } = await import("@toolctrl/plugin-runtime");
|
||||
const { stampPluginCapabilityUseChain } = await import("@hivegrid/plugin-runtime");
|
||||
const seedPrevHash = deps.store.getChainHead
|
||||
? await deps.store.getChainHead(parsed.payload.tenantId).catch(() => null)
|
||||
: null;
|
||||
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
|
||||
import { buildPluginInvokeJob, enqueueJob, queueNames, type QueueEnqueueOptions, type TypedQueue } from "@toolctrl/queue";
|
||||
import { describePluginRuntimeProductionGate } from "@toolctrl/plugin-runtime";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import type { OutboxReceiverIdempotencyStore } from "@hivegrid/db/outbox-receiver";
|
||||
import { buildPluginInvokeJob, enqueueJob, queueNames, type QueueEnqueueOptions, type TypedQueue } from "@hivegrid/queue";
|
||||
import { describePluginRuntimeProductionGate } from "@hivegrid/plugin-runtime";
|
||||
|
||||
import {
|
||||
PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE,
|
||||
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
import { queueNames } from "@toolctrl/queue";
|
||||
import type { OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
|
||||
import { registerHook } from "@toolctrl/plugin-runtime";
|
||||
import { queueNames } from "@hivegrid/queue";
|
||||
import type { OutboxReceiverIdempotencyStore } from "@hivegrid/db/outbox-receiver";
|
||||
import { registerHook } from "@hivegrid/plugin-runtime";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { buildPluginInvokeJob, type QueueEnqueueOptions } from "@toolctrl/queue";
|
||||
import { getPluginHook, isPluginHookProductionAvailable, parsePluginManifest, validatePluginHookPayload } from "@toolctrl/plugin-runtime";
|
||||
import { withOutboxIdempotency, type OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
|
||||
import { buildPluginInvokeJob, type QueueEnqueueOptions } from "@hivegrid/queue";
|
||||
import { getPluginHook, isPluginHookProductionAvailable, parsePluginManifest, validatePluginHookPayload } from "@hivegrid/plugin-runtime";
|
||||
import { withOutboxIdempotency, type OutboxReceiverIdempotencyStore } from "@hivegrid/db/outbox-receiver";
|
||||
|
||||
export const PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE = "outbox.plugin-hook" as const;
|
||||
export const PLUGIN_HOOK_EVENT_PREFIX = "plugin.hook." as const;
|
||||
|
||||
+35
-26
@@ -1,3 +1,12 @@
|
||||
// @vitest-environment node
|
||||
//
|
||||
// This is a backend worker suite. `invokeInSandbox` spawns a worker thread and,
|
||||
// when no compiled sandbox worker exists, bundles the source with esbuild.
|
||||
// esbuild asserts `new TextEncoder().encode("") instanceof Uint8Array`, which is
|
||||
// false under the repo-wide jsdom default (jsdom swaps the `Uint8Array` global
|
||||
// for its own realm). Pin this file to the node environment so the invariant
|
||||
// holds and esbuild can run.
|
||||
|
||||
/**
|
||||
* Phase 06 #21 / F-015 — bootstrapPluginInvokeDeps wiring tests.
|
||||
*
|
||||
@@ -109,7 +118,7 @@ const INFINITE_LOOP_WASM = new Uint8Array([
|
||||
function manifestWithCapabilities(capabilities: Array<{ name: "log" | "audit" }>) {
|
||||
return {
|
||||
schemaVersion: 1 as const,
|
||||
id: "de.toolctrl.test",
|
||||
id: "de.hivegrid.test",
|
||||
version: "1.0.0",
|
||||
name: "Test",
|
||||
entry: "plugin.wasm",
|
||||
@@ -167,12 +176,12 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
logger: noopLogger,
|
||||
});
|
||||
|
||||
await expect(deps.loadCapabilityGrants({ tenantId: "tenant-1", pluginId: "de.toolctrl.test" })).resolves.toEqual([
|
||||
await expect(deps.loadCapabilityGrants({ tenantId: "tenant-1", pluginId: "de.hivegrid.test" })).resolves.toEqual([
|
||||
{ capabilityName: "log", scopeJson: null },
|
||||
{ capabilityName: "http.outbound", scopeJson: { allowedHosts: ["api.example.com"] } },
|
||||
]);
|
||||
expect(prisma.tenantPluginCapabilityGrant.findMany).toHaveBeenCalledWith({
|
||||
where: { tenantId: "tenant-1", pluginId: "de.toolctrl.test" },
|
||||
where: { tenantId: "tenant-1", pluginId: "de.hivegrid.test" },
|
||||
select: { capabilityName: true, scopeJson: true },
|
||||
});
|
||||
});
|
||||
@@ -217,7 +226,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const manifest = manifestWithCapabilities([{ name: "log" }]);
|
||||
prisma.plugin.findUnique = vi.fn(async () => ({
|
||||
tenantId: "t",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
version: "1.0.0",
|
||||
manifest,
|
||||
wasmSha256: "0".repeat(64),
|
||||
@@ -231,9 +240,9 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
logger: noopLogger,
|
||||
});
|
||||
|
||||
await expect(deps.loadPlugin({ tenantId: "t", pluginId: "de.toolctrl.test", version: "1.0.0" })).resolves.toMatchObject({
|
||||
await expect(deps.loadPlugin({ tenantId: "t", pluginId: "de.hivegrid.test", version: "1.0.0" })).resolves.toMatchObject({
|
||||
tenantId: "t",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
manifest,
|
||||
enabled: true,
|
||||
});
|
||||
@@ -243,7 +252,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const prisma = makeFakePrisma();
|
||||
prisma.plugin.findUnique = vi.fn(async () => ({
|
||||
tenantId: "t",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
version: "1.0.0",
|
||||
manifest: { hooks: [{ name: "run", export: "run" }] },
|
||||
wasmSha256: "0".repeat(64),
|
||||
@@ -258,11 +267,11 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
logger,
|
||||
});
|
||||
|
||||
await expect(deps.loadPlugin({ tenantId: "t", pluginId: "de.toolctrl.test", version: "1.0.0" })).resolves.toBeNull();
|
||||
await expect(deps.loadPlugin({ tenantId: "t", pluginId: "de.hivegrid.test", version: "1.0.0" })).resolves.toBeNull();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tenantId: "t",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
version: "1.0.0",
|
||||
}),
|
||||
"plugin.invoke.stored_manifest_invalid"
|
||||
@@ -283,7 +292,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
test("loadBinary refuses plugin paths that escape the configured base directory", async () => {
|
||||
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
||||
const deps = bootstrapPluginInvokeDeps({
|
||||
env: { PLUGIN_BINARY_BASE_PATH: "/tmp/toolctrl-plugins" },
|
||||
env: { PLUGIN_BINARY_BASE_PATH: "/tmp/hivegrid-plugins" },
|
||||
prisma: makeFakePrisma(),
|
||||
store: makeFakeStore() as never,
|
||||
logger,
|
||||
@@ -329,7 +338,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const result = await deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-1",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: HOST_IMPORT_WASM,
|
||||
manifest: manifestWithCapabilities(capabilities),
|
||||
hookName: "run",
|
||||
@@ -346,19 +355,19 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
priority: 20,
|
||||
payload: expect.objectContaining({
|
||||
tenantId: "tenant-1",
|
||||
actorId: "de.toolctrl.test",
|
||||
actorId: "de.hivegrid.test",
|
||||
actorType: "plugin",
|
||||
action: "plugin.audit.recorded",
|
||||
targetType: "plugin",
|
||||
targetId: "de.toolctrl.test",
|
||||
targetId: "de.hivegrid.test",
|
||||
requestId: "inv-1",
|
||||
sourceModule: "worker.plugin-runtime",
|
||||
outcome: "success",
|
||||
dataSensitivity: "internal",
|
||||
regulatoryScope: ["soc2"],
|
||||
metadata: expect.objectContaining({
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginManifestId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
pluginManifestId: "de.hivegrid.test",
|
||||
pluginInvocationId: "inv-1",
|
||||
version: "1.0.0",
|
||||
hookName: "run",
|
||||
@@ -388,7 +397,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
]);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
value: { value: 7 },
|
||||
version: "1.0.0",
|
||||
}),
|
||||
@@ -396,7 +405,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
value: { value: 7 },
|
||||
version: "1.0.0",
|
||||
}),
|
||||
@@ -405,7 +414,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
pluginInvocationId: "inv-1",
|
||||
sequence: 0,
|
||||
}),
|
||||
@@ -429,7 +438,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-audit-down",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: HOST_IMPORT_WASM,
|
||||
manifest: manifestWithCapabilities(capabilities),
|
||||
hookName: "run",
|
||||
@@ -441,7 +450,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
expect.objectContaining({
|
||||
pluginInvocationId: "inv-audit-down",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
entryCount: expect.any(Number),
|
||||
}),
|
||||
"plugin capability-use audit flush failed"
|
||||
@@ -463,7 +472,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-audit-truncated",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: HOST_IMPORT_WASM,
|
||||
manifest: manifestWithCapabilities(capabilities),
|
||||
hookName: "run",
|
||||
@@ -501,7 +510,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const result = await deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-duplicate",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: HOST_IMPORT_WASM,
|
||||
manifest: manifestWithCapabilities(capabilities),
|
||||
hookName: "run",
|
||||
@@ -547,7 +556,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const result = await deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-host-audit-duplicate",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: HOST_IMPORT_WASM,
|
||||
manifest: manifestWithCapabilities(capabilities),
|
||||
hookName: "run",
|
||||
@@ -588,7 +597,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const result = await deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-2",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: HOST_IMPORT_WASM,
|
||||
manifest: manifestWithCapabilities(capabilities),
|
||||
hookName: "run",
|
||||
@@ -642,7 +651,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const result = await deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-missing-grant",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: HOST_IMPORT_WASM,
|
||||
manifest: manifestWithCapabilities(capabilities),
|
||||
hookName: "run",
|
||||
@@ -689,7 +698,7 @@ describe("bootstrapPluginInvokeDeps", () => {
|
||||
const result = await deps.invokeInSandbox({
|
||||
pluginInvocationId: "inv-timeout",
|
||||
tenantId: "tenant-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
wasmBuffer: INFINITE_LOOP_WASM,
|
||||
manifest: manifestWithCapabilities([]),
|
||||
hookName: "run",
|
||||
|
||||
+3
-3
@@ -39,9 +39,9 @@ import {
|
||||
type PluginCapabilityGrant,
|
||||
type PluginCapabilityUseBatch,
|
||||
type PluginInvocationResult,
|
||||
} from "@toolctrl/plugin-runtime";
|
||||
import { AUDIT_EVENT_OUTBOX_TYPE, buildAuditEventCreateData } from "@toolctrl/db/audit-events.server";
|
||||
import type { SharedStore } from "@toolctrl/shared-store";
|
||||
} from "@hivegrid/plugin-runtime";
|
||||
import { AUDIT_EVENT_OUTBOX_TYPE, buildAuditEventCreateData } from "@hivegrid/db/audit-events.server";
|
||||
import type { SharedStore } from "@hivegrid/shared-store";
|
||||
|
||||
import type {
|
||||
PluginInvokeDeps,
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import type { PluginCapabilityGrant, PluginManifest } from "@toolctrl/plugin-runtime";
|
||||
import type { PluginCapabilityGrant, PluginManifest } from "@hivegrid/plugin-runtime";
|
||||
import { handlePluginInvokeJob, type PluginInvokeDeps, type PluginInvokeJobPayload, type PluginInvokeRow } from "./plugin-invoke.js";
|
||||
|
||||
function manifest(): PluginManifest {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
id: "de.toolctrl.test",
|
||||
id: "de.hivegrid.test",
|
||||
version: "1.0.0",
|
||||
name: "Test",
|
||||
entry: "plugin.wasm",
|
||||
@@ -36,7 +36,7 @@ function outboundManifest(): PluginManifest {
|
||||
function row(): PluginInvokeRow {
|
||||
return {
|
||||
tenantId: "t-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
version: "1.0.0",
|
||||
manifest: manifest(),
|
||||
wasmSha256: "0".repeat(64),
|
||||
@@ -55,7 +55,7 @@ function grants(): PluginCapabilityGrant[] {
|
||||
function payload(): PluginInvokeJobPayload {
|
||||
return {
|
||||
tenantId: "t-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
version: "1.0.0",
|
||||
hookName: "ping",
|
||||
input: { x: 1 },
|
||||
@@ -92,7 +92,7 @@ describe("handlePluginInvokeJob", () => {
|
||||
expect(deps.invokeInSandbox).toHaveBeenCalledTimes(1);
|
||||
expect(deps.loadCapabilityGrants).toHaveBeenCalledWith({
|
||||
tenantId: "t-1",
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
});
|
||||
const sandboxArgs = vi.mocked(deps.invokeInSandbox).mock.calls[0]?.[0];
|
||||
const auditArgs = vi.mocked(deps.recordAudit).mock.calls[0]?.[0];
|
||||
@@ -214,7 +214,7 @@ describe("handlePluginInvokeJob", () => {
|
||||
expect(result.reason).toBe("capability_grant_missing");
|
||||
expect(result.message).toContain("hook:tenant.created");
|
||||
}
|
||||
expect(deps.loadCapabilityGrants).toHaveBeenCalledWith({ tenantId: "t-1", pluginId: "de.toolctrl.test" });
|
||||
expect(deps.loadCapabilityGrants).toHaveBeenCalledWith({ tenantId: "t-1", pluginId: "de.hivegrid.test" });
|
||||
expect(deps.loadBinary).not.toHaveBeenCalled();
|
||||
expect(deps.invokeInSandbox).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -386,7 +386,7 @@ describe("handlePluginInvokeJob", () => {
|
||||
await expect(handlePluginInvokeJob(payload(), deps)).rejects.toThrow("audit-down");
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
pluginId: "de.toolctrl.test",
|
||||
pluginId: "de.hivegrid.test",
|
||||
pluginInvocationId: expect.any(String),
|
||||
error: "audit-down",
|
||||
}),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Phase 06 #21 — Plugin-invoke worker job.
|
||||
*
|
||||
* Handles one job from the `toolctrl.plugin-invoke` queue:
|
||||
* Handles one job from the `hivegrid.plugin-invoke` queue:
|
||||
* 1. Loads the Plugin row + .wasm binary for the requested
|
||||
* (tenantId, pluginId, version) tuple.
|
||||
* 2. Verifies the binary's SHA-256 matches the stored hash and the
|
||||
@@ -34,7 +34,7 @@ import {
|
||||
type SanitizationStrictness,
|
||||
type VerifyPluginSignatureResult,
|
||||
validatePluginHookPayload,
|
||||
} from "@toolctrl/plugin-runtime";
|
||||
} from "@hivegrid/plugin-runtime";
|
||||
|
||||
/**
|
||||
* Resolve the sanitization strictness from env. Default is `moderate`
|
||||
|
||||
+9
-2
@@ -1,6 +1,13 @@
|
||||
// @vitest-environment node
|
||||
//
|
||||
// `invokePluginSandboxInWorker` bundles the sandbox worker with esbuild when no
|
||||
// compiled output is present. esbuild asserts
|
||||
// `new TextEncoder().encode("") instanceof Uint8Array`, which is false under the
|
||||
// repo-wide jsdom default (jsdom replaces the `Uint8Array` global). Pin this
|
||||
// backend suite to the node environment so the invariant holds.
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import type { PluginManifest } from "@toolctrl/plugin-runtime";
|
||||
import type { PluginManifest } from "@hivegrid/plugin-runtime";
|
||||
|
||||
import { invokePluginSandboxInWorker, resolveSandboxWorkerLocationForTest } from "./plugin-sandbox-isolated.js";
|
||||
import { runSandboxForTest } from "./plugin-sandbox-worker.js";
|
||||
@@ -29,7 +36,7 @@ const HOST_IMPORT_WASM = new Uint8Array([
|
||||
|
||||
const manifest: PluginManifest = {
|
||||
schemaVersion: 1,
|
||||
id: "de.toolctrl.timeout-test",
|
||||
id: "de.hivegrid.timeout-test",
|
||||
version: "1.0.0",
|
||||
name: "Timeout Test",
|
||||
entry: "plugin.wasm",
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ import { existsSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
import type { CapabilityCallInfo, PluginCapabilityGrant, PluginInvocationResult, PluginManifest } from "@toolctrl/plugin-runtime";
|
||||
import type { CapabilityCallInfo, PluginCapabilityGrant, PluginInvocationResult, PluginManifest } from "@hivegrid/plugin-runtime";
|
||||
|
||||
import type { SandboxWorkerResponse } from "./plugin-sandbox-worker.js";
|
||||
|
||||
|
||||
+9
-2
@@ -1,6 +1,13 @@
|
||||
// @vitest-environment node
|
||||
//
|
||||
// `invokePluginSandboxInWorker` bundles the sandbox worker with esbuild when no
|
||||
// compiled output is present. esbuild asserts
|
||||
// `new TextEncoder().encode("") instanceof Uint8Array`, which is false under the
|
||||
// repo-wide jsdom default (jsdom replaces the `Uint8Array` global). Pin this
|
||||
// backend suite to the node environment so the invariant holds.
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import type { PluginManifest } from "@toolctrl/plugin-runtime";
|
||||
import type { PluginManifest } from "@hivegrid/plugin-runtime";
|
||||
|
||||
import { invokePluginSandboxInWorker } from "./plugin-sandbox-isolated.js";
|
||||
import { createPluginSandboxHostFns, runSandboxForTest } from "./plugin-sandbox-worker.js";
|
||||
@@ -40,7 +47,7 @@ const HOST_IMPORT_WASM = new Uint8Array([
|
||||
function manifest(capabilities: PluginManifest["capabilities"] = []): PluginManifest {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
id: "de.toolctrl.worker-test",
|
||||
id: "de.hivegrid.worker-test",
|
||||
version: "1.0.0",
|
||||
name: "Worker Test",
|
||||
entry: "plugin.wasm",
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import {
|
||||
type PluginCapabilityGrant,
|
||||
type PluginInvocationResult,
|
||||
type PluginManifest,
|
||||
} from "@toolctrl/plugin-runtime";
|
||||
} from "@hivegrid/plugin-runtime";
|
||||
|
||||
type SandboxWorkerRequest = {
|
||||
wasmBuffer: Uint8Array;
|
||||
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { buildExportJob } from "@hivegrid/queue";
|
||||
|
||||
import {
|
||||
createWorkerPrivacyGdprExportService,
|
||||
loadPrivacyGdprBundle,
|
||||
type PrivacyGdprExportPrisma,
|
||||
} from "./privacy-gdpr-export-bootstrap.js";
|
||||
import {
|
||||
PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE,
|
||||
PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
} from "./privacy-gdpr-export.js";
|
||||
|
||||
function prismaMock(): PrivacyGdprExportPrisma {
|
||||
return {
|
||||
user: {
|
||||
findUnique: vi.fn(async () => ({ id: "user_1", email: "person@example.test" })),
|
||||
},
|
||||
tenant: {
|
||||
findUnique: vi.fn(async () => ({ id: "tenant_1", slug: "acme", name: "Acme" })),
|
||||
},
|
||||
tenantUser: {
|
||||
findMany: vi.fn(async () => [{ id: "membership_1", tenantId: "tenant_1", userId: "user_1" }]),
|
||||
},
|
||||
auditEvent: {
|
||||
findMany: vi.fn(async () => [{ id: "audit_1", tenantId: "tenant_1", actorId: "user_1", action: "LOGIN" }]),
|
||||
},
|
||||
notificationPreference: {
|
||||
findMany: vi.fn(async () => [{ id: "pref_1", channel: "EMAIL", enabled: true }]),
|
||||
},
|
||||
notificationDelivery: {
|
||||
findMany: vi.fn(async () => []),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("privacy GDPR export bootstrap", () => {
|
||||
it("loads a scoped self-export bundle from existing tenant tables", async () => {
|
||||
const prisma = prismaMock();
|
||||
const descriptor = buildExportJob({
|
||||
exportJobId: "gdpr_export_1",
|
||||
tenantId: "tenant_1",
|
||||
requestedByUserId: "user_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
format: "json",
|
||||
metadata: { targetUserId: "user_1" },
|
||||
});
|
||||
|
||||
await expect(loadPrivacyGdprBundle(prisma, descriptor.payload)).resolves.toMatchObject({
|
||||
subject: { scope: "user_self", tenantId: "tenant_1", requestedByUserId: "user_1", targetUserId: "user_1" },
|
||||
files: expect.arrayContaining([
|
||||
expect.objectContaining({ path: "account/user.json", rows: [{ id: "user_1", email: "person@example.test" }] }),
|
||||
expect.objectContaining({ path: "activity/audit-events.json" }),
|
||||
]),
|
||||
});
|
||||
expect(prisma.tenantUser?.findMany).toHaveBeenCalledWith(expect.objectContaining({ where: { tenantId: "tenant_1", userId: "user_1" } }));
|
||||
expect(prisma.auditEvent?.findMany).toHaveBeenCalledWith(expect.objectContaining({ where: { tenantId: "tenant_1", actorId: "user_1" }, take: 10_000 }));
|
||||
});
|
||||
|
||||
it("loads a tenant-full bundle without a target user", async () => {
|
||||
const prisma = prismaMock();
|
||||
const descriptor = buildExportJob({
|
||||
exportJobId: "gdpr_export_2",
|
||||
tenantId: "tenant_1",
|
||||
requestedByUserId: "admin_1",
|
||||
exportType: PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE,
|
||||
format: "json",
|
||||
});
|
||||
|
||||
await expect(loadPrivacyGdprBundle(prisma, descriptor.payload)).resolves.toMatchObject({
|
||||
subject: { scope: "tenant_full", tenantId: "tenant_1", requestedByUserId: "admin_1", targetUserId: null },
|
||||
files: expect.arrayContaining([
|
||||
expect.objectContaining({ path: "tenant/tenant.json" }),
|
||||
expect.objectContaining({ path: "tenant/memberships.json" }),
|
||||
]),
|
||||
});
|
||||
expect(prisma.tenant?.findUnique).toHaveBeenCalledWith(expect.objectContaining({ where: { id: "tenant_1" } }));
|
||||
expect(prisma.auditEvent?.findMany).toHaveBeenCalledWith(expect.objectContaining({ where: { tenantId: "tenant_1" }, take: 10_000 }));
|
||||
});
|
||||
|
||||
it("creates a storage-backed service that writes the ZIP artifact", async () => {
|
||||
const send = vi.fn(async (command: { input?: unknown }) => {
|
||||
void command;
|
||||
return {};
|
||||
});
|
||||
const service = createWorkerPrivacyGdprExportService(
|
||||
{ STORAGE_BUCKET: "exports" },
|
||||
{
|
||||
prisma: prismaMock(),
|
||||
s3Client: { send: send as never },
|
||||
now: () => new Date("2026-07-02T12:00:00.000Z"),
|
||||
},
|
||||
);
|
||||
const descriptor = buildExportJob({
|
||||
exportJobId: "gdpr_export_1",
|
||||
tenantId: "tenant_1",
|
||||
requestedByUserId: "user_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
format: "json",
|
||||
});
|
||||
|
||||
await expect(service?.runExport(descriptor.payload)).resolves.toMatchObject({
|
||||
artifactBucket: "exports",
|
||||
artifactKey: "tenants/tenant_1/gdpr/gdpr_export_1.zip",
|
||||
expiresAt: new Date("2026-07-05T12:00:00.000Z"),
|
||||
});
|
||||
expect(send).toHaveBeenCalledTimes(1);
|
||||
const command = send.mock.calls[0]?.[0];
|
||||
expect(command).toBeDefined();
|
||||
expect(command!.input).toMatchObject({
|
||||
Bucket: "exports",
|
||||
Key: "tenants/tenant_1/gdpr/gdpr_export_1.zip",
|
||||
ContentType: "application/zip",
|
||||
Metadata: expect.objectContaining({ tenantId: "tenant_1", exportJobId: "gdpr_export_1" }),
|
||||
});
|
||||
});
|
||||
|
||||
it("skips service creation when storage is not configured", () => {
|
||||
expect(createWorkerPrivacyGdprExportService({}, { prisma: prismaMock(), s3Client: { send: vi.fn() } })).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+334
@@ -0,0 +1,334 @@
|
||||
import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import type { ExportJob } from "@hivegrid/queue";
|
||||
|
||||
import {
|
||||
GDPR_EXPORT_URL_TTL_HOURS,
|
||||
} from "@hivegrid/db/privacy-gdpr.server";
|
||||
|
||||
import {
|
||||
createPrivacyGdprExportRunner,
|
||||
PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE,
|
||||
PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
type PrivacyGdprExportBundle,
|
||||
type PrivacyGdprExportDataFile,
|
||||
type PrivacyGdprExportRunResult,
|
||||
} from "./privacy-gdpr-export.js";
|
||||
|
||||
export type PrivacyGdprExportPrisma = {
|
||||
user?: PrivacyGdprFindUniqueDelegate;
|
||||
tenant?: PrivacyGdprFindUniqueDelegate;
|
||||
tenantUser?: PrivacyGdprFindManyDelegate;
|
||||
auditEvent?: PrivacyGdprFindManyDelegate;
|
||||
notificationPreference?: PrivacyGdprFindManyDelegate;
|
||||
notificationDelivery?: PrivacyGdprFindManyDelegate;
|
||||
};
|
||||
|
||||
type PrivacyGdprFindUniqueDelegate = {
|
||||
findUnique(args: { where: Record<string, unknown>; select?: Record<string, unknown> }): Promise<Record<string, unknown> | null>;
|
||||
};
|
||||
|
||||
type PrivacyGdprFindManyDelegate = {
|
||||
findMany(args: { where: Record<string, unknown>; select?: Record<string, unknown>; orderBy?: Record<string, string>; take?: number }): Promise<Array<Record<string, unknown>>>;
|
||||
};
|
||||
|
||||
export type WorkerPrivacyGdprExportEnv = EnvRecord & {
|
||||
STORAGE_ACCESS_KEY_ID?: string;
|
||||
STORAGE_BUCKET?: string;
|
||||
STORAGE_ENDPOINT?: string;
|
||||
STORAGE_FORCE_PATH_STYLE?: string;
|
||||
STORAGE_REGION?: string;
|
||||
STORAGE_SECRET_ACCESS_KEY?: string;
|
||||
STORAGE_SESSION_TOKEN?: string;
|
||||
};
|
||||
|
||||
export type WorkerPrivacyGdprExportDependencies = {
|
||||
prisma: PrivacyGdprExportPrisma;
|
||||
s3Client?: Pick<S3Client, "send">;
|
||||
now?: () => Date;
|
||||
};
|
||||
|
||||
export type WorkerPrivacyGdprExportService = {
|
||||
runExport(payload: ExportJob): Promise<PrivacyGdprExportRunResult>;
|
||||
};
|
||||
|
||||
const JSON_SELECT_USER = {
|
||||
id: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
email: true,
|
||||
firstName: true,
|
||||
lastName: true,
|
||||
avatar: true,
|
||||
phone: true,
|
||||
defaultTenantId: true,
|
||||
locale: true,
|
||||
lightOrDarkMode: true,
|
||||
active: true,
|
||||
pendingDeletion: true,
|
||||
scheduledDeletionAt: true,
|
||||
deletionRequestedAt: true,
|
||||
pseudonymized: true,
|
||||
pseudonymizedAt: true,
|
||||
};
|
||||
|
||||
const JSON_SELECT_TENANT = {
|
||||
id: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
slug: true,
|
||||
name: true,
|
||||
icon: true,
|
||||
logo: true,
|
||||
subscriptionId: true,
|
||||
active: true,
|
||||
deactivatedReason: true,
|
||||
};
|
||||
|
||||
const JSON_SELECT_TENANT_USER = {
|
||||
id: true,
|
||||
createdAt: true,
|
||||
tenantId: true,
|
||||
userId: true,
|
||||
type: true,
|
||||
joined: true,
|
||||
status: true,
|
||||
};
|
||||
|
||||
const JSON_SELECT_AUDIT_EVENT = {
|
||||
id: true,
|
||||
createdAt: true,
|
||||
tenantId: true,
|
||||
actorId: true,
|
||||
actorType: true,
|
||||
actorEmail: true,
|
||||
action: true,
|
||||
targetType: true,
|
||||
targetId: true,
|
||||
requestId: true,
|
||||
sourceModule: true,
|
||||
outcome: true,
|
||||
dataSensitivity: true,
|
||||
regulatoryScope: true,
|
||||
metadata: true,
|
||||
};
|
||||
|
||||
const JSON_SELECT_NOTIFICATION_PREFERENCE = {
|
||||
id: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
tenantId: true,
|
||||
userId: true,
|
||||
channel: true,
|
||||
category: true,
|
||||
enabled: true,
|
||||
};
|
||||
|
||||
const JSON_SELECT_NOTIFICATION_DELIVERY = {
|
||||
id: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
tenantId: true,
|
||||
userId: true,
|
||||
channel: true,
|
||||
category: true,
|
||||
messageKey: true,
|
||||
sourceKey: true,
|
||||
status: true,
|
||||
attemptsMade: true,
|
||||
sentAt: true,
|
||||
readAt: true,
|
||||
};
|
||||
|
||||
export function createWorkerPrivacyGdprExportService(
|
||||
env: WorkerPrivacyGdprExportEnv,
|
||||
deps: WorkerPrivacyGdprExportDependencies,
|
||||
): WorkerPrivacyGdprExportService | undefined {
|
||||
const bucket = env.STORAGE_BUCKET?.trim();
|
||||
const s3Client = deps.s3Client ?? createPrivacyGdprExportS3Client(env);
|
||||
if (!bucket || !s3Client) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const now = deps.now ?? (() => new Date());
|
||||
return {
|
||||
runExport: createPrivacyGdprExportRunner({
|
||||
loadBundle: (payload) => loadPrivacyGdprBundle(deps.prisma, payload),
|
||||
buildArtifactLocation: (payload) => {
|
||||
const generatedAt = now();
|
||||
return {
|
||||
bucket,
|
||||
key: `tenants/${payload.tenantId}/gdpr/${payload.exportJobId}.zip`,
|
||||
expiresAt: new Date(generatedAt.getTime() + GDPR_EXPORT_URL_TTL_HOURS * 60 * 60 * 1_000),
|
||||
};
|
||||
},
|
||||
putObject: async (input) => {
|
||||
await s3Client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: input.bucket,
|
||||
Key: input.key,
|
||||
Body: input.body,
|
||||
ContentType: input.contentType,
|
||||
Metadata: input.metadata,
|
||||
}),
|
||||
);
|
||||
},
|
||||
now,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadPrivacyGdprBundle(prisma: PrivacyGdprExportPrisma, payload: ExportJob): Promise<PrivacyGdprExportBundle> {
|
||||
const targetUserId = typeof payload.metadata?.targetUserId === "string" ? payload.metadata.targetUserId : payload.requestedByUserId;
|
||||
if (payload.exportType === PRIVACY_GDPR_USER_SELF_EXPORT_TYPE) {
|
||||
return {
|
||||
subject: {
|
||||
scope: "user_self",
|
||||
tenantId: payload.tenantId,
|
||||
requestedByUserId: payload.requestedByUserId,
|
||||
targetUserId,
|
||||
},
|
||||
files: await loadUserSelfFiles(prisma, payload.tenantId, targetUserId),
|
||||
};
|
||||
}
|
||||
|
||||
if (payload.exportType === PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE) {
|
||||
return {
|
||||
subject: {
|
||||
scope: "tenant_full",
|
||||
tenantId: payload.tenantId,
|
||||
requestedByUserId: payload.requestedByUserId,
|
||||
targetUserId: null,
|
||||
},
|
||||
files: await loadTenantFullFiles(prisma, payload.tenantId),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported privacy GDPR export type: ${payload.exportType}`);
|
||||
}
|
||||
|
||||
function createPrivacyGdprExportS3Client(env: WorkerPrivacyGdprExportEnv): S3Client | undefined {
|
||||
if (!env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
|
||||
return undefined;
|
||||
}
|
||||
return new S3Client({
|
||||
endpoint: env.STORAGE_ENDPOINT,
|
||||
region: env.STORAGE_REGION,
|
||||
forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === "true",
|
||||
credentials: {
|
||||
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY,
|
||||
...(env.STORAGE_SESSION_TOKEN ? { sessionToken: env.STORAGE_SESSION_TOKEN } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function loadUserSelfFiles(prisma: PrivacyGdprExportPrisma, tenantId: string, userId: string): Promise<PrivacyGdprExportDataFile[]> {
|
||||
const user = await prisma.user?.findUnique({ where: { id: userId }, select: JSON_SELECT_USER });
|
||||
return compactFiles([
|
||||
{
|
||||
path: "account/user.json",
|
||||
description: "User account profile and privacy flags",
|
||||
rows: user ? [user] : [],
|
||||
},
|
||||
{
|
||||
path: "account/tenant-memberships.json",
|
||||
description: "Tenant memberships for this user",
|
||||
rows: await findMany(prisma.tenantUser, {
|
||||
where: { tenantId, userId },
|
||||
select: JSON_SELECT_TENANT_USER,
|
||||
orderBy: { createdAt: "asc" },
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "activity/audit-events.json",
|
||||
description: "Audit events where this user is the actor in this tenant",
|
||||
rows: await findMany(prisma.auditEvent, {
|
||||
where: { tenantId, actorId: userId },
|
||||
select: JSON_SELECT_AUDIT_EVENT,
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 10_000,
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "notifications/preferences.json",
|
||||
description: "Notification preferences for this user",
|
||||
rows: await findMany(prisma.notificationPreference, {
|
||||
where: { tenantId, userId },
|
||||
select: JSON_SELECT_NOTIFICATION_PREFERENCE,
|
||||
orderBy: { createdAt: "asc" },
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "notifications/deliveries.json",
|
||||
description: "Notification delivery records for this user",
|
||||
rows: await findMany(prisma.notificationDelivery, {
|
||||
where: { tenantId, userId },
|
||||
select: JSON_SELECT_NOTIFICATION_DELIVERY,
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 10_000,
|
||||
}),
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
async function loadTenantFullFiles(prisma: PrivacyGdprExportPrisma, tenantId: string): Promise<PrivacyGdprExportDataFile[]> {
|
||||
const tenant = await prisma.tenant?.findUnique({ where: { id: tenantId }, select: JSON_SELECT_TENANT });
|
||||
return compactFiles([
|
||||
{
|
||||
path: "tenant/tenant.json",
|
||||
description: "Tenant metadata",
|
||||
rows: tenant ? [tenant] : [],
|
||||
},
|
||||
{
|
||||
path: "tenant/memberships.json",
|
||||
description: "Tenant membership rows",
|
||||
rows: await findMany(prisma.tenantUser, {
|
||||
where: { tenantId },
|
||||
select: JSON_SELECT_TENANT_USER,
|
||||
orderBy: { createdAt: "asc" },
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "activity/audit-events.json",
|
||||
description: "Tenant audit events",
|
||||
rows: await findMany(prisma.auditEvent, {
|
||||
where: { tenantId },
|
||||
select: JSON_SELECT_AUDIT_EVENT,
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 10_000,
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "notifications/preferences.json",
|
||||
description: "Tenant notification preferences",
|
||||
rows: await findMany(prisma.notificationPreference, {
|
||||
where: { tenantId },
|
||||
select: JSON_SELECT_NOTIFICATION_PREFERENCE,
|
||||
orderBy: { createdAt: "asc" },
|
||||
take: 10_000,
|
||||
}),
|
||||
},
|
||||
{
|
||||
path: "notifications/deliveries.json",
|
||||
description: "Tenant notification delivery records",
|
||||
rows: await findMany(prisma.notificationDelivery, {
|
||||
where: { tenantId },
|
||||
select: JSON_SELECT_NOTIFICATION_DELIVERY,
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: 10_000,
|
||||
}),
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
async function findMany(
|
||||
delegate: PrivacyGdprFindManyDelegate | undefined,
|
||||
args: Parameters<PrivacyGdprFindManyDelegate["findMany"]>[0],
|
||||
): Promise<Array<Record<string, unknown>>> {
|
||||
return delegate ? delegate.findMany(args) : [];
|
||||
}
|
||||
|
||||
function compactFiles(files: PrivacyGdprExportDataFile[]): PrivacyGdprExportDataFile[] {
|
||||
return files.filter((file) => file.rows.length > 0);
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { buildExportJob } from "@hivegrid/queue";
|
||||
|
||||
import {
|
||||
PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE,
|
||||
PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
type PrivacyGdprExportRunnerDependencies,
|
||||
createPrivacyGdprExportRunner,
|
||||
isPrivacyGdprExportType,
|
||||
} from "./privacy-gdpr-export.js";
|
||||
|
||||
function listZipEntries(zip: Buffer): string[] {
|
||||
const names: string[] = [];
|
||||
let offset = zip.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
|
||||
while (offset >= 0 && zip.readUInt32LE(offset) === 0x02014b50) {
|
||||
const nameLength = zip.readUInt16LE(offset + 28);
|
||||
const extraLength = zip.readUInt16LE(offset + 30);
|
||||
const commentLength = zip.readUInt16LE(offset + 32);
|
||||
names.push(zip.subarray(offset + 46, offset + 46 + nameLength).toString("utf8"));
|
||||
offset += 46 + nameLength + extraLength + commentLength;
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
describe("privacy GDPR export runner", () => {
|
||||
it("builds a ZIP with README, manifest and data files, then stores it", async () => {
|
||||
const expiresAt = new Date("2026-07-05T12:00:00.000Z");
|
||||
const putObject = vi.fn<PrivacyGdprExportRunnerDependencies["putObject"]>(async () => undefined);
|
||||
const descriptor = buildExportJob({
|
||||
exportJobId: "gdpr_export_1",
|
||||
tenantId: "tenant_1",
|
||||
requestedByUserId: "user_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
format: "json",
|
||||
requestId: "req_1",
|
||||
metadata: { targetUserId: "user_1" },
|
||||
});
|
||||
const runner = createPrivacyGdprExportRunner({
|
||||
now: () => new Date("2026-07-02T12:00:00.000Z"),
|
||||
buildArtifactLocation: () => ({ bucket: "exports", key: "tenant_1/gdpr_export_1.zip", expiresAt }),
|
||||
putObject,
|
||||
loadBundle: vi.fn(async () => ({
|
||||
subject: { scope: "user_self" as const, tenantId: "tenant_1", requestedByUserId: "user_1", targetUserId: "user_1" },
|
||||
files: [
|
||||
{
|
||||
path: "account/user.json",
|
||||
description: "User account",
|
||||
rows: [{ id: "user_1", email: "person@example.test" }],
|
||||
},
|
||||
],
|
||||
})),
|
||||
});
|
||||
|
||||
await expect(runner(descriptor.payload)).resolves.toMatchObject({
|
||||
artifactBucket: "exports",
|
||||
artifactKey: "tenant_1/gdpr_export_1.zip",
|
||||
rowCount: 1,
|
||||
expiresAt,
|
||||
});
|
||||
|
||||
const storedObject = putObject.mock.calls[0]?.[0];
|
||||
expect(storedObject).toBeDefined();
|
||||
const body = storedObject!.body as Buffer;
|
||||
expect(listZipEntries(body)).toEqual(["README.md", "manifest.json", "data/account/user.json"]);
|
||||
expect(putObject).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
bucket: "exports",
|
||||
key: "tenant_1/gdpr_export_1.zip",
|
||||
contentType: "application/zip",
|
||||
metadata: expect.objectContaining({
|
||||
tenantId: "tenant_1",
|
||||
exportJobId: "gdpr_export_1",
|
||||
exportType: PRIVACY_GDPR_USER_SELF_EXPORT_TYPE,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("identifies supported privacy export types only", () => {
|
||||
expect(isPrivacyGdprExportType(PRIVACY_GDPR_USER_SELF_EXPORT_TYPE)).toBe(true);
|
||||
expect(isPrivacyGdprExportType(PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE)).toBe(true);
|
||||
expect(isPrivacyGdprExportType("members")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,168 @@
|
||||
import type { ExportJob } from "@hivegrid/queue";
|
||||
import { createStoredZipBuffer, type ZipEntryInput } from "@hivegrid/exports/zip";
|
||||
|
||||
export const PRIVACY_GDPR_USER_SELF_EXPORT_TYPE = "gdpr_user_self";
|
||||
export const PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE = "gdpr_tenant_full";
|
||||
export const PRIVACY_GDPR_EXPORT_TYPES = [PRIVACY_GDPR_USER_SELF_EXPORT_TYPE, PRIVACY_GDPR_TENANT_FULL_EXPORT_TYPE] as const;
|
||||
|
||||
export type PrivacyGdprExportType = (typeof PRIVACY_GDPR_EXPORT_TYPES)[number];
|
||||
|
||||
export type PrivacyGdprExportDataFile = {
|
||||
path: string;
|
||||
description: string;
|
||||
rows: readonly Record<string, unknown>[];
|
||||
};
|
||||
|
||||
export type PrivacyGdprExportBundle = {
|
||||
subject: {
|
||||
scope: "user_self" | "tenant_full";
|
||||
tenantId: string;
|
||||
requestedByUserId: string;
|
||||
targetUserId?: string | null;
|
||||
};
|
||||
files: readonly PrivacyGdprExportDataFile[];
|
||||
};
|
||||
|
||||
export type PrivacyGdprExportRunResult = {
|
||||
artifactBucket: string;
|
||||
artifactKey: string;
|
||||
rowCount: number;
|
||||
sizeBytes: number;
|
||||
expiresAt: Date;
|
||||
};
|
||||
|
||||
export type PrivacyGdprExportRunnerDependencies = {
|
||||
loadBundle(payload: ExportJob): Promise<PrivacyGdprExportBundle>;
|
||||
putObject(input: {
|
||||
bucket: string;
|
||||
key: string;
|
||||
body: Uint8Array;
|
||||
contentType: string;
|
||||
metadata: Record<string, string>;
|
||||
}): Promise<void>;
|
||||
buildArtifactLocation(payload: ExportJob): { bucket: string; key: string; expiresAt: Date };
|
||||
now?: () => Date;
|
||||
};
|
||||
|
||||
type PrivacyGdprManifest = {
|
||||
version: "1.0";
|
||||
exportType: PrivacyGdprExportType;
|
||||
exportJobId: string;
|
||||
tenantId: string;
|
||||
requestedByUserId: string;
|
||||
targetUserId: string | null;
|
||||
generatedAt: string;
|
||||
expiresAt: string;
|
||||
files: Array<{
|
||||
path: string;
|
||||
description: string;
|
||||
rowCount: number;
|
||||
}>;
|
||||
};
|
||||
|
||||
const ZIP_CONTENT_TYPE = "application/zip";
|
||||
|
||||
export function isPrivacyGdprExportType(exportType: string): exportType is PrivacyGdprExportType {
|
||||
return (PRIVACY_GDPR_EXPORT_TYPES as readonly string[]).includes(exportType);
|
||||
}
|
||||
|
||||
export function createPrivacyGdprExportRunner(deps: PrivacyGdprExportRunnerDependencies): (payload: ExportJob) => Promise<PrivacyGdprExportRunResult> {
|
||||
return async function runPrivacyGdprExport(payload: ExportJob): Promise<PrivacyGdprExportRunResult> {
|
||||
if (!isPrivacyGdprExportType(payload.exportType)) {
|
||||
throw new Error(`Unsupported privacy GDPR export type: ${payload.exportType}`);
|
||||
}
|
||||
|
||||
const bundle = await deps.loadBundle(payload);
|
||||
const artifact = deps.buildArtifactLocation(payload);
|
||||
const generatedAt = (deps.now ?? (() => new Date()))();
|
||||
const rowCount = bundle.files.reduce((sum, file) => sum + file.rows.length, 0);
|
||||
const manifest = buildManifest({ payload, bundle, generatedAt, expiresAt: artifact.expiresAt });
|
||||
const zipEntries = buildZipEntries({ manifest, files: bundle.files, generatedAt });
|
||||
const zip = createStoredZipBuffer(zipEntries);
|
||||
|
||||
await deps.putObject({
|
||||
bucket: artifact.bucket,
|
||||
key: artifact.key,
|
||||
body: zip,
|
||||
contentType: ZIP_CONTENT_TYPE,
|
||||
metadata: {
|
||||
tenantId: payload.tenantId,
|
||||
exportJobId: payload.exportJobId,
|
||||
exportType: payload.exportType,
|
||||
requestedByUserId: payload.requestedByUserId,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
artifactBucket: artifact.bucket,
|
||||
artifactKey: artifact.key,
|
||||
rowCount,
|
||||
sizeBytes: zip.byteLength,
|
||||
expiresAt: artifact.expiresAt,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function buildManifest(input: { payload: ExportJob; bundle: PrivacyGdprExportBundle; generatedAt: Date; expiresAt: Date }): PrivacyGdprManifest {
|
||||
const { payload, bundle, generatedAt, expiresAt } = input;
|
||||
if (!isPrivacyGdprExportType(payload.exportType)) {
|
||||
throw new Error(`Unsupported privacy GDPR export type: ${payload.exportType}`);
|
||||
}
|
||||
return {
|
||||
version: "1.0",
|
||||
exportType: payload.exportType,
|
||||
exportJobId: payload.exportJobId,
|
||||
tenantId: payload.tenantId,
|
||||
requestedByUserId: payload.requestedByUserId,
|
||||
targetUserId: bundle.subject.targetUserId ?? null,
|
||||
generatedAt: generatedAt.toISOString(),
|
||||
expiresAt: expiresAt.toISOString(),
|
||||
files: bundle.files.map((file) => ({
|
||||
path: file.path,
|
||||
description: file.description,
|
||||
rowCount: file.rows.length,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function buildZipEntries(input: { manifest: PrivacyGdprManifest; files: readonly PrivacyGdprExportDataFile[]; generatedAt: Date }): ZipEntryInput[] {
|
||||
return [
|
||||
{
|
||||
path: "README.md",
|
||||
data: buildReadme(input.manifest),
|
||||
modifiedAt: input.generatedAt,
|
||||
},
|
||||
{
|
||||
path: "manifest.json",
|
||||
data: `${JSON.stringify(input.manifest, null, 2)}\n`,
|
||||
modifiedAt: input.generatedAt,
|
||||
},
|
||||
...input.files.map((file) => ({
|
||||
path: normalizeDataPath(file.path),
|
||||
data: `${JSON.stringify({ description: file.description, rowCount: file.rows.length, rows: file.rows }, null, 2)}\n`,
|
||||
modifiedAt: input.generatedAt,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
function buildReadme(manifest: PrivacyGdprManifest): string {
|
||||
return [
|
||||
"# GDPR Export",
|
||||
"",
|
||||
`Export job: ${manifest.exportJobId}`,
|
||||
`Scope: ${manifest.exportType}`,
|
||||
`Generated at: ${manifest.generatedAt}`,
|
||||
`Download expires at: ${manifest.expiresAt}`,
|
||||
"",
|
||||
"Files:",
|
||||
...manifest.files.map((file) => `- ${file.path}: ${file.description} (${file.rowCount} rows)`),
|
||||
"",
|
||||
"This archive contains JSON data plus manifest.json. Treat it as confidential personal data.",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function normalizeDataPath(path: string): string {
|
||||
const trimmed = path.trim().replace(/\\/g, "/").replace(/^\/+/, "");
|
||||
return trimmed.startsWith("data/") ? trimmed : `data/${trimmed}`;
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
runPrivacyTenantDeletion,
|
||||
type PrivacyTenantDeletionDependencies,
|
||||
type PrivacyTenantDeletionRequest,
|
||||
} from "./privacy-tenant-deletion.js";
|
||||
|
||||
const request = {
|
||||
id: "tenant_deletion_1",
|
||||
tenantId: "tenant_1",
|
||||
requestedByUserId: "user_1",
|
||||
reason: "voluntary",
|
||||
} satisfies PrivacyTenantDeletionRequest;
|
||||
|
||||
function makeDeps(overrides?: Partial<PrivacyTenantDeletionDependencies>): {
|
||||
deps: PrivacyTenantDeletionDependencies;
|
||||
calls: string[];
|
||||
audits: Array<Parameters<NonNullable<PrivacyTenantDeletionDependencies["writeAuditEvent"]>>[0]>;
|
||||
} {
|
||||
const calls: string[] = [];
|
||||
const audits: Array<Parameters<NonNullable<PrivacyTenantDeletionDependencies["writeAuditEvent"]>>[0]> = [];
|
||||
const deps: PrivacyTenantDeletionDependencies = {
|
||||
markExecuting: vi.fn(async () => {
|
||||
calls.push("markExecuting");
|
||||
}),
|
||||
markCompleted: vi.fn(async () => {
|
||||
calls.push("markCompleted");
|
||||
}),
|
||||
markFailed: vi.fn(async () => {
|
||||
calls.push("markFailed");
|
||||
}),
|
||||
cancelBillingForTenant: vi.fn(async () => {
|
||||
calls.push("cancelBilling");
|
||||
return { cancelledSubscriptions: 1 };
|
||||
}),
|
||||
deleteStoragePrefix: vi.fn(async () => {
|
||||
calls.push("deleteStorage");
|
||||
return { deletedObjects: 2 };
|
||||
}),
|
||||
deleteSearchDocuments: vi.fn(async () => {
|
||||
calls.push("deleteSearch");
|
||||
return { deletedDocuments: 3 };
|
||||
}),
|
||||
destroyTenantDek: vi.fn(async () => {
|
||||
calls.push("destroyDek");
|
||||
return { affectedRows: 1 };
|
||||
}),
|
||||
deleteTenantRows: vi.fn(async () => {
|
||||
calls.push("deleteRows");
|
||||
return { affectedRows: 8 };
|
||||
}),
|
||||
writeAuditEvent: vi.fn(async (input) => {
|
||||
audits.push(input);
|
||||
}),
|
||||
logger: { info: vi.fn(), error: vi.fn() },
|
||||
...overrides,
|
||||
};
|
||||
return { deps, calls, audits };
|
||||
}
|
||||
|
||||
describe("runPrivacyTenantDeletion", () => {
|
||||
it("runs external cleanup, crypto shredding, then tenant row deletion in order", async () => {
|
||||
const { deps, calls, audits } = makeDeps();
|
||||
|
||||
await expect(runPrivacyTenantDeletion(deps, request)).resolves.toEqual({
|
||||
requestId: "tenant_deletion_1",
|
||||
tenantId: "tenant_1",
|
||||
steps: [
|
||||
{ step: "billing.cancel", cancelledSubscriptions: 1 },
|
||||
{ step: "storage.delete-prefix", deletedObjects: 2 },
|
||||
{ step: "search.delete-documents", deletedDocuments: 3 },
|
||||
{ step: "crypto.destroy-dek", affectedRows: 1 },
|
||||
{ step: "tenant.delete-rows", affectedRows: 8 },
|
||||
],
|
||||
});
|
||||
|
||||
expect(calls).toEqual(["markExecuting", "cancelBilling", "deleteStorage", "deleteSearch", "destroyDek", "deleteRows", "markCompleted"]);
|
||||
expect(deps.markExecuting).toHaveBeenCalledWith({ id: "tenant_deletion_1", tenantId: "tenant_1" });
|
||||
expect(deps.markCompleted).toHaveBeenCalledWith({ id: "tenant_deletion_1", tenantId: "tenant_1" });
|
||||
expect(audits[0]).toMatchObject({
|
||||
tenantId: "tenant_1",
|
||||
action: "DELETE_TENANT",
|
||||
targetType: "Tenant",
|
||||
targetId: "tenant_1",
|
||||
regulatoryScope: ["gdpr"],
|
||||
outcome: "success",
|
||||
});
|
||||
});
|
||||
|
||||
it("marks the request failed with redacted diagnostics and stops before tenant rows when an earlier step fails", async () => {
|
||||
const { deps, calls, audits } = makeDeps({
|
||||
deleteSearchDocuments: vi.fn(async () => {
|
||||
calls.push("deleteSearch");
|
||||
throw new Error("search outage token=secret authorization: Bearer abc");
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(runPrivacyTenantDeletion(deps, request)).rejects.toThrow(/search outage/);
|
||||
|
||||
expect(calls).toEqual(["markExecuting", "cancelBilling", "deleteStorage", "deleteSearch", "markFailed"]);
|
||||
expect(deps.deleteTenantRows).not.toHaveBeenCalled();
|
||||
expect(deps.markFailed).toHaveBeenCalledWith({
|
||||
id: "tenant_deletion_1",
|
||||
tenantId: "tenant_1",
|
||||
errorMessage: "search outage token=[redacted] authorization: Bearer [redacted]",
|
||||
});
|
||||
expect(JSON.stringify(audits)).not.toContain("secret");
|
||||
expect(audits.at(-1)).toMatchObject({ outcome: "failure" });
|
||||
});
|
||||
});
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
export type PrivacyTenantDeletionRequest = {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
requestedByUserId: string;
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export type PrivacyTenantDeletionStepName =
|
||||
| "billing.cancel"
|
||||
| "storage.delete-prefix"
|
||||
| "search.delete-documents"
|
||||
| "crypto.destroy-dek"
|
||||
| "tenant.delete-rows";
|
||||
|
||||
export type PrivacyTenantDeletionStepResult = {
|
||||
step: PrivacyTenantDeletionStepName;
|
||||
affectedRows?: number;
|
||||
deletedObjects?: number;
|
||||
cancelledSubscriptions?: number;
|
||||
deletedDocuments?: number;
|
||||
};
|
||||
|
||||
export type PrivacyTenantDeletionResult = {
|
||||
requestId: string;
|
||||
tenantId: string;
|
||||
steps: PrivacyTenantDeletionStepResult[];
|
||||
};
|
||||
|
||||
export type PrivacyTenantDeletionDependencies = {
|
||||
markExecuting(input: { id: string; tenantId: string; now?: Date }): Promise<unknown>;
|
||||
markCompleted(input: { id: string; tenantId: string; now?: Date }): Promise<unknown>;
|
||||
markFailed(input: { id: string; tenantId: string; errorMessage?: string | null }): Promise<unknown>;
|
||||
cancelBillingForTenant(input: { tenantId: string; requestId: string }): Promise<{ cancelledSubscriptions?: number }>;
|
||||
deleteStoragePrefix(input: { tenantId: string; requestId: string }): Promise<{ deletedObjects?: number }>;
|
||||
deleteSearchDocuments(input: { tenantId: string; requestId: string }): Promise<{ deletedDocuments?: number }>;
|
||||
destroyTenantDek(input: { tenantId: string; requestId: string }): Promise<{ affectedRows?: number }>;
|
||||
deleteTenantRows(input: { tenantId: string; requestId: string }): Promise<{ affectedRows?: number }>;
|
||||
writeAuditEvent?: (input: {
|
||||
tenantId: string;
|
||||
actorId: string;
|
||||
actorType: "system";
|
||||
action: "DELETE_TENANT";
|
||||
targetType: "Tenant";
|
||||
targetId: string;
|
||||
regulatoryScope: string[];
|
||||
dataSensitivity: "pii";
|
||||
outcome: "success" | "failure";
|
||||
metadata: Record<string, unknown>;
|
||||
}) => Promise<unknown>;
|
||||
logger?: {
|
||||
info(fields: Record<string, unknown>, message: string): void;
|
||||
error(fields: Record<string, unknown>, message: string): void;
|
||||
};
|
||||
};
|
||||
|
||||
export async function runPrivacyTenantDeletion(
|
||||
deps: PrivacyTenantDeletionDependencies,
|
||||
request: PrivacyTenantDeletionRequest,
|
||||
): Promise<PrivacyTenantDeletionResult> {
|
||||
const steps: PrivacyTenantDeletionStepResult[] = [];
|
||||
await deps.markExecuting({ id: request.id, tenantId: request.tenantId });
|
||||
|
||||
try {
|
||||
const billing = await deps.cancelBillingForTenant({ tenantId: request.tenantId, requestId: request.id });
|
||||
steps.push({ step: "billing.cancel", cancelledSubscriptions: billing.cancelledSubscriptions ?? 0 });
|
||||
|
||||
const storage = await deps.deleteStoragePrefix({ tenantId: request.tenantId, requestId: request.id });
|
||||
steps.push({ step: "storage.delete-prefix", deletedObjects: storage.deletedObjects ?? 0 });
|
||||
|
||||
const search = await deps.deleteSearchDocuments({ tenantId: request.tenantId, requestId: request.id });
|
||||
steps.push({ step: "search.delete-documents", deletedDocuments: search.deletedDocuments ?? 0 });
|
||||
|
||||
const crypto = await deps.destroyTenantDek({ tenantId: request.tenantId, requestId: request.id });
|
||||
steps.push({ step: "crypto.destroy-dek", affectedRows: crypto.affectedRows ?? 0 });
|
||||
|
||||
const tenantRows = await deps.deleteTenantRows({ tenantId: request.tenantId, requestId: request.id });
|
||||
steps.push({ step: "tenant.delete-rows", affectedRows: tenantRows.affectedRows ?? 0 });
|
||||
|
||||
await deps.markCompleted({ id: request.id, tenantId: request.tenantId });
|
||||
await deps.writeAuditEvent?.({
|
||||
tenantId: request.tenantId,
|
||||
actorId: "system",
|
||||
actorType: "system",
|
||||
action: "DELETE_TENANT",
|
||||
targetType: "Tenant",
|
||||
targetId: request.tenantId,
|
||||
regulatoryScope: ["gdpr"],
|
||||
dataSensitivity: "pii",
|
||||
outcome: "success",
|
||||
metadata: { requestId: request.id, requestedByUserId: request.requestedByUserId, reason: request.reason, steps },
|
||||
});
|
||||
deps.logger?.info({ requestId: request.id, tenantId: request.tenantId, steps: steps.length }, "privacy.tenant-deletion.complete");
|
||||
|
||||
return { requestId: request.id, tenantId: request.tenantId, steps };
|
||||
} catch (error) {
|
||||
const message = redactTenantDeletionDiagnostic(error instanceof Error ? error.message : String(error));
|
||||
await deps.markFailed({ id: request.id, tenantId: request.tenantId, errorMessage: message });
|
||||
await deps.writeAuditEvent?.({
|
||||
tenantId: request.tenantId,
|
||||
actorId: "system",
|
||||
actorType: "system",
|
||||
action: "DELETE_TENANT",
|
||||
targetType: "Tenant",
|
||||
targetId: request.tenantId,
|
||||
regulatoryScope: ["gdpr"],
|
||||
dataSensitivity: "pii",
|
||||
outcome: "failure",
|
||||
metadata: { requestId: request.id, requestedByUserId: request.requestedByUserId, reason: request.reason, steps, error: message },
|
||||
});
|
||||
deps.logger?.error({ requestId: request.id, tenantId: request.tenantId, error: message }, "privacy.tenant-deletion.failed");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function redactTenantDeletionDiagnostic(message: string): string {
|
||||
return message
|
||||
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
||||
.replace(/(authorization)(\s*[=:]\s*)(?!Bearer\b)\S+/gi, "$1$2[redacted]")
|
||||
.replace(/(access[_-]?key(?:id)?|secret[_-]?access[_-]?key|session[_-]?token|credential|token|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]");
|
||||
}
|
||||
+19
-23
@@ -15,8 +15,8 @@ function makeRow(overrides: Partial<DeviceTokenRow>): DeviceTokenRow {
|
||||
token: overrides.token ?? "tokAAAAAAAAAAAAAA",
|
||||
userId: overrides.userId ?? "u-1",
|
||||
tenantId: overrides.tenantId ?? null,
|
||||
platform: overrides.platform ?? "ios",
|
||||
pushProvider: overrides.pushProvider ?? "apns",
|
||||
platform: overrides.platform ?? "web",
|
||||
pushProvider: overrides.pushProvider ?? "webpush",
|
||||
revokedAt: overrides.revokedAt ?? null,
|
||||
};
|
||||
}
|
||||
@@ -33,8 +33,6 @@ function makePayload(overrides: Partial<PushPayload> = {}): PushPayload {
|
||||
|
||||
function makeProviders(overrides: Partial<PushProviderSeam> = {}): PushProviderSeam {
|
||||
return {
|
||||
sendApns: overrides.sendApns ?? vi.fn().mockResolvedValue({ success: true, messageId: "apns-1" }),
|
||||
sendFcm: overrides.sendFcm ?? vi.fn().mockResolvedValue({ success: true, messageId: "fcm-1" }),
|
||||
sendWebPush: overrides.sendWebPush ?? vi.fn().mockResolvedValue({ success: true, messageId: "web-1" }),
|
||||
};
|
||||
}
|
||||
@@ -73,26 +71,24 @@ describe("dispatchPushNotification", () => {
|
||||
const result = await dispatchPushNotification(input);
|
||||
expect(result.attempted).toBe(1);
|
||||
expect(result.succeeded).toBe(1);
|
||||
expect(providers.sendApns).toHaveBeenCalledTimes(1);
|
||||
expect(providers.sendWebPush).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("routes ios tokens to APNs, android to FCM, web to WebPush", async () => {
|
||||
it("routes active tokens to WebPush", async () => {
|
||||
const providers = makeProviders();
|
||||
const input: PushFanOutInput = {
|
||||
payload: makePayload(),
|
||||
tokenLister: async () => [
|
||||
makeRow({ token: "tok-ios", platform: "ios", pushProvider: "apns" }),
|
||||
makeRow({ id: "dt-2", token: "tok-and", platform: "android", pushProvider: "fcm" }),
|
||||
makeRow({ id: "dt-3", token: "tok-web", platform: "web", pushProvider: "webpush" }),
|
||||
makeRow({ token: "tok-one", platform: "web", pushProvider: "webpush" }),
|
||||
makeRow({ id: "dt-2", token: "tok-two", platform: "web", pushProvider: "webpush" }),
|
||||
],
|
||||
providers,
|
||||
tokenRevoker: vi.fn(),
|
||||
};
|
||||
const result = await dispatchPushNotification(input);
|
||||
expect(result.succeeded).toBe(3);
|
||||
expect(providers.sendApns).toHaveBeenCalledWith("tok-ios", expect.any(Object));
|
||||
expect(providers.sendFcm).toHaveBeenCalledWith("tok-and", expect.any(Object));
|
||||
expect(providers.sendWebPush).toHaveBeenCalledWith("tok-web", expect.any(Object));
|
||||
expect(result.succeeded).toBe(2);
|
||||
expect(providers.sendWebPush).toHaveBeenCalledWith("tok-one", expect.any(Object));
|
||||
expect(providers.sendWebPush).toHaveBeenCalledWith("tok-two", expect.any(Object));
|
||||
});
|
||||
|
||||
it("starts multiple provider sends before the first delayed send resolves", async () => {
|
||||
@@ -101,12 +97,12 @@ describe("dispatchPushNotification", () => {
|
||||
const completed: string[] = [];
|
||||
let sendIndex = 0;
|
||||
const providers = makeProviders({
|
||||
sendApns: vi.fn(async (token) => {
|
||||
sendWebPush: vi.fn(async (token) => {
|
||||
const release = releases[sendIndex++]!;
|
||||
started.push(token);
|
||||
await release.promise;
|
||||
completed.push(token);
|
||||
return { success: true, messageId: `apns-${token}` };
|
||||
return { success: true, messageId: `web-${token}` };
|
||||
}),
|
||||
});
|
||||
const input: PushFanOutInput = {
|
||||
@@ -141,7 +137,7 @@ describe("dispatchPushNotification", () => {
|
||||
it("revokes tokens when the provider reports an irrecoverable error code", async () => {
|
||||
const revoker = vi.fn();
|
||||
const providers = makeProviders({
|
||||
sendApns: vi.fn().mockResolvedValue({ success: false, errorCode: "Unregistered" }),
|
||||
sendWebPush: vi.fn().mockResolvedValue({ success: false, errorCode: "Unregistered" }),
|
||||
});
|
||||
const input: PushFanOutInput = {
|
||||
payload: makePayload(),
|
||||
@@ -158,7 +154,7 @@ describe("dispatchPushNotification", () => {
|
||||
it("counts other errors as failed without revoking", async () => {
|
||||
const revoker = vi.fn();
|
||||
const providers = makeProviders({
|
||||
sendApns: vi.fn().mockResolvedValue({ success: false, errorCode: "InternalServerError" }),
|
||||
sendWebPush: vi.fn().mockResolvedValue({ success: false, errorCode: "InternalServerError" }),
|
||||
});
|
||||
const input: PushFanOutInput = {
|
||||
payload: makePayload(),
|
||||
@@ -176,9 +172,9 @@ describe("dispatchPushNotification", () => {
|
||||
const info = vi.fn();
|
||||
const warn = vi.fn();
|
||||
const providers = makeProviders({
|
||||
sendApns: vi.fn()
|
||||
sendWebPush: vi.fn()
|
||||
.mockResolvedValueOnce({ success: true, messageId: "id-1" })
|
||||
.mockResolvedValueOnce({ success: false, errorCode: "BadDeviceToken" }),
|
||||
.mockResolvedValueOnce({ success: false, errorCode: "Gone" }),
|
||||
});
|
||||
const input: PushFanOutInput = {
|
||||
payload: makePayload(),
|
||||
@@ -192,7 +188,7 @@ describe("dispatchPushNotification", () => {
|
||||
};
|
||||
await dispatchPushNotification(input);
|
||||
expect(info).toHaveBeenCalledWith(expect.objectContaining({ messageId: "id-1" }), "push delivered");
|
||||
expect(warn).toHaveBeenCalledWith(expect.objectContaining({ errorCode: "BadDeviceToken" }), "push provider reported token revoked");
|
||||
expect(warn).toHaveBeenCalledWith(expect.objectContaining({ errorCode: "Gone" }), "push provider reported token revoked");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -204,9 +200,9 @@ describe("redactToken", () => {
|
||||
});
|
||||
|
||||
describe("REVOCATION_ERROR_CODES", () => {
|
||||
it("includes both APNs and FCM canonical errors", () => {
|
||||
it("includes WebPush permanent failure codes", () => {
|
||||
expect(REVOCATION_ERROR_CODES.has("Unregistered")).toBe(true);
|
||||
expect(REVOCATION_ERROR_CODES.has("BadDeviceToken")).toBe(true);
|
||||
expect(REVOCATION_ERROR_CODES.has("messaging/registration-token-not-registered")).toBe(true);
|
||||
expect(REVOCATION_ERROR_CODES.has("Gone")).toBe(true);
|
||||
expect(REVOCATION_ERROR_CODES.has("ExpiredSubscription")).toBe(true);
|
||||
});
|
||||
});
|
||||
+11
-30
@@ -1,32 +1,21 @@
|
||||
/**
|
||||
* Phase 08 #65 — Push-notification fan-out worker.
|
||||
* Web Push notification fan-out worker.
|
||||
*
|
||||
* Consumes events emitted via the existing notifications pipeline and
|
||||
* dispatches one APNs / FCM / WebPush call per active DeviceToken for
|
||||
* the recipient user. The worker is intentionally provider-pluggable:
|
||||
* the two SDK adapters (firebase-admin for FCM, node-apn for APNs) are
|
||||
* injected through the `providers` seam so unit tests can mock them
|
||||
* without dragging the actual libraries into Vitest's module graph.
|
||||
* dispatches one Web Push call per active DeviceToken for the recipient
|
||||
* user. The provider is injected through the `providers` seam so unit tests
|
||||
* can mock it without dragging provider libraries into Vitest's module graph.
|
||||
*
|
||||
* Provider response codes that signal a permanent invalidation
|
||||
* (`Unregistered`, `InvalidRegistration`, `BadDeviceToken`) flip the
|
||||
* Provider response codes that signal a permanent invalidation flip the
|
||||
* matching DeviceToken's `revokedAt` so subsequent fan-out passes skip
|
||||
* the dead row.
|
||||
*
|
||||
* Bootstrap-time config (apps/worker/src/index.ts) reads:
|
||||
* - FCM_SERVICE_ACCOUNT_JSON (base64 or raw JSON for the FCM key)
|
||||
* - APNS_KEY (PEM contents of the .p8 sign key)
|
||||
* - APNS_KEY_ID (10-char key id from Apple Developer)
|
||||
* - APNS_TEAM_ID (10-char team id)
|
||||
* - APNS_BUNDLE_ID (defaults to de.toolctrl.app)
|
||||
*
|
||||
* @see packages/db/src/device-tokens.server.ts
|
||||
* @see apps/mobile/src/pushRegistration.ts
|
||||
*/
|
||||
|
||||
import { mapWithConcurrency } from "@toolctrl/foundation";
|
||||
import { mapWithConcurrency } from "@hivegrid/foundation";
|
||||
|
||||
export type PushTargetPlatform = "ios" | "android" | "web";
|
||||
export type PushTargetPlatform = "web";
|
||||
export type PushProviderResult = {
|
||||
/** Provider-side message id, when available. */
|
||||
messageId?: string;
|
||||
@@ -56,8 +45,6 @@ export type DeviceTokenRow = {
|
||||
};
|
||||
|
||||
export type PushProviderSeam = {
|
||||
sendFcm: (token: string, payload: PushPayload) => Promise<PushProviderResult>;
|
||||
sendApns: (token: string, payload: PushPayload) => Promise<PushProviderResult>;
|
||||
sendWebPush: (token: string, payload: PushPayload) => Promise<PushProviderResult>;
|
||||
};
|
||||
|
||||
@@ -89,10 +76,8 @@ export type PushFanOutResult = {
|
||||
const REVOCATION_ERROR_CODES = new Set([
|
||||
"Unregistered",
|
||||
"InvalidRegistration",
|
||||
"BadDeviceToken",
|
||||
"DeviceTokenNotForTopic",
|
||||
"messaging/registration-token-not-registered",
|
||||
"messaging/invalid-registration-token",
|
||||
"Gone",
|
||||
"ExpiredSubscription",
|
||||
]);
|
||||
|
||||
export const PUSH_NOTIFICATION_FAN_OUT_CONCURRENCY = 5;
|
||||
@@ -143,12 +128,8 @@ export async function dispatchPushNotification(input: PushFanOutInput): Promise<
|
||||
}
|
||||
|
||||
async function sendForRow(row: DeviceTokenRow, payload: PushPayload, providers: PushProviderSeam): Promise<PushProviderResult> {
|
||||
if (row.platform === "ios" || row.pushProvider === "apns") {
|
||||
return providers.sendApns(row.token, payload);
|
||||
}
|
||||
if (row.platform === "android" || row.pushProvider === "fcm") {
|
||||
return providers.sendFcm(row.token, payload);
|
||||
}
|
||||
void row.platform;
|
||||
void row.pushProvider;
|
||||
return providers.sendWebPush(row.token, payload);
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createSearchClient } from "@toolctrl/search";
|
||||
import { createInMemoryMeili } from "@toolctrl/search/testing";
|
||||
import { createSearchClient } from "@hivegrid/search";
|
||||
import { createInMemoryMeili } from "@hivegrid/search/testing";
|
||||
import { bootstrapSearchIndexer } from "./search-indexer-bootstrap.js";
|
||||
import { SEARCH_INDEX_OUTBOX_QUEUE } from "./search-indexer.js";
|
||||
|
||||
|
||||
+2
-2
@@ -12,8 +12,8 @@
|
||||
* without spinning up a BullMQ Worker.
|
||||
*/
|
||||
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import { createSearchClient, type SearchClient } from "@toolctrl/search";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import { createSearchClient, type SearchClient } from "@hivegrid/search";
|
||||
import {
|
||||
runSearchIndexerJob,
|
||||
SEARCH_INDEX_OUTBOX_QUEUE,
|
||||
|
||||
+162
-165
@@ -1,4 +1,4 @@
|
||||
import { BULK_REINDEX_SUPPORTED_INDEX_NAMES, crmContactsIndex, kbArticlesIndex, rowsIndex, usersIndex } from "@toolctrl/search";
|
||||
import { BULK_REINDEX_SUPPORTED_INDEX_NAMES, crmContactsIndex, kbArticlesIndex, projectsIndex, usersIndex } from "@hivegrid/search";
|
||||
import type { SearchIndexerBulkPager, SearchIndexerDependencies } from "./search-indexer.js";
|
||||
|
||||
type KnowledgeBaseArticleRow = {
|
||||
@@ -15,37 +15,21 @@ type KnowledgeBaseArticleRow = {
|
||||
knowledgeBase: { tenantId: string | null };
|
||||
};
|
||||
|
||||
const SEARCHABLE_ROW_PROPERTY_TYPES = new Set([1, 8]); // TEXT, SELECT
|
||||
|
||||
type RowSearchProperty = {
|
||||
// CRM-contact search rows now come from the typed `Kontakt` table (R-P6 / R-P6B-β,
|
||||
// AD-2) instead of the removed generic entity-builder `Row` engine.
|
||||
type KontaktSearchRow = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: number;
|
||||
};
|
||||
|
||||
type RowSearchValue = {
|
||||
propertyId: string;
|
||||
textValue: string | null;
|
||||
numberValue: { toString(): string } | number | string | null;
|
||||
dateValue: Date | string | null;
|
||||
booleanValue: boolean | null;
|
||||
property: RowSearchProperty;
|
||||
};
|
||||
|
||||
type SearchRow = {
|
||||
id: string;
|
||||
tenantId: string | null;
|
||||
entityId: string;
|
||||
folio: number;
|
||||
tenantId: string;
|
||||
kontaktTyp: string;
|
||||
vorname: string | null;
|
||||
nachname: string | null;
|
||||
orgName: string | null;
|
||||
primaerEmail: string | null;
|
||||
kontaktStatus: string;
|
||||
zustaendigerUserId: string | null;
|
||||
createdByUserId: string | null;
|
||||
createdAt: Date | string;
|
||||
updatedAt: Date | string;
|
||||
deletedAt: Date | string | null;
|
||||
entity: {
|
||||
name: string;
|
||||
properties: RowSearchProperty[];
|
||||
};
|
||||
values: RowSearchValue[];
|
||||
};
|
||||
|
||||
const TENANT_USER_STATUS_ACTIVE = 2;
|
||||
@@ -68,6 +52,26 @@ type TenantUserSearchRow = {
|
||||
};
|
||||
};
|
||||
|
||||
type ProjektSearchRow = {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
projektNummer: string;
|
||||
projektname: string;
|
||||
beschreibung: string | null;
|
||||
projekttyp: string | null;
|
||||
quelle: string | null;
|
||||
pipelineId: string;
|
||||
stageId: string;
|
||||
zustaendigePersonId: string | null;
|
||||
projektleiterId: string | null;
|
||||
archivedAt: Date | string | null;
|
||||
createdAt: Date | string;
|
||||
updatedAt: Date | string;
|
||||
deadline: Date | string | null;
|
||||
potenziellerAuftragswert: unknown;
|
||||
auftragsvolumen: unknown;
|
||||
};
|
||||
|
||||
export type SearchIndexerBulkPagerPrisma = {
|
||||
knowledgeBaseArticle: {
|
||||
findMany(args: {
|
||||
@@ -92,55 +96,30 @@ export type SearchIndexerBulkPagerPrisma = {
|
||||
};
|
||||
}): Promise<KnowledgeBaseArticleRow[]>;
|
||||
};
|
||||
row: {
|
||||
kontakt: {
|
||||
findMany(args: {
|
||||
where: {
|
||||
id?: { gt: string };
|
||||
tenantId: string;
|
||||
deletedAt: null;
|
||||
entity?: { name: string };
|
||||
archivedAt: null;
|
||||
};
|
||||
orderBy: { id: "asc" };
|
||||
take: number;
|
||||
select: {
|
||||
id: true;
|
||||
tenantId: true;
|
||||
entityId: true;
|
||||
folio: true;
|
||||
kontaktTyp: true;
|
||||
vorname: true;
|
||||
nachname: true;
|
||||
orgName: true;
|
||||
primaerEmail: true;
|
||||
kontaktStatus: true;
|
||||
zustaendigerUserId: true;
|
||||
createdByUserId: true;
|
||||
createdAt: true;
|
||||
updatedAt: true;
|
||||
deletedAt: true;
|
||||
entity: {
|
||||
select: {
|
||||
name: true;
|
||||
properties: {
|
||||
select: {
|
||||
id: true;
|
||||
name: true;
|
||||
type: true;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
values: {
|
||||
select: {
|
||||
propertyId: true;
|
||||
textValue: true;
|
||||
numberValue: true;
|
||||
dateValue: true;
|
||||
booleanValue: true;
|
||||
property: {
|
||||
select: {
|
||||
id: true;
|
||||
name: true;
|
||||
type: true;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}): Promise<SearchRow[]>;
|
||||
}): Promise<KontaktSearchRow[]>;
|
||||
};
|
||||
tenantUser: {
|
||||
findMany(args: {
|
||||
@@ -176,14 +155,44 @@ export type SearchIndexerBulkPagerPrisma = {
|
||||
};
|
||||
}): Promise<TenantUserSearchRow[]>;
|
||||
};
|
||||
projekt: {
|
||||
findMany(args: {
|
||||
where: {
|
||||
id?: { gt: string };
|
||||
tenantId: string;
|
||||
archivedAt: null;
|
||||
};
|
||||
orderBy: { id: "asc" };
|
||||
take: number;
|
||||
select: {
|
||||
id: true;
|
||||
tenantId: true;
|
||||
projektNummer: true;
|
||||
projektname: true;
|
||||
beschreibung: true;
|
||||
projekttyp: true;
|
||||
quelle: true;
|
||||
pipelineId: true;
|
||||
stageId: true;
|
||||
zustaendigePersonId: true;
|
||||
projektleiterId: true;
|
||||
archivedAt: true;
|
||||
createdAt: true;
|
||||
updatedAt: true;
|
||||
deadline: true;
|
||||
potenziellerAuftragswert: true;
|
||||
auftragsvolumen: true;
|
||||
};
|
||||
}): Promise<ProjektSearchRow[]>;
|
||||
};
|
||||
};
|
||||
|
||||
export function createSearchIndexerBulkPagers(input: { prisma: SearchIndexerBulkPagerPrisma }): NonNullable<SearchIndexerDependencies["bulkPagers"]> {
|
||||
return {
|
||||
[kbArticlesIndex.name]: createKbArticlesPager(input.prisma),
|
||||
[crmContactsIndex.name]: createCrmContactsPager(input.prisma),
|
||||
[rowsIndex.name]: createRowsPager(input.prisma),
|
||||
[usersIndex.name]: createUsersPager(input.prisma),
|
||||
[projectsIndex.name]: createProjectsPager(input.prisma),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -237,24 +246,31 @@ function createKbArticlesPager(prisma: SearchIndexerBulkPagerPrisma): SearchInde
|
||||
|
||||
function createCrmContactsPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBulkPager {
|
||||
return async ({ tenantId, cursor, pageSize }) => {
|
||||
const rows = await findSearchRows(prisma, {
|
||||
tenantId,
|
||||
cursor,
|
||||
pageSize,
|
||||
entityName: "contact",
|
||||
const rows = await prisma.kontakt.findMany({
|
||||
where: {
|
||||
...(cursor ? { id: { gt: cursor } } : {}),
|
||||
tenantId,
|
||||
archivedAt: null,
|
||||
},
|
||||
orderBy: { id: "asc" },
|
||||
take: pageSize,
|
||||
select: {
|
||||
id: true,
|
||||
tenantId: true,
|
||||
kontaktTyp: true,
|
||||
vorname: true,
|
||||
nachname: true,
|
||||
orgName: true,
|
||||
primaerEmail: true,
|
||||
kontaktStatus: true,
|
||||
zustaendigerUserId: true,
|
||||
createdByUserId: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
},
|
||||
});
|
||||
return {
|
||||
documents: rows.filter((row) => row.tenantId === tenantId && row.entity.name === "contact").map((row) => rowToContactDocument(row, tenantId)),
|
||||
nextCursor: rows.length === pageSize ? (rows[rows.length - 1]?.id ?? null) : null,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function createRowsPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBulkPager {
|
||||
return async ({ tenantId, cursor, pageSize }) => {
|
||||
const rows = await findSearchRows(prisma, { tenantId, cursor, pageSize });
|
||||
return {
|
||||
documents: rows.filter((row) => row.tenantId === tenantId).map((row) => rowToRowsDocument(row, tenantId)),
|
||||
documents: rows.filter((row) => row.tenantId === tenantId).map((row) => kontaktToContactDocument(row, tenantId)),
|
||||
nextCursor: rows.length === pageSize ? (rows[rows.length - 1]?.id ?? null) : null,
|
||||
};
|
||||
};
|
||||
@@ -301,58 +317,41 @@ function createUsersPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBu
|
||||
};
|
||||
}
|
||||
|
||||
function findSearchRows(
|
||||
prisma: SearchIndexerBulkPagerPrisma,
|
||||
input: { tenantId: string; cursor: string | null; pageSize: number; entityName?: string }
|
||||
) {
|
||||
return prisma.row.findMany({
|
||||
where: {
|
||||
...(input.cursor ? { id: { gt: input.cursor } } : {}),
|
||||
tenantId: input.tenantId,
|
||||
deletedAt: null,
|
||||
...(input.entityName ? { entity: { name: input.entityName } } : {}),
|
||||
},
|
||||
orderBy: { id: "asc" },
|
||||
take: input.pageSize,
|
||||
select: {
|
||||
id: true,
|
||||
tenantId: true,
|
||||
entityId: true,
|
||||
folio: true,
|
||||
createdByUserId: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
deletedAt: true,
|
||||
entity: {
|
||||
select: {
|
||||
name: true,
|
||||
properties: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
type: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
function createProjectsPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBulkPager {
|
||||
return async ({ tenantId, cursor, pageSize }) => {
|
||||
const rows = await prisma.projekt.findMany({
|
||||
where: {
|
||||
...(cursor ? { id: { gt: cursor } } : {}),
|
||||
tenantId,
|
||||
archivedAt: null,
|
||||
},
|
||||
values: {
|
||||
select: {
|
||||
propertyId: true,
|
||||
textValue: true,
|
||||
numberValue: true,
|
||||
dateValue: true,
|
||||
booleanValue: true,
|
||||
property: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
type: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: { id: "asc" },
|
||||
take: pageSize,
|
||||
select: {
|
||||
id: true,
|
||||
tenantId: true,
|
||||
projektNummer: true,
|
||||
projektname: true,
|
||||
beschreibung: true,
|
||||
projekttyp: true,
|
||||
quelle: true,
|
||||
pipelineId: true,
|
||||
stageId: true,
|
||||
zustaendigePersonId: true,
|
||||
projektleiterId: true,
|
||||
archivedAt: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
deadline: true,
|
||||
potenziellerAuftragswert: true,
|
||||
auftragsvolumen: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
return {
|
||||
documents: rows.filter((row) => row.tenantId === tenantId).map((row) => projektToProjectDocument(row, tenantId)),
|
||||
nextCursor: rows.length === pageSize ? (rows[rows.length - 1]?.id ?? null) : null,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function tenantUserToUserDocument(row: TenantUserSearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
|
||||
@@ -370,57 +369,55 @@ function tenantUserToUserDocument(row: TenantUserSearchRow, tenantId: string): R
|
||||
};
|
||||
}
|
||||
|
||||
function rowToContactDocument(row: SearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
|
||||
function kontaktToContactDocument(row: KontaktSearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
|
||||
// Project the typed Kontakt columns onto the `crm-contacts` Meilisearch document
|
||||
// keys (German column → English document field), mirroring indexedEntities.
|
||||
return {
|
||||
id: row.id,
|
||||
tenantId,
|
||||
firstName: getTextValue(row, "firstName"),
|
||||
lastName: getTextValue(row, "lastName"),
|
||||
email: getTextValue(row, "email"),
|
||||
company: getTextValue(row, "company"),
|
||||
jobTitle: getTextValue(row, "jobTitle"),
|
||||
status: getTextValue(row, "status"),
|
||||
ownerId: row.createdByUserId ?? "",
|
||||
entityType: row.kontaktTyp === "organisation" ? "company" : "contact",
|
||||
firstName: row.vorname ?? "",
|
||||
lastName: row.nachname ?? "",
|
||||
email: row.primaerEmail ?? "",
|
||||
company: row.orgName ?? "",
|
||||
jobTitle: "",
|
||||
status: row.kontaktStatus,
|
||||
ownerId: row.zustaendigerUserId ?? row.createdByUserId ?? "",
|
||||
createdAt: serializeDate(row.createdAt),
|
||||
updatedAt: serializeDate(row.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function rowToRowsDocument(row: SearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
|
||||
function projektToProjectDocument(row: ProjektSearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
|
||||
return {
|
||||
id: row.id,
|
||||
tenantId,
|
||||
entityId: row.entityId,
|
||||
entityName: row.entity.name,
|
||||
folio: typeof row.folio === "number" ? row.folio : 0,
|
||||
searchable: buildSearchableBlob(row),
|
||||
createdById: row.createdByUserId ?? "",
|
||||
projectNumber: row.projektNummer,
|
||||
name: row.projektname,
|
||||
description: row.beschreibung ?? "",
|
||||
type: row.projekttyp ?? "",
|
||||
source: row.quelle ?? "",
|
||||
pipelineId: row.pipelineId,
|
||||
stageId: row.stageId,
|
||||
ownerId: row.zustaendigePersonId ?? "",
|
||||
projectLeadId: row.projektleiterId ?? "",
|
||||
archived: row.archivedAt !== null,
|
||||
createdAt: serializeDate(row.createdAt),
|
||||
updatedAt: serializeDate(row.updatedAt),
|
||||
deadline: serializeDate(row.deadline),
|
||||
potentialValue: serializeDecimal(row.potenziellerAuftragswert),
|
||||
orderVolume: serializeDecimal(row.auftragsvolumen),
|
||||
};
|
||||
}
|
||||
|
||||
function buildSearchableBlob(row: SearchRow): string {
|
||||
return row.entity.properties
|
||||
.filter((property) => SEARCHABLE_ROW_PROPERTY_TYPES.has(property.type))
|
||||
.map((property) => getTextValue(row, property.name))
|
||||
.filter((value) => value.length > 0)
|
||||
.join(" \n ");
|
||||
}
|
||||
|
||||
function getTextValue(row: SearchRow, propertyName: string): string {
|
||||
const property = row.entity.properties.find((item) => item.name === propertyName);
|
||||
const value = property ? row.values.find((item) => item.propertyId === property.id) : undefined;
|
||||
return normalizeRowValue(value).trim();
|
||||
}
|
||||
|
||||
function normalizeRowValue(value: RowSearchValue | undefined): string {
|
||||
if (!value) return "";
|
||||
if (value.textValue !== null) return value.textValue;
|
||||
if (value.numberValue !== null) return value.numberValue.toString();
|
||||
if (value.dateValue !== null) return serializeDate(value.dateValue) ?? "";
|
||||
if (value.booleanValue !== null) return String(value.booleanValue);
|
||||
return "";
|
||||
function serializeDecimal(value: unknown): number | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
if (typeof value === "number") return value;
|
||||
if (typeof value === "string") return Number(value);
|
||||
if (typeof value === "object" && "toString" in value && typeof value.toString === "function") {
|
||||
return Number(value.toString());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function serializeDate(value: Date | string | null): string | null {
|
||||
|
||||
+35
-95
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createSearchClient, crmContactsIndex, kbArticlesIndex, rowsIndex, searchInIndex, usersIndex } from "@toolctrl/search";
|
||||
import { createInMemoryMeili } from "@toolctrl/search/testing";
|
||||
import { createSearchClient, crmContactsIndex, kbArticlesIndex, searchInIndex, usersIndex } from "@hivegrid/search";
|
||||
import { createInMemoryMeili } from "@hivegrid/search/testing";
|
||||
import { runSearchIndexerJob, SearchIndexerJobError, parseSearchIndexJob } from "./search-indexer.js";
|
||||
import { createSearchIndexerBulkPagers, registeredSearchIndexerBulkPagerNames } from "./search-indexer-bulk-pagers.js";
|
||||
|
||||
@@ -338,75 +338,26 @@ describe("runSearchIndexerJob", () => {
|
||||
expect(result?.nextCursor).toBeNull();
|
||||
});
|
||||
|
||||
it("registers a rows bulk pager that maps searchable tenant rows", async () => {
|
||||
const findRows = vi.fn(async () => [
|
||||
makeSearchRow({
|
||||
id: "row_1",
|
||||
tenantId: "tenant_1",
|
||||
entityName: "account",
|
||||
values: {
|
||||
name: "Acme",
|
||||
status: "active",
|
||||
hiddenNumber: 42,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
const pagers = createSearchIndexerBulkPagers({
|
||||
prisma: fakeSearchPagerPrisma({ row: { findMany: findRows } }),
|
||||
});
|
||||
|
||||
const result = await pagers[rowsIndex.name]?.({
|
||||
indexName: rowsIndex.name,
|
||||
tenantId: "tenant_1",
|
||||
cursor: "row_0",
|
||||
pageSize: 1,
|
||||
});
|
||||
|
||||
expect(registeredSearchIndexerBulkPagerNames).toContain(rowsIndex.name);
|
||||
expect(findRows).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: { id: { gt: "row_0" }, tenantId: "tenant_1", deletedAt: null },
|
||||
take: 1,
|
||||
})
|
||||
);
|
||||
expect(result?.documents).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "row_1",
|
||||
tenantId: "tenant_1",
|
||||
entityId: "entity_1",
|
||||
entityName: "account",
|
||||
folio: 7,
|
||||
searchable: "Acme \n active",
|
||||
createdById: "user_1",
|
||||
}),
|
||||
]);
|
||||
expect(result?.nextCursor).toBe("row_1");
|
||||
});
|
||||
|
||||
it("registers a crm-contacts bulk pager from contact rows only", async () => {
|
||||
const findRows = vi.fn(async () => [
|
||||
makeSearchRow({
|
||||
it("registers a crm-contacts bulk pager from the typed Kontakt table", async () => {
|
||||
const findKontakte = vi.fn(async () => [
|
||||
makeKontaktRow({
|
||||
id: "contact_1",
|
||||
tenantId: "tenant_1",
|
||||
entityName: "contact",
|
||||
values: {
|
||||
firstName: "Ada",
|
||||
lastName: "Lovelace",
|
||||
email: "ada@example.test",
|
||||
company: "Analytical Engines",
|
||||
jobTitle: "Founder",
|
||||
status: "customer",
|
||||
},
|
||||
vorname: "Ada",
|
||||
nachname: "Lovelace",
|
||||
primaerEmail: "ada@example.test",
|
||||
orgName: "Analytical Engines",
|
||||
kontaktStatus: "kunde",
|
||||
zustaendigerUserId: "user_1",
|
||||
}),
|
||||
makeSearchRow({
|
||||
makeKontaktRow({
|
||||
id: "contact_2",
|
||||
tenantId: "tenant_2",
|
||||
entityName: "contact",
|
||||
values: { firstName: "Other" },
|
||||
vorname: "Other",
|
||||
}),
|
||||
]);
|
||||
const pagers = createSearchIndexerBulkPagers({
|
||||
prisma: fakeSearchPagerPrisma({ row: { findMany: findRows } }),
|
||||
prisma: fakeSearchPagerPrisma({ kontakt: { findMany: findKontakte } }),
|
||||
});
|
||||
|
||||
const result = await pagers[crmContactsIndex.name]?.({
|
||||
@@ -417,9 +368,9 @@ describe("runSearchIndexerJob", () => {
|
||||
});
|
||||
|
||||
expect(registeredSearchIndexerBulkPagerNames).toContain(crmContactsIndex.name);
|
||||
expect(findRows).toHaveBeenCalledWith(
|
||||
expect(findKontakte).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: { tenantId: "tenant_1", deletedAt: null, entity: { name: "contact" } },
|
||||
where: { tenantId: "tenant_1", archivedAt: null },
|
||||
orderBy: { id: "asc" },
|
||||
take: 3,
|
||||
})
|
||||
@@ -428,12 +379,13 @@ describe("runSearchIndexerJob", () => {
|
||||
expect.objectContaining({
|
||||
id: "contact_1",
|
||||
tenantId: "tenant_1",
|
||||
entityType: "contact",
|
||||
firstName: "Ada",
|
||||
lastName: "Lovelace",
|
||||
email: "ada@example.test",
|
||||
company: "Analytical Engines",
|
||||
jobTitle: "Founder",
|
||||
status: "customer",
|
||||
jobTitle: "",
|
||||
status: "kunde",
|
||||
ownerId: "user_1",
|
||||
}),
|
||||
]);
|
||||
@@ -531,47 +483,35 @@ describe("runSearchIndexerJob", () => {
|
||||
function fakeSearchPagerPrisma(overrides: Partial<Parameters<typeof createSearchIndexerBulkPagers>[0]["prisma"]>) {
|
||||
return {
|
||||
knowledgeBaseArticle: { findMany: vi.fn(async () => []) },
|
||||
row: { findMany: vi.fn(async () => []) },
|
||||
kontakt: { findMany: vi.fn(async () => []) },
|
||||
tenantUser: { findMany: vi.fn(async () => []) },
|
||||
...overrides,
|
||||
} as Parameters<typeof createSearchIndexerBulkPagers>[0]["prisma"];
|
||||
}
|
||||
|
||||
function makeSearchRow(input: {
|
||||
function makeKontaktRow(input: {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
entityName: string;
|
||||
values: Record<string, string | number | boolean>;
|
||||
vorname?: string | null;
|
||||
nachname?: string | null;
|
||||
orgName?: string | null;
|
||||
primaerEmail?: string | null;
|
||||
kontaktTyp?: string;
|
||||
kontaktStatus?: string;
|
||||
zustaendigerUserId?: string | null;
|
||||
}) {
|
||||
const properties = Object.keys(input.values).map((name, index) => ({
|
||||
id: `property_${name}`,
|
||||
name,
|
||||
type: typeof input.values[name] === "string" ? (name === "hiddenNumber" ? 0 : 1) : 0,
|
||||
order: index,
|
||||
}));
|
||||
return {
|
||||
id: input.id,
|
||||
tenantId: input.tenantId,
|
||||
entityId: "entity_1",
|
||||
folio: 7,
|
||||
kontaktTyp: input.kontaktTyp ?? "privatperson",
|
||||
vorname: input.vorname ?? null,
|
||||
nachname: input.nachname ?? null,
|
||||
orgName: input.orgName ?? null,
|
||||
primaerEmail: input.primaerEmail ?? null,
|
||||
kontaktStatus: input.kontaktStatus ?? "kontakt",
|
||||
zustaendigerUserId: input.zustaendigerUserId ?? null,
|
||||
createdByUserId: "user_1",
|
||||
createdAt: new Date("2026-01-01T00:00:00.000Z"),
|
||||
updatedAt: new Date("2026-01-02T00:00:00.000Z"),
|
||||
deletedAt: null,
|
||||
entity: {
|
||||
name: input.entityName,
|
||||
properties,
|
||||
},
|
||||
values: properties.map((property) => {
|
||||
const value = input.values[property.name];
|
||||
return {
|
||||
propertyId: property.id,
|
||||
textValue: typeof value === "string" ? value : null,
|
||||
numberValue: typeof value === "number" ? value : null,
|
||||
dateValue: null,
|
||||
booleanValue: typeof value === "boolean" ? value : null,
|
||||
property,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -10,11 +10,11 @@
|
||||
* - `search.index.<entity>.bulk-reindex` → page over Postgres in
|
||||
* 1000-row batches and replace the tenant slice
|
||||
*
|
||||
* All writes go through the tenant-scoped wrapper in @toolctrl/search so
|
||||
* All writes go through the tenant-scoped wrapper in @hivegrid/search so
|
||||
* the indexer cannot accidentally cross-pollinate tenants.
|
||||
*/
|
||||
|
||||
import { ALL_SEARCH_INDEXES, bulkIndex, deleteFromIndex, getSearchIndexSchema, type SearchClient, type SearchIndexSchema } from "@toolctrl/search";
|
||||
import { ALL_SEARCH_INDEXES, bulkIndex, deleteFromIndex, getSearchIndexSchema, type SearchClient, type SearchIndexSchema } from "@hivegrid/search";
|
||||
|
||||
export const SEARCH_INDEX_OUTBOX_QUEUE = "outbox.search-index" as const;
|
||||
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
registerTaskReminderSweepCron,
|
||||
resolveTaskReminderSweepConfig,
|
||||
TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
|
||||
TASK_REMINDER_SWEEP_EVERY_MS,
|
||||
TASK_REMINDER_SWEEP_JOB_ID,
|
||||
} from "./task-reminder-cron.js";
|
||||
|
||||
describe("resolveTaskReminderSweepConfig", () => {
|
||||
it("uses the default batch size when the env var is unset", () => {
|
||||
expect(resolveTaskReminderSweepConfig({})).toEqual({ take: TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE });
|
||||
});
|
||||
|
||||
it("parses a positive integer batch size", () => {
|
||||
expect(resolveTaskReminderSweepConfig({ TASK_REMINDER_SWEEP_BATCH_SIZE: "250" })).toEqual({ take: 250 });
|
||||
});
|
||||
|
||||
it("floors a fractional batch size", () => {
|
||||
expect(resolveTaskReminderSweepConfig({ TASK_REMINDER_SWEEP_BATCH_SIZE: "42.9" })).toEqual({ take: 42 });
|
||||
});
|
||||
|
||||
it.each(["0", "-5", "abc", "", "NaN"])(
|
||||
"falls back to the default for the invalid value %j",
|
||||
(raw) => {
|
||||
expect(resolveTaskReminderSweepConfig({ TASK_REMINDER_SWEEP_BATCH_SIZE: raw })).toEqual({
|
||||
take: TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE,
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe("registerTaskReminderSweepCron", () => {
|
||||
it("upserts the hourly scheduler with the resolved batch size", async () => {
|
||||
const upsertJobScheduler = vi.fn(async () => undefined);
|
||||
|
||||
await registerTaskReminderSweepCron({ upsertJobScheduler }, { TASK_REMINDER_SWEEP_BATCH_SIZE: "500" });
|
||||
|
||||
expect(upsertJobScheduler).toHaveBeenCalledTimes(1);
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
TASK_REMINDER_SWEEP_JOB_ID,
|
||||
{ every: TASK_REMINDER_SWEEP_EVERY_MS },
|
||||
{
|
||||
name: TASK_REMINDER_SWEEP_JOB_ID,
|
||||
data: { kind: "cleanup.run", task: TASK_REMINDER_SWEEP_JOB_ID, take: 500 },
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
it("registers with the default batch size when env is empty", async () => {
|
||||
const upsertJobScheduler = vi.fn(async () => undefined);
|
||||
|
||||
await registerTaskReminderSweepCron({ upsertJobScheduler }, {});
|
||||
|
||||
expect(upsertJobScheduler).toHaveBeenCalledWith(
|
||||
TASK_REMINDER_SWEEP_JOB_ID,
|
||||
{ every: TASK_REMINDER_SWEEP_EVERY_MS },
|
||||
expect.objectContaining({
|
||||
data: expect.objectContaining({ take: TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE }),
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
|
||||
import {
|
||||
runTransactionalMailOutboxJob,
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { queueNames } from "@toolctrl/queue";
|
||||
import { queueNames } from "@hivegrid/queue";
|
||||
import {
|
||||
parseTransactionalMailOutboxJob,
|
||||
runTransactionalMailOutboxJob,
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Bridges transactional mail outbox events onto the existing
|
||||
* `toolctrl.email` send-transactional-email processor.
|
||||
* `hivegrid.email` send-transactional-email processor.
|
||||
*/
|
||||
|
||||
import {
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
type QueueEnqueueOptions,
|
||||
type TransactionalMailJobDescriptor,
|
||||
type TransactionalMailJobInput,
|
||||
} from "@toolctrl/queue";
|
||||
} from "@hivegrid/queue";
|
||||
|
||||
export const TRANSACTIONAL_MAIL_OUTBOX_QUEUE = "outbox.mail" as const;
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import { mapWithConcurrency } from "@toolctrl/foundation";
|
||||
import { mapWithConcurrency } from "@hivegrid/foundation";
|
||||
|
||||
/**
|
||||
* Marketing-Funnel Phase 3 (Phase 05 #80) — trial-expiring trigger cron.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import type { EnvRecord } from "@toolctrl/config";
|
||||
import type { EnvRecord } from "@hivegrid/config";
|
||||
import {
|
||||
WORKFLOW_TRIGGER_OUTBOX_QUEUE,
|
||||
runWorkflowTriggerJob,
|
||||
|
||||
+5
-1
@@ -1,8 +1,12 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
// Resolve the repo root from this file's location (apps/worker/src/jobs/) instead of
|
||||
// process.cwd(): vitest runs from the repo root locally but with cwd=apps/worker in CI,
|
||||
// which doubled the path to apps/worker/apps/worker/src → ENOENT.
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../..");
|
||||
const workerSrcRoot = path.join(repoRoot, "apps/worker/src");
|
||||
const workflowTriggerRoot = path.join(workerSrcRoot, "jobs");
|
||||
|
||||
|
||||
+60
-13
@@ -9,15 +9,15 @@
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createDbClient } from "@toolctrl/db/client.server";
|
||||
import { publishOutboxEvent, type OutboxEvent, type OutboxPublisherTransaction } from "@toolctrl/db/outbox";
|
||||
import { claimPendingOutboxEvents } from "@toolctrl/db/outbox-claim";
|
||||
import { createDbClient } from "@hivegrid/db/client.server";
|
||||
import { publishOutboxEvent, type OutboxEvent, type OutboxPublisherTransaction } from "@hivegrid/db/outbox";
|
||||
import { claimPendingOutboxEvents } from "@hivegrid/db/outbox-claim";
|
||||
import {
|
||||
buildWorkflowExecutionContinueRequestedOutboxEvent,
|
||||
buildWorkflowExecutionRequestedOutboxEvent,
|
||||
buildWorkflowTriggerOutboxEvent,
|
||||
queueNames,
|
||||
} from "@toolctrl/queue";
|
||||
} from "@hivegrid/queue";
|
||||
|
||||
import { runDispatcherTick } from "../outbox/dispatcher.js";
|
||||
import { runPluginHookFanoutJob } from "./plugin-hook-fanout.js";
|
||||
@@ -31,7 +31,54 @@ const integrationUrl =
|
||||
process.env.WORKFLOW_TRIGGER_E2E_DATABASE_URL?.trim() ||
|
||||
process.env.OUTBOX_E2E_DATABASE_URL?.trim() ||
|
||||
(DATABASE_URL && !DATABASE_URL.includes("example.invalid") ? DATABASE_URL : undefined);
|
||||
const describeIntegration = integrationUrl ? describe : describe.skip;
|
||||
|
||||
/** Host:port label for logs — never includes credentials. */
|
||||
function databaseTargetLabel(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return `${parsed.hostname}:${parsed.port || "5432"}`;
|
||||
} catch {
|
||||
return "configured database";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Usability probe for the configured Postgres URL.
|
||||
*
|
||||
* The sibling integration suites gate purely on a non-sentinel DATABASE_URL
|
||||
* being set (see `outbox-e2e.test.ts`). That is insufficient for the CI Worker
|
||||
* Service job: it injects a localhost DATABASE_URL but runs no Postgres service,
|
||||
* so gating on the URL alone makes this suite connect and fail with P1001
|
||||
* instead of skipping. We additionally confirm the server is actually usable by
|
||||
* running a trivial `SELECT 1` with the suite's own client factory. Any
|
||||
* connection problem (unreachable / auth failure / missing database / timeout)
|
||||
* resolves to "not reachable" so local runs with a real DB still execute while
|
||||
* CI-without-DB skips. `onError` is silenced so the probe stays quiet on the
|
||||
* skip path.
|
||||
*/
|
||||
async function isDatabaseReachable(url: string, timeoutMs = 4_000): Promise<boolean> {
|
||||
const probe = createDbClient({ connectionString: url, onError: () => undefined });
|
||||
// The query promise resolves to true/false and never rejects, so the losing
|
||||
// race branch can't surface as an unhandled rejection.
|
||||
const query = probe.$queryRaw`SELECT 1`.then(
|
||||
() => true,
|
||||
() => false
|
||||
);
|
||||
const timeout = new Promise<boolean>((resolve) => {
|
||||
setTimeout(() => resolve(false), timeoutMs).unref?.();
|
||||
});
|
||||
try {
|
||||
return await Promise.race([query, timeout]);
|
||||
} finally {
|
||||
await probe.$disconnect().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
const databaseReachable = integrationUrl ? await isDatabaseReachable(integrationUrl) : false;
|
||||
if (integrationUrl && !databaseReachable) {
|
||||
console.warn(`[workflow-trigger-e2e] skipped: no database reachable at ${databaseTargetLabel(integrationUrl)}`);
|
||||
}
|
||||
const describeIntegration = databaseReachable ? describe : describe.skip;
|
||||
|
||||
const TEST_SLUG_PREFIX = "workflow-trigger-e2e-";
|
||||
|
||||
@@ -459,8 +506,8 @@ describeIntegration("workflow trigger outbox bridge (integration)", () => {
|
||||
"Idempotency-Key": `workflow-trigger:${enqueued[0]!.jobData.outboxEventId}:${workflow.id}:${tenant.id}`,
|
||||
"X-Api-Key": "service-secret",
|
||||
"X-Tenant-Id": tenant.id,
|
||||
"x-toolctrl-internal-web-api-proxy": "1",
|
||||
"x-toolctrl-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
"x-hivegrid-internal-web-api-proxy": "1",
|
||||
"x-hivegrid-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
method: "POST",
|
||||
url: expectedUrl,
|
||||
secret: "service-secret",
|
||||
@@ -654,20 +701,20 @@ describeIntegration("workflow trigger outbox bridge (integration)", () => {
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
const gptBlock = await prisma.workflowBlock.create({
|
||||
const unsupportedBlock = await prisma.workflowBlock.create({
|
||||
data: {
|
||||
workflowId: workflow.id,
|
||||
type: "gpt",
|
||||
description: "Unsupported native GPT block",
|
||||
type: "rowDelete",
|
||||
description: "Unsupported native rowDelete block",
|
||||
isBlock: true,
|
||||
input: JSON.stringify({ model: "gpt-4.1-mini", prompt: "Summarize" }),
|
||||
input: JSON.stringify({ entity: "employee", id: "row_1" }),
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
await prisma.workflowBlockToBlock.create({
|
||||
data: {
|
||||
fromBlockId: manualBlock.id,
|
||||
toBlockId: gptBlock.id,
|
||||
toBlockId: unsupportedBlock.id,
|
||||
condition: null,
|
||||
},
|
||||
});
|
||||
@@ -750,7 +797,7 @@ describeIntegration("workflow trigger outbox bridge (integration)", () => {
|
||||
|
||||
const failedExecution = await prisma.workflowExecution.findUniqueOrThrow({ where: { id: execution.id } });
|
||||
expect(failedExecution.status).toBe("error");
|
||||
expect(failedExecution.error).toContain('unsupported block type "gpt"');
|
||||
expect(failedExecution.error).toContain('unsupported block type "rowDelete"');
|
||||
expect(pluginHookEvents).toHaveLength(2);
|
||||
expect(pluginHookEvents).toEqual([
|
||||
expect.objectContaining({
|
||||
|
||||
+2
-2
@@ -291,7 +291,7 @@ describe("workflow trigger executor factory", () => {
|
||||
continueExecution: true,
|
||||
executionType: "api",
|
||||
userId: "user_1",
|
||||
capabilityErrors: ['unsupported block type "waitForInput" on block wait_1', 'unsupported block type "gpt" on block gpt_1'],
|
||||
capabilityErrors: ['unsupported block type "waitForInput" on block wait_1', 'unsupported block type "rowCreate" on block row_create_1'],
|
||||
});
|
||||
|
||||
const nativeExecutor = vi.mocked(createWorkflowTriggerNativeExecutor).mock.results[0]?.value;
|
||||
@@ -301,7 +301,7 @@ describe("workflow trigger executor factory", () => {
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
executionId: "execution_1",
|
||||
capabilityErrors: ['unsupported block type "waitForInput" on block wait_1', 'unsupported block type "gpt" on block gpt_1'],
|
||||
capabilityErrors: ['unsupported block type "waitForInput" on block wait_1', 'unsupported block type "rowCreate" on block row_create_1'],
|
||||
}),
|
||||
"workflow trigger hybrid executor using HTTP bridge for unsupported queued continuation"
|
||||
);
|
||||
|
||||
+10
-10
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { buildWorkflowTriggerOutboxEvent } from "@toolctrl/queue";
|
||||
import type { OutboxEvent } from "@toolctrl/db/outbox";
|
||||
import { buildWorkflowTriggerOutboxEvent } from "@hivegrid/queue";
|
||||
import type { OutboxEvent } from "@hivegrid/db/outbox";
|
||||
|
||||
import {
|
||||
createWorkflowTriggerHttpExecutor,
|
||||
@@ -46,8 +46,8 @@ describe("workflow trigger HTTP executor", () => {
|
||||
"Idempotency-Key": "workflow-trigger:evt_1:workflow_1:tenant_1",
|
||||
"X-Api-Key": "service-secret",
|
||||
"X-Tenant-Id": "tenant_1",
|
||||
"x-toolctrl-internal-web-api-proxy": "1",
|
||||
"x-toolctrl-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
"x-hivegrid-internal-web-api-proxy": "1",
|
||||
"x-hivegrid-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
method: "POST",
|
||||
url: "https://app.example.com/internal/api/workflows/trigger/workflow_1?tenantId=tenant_1",
|
||||
secret: "service-secret",
|
||||
@@ -99,7 +99,7 @@ describe("workflow trigger HTTP executor", () => {
|
||||
expect(init?.body).toBe(JSON.stringify({ workflowId: "workflow_1", type: "manual", input: { approved: true }, userId: "user_1" }));
|
||||
expect(init?.headers).toMatchObject({
|
||||
"Idempotency-Key": "workflow-execution:outbox_1:execution_1:tenant_1",
|
||||
"x-toolctrl-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
"x-hivegrid-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
method: "POST",
|
||||
url: "https://app.example.com/internal/api/workflows/executions/execution_1/start?tenantId=tenant_1",
|
||||
secret: "service-secret",
|
||||
@@ -131,7 +131,7 @@ describe("workflow trigger HTTP executor", () => {
|
||||
expect(init?.body).toBe(JSON.stringify({ workflowId: "workflow_1", type: "api", input: { approved: true }, userId: "user_1" }));
|
||||
expect(init?.headers).toMatchObject({
|
||||
"Idempotency-Key": "workflow-continuation:outbox_1:execution_1:tenant_1",
|
||||
"x-toolctrl-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
"x-hivegrid-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
method: "POST",
|
||||
url: "https://app.example.com/internal/api/workflows/executions/execution_1/continue/start?tenantId=tenant_1",
|
||||
secret: "service-secret",
|
||||
@@ -355,8 +355,8 @@ describe("workflow trigger HTTP executor", () => {
|
||||
"Idempotency-Key": "workflow-trigger:evt_1:workflow_1:tenant_1",
|
||||
"X-Api-Key": "service-secret",
|
||||
"X-Tenant-Id": "tenant_1",
|
||||
"x-toolctrl-internal-web-api-proxy": "1",
|
||||
"x-toolctrl-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
"x-hivegrid-internal-web-api-proxy": "1",
|
||||
"x-hivegrid-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
method: "POST",
|
||||
url: "https://app.example.com/internal/api/workflows/trigger/workflow_1?tenantId=tenant_1",
|
||||
secret: "service-secret",
|
||||
@@ -466,8 +466,8 @@ describe("workflow trigger HTTP executor", () => {
|
||||
expect(init?.headers).toMatchObject({
|
||||
"Idempotency-Key": "workflow-trigger:evt_workflow_1:workflow_1:tenant_1",
|
||||
"X-Tenant-Id": "tenant_1",
|
||||
"x-toolctrl-internal-web-api-proxy": "1",
|
||||
"x-toolctrl-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
"x-hivegrid-internal-web-api-proxy": "1",
|
||||
"x-hivegrid-internal-web-api-proxy-signature": createWorkflowTriggerProxySignature({
|
||||
method: "POST",
|
||||
url: "https://app.example.com/internal/api/workflows/trigger/workflow_1?tenantId=tenant_1",
|
||||
secret: "service-secret",
|
||||
|
||||
Loaded 100 of 3781 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user