Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -15,6 +15,7 @@ This package defines stable queue names, typed payloads, job descriptors, produc
| `hivegrid.file-scan` | Storage file scan jobs. |
| `hivegrid.exports` | Export job execution. |
| `hivegrid.imports` | Import job execution. |
| `hivegrid.kwp-imports` | Isolated KWP provider synchronization jobs. |
| `hivegrid.plugin-invoke` | Isolated third-party plugin invocation jobs. |
## Browser Jobs
@@ -27,6 +28,7 @@ Stable job names:
- `scan-file`
- `run-export`
- `run-import`
- `run-kwp-import`
These are typed contracts and descriptors. Concrete processing is owned by `apps/worker` handlers and may still be disabled until the matching service flag or provider configuration is present.
@@ -1,9 +1,15 @@
import { describe, expect, test } from "vitest";
import { queueNames, type QueueName } from "./names.js";
import {
operationalQueueNames,
productAcquisitionProviderOutboxQueueName,
queueNames,
type QueueName,
} from "./names.js";
import {
WORKER_QUEUE_GROUPS,
WORKER_QUEUE_GROUP_NAMES,
getGroupForQueue,
getOperationalGroupForQueue,
getQueueNamesForGroup,
isWorkerQueueGroup,
} from "./groups.js";
@@ -58,8 +64,8 @@ describe("getQueueNamesForGroup", () => {
expect(getQueueNamesForGroup("mail")).toEqual([queueNames.email]);
});
test("returns import + export queues for the import group", () => {
expect(getQueueNamesForGroup("import")).toEqual([queueNames.imports, queueNames.exports]);
test("returns dedicated KWP plus generic import and export queues for the import group", () => {
expect(getQueueNamesForGroup("import")).toEqual([queueNames.kwpImports, queueNames.imports, queueNames.exports]);
});
});
@@ -72,6 +78,19 @@ describe("getGroupForQueue", () => {
});
});
describe("getOperationalGroupForQueue", () => {
test("assigns every operator-visible queue to an explicit group", () => {
for (const queue of operationalQueueNames) {
expect(WORKER_QUEUE_GROUP_NAMES).toContain(getOperationalGroupForQueue(queue));
}
});
test("assigns the dedicated product-provider outbox queue to general", () => {
expect(getOperationalGroupForQueue(productAcquisitionProviderOutboxQueueName)).toBe("general");
expect(WORKER_QUEUE_GROUPS.general).not.toContain(productAcquisitionProviderOutboxQueueName);
});
});
describe("isWorkerQueueGroup", () => {
test("accepts every defined group", () => {
for (const group of WORKER_QUEUE_GROUP_NAMES) {
@@ -1,4 +1,10 @@
import { queueNames, type QueueName } from "./names.js";
import {
productAcquisitionProviderOutboxQueueName,
recurringInvoiceCollectionOutboxQueueName,
queueNames,
type OperationalQueueName,
type QueueName,
} from "./names.js";
/**
* Worker queue groups — single source of truth for which BullMQ queues a
@@ -12,10 +18,10 @@ import { queueNames, type QueueName } from "./names.js";
* cleanup, outbox dispatcher cron). Concurrency ~10.
* - `mail` — outbound mail and webhook delivery (SMTP/HTTP throughput-
* bound). Concurrency ~5; replicate horizontally for volume.
* - `import` — long-running CPU/memory-bound imports/exports. Concurrency
* 1 — head-of-line-blocking is acceptable inside this group
* because all jobs are heavy; we isolate them so they don't
* stall the rest of the system.
* - `import` — provider sync plus long-running CPU/memory-bound
* imports/exports. KWP uses a dedicated queue so its
* read-only synchronization can run while generic imports
* remain gated off.
* - `webhook` — external API integrations (browser automation) that are
* I/O-bound and benefit from high parallelism.
* - `plugin` — third-party WebAssembly plugin invocations (Phase 06
@@ -30,7 +36,7 @@ import { queueNames, type QueueName } from "./names.js";
export const WORKER_QUEUE_GROUPS = {
general: [queueNames.notifications, queueNames.maintenance, queueNames.fileScan, queueNames.filePreview],
mail: [queueNames.email],
import: [queueNames.imports, queueNames.exports],
import: [queueNames.kwpImports, queueNames.imports, queueNames.exports],
webhook: [queueNames.browserAutomation],
plugin: [queueNames.pluginInvoke],
} as const satisfies Record<string, readonly QueueName[]>;
@@ -66,6 +72,18 @@ export function getGroupForQueue(queue: QueueName): WorkerQueueGroup {
return QUEUE_TO_GROUP[queue];
}
/**
* Reverse lookup for operator-visible queues. Dedicated outbox consumers are
* operationally owned by the general worker group without being added to the
* generic group subscription list.
*/
export function getOperationalGroupForQueue(queue: OperationalQueueName): WorkerQueueGroup {
if (queue === productAcquisitionProviderOutboxQueueName || queue === recurringInvoiceCollectionOutboxQueueName) {
return "general";
}
return getGroupForQueue(queue);
}
/**
* Type guard — does this string identify a known worker queue group?
* Used by the runtime entry point to validate `WORKER_QUEUE_GROUP` env.
@@ -9,8 +9,18 @@ export {
buildFilePreviewJob,
buildFileScanJob,
buildImportJob,
buildAdminSalesContactsImportJob,
buildAdminSalesContactsExportJob,
buildKwpArchiveDiscoveryJob,
buildKwpContractSentinelJob,
buildKwpArchiveFileSyncJob,
buildKwpProjectCatalogSyncJob,
buildKwpProjectSyncJob,
buildNotificationJob,
buildPlatformNotificationJob,
buildPlatformNotificationRequestedOutboxEvent,
buildPluginInvokeJob,
buildPortalDocumentReadyOutboxEvent,
buildRetryJobOptions,
buildIdempotentJobId,
buildTransactionalMailJob,
@@ -22,6 +32,8 @@ export {
getEnqueueObserver,
setEnqueueObserver,
} from "./producers.js";
export { fromBullMqJobId, toBullMqJobId } from "./job-id.js";
export { PORTAL_DOCUMENT_READY_EVENT } from "./producers.js";
export type { EnqueueObserver } from "./producers.js";
export type {
BrowserAutomationProducerJobDescriptor,
@@ -40,10 +52,23 @@ export type {
FilePreviewJobInput,
FileScanJobDescriptor,
FileScanJobInput,
ImportJobDescriptor,
ImportJobInput,
AdminSalesContactsImportJobInput,
AdminSalesContactsExportJobInput,
KwpImportJobDescriptor,
KwpArchiveDiscoveryJobInput,
KwpContractSentinelJobInput,
KwpArchiveFileSyncJobInput,
KwpProjectCatalogSyncJobInput,
KwpProjectSyncJobInput,
NotificationJobDescriptor,
NotificationJobInput,
PlatformNotificationJobInput,
PlatformNotificationRequestedOutboxEventInput,
PluginInvokeJobDescriptor,
PluginInvokeJobInput,
PortalDocumentReadyOutboxEventInput,
ProducerJobDescriptor,
QueueEnqueueOptions,
TransactionalMailJobDescriptor,
@@ -54,12 +79,20 @@ export type {
WorkflowExecutionContinueRequestedOutboxEventInput,
WorkflowExecutionRequestedOutboxEventInput,
} from "./producers.js";
export { isQueueName, queueNames } from "./names.js";
export type { QueueKey, QueueName } from "./names.js";
export {
isOperationalQueueName,
isQueueName,
operationalQueueNames,
productAcquisitionProviderOutboxQueueName,
recurringInvoiceCollectionOutboxQueueName,
queueNames,
} from "./names.js";
export type { OperationalQueueName, QueueKey, QueueName } from "./names.js";
export {
WORKER_QUEUE_GROUPS,
WORKER_QUEUE_GROUP_NAMES,
getGroupForQueue,
getOperationalGroupForQueue,
getQueueNamesForGroup,
isWorkerQueueGroup,
} from "./groups.js";
@@ -77,14 +110,42 @@ export {
setTenantPlanLookup,
} from "./priority.js";
export type { PriorityOverride, TenantPlanIdentifier, TenantPlanLookup } from "./priority.js";
export {
TELEPHONY_WEBHOOK_JOB_NAME,
TELEPHONY_WEBHOOK_QUEUE_NAME,
buildTelephonyWebhookJob,
enqueueTelephonyWebhookJob,
recoverFailedTelephonyWebhookJob,
} from "./telephony.js";
export type {
TelephonyWebhookJobDescriptor,
TelephonyWebhookJobInput,
TelephonyWebhookProcessJob,
TelephonyWebhookQueue,
TelephonyWebhookRecoveryQueue,
TelephonyWebhookRetainedJob,
} from "./telephony.js";
export { parseRedisUrl, resolveRedisConnectionOptions, resolveRedisConnectionOptionsFromProcessEnv } from "./connection.js";
export type { ConnectionOptions, RedisConnectionEnv, ResolveRedisConnectionOptions } from "./connection.js";
export type {
ExportJob,
FilePreviewJob,
FileScanJob,
FileImportJob,
AdminSalesContactsImportJob,
AdminSalesContactsExportJob,
ImportJob,
KwpImportJob,
KwpArchiveDiscoveryJob,
KwpContractSentinelJob,
KwpArchiveFileSyncJob,
KwpProjectCatalogSyncJob,
KwpProjectSyncJob,
PluginInvokeJob,
PlatformNotificationJob,
PlatformNotificationRequestedOutboxEvent,
PlatformNotificationRequestedOutboxPayload,
PlatformNotificationSafeMetadata,
QueueJsonValue,
QueueJobDescriptor,
QueueJobName,
@@ -0,0 +1,41 @@
import { describe, expect, it } from "vitest";
import { fromBullMqJobId, toBullMqJobId } from "./job-id.js";
describe("BullMQ job id mapping", () => {
it("escapes colons so BullMQ accepts idempotency keys as custom ids", () => {
expect(toBullMqJobId("portal-deletion-confirmation:3d1c2427")).toBe("portal-deletion-confirmation%3A3d1c2427");
});
it("prefixes integer-only keys, which BullMQ rejects as custom ids", () => {
expect(toBullMqJobId("12345")).toBe("job-12345");
});
it("keeps the lossy integer prefix visible instead of guessing the original key", () => {
// `42` and `job-42` both encode to `job-42`, so decoding cannot tell them
// apart. Showing the wire form is honest; stripping would display a key
// that may never have been enqueued.
expect(toBullMqJobId("42")).toBe(toBullMqJobId("job-42"));
expect(fromBullMqJobId("job-42")).toBe("job-42");
});
it("shows operators the original key instead of the wire form", () => {
// The admin jobs list rendered "portal-deletion-confirmation%3A3d1c2427".
expect(fromBullMqJobId("portal-deletion-confirmation%3A3d1c2427")).toBe("portal-deletion-confirmation:3d1c2427");
});
it("round-trips keys that contain colons, percent signs and literal escape sequences", () => {
for (const key of ["plain-key", "a:b:c", "100%-key", "literal-%3A-inside", "literal-%25-inside", "mixed:%3A:%25", "job-abc:1"]) {
expect(fromBullMqJobId(toBullMqJobId(key))).toBe(key);
}
});
it("leaves a key without escape sequences untouched in both directions", () => {
expect(toBullMqJobId("outbox-abc-1")).toBe("outbox-abc-1");
expect(fromBullMqJobId("outbox-abc-1")).toBe("outbox-abc-1");
});
it("keeps a hyphenated job prefix that is not an escaped integer", () => {
expect(fromBullMqJobId("job-abc")).toBe("job-abc");
});
});
@@ -0,0 +1,34 @@
/**
* Bijective mapping between HiveGrid idempotency keys and BullMQ custom job ids.
*
* BullMQ rejects integer-only custom ids and most ids containing a colon, while
* HiveGrid idempotency keys use colons throughout (`portal-deletion-confirmation:<uuid>`).
* The escape is intentionally reversible so operator surfaces can show the
* original key instead of the wire form.
*/
const NUMERIC_ID = /^\d+$/u;
const NUMERIC_JOB_PREFIX = "job-";
const ESCAPED_TOKEN = /%25|%3A/gu;
/** Encodes an idempotency key into a BullMQ-safe custom job id. */
export function toBullMqJobId(value: string): string {
const escaped = value.replaceAll("%", "%25").replaceAll(":", "%3A");
return NUMERIC_ID.test(value) ? `${NUMERIC_JOB_PREFIX}${escaped}` : escaped;
}
/**
* Reverses the character escaping of {@link toBullMqJobId} for display.
*
* Decoding happens in a single left-to-right pass: replacing `%3A` before `%25`
* would corrupt keys that legitimately contain the literal text `%3A`.
*
* The `job-` prefix that {@link toBullMqJobId} adds to integer-only keys is
* deliberately *not* stripped, because that part of the encoding is lossy — the
* keys `42` and `job-42` both encode to `job-42`. Stripping would show one of
* them as a key that was never enqueued, so an integer-only key stays in its
* wire form rather than being guessed.
*/
export function fromBullMqJobId(value: string): string {
return value.replace(ESCAPED_TOKEN, (token) => (token === "%25" ? "%" : ":"));
}
@@ -7,14 +7,39 @@ export const queueNames = {
filePreview: "hivegrid.file-preview",
exports: "hivegrid.exports",
imports: "hivegrid.imports",
kwpImports: "hivegrid.kwp-imports",
pluginInvoke: "hivegrid.plugin-invoke",
} as const;
export type QueueKey = keyof typeof queueNames;
export type QueueName = (typeof queueNames)[QueueKey];
/**
* Durable product-acquisition provider side effects are consumed by a
* dedicated worker rather than the typed producer registry. Keep the name
* centralised so operator surfaces can allowlist it without making it a
* generic worker-group subscription.
*/
export const productAcquisitionProviderOutboxQueueName = "outbox.billing-product-provider" as const;
export const recurringInvoiceCollectionOutboxQueueName = "outbox.billing-recurring-payment" as const;
/** Queues visible to operators, including durable outbox consumers that are
* created directly rather than through the typed producer registry. */
export const operationalQueueNames = [
...Object.values(queueNames),
productAcquisitionProviderOutboxQueueName,
recurringInvoiceCollectionOutboxQueueName,
] as const;
export type OperationalQueueName = (typeof operationalQueueNames)[number];
const queueNameValues = new Set<string>(Object.values(queueNames));
const operationalQueueNameValues = new Set<string>(operationalQueueNames);
export function isQueueName(value: string): value is QueueName {
return queueNameValues.has(value);
}
export function isOperationalQueueName(value: string): value is OperationalQueueName {
return operationalQueueNameValues.has(value);
}
@@ -54,7 +54,7 @@ export type WebhookDeliveryJob = {
metadata?: Record<string, unknown>;
};
export type NotificationJob = {
export type TenantNotificationJob = {
kind: "notification.send";
userId: string;
channel: "email" | "in-app";
@@ -71,6 +71,40 @@ export type NotificationJob = {
sourceKey?: string;
};
export type PlatformNotificationSafeMetadata = {
deliveryId?: string;
eventId?: string;
eventType?: string;
leadId?: string;
taskId?: string;
emailMessageId?: string;
emailEventId?: string;
requestId?: string;
sourceId?: string;
deeplink?: string;
replay?: boolean;
};
export type PlatformNotificationJob = {
kind: "notification.platform.send";
tenantId: null;
userId?: string;
adminUserId?: string;
sourceKey: string;
channel: "email" | "in-app";
messageKey: string;
category?: string;
email?: string;
subject?: string;
text?: string;
html?: string;
templateId?: string;
templateData?: Record<string, unknown>;
metadata?: PlatformNotificationSafeMetadata;
};
export type NotificationJob = TenantNotificationJob | PlatformNotificationJob;
export type CleanupJob = {
kind: "cleanup.run";
task:
@@ -79,12 +113,28 @@ export type CleanupJob = {
| "audit-log-retention"
| "temporary-files"
| "expired-export-artifacts"
| "privacy-gdpr-export-retention"
| "portal-gdpr-export-retention"
| "dsgvo-grace-period-scan"
| "privacy-tenant-deletion-scan"
| "portal-lifecycle-scan"
| "ai-cost-billing-report"
| "outbox-done-retention"
| "workflow-execution-stalled-reconcile"
| "automation-scheduler-sweep"
| "task-archive-retention"
| "project-auto-archive"
| "task-reminder-sweep"
| "admin-sales-task-reminder-sweep"
| "admin-booking-reminder-sweep"
| "admin-calendar-sync"
| "admin-email-delivery"
| "admin-email-event-retention"
| "admin-email-sync"
| "admin-email-domain-check"
| "admin-call-recording-process"
| "admin-call-recording-retention"
| "admin-outbound-call-process"
| "integrations-token-refresh"
| "integrations-health-sweep"
| "calendar-token-refresh"
@@ -99,7 +149,11 @@ export type CleanupJob = {
| "email-inbox-event-cleanup"
| "hr-upload-reminder-dispatch"
| "email-domain-probe"
| "lead-retention-sweep";
| "lead-retention-sweep"
| "billing-dunning-sweep"
| "product-acquisition-reconcile"
| "kwp-project-maintenance"
| "telephony-webhook-maintenance";
olderThanHours?: number;
olderThanMinutes?: number;
take?: number;
@@ -161,7 +215,7 @@ export type ExportJob = {
filters?: Record<string, unknown>;
};
export type ImportJob = {
export type FileImportJob = {
kind: "import.run";
importJobId: string;
tenantId: string;
@@ -177,6 +231,101 @@ export type ImportJob = {
metadata?: Record<string, unknown>;
};
export type AdminSalesContactsImportJob = {
kind: "admin-sales.contacts-import";
importJobId: string;
requestedByAdminUserId: string;
requestId?: string;
metadata?: Record<string, unknown>;
};
export type AdminSalesContactsExportJob = {
kind: "admin-sales.contacts-export";
exportJobId: string;
requestedByAdminUserId: string;
requestId?: string;
metadata?: Record<string, unknown>;
};
/** KWP provider calls run on the import worker. The maintenance worker only
* creates these stable, replay-safe descriptors. */
export type KwpProjectSyncJob = {
kind: "kwp.project-sync";
tenantId: string;
bindingId: string;
dataConnectionId: string;
cadenceBucket: string;
requestId?: string;
metadata?: Record<string, unknown>;
};
export type KwpProjectCatalogSyncJob = {
kind: "kwp.project-catalog-sync";
tenantId: string;
dataConnectionId: string;
cadenceBucket: string;
requestId?: string;
metadata?: Record<string, unknown>;
};
export type KwpContractSentinelJob = {
kind: "kwp.contract-sentinel";
tenantId: string;
dataConnectionId: string;
cadenceBucket: string;
requestId?: string;
metadata?: Record<string, unknown>;
};
export type KwpArchiveFileSyncJob = {
kind: "kwp.archive-file-sync";
tenantId: string;
bindingId: string;
dataConnectionId: string;
cadenceBucket: string;
archiveFile: {
externalId: string;
headerId: number;
revisionId: number;
revision: number;
folderId: number;
displayName: string;
documentType: string | null;
providerFileType: string | null;
providerSizeBytes: number | null;
lastModifiedAt: string | null;
};
requestId?: string;
metadata?: Record<string, unknown>;
};
export type KwpArchiveDiscoveryJob = {
kind: "kwp.archive-discovery";
tenantId: string;
bindingId: string;
dataConnectionId: string;
cadenceBucket: string;
batchNumber: number;
cursor?: {
pendingFolderIds: number[];
visitedFolderIds: number[];
};
requestId?: string;
metadata?: Record<string, unknown>;
};
export type ImportJob =
| FileImportJob
| AdminSalesContactsImportJob
| AdminSalesContactsExportJob;
export type KwpImportJob =
| KwpProjectSyncJob
| KwpProjectCatalogSyncJob
| KwpContractSentinelJob
| KwpArchiveDiscoveryJob
| KwpArchiveFileSyncJob;
/**
* Plugin invocation payload — Phase 06 #21 foundation. Carries the
* minimal identifying tuple a worker needs to load the right Plugin
@@ -227,6 +376,24 @@ export type WorkflowExecutionRequestedOutboxEvent = {
export type WorkflowExecutionContinueRequestedOutboxPayload = WorkflowExecutionRequestedOutboxPayload;
export type PlatformNotificationRequestedOutboxPayload = {
deliveryId: string;
userId?: string;
adminUserId?: string;
sourceKey: string;
channel: "email" | "in-app";
messageKey: string;
category?: string;
metadata?: Omit<PlatformNotificationSafeMetadata, "deliveryId">;
};
export type PlatformNotificationRequestedOutboxEvent = {
tenantId: null;
type: "notification.platform.requested.v1";
payload: PlatformNotificationRequestedOutboxPayload;
dispatchKey: `platform-notification:${string}`;
};
export type WorkflowExecutionContinueRequestedOutboxEvent = {
tenantId: string | null;
type: "workflow.execution.continue.requested";
@@ -243,18 +410,26 @@ export type QueueJobPayloads = {
[queueNames.filePreview]: FilePreviewJob;
[queueNames.exports]: ExportJob;
[queueNames.imports]: ImportJob;
[queueNames.kwpImports]: KwpImportJob;
[queueNames.pluginInvoke]: PluginInvokeJob;
};
export type QueueJobNames = {
[queueNames.email]: "send-transactional-email" | "deliver-webhook";
[queueNames.notifications]: "send-notification";
[queueNames.maintenance]: "run-cleanup" | "dsgvo-grace-period-scan";
[queueNames.maintenance]:
| "run-cleanup"
| "dsgvo-grace-period-scan"
| "privacy-tenant-deletion-scan"
| "portal-lifecycle-scan"
| "product-acquisition-reconcile"
| "telephony-webhook-maintenance";
[queueNames.browserAutomation]: "browser-session-run" | "browser-session-cleanup" | "usage-flush";
[queueNames.fileScan]: "scan-file";
[queueNames.filePreview]: "generate-file-preview";
[queueNames.exports]: "run-export";
[queueNames.imports]: "run-import";
[queueNames.kwpImports]: "run-kwp-import";
[queueNames.pluginInvoke]: "invoke-plugin";
};
@@ -12,8 +12,18 @@ import {
buildFileScanJob,
buildIdempotentJobId,
buildImportJob,
buildAdminSalesContactsImportJob,
buildAdminSalesContactsExportJob,
buildKwpArchiveDiscoveryJob,
buildKwpArchiveFileSyncJob,
buildKwpContractSentinelJob,
buildKwpProjectCatalogSyncJob,
buildKwpProjectSyncJob,
buildNotificationJob,
buildPlatformNotificationJob,
buildPlatformNotificationRequestedOutboxEvent,
buildPluginInvokeJob,
buildPortalDocumentReadyOutboxEvent,
buildRetryJobOptions,
buildTransactionalMailJob,
buildWebhookDeliveryJob,
@@ -132,6 +142,178 @@ describe("queue producers", () => {
).toThrow("Notification jobs require tenantId.");
});
test("builds a strictly tenantless platform notification descriptor", () => {
expect(
buildPlatformNotificationJob({
userId: " user_haris ",
sourceKey: " platform-lead:lead_1:email ",
channel: "email",
messageKey: "platform.leads.captured",
category: "Leads",
email: "haris@example.test",
subject: "Neuer Lead in HiveGrid",
text: "Ein neuer Lead ist eingegangen.",
metadata: {
deliveryId: "delivery_1",
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
},
})
).toEqual({
queueName: queueNames.notifications,
jobName: "send-notification",
payload: {
kind: "notification.platform.send",
tenantId: null,
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
email: "haris@example.test",
subject: "Neuer Lead in HiveGrid",
text: "Ein neuer Lead ist eingegangen.",
metadata: {
deliveryId: "delivery_1",
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
},
},
});
});
test("preserves a safe task subject for platform reminder jobs", () => {
expect(
buildPlatformNotificationJob({
adminUserId: "admin_1",
sourceKey: "admin-sales:task-reminder:task_1:2026-07-25",
channel: "in-app",
messageKey: "admin.sales.notifications.taskreminder",
metadata: { taskId: "task_1", deeplink: "/sales/tasks/today" },
}).payload
).toMatchObject({
adminUserId: "admin_1",
metadata: {
taskId: "task_1",
deeplink: "/sales/tasks/today",
},
});
});
test("requires exactly one tenant or admin recipient identity", () => {
expect(() =>
buildPlatformNotificationJob({
sourceKey: "admin-email-open:event_1",
channel: "in-app",
messageKey: "admin.sales.email.notifications.opened",
})
).toThrow("exactly one recipient identity");
expect(() =>
buildPlatformNotificationRequestedOutboxEvent({
deliveryId: "delivery_1",
userId: "user_1",
adminUserId: "admin_1",
sourceKey: "admin-email-open:event_1",
channel: "in-app",
messageKey: "admin.sales.email.notifications.opened",
})
).toThrow("exactly one recipient identity");
});
test("rejects invalid platform scope and strips non-allowlisted queue metadata", () => {
expect(() =>
buildPlatformNotificationJob({
userId: " ",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
})
).toThrow("userId to be a stable non-secret identifier");
expect(() =>
buildPlatformNotificationJob({
userId: "user_haris",
sourceKey: "Bearer secret/value",
channel: "email",
messageKey: "platform.leads.captured",
})
).toThrow("sourceKey to be a stable non-secret identifier");
const descriptor = buildPlatformNotificationJob({
tenantId: "tenant_escape",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:in-app",
channel: "in-app",
messageKey: "platform.leads.captured",
metadata: {
leadId: "lead_1",
email: "lead@example.test",
phone: "+491701234567",
message: "private message",
token: "secret-token",
},
} as never);
expect(descriptor.payload.tenantId).toBeNull();
expect(descriptor.payload.metadata).toEqual({ leadId: "lead_1" });
expect(JSON.stringify(descriptor)).not.toContain("lead@example.test");
expect(JSON.stringify(descriptor)).not.toContain("private message");
expect(JSON.stringify(descriptor)).not.toContain("secret-token");
});
test("builds a PII-minimized platform notification outbox event", () => {
const event = buildPlatformNotificationRequestedOutboxEvent({
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
metadata: {
deliveryId: "must_not_be_nested",
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
email: "lead@example.test",
},
} as never);
expect(event).toEqual({
tenantId: null,
type: "notification.platform.requested.v1",
payload: {
deliveryId: "delivery_1",
userId: "user_haris",
sourceKey: "platform-lead:lead_1:email",
channel: "email",
messageKey: "platform.leads.captured",
category: "leads",
metadata: {
leadId: "lead_1",
sourceId: "hivegrid-website",
deeplink: "/leads/lead_1",
},
},
dispatchKey: "platform-notification:delivery_1",
});
expect(JSON.stringify(event)).not.toContain("lead@example.test");
});
test("uses the recipient-specific delivery id for platform outbox dispatch dedupe", () => {
const common = {
sourceKey: "platform-lead:lead_1:email",
channel: "email" as const,
messageKey: "platform.leads.captured",
};
const first = buildPlatformNotificationRequestedOutboxEvent({ ...common, deliveryId: "delivery_user_1", userId: "user_1" });
const second = buildPlatformNotificationRequestedOutboxEvent({ ...common, deliveryId: "delivery_user_2", userId: "user_2" });
expect(first.dispatchKey).toBe("platform-notification:delivery_user_1");
expect(second.dispatchKey).toBe("platform-notification:delivery_user_2");
});
test("builds maintenance cleanup descriptors", () => {
expect(
buildCleanupJob({
@@ -208,6 +390,33 @@ describe("queue producers", () => {
});
});
test("builds telephony webhook maintenance descriptors on the maintenance queue", () => {
const descriptor = buildCleanupJob({ task: "telephony-webhook-maintenance" });
expect(descriptor).toEqual({
queueName: queueNames.maintenance,
jobName: "run-cleanup",
payload: {
kind: "cleanup.run",
task: "telephony-webhook-maintenance",
},
});
});
test("builds product acquisition reconciliation descriptors on the maintenance queue", () => {
const descriptor = buildCleanupJob({ task: "product-acquisition-reconcile", take: 50 });
expect(descriptor).toEqual({
queueName: queueNames.maintenance,
jobName: "run-cleanup",
payload: {
kind: "cleanup.run",
task: "product-acquisition-reconcile",
take: 50,
},
});
});
test("builds expired export artifact retention descriptors on the maintenance queue", () => {
const descriptor = buildCleanupJob({
task: "expired-export-artifacts",
@@ -453,6 +662,39 @@ describe("queue producers", () => {
});
});
test("builds tenant-less admin sales data jobs on the imports queue", () => {
expect(
buildAdminSalesContactsImportJob({
importJobId: "admin_import_1",
requestedByAdminUserId: "admin_1",
requestId: "request_1",
})
).toEqual({
queueName: queueNames.imports,
jobName: "run-import",
payload: {
kind: "admin-sales.contacts-import",
importJobId: "admin_import_1",
requestedByAdminUserId: "admin_1",
requestId: "request_1",
},
});
expect(
buildAdminSalesContactsExportJob({
exportJobId: "admin_export_1",
requestedByAdminUserId: "admin_1",
})
).toEqual({
queueName: queueNames.imports,
jobName: "run-import",
payload: {
kind: "admin-sales.contacts-export",
exportJobId: "admin_export_1",
requestedByAdminUserId: "admin_1",
},
});
});
test("keeps import descriptors off export and email queues", () => {
const descriptor = buildImportJob({
importJobId: "import_job_queue_guard_1",
@@ -472,6 +714,114 @@ describe("queue producers", () => {
expect(descriptor.payload.requestId).toBe("req_import_queue_guard_1");
});
test("rejects every unstable KWP project-sync identifier", () => {
const valid = {
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T11:30Z",
};
expect(buildKwpProjectSyncJob(valid)).toMatchObject({
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.project-sync", ...valid },
});
for (const field of ["tenantId", "bindingId", "dataConnectionId", "cadenceBucket"] as const) {
expect(() => buildKwpProjectSyncJob({ ...valid, [field]: " " }))
.toThrow("KWP project sync jobs require stable tenant, binding, connection and cadence identifiers.");
}
});
test("rejects every unstable KWP contract-sentinel identifier", () => {
const valid = {
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23",
};
expect(buildKwpContractSentinelJob(valid)).toMatchObject({
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.contract-sentinel", ...valid },
});
for (const field of ["tenantId", "dataConnectionId", "cadenceBucket"] as const) {
expect(() => buildKwpContractSentinelJob({ ...valid, [field]: " " }))
.toThrow("KWP contract sentinel jobs require stable tenant, connection and cadence identifiers.");
}
});
test("builds a stable KWP project-catalog descriptor and rejects unstable identifiers", () => {
const valid = {
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T11:30Z",
};
expect(buildKwpProjectCatalogSyncJob(valid)).toMatchObject({
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.project-catalog-sync", ...valid },
});
for (const field of ["tenantId", "dataConnectionId", "cadenceBucket"] as const) {
expect(() => buildKwpProjectCatalogSyncJob({ ...valid, [field]: " " }))
.toThrow("KWP project catalog jobs require stable tenant, connection and cadence identifiers.");
}
});
test("builds a stable KWP archive-file descriptor and rejects an unstable revision", () => {
const input = {
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T11:30Z",
archiveFile: {
externalId: "1:2:3",
headerId: 1,
revisionId: 3,
revision: 1,
folderId: 0,
displayName: "plan.pdf",
documentType: null,
providerFileType: "PDF",
providerSizeBytes: 123,
lastModifiedAt: null,
},
};
expect(buildKwpArchiveFileSyncJob(input)).toEqual({
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.archive-file-sync", ...input },
});
expect(() => buildKwpArchiveFileSyncJob({
...input,
archiveFile: { ...input.archiveFile, revisionId: 1.5 },
})).toThrow("KWP archive file jobs require stable tenant, binding, connection, cadence and revision identifiers.");
});
test("builds bounded resumable KWP archive-discovery descriptors", () => {
const input = {
tenantId: "tenant_1",
bindingId: "binding_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T11:30Z",
batchNumber: 2,
cursor: { pendingFolderIds: [-5, 6], visitedFolderIds: [0, -2, -1, 3, 4] },
};
expect(buildKwpArchiveDiscoveryJob(input)).toEqual({
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.archive-discovery", ...input },
});
expect(() => buildKwpArchiveDiscoveryJob({
...input,
cursor: { pendingFolderIds: [], visitedFolderIds: [0] },
})).toThrow("KWP archive discovery jobs require a stable project binding and a bounded folder cursor.");
expect(() => buildKwpArchiveDiscoveryJob({
...input,
cursor: { pendingFolderIds: [2_147_483_648], visitedFolderIds: [0] },
})).toThrow("KWP archive discovery jobs require a stable project binding and a bounded folder cursor.");
});
test("builds plugin invoke descriptors for the isolated plugin queue", () => {
expect(
buildPluginInvokeJob({
@@ -518,6 +868,28 @@ describe("queue producers", () => {
});
});
test("builds DB-authoritative portal document-ready events without a caller-supplied document type", () => {
expect(
buildPortalDocumentReadyOutboxEvent({
tenantId: "tenant_1",
projectId: "project_1",
fileId: "file_1",
fileEntityLinkId: "link_1",
linkVersion: new Date("2026-07-16T10:00:00.000Z"),
userId: "user_1",
})
).toEqual({
tenantId: "tenant_1",
type: "workflow.trigger.event",
payload: {
event: "portal.document_ready",
data: { projectId: "project_1", fileId: "file_1", fileEntityLinkId: "link_1" },
userId: "user_1",
},
dispatchKey: "workflow-trigger:portal-document-ready:link_1:1784196000000",
});
});
test("builds workflow execution request outbox events for queued manual runs", () => {
expect(
buildWorkflowExecutionRequestedOutboxEvent({
@@ -602,15 +974,9 @@ describe("queue producers", () => {
sourceId: "event_1",
};
expect(() => buildWorkflowTriggerOutboxEvent({ ...input, event: " " })).toThrow(
"workflow.trigger.event outbox events require event."
);
expect(() => buildWorkflowTriggerOutboxEvent({ ...input, sourceId: "" })).toThrow(
"workflow.trigger.event outbox events require sourceId."
);
expect(() => buildWorkflowTriggerOutboxEvent({ ...input, sourceId: "event 1" })).toThrow(
"workflow.trigger.event outbox events require sourceId to match"
);
expect(() => buildWorkflowTriggerOutboxEvent({ ...input, event: " " })).toThrow("workflow.trigger.event outbox events require event.");
expect(() => buildWorkflowTriggerOutboxEvent({ ...input, sourceId: "" })).toThrow("workflow.trigger.event outbox events require sourceId.");
expect(() => buildWorkflowTriggerOutboxEvent({ ...input, sourceId: "event 1" })).toThrow("workflow.trigger.event outbox events require sourceId to match");
});
test("rejects malformed workflow execution request outbox event keys before publishing", () => {
@@ -824,7 +1190,7 @@ describe("queue producers", () => {
await enqueueJob(queue, descriptor, { idempotencyKey: "safe-key", jobId: "explicit-job-id" });
expect(add).toHaveBeenCalledWith("deliver-webhook", descriptor.payload, {
...buildRetryJobOptions(descriptor),
removeOnComplete: false,
removeOnComplete: 10_000,
jobId: "explicit-job-id",
});
const enqueuedPayload = add.mock.calls[0]?.[1];
@@ -846,7 +1212,21 @@ describe("queue producers", () => {
expect(buildIdempotentJobId(descriptor)).toBe("metadata-key");
});
test("keeps completed idempotent jobs unless callers override completion retention", async () => {
test("encodes BullMQ-reserved custom job id characters without collisions", () => {
const descriptor = buildNotificationJob({
tenantId: "tenant_1",
userId: "user_1",
channel: "in-app",
messageKey: "message.ready",
metadata: { idempotencyKey: "notification:delivery%3A1" },
});
expect(buildIdempotentJobId(descriptor)).toBe("notification%3Adelivery%253A1");
expect(buildIdempotentJobId(descriptor, { idempotencyKey: "123" })).toBe("job-123");
expect(buildIdempotentJobId(descriptor, { idempotencyKey: "notification%3Adelivery%253A1" })).not.toBe(buildIdempotentJobId(descriptor));
});
test("bounds completed idempotent jobs unless callers override completion retention", async () => {
const add = vi.fn(async () => ({ id: "job_idempotent" }));
const queue = { add } as unknown as TypedQueue<typeof queueNames.notifications>;
const descriptor = buildNotificationJob({
@@ -860,7 +1240,7 @@ describe("queue producers", () => {
await enqueueJob(queue, descriptor);
expect(add).toHaveBeenLastCalledWith("send-notification", descriptor.payload, {
...buildRetryJobOptions(descriptor),
removeOnComplete: false,
removeOnComplete: 10_000,
jobId: "metadata-key",
});
@@ -915,7 +1295,7 @@ describe("queue producers", () => {
await enqueueJob(queue, descriptor);
expect(add).toHaveBeenCalledWith("run-export", descriptor.payload, {
...buildRetryJobOptions(descriptor),
removeOnComplete: false,
removeOnComplete: 10_000,
jobId: "export_job_enqueue_1",
});
});
@@ -960,7 +1340,7 @@ describe("queue producers", () => {
await enqueueJob(queue, descriptor);
expect(add).toHaveBeenCalledWith("run-import", descriptor.payload, {
...buildRetryJobOptions(descriptor),
removeOnComplete: false,
removeOnComplete: 10_000,
jobId: "import_job_enqueue_1",
});
});
@@ -1,17 +1,31 @@
import type { JobsOptions, TypedQueue } from "./types.js";
import { toBullMqJobId } from "./job-id.js";
import { queueNames, type QueueName } from "./names.js";
import type {
AdminSalesContactsExportJob,
AdminSalesContactsImportJob,
BrowserAutomationJob,
CleanupJob,
ExportJob,
FileImportJob,
FilePreviewJob,
FileScanJob,
ImportJob,
KwpImportJob,
KwpArchiveDiscoveryJob,
KwpArchiveFileSyncJob,
KwpContractSentinelJob,
KwpProjectCatalogSyncJob,
KwpProjectSyncJob,
NotificationJob,
PlatformNotificationJob,
PlatformNotificationRequestedOutboxEvent,
PlatformNotificationSafeMetadata,
PluginInvokeJob,
QueueJobDescriptor,
QueueJsonValue,
TransactionalMailJob,
TenantNotificationJob,
WebhookDeliveryJob,
WorkflowExecutionContinueRequestedOutboxEvent,
WorkflowExecutionRequestedOutboxEvent,
@@ -19,11 +33,15 @@ import type {
} from "./payloads.js";
import { JOB_PRIORITY_CRITICAL, priorityFromOverride, priorityLabel, resolveJobPriorityForTenant, type PriorityOverride } from "./priority.js";
/** Keep idempotency effective without allowing completed BullMQ jobs to grow forever. */
export const IDEMPOTENT_JOB_RETENTION_COUNT = 10_000;
type BrowserAutomationJobForKind<Kind extends BrowserAutomationJob["kind"]> = BrowserAutomationJob & { kind: Kind };
export type TransactionalMailJobInput = Omit<TransactionalMailJob, "kind">;
export type WebhookDeliveryJobInput = Omit<WebhookDeliveryJob, "kind">;
export type NotificationJobInput = Omit<NotificationJob, "kind">;
export type NotificationJobInput = Omit<TenantNotificationJob, "kind">;
export type PlatformNotificationJobInput = Omit<PlatformNotificationJob, "kind" | "tenantId">;
export type CleanupJobInput = Omit<CleanupJob, "kind">;
export type BrowserSessionRunJobInput = Omit<BrowserAutomationJobForKind<"browser.session-run">, "kind">;
export type BrowserSessionCleanupJobInput = Omit<BrowserAutomationJobForKind<"browser.session-cleanup">, "kind">;
@@ -31,7 +49,20 @@ export type BrowserUsageFlushJobInput = Omit<BrowserAutomationJobForKind<"usage.
export type FileScanJobInput = Omit<FileScanJob, "kind">;
export type FilePreviewJobInput = Omit<FilePreviewJob, "kind">;
export type ExportJobInput = Omit<ExportJob, "kind">;
export type ImportJobInput = Omit<ImportJob, "kind">;
export type ImportJobInput = Omit<FileImportJob, "kind">;
export type AdminSalesContactsImportJobInput = Omit<
AdminSalesContactsImportJob,
"kind"
>;
export type AdminSalesContactsExportJobInput = Omit<
AdminSalesContactsExportJob,
"kind"
>;
export type KwpProjectSyncJobInput = Omit<KwpProjectSyncJob, "kind">;
export type KwpProjectCatalogSyncJobInput = Omit<KwpProjectCatalogSyncJob, "kind">;
export type KwpContractSentinelJobInput = Omit<KwpContractSentinelJob, "kind">;
export type KwpArchiveDiscoveryJobInput = Omit<KwpArchiveDiscoveryJob, "kind">;
export type KwpArchiveFileSyncJobInput = Omit<KwpArchiveFileSyncJob, "kind">;
export type PluginInvokeJobInput = Omit<PluginInvokeJob, "kind">;
export type WorkflowTriggerOutboxEventInput = {
tenantId: string | null;
@@ -41,6 +72,17 @@ export type WorkflowTriggerOutboxEventInput = {
sourceId: string;
};
export const PORTAL_DOCUMENT_READY_EVENT = "portal.document_ready" as const;
export type PortalDocumentReadyOutboxEventInput = {
tenantId: string;
projectId: string;
fileId: string;
fileEntityLinkId: string;
linkVersion: Date;
userId: string | null;
};
export type WorkflowExecutionRequestedOutboxEventInput = {
tenantId: string | null;
workflowId: string;
@@ -53,7 +95,14 @@ export type WorkflowExecutionRequestedOutboxEventInput = {
export type WorkflowExecutionContinueRequestedOutboxEventInput = WorkflowExecutionRequestedOutboxEventInput;
export type PlatformNotificationRequestedOutboxEventInput = Omit<PlatformNotificationRequestedOutboxEvent["payload"], "metadata"> & {
metadata?: PlatformNotificationSafeMetadata;
};
const WORKFLOW_OUTBOX_SOURCE_ID_PATTERN = /^[A-Za-z0-9._:-]{1,234}$/u;
const PLATFORM_NOTIFICATION_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/u;
const PLATFORM_NOTIFICATION_MESSAGE_KEY_PATTERN = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/u;
const PLATFORM_NOTIFICATION_DEEPLINK_PATTERN = /^\/[A-Za-z0-9/_-]{1,239}$/u;
function requireWorkflowOutboxText(value: string, fieldName: string, eventType: string): string {
const trimmed = value.trim();
@@ -71,6 +120,50 @@ function requireWorkflowOutboxSourceId(value: string, eventType: string): string
return sourceId;
}
function requirePlatformNotificationIdentifier(value: string, field: string): string {
const normalized = value?.trim();
if (!normalized || !PLATFORM_NOTIFICATION_IDENTIFIER_PATTERN.test(normalized)) {
throw new Error(`Platform notifications require ${field} to be a stable non-secret identifier.`);
}
return normalized;
}
function requirePlatformNotificationMessageKey(value: string): string {
const normalized = value?.trim();
if (!normalized || !PLATFORM_NOTIFICATION_MESSAGE_KEY_PATTERN.test(normalized)) {
throw new Error("Platform notifications require a valid messageKey.");
}
return normalized;
}
function normalizePlatformNotificationChannel(value: string): "email" | "in-app" {
if (value === "email" || value === "in-app") {
return value;
}
throw new Error(`Unsupported platform notification channel: ${value}`);
}
function sanitizePlatformNotificationMetadata(metadata?: PlatformNotificationSafeMetadata): PlatformNotificationSafeMetadata | undefined {
if (!metadata) return undefined;
const sanitized: PlatformNotificationSafeMetadata = {};
for (const key of ["deliveryId", "eventId", "leadId", "taskId", "requestId", "sourceId"] as const) {
const value = metadata[key];
if (typeof value === "string" && PLATFORM_NOTIFICATION_IDENTIFIER_PATTERN.test(value.trim())) {
sanitized[key] = value.trim();
}
}
if (typeof metadata.eventType === "string" && /^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$/u.test(metadata.eventType.trim())) {
sanitized.eventType = metadata.eventType.trim();
}
if (typeof metadata.deeplink === "string" && PLATFORM_NOTIFICATION_DEEPLINK_PATTERN.test(metadata.deeplink.trim())) {
sanitized.deeplink = metadata.deeplink.trim();
}
if (typeof metadata.replay === "boolean") {
sanitized.replay = metadata.replay;
}
return Object.keys(sanitized).length > 0 ? sanitized : undefined;
}
export type TransactionalMailJobDescriptor = QueueJobDescriptor<typeof queueNames.email> & {
jobName: "send-transactional-email";
payload: TransactionalMailJob;
@@ -126,6 +219,11 @@ export type ImportJobDescriptor = QueueJobDescriptor<typeof queueNames.imports>
payload: ImportJob;
};
export type KwpImportJobDescriptor = QueueJobDescriptor<typeof queueNames.kwpImports> & {
jobName: "run-kwp-import";
payload: KwpImportJob;
};
export type PluginInvokeJobDescriptor = QueueJobDescriptor<typeof queueNames.pluginInvoke> & {
jobName: "invoke-plugin";
payload: PluginInvokeJob;
@@ -143,6 +241,7 @@ export type ProducerJobDescriptor =
| FilePreviewJobDescriptor
| ExportJobDescriptor
| ImportJobDescriptor
| KwpImportJobDescriptor
| PluginInvokeJobDescriptor;
const retryJobOptionsByQueueAndJobName: {
@@ -162,6 +261,15 @@ const retryJobOptionsByQueueAndJobName: {
// one retry on transient Prisma/Redis hiccups before letting the
// next 24h tick recover the run.
"dsgvo-grace-period-scan": { attempts: 2, backoff: { type: "fixed", delay: 30_000 } },
// AK-P21 tenant hard-delete scan: idempotent per tick (only overdue
// pending requests match); one retry for transient infra hiccups.
"privacy-tenant-deletion-scan": { attempts: 2, backoff: { type: "fixed", delay: 30_000 } },
// VA-P26b portal lifecycle scan: every mutation is claimed/idempotent and
// the daily scheduler provides the long-stop retry. One near-term retry
// covers transient database or object-storage failures.
"portal-lifecycle-scan": { attempts: 2, backoff: { type: "fixed", delay: 30_000 } },
"product-acquisition-reconcile": { attempts: 2, backoff: { type: "fixed", delay: 5_000 } },
"telephony-webhook-maintenance": { attempts: 2, backoff: { type: "fixed", delay: 5_000 } },
},
[queueNames.browserAutomation]: {
"browser-session-run": { attempts: 2, backoff: { type: "exponential", delay: 5_000 } },
@@ -183,6 +291,9 @@ const retryJobOptionsByQueueAndJobName: {
[queueNames.imports]: {
"run-import": { attempts: 2, backoff: { type: "exponential", delay: 10_000 } },
},
[queueNames.kwpImports]: {
"run-kwp-import": { attempts: 5, backoff: { type: "exponential", delay: 30_000 } },
},
[queueNames.pluginInvoke]: {
"invoke-plugin": { attempts: 3, backoff: { type: "exponential", delay: 2_000 } },
},
@@ -241,6 +352,45 @@ export function buildNotificationJob(input: NotificationJobInput): NotificationJ
};
}
export function buildPlatformNotificationJob(input: PlatformNotificationJobInput): NotificationJobDescriptor {
const userId = input.userId
? requirePlatformNotificationIdentifier(input.userId, "userId")
: undefined;
const adminUserId = input.adminUserId
? requirePlatformNotificationIdentifier(input.adminUserId, "adminUserId")
: undefined;
if (Boolean(userId) === Boolean(adminUserId)) {
throw new Error(
"Platform notification jobs require exactly one recipient identity."
);
}
const sourceKey = requirePlatformNotificationIdentifier(input.sourceKey, "sourceKey");
const channel = normalizePlatformNotificationChannel(input.channel);
const messageKey = requirePlatformNotificationMessageKey(input.messageKey);
const metadata = sanitizePlatformNotificationMetadata(input.metadata);
return {
queueName: queueNames.notifications,
jobName: "send-notification",
payload: {
kind: "notification.platform.send",
tenantId: null,
...(userId ? { userId } : { adminUserId }),
sourceKey,
channel,
messageKey,
...(input.category?.trim() ? { category: input.category.trim().toLowerCase() } : {}),
...(input.email?.trim() ? { email: input.email.trim() } : {}),
...(input.subject ? { subject: input.subject } : {}),
...(input.text ? { text: input.text } : {}),
...(input.html ? { html: input.html } : {}),
...(input.templateId ? { templateId: input.templateId } : {}),
...(input.templateData ? { templateData: input.templateData } : {}),
...(metadata ? { metadata } : {}),
},
};
}
export function buildCleanupJob(input: CleanupJobInput): CleanupJobDescriptor {
return {
queueName: queueNames.maintenance,
@@ -335,6 +485,124 @@ export function buildImportJob(input: ImportJobInput): ImportJobDescriptor {
};
}
export function buildAdminSalesContactsImportJob(
input: AdminSalesContactsImportJobInput
): ImportJobDescriptor {
requireAdminSalesDataJobInput(
input.importJobId,
input.requestedByAdminUserId,
"import"
);
return {
queueName: queueNames.imports,
jobName: "run-import",
payload: { kind: "admin-sales.contacts-import", ...input },
};
}
export function buildAdminSalesContactsExportJob(
input: AdminSalesContactsExportJobInput
): ImportJobDescriptor {
requireAdminSalesDataJobInput(
input.exportJobId,
input.requestedByAdminUserId,
"export"
);
return {
queueName: queueNames.imports,
jobName: "run-import",
payload: { kind: "admin-sales.contacts-export", ...input },
};
}
function requireAdminSalesDataJobInput(
jobId: string,
requestedByAdminUserId: string,
operation: "import" | "export"
): void {
if (!jobId?.trim() || !requestedByAdminUserId?.trim()) {
throw new Error(
`Admin sales contacts ${operation} jobs require stable job and operator identifiers.`
);
}
}
export function buildKwpProjectSyncJob(input: KwpProjectSyncJobInput): KwpImportJobDescriptor {
if (!input.tenantId.trim() || !input.bindingId.trim() || !input.dataConnectionId.trim() || !input.cadenceBucket.trim()) {
throw new Error("KWP project sync jobs require stable tenant, binding, connection and cadence identifiers.");
}
return {
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.project-sync", ...input },
};
}
export function buildKwpProjectCatalogSyncJob(input: KwpProjectCatalogSyncJobInput): KwpImportJobDescriptor {
if (!input.tenantId.trim() || !input.dataConnectionId.trim() || !input.cadenceBucket.trim()) {
throw new Error("KWP project catalog jobs require stable tenant, connection and cadence identifiers.");
}
return {
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.project-catalog-sync", ...input },
};
}
export function buildKwpContractSentinelJob(input: KwpContractSentinelJobInput): KwpImportJobDescriptor {
if (!input.tenantId.trim() || !input.dataConnectionId.trim() || !input.cadenceBucket.trim()) {
throw new Error("KWP contract sentinel jobs require stable tenant, connection and cadence identifiers.");
}
return {
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.contract-sentinel", ...input },
};
}
export function buildKwpArchiveDiscoveryJob(input: KwpArchiveDiscoveryJobInput): KwpImportJobDescriptor {
const folderIds = input.cursor
? [...input.cursor.pendingFolderIds, ...input.cursor.visitedFolderIds]
: [];
if (
!input.tenantId.trim()
|| !input.bindingId.trim()
|| !input.dataConnectionId.trim()
|| !input.cadenceBucket.trim()
|| !Number.isInteger(input.batchNumber)
|| input.batchNumber < 0
|| folderIds.length > 1_000
|| folderIds.some((folderId) => !Number.isInteger(folderId) || folderId < -2_147_483_648 || folderId > 2_147_483_647)
|| (input.cursor !== undefined && input.cursor.pendingFolderIds.length === 0)
) {
throw new Error("KWP archive discovery jobs require a stable project binding and a bounded folder cursor.");
}
return {
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.archive-discovery", ...input },
};
}
export function buildKwpArchiveFileSyncJob(input: KwpArchiveFileSyncJobInput): KwpImportJobDescriptor {
if (
!input.tenantId.trim()
|| !input.bindingId.trim()
|| !input.dataConnectionId.trim()
|| !input.cadenceBucket.trim()
|| !input.archiveFile.externalId.trim()
|| !input.archiveFile.displayName.trim()
|| !Number.isInteger(input.archiveFile.revisionId)
) {
throw new Error("KWP archive file jobs require stable tenant, binding, connection, cadence and revision identifiers.");
}
return {
queueName: queueNames.kwpImports,
jobName: "run-kwp-import",
payload: { kind: "kwp.archive-file-sync", ...input },
};
}
export function buildPluginInvokeJob(input: PluginInvokeJobInput): PluginInvokeJobDescriptor {
return {
queueName: queueNames.pluginInvoke,
@@ -362,6 +630,25 @@ export function buildWorkflowTriggerOutboxEvent(input: WorkflowTriggerOutboxEven
};
}
/**
* Internal, DB-backed producer contract for customer-portal auto release.
* It deliberately carries no document type; the consumer must resolve the
* authoritative FileEntityLink.linkRole inside its tenant transaction.
*/
export function buildPortalDocumentReadyOutboxEvent(input: PortalDocumentReadyOutboxEventInput): WorkflowTriggerOutboxEvent {
return buildWorkflowTriggerOutboxEvent({
tenantId: input.tenantId,
event: PORTAL_DOCUMENT_READY_EVENT,
data: {
projectId: input.projectId,
fileId: input.fileId,
fileEntityLinkId: input.fileEntityLinkId,
},
userId: input.userId,
sourceId: `portal-document-ready:${input.fileEntityLinkId}:${input.linkVersion.getTime()}`,
});
}
export function buildWorkflowExecutionRequestedOutboxEvent(input: WorkflowExecutionRequestedOutboxEventInput): WorkflowExecutionRequestedOutboxEvent {
const eventType = "workflow.execution.requested";
const workflowId = requireWorkflowOutboxText(input.workflowId, "workflowId", eventType);
@@ -402,6 +689,44 @@ export function buildWorkflowExecutionContinueRequestedOutboxEvent(
};
}
export function buildPlatformNotificationRequestedOutboxEvent(input: PlatformNotificationRequestedOutboxEventInput): PlatformNotificationRequestedOutboxEvent {
const deliveryId = requirePlatformNotificationIdentifier(input.deliveryId, "deliveryId");
const userId = input.userId
? requirePlatformNotificationIdentifier(input.userId, "userId")
: undefined;
const adminUserId = input.adminUserId
? requirePlatformNotificationIdentifier(input.adminUserId, "adminUserId")
: undefined;
if (Boolean(userId) === Boolean(adminUserId)) {
throw new Error(
"Platform notification outbox events require exactly one recipient identity."
);
}
const sourceKey = requirePlatformNotificationIdentifier(input.sourceKey, "sourceKey");
const channel = normalizePlatformNotificationChannel(input.channel);
const messageKey = requirePlatformNotificationMessageKey(input.messageKey);
const metadata = sanitizePlatformNotificationMetadata(input.metadata);
const outboxMetadata = { ...(metadata ?? {}) };
delete outboxMetadata.deliveryId;
return {
tenantId: null,
type: "notification.platform.requested.v1",
payload: {
deliveryId,
...(userId ? { userId } : { adminUserId }),
sourceKey,
channel,
messageKey,
...(input.category?.trim() ? { category: input.category.trim().toLowerCase() } : {}),
...(Object.keys(outboxMetadata).length > 0 ? { metadata: outboxMetadata } : {}),
},
// The delivery id is stable across replay and distinct per recipient, while
// sourceKey may intentionally be shared by multiple configured recipients.
dispatchKey: `platform-notification:${deliveryId}`,
};
}
export type QueueEnqueueOptions = JobsOptions & {
idempotencyKey?: string;
/**
@@ -439,24 +764,24 @@ export function getEnqueueObserver(): EnqueueObserver | null {
export function buildIdempotentJobId(descriptor: ProducerJobDescriptor, options?: QueueEnqueueOptions): string | undefined {
if (typeof options?.jobId === "string") {
return options.jobId;
return toBullMqJobId(options.jobId);
}
if (typeof options?.idempotencyKey === "string") {
return options.idempotencyKey;
return toBullMqJobId(options.idempotencyKey);
}
if (descriptor.payload.kind === "export.run") {
return descriptor.payload.exportJobId;
return toBullMqJobId(descriptor.payload.exportJobId);
}
if (descriptor.payload.kind === "import.run") {
return descriptor.payload.importJobId;
return toBullMqJobId(descriptor.payload.importJobId);
}
const metadata = descriptor.payload.metadata;
const metadata = descriptor.payload.metadata as Record<string, unknown> | undefined;
if (metadata && typeof metadata.idempotencyKey === "string") {
return metadata.idempotencyKey;
return toBullMqJobId(metadata.idempotencyKey);
}
return undefined;
@@ -495,7 +820,7 @@ export async function enqueueJob<Name extends QueueName>(queue: TypedQueue<Name>
delete (callerOptions as QueueEnqueueOptions).priorityOverride;
const jobId = buildIdempotentJobId(producerDescriptor, options);
const idempotentCompletionRetention: Pick<JobsOptions, "removeOnComplete"> =
jobId && callerOptions.removeOnComplete === undefined ? { removeOnComplete: false } : {};
jobId && callerOptions.removeOnComplete === undefined ? { removeOnComplete: IDEMPOTENT_JOB_RETENTION_COUNT } : {};
const resolvedPriority = await resolvePriorityForEnqueue(options);
const priorityOptions: Pick<JobsOptions, "priority"> = resolvedPriority !== undefined ? { priority: resolvedPriority } : {};
const enqueueOptions: JobsOptions = {
@@ -0,0 +1,98 @@
import { describe, expect, it, vi } from "vitest";
import {
TELEPHONY_WEBHOOK_JOB_NAME,
TELEPHONY_WEBHOOK_QUEUE_NAME,
buildTelephonyWebhookJob,
enqueueTelephonyWebhookJob,
recoverFailedTelephonyWebhookJob,
type TelephonyWebhookQueue,
type TelephonyWebhookRecoveryQueue,
} from "./telephony.js";
function input() {
return {
tenantId: "tenant_1",
connectionId: "connection_1",
webhookEventId: "event_1",
};
}
describe("telephony webhook queue producer", () => {
it("builds a typed minimal descriptor without provider payload data", () => {
expect(buildTelephonyWebhookJob(input())).toEqual({
queueName: TELEPHONY_WEBHOOK_QUEUE_NAME,
jobName: TELEPHONY_WEBHOOK_JOB_NAME,
payload: {
kind: "telephony.webhook.process",
...input(),
},
});
});
it("uses the persisted event id as a stable BullMQ idempotency key", async () => {
const add = vi.fn<TelephonyWebhookQueue["add"]>(async () => ({ id: "job_1" }));
const queue = { add, close: vi.fn(async () => {}) } as TelephonyWebhookQueue;
const descriptor = buildTelephonyWebhookJob(input());
await enqueueTelephonyWebhookJob(queue, descriptor);
await enqueueTelephonyWebhookJob(queue, descriptor);
expect(add).toHaveBeenCalledTimes(2);
expect(add.mock.calls[0]?.[2]).toMatchObject({
attempts: 5,
backoff: { type: "exponential", delay: 2_000 },
jobId: "telephony-webhook-event_1",
removeOnComplete: 10_000,
});
expect(add.mock.calls[1]?.[2]?.jobId).toBe(add.mock.calls[0]?.[2]?.jobId);
});
it("manually retries a retained terminal failure with reset attempt counters", async () => {
const retry = vi.fn(async () => undefined);
const retainedJob = {
id: "telephony-webhook-event_1",
getState: vi.fn(async () => "failed"),
retry,
};
const add = vi.fn<TelephonyWebhookQueue["add"]>(async () => ({ id: "unexpected" }));
const queue = {
add,
getJob: vi.fn(async () => retainedJob),
close: vi.fn(async () => undefined),
} as TelephonyWebhookRecoveryQueue;
await expect(recoverFailedTelephonyWebhookJob(queue, buildTelephonyWebhookJob(input()))).resolves.toEqual({
id: "telephony-webhook-event_1",
});
expect(queue.getJob).toHaveBeenCalledWith("telephony-webhook-event_1");
expect(retry).toHaveBeenCalledWith("failed", {
resetAttemptsMade: true,
resetAttemptsStarted: true,
});
expect(add).not.toHaveBeenCalled();
});
it("adds a fresh stable job when no retained terminal failure exists", async () => {
const add = vi.fn<TelephonyWebhookQueue["add"]>(async () => ({ id: "telephony-webhook-event_1" }));
const queue = {
add,
getJob: vi.fn(async () => undefined),
close: vi.fn(async () => undefined),
} as TelephonyWebhookRecoveryQueue;
await recoverFailedTelephonyWebhookJob(queue, buildTelephonyWebhookJob(input()));
expect(add).toHaveBeenCalledWith(
TELEPHONY_WEBHOOK_JOB_NAME,
expect.objectContaining({ webhookEventId: "event_1" }),
expect.objectContaining({ jobId: "telephony-webhook-event_1" })
);
});
it("rejects empty or oversized identifiers before touching BullMQ", () => {
expect(() => buildTelephonyWebhookJob({ ...input(), tenantId: " " })).toThrow("valid tenantId");
expect(() => buildTelephonyWebhookJob({ ...input(), webhookEventId: "x".repeat(192) })).toThrow("valid webhookEventId");
});
});
@@ -0,0 +1,90 @@
import type { JobsOptions } from "./types.js";
/** Queue contract shared by the Aircall ingress and telephony webhook worker. */
export const TELEPHONY_WEBHOOK_QUEUE_NAME = "hivegrid.telephony-webhooks" as const;
export const TELEPHONY_WEBHOOK_JOB_NAME = "process-telephony-webhook" as const;
export type TelephonyWebhookProcessJob = {
readonly kind: "telephony.webhook.process";
readonly tenantId: string;
readonly connectionId: string;
readonly webhookEventId: string;
};
export type TelephonyWebhookJobDescriptor = {
readonly queueName: typeof TELEPHONY_WEBHOOK_QUEUE_NAME;
readonly jobName: typeof TELEPHONY_WEBHOOK_JOB_NAME;
readonly payload: TelephonyWebhookProcessJob;
};
export type TelephonyWebhookQueue = {
add(jobName: typeof TELEPHONY_WEBHOOK_JOB_NAME, payload: TelephonyWebhookProcessJob, options?: JobsOptions): Promise<{ id?: string }>;
close(): Promise<void>;
};
export type TelephonyWebhookRetainedJob = {
readonly id?: string;
getState(): Promise<string>;
retry(state?: "failed" | "completed", options?: { resetAttemptsMade?: boolean; resetAttemptsStarted?: boolean }): Promise<void>;
};
export type TelephonyWebhookRecoveryQueue = TelephonyWebhookQueue & {
getJob(jobId: string): Promise<TelephonyWebhookRetainedJob | undefined>;
};
export type TelephonyWebhookJobInput = Omit<TelephonyWebhookProcessJob, "kind">;
const MAX_IDENTIFIER_LENGTH = 191;
function requireIdentifier(value: string, name: keyof TelephonyWebhookJobInput): string {
const normalized = value.trim();
if (!normalized || normalized.length > MAX_IDENTIFIER_LENGTH || normalized.includes("\u0000")) {
throw new Error(`Telephony webhook jobs require a valid ${name}.`);
}
return normalized;
}
export function buildTelephonyWebhookJob(input: TelephonyWebhookJobInput): TelephonyWebhookJobDescriptor {
return {
queueName: TELEPHONY_WEBHOOK_QUEUE_NAME,
jobName: TELEPHONY_WEBHOOK_JOB_NAME,
payload: {
kind: "telephony.webhook.process",
tenantId: requireIdentifier(input.tenantId, "tenantId"),
connectionId: requireIdentifier(input.connectionId, "connectionId"),
webhookEventId: requireIdentifier(input.webhookEventId, "webhookEventId"),
},
};
}
/** Enqueue by persisted event id so webhook retries collapse into one job. */
export async function enqueueTelephonyWebhookJob(queue: TelephonyWebhookQueue, descriptor: TelephonyWebhookJobDescriptor): Promise<{ id?: string }> {
const jobId = `telephony-webhook-${descriptor.payload.webhookEventId}`;
return queue.add(descriptor.jobName, descriptor.payload, {
attempts: 5,
backoff: { type: "exponential", delay: 2_000 },
jobId,
removeOnComplete: 10_000,
removeOnFail: 1_000,
});
}
/**
* Requeue an event whose retained BullMQ job exhausted its attempts.
*
* A plain add with the stable event job id returns the retained failed job and
* never schedules work. Manual retry keeps that idempotency key, while the
* persisted webhook row remains the source of failure diagnostics.
*/
export async function recoverFailedTelephonyWebhookJob(
queue: TelephonyWebhookRecoveryQueue,
descriptor: TelephonyWebhookJobDescriptor
): Promise<{ id?: string }> {
const jobId = `telephony-webhook-${descriptor.payload.webhookEventId}`;
const retainedJob = await queue.getJob(jobId);
if (retainedJob && (await retainedJob.getState()) === "failed") {
await retainedJob.retry("failed", { resetAttemptsMade: true, resetAttemptsStarted: true });
return { id: retainedJob.id ?? jobId };
}
return enqueueTelephonyWebhookJob(queue, descriptor);
}
@@ -7,6 +7,7 @@ export type JobsOptions = {
delay?: number;
type: string;
};
delay?: number;
jobId?: string;
removeOnComplete?: boolean | number;
removeOnFail?: boolean | number;
@@ -1,5 +1,7 @@
import { describe, expect, test, vi } from "vitest";
import {
buildAdminSalesContactsExportJob,
buildAdminSalesContactsImportJob,
buildBrowserSessionCleanupJob,
buildBrowserSessionRunJob,
buildBrowserUsageFlushJob,
@@ -8,7 +10,13 @@ import {
buildFilePreviewJob,
buildFileScanJob,
buildImportJob,
buildKwpArchiveDiscoveryJob,
buildKwpArchiveFileSyncJob,
buildKwpContractSentinelJob,
buildKwpProjectCatalogSyncJob,
buildKwpProjectSyncJob,
buildNotificationJob,
buildPlatformNotificationJob,
buildPluginInvokeJob,
buildRetryJobOptions,
buildTransactionalMailJob,
@@ -27,6 +35,16 @@ type RetryMetrics = Pick<
>;
const descriptorBuilders = {
buildAdminSalesContactsExportJob: () =>
buildAdminSalesContactsExportJob({
exportJobId: "admin_sales_export_metrics_1",
requestedByAdminUserId: "admin_1",
}),
buildAdminSalesContactsImportJob: () =>
buildAdminSalesContactsImportJob({
importJobId: "admin_sales_import_metrics_1",
requestedByAdminUserId: "admin_1",
}),
buildBrowserSessionCleanupJob: () => buildBrowserSessionCleanupJob({ tenantId: "tenant_1", sessionId: "session_1" }),
buildBrowserSessionRunJob: () =>
buildBrowserSessionRunJob({ tenantId: "tenant_1", sessionId: "session_1", task: { action: "capture" } }),
@@ -53,7 +71,60 @@ const descriptorBuilders = {
sourceBucket: "uploads",
sourceKey: "tenant_1/imports/members.csv",
}),
buildKwpArchiveDiscoveryJob: () =>
buildKwpArchiveDiscoveryJob({
tenantId: "tenant_1",
dataConnectionId: "connection_1",
bindingId: "binding_1",
cadenceBucket: "2026-07-23T12:00",
batchNumber: 0,
}),
buildKwpArchiveFileSyncJob: () =>
buildKwpArchiveFileSyncJob({
tenantId: "tenant_1",
dataConnectionId: "connection_1",
bindingId: "binding_1",
cadenceBucket: "2026-07-23T13:00",
archiveFile: {
externalId: "1:2:3",
headerId: 1,
revisionId: 3,
revision: 1,
folderId: 0,
displayName: "plan.pdf",
documentType: "PLAN",
providerFileType: "PDF",
providerSizeBytes: 123,
lastModifiedAt: null,
},
}),
buildKwpContractSentinelJob: () =>
buildKwpContractSentinelJob({
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23",
}),
buildKwpProjectSyncJob: () =>
buildKwpProjectSyncJob({
tenantId: "tenant_1",
dataConnectionId: "connection_1",
bindingId: "binding_1",
cadenceBucket: "2026-07-23T13:00",
}),
buildKwpProjectCatalogSyncJob: () =>
buildKwpProjectCatalogSyncJob({
tenantId: "tenant_1",
dataConnectionId: "connection_1",
cadenceBucket: "2026-07-23T13:00",
}),
buildNotificationJob: () => buildNotificationJob({ tenantId: "tenant_1", userId: "user_1", channel: "in-app", messageKey: "message.ready" }),
buildPlatformNotificationJob: () =>
buildPlatformNotificationJob({
userId: "operator_1",
sourceKey: "platform-lead:lead_1",
channel: "in-app",
messageKey: "platform.leads.captured",
}),
buildPluginInvokeJob: () => buildPluginInvokeJob({ tenantId: "tenant_1", pluginId: "plugin_1", version: "1.0.0", hookName: "onEvent" }),
buildTransactionalMailJob: () => buildTransactionalMailJob({ to: ["user@example.com"], templateId: "welcome" }),
buildWebhookDeliveryJob: () => buildWebhookDeliveryJob({ url: "https://example.com/webhook", eventName: "entity.created" }),
@@ -184,6 +255,15 @@ describe("queue retry metrics snapshot", () => {
"removeOnComplete": true,
"removeOnFail": 1000,
},
"${queueNames.kwpImports}:run-kwp-import": {
"attempts": 5,
"backoff": {
"delay": 30000,
"type": "exponential",
},
"removeOnComplete": true,
"removeOnFail": 1000,
},
"${queueNames.maintenance}:run-cleanup": {
"attempts": 2,
"backoff": {