Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/worker/src/handlers.ts
T
2026-07-08 16:15:37 +02:00

2424 lines
97 KiB
TypeScript

import {
updateLegacyEventWebhookAttempt as updateLegacyEventWebhookAttemptInDb,
type LegacyEventWebhookAttemptUpdate,
} from "@hivegrid/db/event-webhook-attempts.server";
import {
listExpiredCompletedExportJobs as listExpiredCompletedExportJobsInDb,
listSoftDeletedExportJobsForHardDelete as listSoftDeletedExportJobsForHardDeleteInDb,
markExportJobExpired as markExportJobExpiredInDb,
markExportJobFailed as markExportJobFailedInDb,
markExportJobHardDeleteFailed as markExportJobHardDeleteFailedInDb,
markExportJobHardDeleteSucceeded as markExportJobHardDeleteSucceededInDb,
markExportJobPaused as markExportJobPausedInDb,
markExportJobResumed as markExportJobResumedInDb,
markExportJobRunning as markExportJobRunningInDb,
markExportJobSucceeded as markExportJobSucceededInDb,
type ExportJobExpiredInput,
type ExportJobFailedInput,
type ExportJobHardDeleteFailedInput,
type ExportJobHardDeleteSucceededInput,
type ExportJobPausedInput,
type ExportJobResumedInput,
type ExportJobRetentionListInput,
type ExportJobRetentionRecord,
type ExportJobRunningInput,
type ExportJobSucceededInput,
} from "@hivegrid/db/export-jobs.server";
import {
markGdprExportFailed as markGdprExportFailedInDb,
markGdprExportProcessing as markGdprExportProcessingInDb,
markGdprExportReady as markGdprExportReadyInDb,
type MarkGdprExportFailedInput,
type MarkGdprExportProcessingInput,
type MarkGdprExportReadyInput,
} from "@hivegrid/db/privacy-gdpr.server";
import {
markImportJobFailed as markImportJobFailedInDb,
markImportJobRunning as markImportJobRunningInDb,
markImportJobSucceeded as markImportJobSucceededInDb,
type ImportJobFailedInput,
type ImportJobRunningInput,
type ImportJobSucceededInput,
} from "@hivegrid/db/import-jobs.server";
import {
createNotificationDelivery as createNotificationDeliveryInDb,
getUserNotificationPreferences as getUserNotificationPreferencesInDb,
markNotificationDeliveryStatus as markNotificationDeliveryStatusInDb,
redactNotificationDiagnosticText,
type CreateNotificationDeliveryInput,
type MarkNotificationDeliveryStatusInput,
} from "@hivegrid/db/notifications.server";
import {
deleteDoneOutboxEvents as deleteDoneOutboxEventsInDb,
type DeleteDoneOutboxEventsOptions,
type DeleteDoneOutboxEventsResult,
} from "@hivegrid/db/outbox-retention";
import { type EnvRecord } from "@hivegrid/config";
import type { ValidationReport } from "@hivegrid/imports";
import { createMailjetProviderFromEnv, type MailMetricsHook, type MailProvider } from "@hivegrid/mail";
import {
queueNames,
type ExportJob,
type FilePreviewJob,
type ImportJob,
type PluginInvokeJob,
type QueueName,
type QueuePayload,
type TypedProcessor,
} from "@hivegrid/queue";
import {
authorizeStorageObjectAccess,
type DeleteObjectInput,
type DeleteObjectResult,
type StorageScanDiagnostic,
type StorageScanResult,
} from "@hivegrid/storage";
import { mapWithConcurrency, notify, type NotificationClient } from "@hivegrid/foundation";
import { OutboundHttpUrlValidationError, fetchOutboundHttpWithRedirects, type OutboundHttpAddressResolver } from "@hivegrid/foundation/outbound-http";
import {
drainCalendarWebhookEventsForTenant,
refreshCalendarTokensForTenant,
renewCalendarWebhooksForTenant,
syncInternalCalendarEventsForTenant,
} from "@hivegrid/integrations/calendar-worker-tasks.server";
import {
cleanupEmailWebhookEventsForTenant,
drainEmailWebhookEventsForTenant,
pollStaleEmailMailboxesForTenant,
refreshEmailTokensForTenant,
renewEmailWebhooksForTenant,
runEmailInitialSyncForTenant,
} from "@hivegrid/integrations/email-worker-tasks.server";
import { createEmailAutoMatchHook } from "@hivegrid/integrations/email-match-hook.server";
import { refreshExpiringTokensForTenant, sweepConnectionHealthForTenant } from "@hivegrid/integrations/worker-tasks.server";
import { withTenantRLS } from "./db/tenantRls.js";
import { runDsgvoGraceCron, type DsgvoGraceCandidate } from "./jobs/dsgvo-grace-cron.js";
import { reminderChannel, reminderMessageKey, reminderScanBounds, reminderSourceKey, selectReminderWindows } from "./jobs/task-reminder-cron.js";
import { audienceMatches, nextReminderDue, HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS } from "./jobs/hr-upload-reminder-cron.js";
import { createDnsDomainResolver, decideDomainProbeStatus, type DomainDnsResolver } from "./jobs/email-domain-probe-cron.js";
import { handlePluginInvokeJob, type PluginInvokeDeps } from "./jobs/plugin-invoke.js";
import {
reconcileWorkflowExecutionStalls,
type WorkflowExecutionStalledReconcileInput,
type WorkflowExecutionStalledReconcileResult,
type WorkflowExecutionStalledReconcilerPrisma,
} from "./jobs/workflow-execution-stalled-reconciler.js";
import {
isPrivacyGdprExportType,
type PrivacyGdprExportRunResult,
} from "./jobs/privacy-gdpr-export.js";
const DEFAULT_WORKER_FETCH_TIMEOUT_MS = 10_000;
const WEBHOOK_RESPONSE_BODY_LIMIT_BYTES = 4_096;
const WEBHOOK_REDIRECT_LIMIT = 5;
const EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY = 5;
export type HandlerResult = {
handled: true;
action: string;
details?: Record<string, unknown>;
};
export type WebhookDeliveryMetricsHook = (input: { status: "success" | "retry" | "dlq" | "rejected" }) => void;
export type HandlerDependencies = {
mailProvider?: MailProvider;
/**
* Optional metrics hook forwarded to the lazily-constructed Mailjet
* provider. When the worker entry wires this to the shared Prometheus
* registry, every send increments mail_sent_total{provider, status}.
*/
mailMetricsHook?: MailMetricsHook;
/**
* Optional metrics hook fired for each outbound webhook attempt. Status:
* - success : 2xx response
* - retry : retryable failure (5xx, transient network) below the
* attempts cap, BullMQ will requeue
* - dlq : terminal failure (final attempt or non-retryable status)
* - rejected : URL validation failure (SSRF guard, etc.)
*/
webhookDeliveryMetricsHook?: WebhookDeliveryMetricsHook;
webhookAddressResolver?: OutboundHttpAddressResolver;
fetch?: typeof fetch;
env?: EnvRecord;
updateLegacyEventWebhookAttempt?: (attempt: { id: string; tenantId: string | null }, data: LegacyEventWebhookAttemptUpdate) => Promise<void>;
scanService?: {
scanObject(target: WorkerStorageScanTarget): Promise<StorageScanResult>;
};
/**
* AK-P4a §5 — writes the scan verdict (clean|infected) back onto
* File.scanStatus so the fail-closed media proxy (AD-23) can serve the
* object. Composed in the `general` group bootstrap (jobs/file-scan-status.ts)
* with a Prisma seam. When unset (or the job carries no fileId) the handler
* skips the write-back — e.g. legacy import-path scans with no File row.
*/
fileScanStatusWriter?: {
recordVerdict(input: { fileId: string; tenantId?: string; verdict: "clean" | "infected" }): Promise<void>;
};
/**
* AK-P4a §4 — file-preview generator. Composed in the `general` worker
* group bootstrap (jobs/file-preview.ts) with Prisma + storage seams.
* The handler stays thin: it delegates to this service, which loads the
* object, renders the preview (HEIC->JPG, image->WebP, PDF->PNG) and
* writes previewStatus + preview/thumbnail keys back to the File row.
* When unset, the handler throws so the job dead-letters loudly rather
* than silently leaving previewStatus stuck on `generating`.
*/
filePreviewService?: {
generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult>;
};
exportService?: {
runExport(payload: ExportJob): Promise<WorkerExportRunResult>;
};
privacyGdprExportService?: {
runExport(payload: ExportJob): Promise<PrivacyGdprExportRunResult>;
};
importService?: {
runImport(payload: ImportJob): Promise<WorkerImportRunResult>;
};
storageDeleteService?: {
deleteObject(input: DeleteObjectInput): Promise<DeleteObjectResult>;
};
listSoftDeletedExportJobsForHardDelete?: (data: ExportJobRetentionListInput) => Promise<ExportJobRetentionRecord[]>;
listExpiredCompletedExportJobs?: (data: ExportJobRetentionListInput) => Promise<ExportJobRetentionRecord[]>;
markExportJobExpired?: (data: ExportJobExpiredInput) => Promise<unknown>;
markExportJobHardDeleteSucceeded?: (data: ExportJobHardDeleteSucceededInput) => Promise<unknown>;
markExportJobHardDeleteFailed?: (data: ExportJobHardDeleteFailedInput) => Promise<unknown>;
markExportJobPaused?: (data: ExportJobPausedInput) => Promise<unknown>;
markExportJobResumed?: (data: ExportJobResumedInput) => Promise<unknown>;
markExportJobRunning?: (data: ExportJobRunningInput) => Promise<unknown>;
markExportJobSucceeded?: (data: ExportJobSucceededInput) => Promise<unknown>;
markExportJobFailed?: (data: ExportJobFailedInput) => Promise<unknown>;
markGdprExportProcessing?: (data: MarkGdprExportProcessingInput) => Promise<unknown>;
markGdprExportReady?: (data: MarkGdprExportReadyInput) => Promise<unknown>;
markGdprExportFailed?: (data: MarkGdprExportFailedInput) => Promise<unknown>;
markImportJobRunning?: (data: ImportJobRunningInput) => Promise<unknown>;
markImportJobSucceeded?: (data: ImportJobSucceededInput) => Promise<unknown>;
markImportJobFailed?: (data: ImportJobFailedInput) => Promise<unknown>;
getUserNotificationPreferences?: (filters: { tenantId: string; userId: string; channel?: string | null; category?: string | null }) => Promise<unknown[]>;
createNotificationDelivery?: (data: CreateNotificationDeliveryInput) => Promise<unknown>;
markNotificationDeliveryStatus?: (data: MarkNotificationDeliveryStatusInput) => Promise<unknown>;
/**
* Phase 10 — DSGVO grace-period cron service. The maintenance handler
* looks up overdue User rows and dispatches cascade-deletions through
* this slot. When unset, the handler falls back to a Prisma-backed
* implementation that lists candidates and logs them without running
* the destructive cascade — the cascade-runtime wiring lands in a
* follow-up. The split keeps the cron observable from day-one without
* blocking on the full cascade implementation.
*/
dsgvoGraceCronService?: {
listOverdueUsers: (now: Date) => Promise<DsgvoGraceCandidate[]>;
enqueueDeletion: (input: {
userId: string;
tenantId: string | null;
requestId: string;
actorType: "system";
actorId: string;
mode: "delete";
reason: string;
}) => Promise<{ jobId?: string }>;
};
deleteDoneOutboxEvents?: (data: DeleteDoneOutboxEventsOptions) => Promise<DeleteDoneOutboxEventsResult>;
reconcileWorkflowExecutionStalls?: (data: WorkflowExecutionStalledReconcileInput) => Promise<WorkflowExecutionStalledReconcileResult>;
/**
* Phase 06 #21 / F-015 — production dependencies for the plugin-invoke
* handler. The `plugin` worker group bootstrap (`apps/worker/src/jobs/
* plugin-invoke-bootstrap.ts`) composes the real Prisma / filesystem /
* SharedStore / sandbox seams and injects them here. When the slot is
* unset (e.g. a misconfigured `general` worker receives a plugin-invoke
* job because someone changed group routing) the registry entry throws
* so the job dead-letters loudly rather than being silently acked.
*/
pluginInvokeDeps?: PluginInvokeDeps;
/**
* Optional structured logger used by handlers that need to emit
* non-error progress (currently only the dsgvo-grace-cron branch). A
* pino-shaped surface keeps this compatible with the worker bootstrap
* logger without dragging the pino type-graph into this module.
*/
logger?: {
info: (fields: Record<string, unknown>, msg: string) => void;
error: (fields: Record<string, unknown>, msg: string) => void;
};
};
type WorkerExportRunResult = {
artifactBucket: string;
artifactKey: string;
rowCount: number;
expiresAt: Date;
};
type WorkerImportRunResult = {
rowCount: number;
acceptedRows: number;
rejectedRows: number;
duplicateCount: number;
validationReport?: Pick<ValidationReport, "summary" | "errorReport">;
errorReportBucket?: string;
errorReportKey?: string;
};
type WorkerStorageScanTarget = {
bucket: string;
key: string;
tenantId?: string;
uploadId?: string;
contentType?: string;
metadata?: Record<string, unknown>;
};
export type FilePreviewServiceResult = {
/** Terminal preview state written to the File row by the service. */
previewStatus: "ready" | "failed" | "not_supported";
previewStorageKey?: string;
thumbnailStorageKey?: string;
/** Optional diagnostic reason (e.g. why generation failed). */
reason?: string;
};
class WebhookDeliveryError extends Error {}
class WebhookUrlValidationError extends Error {}
class NotificationHandlerError extends Error {
readonly action = "notification.rejected";
}
class UnsupportedWorkerJobError extends Error {
readonly action = "worker.unsupported";
readonly details: Record<string, unknown>;
constructor(message: string, details: Record<string, unknown>) {
super(message);
this.name = "UnsupportedWorkerJobError";
this.details = details;
}
}
class PluginInvokePayloadError extends Error {
readonly action = "plugin.invoke.invalid_payload";
readonly details: Record<string, unknown>;
constructor(message: string, details: Record<string, unknown>) {
super(message);
this.name = "PluginInvokePayloadError";
this.details = details;
}
}
function getNotificationDeliveryId(record: unknown): string | undefined {
return record && typeof record === "object" && typeof (record as { id?: unknown }).id === "string" ? (record as { id: string }).id : undefined;
}
async function createSkippedNotificationDelivery(
createNotificationDelivery: NonNullable<HandlerDependencies["createNotificationDelivery"]>,
payload: QueuePayload<typeof queueNames.notifications>,
payloadMetadata: Record<string, string>,
reason: string,
sourceKey?: string
) {
return createNotificationDelivery({
tenantId: payload.tenantId ?? "",
userId: payload.userId,
channel: payload.channel,
category: payload.category,
messageKey: payload.messageKey,
sourceKey,
status: "SKIPPED",
attemptsMade: 0,
lastError: reason,
payloadMetadata: { ...payloadMetadata, reason },
});
}
function isNotificationPreferenceDisabled(preferences: unknown[]): boolean {
return preferences.some((preference) => preference && typeof preference === "object" && (preference as { enabled?: unknown }).enabled === false);
}
function getNotificationErrorMessage(error: unknown): string {
return redactNotificationDiagnosticText(error instanceof Error ? error.message : "Notification email delivery failed.");
}
function getPreferenceChannel(channel: string): "email" | "in-app" {
if (channel === "email" || channel === "in-app") {
return channel;
}
throw new NotificationHandlerError(`Unsupported notification channel: ${channel}`);
}
async function getNotificationPreferencesForPayload(
payload: QueuePayload<typeof queueNames.notifications>,
getUserNotificationPreferences: NonNullable<HandlerDependencies["getUserNotificationPreferences"]>
): Promise<unknown[]> {
if (!payload.tenantId) {
return [];
}
const channel = getPreferenceChannel(payload.channel);
const category = payload.category ?? "default";
const preferences = await getUserNotificationPreferences({
tenantId: payload.tenantId,
userId: payload.userId,
channel,
category,
});
const defaultPreferences =
payload.category && payload.category !== "default"
? await getUserNotificationPreferences({
tenantId: payload.tenantId,
userId: payload.userId,
channel,
category: "default",
})
: [];
return preferences.length > 0 ? preferences : defaultPreferences;
}
export class ScanHandlerError extends Error {
readonly action = "scan.error";
readonly details: Record<string, unknown>;
constructor(message: string, details: Record<string, unknown>) {
super(message);
this.name = "ScanHandlerError";
this.details = details;
}
}
export class PreviewHandlerError extends Error {
readonly action = "preview.error";
readonly details: Record<string, unknown>;
constructor(message: string, details: Record<string, unknown>) {
super(message);
this.name = "PreviewHandlerError";
this.details = details;
}
}
export class ExportHandlerError extends Error {
readonly action = "export.error";
readonly details: Record<string, unknown>;
constructor(message: string, details: Record<string, unknown>) {
super(message);
this.name = "ExportHandlerError";
this.details = details;
}
}
export class ImportHandlerError extends Error {
readonly action = "import.error";
readonly details: Record<string, unknown>;
constructor(message: string, details: Record<string, unknown>) {
super(message);
this.name = "ImportHandlerError";
this.details = details;
}
}
type HandlerExecutionContext<Name extends QueueName = QueueName> = {
attemptsMade?: number;
attemptsLimit?: number;
jobId?: string;
updatePayload?: (payload: QueuePayload<Name>) => Promise<void>;
};
type Handler<Name extends QueueName> = (
payload: QueuePayload<Name>,
dependencies: HandlerDependencies,
context?: HandlerExecutionContext<Name>
) => Promise<HandlerResult>;
type HandlerRegistry = {
[Name in QueueName]: Handler<Name>;
};
function getAttemptNumber(context?: { attemptsMade?: number }): number {
return Math.max(1, (context?.attemptsMade ?? 0) + 1);
}
function getAttemptsLimit(context?: { attemptsLimit?: number }): number {
return Math.max(1, context?.attemptsLimit ?? 1);
}
function isRetryableWebhookStatus(status: number): boolean {
return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500;
}
function getPayloadNotificationDeliveryId(payload: QueuePayload<typeof queueNames.notifications>): string | undefined {
return typeof payload.metadata?.deliveryId === "string" ? payload.metadata.deliveryId : undefined;
}
function getNotificationDeliverySourceKey(
payload: QueuePayload<typeof queueNames.notifications>,
context: HandlerExecutionContext<typeof queueNames.notifications> | undefined
): string | undefined {
if (typeof payload.metadata?.idempotencyKey === "string" && payload.metadata.idempotencyKey.trim()) {
return payload.metadata.idempotencyKey.trim();
}
if (context?.jobId?.trim()) {
return `job:${context.jobId.trim()}`;
}
if (typeof payload.metadata?.requestId === "string" && payload.metadata.requestId.trim()) {
return `request:${payload.metadata.requestId.trim()}`;
}
return undefined;
}
async function persistPayloadNotificationDeliveryId(
payload: QueuePayload<typeof queueNames.notifications>,
context: HandlerExecutionContext<typeof queueNames.notifications> | undefined,
deliveryId: string | undefined
): Promise<void> {
if (!deliveryId || getPayloadNotificationDeliveryId(payload) || !context?.updatePayload) {
return;
}
await context.updatePayload({
...payload,
metadata: {
...payload.metadata,
deliveryId,
},
});
}
export function buildHandlerRegistry(dependencies: HandlerDependencies = {}): HandlerRegistry {
const deps = dependencies;
return {
[queueNames.email]: async (payload, _dependencies, context) => {
if (payload.kind === "mail.transactional") {
const provider =
deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined);
const recipients = payload.to.map(addressToEmail);
const result = await provider.send({
from: payload.from,
to: payload.to,
cc: payload.cc,
bcc: payload.bcc,
replyTo: payload.replyTo,
subject: payload.subject,
text: payload.text,
html: payload.html,
templateId: payload.templateId,
attachments: payload.attachments,
tags: payload.tags,
precedenceBulk: payload.precedenceBulk,
sendAt: payload.sendAt,
inReplyTo: payload.inReplyTo,
references: payload.references,
settings: payload.settings,
headers: payload.headers,
listUnsubscribe: payload.listUnsubscribe,
...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}),
personalization: buildMailPersonalization(recipients, payload.templateData),
});
return {
handled: true,
action: "mail.sent",
details: { provider: result.provider, messageId: result.messageId },
};
}
// Only an injected (test) fetch is passed through; in production we leave
// it undefined so the outbound guard uses its own undici fetch with the
// DNS-rebinding IP pin engaged (see fetchWebhookWithRedirects).
const fetchImpl = deps.fetch;
const updateLegacyEventWebhookAttempt = deps.updateLegacyEventWebhookAttempt ?? updateLegacyEventWebhookAttemptInDb;
const legacyAttemptId = typeof payload.metadata?.legacyAttemptId === "string" ? payload.metadata.legacyAttemptId : undefined;
const legacyAttemptTenantId = typeof payload.metadata?.legacyAttemptTenantId === "string" ? payload.metadata.legacyAttemptTenantId : null;
const attemptNumber = getAttemptNumber(context);
const isTerminalFailure = attemptNumber >= getAttemptsLimit(context);
const emitWebhookMetric = (status: "success" | "retry" | "dlq" | "rejected") => {
if (!deps.webhookDeliveryMetricsHook) return;
try {
deps.webhookDeliveryMetricsHook({ status });
} catch {
// never block delivery
}
};
try {
const response = await fetchWebhookWithRedirects(
fetchImpl,
payload.url,
{
method: payload.method ?? "POST",
headers: {
"content-type": "application/json",
...payload.headers,
},
body: payload.body === undefined ? undefined : JSON.stringify(payload.body),
},
deps.webhookAddressResolver
);
const responseBody = await readLimitedResponseText(response, WEBHOOK_RESPONSE_BODY_LIMIT_BYTES);
const responseDiagnostics = serializeWebhookResponseDiagnostics(response, responseBody);
if (!response.ok) {
const message = `Webhook delivery failed with ${response.status}.`;
const retryableFailure = isRetryableWebhookStatus(response.status);
const shouldFinishAttempt = !retryableFailure || isTerminalFailure;
if (legacyAttemptId) {
await updateLegacyEventWebhookAttempt(
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
{
...(shouldFinishAttempt ? { finishedAt: new Date(), success: false } : {}),
status: response.status,
message,
body: responseDiagnostics,
}
);
}
if (!retryableFailure) {
emitWebhookMetric("dlq");
return {
handled: true,
action: "webhook.failed",
details: { status: response.status, eventName: payload.eventName, retryable: false },
};
}
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
throw new WebhookDeliveryError(message);
}
if (legacyAttemptId) {
await updateLegacyEventWebhookAttempt(
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
{
finishedAt: new Date(),
success: true,
status: response.status,
message: response.statusText,
body: responseDiagnostics,
}
);
}
emitWebhookMetric("success");
return {
handled: true,
action: "webhook.delivered",
details: { status: response.status, eventName: payload.eventName },
};
} catch (error) {
if (error instanceof WebhookUrlValidationError) {
const message = redactSecretLikeText(error.message);
if (legacyAttemptId) {
await updateLegacyEventWebhookAttempt(
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
{
finishedAt: new Date(),
success: false,
message,
}
);
}
emitWebhookMetric("rejected");
return {
handled: true,
action: "webhook.rejected",
details: { eventName: payload.eventName, reason: message },
};
}
if (error instanceof WebhookDeliveryError) {
// Already counted at throw-site (retry / dlq).
} else if (legacyAttemptId) {
await updateLegacyEventWebhookAttempt(
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
{
...(isTerminalFailure ? { finishedAt: new Date(), success: false } : {}),
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
}
);
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
} else if (!legacyAttemptId) {
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
}
throw error;
}
},
[queueNames.notifications]: async (payload, _dependencies, context) => {
const getUserNotificationPreferences = deps.getUserNotificationPreferences ?? getUserNotificationPreferencesInDb;
const createNotificationDelivery = deps.createNotificationDelivery ?? createNotificationDeliveryInDb;
const markNotificationDeliveryStatus = deps.markNotificationDeliveryStatus ?? markNotificationDeliveryStatusInDb;
const payloadMetadata = {
...(typeof payload.metadata?.requestId === "string" ? { requestId: payload.metadata.requestId } : {}),
...(typeof payload.metadata?.source === "string" ? { source: payload.metadata.source } : {}),
...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}),
};
const sourceKey = getNotificationDeliverySourceKey(payload, context);
getPreferenceChannel(payload.channel);
if (!payload.tenantId) {
throw new NotificationHandlerError("Notification jobs require tenantId.");
}
if (payload.channel !== "email") {
const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences);
if (isNotificationPreferenceDisabled(preferences)) {
const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey);
return {
handled: true,
action: "notification.in-app.skipped",
details: {
userId: payload.userId,
channel: payload.channel,
messageKey: payload.messageKey,
deliveryId: getNotificationDeliveryId(delivery),
reason: "preference_disabled",
},
};
}
const delivery = await createNotificationDelivery({
tenantId: payload.tenantId ?? null,
userId: payload.userId,
channel: payload.channel,
category: payload.category,
messageKey: payload.messageKey,
sourceKey,
status: "SENT",
attemptsMade: 1,
payloadMetadata,
sentAt: new Date(),
});
return {
handled: true,
action: "notification.accepted",
details: {
userId: payload.userId,
channel: payload.channel,
messageKey: payload.messageKey,
deliveryId: getNotificationDeliveryId(delivery),
},
};
}
if (!payload.email) {
throw new NotificationHandlerError("Email notifications require email recipient.");
}
const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences);
if (isNotificationPreferenceDisabled(preferences)) {
const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey);
return {
handled: true,
action: "notification.email.skipped",
details: {
userId: payload.userId,
channel: payload.channel,
messageKey: payload.messageKey,
deliveryId: getNotificationDeliveryId(delivery),
reason: "preference_disabled",
},
};
}
let deliveryId = getPayloadNotificationDeliveryId(payload);
if (!deliveryId) {
const delivery = await createNotificationDelivery({
tenantId: payload.tenantId,
userId: payload.userId,
channel: "email",
category: payload.category,
messageKey: payload.messageKey,
sourceKey,
status: "PENDING",
attemptsMade: 0,
payloadMetadata,
});
deliveryId = getNotificationDeliveryId(delivery);
await persistPayloadNotificationDeliveryId(payload, context, deliveryId);
}
const attemptNumber = getAttemptNumber(context);
const isTerminalFailure = attemptNumber >= getAttemptsLimit(context);
let result: Awaited<ReturnType<MailProvider["send"]>>;
try {
const provider =
deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined);
result = await provider.send({
to: [payload.email],
subject: payload.subject,
text: payload.text,
html: payload.html,
templateId: payload.templateId,
idempotencyKey: `notification-delivery:${deliveryId}`,
personalization: payload.templateData ? [{ email: payload.email, data: payload.templateData }] : undefined,
});
} catch (error) {
if (deliveryId) {
try {
await markNotificationDeliveryStatus({
id: deliveryId,
tenantId: payload.tenantId,
status: isTerminalFailure ? "FAILED" : "PENDING",
attemptsMade: attemptNumber,
lastError: getNotificationErrorMessage(error),
});
} catch {
// Preserve the provider failure as the retry reason.
}
}
throw error;
}
if (deliveryId) {
try {
await markNotificationDeliveryStatus({
id: deliveryId,
tenantId: payload.tenantId,
status: "SENT",
attemptsMade: attemptNumber,
providerMessageId: result.messageId ?? null,
sentAt: new Date(),
});
} catch (error) {
return {
handled: true,
action: "notification.email.sent.diagnostic_failed",
details: {
provider: result.provider,
messageId: result.messageId,
userId: payload.userId,
messageKey: payload.messageKey,
deliveryId,
diagnosticError: getNotificationErrorMessage(error),
},
};
}
}
return {
handled: true,
action: "notification.email.sent",
details: { provider: result.provider, messageId: result.messageId, userId: payload.userId, messageKey: payload.messageKey },
};
},
[queueNames.maintenance]: async (payload) => {
if (payload.task === "expired-export-artifacts") {
return runExportArtifactRetentionHandler(payload, deps);
}
if (payload.task === "dsgvo-grace-period-scan") {
return runDsgvoGracePeriodScanHandler(payload, deps);
}
if (payload.task === "outbox-done-retention") {
return runOutboxDoneRetentionHandler(payload, deps);
}
if (payload.task === "workflow-execution-stalled-reconcile") {
return runWorkflowExecutionStalledReconcileHandler(payload, deps);
}
if (payload.task === "task-archive-retention") {
return runTaskArchiveRetentionHandler(payload, deps);
}
if (payload.task === "lead-retention-sweep") {
return runLeadRetentionHandler(payload, deps);
}
if (payload.task === "task-reminder-sweep") {
return runTaskReminderSweepHandler(payload, deps);
}
if (payload.task === "integrations-token-refresh") {
return runIntegrationsTokenRefreshHandler(payload, deps);
}
if (payload.task === "integrations-health-sweep") {
return runIntegrationsHealthSweepHandler(payload, deps);
}
if (payload.task === "calendar-token-refresh") {
return runCalendarTokenRefreshHandler(payload, deps);
}
if (payload.task === "calendar-webhook-renewal") {
return runCalendarWebhookRenewalHandler(payload, deps);
}
if (payload.task === "calendar-webhook-drain") {
return runCalendarWebhookDrainHandler(payload, deps);
}
if (payload.task === "calendar-internal-write-sync") {
return runCalendarInternalWriteSyncHandler(payload, deps);
}
if (payload.task === "email-inbox-token-refresh") {
return runEmailInboxTokenRefreshHandler(payload, deps);
}
if (payload.task === "email-inbox-webhook-renewal") {
return runEmailInboxWebhookRenewalHandler(payload, deps);
}
if (payload.task === "email-inbox-webhook-drain") {
return runEmailInboxWebhookDrainHandler(payload, deps);
}
if (payload.task === "email-inbox-initial-sync") {
return runEmailInboxInitialSyncHandler(payload, deps);
}
if (payload.task === "email-inbox-stale-poll") {
return runEmailInboxStalePollHandler(payload, deps);
}
if (payload.task === "email-inbox-event-cleanup") {
return runEmailInboxEventCleanupHandler(payload, deps);
}
if (payload.task === "hr-upload-reminder-dispatch") {
return runHrUploadReminderDispatchHandler(payload, deps);
}
if (payload.task === "email-domain-probe") {
return runEmailDomainProbeHandler(payload, deps);
}
throw new UnsupportedWorkerJobError("Unsupported maintenance task.", {
queueName: queueNames.maintenance,
task: payload.task,
olderThanHours: payload.olderThanHours,
olderThanMinutes: payload.olderThanMinutes,
});
},
[queueNames.browserAutomation]: async (payload) => {
throw new UnsupportedWorkerJobError("Browser automation worker job is not configured.", {
queueName: queueNames.browserAutomation,
kind: payload.kind,
sessionId: payload.sessionId,
});
},
[queueNames.fileScan]: async (payload) => {
if (payload.kind !== "scan.file") {
throw new ScanHandlerError("Unsupported file scan payload kind.", {
verdict: "error",
code: "invalid_payload",
message: "Unsupported file scan payload kind.",
});
}
if (!deps.scanService) {
throw new ScanHandlerError("File scan service is not configured.", {
verdict: "error",
bucket: payload.bucket,
key: payload.key,
uploadId: payload.uploadId,
code: "scan_service_missing",
message: "File scan service is not configured.",
});
}
let result: StorageScanResult;
try {
result = await deps.scanService.scanObject({
bucket: payload.bucket,
key: payload.key,
tenantId: payload.tenantId,
uploadId: payload.uploadId,
contentType: payload.contentType,
metadata: payload.metadata,
});
} catch (error) {
throw new ScanHandlerError("File scan failed.", buildRejectedScanDetails(payload, error));
}
const details = buildScanResultDetails(result, payload.uploadId);
if (result.verdict === "clean" || result.verdict === "infected") {
// AK-P4a §5 — persist the verdict so the fail-closed media proxy can
// serve clean files (and keep refusing infected ones). Skipped when the
// job carries no fileId or the writer seam is unwired (legacy scans).
if (payload.fileId && deps.fileScanStatusWriter) {
await deps.fileScanStatusWriter.recordVerdict({
fileId: payload.fileId,
tenantId: payload.tenantId,
verdict: result.verdict,
});
}
return { handled: true, action: `scan.${result.verdict}`, details };
}
throw new ScanHandlerError("File scan failed.", details);
},
[queueNames.filePreview]: async (payload) => {
if (payload.kind !== "preview.file") {
throw new PreviewHandlerError("Unsupported file preview payload kind.", {
code: "invalid_payload",
message: "Unsupported file preview payload kind.",
});
}
if (!deps.filePreviewService) {
throw new PreviewHandlerError("File preview service is not configured.", {
fileId: payload.fileId,
tenantId: payload.tenantId,
code: "preview_service_missing",
message: "File preview service is not configured.",
});
}
let result: FilePreviewServiceResult;
try {
result = await deps.filePreviewService.generatePreview(payload);
} catch (error) {
throw new PreviewHandlerError("File preview generation failed.", {
fileId: payload.fileId,
tenantId: payload.tenantId,
code: "preview_failed",
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
});
}
return {
handled: true,
action: `preview.${result.previewStatus}`,
details: {
fileId: payload.fileId,
tenantId: payload.tenantId,
previewStatus: result.previewStatus,
previewStorageKey: result.previewStorageKey,
thumbnailStorageKey: result.thumbnailStorageKey,
...(result.reason ? { reason: redactSecretLikeText(result.reason) } : {}),
},
};
},
[queueNames.exports]: async (payload, _dependencies, context) => runExportHandler(payload, deps, context),
[queueNames.imports]: async (payload, _dependencies, context) => runImportHandler(payload, deps, context),
// Phase 06 #21 / F-015 — real plugin-invoke dispatch. The handler is
// dependency-injected via `deps.pluginInvokeDeps` (composed in the
// `plugin` worker group bootstrap). When the slot is missing the
// handler throws so BullMQ dead-letters the job instead of silently
// acknowledging it — silent ack was the F-015 Critical finding.
[queueNames.pluginInvoke]: async (payload, _dependencies, context) => {
void context;
assertPluginInvokePayload(payload, deps);
if (!deps.pluginInvokeDeps) {
const message =
"plugin-invoke job received but pluginInvokeDeps is not configured on this worker. " +
"Set WORKER_QUEUE_GROUP=plugin and ensure the plugin bootstrap injected pluginInvokeDeps.";
deps.logger?.error?.({ queue: queueNames.pluginInvoke, pluginId: payload.pluginId, hookName: payload.hookName }, message);
throw new Error(message);
}
const outcome = await handlePluginInvokeJob(
{
tenantId: payload.tenantId,
pluginId: payload.pluginId,
version: payload.version,
hookName: payload.hookName,
input: payload.input,
metadata: payload.metadata,
},
deps.pluginInvokeDeps
);
if (outcome.ok) {
return {
handled: true,
action: "plugin.invoke.completed",
details: {
pluginId: payload.pluginId,
hookName: payload.hookName,
fuelUsed: outcome.fuelUsed,
durationMs: outcome.durationMs,
},
};
}
return {
handled: true,
action: "plugin.invoke.failed",
details: {
pluginId: payload.pluginId,
hookName: payload.hookName,
reason: outcome.reason,
message: outcome.message,
durationMs: outcome.durationMs,
},
};
},
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === "object" && !Array.isArray(value);
}
function isNonEmptyString(value: unknown): value is string {
return typeof value === "string" && value.trim().length > 0;
}
function assertPluginInvokePayload(payload: unknown, deps: HandlerDependencies): asserts payload is PluginInvokeJob {
const record = isRecord(payload) ? payload : {};
const invalidFields: string[] = [];
if (record.kind !== "plugin.invoke") {
invalidFields.push("kind");
}
for (const field of ["tenantId", "pluginId", "version", "hookName"] as const) {
if (!isNonEmptyString(record[field])) {
invalidFields.push(field);
}
}
if (invalidFields.length === 0) {
return;
}
const details = {
queueName: queueNames.pluginInvoke,
code: "invalid_payload",
invalidFields,
};
deps.logger?.error?.(
{
...details,
payload: summarizePluginInvokePayloadForLog(record),
},
"Invalid plugin-invoke queue payload."
);
throw new PluginInvokePayloadError("Invalid plugin-invoke queue payload.", details);
}
function summarizePluginInvokePayloadForLog(record: Record<string, unknown>): Record<string, unknown> {
return {
kind: typeof record.kind === "string" ? record.kind : typeof record.kind,
tenantId: typeof record.tenantId === "string" ? record.tenantId : typeof record.tenantId,
pluginId: typeof record.pluginId === "string" ? record.pluginId : typeof record.pluginId,
version: typeof record.version === "string" ? record.version : typeof record.version,
hookName: typeof record.hookName === "string" ? record.hookName : typeof record.hookName,
};
}
function addressToEmail(address: string | { email: string }) {
return typeof address === "string" ? address : address.email;
}
function buildMailPersonalization(to: readonly string[], templateData: Record<string, unknown> | undefined) {
if (!templateData) {
return undefined;
}
const hasPerRecipientData = to.some((email) => isRecord(templateData[email]));
return to.map((email) => ({
email,
data: hasPerRecipientData && isRecord(templateData[email]) ? templateData[email] : templateData,
}));
}
async function runExportArtifactRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const now = new Date();
const retentionCutoff = getRetentionCutoff(now, payload.olderThanHours);
const take = payload.take ?? undefined;
const listExpiredCompleted = deps.listExpiredCompletedExportJobs ?? listExpiredCompletedExportJobsInDb;
const markExpired = deps.markExportJobExpired ?? markExportJobExpiredInDb;
const listPending = deps.listSoftDeletedExportJobsForHardDelete ?? listSoftDeletedExportJobsForHardDeleteInDb;
const markDeleted = deps.markExportJobHardDeleteSucceeded ?? markExportJobHardDeleteSucceededInDb;
const markFailed = deps.markExportJobHardDeleteFailed ?? markExportJobHardDeleteFailedInDb;
const expiredCompletedJobs = await listExpiredCompleted({ now, take });
await Promise.all(expiredCompletedJobs.map((job) => markExpired({ id: job.id, tenantId: job.tenantId, deletedAt: now })));
const jobs = await listPending({ now: retentionCutoff, take });
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
const details: Record<string, unknown>[] = new Array(jobs.length);
if (jobs.length > 0 && !deps.storageDeleteService) {
throw new Error("Export artifact storage delete service is not configured.");
}
await mapWithConcurrency(jobs, EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY, async (job, index) => {
try {
await deps.storageDeleteService!.deleteObject({ bucket: job.artifactBucket, key: job.artifactKey });
await markDeleted({ id: job.id, tenantId: job.tenantId });
details[index] = { exportJobId: job.id, status: "deleted" };
} catch (error) {
const message = redactSecretLikeText(error instanceof Error ? error.message : String(error));
await markFailed({ id: job.id, tenantId: job.tenantId, errorMessage: message });
details[index] = { exportJobId: job.id, status: "failed", message };
}
});
const deleted = details.filter((job) => job?.status === "deleted").length;
const failed = details.filter((job) => job?.status === "failed").length;
return {
handled: true,
action: "export-artifact-retention.completed",
details: {
task: payload.task,
requestId,
expired: expiredCompletedJobs.length,
processed: jobs.length,
deleted,
failed,
jobs: details,
},
};
}
function getRetentionCutoff(now: Date, olderThanHours: number | undefined) {
if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) {
return now;
}
return new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
}
async function runOutboxDoneRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const now = new Date();
const olderThanHours = resolveOutboxDoneRetentionHours(payload.olderThanHours);
const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
const deleteDoneOutboxEvents = deps.deleteDoneOutboxEvents ?? (async (options) => deleteDoneOutboxEventsInDb(await getDefaultDbClient(), options));
const result = await deleteDoneOutboxEvents({ cutoff, take: payload.take });
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
return {
handled: true,
action: "outbox.done-retention.completed",
details: {
task: payload.task,
requestId,
olderThanHours,
cutoff: result.cutoff.toISOString(),
take: result.take,
deleted: result.deleted,
},
};
}
function resolveOutboxDoneRetentionHours(olderThanHours: number | undefined): number {
if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) {
return 720;
}
return Math.floor(olderThanHours);
}
async function runWorkflowExecutionStalledReconcileHandler(
payload: QueuePayload<typeof queueNames.maintenance>,
deps: HandlerDependencies
): Promise<HandlerResult> {
const runReconcile =
deps.reconcileWorkflowExecutionStalls ??
(async (options) => reconcileWorkflowExecutionStalls((await getDefaultDbClient()) as unknown as WorkflowExecutionStalledReconcilerPrisma, options));
const result = await runReconcile({
olderThanMinutes: payload.olderThanMinutes,
take: payload.take,
});
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
return {
handled: true,
action: "workflow.execution-stalled-reconcile.completed",
details: {
task: payload.task,
requestId,
olderThanMinutes: payload.olderThanMinutes,
cutoff: result.cutoff.toISOString(),
take: result.take,
inspected: result.inspected,
failedOutboxEvents: result.failedOutboxEvents,
repaired: result.repaired,
skippedWithoutFailedOutbox: result.skippedWithoutFailedOutbox,
updateMisses: result.updateMisses,
},
};
}
async function getDefaultDbClient() {
const { getDbClient } = await import("@hivegrid/db/client.server");
return getDbClient();
}
/**
* P4 Tasks (AK-P4b) §7 — 30-day auto-archive (DoD 5). Archives completed tasks
* whose `completedAt` is older than the retention window by stamping
* `archivedAt`. Idempotent: already-archived rows are excluded by the filter.
*
* RLS-ready (AK-P4b "Teil A"): a cross-tenant sweep cannot run under the
* restricted app role once Task RLS is active (an unscoped query returns zero
* rows). So we enumerate active tenants from the global Tenant registry (not
* RLS-scoped) and run the scoped archive per tenant inside withTenantRLS(), which
* sets `app.tenant_id` so the policy binds. Functional no-op until the policy is
* activated (prisma/rls/task-isolation.sql, P5 cutover — at which point
* hivegrid_app needs SELECT on "Tenant" or a privileged maintenance connection;
* see that file's Teil-B notes).
*/
async function runTaskArchiveRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const olderThanHours = typeof payload.olderThanHours === "number" && payload.olderThanHours > 0 ? Math.floor(payload.olderThanHours) : 720;
const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let archived = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) =>
tx.task.updateMany({
where: { tenantId, status: "done", completedAt: { lte: cutoff }, archivedAt: null },
data: { archivedAt: now },
}),
);
archived += result.count;
}
logger.info({ task: payload.task, cutoff: cutoff.toISOString(), tenants: tenants.length, archived }, "tasks.archive-retention.completed");
return {
handled: true,
action: "tasks.archive-retention.completed",
details: { task: payload.task, olderThanHours, cutoff: cutoff.toISOString(), tenants: tenants.length, archived },
};
}
async function runLeadRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let archived = 0;
for (const { id: tenantId } of tenants) {
const dueIds = await withTenantRLS(db, tenantId, (tx) =>
tx.inboxLead.findMany({
where: { tenantId, archivedAt: null, autoDeleteAt: { lte: now } },
select: { id: true },
orderBy: { autoDeleteAt: "asc" },
take,
}),
);
if (dueIds.length === 0) continue;
const result = await withTenantRLS(db, tenantId, (tx) =>
tx.inboxLead.updateMany({
where: { tenantId, id: { in: dueIds.map((row) => row.id) }, archivedAt: null },
data: { archivedAt: now },
}),
);
archived += result.count;
}
logger.info({ task: payload.task, tenants: tenants.length, archived }, "leads.retention.completed");
return {
handled: true,
action: "leads.retention.completed",
details: { task: payload.task, tenants: tenants.length, archived, take },
};
}
/**
* R-P5 §P2d — refresh OAuth tokens nearing expiry across all tenants. Enumerates
* active tenants from the global registry (not RLS-scoped) and runs the scoped
* refresh per tenant inside withTenantRLS() (so the policy binds at cutover). The
* per-connection logic (decrypt → adapter.refreshToken → re-encrypt) lives in
* @hivegrid/integrations/worker-tasks.server, shared with the web side.
*/
async function runIntegrationsTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let refreshed = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => refreshExpiringTokensForTenant(tx as unknown as Parameters<typeof refreshExpiringTokensForTenant>[0], tenantId, { now, windowMs }));
refreshed += result.refreshed;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "integrations.token-refresh.completed");
return { handled: true, action: "integrations.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
}
/**
* R-P5 §P2d — probe every active connection's health across all tenants, persist
* the status, and notify owners on degraded/broken. Same per-tenant RLS sweep
* pattern; per-connection logic shared via worker-tasks.server.
*/
async function runIntegrationsHealthSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let checked = 0;
let unhealthy = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => sweepConnectionHealthForTenant(tx as unknown as Parameters<typeof sweepConnectionHealthForTenant>[0], tenantId, { now }));
checked += result.checked;
unhealthy += result.unhealthy;
}
logger.info({ task: payload.task, tenants: tenants.length, checked, unhealthy }, "integrations.health-sweep.completed");
return { handled: true, action: "integrations.health-sweep.completed", details: { task: payload.task, tenants: tenants.length, checked, unhealthy } };
}
async function runCalendarTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let refreshed = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => refreshCalendarTokensForTenant(tx as never, tenantId, { now, windowMs }));
refreshed += result.refreshed;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "calendar.token-refresh.completed");
return { handled: true, action: "calendar.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
}
async function runCalendarWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
const webhookUrl = process.env.GOOGLE_CALENDAR_WEBHOOK_URL;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let checked = 0;
let renewed = 0;
let failed = 0;
let skipped = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => renewCalendarWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, webhookUrl }));
checked += result.checked;
renewed += result.renewed;
failed += result.failed;
skipped += result.skipped;
}
logger.info({ task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped }, "calendar.webhook-renewal.completed");
return { handled: true, action: "calendar.webhook-renewal.completed", details: { task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped } };
}
async function runCalendarWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let claimed = 0;
let completed = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => drainCalendarWebhookEventsForTenant(tx as never, tenantId, { now, take }));
claimed += result.claimed;
completed += result.completed;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, claimed, completed, failed }, "calendar.webhook-drain.completed");
return { handled: true, action: "calendar.webhook-drain.completed", details: { task: payload.task, tenants: tenants.length, claimed, completed, failed } };
}
async function runCalendarInternalWriteSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let scanned = 0;
let created = 0;
let updated = 0;
let deleted = 0;
let conflicts = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => syncInternalCalendarEventsForTenant(tx as never, tenantId, { now, take }));
scanned += result.scanned;
created += result.created;
updated += result.updated;
deleted += result.deleted;
conflicts += result.conflicts;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, scanned, created, updated, deleted, conflicts, failed }, "calendar.internal-write-sync.completed");
return {
handled: true,
action: "calendar.internal-write-sync.completed",
details: { task: payload.task, tenants: tenants.length, scanned, created, updated, deleted, conflicts, failed },
};
}
async function runEmailInboxTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let refreshed = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => refreshEmailTokensForTenant(tx as never, tenantId, { now, windowMs }));
refreshed += result.refreshed;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "email-inbox.token-refresh.completed");
return { handled: true, action: "email-inbox.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
}
async function runEmailInboxWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
const graphWebhookUrl = process.env.MICROSOFT_MAIL_WEBHOOK_URL;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let checked = 0;
let renewed = 0;
let failed = 0;
let skipped = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => renewEmailWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, graphWebhookUrl }));
checked += result.checked;
renewed += result.renewed;
failed += result.failed;
skipped += result.skipped;
}
logger.info({ task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped }, "email-inbox.webhook-renewal.completed");
return { handled: true, action: "email-inbox.webhook-renewal.completed", details: { task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped } };
}
async function runEmailInboxWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let claimed = 0;
let completed = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => drainEmailWebhookEventsForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
claimed += result.claimed;
completed += result.completed;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, claimed, completed, failed }, "email-inbox.webhook-drain.completed");
return { handled: true, action: "email-inbox.webhook-drain.completed", details: { task: payload.task, tenants: tenants.length, claimed, completed, failed } };
}
async function runEmailInboxInitialSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 10;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let folders = 0;
let fetched = 0;
let completed = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => runEmailInitialSyncForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
folders += result.folders;
fetched += result.fetched;
completed += result.completed;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, folders, fetched, completed, failed }, "email-inbox.initial-sync.completed");
return { handled: true, action: "email-inbox.initial-sync.completed", details: { task: payload.task, tenants: tenants.length, folders, fetched, completed, failed } };
}
async function runEmailInboxStalePollHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let polled = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => pollStaleEmailMailboxesForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
polled += result.polled;
failed += result.failed;
}
logger.info({ task: payload.task, tenants: tenants.length, polled, failed }, "email-inbox.stale-poll.completed");
return { handled: true, action: "email-inbox.stale-poll.completed", details: { task: payload.task, tenants: tenants.length, polled, failed } };
}
async function runEmailInboxEventCleanupHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let deleted = 0;
for (const { id: tenantId } of tenants) {
const result = await withTenantRLS(db, tenantId, (tx) => cleanupEmailWebhookEventsForTenant(tx as never, tenantId, { now }));
deleted += result.deleted;
}
logger.info({ task: payload.task, tenants: tenants.length, deleted }, "email-inbox.event-cleanup.completed");
return { handled: true, action: "email-inbox.event-cleanup.completed", details: { task: payload.task, tenants: tenants.length, deleted } };
}
// VA-P23 · auto-match + activity logging seam of the sync engine (M09 Block E/H).
const emailInboxMessageHook = createEmailAutoMatchHook();
/**
* P4 Tasks (AK-P4b) §7 — reminder sweep (DoD 4). For each open task with a
* deadline in scan range, sends reminders to the assignee (or assigned group's
* members) + watchers at the 24h-before / due / 24h-after windows. The
* after-window escalates to EMAIL. notify()'s sourceKey upsert dedups, so the
* hourly sweep never double-sends.
*
* RLS-ready (AK-P4b "Teil A"): like the archive sweep, the cross-tenant scan is
* driven per tenant from the global Tenant registry, and the Task/TaskWatcher
* reads run inside withTenantRLS() so the policy binds once activated. notify()
* runs OUTSIDE the transaction (slow I/O, and it has its own tenant/RLS story in
* R-P1b). `take` now bounds tasks per tenant rather than globally.
*/
async function runTaskReminderSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const bounds = reminderScanBounds(now);
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000;
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let sent = 0;
let scanned = 0;
for (const { id: tenantId } of tenants) {
const tasks = await withTenantRLS(db, tenantId, (tx) =>
tx.task.findMany({
where: {
tenantId,
deadline: { not: null, gte: bounds.deadlineGte, lte: bounds.deadlineLte },
status: { in: ["open", "in_progress", "blocked"] },
archivedAt: null,
},
select: { id: true, tenantId: true, deadline: true, assignedToUserId: true, assignedToGroupId: true },
take,
}),
);
scanned += tasks.length;
for (const task of tasks) {
if (!task.deadline) continue;
const windows = selectReminderWindows(task.deadline, now);
if (windows.length === 0) continue;
// Recipients: the assigned user (or every member of the assigned group)
// plus all watchers, de-duplicated. TaskWatcher is RLS-scoped, so the reads
// run inside the tenant transaction; group membership is read alongside.
const recipientIds = new Set<string>();
if (task.assignedToUserId) recipientIds.add(task.assignedToUserId);
const { groupMembers, watchers } = await withTenantRLS(db, tenantId, async (tx) => {
const groupMembers = task.assignedToGroupId
? await tx.groupUser.findMany({ where: { groupId: task.assignedToGroupId }, select: { userId: true } })
: [];
const watcherRows = await tx.taskWatcher.findMany({ where: { tenantId, taskId: task.id }, select: { userId: true } });
return { groupMembers, watchers: watcherRows };
});
for (const member of groupMembers) recipientIds.add(member.userId);
for (const watcher of watchers) recipientIds.add(watcher.userId);
for (const window of windows) {
for (const userId of recipientIds) {
try {
// notify() runs outside the tenant transaction (slow I/O; own RLS story).
await notify(
{
tenantId: task.tenantId,
userId,
channel: reminderChannel(window),
category: "tasks.reminder",
messageKey: reminderMessageKey(window),
sourceKey: reminderSourceKey(task.id, window),
payloadMetadata: { taskId: task.id, deadline: task.deadline.toISOString() },
},
db as unknown as NotificationClient
);
sent += 1;
} catch (err) {
logger.error(
{ task: payload.task, taskId: task.id, userId, window, err: err instanceof Error ? err.message : String(err) },
"tasks.reminder.notify-failed"
);
}
}
}
}
}
logger.info({ task: payload.task, tenants: tenants.length, scanned, sent }, "tasks.reminder-sweep.completed");
return { handled: true, action: "tasks.reminder-sweep.completed", details: { task: payload.task, tenants: tenants.length, scanned, sent } };
}
/**
* P17 HR-Hub (AK-P17b) — upload-reminder dispatch. For every active reminder
* past its `dueAt`, fan out per-employee dispatch rows (idempotent via the unique
* reminder+employee+dueAt), notify each employee, advance the reminder by its
* cadence (or complete a one-off), then escalate overdue mandatory dispatches to
* the employee's manager. Per-tenant from the global Tenant registry; all HR
* reads/writes run inside withTenantRLS() so the policy binds once activated.
* notify() runs outside the tenant transaction (slow I/O), deduped by sourceKey.
*/
async function runHrUploadReminderDispatchHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const db = await getDefaultDbClient();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let dispatched = 0;
let escalated = 0;
for (const { id: tenantId } of tenants) {
const due = await withTenantRLS(db, tenantId, (tx) =>
tx.employeeUploadReminder.findMany({ where: { tenantId, status: "active", archivedAt: null, dueAt: { not: null, lte: now } } }),
);
const profiles =
due.length > 0
? await withTenantRLS(db, tenantId, (tx) =>
tx.hrEmployeeProfile.findMany({ where: { tenantId }, select: { userId: true, team: true, locationId: true } }),
)
: [];
for (const reminder of due) {
const dueAt = reminder.dueAt ?? now;
const members = profiles.filter((profile) => audienceMatches(reminder.audienceKind, reminder.audienceRef, profile));
for (const member of members) {
const created = await withTenantRLS(db, tenantId, (tx) =>
tx.employeeUploadReminderDispatch.createMany({
data: [{ tenantId, reminderId: reminder.id, employeeUserId: member.userId, dueAt, status: "sent", sentAt: now }],
skipDuplicates: true,
}),
);
if (created.count > 0) {
dispatched += created.count;
await notify(
{
tenantId,
userId: member.userId,
channel: "IN_APP",
category: "hr",
messageKey: "hr.notify.reminder.due",
sourceKey: `hr-reminder:${reminder.id}:${dueAt.getTime()}:${member.userId}`,
payloadMetadata: { reminderId: reminder.id },
},
db as unknown as NotificationClient,
).catch((err) => logger.error({ task: payload.task, reminderId: reminder.id, userId: member.userId, err: err instanceof Error ? err.message : String(err) }, "hr.upload-reminder.notify-failed"));
}
}
const next = nextReminderDue({ cadenceKind: reminder.cadenceKind, intervalMonths: reminder.intervalMonths, from: dueAt });
await withTenantRLS(db, tenantId, (tx) =>
tx.employeeUploadReminder.update({ where: { id: reminder.id }, data: next ? { dueAt: next } : { dueAt: null, status: "completed" } }),
);
}
// Escalate still-unacknowledged mandatory dispatches past the grace window.
const cutoff = new Date(now.getTime() - HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS);
const stale = await withTenantRLS(db, tenantId, (tx) =>
tx.employeeUploadReminderDispatch.findMany({ where: { tenantId, status: "sent", dueAt: { lt: cutoff }, reminder: { escalateToManager: true, mandatory: true } } }),
);
const staleEmployeeUserIds = [...new Set(stale.map((dispatch) => dispatch.employeeUserId))];
const escalationProfiles =
staleEmployeeUserIds.length > 0
? await withTenantRLS(db, tenantId, (tx) =>
tx.hrEmployeeProfile.findMany({
where: { tenantId, userId: { in: staleEmployeeUserIds } },
select: { userId: true, managerUserId: true },
}),
)
: [];
const managerUserIdsByEmployee = new Map(escalationProfiles.map((profile) => [profile.userId, profile.managerUserId]));
for (const dispatch of stale) {
await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminderDispatch.update({ where: { id: dispatch.id }, data: { status: "escalated", escalatedAt: now } }));
escalated += 1;
const managerUserId = managerUserIdsByEmployee.get(dispatch.employeeUserId);
if (managerUserId) {
await notify(
{
tenantId,
userId: managerUserId,
channel: "IN_APP",
category: "hr",
messageKey: "hr.notify.reminder.escalated",
sourceKey: `hr-reminder-escalate:${dispatch.id}`,
payloadMetadata: { dispatchId: dispatch.id },
},
db as unknown as NotificationClient,
).catch((err) => logger.error({ task: payload.task, dispatchId: dispatch.id, err: err instanceof Error ? err.message : String(err) }, "hr.upload-reminder.escalate-notify-failed"));
}
}
}
logger.info({ task: payload.task, tenants: tenants.length, dispatched, escalated }, "hr.upload-reminder-dispatch.completed");
return { handled: true, action: "hr.upload-reminder-dispatch.completed", details: { task: payload.task, tenants: tenants.length, dispatched, escalated } };
}
/**
* AK-P14a · P14 Email — sending-domain DNS auto-probe (5-minute cadence). For
* each active tenant, resolves DKIM/SPF/DMARC for every not-yet-settled sending
* domain and advances `verificationStatus` via the pure decideDomainProbeStatus()
* helper. Reads/writes run inside withTenantRLS() so the policy binds once the
* P5 cutover activates RLS; the domain UPDATE is auto-audited by the M001 Prisma
* middleware. A tenant-level notification fires only on the verified/failed
* transition (not on every probe). The DNS resolver is the node:dns binding; the
* decision logic is unit-tested without DNS/DB.
*/
async function runEmailDomainProbeHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const now = new Date();
const db = await getDefaultDbClient();
const resolver: DomainDnsResolver = createDnsDomainResolver();
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
let probed = 0;
let verified = 0;
let failed = 0;
for (const { id: tenantId } of tenants) {
const domains = await withTenantRLS(db, tenantId, (tx) =>
tx.tenantSendingDomain.findMany({ where: { tenantId, archivedAt: null, verificationStatus: { in: ["pending", "verifying"] } } }),
);
for (const domain of domains) {
probed += 1;
const dns = await resolver({ domain: domain.domain, dkimSelector: domain.dkimSelector });
const firstProbeAt = domain.firstProbeAt ?? now;
const decision = decideDomainProbeStatus({ dns, firstProbeAt, now });
const changed = decision.status !== domain.verificationStatus;
await withTenantRLS(db, tenantId, (tx) =>
tx.tenantSendingDomain.update({
where: { id: domain.id },
data: { verificationStatus: decision.status, lastError: decision.lastError, lastProbedAt: now, firstProbeAt },
}),
);
if (changed && (decision.status === "verified" || decision.status === "failed")) {
if (decision.status === "verified") verified += 1;
else failed += 1;
await notify(
{
tenantId,
channel: "IN_APP",
category: "email",
messageKey: decision.status === "verified" ? "email.domain.notify.verified" : "email.domain.notify.failed",
sourceKey: `email-domain-${decision.status}:${domain.id}`,
payloadMetadata: { domainId: domain.id, domain: domain.domain },
},
db as unknown as NotificationClient,
).catch((err) => logger.error({ task: payload.task, domainId: domain.id, err: err instanceof Error ? err.message : String(err) }, "email.domain-probe.notify-failed"));
}
}
}
logger.info({ task: payload.task, tenants: tenants.length, probed, verified, failed }, "email.domain-probe.completed");
return { handled: true, action: "email.domain-probe.completed", details: { task: payload.task, tenants: tenants.length, probed, verified, failed } };
}
/**
* Phase 10 #68 — maintenance-handler adapter for the DSGVO grace-period
* scan. Wraps `runDsgvoGraceCron` (the pure unit-tested core) with the
* production HandlerDependencies surface.
*
* Why a separate adapter: `runDsgvoGraceCron` is intentionally
* host-agnostic (no BullMQ, no Prisma). The maintenance queue receives
* a CleanupJob discriminator and this adapter is what binds the cron
* to the worker's logger + Prisma-backed candidate-reader + cascade
* dispatcher (via `deps.dsgvoGraceCronService`).
*
* When `deps.dsgvoGraceCronService` is not configured we still need to
* acknowledge the job (otherwise BullMQ retries forever and we are
* back to the UnsupportedWorkerJobError loop that motivated this fix).
* In that case we log a single warning and return `scanned=0`. The
* follow-up phase wires the real Prisma reader + cascade dispatcher.
*/
async function runDsgvoGracePeriodScanHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
const logger = deps.logger ?? noopHandlerLogger;
const service = deps.dsgvoGraceCronService;
if (!service) {
// Deliberately not throwing: we want the cron to ack so BullMQ does
// not retry; the missing service is a deployment-config gap, not a
// payload-shape bug. Emit an explicit log for operators.
logger.info({ task: payload.task, requestId: payload.metadata?.requestId ?? null }, "dsgvo-grace-cron.skipped: dsgvoGraceCronService not configured");
return {
handled: true,
action: "dsgvo.grace-period.skipped",
details: { task: payload.task, reason: "dsgvoGraceCronService not configured", scanned: 0, dispatched: 0, failed: 0 },
};
}
const result = await runDsgvoGraceCron({
listOverdueUsers: (now: Date): Promise<DsgvoGraceCandidate[]> => service.listOverdueUsers(now),
enqueueDeletion: (input) => service.enqueueDeletion(input),
logger,
});
return {
handled: true,
action: "dsgvo.grace-period.scanned",
details: {
task: payload.task,
scanned: result.scanned,
dispatched: result.dispatched,
failed: result.failed,
// `failures` contains userIds — keep them so audit can correlate
// but redact in case any future error message embeds secrets.
failures: result.failures.map((f) => ({ userId: f.userId, error: redactSecretLikeText(f.error) })),
},
};
}
const noopHandlerLogger = {
info: () => undefined,
error: () => undefined,
} satisfies NonNullable<HandlerDependencies["logger"]>;
async function runExportHandler(
payload: QueuePayload<typeof queueNames.exports>,
deps: HandlerDependencies,
context?: HandlerExecutionContext<typeof queueNames.exports>
): Promise<HandlerResult> {
if (payload.kind !== "export.run") {
throw new ExportHandlerError("Unsupported export payload kind.", {
code: "invalid_payload",
message: "Unsupported export payload kind.",
});
}
if (isPrivacyGdprExportType(payload.exportType)) {
return runPrivacyGdprExportHandler(payload, deps, context);
}
const markPaused = deps.markExportJobPaused ?? markExportJobPausedInDb;
const markResumed = deps.markExportJobResumed ?? markExportJobResumedInDb;
const markRunning = deps.markExportJobRunning ?? markExportJobRunningInDb;
const markSucceeded = deps.markExportJobSucceeded ?? markExportJobSucceededInDb;
const markFailed = deps.markExportJobFailed ?? markExportJobFailedInDb;
const attemptsMade = readAttemptsMade(payload.metadata, context);
const complianceGate = payload.complianceGate;
if (complianceGate?.state === "paused") {
const pauseReason = redactSecretLikeText(complianceGate.reason ?? "Export paused by compliance gate.");
await markPaused({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade, reason: pauseReason });
return {
handled: true,
action: "export.paused",
details: buildExportDetails(payload, {
attemptsMade,
pauseReason,
pauseRequestId: complianceGate.requestId,
status: "paused",
}),
};
}
if (complianceGate?.state === "active" || complianceGate?.resumedAt) {
await markResumed({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade });
}
await markRunning({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade });
if (!deps.exportService) {
const message = "Export service is not configured.";
await markFailed({
id: payload.exportJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName: "ExportHandlerError",
errorMessage: message,
});
throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "export_service_missing" }));
}
try {
const result = await deps.exportService.runExport(payload);
await markSucceeded({
id: payload.exportJobId,
tenantId: payload.tenantId,
artifactBucket: result.artifactBucket,
artifactKey: result.artifactKey,
rowCount: result.rowCount,
attemptsMade,
expiresAt: result.expiresAt,
});
return {
handled: true,
action: "export.completed",
details: buildExportDetails(payload, {
artifactBucket: result.artifactBucket,
artifactKey: result.artifactKey,
attemptsMade,
expiresAt: result.expiresAt.toISOString(),
rowCount: result.rowCount,
status: "completed",
}),
};
} catch (error) {
const errorName = error instanceof Error ? error.name : "Error";
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
if (isTerminalJobAttempt(attemptsMade, context)) {
await markFailed({
id: payload.exportJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName,
errorMessage,
});
}
throw new ExportHandlerError("Export job failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }));
}
}
async function runPrivacyGdprExportHandler(
payload: ExportJob,
deps: HandlerDependencies,
context?: HandlerExecutionContext<typeof queueNames.exports>
): Promise<HandlerResult> {
const attemptsMade = readAttemptsMade(payload.metadata, context);
const markProcessing = deps.markGdprExportProcessing ?? markGdprExportProcessingInDb;
const markReady = deps.markGdprExportReady ?? markGdprExportReadyInDb;
const markFailed = deps.markGdprExportFailed ?? markGdprExportFailedInDb;
await markProcessing({ id: payload.exportJobId, tenantId: payload.tenantId });
if (!deps.privacyGdprExportService) {
const message = "Privacy GDPR export service is not configured.";
await markFailed({
id: payload.exportJobId,
tenantId: payload.tenantId,
errorMessage: message,
});
throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "privacy_gdpr_export_service_missing" }));
}
try {
const result = await deps.privacyGdprExportService.runExport(payload);
await markReady({
id: payload.exportJobId,
tenantId: payload.tenantId,
resultBucket: result.artifactBucket,
resultKey: result.artifactKey,
resultSizeBytes: result.sizeBytes,
resultExpiresAt: result.expiresAt,
});
return {
handled: true,
action: "privacy-gdpr-export.completed",
details: buildExportDetails(payload, {
artifactBucket: result.artifactBucket,
artifactKey: result.artifactKey,
attemptsMade,
expiresAt: result.expiresAt.toISOString(),
rowCount: result.rowCount,
sizeBytes: result.sizeBytes,
status: "ready",
}),
};
} catch (error) {
const errorName = error instanceof Error ? error.name : "Error";
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
if (isTerminalJobAttempt(attemptsMade, context)) {
await markFailed({
id: payload.exportJobId,
tenantId: payload.tenantId,
errorMessage,
});
}
throw new ExportHandlerError("Privacy GDPR export job failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }));
}
}
async function runImportHandler(
payload: QueuePayload<typeof queueNames.imports>,
deps: HandlerDependencies,
context?: HandlerExecutionContext<typeof queueNames.imports>
): Promise<HandlerResult> {
if (payload.kind !== "import.run") {
throw new ImportHandlerError("Unsupported import payload kind.", {
code: "invalid_payload",
message: "Unsupported import payload kind.",
});
}
const markRunning = deps.markImportJobRunning ?? markImportJobRunningInDb;
const markSucceeded = deps.markImportJobSucceeded ?? markImportJobSucceededInDb;
const markFailed = deps.markImportJobFailed ?? markImportJobFailedInDb;
const attemptsMade = readAttemptsMade(payload.metadata, context);
await markRunning({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade });
const accessDecision = authorizeStorageObjectAccess({
tenantId: payload.tenantId,
key: payload.sourceKey,
operation: "download",
});
if (!accessDecision.allowed) {
const message = "Import source storage access denied.";
await markFailed({
id: payload.importJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName: "ImportHandlerError",
errorMessage: `${message} reason=${accessDecision.reason}`,
});
throw new ImportHandlerError(
message,
buildImportDetails(payload, { status: "failed", attemptsMade, code: "source_access_denied", reason: accessDecision.reason, message })
);
}
if (!deps.importService) {
const message = "Import service is not configured.";
await markFailed({
id: payload.importJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName: "ImportHandlerError",
errorMessage: message,
});
throw new ImportHandlerError(message, buildImportDetails(payload, { status: "failed", attemptsMade, code: "import_service_missing" }));
}
if (!deps.scanService) {
const details = buildImportScanFailureDetails(payload, attemptsMade, {
verdict: "error",
code: "scan_service_missing",
message: "Import scan service is not configured.",
});
await markFailed({
id: payload.importJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName: "ImportHandlerError",
errorMessage: buildImportScanFailureMessage(details),
});
throw new ImportHandlerError("Import source scan failed.", details);
}
let scanResult: StorageScanResult;
try {
scanResult = await deps.scanService.scanObject({
bucket: payload.sourceBucket,
key: accessDecision.key,
tenantId: payload.tenantId,
uploadId: payload.importJobId,
contentType: payload.contentType,
metadata: {
...payload.metadata,
requestId: payload.requestId,
importJobId: payload.importJobId,
importType: payload.importType,
format: payload.format,
},
});
} catch (error) {
const details = buildImportScanFailureDetails(payload, attemptsMade, {
verdict: "error",
code: "scanner_rejected",
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
});
if (isTerminalJobAttempt(attemptsMade, context)) {
await markFailed({
id: payload.importJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName: "ImportHandlerError",
errorMessage: buildImportScanFailureMessage(details),
});
}
throw new ImportHandlerError("Import source scan failed.", details);
}
if (scanResult.verdict !== "clean") {
const details = buildImportScanFailureDetails(payload, attemptsMade, buildImportScanResultDetails(scanResult));
await markFailed({
id: payload.importJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName: "ImportHandlerError",
errorMessage: buildImportScanFailureMessage(details),
});
throw new ImportHandlerError("Import source scan failed.", details);
}
try {
const result = await deps.importService.runImport(payload);
await markSucceeded({
id: payload.importJobId,
tenantId: payload.tenantId,
rowCount: result.rowCount,
acceptedRows: result.acceptedRows,
rejectedRows: result.rejectedRows,
attemptsMade,
});
return {
handled: true,
action: "import.completed",
details: buildImportDetails(payload, {
status: "completed",
attemptsMade,
rowCount: result.rowCount,
acceptedRows: result.acceptedRows,
rejectedRows: result.rejectedRows,
duplicateCount: result.duplicateCount,
errorReportBucket: result.errorReportBucket,
errorReportKey: result.errorReportKey ?? result.validationReport?.errorReport?.errorReportKey,
}),
};
} catch (error) {
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
if (isTerminalJobAttempt(attemptsMade, context)) {
await markFailed({
id: payload.importJobId,
tenantId: payload.tenantId,
attemptsMade,
errorName: error instanceof Error ? error.name : "Error",
errorMessage,
});
}
throw new ImportHandlerError("Import job failed.", buildImportDetails(payload, { status: "failed", attemptsMade, message: errorMessage }));
}
}
function readAttemptsMade(metadata: Record<string, unknown> | undefined, context?: { attemptsMade?: number }): number {
if (context?.attemptsMade !== undefined) {
return Math.max(1, context.attemptsMade + 1);
}
return typeof metadata?.attemptsMade === "number" && Number.isFinite(metadata.attemptsMade) ? metadata.attemptsMade : 0;
}
function isTerminalJobAttempt(attemptsMade: number, context?: { attemptsLimit?: number }): boolean {
if (context?.attemptsLimit === undefined) {
return true;
}
return attemptsMade >= Math.max(1, context.attemptsLimit);
}
function buildExportDetails(payload: ExportJob, details: Record<string, unknown>): Record<string, unknown> {
return {
exportJobId: payload.exportJobId,
tenantId: payload.tenantId,
exportType: payload.exportType,
format: payload.format,
requestId: payload.requestId,
...details,
};
}
function buildImportDetails(payload: ImportJob, details: Record<string, unknown>): Record<string, unknown> {
return {
importJobId: payload.importJobId,
tenantId: payload.tenantId,
importType: payload.importType,
format: payload.format,
requestId: payload.requestId,
...details,
};
}
function buildScanResultDetails(result: StorageScanResult, uploadId: string | undefined): Record<string, unknown> {
const base = {
verdict: result.verdict,
bucket: result.bucket,
key: result.key,
uploadId,
};
if (result.verdict === "clean") {
return {
...base,
engine: result.engine,
};
}
if (result.verdict === "infected") {
return {
...base,
engine: result.engine,
signature: result.signature,
quarantineKey: result.quarantineKey,
};
}
return {
...base,
...buildRedactedErrorDetails(result.error),
};
}
function buildImportScanResultDetails(result: StorageScanResult): Record<string, unknown> {
if (result.verdict === "clean") {
return {
verdict: result.verdict,
engine: result.engine,
};
}
if (result.verdict === "infected") {
return {
verdict: result.verdict,
code: "source_infected",
engine: result.engine,
signature: result.signature,
quarantineKey: result.quarantineKey,
message: "Import source object failed malware scan.",
};
}
return {
verdict: result.verdict,
...buildRedactedErrorDetails(result.error),
};
}
function buildImportScanFailureDetails(payload: ImportJob, attemptsMade: number, scanDetails: Record<string, unknown>): Record<string, unknown> {
return buildImportDetails(payload, {
status: "failed",
attemptsMade,
scan: scanDetails,
});
}
function buildImportScanFailureMessage(details: Record<string, unknown>): string {
const scan = details.scan && typeof details.scan === "object" ? (details.scan as Record<string, unknown>) : {};
const parts = ["Import source scan failed.", `verdict=${String(scan.verdict ?? "error")}`, `code=${String(scan.code ?? "scan_failed")}`];
const requestId = typeof details.requestId === "string" ? details.requestId : undefined;
if (requestId) {
parts.push(`requestId=${requestId}`);
}
const message = typeof scan.message === "string" ? scan.message : undefined;
if (message) {
parts.push(`message=${redactSecretLikeText(message)}`);
}
return parts.join(" ");
}
function buildRejectedScanDetails(payload: QueuePayload<typeof queueNames.fileScan>, error: unknown): Record<string, unknown> {
return {
verdict: "error",
bucket: payload.bucket,
key: payload.key,
uploadId: payload.uploadId,
code: "scanner_rejected",
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
};
}
function buildRedactedErrorDetails(error: StorageScanDiagnostic): Record<string, unknown> {
return {
code: error.code,
message: redactSecretLikeText(error.message),
upstream: error.upstream
? {
name: error.upstream.name,
message: error.upstream.message ? redactSecretLikeText(error.upstream.message) : undefined,
status: error.upstream.status,
requestId: error.upstream.requestId,
}
: undefined,
};
}
function redactSecretLikeText(message: string): string {
return message
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
.replace(/(access[_-]?key(?:id)?|secret[_-]?access[_-]?key|session[_-]?token|credential|authorization|token|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
.replace(/(body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]");
}
function serializeWebhookResponseDiagnostics(response: Response, body: string) {
return JSON.stringify({
status: response.status,
statusText: response.statusText,
contentType: response.headers.get("content-type"),
bodyBytes: new TextEncoder().encode(body).byteLength,
});
}
async function fetchWebhookWithRedirects(
fetchImpl: typeof fetch | undefined,
input: string,
init: RequestInit,
addressResolver?: OutboundHttpAddressResolver
): Promise<Response> {
try {
return await fetchOutboundHttpWithRedirects(input, init, {
// In production fetchImpl is undefined, so the guard uses its pinned
// undici fetch; a test-injected fetchImpl disables pinning.
...(fetchImpl ? { fetchImpl } : {}),
...(addressResolver ? { addressResolver } : {}),
timeoutMs: DEFAULT_WORKER_FETCH_TIMEOUT_MS,
redirectLimit: WEBHOOK_REDIRECT_LIMIT,
});
} catch (error) {
if (error instanceof OutboundHttpUrlValidationError) {
throw new WebhookUrlValidationError(toWebhookUrlValidationMessage(error.message));
}
throw error;
}
}
function toWebhookUrlValidationMessage(message: string): string {
return message.replace(/^Outbound HTTP endpoint/u, "Webhook endpoint").replace(/^Outbound HTTP redirect/u, "Webhook redirect");
}
async function readLimitedResponseText(response: Response, limitBytes: number): Promise<string> {
if (!response.body) {
return "";
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
const chunks: string[] = [];
let bytes = 0;
while (bytes < limitBytes) {
const result = await reader.read();
if (result.done) {
chunks.push(decoder.decode());
return chunks.join("");
}
const chunk = result.value;
const remaining = limitBytes - bytes;
const slice = chunk.byteLength > remaining ? chunk.slice(0, remaining) : chunk;
chunks.push(decoder.decode(slice, { stream: chunk.byteLength <= remaining }));
bytes += slice.byteLength;
if (chunk.byteLength > remaining) {
await reader.cancel();
break;
}
}
chunks.push(decoder.decode());
return chunks.join("");
}
export type CreateRegisteredProcessorOptions = {
dependencies?: HandlerDependencies;
registry?: HandlerRegistry;
};
export function createRegisteredProcessor<Name extends QueueName>(queueName: Name, options: CreateRegisteredProcessorOptions = {}): TypedProcessor<Name> {
const dependencies = options.dependencies ?? {};
const registry = options.registry ?? buildHandlerRegistry(dependencies);
return async (job) => {
return registry[queueName](job.data, dependencies, {
attemptsMade: job.attemptsMade,
attemptsLimit: job.opts?.attempts,
jobId: job.id === undefined ? undefined : String(job.id),
updatePayload: job.updateData
? async (payload) => {
await job.updateData?.(payload);
}
: undefined,
});
};
}