2424 lines
97 KiB
TypeScript
2424 lines
97 KiB
TypeScript
import {
|
|
updateLegacyEventWebhookAttempt as updateLegacyEventWebhookAttemptInDb,
|
|
type LegacyEventWebhookAttemptUpdate,
|
|
} from "@hivegrid/db/event-webhook-attempts.server";
|
|
import {
|
|
listExpiredCompletedExportJobs as listExpiredCompletedExportJobsInDb,
|
|
listSoftDeletedExportJobsForHardDelete as listSoftDeletedExportJobsForHardDeleteInDb,
|
|
markExportJobExpired as markExportJobExpiredInDb,
|
|
markExportJobFailed as markExportJobFailedInDb,
|
|
markExportJobHardDeleteFailed as markExportJobHardDeleteFailedInDb,
|
|
markExportJobHardDeleteSucceeded as markExportJobHardDeleteSucceededInDb,
|
|
markExportJobPaused as markExportJobPausedInDb,
|
|
markExportJobResumed as markExportJobResumedInDb,
|
|
markExportJobRunning as markExportJobRunningInDb,
|
|
markExportJobSucceeded as markExportJobSucceededInDb,
|
|
type ExportJobExpiredInput,
|
|
type ExportJobFailedInput,
|
|
type ExportJobHardDeleteFailedInput,
|
|
type ExportJobHardDeleteSucceededInput,
|
|
type ExportJobPausedInput,
|
|
type ExportJobResumedInput,
|
|
type ExportJobRetentionListInput,
|
|
type ExportJobRetentionRecord,
|
|
type ExportJobRunningInput,
|
|
type ExportJobSucceededInput,
|
|
} from "@hivegrid/db/export-jobs.server";
|
|
import {
|
|
markGdprExportFailed as markGdprExportFailedInDb,
|
|
markGdprExportProcessing as markGdprExportProcessingInDb,
|
|
markGdprExportReady as markGdprExportReadyInDb,
|
|
type MarkGdprExportFailedInput,
|
|
type MarkGdprExportProcessingInput,
|
|
type MarkGdprExportReadyInput,
|
|
} from "@hivegrid/db/privacy-gdpr.server";
|
|
import {
|
|
markImportJobFailed as markImportJobFailedInDb,
|
|
markImportJobRunning as markImportJobRunningInDb,
|
|
markImportJobSucceeded as markImportJobSucceededInDb,
|
|
type ImportJobFailedInput,
|
|
type ImportJobRunningInput,
|
|
type ImportJobSucceededInput,
|
|
} from "@hivegrid/db/import-jobs.server";
|
|
import {
|
|
createNotificationDelivery as createNotificationDeliveryInDb,
|
|
getUserNotificationPreferences as getUserNotificationPreferencesInDb,
|
|
markNotificationDeliveryStatus as markNotificationDeliveryStatusInDb,
|
|
redactNotificationDiagnosticText,
|
|
type CreateNotificationDeliveryInput,
|
|
type MarkNotificationDeliveryStatusInput,
|
|
} from "@hivegrid/db/notifications.server";
|
|
import {
|
|
deleteDoneOutboxEvents as deleteDoneOutboxEventsInDb,
|
|
type DeleteDoneOutboxEventsOptions,
|
|
type DeleteDoneOutboxEventsResult,
|
|
} from "@hivegrid/db/outbox-retention";
|
|
import { type EnvRecord } from "@hivegrid/config";
|
|
import type { ValidationReport } from "@hivegrid/imports";
|
|
import { createMailjetProviderFromEnv, type MailMetricsHook, type MailProvider } from "@hivegrid/mail";
|
|
import {
|
|
queueNames,
|
|
type ExportJob,
|
|
type FilePreviewJob,
|
|
type ImportJob,
|
|
type PluginInvokeJob,
|
|
type QueueName,
|
|
type QueuePayload,
|
|
type TypedProcessor,
|
|
} from "@hivegrid/queue";
|
|
import {
|
|
authorizeStorageObjectAccess,
|
|
type DeleteObjectInput,
|
|
type DeleteObjectResult,
|
|
type StorageScanDiagnostic,
|
|
type StorageScanResult,
|
|
} from "@hivegrid/storage";
|
|
import { mapWithConcurrency, notify, type NotificationClient } from "@hivegrid/foundation";
|
|
import { OutboundHttpUrlValidationError, fetchOutboundHttpWithRedirects, type OutboundHttpAddressResolver } from "@hivegrid/foundation/outbound-http";
|
|
|
|
import {
|
|
drainCalendarWebhookEventsForTenant,
|
|
refreshCalendarTokensForTenant,
|
|
renewCalendarWebhooksForTenant,
|
|
syncInternalCalendarEventsForTenant,
|
|
} from "@hivegrid/integrations/calendar-worker-tasks.server";
|
|
import {
|
|
cleanupEmailWebhookEventsForTenant,
|
|
drainEmailWebhookEventsForTenant,
|
|
pollStaleEmailMailboxesForTenant,
|
|
refreshEmailTokensForTenant,
|
|
renewEmailWebhooksForTenant,
|
|
runEmailInitialSyncForTenant,
|
|
} from "@hivegrid/integrations/email-worker-tasks.server";
|
|
import { createEmailAutoMatchHook } from "@hivegrid/integrations/email-match-hook.server";
|
|
import { refreshExpiringTokensForTenant, sweepConnectionHealthForTenant } from "@hivegrid/integrations/worker-tasks.server";
|
|
import { withTenantRLS } from "./db/tenantRls.js";
|
|
import { runDsgvoGraceCron, type DsgvoGraceCandidate } from "./jobs/dsgvo-grace-cron.js";
|
|
import { reminderChannel, reminderMessageKey, reminderScanBounds, reminderSourceKey, selectReminderWindows } from "./jobs/task-reminder-cron.js";
|
|
import { audienceMatches, nextReminderDue, HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS } from "./jobs/hr-upload-reminder-cron.js";
|
|
import { createDnsDomainResolver, decideDomainProbeStatus, type DomainDnsResolver } from "./jobs/email-domain-probe-cron.js";
|
|
import { handlePluginInvokeJob, type PluginInvokeDeps } from "./jobs/plugin-invoke.js";
|
|
import {
|
|
reconcileWorkflowExecutionStalls,
|
|
type WorkflowExecutionStalledReconcileInput,
|
|
type WorkflowExecutionStalledReconcileResult,
|
|
type WorkflowExecutionStalledReconcilerPrisma,
|
|
} from "./jobs/workflow-execution-stalled-reconciler.js";
|
|
import {
|
|
isPrivacyGdprExportType,
|
|
type PrivacyGdprExportRunResult,
|
|
} from "./jobs/privacy-gdpr-export.js";
|
|
|
|
const DEFAULT_WORKER_FETCH_TIMEOUT_MS = 10_000;
|
|
const WEBHOOK_RESPONSE_BODY_LIMIT_BYTES = 4_096;
|
|
const WEBHOOK_REDIRECT_LIMIT = 5;
|
|
const EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY = 5;
|
|
|
|
export type HandlerResult = {
|
|
handled: true;
|
|
action: string;
|
|
details?: Record<string, unknown>;
|
|
};
|
|
|
|
export type WebhookDeliveryMetricsHook = (input: { status: "success" | "retry" | "dlq" | "rejected" }) => void;
|
|
|
|
export type HandlerDependencies = {
|
|
mailProvider?: MailProvider;
|
|
/**
|
|
* Optional metrics hook forwarded to the lazily-constructed Mailjet
|
|
* provider. When the worker entry wires this to the shared Prometheus
|
|
* registry, every send increments mail_sent_total{provider, status}.
|
|
*/
|
|
mailMetricsHook?: MailMetricsHook;
|
|
/**
|
|
* Optional metrics hook fired for each outbound webhook attempt. Status:
|
|
* - success : 2xx response
|
|
* - retry : retryable failure (5xx, transient network) below the
|
|
* attempts cap, BullMQ will requeue
|
|
* - dlq : terminal failure (final attempt or non-retryable status)
|
|
* - rejected : URL validation failure (SSRF guard, etc.)
|
|
*/
|
|
webhookDeliveryMetricsHook?: WebhookDeliveryMetricsHook;
|
|
webhookAddressResolver?: OutboundHttpAddressResolver;
|
|
fetch?: typeof fetch;
|
|
env?: EnvRecord;
|
|
updateLegacyEventWebhookAttempt?: (attempt: { id: string; tenantId: string | null }, data: LegacyEventWebhookAttemptUpdate) => Promise<void>;
|
|
scanService?: {
|
|
scanObject(target: WorkerStorageScanTarget): Promise<StorageScanResult>;
|
|
};
|
|
/**
|
|
* AK-P4a §5 — writes the scan verdict (clean|infected) back onto
|
|
* File.scanStatus so the fail-closed media proxy (AD-23) can serve the
|
|
* object. Composed in the `general` group bootstrap (jobs/file-scan-status.ts)
|
|
* with a Prisma seam. When unset (or the job carries no fileId) the handler
|
|
* skips the write-back — e.g. legacy import-path scans with no File row.
|
|
*/
|
|
fileScanStatusWriter?: {
|
|
recordVerdict(input: { fileId: string; tenantId?: string; verdict: "clean" | "infected" }): Promise<void>;
|
|
};
|
|
/**
|
|
* AK-P4a §4 — file-preview generator. Composed in the `general` worker
|
|
* group bootstrap (jobs/file-preview.ts) with Prisma + storage seams.
|
|
* The handler stays thin: it delegates to this service, which loads the
|
|
* object, renders the preview (HEIC->JPG, image->WebP, PDF->PNG) and
|
|
* writes previewStatus + preview/thumbnail keys back to the File row.
|
|
* When unset, the handler throws so the job dead-letters loudly rather
|
|
* than silently leaving previewStatus stuck on `generating`.
|
|
*/
|
|
filePreviewService?: {
|
|
generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult>;
|
|
};
|
|
exportService?: {
|
|
runExport(payload: ExportJob): Promise<WorkerExportRunResult>;
|
|
};
|
|
privacyGdprExportService?: {
|
|
runExport(payload: ExportJob): Promise<PrivacyGdprExportRunResult>;
|
|
};
|
|
importService?: {
|
|
runImport(payload: ImportJob): Promise<WorkerImportRunResult>;
|
|
};
|
|
storageDeleteService?: {
|
|
deleteObject(input: DeleteObjectInput): Promise<DeleteObjectResult>;
|
|
};
|
|
listSoftDeletedExportJobsForHardDelete?: (data: ExportJobRetentionListInput) => Promise<ExportJobRetentionRecord[]>;
|
|
listExpiredCompletedExportJobs?: (data: ExportJobRetentionListInput) => Promise<ExportJobRetentionRecord[]>;
|
|
markExportJobExpired?: (data: ExportJobExpiredInput) => Promise<unknown>;
|
|
markExportJobHardDeleteSucceeded?: (data: ExportJobHardDeleteSucceededInput) => Promise<unknown>;
|
|
markExportJobHardDeleteFailed?: (data: ExportJobHardDeleteFailedInput) => Promise<unknown>;
|
|
markExportJobPaused?: (data: ExportJobPausedInput) => Promise<unknown>;
|
|
markExportJobResumed?: (data: ExportJobResumedInput) => Promise<unknown>;
|
|
markExportJobRunning?: (data: ExportJobRunningInput) => Promise<unknown>;
|
|
markExportJobSucceeded?: (data: ExportJobSucceededInput) => Promise<unknown>;
|
|
markExportJobFailed?: (data: ExportJobFailedInput) => Promise<unknown>;
|
|
markGdprExportProcessing?: (data: MarkGdprExportProcessingInput) => Promise<unknown>;
|
|
markGdprExportReady?: (data: MarkGdprExportReadyInput) => Promise<unknown>;
|
|
markGdprExportFailed?: (data: MarkGdprExportFailedInput) => Promise<unknown>;
|
|
markImportJobRunning?: (data: ImportJobRunningInput) => Promise<unknown>;
|
|
markImportJobSucceeded?: (data: ImportJobSucceededInput) => Promise<unknown>;
|
|
markImportJobFailed?: (data: ImportJobFailedInput) => Promise<unknown>;
|
|
getUserNotificationPreferences?: (filters: { tenantId: string; userId: string; channel?: string | null; category?: string | null }) => Promise<unknown[]>;
|
|
createNotificationDelivery?: (data: CreateNotificationDeliveryInput) => Promise<unknown>;
|
|
markNotificationDeliveryStatus?: (data: MarkNotificationDeliveryStatusInput) => Promise<unknown>;
|
|
/**
|
|
* Phase 10 — DSGVO grace-period cron service. The maintenance handler
|
|
* looks up overdue User rows and dispatches cascade-deletions through
|
|
* this slot. When unset, the handler falls back to a Prisma-backed
|
|
* implementation that lists candidates and logs them without running
|
|
* the destructive cascade — the cascade-runtime wiring lands in a
|
|
* follow-up. The split keeps the cron observable from day-one without
|
|
* blocking on the full cascade implementation.
|
|
*/
|
|
dsgvoGraceCronService?: {
|
|
listOverdueUsers: (now: Date) => Promise<DsgvoGraceCandidate[]>;
|
|
enqueueDeletion: (input: {
|
|
userId: string;
|
|
tenantId: string | null;
|
|
requestId: string;
|
|
actorType: "system";
|
|
actorId: string;
|
|
mode: "delete";
|
|
reason: string;
|
|
}) => Promise<{ jobId?: string }>;
|
|
};
|
|
deleteDoneOutboxEvents?: (data: DeleteDoneOutboxEventsOptions) => Promise<DeleteDoneOutboxEventsResult>;
|
|
reconcileWorkflowExecutionStalls?: (data: WorkflowExecutionStalledReconcileInput) => Promise<WorkflowExecutionStalledReconcileResult>;
|
|
/**
|
|
* Phase 06 #21 / F-015 — production dependencies for the plugin-invoke
|
|
* handler. The `plugin` worker group bootstrap (`apps/worker/src/jobs/
|
|
* plugin-invoke-bootstrap.ts`) composes the real Prisma / filesystem /
|
|
* SharedStore / sandbox seams and injects them here. When the slot is
|
|
* unset (e.g. a misconfigured `general` worker receives a plugin-invoke
|
|
* job because someone changed group routing) the registry entry throws
|
|
* so the job dead-letters loudly rather than being silently acked.
|
|
*/
|
|
pluginInvokeDeps?: PluginInvokeDeps;
|
|
/**
|
|
* Optional structured logger used by handlers that need to emit
|
|
* non-error progress (currently only the dsgvo-grace-cron branch). A
|
|
* pino-shaped surface keeps this compatible with the worker bootstrap
|
|
* logger without dragging the pino type-graph into this module.
|
|
*/
|
|
logger?: {
|
|
info: (fields: Record<string, unknown>, msg: string) => void;
|
|
error: (fields: Record<string, unknown>, msg: string) => void;
|
|
};
|
|
};
|
|
|
|
type WorkerExportRunResult = {
|
|
artifactBucket: string;
|
|
artifactKey: string;
|
|
rowCount: number;
|
|
expiresAt: Date;
|
|
};
|
|
|
|
type WorkerImportRunResult = {
|
|
rowCount: number;
|
|
acceptedRows: number;
|
|
rejectedRows: number;
|
|
duplicateCount: number;
|
|
validationReport?: Pick<ValidationReport, "summary" | "errorReport">;
|
|
errorReportBucket?: string;
|
|
errorReportKey?: string;
|
|
};
|
|
|
|
type WorkerStorageScanTarget = {
|
|
bucket: string;
|
|
key: string;
|
|
tenantId?: string;
|
|
uploadId?: string;
|
|
contentType?: string;
|
|
metadata?: Record<string, unknown>;
|
|
};
|
|
|
|
export type FilePreviewServiceResult = {
|
|
/** Terminal preview state written to the File row by the service. */
|
|
previewStatus: "ready" | "failed" | "not_supported";
|
|
previewStorageKey?: string;
|
|
thumbnailStorageKey?: string;
|
|
/** Optional diagnostic reason (e.g. why generation failed). */
|
|
reason?: string;
|
|
};
|
|
|
|
class WebhookDeliveryError extends Error {}
|
|
class WebhookUrlValidationError extends Error {}
|
|
|
|
class NotificationHandlerError extends Error {
|
|
readonly action = "notification.rejected";
|
|
}
|
|
|
|
class UnsupportedWorkerJobError extends Error {
|
|
readonly action = "worker.unsupported";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "UnsupportedWorkerJobError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
class PluginInvokePayloadError extends Error {
|
|
readonly action = "plugin.invoke.invalid_payload";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "PluginInvokePayloadError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
function getNotificationDeliveryId(record: unknown): string | undefined {
|
|
return record && typeof record === "object" && typeof (record as { id?: unknown }).id === "string" ? (record as { id: string }).id : undefined;
|
|
}
|
|
|
|
async function createSkippedNotificationDelivery(
|
|
createNotificationDelivery: NonNullable<HandlerDependencies["createNotificationDelivery"]>,
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
payloadMetadata: Record<string, string>,
|
|
reason: string,
|
|
sourceKey?: string
|
|
) {
|
|
return createNotificationDelivery({
|
|
tenantId: payload.tenantId ?? "",
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey,
|
|
status: "SKIPPED",
|
|
attemptsMade: 0,
|
|
lastError: reason,
|
|
payloadMetadata: { ...payloadMetadata, reason },
|
|
});
|
|
}
|
|
|
|
function isNotificationPreferenceDisabled(preferences: unknown[]): boolean {
|
|
return preferences.some((preference) => preference && typeof preference === "object" && (preference as { enabled?: unknown }).enabled === false);
|
|
}
|
|
|
|
function getNotificationErrorMessage(error: unknown): string {
|
|
return redactNotificationDiagnosticText(error instanceof Error ? error.message : "Notification email delivery failed.");
|
|
}
|
|
|
|
function getPreferenceChannel(channel: string): "email" | "in-app" {
|
|
if (channel === "email" || channel === "in-app") {
|
|
return channel;
|
|
}
|
|
|
|
throw new NotificationHandlerError(`Unsupported notification channel: ${channel}`);
|
|
}
|
|
|
|
async function getNotificationPreferencesForPayload(
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
getUserNotificationPreferences: NonNullable<HandlerDependencies["getUserNotificationPreferences"]>
|
|
): Promise<unknown[]> {
|
|
if (!payload.tenantId) {
|
|
return [];
|
|
}
|
|
|
|
const channel = getPreferenceChannel(payload.channel);
|
|
const category = payload.category ?? "default";
|
|
const preferences = await getUserNotificationPreferences({
|
|
tenantId: payload.tenantId,
|
|
userId: payload.userId,
|
|
channel,
|
|
category,
|
|
});
|
|
const defaultPreferences =
|
|
payload.category && payload.category !== "default"
|
|
? await getUserNotificationPreferences({
|
|
tenantId: payload.tenantId,
|
|
userId: payload.userId,
|
|
channel,
|
|
category: "default",
|
|
})
|
|
: [];
|
|
|
|
return preferences.length > 0 ? preferences : defaultPreferences;
|
|
}
|
|
|
|
export class ScanHandlerError extends Error {
|
|
readonly action = "scan.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "ScanHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
export class PreviewHandlerError extends Error {
|
|
readonly action = "preview.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "PreviewHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
export class ExportHandlerError extends Error {
|
|
readonly action = "export.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "ExportHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
export class ImportHandlerError extends Error {
|
|
readonly action = "import.error";
|
|
readonly details: Record<string, unknown>;
|
|
|
|
constructor(message: string, details: Record<string, unknown>) {
|
|
super(message);
|
|
this.name = "ImportHandlerError";
|
|
this.details = details;
|
|
}
|
|
}
|
|
|
|
type HandlerExecutionContext<Name extends QueueName = QueueName> = {
|
|
attemptsMade?: number;
|
|
attemptsLimit?: number;
|
|
jobId?: string;
|
|
updatePayload?: (payload: QueuePayload<Name>) => Promise<void>;
|
|
};
|
|
type Handler<Name extends QueueName> = (
|
|
payload: QueuePayload<Name>,
|
|
dependencies: HandlerDependencies,
|
|
context?: HandlerExecutionContext<Name>
|
|
) => Promise<HandlerResult>;
|
|
type HandlerRegistry = {
|
|
[Name in QueueName]: Handler<Name>;
|
|
};
|
|
|
|
function getAttemptNumber(context?: { attemptsMade?: number }): number {
|
|
return Math.max(1, (context?.attemptsMade ?? 0) + 1);
|
|
}
|
|
|
|
function getAttemptsLimit(context?: { attemptsLimit?: number }): number {
|
|
return Math.max(1, context?.attemptsLimit ?? 1);
|
|
}
|
|
|
|
function isRetryableWebhookStatus(status: number): boolean {
|
|
return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500;
|
|
}
|
|
|
|
function getPayloadNotificationDeliveryId(payload: QueuePayload<typeof queueNames.notifications>): string | undefined {
|
|
return typeof payload.metadata?.deliveryId === "string" ? payload.metadata.deliveryId : undefined;
|
|
}
|
|
|
|
function getNotificationDeliverySourceKey(
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
context: HandlerExecutionContext<typeof queueNames.notifications> | undefined
|
|
): string | undefined {
|
|
if (typeof payload.metadata?.idempotencyKey === "string" && payload.metadata.idempotencyKey.trim()) {
|
|
return payload.metadata.idempotencyKey.trim();
|
|
}
|
|
if (context?.jobId?.trim()) {
|
|
return `job:${context.jobId.trim()}`;
|
|
}
|
|
if (typeof payload.metadata?.requestId === "string" && payload.metadata.requestId.trim()) {
|
|
return `request:${payload.metadata.requestId.trim()}`;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
async function persistPayloadNotificationDeliveryId(
|
|
payload: QueuePayload<typeof queueNames.notifications>,
|
|
context: HandlerExecutionContext<typeof queueNames.notifications> | undefined,
|
|
deliveryId: string | undefined
|
|
): Promise<void> {
|
|
if (!deliveryId || getPayloadNotificationDeliveryId(payload) || !context?.updatePayload) {
|
|
return;
|
|
}
|
|
|
|
await context.updatePayload({
|
|
...payload,
|
|
metadata: {
|
|
...payload.metadata,
|
|
deliveryId,
|
|
},
|
|
});
|
|
}
|
|
|
|
export function buildHandlerRegistry(dependencies: HandlerDependencies = {}): HandlerRegistry {
|
|
const deps = dependencies;
|
|
|
|
return {
|
|
[queueNames.email]: async (payload, _dependencies, context) => {
|
|
if (payload.kind === "mail.transactional") {
|
|
const provider =
|
|
deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined);
|
|
const recipients = payload.to.map(addressToEmail);
|
|
const result = await provider.send({
|
|
from: payload.from,
|
|
to: payload.to,
|
|
cc: payload.cc,
|
|
bcc: payload.bcc,
|
|
replyTo: payload.replyTo,
|
|
subject: payload.subject,
|
|
text: payload.text,
|
|
html: payload.html,
|
|
templateId: payload.templateId,
|
|
attachments: payload.attachments,
|
|
tags: payload.tags,
|
|
precedenceBulk: payload.precedenceBulk,
|
|
sendAt: payload.sendAt,
|
|
inReplyTo: payload.inReplyTo,
|
|
references: payload.references,
|
|
settings: payload.settings,
|
|
headers: payload.headers,
|
|
listUnsubscribe: payload.listUnsubscribe,
|
|
...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}),
|
|
personalization: buildMailPersonalization(recipients, payload.templateData),
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "mail.sent",
|
|
details: { provider: result.provider, messageId: result.messageId },
|
|
};
|
|
}
|
|
|
|
// Only an injected (test) fetch is passed through; in production we leave
|
|
// it undefined so the outbound guard uses its own undici fetch with the
|
|
// DNS-rebinding IP pin engaged (see fetchWebhookWithRedirects).
|
|
const fetchImpl = deps.fetch;
|
|
const updateLegacyEventWebhookAttempt = deps.updateLegacyEventWebhookAttempt ?? updateLegacyEventWebhookAttemptInDb;
|
|
const legacyAttemptId = typeof payload.metadata?.legacyAttemptId === "string" ? payload.metadata.legacyAttemptId : undefined;
|
|
const legacyAttemptTenantId = typeof payload.metadata?.legacyAttemptTenantId === "string" ? payload.metadata.legacyAttemptTenantId : null;
|
|
const attemptNumber = getAttemptNumber(context);
|
|
const isTerminalFailure = attemptNumber >= getAttemptsLimit(context);
|
|
const emitWebhookMetric = (status: "success" | "retry" | "dlq" | "rejected") => {
|
|
if (!deps.webhookDeliveryMetricsHook) return;
|
|
try {
|
|
deps.webhookDeliveryMetricsHook({ status });
|
|
} catch {
|
|
// never block delivery
|
|
}
|
|
};
|
|
|
|
try {
|
|
const response = await fetchWebhookWithRedirects(
|
|
fetchImpl,
|
|
payload.url,
|
|
{
|
|
method: payload.method ?? "POST",
|
|
headers: {
|
|
"content-type": "application/json",
|
|
...payload.headers,
|
|
},
|
|
body: payload.body === undefined ? undefined : JSON.stringify(payload.body),
|
|
},
|
|
deps.webhookAddressResolver
|
|
);
|
|
const responseBody = await readLimitedResponseText(response, WEBHOOK_RESPONSE_BODY_LIMIT_BYTES);
|
|
const responseDiagnostics = serializeWebhookResponseDiagnostics(response, responseBody);
|
|
|
|
if (!response.ok) {
|
|
const message = `Webhook delivery failed with ${response.status}.`;
|
|
const retryableFailure = isRetryableWebhookStatus(response.status);
|
|
const shouldFinishAttempt = !retryableFailure || isTerminalFailure;
|
|
if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
...(shouldFinishAttempt ? { finishedAt: new Date(), success: false } : {}),
|
|
status: response.status,
|
|
message,
|
|
body: responseDiagnostics,
|
|
}
|
|
);
|
|
}
|
|
if (!retryableFailure) {
|
|
emitWebhookMetric("dlq");
|
|
return {
|
|
handled: true,
|
|
action: "webhook.failed",
|
|
details: { status: response.status, eventName: payload.eventName, retryable: false },
|
|
};
|
|
}
|
|
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
|
|
throw new WebhookDeliveryError(message);
|
|
}
|
|
|
|
if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
finishedAt: new Date(),
|
|
success: true,
|
|
status: response.status,
|
|
message: response.statusText,
|
|
body: responseDiagnostics,
|
|
}
|
|
);
|
|
}
|
|
|
|
emitWebhookMetric("success");
|
|
return {
|
|
handled: true,
|
|
action: "webhook.delivered",
|
|
details: { status: response.status, eventName: payload.eventName },
|
|
};
|
|
} catch (error) {
|
|
if (error instanceof WebhookUrlValidationError) {
|
|
const message = redactSecretLikeText(error.message);
|
|
if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
finishedAt: new Date(),
|
|
success: false,
|
|
message,
|
|
}
|
|
);
|
|
}
|
|
emitWebhookMetric("rejected");
|
|
return {
|
|
handled: true,
|
|
action: "webhook.rejected",
|
|
details: { eventName: payload.eventName, reason: message },
|
|
};
|
|
}
|
|
|
|
if (error instanceof WebhookDeliveryError) {
|
|
// Already counted at throw-site (retry / dlq).
|
|
} else if (legacyAttemptId) {
|
|
await updateLegacyEventWebhookAttempt(
|
|
{ id: legacyAttemptId, tenantId: legacyAttemptTenantId },
|
|
{
|
|
...(isTerminalFailure ? { finishedAt: new Date(), success: false } : {}),
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
}
|
|
);
|
|
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
|
|
} else if (!legacyAttemptId) {
|
|
emitWebhookMetric(isTerminalFailure ? "dlq" : "retry");
|
|
}
|
|
throw error;
|
|
}
|
|
},
|
|
[queueNames.notifications]: async (payload, _dependencies, context) => {
|
|
const getUserNotificationPreferences = deps.getUserNotificationPreferences ?? getUserNotificationPreferencesInDb;
|
|
const createNotificationDelivery = deps.createNotificationDelivery ?? createNotificationDeliveryInDb;
|
|
const markNotificationDeliveryStatus = deps.markNotificationDeliveryStatus ?? markNotificationDeliveryStatusInDb;
|
|
const payloadMetadata = {
|
|
...(typeof payload.metadata?.requestId === "string" ? { requestId: payload.metadata.requestId } : {}),
|
|
...(typeof payload.metadata?.source === "string" ? { source: payload.metadata.source } : {}),
|
|
...(typeof payload.metadata?.idempotencyKey === "string" ? { idempotencyKey: payload.metadata.idempotencyKey } : {}),
|
|
};
|
|
const sourceKey = getNotificationDeliverySourceKey(payload, context);
|
|
|
|
getPreferenceChannel(payload.channel);
|
|
if (!payload.tenantId) {
|
|
throw new NotificationHandlerError("Notification jobs require tenantId.");
|
|
}
|
|
|
|
if (payload.channel !== "email") {
|
|
const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences);
|
|
|
|
if (isNotificationPreferenceDisabled(preferences)) {
|
|
const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey);
|
|
return {
|
|
handled: true,
|
|
action: "notification.in-app.skipped",
|
|
details: {
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
messageKey: payload.messageKey,
|
|
deliveryId: getNotificationDeliveryId(delivery),
|
|
reason: "preference_disabled",
|
|
},
|
|
};
|
|
}
|
|
|
|
const delivery = await createNotificationDelivery({
|
|
tenantId: payload.tenantId ?? null,
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey,
|
|
status: "SENT",
|
|
attemptsMade: 1,
|
|
payloadMetadata,
|
|
sentAt: new Date(),
|
|
});
|
|
return {
|
|
handled: true,
|
|
action: "notification.accepted",
|
|
details: {
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
messageKey: payload.messageKey,
|
|
deliveryId: getNotificationDeliveryId(delivery),
|
|
},
|
|
};
|
|
}
|
|
|
|
if (!payload.email) {
|
|
throw new NotificationHandlerError("Email notifications require email recipient.");
|
|
}
|
|
|
|
const preferences = await getNotificationPreferencesForPayload(payload, getUserNotificationPreferences);
|
|
|
|
if (isNotificationPreferenceDisabled(preferences)) {
|
|
const delivery = await createSkippedNotificationDelivery(createNotificationDelivery, payload, payloadMetadata, "preference_disabled", sourceKey);
|
|
return {
|
|
handled: true,
|
|
action: "notification.email.skipped",
|
|
details: {
|
|
userId: payload.userId,
|
|
channel: payload.channel,
|
|
messageKey: payload.messageKey,
|
|
deliveryId: getNotificationDeliveryId(delivery),
|
|
reason: "preference_disabled",
|
|
},
|
|
};
|
|
}
|
|
let deliveryId = getPayloadNotificationDeliveryId(payload);
|
|
if (!deliveryId) {
|
|
const delivery = await createNotificationDelivery({
|
|
tenantId: payload.tenantId,
|
|
userId: payload.userId,
|
|
channel: "email",
|
|
category: payload.category,
|
|
messageKey: payload.messageKey,
|
|
sourceKey,
|
|
status: "PENDING",
|
|
attemptsMade: 0,
|
|
payloadMetadata,
|
|
});
|
|
deliveryId = getNotificationDeliveryId(delivery);
|
|
await persistPayloadNotificationDeliveryId(payload, context, deliveryId);
|
|
}
|
|
|
|
const attemptNumber = getAttemptNumber(context);
|
|
const isTerminalFailure = attemptNumber >= getAttemptsLimit(context);
|
|
|
|
let result: Awaited<ReturnType<MailProvider["send"]>>;
|
|
try {
|
|
const provider =
|
|
deps.mailProvider ?? createMailjetProviderFromEnv(deps.env ?? {}, deps.mailMetricsHook ? { onMetric: deps.mailMetricsHook } : undefined);
|
|
result = await provider.send({
|
|
to: [payload.email],
|
|
subject: payload.subject,
|
|
text: payload.text,
|
|
html: payload.html,
|
|
templateId: payload.templateId,
|
|
idempotencyKey: `notification-delivery:${deliveryId}`,
|
|
personalization: payload.templateData ? [{ email: payload.email, data: payload.templateData }] : undefined,
|
|
});
|
|
} catch (error) {
|
|
if (deliveryId) {
|
|
try {
|
|
await markNotificationDeliveryStatus({
|
|
id: deliveryId,
|
|
tenantId: payload.tenantId,
|
|
status: isTerminalFailure ? "FAILED" : "PENDING",
|
|
attemptsMade: attemptNumber,
|
|
lastError: getNotificationErrorMessage(error),
|
|
});
|
|
} catch {
|
|
// Preserve the provider failure as the retry reason.
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
if (deliveryId) {
|
|
try {
|
|
await markNotificationDeliveryStatus({
|
|
id: deliveryId,
|
|
tenantId: payload.tenantId,
|
|
status: "SENT",
|
|
attemptsMade: attemptNumber,
|
|
providerMessageId: result.messageId ?? null,
|
|
sentAt: new Date(),
|
|
});
|
|
} catch (error) {
|
|
return {
|
|
handled: true,
|
|
action: "notification.email.sent.diagnostic_failed",
|
|
details: {
|
|
provider: result.provider,
|
|
messageId: result.messageId,
|
|
userId: payload.userId,
|
|
messageKey: payload.messageKey,
|
|
deliveryId,
|
|
diagnosticError: getNotificationErrorMessage(error),
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
return {
|
|
handled: true,
|
|
action: "notification.email.sent",
|
|
details: { provider: result.provider, messageId: result.messageId, userId: payload.userId, messageKey: payload.messageKey },
|
|
};
|
|
},
|
|
[queueNames.maintenance]: async (payload) => {
|
|
if (payload.task === "expired-export-artifacts") {
|
|
return runExportArtifactRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "dsgvo-grace-period-scan") {
|
|
return runDsgvoGracePeriodScanHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "outbox-done-retention") {
|
|
return runOutboxDoneRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "workflow-execution-stalled-reconcile") {
|
|
return runWorkflowExecutionStalledReconcileHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "task-archive-retention") {
|
|
return runTaskArchiveRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "lead-retention-sweep") {
|
|
return runLeadRetentionHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "task-reminder-sweep") {
|
|
return runTaskReminderSweepHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "integrations-token-refresh") {
|
|
return runIntegrationsTokenRefreshHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "integrations-health-sweep") {
|
|
return runIntegrationsHealthSweepHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "calendar-token-refresh") {
|
|
return runCalendarTokenRefreshHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "calendar-webhook-renewal") {
|
|
return runCalendarWebhookRenewalHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "calendar-webhook-drain") {
|
|
return runCalendarWebhookDrainHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "calendar-internal-write-sync") {
|
|
return runCalendarInternalWriteSyncHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "email-inbox-token-refresh") {
|
|
return runEmailInboxTokenRefreshHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "email-inbox-webhook-renewal") {
|
|
return runEmailInboxWebhookRenewalHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "email-inbox-webhook-drain") {
|
|
return runEmailInboxWebhookDrainHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "email-inbox-initial-sync") {
|
|
return runEmailInboxInitialSyncHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "email-inbox-stale-poll") {
|
|
return runEmailInboxStalePollHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "email-inbox-event-cleanup") {
|
|
return runEmailInboxEventCleanupHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "hr-upload-reminder-dispatch") {
|
|
return runHrUploadReminderDispatchHandler(payload, deps);
|
|
}
|
|
|
|
if (payload.task === "email-domain-probe") {
|
|
return runEmailDomainProbeHandler(payload, deps);
|
|
}
|
|
|
|
throw new UnsupportedWorkerJobError("Unsupported maintenance task.", {
|
|
queueName: queueNames.maintenance,
|
|
task: payload.task,
|
|
olderThanHours: payload.olderThanHours,
|
|
olderThanMinutes: payload.olderThanMinutes,
|
|
});
|
|
},
|
|
[queueNames.browserAutomation]: async (payload) => {
|
|
throw new UnsupportedWorkerJobError("Browser automation worker job is not configured.", {
|
|
queueName: queueNames.browserAutomation,
|
|
kind: payload.kind,
|
|
sessionId: payload.sessionId,
|
|
});
|
|
},
|
|
[queueNames.fileScan]: async (payload) => {
|
|
if (payload.kind !== "scan.file") {
|
|
throw new ScanHandlerError("Unsupported file scan payload kind.", {
|
|
verdict: "error",
|
|
code: "invalid_payload",
|
|
message: "Unsupported file scan payload kind.",
|
|
});
|
|
}
|
|
|
|
if (!deps.scanService) {
|
|
throw new ScanHandlerError("File scan service is not configured.", {
|
|
verdict: "error",
|
|
bucket: payload.bucket,
|
|
key: payload.key,
|
|
uploadId: payload.uploadId,
|
|
code: "scan_service_missing",
|
|
message: "File scan service is not configured.",
|
|
});
|
|
}
|
|
|
|
let result: StorageScanResult;
|
|
try {
|
|
result = await deps.scanService.scanObject({
|
|
bucket: payload.bucket,
|
|
key: payload.key,
|
|
tenantId: payload.tenantId,
|
|
uploadId: payload.uploadId,
|
|
contentType: payload.contentType,
|
|
metadata: payload.metadata,
|
|
});
|
|
} catch (error) {
|
|
throw new ScanHandlerError("File scan failed.", buildRejectedScanDetails(payload, error));
|
|
}
|
|
|
|
const details = buildScanResultDetails(result, payload.uploadId);
|
|
|
|
if (result.verdict === "clean" || result.verdict === "infected") {
|
|
// AK-P4a §5 — persist the verdict so the fail-closed media proxy can
|
|
// serve clean files (and keep refusing infected ones). Skipped when the
|
|
// job carries no fileId or the writer seam is unwired (legacy scans).
|
|
if (payload.fileId && deps.fileScanStatusWriter) {
|
|
await deps.fileScanStatusWriter.recordVerdict({
|
|
fileId: payload.fileId,
|
|
tenantId: payload.tenantId,
|
|
verdict: result.verdict,
|
|
});
|
|
}
|
|
return { handled: true, action: `scan.${result.verdict}`, details };
|
|
}
|
|
|
|
throw new ScanHandlerError("File scan failed.", details);
|
|
},
|
|
[queueNames.filePreview]: async (payload) => {
|
|
if (payload.kind !== "preview.file") {
|
|
throw new PreviewHandlerError("Unsupported file preview payload kind.", {
|
|
code: "invalid_payload",
|
|
message: "Unsupported file preview payload kind.",
|
|
});
|
|
}
|
|
|
|
if (!deps.filePreviewService) {
|
|
throw new PreviewHandlerError("File preview service is not configured.", {
|
|
fileId: payload.fileId,
|
|
tenantId: payload.tenantId,
|
|
code: "preview_service_missing",
|
|
message: "File preview service is not configured.",
|
|
});
|
|
}
|
|
|
|
let result: FilePreviewServiceResult;
|
|
try {
|
|
result = await deps.filePreviewService.generatePreview(payload);
|
|
} catch (error) {
|
|
throw new PreviewHandlerError("File preview generation failed.", {
|
|
fileId: payload.fileId,
|
|
tenantId: payload.tenantId,
|
|
code: "preview_failed",
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
});
|
|
}
|
|
|
|
return {
|
|
handled: true,
|
|
action: `preview.${result.previewStatus}`,
|
|
details: {
|
|
fileId: payload.fileId,
|
|
tenantId: payload.tenantId,
|
|
previewStatus: result.previewStatus,
|
|
previewStorageKey: result.previewStorageKey,
|
|
thumbnailStorageKey: result.thumbnailStorageKey,
|
|
...(result.reason ? { reason: redactSecretLikeText(result.reason) } : {}),
|
|
},
|
|
};
|
|
},
|
|
[queueNames.exports]: async (payload, _dependencies, context) => runExportHandler(payload, deps, context),
|
|
[queueNames.imports]: async (payload, _dependencies, context) => runImportHandler(payload, deps, context),
|
|
// Phase 06 #21 / F-015 — real plugin-invoke dispatch. The handler is
|
|
// dependency-injected via `deps.pluginInvokeDeps` (composed in the
|
|
// `plugin` worker group bootstrap). When the slot is missing the
|
|
// handler throws so BullMQ dead-letters the job instead of silently
|
|
// acknowledging it — silent ack was the F-015 Critical finding.
|
|
[queueNames.pluginInvoke]: async (payload, _dependencies, context) => {
|
|
void context;
|
|
assertPluginInvokePayload(payload, deps);
|
|
if (!deps.pluginInvokeDeps) {
|
|
const message =
|
|
"plugin-invoke job received but pluginInvokeDeps is not configured on this worker. " +
|
|
"Set WORKER_QUEUE_GROUP=plugin and ensure the plugin bootstrap injected pluginInvokeDeps.";
|
|
deps.logger?.error?.({ queue: queueNames.pluginInvoke, pluginId: payload.pluginId, hookName: payload.hookName }, message);
|
|
throw new Error(message);
|
|
}
|
|
const outcome = await handlePluginInvokeJob(
|
|
{
|
|
tenantId: payload.tenantId,
|
|
pluginId: payload.pluginId,
|
|
version: payload.version,
|
|
hookName: payload.hookName,
|
|
input: payload.input,
|
|
metadata: payload.metadata,
|
|
},
|
|
deps.pluginInvokeDeps
|
|
);
|
|
if (outcome.ok) {
|
|
return {
|
|
handled: true,
|
|
action: "plugin.invoke.completed",
|
|
details: {
|
|
pluginId: payload.pluginId,
|
|
hookName: payload.hookName,
|
|
fuelUsed: outcome.fuelUsed,
|
|
durationMs: outcome.durationMs,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
handled: true,
|
|
action: "plugin.invoke.failed",
|
|
details: {
|
|
pluginId: payload.pluginId,
|
|
hookName: payload.hookName,
|
|
reason: outcome.reason,
|
|
message: outcome.message,
|
|
durationMs: outcome.durationMs,
|
|
},
|
|
};
|
|
},
|
|
};
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
function isNonEmptyString(value: unknown): value is string {
|
|
return typeof value === "string" && value.trim().length > 0;
|
|
}
|
|
|
|
function assertPluginInvokePayload(payload: unknown, deps: HandlerDependencies): asserts payload is PluginInvokeJob {
|
|
const record = isRecord(payload) ? payload : {};
|
|
const invalidFields: string[] = [];
|
|
if (record.kind !== "plugin.invoke") {
|
|
invalidFields.push("kind");
|
|
}
|
|
for (const field of ["tenantId", "pluginId", "version", "hookName"] as const) {
|
|
if (!isNonEmptyString(record[field])) {
|
|
invalidFields.push(field);
|
|
}
|
|
}
|
|
|
|
if (invalidFields.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const details = {
|
|
queueName: queueNames.pluginInvoke,
|
|
code: "invalid_payload",
|
|
invalidFields,
|
|
};
|
|
deps.logger?.error?.(
|
|
{
|
|
...details,
|
|
payload: summarizePluginInvokePayloadForLog(record),
|
|
},
|
|
"Invalid plugin-invoke queue payload."
|
|
);
|
|
throw new PluginInvokePayloadError("Invalid plugin-invoke queue payload.", details);
|
|
}
|
|
|
|
function summarizePluginInvokePayloadForLog(record: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
kind: typeof record.kind === "string" ? record.kind : typeof record.kind,
|
|
tenantId: typeof record.tenantId === "string" ? record.tenantId : typeof record.tenantId,
|
|
pluginId: typeof record.pluginId === "string" ? record.pluginId : typeof record.pluginId,
|
|
version: typeof record.version === "string" ? record.version : typeof record.version,
|
|
hookName: typeof record.hookName === "string" ? record.hookName : typeof record.hookName,
|
|
};
|
|
}
|
|
|
|
function addressToEmail(address: string | { email: string }) {
|
|
return typeof address === "string" ? address : address.email;
|
|
}
|
|
|
|
function buildMailPersonalization(to: readonly string[], templateData: Record<string, unknown> | undefined) {
|
|
if (!templateData) {
|
|
return undefined;
|
|
}
|
|
|
|
const hasPerRecipientData = to.some((email) => isRecord(templateData[email]));
|
|
return to.map((email) => ({
|
|
email,
|
|
data: hasPerRecipientData && isRecord(templateData[email]) ? templateData[email] : templateData,
|
|
}));
|
|
}
|
|
|
|
async function runExportArtifactRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const retentionCutoff = getRetentionCutoff(now, payload.olderThanHours);
|
|
const take = payload.take ?? undefined;
|
|
const listExpiredCompleted = deps.listExpiredCompletedExportJobs ?? listExpiredCompletedExportJobsInDb;
|
|
const markExpired = deps.markExportJobExpired ?? markExportJobExpiredInDb;
|
|
const listPending = deps.listSoftDeletedExportJobsForHardDelete ?? listSoftDeletedExportJobsForHardDeleteInDb;
|
|
const markDeleted = deps.markExportJobHardDeleteSucceeded ?? markExportJobHardDeleteSucceededInDb;
|
|
const markFailed = deps.markExportJobHardDeleteFailed ?? markExportJobHardDeleteFailedInDb;
|
|
const expiredCompletedJobs = await listExpiredCompleted({ now, take });
|
|
|
|
await Promise.all(expiredCompletedJobs.map((job) => markExpired({ id: job.id, tenantId: job.tenantId, deletedAt: now })));
|
|
|
|
const jobs = await listPending({ now: retentionCutoff, take });
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
const details: Record<string, unknown>[] = new Array(jobs.length);
|
|
|
|
if (jobs.length > 0 && !deps.storageDeleteService) {
|
|
throw new Error("Export artifact storage delete service is not configured.");
|
|
}
|
|
|
|
await mapWithConcurrency(jobs, EXPORT_ARTIFACT_RETENTION_DELETE_CONCURRENCY, async (job, index) => {
|
|
try {
|
|
await deps.storageDeleteService!.deleteObject({ bucket: job.artifactBucket, key: job.artifactKey });
|
|
await markDeleted({ id: job.id, tenantId: job.tenantId });
|
|
details[index] = { exportJobId: job.id, status: "deleted" };
|
|
} catch (error) {
|
|
const message = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
await markFailed({ id: job.id, tenantId: job.tenantId, errorMessage: message });
|
|
details[index] = { exportJobId: job.id, status: "failed", message };
|
|
}
|
|
});
|
|
|
|
const deleted = details.filter((job) => job?.status === "deleted").length;
|
|
const failed = details.filter((job) => job?.status === "failed").length;
|
|
|
|
return {
|
|
handled: true,
|
|
action: "export-artifact-retention.completed",
|
|
details: {
|
|
task: payload.task,
|
|
requestId,
|
|
expired: expiredCompletedJobs.length,
|
|
processed: jobs.length,
|
|
deleted,
|
|
failed,
|
|
jobs: details,
|
|
},
|
|
};
|
|
}
|
|
|
|
function getRetentionCutoff(now: Date, olderThanHours: number | undefined) {
|
|
if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) {
|
|
return now;
|
|
}
|
|
|
|
return new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
|
|
}
|
|
|
|
async function runOutboxDoneRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const now = new Date();
|
|
const olderThanHours = resolveOutboxDoneRetentionHours(payload.olderThanHours);
|
|
const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
|
|
const deleteDoneOutboxEvents = deps.deleteDoneOutboxEvents ?? (async (options) => deleteDoneOutboxEventsInDb(await getDefaultDbClient(), options));
|
|
const result = await deleteDoneOutboxEvents({ cutoff, take: payload.take });
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
|
|
return {
|
|
handled: true,
|
|
action: "outbox.done-retention.completed",
|
|
details: {
|
|
task: payload.task,
|
|
requestId,
|
|
olderThanHours,
|
|
cutoff: result.cutoff.toISOString(),
|
|
take: result.take,
|
|
deleted: result.deleted,
|
|
},
|
|
};
|
|
}
|
|
|
|
function resolveOutboxDoneRetentionHours(olderThanHours: number | undefined): number {
|
|
if (typeof olderThanHours !== "number" || !Number.isFinite(olderThanHours) || olderThanHours <= 0) {
|
|
return 720;
|
|
}
|
|
return Math.floor(olderThanHours);
|
|
}
|
|
|
|
async function runWorkflowExecutionStalledReconcileHandler(
|
|
payload: QueuePayload<typeof queueNames.maintenance>,
|
|
deps: HandlerDependencies
|
|
): Promise<HandlerResult> {
|
|
const runReconcile =
|
|
deps.reconcileWorkflowExecutionStalls ??
|
|
(async (options) => reconcileWorkflowExecutionStalls((await getDefaultDbClient()) as unknown as WorkflowExecutionStalledReconcilerPrisma, options));
|
|
const result = await runReconcile({
|
|
olderThanMinutes: payload.olderThanMinutes,
|
|
take: payload.take,
|
|
});
|
|
const requestId = typeof payload.metadata?.requestId === "string" ? payload.metadata.requestId : undefined;
|
|
|
|
return {
|
|
handled: true,
|
|
action: "workflow.execution-stalled-reconcile.completed",
|
|
details: {
|
|
task: payload.task,
|
|
requestId,
|
|
olderThanMinutes: payload.olderThanMinutes,
|
|
cutoff: result.cutoff.toISOString(),
|
|
take: result.take,
|
|
inspected: result.inspected,
|
|
failedOutboxEvents: result.failedOutboxEvents,
|
|
repaired: result.repaired,
|
|
skippedWithoutFailedOutbox: result.skippedWithoutFailedOutbox,
|
|
updateMisses: result.updateMisses,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function getDefaultDbClient() {
|
|
const { getDbClient } = await import("@hivegrid/db/client.server");
|
|
return getDbClient();
|
|
}
|
|
|
|
/**
|
|
* P4 Tasks (AK-P4b) §7 — 30-day auto-archive (DoD 5). Archives completed tasks
|
|
* whose `completedAt` is older than the retention window by stamping
|
|
* `archivedAt`. Idempotent: already-archived rows are excluded by the filter.
|
|
*
|
|
* RLS-ready (AK-P4b "Teil A"): a cross-tenant sweep cannot run under the
|
|
* restricted app role once Task RLS is active (an unscoped query returns zero
|
|
* rows). So we enumerate active tenants from the global Tenant registry (not
|
|
* RLS-scoped) and run the scoped archive per tenant inside withTenantRLS(), which
|
|
* sets `app.tenant_id` so the policy binds. Functional no-op until the policy is
|
|
* activated (prisma/rls/task-isolation.sql, P5 cutover — at which point
|
|
* hivegrid_app needs SELECT on "Tenant" or a privileged maintenance connection;
|
|
* see that file's Teil-B notes).
|
|
*/
|
|
async function runTaskArchiveRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const olderThanHours = typeof payload.olderThanHours === "number" && payload.olderThanHours > 0 ? Math.floor(payload.olderThanHours) : 720;
|
|
const cutoff = new Date(now.getTime() - olderThanHours * 60 * 60 * 1000);
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let archived = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.task.updateMany({
|
|
where: { tenantId, status: "done", completedAt: { lte: cutoff }, archivedAt: null },
|
|
data: { archivedAt: now },
|
|
}),
|
|
);
|
|
archived += result.count;
|
|
}
|
|
logger.info({ task: payload.task, cutoff: cutoff.toISOString(), tenants: tenants.length, archived }, "tasks.archive-retention.completed");
|
|
return {
|
|
handled: true,
|
|
action: "tasks.archive-retention.completed",
|
|
details: { task: payload.task, olderThanHours, cutoff: cutoff.toISOString(), tenants: tenants.length, archived },
|
|
};
|
|
}
|
|
|
|
async function runLeadRetentionHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let archived = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const dueIds = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.inboxLead.findMany({
|
|
where: { tenantId, archivedAt: null, autoDeleteAt: { lte: now } },
|
|
select: { id: true },
|
|
orderBy: { autoDeleteAt: "asc" },
|
|
take,
|
|
}),
|
|
);
|
|
if (dueIds.length === 0) continue;
|
|
const result = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.inboxLead.updateMany({
|
|
where: { tenantId, id: { in: dueIds.map((row) => row.id) }, archivedAt: null },
|
|
data: { archivedAt: now },
|
|
}),
|
|
);
|
|
archived += result.count;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, archived }, "leads.retention.completed");
|
|
return {
|
|
handled: true,
|
|
action: "leads.retention.completed",
|
|
details: { task: payload.task, tenants: tenants.length, archived, take },
|
|
};
|
|
}
|
|
|
|
/**
|
|
* R-P5 §P2d — refresh OAuth tokens nearing expiry across all tenants. Enumerates
|
|
* active tenants from the global registry (not RLS-scoped) and runs the scoped
|
|
* refresh per tenant inside withTenantRLS() (so the policy binds at cutover). The
|
|
* per-connection logic (decrypt → adapter.refreshToken → re-encrypt) lives in
|
|
* @hivegrid/integrations/worker-tasks.server, shared with the web side.
|
|
*/
|
|
async function runIntegrationsTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let refreshed = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => refreshExpiringTokensForTenant(tx as unknown as Parameters<typeof refreshExpiringTokensForTenant>[0], tenantId, { now, windowMs }));
|
|
refreshed += result.refreshed;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "integrations.token-refresh.completed");
|
|
return { handled: true, action: "integrations.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
|
|
}
|
|
|
|
/**
|
|
* R-P5 §P2d — probe every active connection's health across all tenants, persist
|
|
* the status, and notify owners on degraded/broken. Same per-tenant RLS sweep
|
|
* pattern; per-connection logic shared via worker-tasks.server.
|
|
*/
|
|
async function runIntegrationsHealthSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let checked = 0;
|
|
let unhealthy = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => sweepConnectionHealthForTenant(tx as unknown as Parameters<typeof sweepConnectionHealthForTenant>[0], tenantId, { now }));
|
|
checked += result.checked;
|
|
unhealthy += result.unhealthy;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, checked, unhealthy }, "integrations.health-sweep.completed");
|
|
return { handled: true, action: "integrations.health-sweep.completed", details: { task: payload.task, tenants: tenants.length, checked, unhealthy } };
|
|
}
|
|
|
|
async function runCalendarTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let refreshed = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => refreshCalendarTokensForTenant(tx as never, tenantId, { now, windowMs }));
|
|
refreshed += result.refreshed;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "calendar.token-refresh.completed");
|
|
return { handled: true, action: "calendar.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
|
|
}
|
|
|
|
async function runCalendarWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
|
|
const webhookUrl = process.env.GOOGLE_CALENDAR_WEBHOOK_URL;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let checked = 0;
|
|
let renewed = 0;
|
|
let failed = 0;
|
|
let skipped = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => renewCalendarWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, webhookUrl }));
|
|
checked += result.checked;
|
|
renewed += result.renewed;
|
|
failed += result.failed;
|
|
skipped += result.skipped;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped }, "calendar.webhook-renewal.completed");
|
|
return { handled: true, action: "calendar.webhook-renewal.completed", details: { task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped } };
|
|
}
|
|
|
|
async function runCalendarWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let claimed = 0;
|
|
let completed = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => drainCalendarWebhookEventsForTenant(tx as never, tenantId, { now, take }));
|
|
claimed += result.claimed;
|
|
completed += result.completed;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, claimed, completed, failed }, "calendar.webhook-drain.completed");
|
|
return { handled: true, action: "calendar.webhook-drain.completed", details: { task: payload.task, tenants: tenants.length, claimed, completed, failed } };
|
|
}
|
|
|
|
async function runCalendarInternalWriteSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 50;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let scanned = 0;
|
|
let created = 0;
|
|
let updated = 0;
|
|
let deleted = 0;
|
|
let conflicts = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => syncInternalCalendarEventsForTenant(tx as never, tenantId, { now, take }));
|
|
scanned += result.scanned;
|
|
created += result.created;
|
|
updated += result.updated;
|
|
deleted += result.deleted;
|
|
conflicts += result.conflicts;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, scanned, created, updated, deleted, conflicts, failed }, "calendar.internal-write-sync.completed");
|
|
return {
|
|
handled: true,
|
|
action: "calendar.internal-write-sync.completed",
|
|
details: { task: payload.task, tenants: tenants.length, scanned, created, updated, deleted, conflicts, failed },
|
|
};
|
|
}
|
|
|
|
async function runEmailInboxTokenRefreshHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const windowMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 10) * 60_000;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let refreshed = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => refreshEmailTokensForTenant(tx as never, tenantId, { now, windowMs }));
|
|
refreshed += result.refreshed;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, refreshed, failed }, "email-inbox.token-refresh.completed");
|
|
return { handled: true, action: "email-inbox.token-refresh.completed", details: { task: payload.task, tenants: tenants.length, refreshed, failed } };
|
|
}
|
|
|
|
async function runEmailInboxWebhookRenewalHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const thresholdMs = (typeof payload.olderThanMinutes === "number" && payload.olderThanMinutes > 0 ? Math.floor(payload.olderThanMinutes) : 24 * 60) * 60_000;
|
|
const graphWebhookUrl = process.env.MICROSOFT_MAIL_WEBHOOK_URL;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let checked = 0;
|
|
let renewed = 0;
|
|
let failed = 0;
|
|
let skipped = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => renewEmailWebhooksForTenant(tx as never, tenantId, { now, thresholdMs, graphWebhookUrl }));
|
|
checked += result.checked;
|
|
renewed += result.renewed;
|
|
failed += result.failed;
|
|
skipped += result.skipped;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped }, "email-inbox.webhook-renewal.completed");
|
|
return { handled: true, action: "email-inbox.webhook-renewal.completed", details: { task: payload.task, tenants: tenants.length, checked, renewed, failed, skipped } };
|
|
}
|
|
|
|
async function runEmailInboxWebhookDrainHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let claimed = 0;
|
|
let completed = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => drainEmailWebhookEventsForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
|
|
claimed += result.claimed;
|
|
completed += result.completed;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, claimed, completed, failed }, "email-inbox.webhook-drain.completed");
|
|
return { handled: true, action: "email-inbox.webhook-drain.completed", details: { task: payload.task, tenants: tenants.length, claimed, completed, failed } };
|
|
}
|
|
|
|
async function runEmailInboxInitialSyncHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 10;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let folders = 0;
|
|
let fetched = 0;
|
|
let completed = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => runEmailInitialSyncForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
|
|
folders += result.folders;
|
|
fetched += result.fetched;
|
|
completed += result.completed;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, folders, fetched, completed, failed }, "email-inbox.initial-sync.completed");
|
|
return { handled: true, action: "email-inbox.initial-sync.completed", details: { task: payload.task, tenants: tenants.length, folders, fetched, completed, failed } };
|
|
}
|
|
|
|
async function runEmailInboxStalePollHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 25;
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let polled = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => pollStaleEmailMailboxesForTenant(tx as never, tenantId, { now, take, hook: emailInboxMessageHook }));
|
|
polled += result.polled;
|
|
failed += result.failed;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, polled, failed }, "email-inbox.stale-poll.completed");
|
|
return { handled: true, action: "email-inbox.stale-poll.completed", details: { task: payload.task, tenants: tenants.length, polled, failed } };
|
|
}
|
|
|
|
async function runEmailInboxEventCleanupHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
let deleted = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const result = await withTenantRLS(db, tenantId, (tx) => cleanupEmailWebhookEventsForTenant(tx as never, tenantId, { now }));
|
|
deleted += result.deleted;
|
|
}
|
|
logger.info({ task: payload.task, tenants: tenants.length, deleted }, "email-inbox.event-cleanup.completed");
|
|
return { handled: true, action: "email-inbox.event-cleanup.completed", details: { task: payload.task, tenants: tenants.length, deleted } };
|
|
}
|
|
|
|
// VA-P23 · auto-match + activity logging seam of the sync engine (M09 Block E/H).
|
|
const emailInboxMessageHook = createEmailAutoMatchHook();
|
|
|
|
/**
|
|
* P4 Tasks (AK-P4b) §7 — reminder sweep (DoD 4). For each open task with a
|
|
* deadline in scan range, sends reminders to the assignee (or assigned group's
|
|
* members) + watchers at the 24h-before / due / 24h-after windows. The
|
|
* after-window escalates to EMAIL. notify()'s sourceKey upsert dedups, so the
|
|
* hourly sweep never double-sends.
|
|
*
|
|
* RLS-ready (AK-P4b "Teil A"): like the archive sweep, the cross-tenant scan is
|
|
* driven per tenant from the global Tenant registry, and the Task/TaskWatcher
|
|
* reads run inside withTenantRLS() so the policy binds once activated. notify()
|
|
* runs OUTSIDE the transaction (slow I/O, and it has its own tenant/RLS story in
|
|
* R-P1b). `take` now bounds tasks per tenant rather than globally.
|
|
*/
|
|
async function runTaskReminderSweepHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const bounds = reminderScanBounds(now);
|
|
const take = typeof payload.take === "number" && payload.take > 0 ? Math.floor(payload.take) : 1000;
|
|
const db = await getDefaultDbClient();
|
|
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
|
|
let sent = 0;
|
|
let scanned = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const tasks = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.task.findMany({
|
|
where: {
|
|
tenantId,
|
|
deadline: { not: null, gte: bounds.deadlineGte, lte: bounds.deadlineLte },
|
|
status: { in: ["open", "in_progress", "blocked"] },
|
|
archivedAt: null,
|
|
},
|
|
select: { id: true, tenantId: true, deadline: true, assignedToUserId: true, assignedToGroupId: true },
|
|
take,
|
|
}),
|
|
);
|
|
scanned += tasks.length;
|
|
|
|
for (const task of tasks) {
|
|
if (!task.deadline) continue;
|
|
const windows = selectReminderWindows(task.deadline, now);
|
|
if (windows.length === 0) continue;
|
|
|
|
// Recipients: the assigned user (or every member of the assigned group)
|
|
// plus all watchers, de-duplicated. TaskWatcher is RLS-scoped, so the reads
|
|
// run inside the tenant transaction; group membership is read alongside.
|
|
const recipientIds = new Set<string>();
|
|
if (task.assignedToUserId) recipientIds.add(task.assignedToUserId);
|
|
const { groupMembers, watchers } = await withTenantRLS(db, tenantId, async (tx) => {
|
|
const groupMembers = task.assignedToGroupId
|
|
? await tx.groupUser.findMany({ where: { groupId: task.assignedToGroupId }, select: { userId: true } })
|
|
: [];
|
|
const watcherRows = await tx.taskWatcher.findMany({ where: { tenantId, taskId: task.id }, select: { userId: true } });
|
|
return { groupMembers, watchers: watcherRows };
|
|
});
|
|
for (const member of groupMembers) recipientIds.add(member.userId);
|
|
for (const watcher of watchers) recipientIds.add(watcher.userId);
|
|
|
|
for (const window of windows) {
|
|
for (const userId of recipientIds) {
|
|
try {
|
|
// notify() runs outside the tenant transaction (slow I/O; own RLS story).
|
|
await notify(
|
|
{
|
|
tenantId: task.tenantId,
|
|
userId,
|
|
channel: reminderChannel(window),
|
|
category: "tasks.reminder",
|
|
messageKey: reminderMessageKey(window),
|
|
sourceKey: reminderSourceKey(task.id, window),
|
|
payloadMetadata: { taskId: task.id, deadline: task.deadline.toISOString() },
|
|
},
|
|
db as unknown as NotificationClient
|
|
);
|
|
sent += 1;
|
|
} catch (err) {
|
|
logger.error(
|
|
{ task: payload.task, taskId: task.id, userId, window, err: err instanceof Error ? err.message : String(err) },
|
|
"tasks.reminder.notify-failed"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.info({ task: payload.task, tenants: tenants.length, scanned, sent }, "tasks.reminder-sweep.completed");
|
|
return { handled: true, action: "tasks.reminder-sweep.completed", details: { task: payload.task, tenants: tenants.length, scanned, sent } };
|
|
}
|
|
|
|
/**
|
|
* P17 HR-Hub (AK-P17b) — upload-reminder dispatch. For every active reminder
|
|
* past its `dueAt`, fan out per-employee dispatch rows (idempotent via the unique
|
|
* reminder+employee+dueAt), notify each employee, advance the reminder by its
|
|
* cadence (or complete a one-off), then escalate overdue mandatory dispatches to
|
|
* the employee's manager. Per-tenant from the global Tenant registry; all HR
|
|
* reads/writes run inside withTenantRLS() so the policy binds once activated.
|
|
* notify() runs outside the tenant transaction (slow I/O), deduped by sourceKey.
|
|
*/
|
|
async function runHrUploadReminderDispatchHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
|
|
let dispatched = 0;
|
|
let escalated = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const due = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminder.findMany({ where: { tenantId, status: "active", archivedAt: null, dueAt: { not: null, lte: now } } }),
|
|
);
|
|
const profiles =
|
|
due.length > 0
|
|
? await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.hrEmployeeProfile.findMany({ where: { tenantId }, select: { userId: true, team: true, locationId: true } }),
|
|
)
|
|
: [];
|
|
|
|
for (const reminder of due) {
|
|
const dueAt = reminder.dueAt ?? now;
|
|
const members = profiles.filter((profile) => audienceMatches(reminder.audienceKind, reminder.audienceRef, profile));
|
|
|
|
for (const member of members) {
|
|
const created = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminderDispatch.createMany({
|
|
data: [{ tenantId, reminderId: reminder.id, employeeUserId: member.userId, dueAt, status: "sent", sentAt: now }],
|
|
skipDuplicates: true,
|
|
}),
|
|
);
|
|
if (created.count > 0) {
|
|
dispatched += created.count;
|
|
await notify(
|
|
{
|
|
tenantId,
|
|
userId: member.userId,
|
|
channel: "IN_APP",
|
|
category: "hr",
|
|
messageKey: "hr.notify.reminder.due",
|
|
sourceKey: `hr-reminder:${reminder.id}:${dueAt.getTime()}:${member.userId}`,
|
|
payloadMetadata: { reminderId: reminder.id },
|
|
},
|
|
db as unknown as NotificationClient,
|
|
).catch((err) => logger.error({ task: payload.task, reminderId: reminder.id, userId: member.userId, err: err instanceof Error ? err.message : String(err) }, "hr.upload-reminder.notify-failed"));
|
|
}
|
|
}
|
|
|
|
const next = nextReminderDue({ cadenceKind: reminder.cadenceKind, intervalMonths: reminder.intervalMonths, from: dueAt });
|
|
await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminder.update({ where: { id: reminder.id }, data: next ? { dueAt: next } : { dueAt: null, status: "completed" } }),
|
|
);
|
|
}
|
|
|
|
// Escalate still-unacknowledged mandatory dispatches past the grace window.
|
|
const cutoff = new Date(now.getTime() - HR_UPLOAD_REMINDER_ESCALATION_GRACE_MS);
|
|
const stale = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.employeeUploadReminderDispatch.findMany({ where: { tenantId, status: "sent", dueAt: { lt: cutoff }, reminder: { escalateToManager: true, mandatory: true } } }),
|
|
);
|
|
const staleEmployeeUserIds = [...new Set(stale.map((dispatch) => dispatch.employeeUserId))];
|
|
const escalationProfiles =
|
|
staleEmployeeUserIds.length > 0
|
|
? await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.hrEmployeeProfile.findMany({
|
|
where: { tenantId, userId: { in: staleEmployeeUserIds } },
|
|
select: { userId: true, managerUserId: true },
|
|
}),
|
|
)
|
|
: [];
|
|
const managerUserIdsByEmployee = new Map(escalationProfiles.map((profile) => [profile.userId, profile.managerUserId]));
|
|
for (const dispatch of stale) {
|
|
await withTenantRLS(db, tenantId, (tx) => tx.employeeUploadReminderDispatch.update({ where: { id: dispatch.id }, data: { status: "escalated", escalatedAt: now } }));
|
|
escalated += 1;
|
|
const managerUserId = managerUserIdsByEmployee.get(dispatch.employeeUserId);
|
|
if (managerUserId) {
|
|
await notify(
|
|
{
|
|
tenantId,
|
|
userId: managerUserId,
|
|
channel: "IN_APP",
|
|
category: "hr",
|
|
messageKey: "hr.notify.reminder.escalated",
|
|
sourceKey: `hr-reminder-escalate:${dispatch.id}`,
|
|
payloadMetadata: { dispatchId: dispatch.id },
|
|
},
|
|
db as unknown as NotificationClient,
|
|
).catch((err) => logger.error({ task: payload.task, dispatchId: dispatch.id, err: err instanceof Error ? err.message : String(err) }, "hr.upload-reminder.escalate-notify-failed"));
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.info({ task: payload.task, tenants: tenants.length, dispatched, escalated }, "hr.upload-reminder-dispatch.completed");
|
|
return { handled: true, action: "hr.upload-reminder-dispatch.completed", details: { task: payload.task, tenants: tenants.length, dispatched, escalated } };
|
|
}
|
|
|
|
/**
|
|
* AK-P14a · P14 Email — sending-domain DNS auto-probe (5-minute cadence). For
|
|
* each active tenant, resolves DKIM/SPF/DMARC for every not-yet-settled sending
|
|
* domain and advances `verificationStatus` via the pure decideDomainProbeStatus()
|
|
* helper. Reads/writes run inside withTenantRLS() so the policy binds once the
|
|
* P5 cutover activates RLS; the domain UPDATE is auto-audited by the M001 Prisma
|
|
* middleware. A tenant-level notification fires only on the verified/failed
|
|
* transition (not on every probe). The DNS resolver is the node:dns binding; the
|
|
* decision logic is unit-tested without DNS/DB.
|
|
*/
|
|
async function runEmailDomainProbeHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const now = new Date();
|
|
const db = await getDefaultDbClient();
|
|
const resolver: DomainDnsResolver = createDnsDomainResolver();
|
|
const tenants = await db.tenant.findMany({ where: { active: true }, select: { id: true } });
|
|
|
|
let probed = 0;
|
|
let verified = 0;
|
|
let failed = 0;
|
|
for (const { id: tenantId } of tenants) {
|
|
const domains = await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.tenantSendingDomain.findMany({ where: { tenantId, archivedAt: null, verificationStatus: { in: ["pending", "verifying"] } } }),
|
|
);
|
|
|
|
for (const domain of domains) {
|
|
probed += 1;
|
|
const dns = await resolver({ domain: domain.domain, dkimSelector: domain.dkimSelector });
|
|
const firstProbeAt = domain.firstProbeAt ?? now;
|
|
const decision = decideDomainProbeStatus({ dns, firstProbeAt, now });
|
|
const changed = decision.status !== domain.verificationStatus;
|
|
|
|
await withTenantRLS(db, tenantId, (tx) =>
|
|
tx.tenantSendingDomain.update({
|
|
where: { id: domain.id },
|
|
data: { verificationStatus: decision.status, lastError: decision.lastError, lastProbedAt: now, firstProbeAt },
|
|
}),
|
|
);
|
|
|
|
if (changed && (decision.status === "verified" || decision.status === "failed")) {
|
|
if (decision.status === "verified") verified += 1;
|
|
else failed += 1;
|
|
await notify(
|
|
{
|
|
tenantId,
|
|
channel: "IN_APP",
|
|
category: "email",
|
|
messageKey: decision.status === "verified" ? "email.domain.notify.verified" : "email.domain.notify.failed",
|
|
sourceKey: `email-domain-${decision.status}:${domain.id}`,
|
|
payloadMetadata: { domainId: domain.id, domain: domain.domain },
|
|
},
|
|
db as unknown as NotificationClient,
|
|
).catch((err) => logger.error({ task: payload.task, domainId: domain.id, err: err instanceof Error ? err.message : String(err) }, "email.domain-probe.notify-failed"));
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.info({ task: payload.task, tenants: tenants.length, probed, verified, failed }, "email.domain-probe.completed");
|
|
return { handled: true, action: "email.domain-probe.completed", details: { task: payload.task, tenants: tenants.length, probed, verified, failed } };
|
|
}
|
|
|
|
/**
|
|
* Phase 10 #68 — maintenance-handler adapter for the DSGVO grace-period
|
|
* scan. Wraps `runDsgvoGraceCron` (the pure unit-tested core) with the
|
|
* production HandlerDependencies surface.
|
|
*
|
|
* Why a separate adapter: `runDsgvoGraceCron` is intentionally
|
|
* host-agnostic (no BullMQ, no Prisma). The maintenance queue receives
|
|
* a CleanupJob discriminator and this adapter is what binds the cron
|
|
* to the worker's logger + Prisma-backed candidate-reader + cascade
|
|
* dispatcher (via `deps.dsgvoGraceCronService`).
|
|
*
|
|
* When `deps.dsgvoGraceCronService` is not configured we still need to
|
|
* acknowledge the job (otherwise BullMQ retries forever and we are
|
|
* back to the UnsupportedWorkerJobError loop that motivated this fix).
|
|
* In that case we log a single warning and return `scanned=0`. The
|
|
* follow-up phase wires the real Prisma reader + cascade dispatcher.
|
|
*/
|
|
async function runDsgvoGracePeriodScanHandler(payload: QueuePayload<typeof queueNames.maintenance>, deps: HandlerDependencies): Promise<HandlerResult> {
|
|
const logger = deps.logger ?? noopHandlerLogger;
|
|
const service = deps.dsgvoGraceCronService;
|
|
|
|
if (!service) {
|
|
// Deliberately not throwing: we want the cron to ack so BullMQ does
|
|
// not retry; the missing service is a deployment-config gap, not a
|
|
// payload-shape bug. Emit an explicit log for operators.
|
|
logger.info({ task: payload.task, requestId: payload.metadata?.requestId ?? null }, "dsgvo-grace-cron.skipped: dsgvoGraceCronService not configured");
|
|
return {
|
|
handled: true,
|
|
action: "dsgvo.grace-period.skipped",
|
|
details: { task: payload.task, reason: "dsgvoGraceCronService not configured", scanned: 0, dispatched: 0, failed: 0 },
|
|
};
|
|
}
|
|
|
|
const result = await runDsgvoGraceCron({
|
|
listOverdueUsers: (now: Date): Promise<DsgvoGraceCandidate[]> => service.listOverdueUsers(now),
|
|
enqueueDeletion: (input) => service.enqueueDeletion(input),
|
|
logger,
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "dsgvo.grace-period.scanned",
|
|
details: {
|
|
task: payload.task,
|
|
scanned: result.scanned,
|
|
dispatched: result.dispatched,
|
|
failed: result.failed,
|
|
// `failures` contains userIds — keep them so audit can correlate
|
|
// but redact in case any future error message embeds secrets.
|
|
failures: result.failures.map((f) => ({ userId: f.userId, error: redactSecretLikeText(f.error) })),
|
|
},
|
|
};
|
|
}
|
|
|
|
const noopHandlerLogger = {
|
|
info: () => undefined,
|
|
error: () => undefined,
|
|
} satisfies NonNullable<HandlerDependencies["logger"]>;
|
|
|
|
async function runExportHandler(
|
|
payload: QueuePayload<typeof queueNames.exports>,
|
|
deps: HandlerDependencies,
|
|
context?: HandlerExecutionContext<typeof queueNames.exports>
|
|
): Promise<HandlerResult> {
|
|
if (payload.kind !== "export.run") {
|
|
throw new ExportHandlerError("Unsupported export payload kind.", {
|
|
code: "invalid_payload",
|
|
message: "Unsupported export payload kind.",
|
|
});
|
|
}
|
|
|
|
if (isPrivacyGdprExportType(payload.exportType)) {
|
|
return runPrivacyGdprExportHandler(payload, deps, context);
|
|
}
|
|
|
|
const markPaused = deps.markExportJobPaused ?? markExportJobPausedInDb;
|
|
const markResumed = deps.markExportJobResumed ?? markExportJobResumedInDb;
|
|
const markRunning = deps.markExportJobRunning ?? markExportJobRunningInDb;
|
|
const markSucceeded = deps.markExportJobSucceeded ?? markExportJobSucceededInDb;
|
|
const markFailed = deps.markExportJobFailed ?? markExportJobFailedInDb;
|
|
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
|
const complianceGate = payload.complianceGate;
|
|
|
|
if (complianceGate?.state === "paused") {
|
|
const pauseReason = redactSecretLikeText(complianceGate.reason ?? "Export paused by compliance gate.");
|
|
await markPaused({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade, reason: pauseReason });
|
|
|
|
return {
|
|
handled: true,
|
|
action: "export.paused",
|
|
details: buildExportDetails(payload, {
|
|
attemptsMade,
|
|
pauseReason,
|
|
pauseRequestId: complianceGate.requestId,
|
|
status: "paused",
|
|
}),
|
|
};
|
|
}
|
|
|
|
if (complianceGate?.state === "active" || complianceGate?.resumedAt) {
|
|
await markResumed({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade });
|
|
}
|
|
|
|
await markRunning({ id: payload.exportJobId, tenantId: payload.tenantId, attemptsMade });
|
|
|
|
if (!deps.exportService) {
|
|
const message = "Export service is not configured.";
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ExportHandlerError",
|
|
errorMessage: message,
|
|
});
|
|
throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "export_service_missing" }));
|
|
}
|
|
|
|
try {
|
|
const result = await deps.exportService.runExport(payload);
|
|
await markSucceeded({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
rowCount: result.rowCount,
|
|
attemptsMade,
|
|
expiresAt: result.expiresAt,
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "export.completed",
|
|
details: buildExportDetails(payload, {
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
attemptsMade,
|
|
expiresAt: result.expiresAt.toISOString(),
|
|
rowCount: result.rowCount,
|
|
status: "completed",
|
|
}),
|
|
};
|
|
} catch (error) {
|
|
const errorName = error instanceof Error ? error.name : "Error";
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
if (isTerminalJobAttempt(attemptsMade, context)) {
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName,
|
|
errorMessage,
|
|
});
|
|
}
|
|
throw new ExportHandlerError("Export job failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }));
|
|
}
|
|
}
|
|
|
|
async function runPrivacyGdprExportHandler(
|
|
payload: ExportJob,
|
|
deps: HandlerDependencies,
|
|
context?: HandlerExecutionContext<typeof queueNames.exports>
|
|
): Promise<HandlerResult> {
|
|
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
|
const markProcessing = deps.markGdprExportProcessing ?? markGdprExportProcessingInDb;
|
|
const markReady = deps.markGdprExportReady ?? markGdprExportReadyInDb;
|
|
const markFailed = deps.markGdprExportFailed ?? markGdprExportFailedInDb;
|
|
|
|
await markProcessing({ id: payload.exportJobId, tenantId: payload.tenantId });
|
|
|
|
if (!deps.privacyGdprExportService) {
|
|
const message = "Privacy GDPR export service is not configured.";
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
errorMessage: message,
|
|
});
|
|
throw new ExportHandlerError(message, buildExportDetails(payload, { status: "failed", attemptsMade, code: "privacy_gdpr_export_service_missing" }));
|
|
}
|
|
|
|
try {
|
|
const result = await deps.privacyGdprExportService.runExport(payload);
|
|
await markReady({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
resultBucket: result.artifactBucket,
|
|
resultKey: result.artifactKey,
|
|
resultSizeBytes: result.sizeBytes,
|
|
resultExpiresAt: result.expiresAt,
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "privacy-gdpr-export.completed",
|
|
details: buildExportDetails(payload, {
|
|
artifactBucket: result.artifactBucket,
|
|
artifactKey: result.artifactKey,
|
|
attemptsMade,
|
|
expiresAt: result.expiresAt.toISOString(),
|
|
rowCount: result.rowCount,
|
|
sizeBytes: result.sizeBytes,
|
|
status: "ready",
|
|
}),
|
|
};
|
|
} catch (error) {
|
|
const errorName = error instanceof Error ? error.name : "Error";
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
if (isTerminalJobAttempt(attemptsMade, context)) {
|
|
await markFailed({
|
|
id: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
errorMessage,
|
|
});
|
|
}
|
|
throw new ExportHandlerError("Privacy GDPR export job failed.", buildExportDetails(payload, { status: "failed", attemptsMade, errorName, message: errorMessage }));
|
|
}
|
|
}
|
|
|
|
async function runImportHandler(
|
|
payload: QueuePayload<typeof queueNames.imports>,
|
|
deps: HandlerDependencies,
|
|
context?: HandlerExecutionContext<typeof queueNames.imports>
|
|
): Promise<HandlerResult> {
|
|
if (payload.kind !== "import.run") {
|
|
throw new ImportHandlerError("Unsupported import payload kind.", {
|
|
code: "invalid_payload",
|
|
message: "Unsupported import payload kind.",
|
|
});
|
|
}
|
|
|
|
const markRunning = deps.markImportJobRunning ?? markImportJobRunningInDb;
|
|
const markSucceeded = deps.markImportJobSucceeded ?? markImportJobSucceededInDb;
|
|
const markFailed = deps.markImportJobFailed ?? markImportJobFailedInDb;
|
|
const attemptsMade = readAttemptsMade(payload.metadata, context);
|
|
await markRunning({ id: payload.importJobId, tenantId: payload.tenantId, attemptsMade });
|
|
|
|
const accessDecision = authorizeStorageObjectAccess({
|
|
tenantId: payload.tenantId,
|
|
key: payload.sourceKey,
|
|
operation: "download",
|
|
});
|
|
|
|
if (!accessDecision.allowed) {
|
|
const message = "Import source storage access denied.";
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: `${message} reason=${accessDecision.reason}`,
|
|
});
|
|
throw new ImportHandlerError(
|
|
message,
|
|
buildImportDetails(payload, { status: "failed", attemptsMade, code: "source_access_denied", reason: accessDecision.reason, message })
|
|
);
|
|
}
|
|
|
|
if (!deps.importService) {
|
|
const message = "Import service is not configured.";
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: message,
|
|
});
|
|
throw new ImportHandlerError(message, buildImportDetails(payload, { status: "failed", attemptsMade, code: "import_service_missing" }));
|
|
}
|
|
|
|
if (!deps.scanService) {
|
|
const details = buildImportScanFailureDetails(payload, attemptsMade, {
|
|
verdict: "error",
|
|
code: "scan_service_missing",
|
|
message: "Import scan service is not configured.",
|
|
});
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: buildImportScanFailureMessage(details),
|
|
});
|
|
throw new ImportHandlerError("Import source scan failed.", details);
|
|
}
|
|
|
|
let scanResult: StorageScanResult;
|
|
try {
|
|
scanResult = await deps.scanService.scanObject({
|
|
bucket: payload.sourceBucket,
|
|
key: accessDecision.key,
|
|
tenantId: payload.tenantId,
|
|
uploadId: payload.importJobId,
|
|
contentType: payload.contentType,
|
|
metadata: {
|
|
...payload.metadata,
|
|
requestId: payload.requestId,
|
|
importJobId: payload.importJobId,
|
|
importType: payload.importType,
|
|
format: payload.format,
|
|
},
|
|
});
|
|
} catch (error) {
|
|
const details = buildImportScanFailureDetails(payload, attemptsMade, {
|
|
verdict: "error",
|
|
code: "scanner_rejected",
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
});
|
|
if (isTerminalJobAttempt(attemptsMade, context)) {
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: buildImportScanFailureMessage(details),
|
|
});
|
|
}
|
|
throw new ImportHandlerError("Import source scan failed.", details);
|
|
}
|
|
|
|
if (scanResult.verdict !== "clean") {
|
|
const details = buildImportScanFailureDetails(payload, attemptsMade, buildImportScanResultDetails(scanResult));
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: "ImportHandlerError",
|
|
errorMessage: buildImportScanFailureMessage(details),
|
|
});
|
|
throw new ImportHandlerError("Import source scan failed.", details);
|
|
}
|
|
|
|
try {
|
|
const result = await deps.importService.runImport(payload);
|
|
await markSucceeded({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
rowCount: result.rowCount,
|
|
acceptedRows: result.acceptedRows,
|
|
rejectedRows: result.rejectedRows,
|
|
attemptsMade,
|
|
});
|
|
|
|
return {
|
|
handled: true,
|
|
action: "import.completed",
|
|
details: buildImportDetails(payload, {
|
|
status: "completed",
|
|
attemptsMade,
|
|
rowCount: result.rowCount,
|
|
acceptedRows: result.acceptedRows,
|
|
rejectedRows: result.rejectedRows,
|
|
duplicateCount: result.duplicateCount,
|
|
errorReportBucket: result.errorReportBucket,
|
|
errorReportKey: result.errorReportKey ?? result.validationReport?.errorReport?.errorReportKey,
|
|
}),
|
|
};
|
|
} catch (error) {
|
|
const errorMessage = redactSecretLikeText(error instanceof Error ? error.message : String(error));
|
|
if (isTerminalJobAttempt(attemptsMade, context)) {
|
|
await markFailed({
|
|
id: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
attemptsMade,
|
|
errorName: error instanceof Error ? error.name : "Error",
|
|
errorMessage,
|
|
});
|
|
}
|
|
throw new ImportHandlerError("Import job failed.", buildImportDetails(payload, { status: "failed", attemptsMade, message: errorMessage }));
|
|
}
|
|
}
|
|
|
|
function readAttemptsMade(metadata: Record<string, unknown> | undefined, context?: { attemptsMade?: number }): number {
|
|
if (context?.attemptsMade !== undefined) {
|
|
return Math.max(1, context.attemptsMade + 1);
|
|
}
|
|
return typeof metadata?.attemptsMade === "number" && Number.isFinite(metadata.attemptsMade) ? metadata.attemptsMade : 0;
|
|
}
|
|
|
|
function isTerminalJobAttempt(attemptsMade: number, context?: { attemptsLimit?: number }): boolean {
|
|
if (context?.attemptsLimit === undefined) {
|
|
return true;
|
|
}
|
|
return attemptsMade >= Math.max(1, context.attemptsLimit);
|
|
}
|
|
|
|
function buildExportDetails(payload: ExportJob, details: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
exportJobId: payload.exportJobId,
|
|
tenantId: payload.tenantId,
|
|
exportType: payload.exportType,
|
|
format: payload.format,
|
|
requestId: payload.requestId,
|
|
...details,
|
|
};
|
|
}
|
|
|
|
function buildImportDetails(payload: ImportJob, details: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
importJobId: payload.importJobId,
|
|
tenantId: payload.tenantId,
|
|
importType: payload.importType,
|
|
format: payload.format,
|
|
requestId: payload.requestId,
|
|
...details,
|
|
};
|
|
}
|
|
|
|
function buildScanResultDetails(result: StorageScanResult, uploadId: string | undefined): Record<string, unknown> {
|
|
const base = {
|
|
verdict: result.verdict,
|
|
bucket: result.bucket,
|
|
key: result.key,
|
|
uploadId,
|
|
};
|
|
|
|
if (result.verdict === "clean") {
|
|
return {
|
|
...base,
|
|
engine: result.engine,
|
|
};
|
|
}
|
|
|
|
if (result.verdict === "infected") {
|
|
return {
|
|
...base,
|
|
engine: result.engine,
|
|
signature: result.signature,
|
|
quarantineKey: result.quarantineKey,
|
|
};
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
...buildRedactedErrorDetails(result.error),
|
|
};
|
|
}
|
|
|
|
function buildImportScanResultDetails(result: StorageScanResult): Record<string, unknown> {
|
|
if (result.verdict === "clean") {
|
|
return {
|
|
verdict: result.verdict,
|
|
engine: result.engine,
|
|
};
|
|
}
|
|
|
|
if (result.verdict === "infected") {
|
|
return {
|
|
verdict: result.verdict,
|
|
code: "source_infected",
|
|
engine: result.engine,
|
|
signature: result.signature,
|
|
quarantineKey: result.quarantineKey,
|
|
message: "Import source object failed malware scan.",
|
|
};
|
|
}
|
|
|
|
return {
|
|
verdict: result.verdict,
|
|
...buildRedactedErrorDetails(result.error),
|
|
};
|
|
}
|
|
|
|
function buildImportScanFailureDetails(payload: ImportJob, attemptsMade: number, scanDetails: Record<string, unknown>): Record<string, unknown> {
|
|
return buildImportDetails(payload, {
|
|
status: "failed",
|
|
attemptsMade,
|
|
scan: scanDetails,
|
|
});
|
|
}
|
|
|
|
function buildImportScanFailureMessage(details: Record<string, unknown>): string {
|
|
const scan = details.scan && typeof details.scan === "object" ? (details.scan as Record<string, unknown>) : {};
|
|
const parts = ["Import source scan failed.", `verdict=${String(scan.verdict ?? "error")}`, `code=${String(scan.code ?? "scan_failed")}`];
|
|
const requestId = typeof details.requestId === "string" ? details.requestId : undefined;
|
|
if (requestId) {
|
|
parts.push(`requestId=${requestId}`);
|
|
}
|
|
const message = typeof scan.message === "string" ? scan.message : undefined;
|
|
if (message) {
|
|
parts.push(`message=${redactSecretLikeText(message)}`);
|
|
}
|
|
return parts.join(" ");
|
|
}
|
|
|
|
function buildRejectedScanDetails(payload: QueuePayload<typeof queueNames.fileScan>, error: unknown): Record<string, unknown> {
|
|
return {
|
|
verdict: "error",
|
|
bucket: payload.bucket,
|
|
key: payload.key,
|
|
uploadId: payload.uploadId,
|
|
code: "scanner_rejected",
|
|
message: redactSecretLikeText(error instanceof Error ? error.message : String(error)),
|
|
};
|
|
}
|
|
|
|
function buildRedactedErrorDetails(error: StorageScanDiagnostic): Record<string, unknown> {
|
|
return {
|
|
code: error.code,
|
|
message: redactSecretLikeText(error.message),
|
|
upstream: error.upstream
|
|
? {
|
|
name: error.upstream.name,
|
|
message: error.upstream.message ? redactSecretLikeText(error.upstream.message) : undefined,
|
|
status: error.upstream.status,
|
|
requestId: error.upstream.requestId,
|
|
}
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
function redactSecretLikeText(message: string): string {
|
|
return message
|
|
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
|
.replace(/(access[_-]?key(?:id)?|secret[_-]?access[_-]?key|session[_-]?token|credential|authorization|token|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
|
|
.replace(/(body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]");
|
|
}
|
|
|
|
function serializeWebhookResponseDiagnostics(response: Response, body: string) {
|
|
return JSON.stringify({
|
|
status: response.status,
|
|
statusText: response.statusText,
|
|
contentType: response.headers.get("content-type"),
|
|
bodyBytes: new TextEncoder().encode(body).byteLength,
|
|
});
|
|
}
|
|
|
|
async function fetchWebhookWithRedirects(
|
|
fetchImpl: typeof fetch | undefined,
|
|
input: string,
|
|
init: RequestInit,
|
|
addressResolver?: OutboundHttpAddressResolver
|
|
): Promise<Response> {
|
|
try {
|
|
return await fetchOutboundHttpWithRedirects(input, init, {
|
|
// In production fetchImpl is undefined, so the guard uses its pinned
|
|
// undici fetch; a test-injected fetchImpl disables pinning.
|
|
...(fetchImpl ? { fetchImpl } : {}),
|
|
...(addressResolver ? { addressResolver } : {}),
|
|
timeoutMs: DEFAULT_WORKER_FETCH_TIMEOUT_MS,
|
|
redirectLimit: WEBHOOK_REDIRECT_LIMIT,
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof OutboundHttpUrlValidationError) {
|
|
throw new WebhookUrlValidationError(toWebhookUrlValidationMessage(error.message));
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function toWebhookUrlValidationMessage(message: string): string {
|
|
return message.replace(/^Outbound HTTP endpoint/u, "Webhook endpoint").replace(/^Outbound HTTP redirect/u, "Webhook redirect");
|
|
}
|
|
|
|
async function readLimitedResponseText(response: Response, limitBytes: number): Promise<string> {
|
|
if (!response.body) {
|
|
return "";
|
|
}
|
|
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
const chunks: string[] = [];
|
|
let bytes = 0;
|
|
|
|
while (bytes < limitBytes) {
|
|
const result = await reader.read();
|
|
if (result.done) {
|
|
chunks.push(decoder.decode());
|
|
return chunks.join("");
|
|
}
|
|
|
|
const chunk = result.value;
|
|
const remaining = limitBytes - bytes;
|
|
const slice = chunk.byteLength > remaining ? chunk.slice(0, remaining) : chunk;
|
|
chunks.push(decoder.decode(slice, { stream: chunk.byteLength <= remaining }));
|
|
bytes += slice.byteLength;
|
|
|
|
if (chunk.byteLength > remaining) {
|
|
await reader.cancel();
|
|
break;
|
|
}
|
|
}
|
|
|
|
chunks.push(decoder.decode());
|
|
return chunks.join("");
|
|
}
|
|
|
|
export type CreateRegisteredProcessorOptions = {
|
|
dependencies?: HandlerDependencies;
|
|
registry?: HandlerRegistry;
|
|
};
|
|
|
|
export function createRegisteredProcessor<Name extends QueueName>(queueName: Name, options: CreateRegisteredProcessorOptions = {}): TypedProcessor<Name> {
|
|
const dependencies = options.dependencies ?? {};
|
|
const registry = options.registry ?? buildHandlerRegistry(dependencies);
|
|
|
|
return async (job) => {
|
|
return registry[queueName](job.data, dependencies, {
|
|
attemptsMade: job.attemptsMade,
|
|
attemptsLimit: job.opts?.attempts,
|
|
jobId: job.id === undefined ? undefined : String(job.id),
|
|
updatePayload: job.updateData
|
|
? async (payload) => {
|
|
await job.updateData?.(payload);
|
|
}
|
|
: undefined,
|
|
});
|
|
};
|
|
}
|