import type { ActionFunctionArgs } from "react-router"; import { createImportDryRunPreview, createRedactedParserDiagnostic, isSupportedImportFormat, redactDiagnosticMessage, type ColumnMapping, type ImportDryRunFieldDefinition, type ImportDryRunPreviewInput, type ImportPreviewInputRow, type NormalizedImportRow, type NormalizedRowValue, type ParserDiagnostic, } from "@toolctrl/imports"; import { createImportJob, markImportJobFailed, markImportJobRunning, markImportJobSucceeded } from "@toolctrl/db/import-jobs.server"; import { authorizeStorageObjectAccess } from "@toolctrl/storage"; import { requireAuth } from "~/utils/loaders.middleware"; import { verifyUserHasPermission } from "~/utils/helpers/.server/PermissionsService"; import type { DefaultPermission } from "~/application/dtos/shared/DefaultPermissions"; import { validateCSRFToken } from "~/utils/session.server"; import { getTranslations } from "~/locale/i18next.server"; import type { TFunction } from "i18next"; const MAX_PREVIEW_ROWS = 500; const MAX_MAPPINGS = 200; const MAX_FIELDS = 200; const DRY_RUN_SOURCE_BUCKET = "import-dry-run"; type DryRunBody = { format?: unknown; importType?: unknown; source?: unknown; mappings?: unknown; rows?: unknown; fields?: unknown; }; type DryRunSource = { tenantId: string; sourceKey: string; fileName?: string | null; contentType?: string | null; requestId?: string | null; }; function json(payload: unknown, status = 200) { return Response.json(payload, { status }); } function invalidInput(message: string, diagnostics?: readonly ParserDiagnostic[]) { return json({ error: message, ...(diagnostics && diagnostics.length > 0 ? { diagnostics } : {}) }, 400); } function methodNotAllowed(t: TFunction) { return json({ error: t("models.row.import.errors.methodNotAllowed") }, 405); } function storageAccessDenied(t: TFunction, reason: string) { const status = reason === "invalid_tenant" || reason === "invalid_key" ? 400 : 403; return json({ error: t("models.row.import.errors.sourceStorageAccessDenied"), reason }, status); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function isNormalizedRowValue(value: unknown): value is NormalizedRowValue { if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") { return true; } return Array.isArray(value) && value.every((item) => item === null || typeof item === "string" || typeof item === "number" || typeof item === "boolean"); } function parseString(value: unknown, fieldName: string, t: TFunction): string { if (typeof value !== "string" || value.trim().length === 0) { throw new Error(t("models.row.import.errors.requiredField", { field: fieldName })); } return value.trim(); } function parseOptionalString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } function parseSource(value: unknown, t: TFunction): DryRunSource { if (!isRecord(value)) { throw new Error(t("models.row.import.errors.sourceRequired")); } return { tenantId: parseString(value.tenantId, "source.tenantId", t), sourceKey: parseString(value.sourceKey, "source.sourceKey", t), fileName: parseOptionalString(value.fileName), contentType: parseOptionalString(value.contentType), requestId: parseOptionalString(value.requestId), }; } function parseMappings(value: unknown, t: TFunction): ColumnMapping[] { if (!Array.isArray(value) || value.length === 0) { throw new Error(t("models.row.import.errors.mappingsRequired")); } if (value.length > MAX_MAPPINGS) { throw new Error(t("models.row.import.errors.maxMappings", { max: MAX_MAPPINGS })); } return value.map((item, index) => { if (!isRecord(item)) { throw new Error(t("models.row.import.errors.mappingMustBeObject", { index })); } return { sourceColumn: parseString(item.sourceColumn, `mappings[${index}].sourceColumn`, t), targetField: parseString(item.targetField, `mappings[${index}].targetField`, t), required: item.required === true, label: parseOptionalString(item.label) ?? undefined, }; }); } function parseFields(value: unknown, t: TFunction): ImportDryRunFieldDefinition[] { if (value === undefined) { return []; } if (!Array.isArray(value)) { throw new Error(t("models.row.import.errors.fieldsMustBeArray")); } if (value.length > MAX_FIELDS) { throw new Error(t("models.row.import.errors.maxFields", { max: MAX_FIELDS })); } return value.map((item, index) => { if (!isRecord(item)) { throw new Error(t("models.row.import.errors.fieldMustBeObject", { index })); } return { targetField: parseString(item.targetField, `fields[${index}].targetField`, t), required: item.required === true, label: parseOptionalString(item.label) ?? undefined, }; }); } function parseRows(value: unknown, t: TFunction): ImportPreviewInputRow[] { if (!Array.isArray(value) || value.length === 0) { throw new Error(t("models.row.import.errors.rowsRequired")); } if (value.length > MAX_PREVIEW_ROWS) { throw new Error(t("models.row.import.errors.maxRows", { max: MAX_PREVIEW_ROWS })); } return value.map((item, index) => { if (!isRecord(item) || !isRecord(item.values)) { throw new Error(t("models.row.import.errors.rowValuesMustBeObject", { index })); } const values: NormalizedImportRow = {}; for (const [key, rowValue] of Object.entries(item.values)) { if (!isNormalizedRowValue(rowValue)) { throw new Error(t("models.row.import.errors.rowValueMustBeScalar", { index, field: key })); } values[key] = rowValue; } const rowNumber = typeof item.rowNumber === "number" && Number.isInteger(item.rowNumber) && item.rowNumber > 0 ? item.rowNumber : undefined; return { rowNumber, values }; }); } function validationDiagnostic(message: string): ParserDiagnostic[] { return [createRedactedParserDiagnostic("invalid_dry_run_request", message)]; } function buildPreviewInput(body: DryRunBody, t: TFunction): ImportDryRunPreviewInput & { source: DryRunSource; importType: string } { const format = parseString(body.format, "format", t); if (!isSupportedImportFormat(format)) { throw new Error(t("models.row.import.errors.unsupportedFormat")); } return { format, importType: parseString(body.importType, "importType", t), source: parseSource(body.source, t), mappings: parseMappings(body.mappings, t), fields: parseFields(body.fields, t), rows: parseRows(body.rows, t), }; } function getImportPermission(importType: string, t: TFunction): DefaultPermission { switch (importType) { case "members": return "app.settings.members.create"; default: throw new Error(t("models.row.import.errors.unsupportedType", { importType })); } } function getCreatedJobId(createdJob: unknown, t: TFunction): string { if (isRecord(createdJob) && typeof createdJob.id === "string" && createdJob.id.length > 0) { return createdJob.id; } throw new Error(t("models.row.import.errors.importJobCreateFailed")); } function toErrorMessage(error: unknown, t: TFunction): string { return error instanceof Error ? error.message : t("models.row.import.errors.dryRunFailed"); } function sanitizeDiagnosticMessage(message: string): string { return redactDiagnosticMessage(message) .replace(/authorization:\s*\[REDACTED\]\s+[^\s,;]+/gi, "authorization: [REDACTED]") .replace(/Bearer\s+[^\s,;]+/gi, "Bearer [REDACTED]") .replace(/rawBody\s*=\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)/gi, "rawBody=[REDACTED]"); } async function markFailureIfPossible( jobId: string | undefined, tenantId: string, attemptsMade: number, error: unknown, t: TFunction, counters?: { rowCount?: number; acceptedRows?: number; rejectedRows?: number } ) { if (!jobId) { return; } await markImportJobFailed({ id: jobId, tenantId, attemptsMade, rowCount: counters?.rowCount, acceptedRows: counters?.acceptedRows, rejectedRows: counters?.rejectedRows, errorName: error instanceof Error ? error.name : "Error", errorMessage: sanitizeDiagnosticMessage(toErrorMessage(error, t)), }); } export const action = async ({ request, params }: ActionFunctionArgs) => { const { t } = await getTranslations(request); if (request.method !== "POST") { return methodNotAllowed(t); } let userInfo: Awaited>; try { userInfo = await requireAuth({ request, params }); } catch (error) { throw error; } let body: DryRunBody; try { await validateCSRFToken(request); const parsed = await request.json(); if (!isRecord(parsed)) { return invalidInput(t("models.row.import.errors.jsonBodyMustBeObject")); } body = parsed; } catch { return invalidInput(t("models.row.import.errors.invalidJsonBody")); } let previewInput: ImportDryRunPreviewInput & { source: DryRunSource; importType: string }; try { previewInput = buildPreviewInput(body, t); } catch (error) { const message = sanitizeDiagnosticMessage(toErrorMessage(error, t)); return invalidInput(message, validationDiagnostic(message)); } let permission: DefaultPermission; try { permission = getImportPermission(previewInput.importType, t); } catch (error) { const message = sanitizeDiagnosticMessage(toErrorMessage(error, t)); return invalidInput(message, validationDiagnostic(message)); } await verifyUserHasPermission(request, permission, previewInput.source.tenantId); const accessDecision = authorizeStorageObjectAccess({ tenantId: previewInput.source.tenantId, key: previewInput.source.sourceKey, operation: "download", }); if (!accessDecision.allowed) { return storageAccessDenied(t, accessDecision.reason); } const requestId = previewInput.source.requestId ?? crypto.randomUUID(); let jobId: string | undefined; const attemptsMade = 1; try { const createdJob = await createImportJob({ tenantId: previewInput.source.tenantId, requestedByUserId: userInfo.userId ?? null, importType: previewInput.importType, format: previewInput.format, requestId, sourceBucket: DRY_RUN_SOURCE_BUCKET, sourceKey: accessDecision.key, fileName: previewInput.source.fileName, contentType: previewInput.source.contentType, attemptsMade: 0, }); jobId = getCreatedJobId(createdJob, t); await markImportJobRunning({ id: jobId, tenantId: previewInput.source.tenantId, attemptsMade }); const preview = createImportDryRunPreview(previewInput); await markImportJobSucceeded({ id: jobId, tenantId: previewInput.source.tenantId, rowCount: preview.summary.totalRows, acceptedRows: preview.summary.acceptedRows, rejectedRows: preview.summary.rejectedRows, attemptsMade, }); return json({ jobId, status: "COMPLETED", summary: preview.summary, rows: preview.rows, diagnostics: preview.summary.diagnostics, }); } catch (error) { try { await markFailureIfPossible(jobId, previewInput.source.tenantId, attemptsMade, error, t); } catch { // Preserve the original dry-run failure while avoiding diagnostic leakage from persistence failures. } return json( { error: t("models.row.import.errors.dryRunFailed"), diagnostics: [createRedactedParserDiagnostic("dry_run_failed", t("models.row.import.errors.dryRunFailedBeforeCompletion"))], }, 500 ); } };