export * from "./dedupe.js"; export const IMPORT_FORMATS = ["csv", "json", "xlsx"] as const; export type ImportFormat = (typeof IMPORT_FORMATS)[number]; export type ImportSourceDescriptor = { tenantId: string; requesterId: string; sourceKey: string; fileName?: string; contentType?: string; byteLength?: number; requestId?: string; }; export type ColumnMapping = { sourceColumn: string; targetField: string; required?: boolean; label?: string; }; export type NormalizedRowPrimitive = string | number | boolean | null; export type NormalizedRowValue = NormalizedRowPrimitive | readonly NormalizedRowPrimitive[]; export type NormalizedImportRow = Record; export type ImportRowSuccessResult = { ok: true; rowNumber: number; normalized: NormalizedImportRow; warnings?: readonly ParserDiagnostic[]; }; export type ImportRowErrorResult = { ok: false; rowNumber: number; errors: readonly ParserDiagnostic[]; rejectedValues?: NormalizedImportRow; }; export type ImportRowResult = ImportRowSuccessResult | ImportRowErrorResult; export type ParserDiagnosticSeverity = "info" | "warning" | "error"; export type ParserDiagnostic = { severity: ParserDiagnosticSeverity; code: string; message: string; rowNumber?: number; column?: string; valuePreview?: string; }; export type ParserErrorReport = { diagnostics: readonly ParserDiagnostic[]; rejectedRows: number; errorReportKey?: string; }; export type ImportDryRunSummary = { format: ImportFormat; totalRows: number; acceptedRows: number; rejectedRows: number; diagnostics: readonly ParserDiagnostic[]; }; export type ImportParserInput = { source: ImportSourceDescriptor; format: ImportFormat; mappings: readonly ColumnMapping[]; text?: string; bytes?: Uint8Array; }; export type ImportParserOutput = { rows: readonly ImportRowResult[]; summary: ImportDryRunSummary; errorReport?: ParserErrorReport; }; export type ImportParser = { readonly format: ImportFormat; parse(input: ImportParserInput): Promise | ImportParserOutput; }; export type ImportDryRunFieldDefinition = { targetField: string; required?: boolean; label?: string; }; export type ImportPreviewInputRow = { rowNumber?: number; values: NormalizedImportRow; }; export type ImportMappedPreviewRow = { rowNumber: number; normalized: NormalizedImportRow; sourceColumns: readonly string[]; }; export type ImportDryRunPreviewInput = { format: ImportFormat; fields: readonly ImportDryRunFieldDefinition[]; mappings: readonly ColumnMapping[]; rows: readonly ImportPreviewInputRow[]; }; export type ImportDryRunPreviewOutput = ImportParserOutput & { mappedRows: readonly ImportMappedPreviewRow[]; }; const EXTENSION_FORMATS: Readonly> = { csv: "csv", json: "json", xlsx: "xlsx", }; const CONTENT_TYPE_FORMATS: Readonly> = { "application/csv": "csv", "application/json": "json", "application/vnd.ms-excel": "csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", "text/csv": "csv", }; const SECRET_PATTERN = /(?:body|payload|raw(?:file|body|payload|rows)?)\s*[:=]\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)|(?:api[_-]?key|authorization|bearer|password|secret|token)\s*[:=]\s*[^\s,;]+/gi; const REDACTED = "[REDACTED]"; export function isSupportedImportFormat(value: string): value is ImportFormat { return (IMPORT_FORMATS as readonly string[]).includes(value); } export function detectImportFormatFromExtension(fileNameOrExtension: string | undefined): ImportFormat | undefined { if (!fileNameOrExtension) { return undefined; } const extension = fileNameOrExtension.trim().toLowerCase().replace(/^.*\./, ""); return EXTENSION_FORMATS[extension]; } export function detectImportFormatFromContentType(contentType: string | undefined): ImportFormat | undefined { if (!contentType) { return undefined; } const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase(); return normalized ? CONTENT_TYPE_FORMATS[normalized] : undefined; } export function summarizeRowResults(rows: readonly ImportRowResult[]): Pick { const acceptedRows = rows.filter((row) => row.ok).length; return { totalRows: rows.length, acceptedRows, rejectedRows: rows.length - acceptedRows, }; } function isBlankPreviewValue(value: NormalizedRowValue | undefined): boolean { if (value === undefined || value === null) { return true; } if (typeof value === "string") { return value.trim().length === 0; } if (Array.isArray(value)) { return value.length === 0 || value.every((item) => isBlankPreviewValue(item)); } return false; } function previewValue(value: NormalizedRowValue | undefined): string | undefined { if (value === undefined) { return undefined; } return redactDiagnosticMessage(String(value), 80); } function buildRequiredFieldDefinitions( fields: readonly ImportDryRunFieldDefinition[], mappings: readonly ColumnMapping[], ): Map { const requiredFields = new Map(); for (const field of fields) { if (field.required) { requiredFields.set(field.targetField, field); } } for (const mapping of mappings) { if (mapping.required) { requiredFields.set(mapping.targetField, { targetField: mapping.targetField, required: true, label: mapping.label, }); } } return requiredFields; } export function createImportDryRunPreview(input: ImportDryRunPreviewInput): ImportDryRunPreviewOutput { const requiredFields = buildRequiredFieldDefinitions(input.fields, input.mappings); const rows = input.rows.map((row, index) => { const rowNumber = row.rowNumber ?? index + 1; const normalized: NormalizedImportRow = {}; const sourceColumns: string[] = []; for (const mapping of input.mappings) { if (!mapping.sourceColumn || !mapping.targetField) { continue; } if (Object.prototype.hasOwnProperty.call(row.values, mapping.sourceColumn)) { normalized[mapping.targetField] = row.values[mapping.sourceColumn] ?? null; sourceColumns.push(mapping.sourceColumn); } } const errors: ParserDiagnostic[] = []; for (const [targetField, field] of requiredFields) { if (isBlankPreviewValue(normalized[targetField])) { errors.push( createRedactedParserDiagnostic( "missing_required_field", `${field.label ?? targetField} is required for import preview.`, { rowNumber, column: targetField, valuePreview: previewValue(normalized[targetField]), }, ), ); } } if (errors.length > 0) { return { ok: false, rowNumber, errors, }; } return { ok: true, rowNumber, normalized, warnings: undefined, }; }); const mappedRows = rows .filter((row): row is ImportRowSuccessResult => row.ok) .map((row) => ({ rowNumber: row.rowNumber, normalized: row.normalized, sourceColumns: input.mappings .filter((mapping) => Object.prototype.hasOwnProperty.call(row.normalized, mapping.targetField)) .map((mapping) => mapping.sourceColumn), })); const diagnostics = rows.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors)); const summaryCounts = summarizeRowResults(rows); return { rows, mappedRows, summary: { format: input.format, ...summaryCounts, diagnostics, }, errorReport: summaryCounts.rejectedRows > 0 ? { diagnostics, rejectedRows: summaryCounts.rejectedRows, } : undefined, }; } export function createRedactedParserDiagnostic( code: string, message: string, options: Omit, "code" | "message"> = {}, ): ParserDiagnostic { return { severity: "error", ...options, code, message: redactDiagnosticMessage(message), }; } export function redactDiagnosticMessage(message: string, maxLength = 300): string { const redacted = message.replace(SECRET_PATTERN, (match) => { const colonIndex = match.indexOf(":"); const equalsIndex = match.indexOf("="); const separatorIndex = colonIndex === -1 ? equalsIndex : equalsIndex === -1 ? colonIndex : Math.min(colonIndex, equalsIndex); const prefix = separatorIndex >= 0 ? match.slice(0, separatorIndex + 1) : "secret:"; return `${prefix} ${REDACTED}`; }); return redacted.length > maxLength ? `${redacted.slice(0, maxLength - 1)}…` : redacted; }