import { createRedactedParserDiagnostic, redactDiagnosticMessage, summarizeRowResults, type ImportDryRunSummary, type ImportFormat, type ImportRowErrorResult, type ImportRowResult, type NormalizedImportRow, type NormalizedRowPrimitive, type NormalizedRowValue, type ParserDiagnostic, type ParserErrorReport, } from "./index.js"; export type DuplicateKeyField = { field: string; label?: string; }; export type DuplicateDetectionRow = { rowNumber?: number; normalized: NormalizedImportRow; }; export type DuplicateKeyPart = { field: string; valuePreview: string; }; export type DuplicateGroup = { id: string; key: readonly DuplicateKeyPart[]; rowNumbers: readonly number[]; }; export type DuplicateDetectionSummary = { totalRows: number; duplicateGroups: number; duplicateRows: number; diagnostics: readonly ParserDiagnostic[]; }; export type DuplicateDetectionReport = { groups: readonly DuplicateGroup[]; rowResults: readonly ImportRowResult[]; summary: DuplicateDetectionSummary; errorReport?: ParserErrorReport; }; export type ValidationReportSummary = ImportDryRunSummary & { duplicateGroups: number; duplicateRows: number; }; export type ValidationReport = { summary: ValidationReportSummary; diagnostics: readonly ParserDiagnostic[]; duplicateGroups: readonly DuplicateGroup[]; errorReport?: ParserErrorReport; }; type DuplicateKeySignature = { signature: string; parts: readonly DuplicateKeyPart[]; }; function normalizeDuplicateKeyValue(value: NormalizedRowValue | undefined): string | undefined { if (value === undefined || value === null) { return undefined; } if (typeof value === "string") { const trimmed = value.trim(); return trimmed.length > 0 ? trimmed : undefined; } if (Array.isArray(value)) { const normalizedItems = value .map((item) => normalizeDuplicateKeyPrimitive(item)) .filter((item): item is string => item !== undefined); return normalizedItems.length > 0 ? JSON.stringify(normalizedItems) : undefined; } return String(value); } function normalizeDuplicateKeyPrimitive(value: NormalizedRowPrimitive): string | undefined { if (value === null) { return undefined; } if (typeof value === "string") { const trimmed = value.trim(); return trimmed.length > 0 ? trimmed : undefined; } return String(value); } function toDuplicateKeyField(field: string | DuplicateKeyField): DuplicateKeyField { return typeof field === "string" ? { field } : field; } function createDuplicateKey(row: NormalizedImportRow, fields: readonly DuplicateKeyField[]): DuplicateKeySignature | undefined { const parts: DuplicateKeyPart[] = []; const signatureParts: string[] = []; for (const field of fields) { if (!field.field) { continue; } const normalizedValue = normalizeDuplicateKeyValue(row[field.field]); if (normalizedValue === undefined) { continue; } parts.push({ field: field.field, valuePreview: redactDiagnosticMessage(normalizedValue, 80) }); signatureParts.push(`${JSON.stringify(field.field)}:${JSON.stringify(normalizedValue)}`); } if (signatureParts.length === 0) { return undefined; } return { signature: signatureParts.join("|"), parts, }; } function duplicateGroupId(index: number): string { return `duplicate-group-${String(index + 1).padStart(4, "0")}`; } function createDuplicateDiagnostic(group: DuplicateGroup, rowNumber: number): ParserDiagnostic { return createRedactedParserDiagnostic( "duplicate_key", `Duplicate import row matches ${group.id} on ${group.key.map((part) => `${part.field}=${part.valuePreview}`).join(", ")}.`, { rowNumber, valuePreview: group.id, }, ); } export function detectDuplicateRows( rows: readonly DuplicateDetectionRow[], keyFields: readonly (string | DuplicateKeyField)[], ): DuplicateDetectionReport { const fields = keyFields.map(toDuplicateKeyField).filter((field) => field.field.trim().length > 0); const rowResults: ImportRowResult[] = rows.map((row, index) => ({ ok: true, rowNumber: row.rowNumber ?? index + 1, normalized: row.normalized, warnings: undefined, })); if (fields.length === 0) { return { groups: [], rowResults, summary: { totalRows: rows.length, duplicateGroups: 0, duplicateRows: 0, diagnostics: [], }, errorReport: undefined, }; } const bySignature = new Map(); for (let index = 0; index < rows.length; index += 1) { const row = rows[index]; const key = createDuplicateKey(row.normalized, fields); if (!key) { continue; } const rowNumber = row.rowNumber ?? index + 1; const existing = bySignature.get(key.signature); if (existing) { existing.rowNumbers.push(rowNumber); } else { bySignature.set(key.signature, { parts: key.parts, rowNumbers: [rowNumber] }); } } const groups = Array.from(bySignature.values()) .filter((group) => group.rowNumbers.length > 1) .sort((left, right) => left.rowNumbers[0] - right.rowNumbers[0]) .map((group, index) => ({ id: duplicateGroupId(index), key: group.parts, rowNumbers: [...group.rowNumbers].sort((left, right) => left - right), })); const diagnostics = groups.flatMap((group) => group.rowNumbers.map((rowNumber) => createDuplicateDiagnostic(group, rowNumber))); const diagnosticsByRowNumber = new Map(); for (const diagnostic of diagnostics) { if (diagnostic.rowNumber === undefined) { continue; } diagnosticsByRowNumber.set(diagnostic.rowNumber, [...(diagnosticsByRowNumber.get(diagnostic.rowNumber) ?? []), diagnostic]); } const duplicateRowResults = rowResults.map((row) => { const errors = diagnosticsByRowNumber.get(row.rowNumber); if (!errors) { return row; } return { ok: false, rowNumber: row.rowNumber, errors, } satisfies ImportRowErrorResult; }); return { groups, rowResults: duplicateRowResults, summary: { totalRows: rows.length, duplicateGroups: groups.length, duplicateRows: new Set(groups.flatMap((group) => group.rowNumbers)).size, diagnostics, }, errorReport: diagnostics.length > 0 ? { diagnostics, rejectedRows: diagnosticsByRowNumber.size, } : undefined, }; } export function createValidationReport( format: ImportFormat, rowResults: readonly ImportRowResult[], duplicateReport: Pick, ): ValidationReport { const summaryCounts = summarizeRowResults(rowResults); const diagnostics = [ ...rowResults.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors)), ...duplicateReport.summary.diagnostics, ]; return { summary: { format, ...summaryCounts, duplicateGroups: duplicateReport.summary.duplicateGroups, duplicateRows: duplicateReport.summary.duplicateRows, diagnostics, }, diagnostics, duplicateGroups: duplicateReport.groups, errorReport: diagnostics.length > 0 ? { diagnostics, rejectedRows: summaryCounts.rejectedRows, } : undefined, }; }