273 lines
7.3 KiB
TypeScript
273 lines
7.3 KiB
TypeScript
import {
|
|
createRedactedParserDiagnostic,
|
|
redactDiagnosticMessage,
|
|
summarizeRowResults,
|
|
type ImportDryRunSummary,
|
|
type ImportFormat,
|
|
type ImportRowErrorResult,
|
|
type ImportRowResult,
|
|
type NormalizedImportRow,
|
|
type NormalizedRowPrimitive,
|
|
type NormalizedRowValue,
|
|
type ParserDiagnostic,
|
|
type ParserErrorReport,
|
|
} from "./index.js";
|
|
|
|
export type DuplicateKeyField = {
|
|
field: string;
|
|
label?: string;
|
|
};
|
|
|
|
export type DuplicateDetectionRow = {
|
|
rowNumber?: number;
|
|
normalized: NormalizedImportRow;
|
|
};
|
|
|
|
export type DuplicateKeyPart = {
|
|
field: string;
|
|
valuePreview: string;
|
|
};
|
|
|
|
export type DuplicateGroup = {
|
|
id: string;
|
|
key: readonly DuplicateKeyPart[];
|
|
rowNumbers: readonly number[];
|
|
};
|
|
|
|
export type DuplicateDetectionSummary = {
|
|
totalRows: number;
|
|
duplicateGroups: number;
|
|
duplicateRows: number;
|
|
diagnostics: readonly ParserDiagnostic[];
|
|
};
|
|
|
|
export type DuplicateDetectionReport = {
|
|
groups: readonly DuplicateGroup[];
|
|
rowResults: readonly ImportRowResult[];
|
|
summary: DuplicateDetectionSummary;
|
|
errorReport?: ParserErrorReport;
|
|
};
|
|
|
|
export type ValidationReportSummary = ImportDryRunSummary & {
|
|
duplicateGroups: number;
|
|
duplicateRows: number;
|
|
};
|
|
|
|
export type ValidationReport = {
|
|
summary: ValidationReportSummary;
|
|
diagnostics: readonly ParserDiagnostic[];
|
|
duplicateGroups: readonly DuplicateGroup[];
|
|
errorReport?: ParserErrorReport;
|
|
};
|
|
|
|
type DuplicateKeySignature = {
|
|
signature: string;
|
|
parts: readonly DuplicateKeyPart[];
|
|
};
|
|
|
|
function normalizeDuplicateKeyValue(value: NormalizedRowValue | undefined): string | undefined {
|
|
if (value === undefined || value === null) {
|
|
return undefined;
|
|
}
|
|
|
|
if (typeof value === "string") {
|
|
const trimmed = value.trim();
|
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
}
|
|
|
|
if (Array.isArray(value)) {
|
|
const normalizedItems = value
|
|
.map((item) => normalizeDuplicateKeyPrimitive(item))
|
|
.filter((item): item is string => item !== undefined);
|
|
|
|
return normalizedItems.length > 0 ? JSON.stringify(normalizedItems) : undefined;
|
|
}
|
|
|
|
return String(value);
|
|
}
|
|
|
|
function normalizeDuplicateKeyPrimitive(value: NormalizedRowPrimitive): string | undefined {
|
|
if (value === null) {
|
|
return undefined;
|
|
}
|
|
|
|
if (typeof value === "string") {
|
|
const trimmed = value.trim();
|
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
}
|
|
|
|
return String(value);
|
|
}
|
|
|
|
function toDuplicateKeyField(field: string | DuplicateKeyField): DuplicateKeyField {
|
|
return typeof field === "string" ? { field } : field;
|
|
}
|
|
|
|
function createDuplicateKey(row: NormalizedImportRow, fields: readonly DuplicateKeyField[]): DuplicateKeySignature | undefined {
|
|
const parts: DuplicateKeyPart[] = [];
|
|
const signatureParts: string[] = [];
|
|
|
|
for (const field of fields) {
|
|
if (!field.field) {
|
|
continue;
|
|
}
|
|
|
|
const normalizedValue = normalizeDuplicateKeyValue(row[field.field]);
|
|
if (normalizedValue === undefined) {
|
|
continue;
|
|
}
|
|
|
|
parts.push({ field: field.field, valuePreview: redactDiagnosticMessage(normalizedValue, 80) });
|
|
signatureParts.push(`${JSON.stringify(field.field)}:${JSON.stringify(normalizedValue)}`);
|
|
}
|
|
|
|
if (signatureParts.length === 0) {
|
|
return undefined;
|
|
}
|
|
|
|
return {
|
|
signature: signatureParts.join("|"),
|
|
parts,
|
|
};
|
|
}
|
|
|
|
function duplicateGroupId(index: number): string {
|
|
return `duplicate-group-${String(index + 1).padStart(4, "0")}`;
|
|
}
|
|
|
|
function createDuplicateDiagnostic(group: DuplicateGroup, rowNumber: number): ParserDiagnostic {
|
|
return createRedactedParserDiagnostic(
|
|
"duplicate_key",
|
|
`Duplicate import row matches ${group.id} on ${group.key.map((part) => `${part.field}=${part.valuePreview}`).join(", ")}.`,
|
|
{
|
|
rowNumber,
|
|
valuePreview: group.id,
|
|
},
|
|
);
|
|
}
|
|
|
|
export function detectDuplicateRows(
|
|
rows: readonly DuplicateDetectionRow[],
|
|
keyFields: readonly (string | DuplicateKeyField)[],
|
|
): DuplicateDetectionReport {
|
|
const fields = keyFields.map(toDuplicateKeyField).filter((field) => field.field.trim().length > 0);
|
|
const rowResults: ImportRowResult[] = rows.map((row, index) => ({
|
|
ok: true,
|
|
rowNumber: row.rowNumber ?? index + 1,
|
|
normalized: row.normalized,
|
|
warnings: undefined,
|
|
}));
|
|
|
|
if (fields.length === 0) {
|
|
return {
|
|
groups: [],
|
|
rowResults,
|
|
summary: {
|
|
totalRows: rows.length,
|
|
duplicateGroups: 0,
|
|
duplicateRows: 0,
|
|
diagnostics: [],
|
|
},
|
|
errorReport: undefined,
|
|
};
|
|
}
|
|
|
|
const bySignature = new Map<string, { parts: readonly DuplicateKeyPart[]; rowNumbers: number[] }>();
|
|
|
|
for (let index = 0; index < rows.length; index += 1) {
|
|
const row = rows[index];
|
|
const key = createDuplicateKey(row.normalized, fields);
|
|
if (!key) {
|
|
continue;
|
|
}
|
|
|
|
const rowNumber = row.rowNumber ?? index + 1;
|
|
const existing = bySignature.get(key.signature);
|
|
if (existing) {
|
|
existing.rowNumbers.push(rowNumber);
|
|
} else {
|
|
bySignature.set(key.signature, { parts: key.parts, rowNumbers: [rowNumber] });
|
|
}
|
|
}
|
|
|
|
const groups = Array.from(bySignature.values())
|
|
.filter((group) => group.rowNumbers.length > 1)
|
|
.sort((left, right) => left.rowNumbers[0] - right.rowNumbers[0])
|
|
.map<DuplicateGroup>((group, index) => ({
|
|
id: duplicateGroupId(index),
|
|
key: group.parts,
|
|
rowNumbers: [...group.rowNumbers].sort((left, right) => left - right),
|
|
}));
|
|
|
|
const diagnostics = groups.flatMap((group) => group.rowNumbers.map((rowNumber) => createDuplicateDiagnostic(group, rowNumber)));
|
|
const diagnosticsByRowNumber = new Map<number, ParserDiagnostic[]>();
|
|
for (const diagnostic of diagnostics) {
|
|
if (diagnostic.rowNumber === undefined) {
|
|
continue;
|
|
}
|
|
diagnosticsByRowNumber.set(diagnostic.rowNumber, [...(diagnosticsByRowNumber.get(diagnostic.rowNumber) ?? []), diagnostic]);
|
|
}
|
|
|
|
const duplicateRowResults = rowResults.map<ImportRowResult>((row) => {
|
|
const errors = diagnosticsByRowNumber.get(row.rowNumber);
|
|
if (!errors) {
|
|
return row;
|
|
}
|
|
|
|
return {
|
|
ok: false,
|
|
rowNumber: row.rowNumber,
|
|
errors,
|
|
} satisfies ImportRowErrorResult;
|
|
});
|
|
|
|
return {
|
|
groups,
|
|
rowResults: duplicateRowResults,
|
|
summary: {
|
|
totalRows: rows.length,
|
|
duplicateGroups: groups.length,
|
|
duplicateRows: new Set(groups.flatMap((group) => group.rowNumbers)).size,
|
|
diagnostics,
|
|
},
|
|
errorReport:
|
|
diagnostics.length > 0
|
|
? {
|
|
diagnostics,
|
|
rejectedRows: diagnosticsByRowNumber.size,
|
|
}
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
export function createValidationReport(
|
|
format: ImportFormat,
|
|
rowResults: readonly ImportRowResult[],
|
|
duplicateReport: Pick<DuplicateDetectionReport, "groups" | "summary">,
|
|
): ValidationReport {
|
|
const summaryCounts = summarizeRowResults(rowResults);
|
|
const diagnostics = [
|
|
...rowResults.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors)),
|
|
...duplicateReport.summary.diagnostics,
|
|
];
|
|
|
|
return {
|
|
summary: {
|
|
format,
|
|
...summaryCounts,
|
|
duplicateGroups: duplicateReport.summary.duplicateGroups,
|
|
duplicateRows: duplicateReport.summary.duplicateRows,
|
|
diagnostics,
|
|
},
|
|
diagnostics,
|
|
duplicateGroups: duplicateReport.groups,
|
|
errorReport:
|
|
diagnostics.length > 0
|
|
? {
|
|
diagnostics,
|
|
rejectedRows: summaryCounts.rejectedRows,
|
|
}
|
|
: undefined,
|
|
};
|
|
}
|