Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,272 @@
|
||||
import {
|
||||
createRedactedParserDiagnostic,
|
||||
redactDiagnosticMessage,
|
||||
summarizeRowResults,
|
||||
type ImportDryRunSummary,
|
||||
type ImportFormat,
|
||||
type ImportRowErrorResult,
|
||||
type ImportRowResult,
|
||||
type NormalizedImportRow,
|
||||
type NormalizedRowPrimitive,
|
||||
type NormalizedRowValue,
|
||||
type ParserDiagnostic,
|
||||
type ParserErrorReport,
|
||||
} from "./index.js";
|
||||
|
||||
export type DuplicateKeyField = {
|
||||
field: string;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type DuplicateDetectionRow = {
|
||||
rowNumber?: number;
|
||||
normalized: NormalizedImportRow;
|
||||
};
|
||||
|
||||
export type DuplicateKeyPart = {
|
||||
field: string;
|
||||
valuePreview: string;
|
||||
};
|
||||
|
||||
export type DuplicateGroup = {
|
||||
id: string;
|
||||
key: readonly DuplicateKeyPart[];
|
||||
rowNumbers: readonly number[];
|
||||
};
|
||||
|
||||
export type DuplicateDetectionSummary = {
|
||||
totalRows: number;
|
||||
duplicateGroups: number;
|
||||
duplicateRows: number;
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
};
|
||||
|
||||
export type DuplicateDetectionReport = {
|
||||
groups: readonly DuplicateGroup[];
|
||||
rowResults: readonly ImportRowResult[];
|
||||
summary: DuplicateDetectionSummary;
|
||||
errorReport?: ParserErrorReport;
|
||||
};
|
||||
|
||||
export type ValidationReportSummary = ImportDryRunSummary & {
|
||||
duplicateGroups: number;
|
||||
duplicateRows: number;
|
||||
};
|
||||
|
||||
export type ValidationReport = {
|
||||
summary: ValidationReportSummary;
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
duplicateGroups: readonly DuplicateGroup[];
|
||||
errorReport?: ParserErrorReport;
|
||||
};
|
||||
|
||||
type DuplicateKeySignature = {
|
||||
signature: string;
|
||||
parts: readonly DuplicateKeyPart[];
|
||||
};
|
||||
|
||||
function normalizeDuplicateKeyValue(value: NormalizedRowValue | undefined): string | undefined {
|
||||
if (value === undefined || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
const normalizedItems = value
|
||||
.map((item) => normalizeDuplicateKeyPrimitive(item))
|
||||
.filter((item): item is string => item !== undefined);
|
||||
|
||||
return normalizedItems.length > 0 ? JSON.stringify(normalizedItems) : undefined;
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function normalizeDuplicateKeyPrimitive(value: NormalizedRowPrimitive): string | undefined {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function toDuplicateKeyField(field: string | DuplicateKeyField): DuplicateKeyField {
|
||||
return typeof field === "string" ? { field } : field;
|
||||
}
|
||||
|
||||
function createDuplicateKey(row: NormalizedImportRow, fields: readonly DuplicateKeyField[]): DuplicateKeySignature | undefined {
|
||||
const parts: DuplicateKeyPart[] = [];
|
||||
const signatureParts: string[] = [];
|
||||
|
||||
for (const field of fields) {
|
||||
if (!field.field) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalizedValue = normalizeDuplicateKeyValue(row[field.field]);
|
||||
if (normalizedValue === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
parts.push({ field: field.field, valuePreview: redactDiagnosticMessage(normalizedValue, 80) });
|
||||
signatureParts.push(`${JSON.stringify(field.field)}:${JSON.stringify(normalizedValue)}`);
|
||||
}
|
||||
|
||||
if (signatureParts.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
signature: signatureParts.join("|"),
|
||||
parts,
|
||||
};
|
||||
}
|
||||
|
||||
function duplicateGroupId(index: number): string {
|
||||
return `duplicate-group-${String(index + 1).padStart(4, "0")}`;
|
||||
}
|
||||
|
||||
function createDuplicateDiagnostic(group: DuplicateGroup, rowNumber: number): ParserDiagnostic {
|
||||
return createRedactedParserDiagnostic(
|
||||
"duplicate_key",
|
||||
`Duplicate import row matches ${group.id} on ${group.key.map((part) => `${part.field}=${part.valuePreview}`).join(", ")}.`,
|
||||
{
|
||||
rowNumber,
|
||||
valuePreview: group.id,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function detectDuplicateRows(
|
||||
rows: readonly DuplicateDetectionRow[],
|
||||
keyFields: readonly (string | DuplicateKeyField)[],
|
||||
): DuplicateDetectionReport {
|
||||
const fields = keyFields.map(toDuplicateKeyField).filter((field) => field.field.trim().length > 0);
|
||||
const rowResults: ImportRowResult[] = rows.map((row, index) => ({
|
||||
ok: true,
|
||||
rowNumber: row.rowNumber ?? index + 1,
|
||||
normalized: row.normalized,
|
||||
warnings: undefined,
|
||||
}));
|
||||
|
||||
if (fields.length === 0) {
|
||||
return {
|
||||
groups: [],
|
||||
rowResults,
|
||||
summary: {
|
||||
totalRows: rows.length,
|
||||
duplicateGroups: 0,
|
||||
duplicateRows: 0,
|
||||
diagnostics: [],
|
||||
},
|
||||
errorReport: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const bySignature = new Map<string, { parts: readonly DuplicateKeyPart[]; rowNumbers: number[] }>();
|
||||
|
||||
for (let index = 0; index < rows.length; index += 1) {
|
||||
const row = rows[index];
|
||||
const key = createDuplicateKey(row.normalized, fields);
|
||||
if (!key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const rowNumber = row.rowNumber ?? index + 1;
|
||||
const existing = bySignature.get(key.signature);
|
||||
if (existing) {
|
||||
existing.rowNumbers.push(rowNumber);
|
||||
} else {
|
||||
bySignature.set(key.signature, { parts: key.parts, rowNumbers: [rowNumber] });
|
||||
}
|
||||
}
|
||||
|
||||
const groups = Array.from(bySignature.values())
|
||||
.filter((group) => group.rowNumbers.length > 1)
|
||||
.sort((left, right) => left.rowNumbers[0] - right.rowNumbers[0])
|
||||
.map<DuplicateGroup>((group, index) => ({
|
||||
id: duplicateGroupId(index),
|
||||
key: group.parts,
|
||||
rowNumbers: [...group.rowNumbers].sort((left, right) => left - right),
|
||||
}));
|
||||
|
||||
const diagnostics = groups.flatMap((group) => group.rowNumbers.map((rowNumber) => createDuplicateDiagnostic(group, rowNumber)));
|
||||
const diagnosticsByRowNumber = new Map<number, ParserDiagnostic[]>();
|
||||
for (const diagnostic of diagnostics) {
|
||||
if (diagnostic.rowNumber === undefined) {
|
||||
continue;
|
||||
}
|
||||
diagnosticsByRowNumber.set(diagnostic.rowNumber, [...(diagnosticsByRowNumber.get(diagnostic.rowNumber) ?? []), diagnostic]);
|
||||
}
|
||||
|
||||
const duplicateRowResults = rowResults.map<ImportRowResult>((row) => {
|
||||
const errors = diagnosticsByRowNumber.get(row.rowNumber);
|
||||
if (!errors) {
|
||||
return row;
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
rowNumber: row.rowNumber,
|
||||
errors,
|
||||
} satisfies ImportRowErrorResult;
|
||||
});
|
||||
|
||||
return {
|
||||
groups,
|
||||
rowResults: duplicateRowResults,
|
||||
summary: {
|
||||
totalRows: rows.length,
|
||||
duplicateGroups: groups.length,
|
||||
duplicateRows: new Set(groups.flatMap((group) => group.rowNumbers)).size,
|
||||
diagnostics,
|
||||
},
|
||||
errorReport:
|
||||
diagnostics.length > 0
|
||||
? {
|
||||
diagnostics,
|
||||
rejectedRows: diagnosticsByRowNumber.size,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function createValidationReport(
|
||||
format: ImportFormat,
|
||||
rowResults: readonly ImportRowResult[],
|
||||
duplicateReport: Pick<DuplicateDetectionReport, "groups" | "summary">,
|
||||
): ValidationReport {
|
||||
const summaryCounts = summarizeRowResults(rowResults);
|
||||
const diagnostics = [
|
||||
...rowResults.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors)),
|
||||
...duplicateReport.summary.diagnostics,
|
||||
];
|
||||
|
||||
return {
|
||||
summary: {
|
||||
format,
|
||||
...summaryCounts,
|
||||
duplicateGroups: duplicateReport.summary.duplicateGroups,
|
||||
duplicateRows: duplicateReport.summary.duplicateRows,
|
||||
diagnostics,
|
||||
},
|
||||
diagnostics,
|
||||
duplicateGroups: duplicateReport.groups,
|
||||
errorReport:
|
||||
diagnostics.length > 0
|
||||
? {
|
||||
diagnostics,
|
||||
rejectedRows: summaryCounts.rejectedRows,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user