Reorganize repo structure and standards references

This commit is contained in:
david committed 2026-06-29 05:03:42 +02:00
1 parent 3cb93a81be
commit be2054274d
5885 files changed
+2588 -195

No files matched your search

@@ -0,0 +1,272 @@
import {
createRedactedParserDiagnostic,
redactDiagnosticMessage,
summarizeRowResults,
type ImportDryRunSummary,
type ImportFormat,
type ImportRowErrorResult,
type ImportRowResult,
type NormalizedImportRow,
type NormalizedRowPrimitive,
type NormalizedRowValue,
type ParserDiagnostic,
type ParserErrorReport,
} from "./index.js";
export type DuplicateKeyField = {
field: string;
label?: string;
};
export type DuplicateDetectionRow = {
rowNumber?: number;
normalized: NormalizedImportRow;
};
export type DuplicateKeyPart = {
field: string;
valuePreview: string;
};
export type DuplicateGroup = {
id: string;
key: readonly DuplicateKeyPart[];
rowNumbers: readonly number[];
};
export type DuplicateDetectionSummary = {
totalRows: number;
duplicateGroups: number;
duplicateRows: number;
diagnostics: readonly ParserDiagnostic[];
};
export type DuplicateDetectionReport = {
groups: readonly DuplicateGroup[];
rowResults: readonly ImportRowResult[];
summary: DuplicateDetectionSummary;
errorReport?: ParserErrorReport;
};
export type ValidationReportSummary = ImportDryRunSummary & {
duplicateGroups: number;
duplicateRows: number;
};
export type ValidationReport = {
summary: ValidationReportSummary;
diagnostics: readonly ParserDiagnostic[];
duplicateGroups: readonly DuplicateGroup[];
errorReport?: ParserErrorReport;
};
type DuplicateKeySignature = {
signature: string;
parts: readonly DuplicateKeyPart[];
};
function normalizeDuplicateKeyValue(value: NormalizedRowValue | undefined): string | undefined {
if (value === undefined || value === null) {
return undefined;
}
if (typeof value === "string") {
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : undefined;
}
if (Array.isArray(value)) {
const normalizedItems = value
.map((item) => normalizeDuplicateKeyPrimitive(item))
.filter((item): item is string => item !== undefined);
return normalizedItems.length > 0 ? JSON.stringify(normalizedItems) : undefined;
}
return String(value);
}
function normalizeDuplicateKeyPrimitive(value: NormalizedRowPrimitive): string | undefined {
if (value === null) {
return undefined;
}
if (typeof value === "string") {
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : undefined;
}
return String(value);
}
function toDuplicateKeyField(field: string | DuplicateKeyField): DuplicateKeyField {
return typeof field === "string" ? { field } : field;
}
function createDuplicateKey(row: NormalizedImportRow, fields: readonly DuplicateKeyField[]): DuplicateKeySignature | undefined {
const parts: DuplicateKeyPart[] = [];
const signatureParts: string[] = [];
for (const field of fields) {
if (!field.field) {
continue;
}
const normalizedValue = normalizeDuplicateKeyValue(row[field.field]);
if (normalizedValue === undefined) {
continue;
}
parts.push({ field: field.field, valuePreview: redactDiagnosticMessage(normalizedValue, 80) });
signatureParts.push(`${JSON.stringify(field.field)}:${JSON.stringify(normalizedValue)}`);
}
if (signatureParts.length === 0) {
return undefined;
}
return {
signature: signatureParts.join("|"),
parts,
};
}
function duplicateGroupId(index: number): string {
return `duplicate-group-${String(index + 1).padStart(4, "0")}`;
}
function createDuplicateDiagnostic(group: DuplicateGroup, rowNumber: number): ParserDiagnostic {
return createRedactedParserDiagnostic(
"duplicate_key",
`Duplicate import row matches ${group.id} on ${group.key.map((part) => `${part.field}=${part.valuePreview}`).join(", ")}.`,
{
rowNumber,
valuePreview: group.id,
},
);
}
export function detectDuplicateRows(
rows: readonly DuplicateDetectionRow[],
keyFields: readonly (string | DuplicateKeyField)[],
): DuplicateDetectionReport {
const fields = keyFields.map(toDuplicateKeyField).filter((field) => field.field.trim().length > 0);
const rowResults: ImportRowResult[] = rows.map((row, index) => ({
ok: true,
rowNumber: row.rowNumber ?? index + 1,
normalized: row.normalized,
warnings: undefined,
}));
if (fields.length === 0) {
return {
groups: [],
rowResults,
summary: {
totalRows: rows.length,
duplicateGroups: 0,
duplicateRows: 0,
diagnostics: [],
},
errorReport: undefined,
};
}
const bySignature = new Map<string, { parts: readonly DuplicateKeyPart[]; rowNumbers: number[] }>();
for (let index = 0; index < rows.length; index += 1) {
const row = rows[index];
const key = createDuplicateKey(row.normalized, fields);
if (!key) {
continue;
}
const rowNumber = row.rowNumber ?? index + 1;
const existing = bySignature.get(key.signature);
if (existing) {
existing.rowNumbers.push(rowNumber);
} else {
bySignature.set(key.signature, { parts: key.parts, rowNumbers: [rowNumber] });
}
}
const groups = Array.from(bySignature.values())
.filter((group) => group.rowNumbers.length > 1)
.sort((left, right) => left.rowNumbers[0] - right.rowNumbers[0])
.map<DuplicateGroup>((group, index) => ({
id: duplicateGroupId(index),
key: group.parts,
rowNumbers: [...group.rowNumbers].sort((left, right) => left - right),
}));
const diagnostics = groups.flatMap((group) => group.rowNumbers.map((rowNumber) => createDuplicateDiagnostic(group, rowNumber)));
const diagnosticsByRowNumber = new Map<number, ParserDiagnostic[]>();
for (const diagnostic of diagnostics) {
if (diagnostic.rowNumber === undefined) {
continue;
}
diagnosticsByRowNumber.set(diagnostic.rowNumber, [...(diagnosticsByRowNumber.get(diagnostic.rowNumber) ?? []), diagnostic]);
}
const duplicateRowResults = rowResults.map<ImportRowResult>((row) => {
const errors = diagnosticsByRowNumber.get(row.rowNumber);
if (!errors) {
return row;
}
return {
ok: false,
rowNumber: row.rowNumber,
errors,
} satisfies ImportRowErrorResult;
});
return {
groups,
rowResults: duplicateRowResults,
summary: {
totalRows: rows.length,
duplicateGroups: groups.length,
duplicateRows: new Set(groups.flatMap((group) => group.rowNumbers)).size,
diagnostics,
},
errorReport:
diagnostics.length > 0
? {
diagnostics,
rejectedRows: diagnosticsByRowNumber.size,
}
: undefined,
};
}
export function createValidationReport(
format: ImportFormat,
rowResults: readonly ImportRowResult[],
duplicateReport: Pick<DuplicateDetectionReport, "groups" | "summary">,
): ValidationReport {
const summaryCounts = summarizeRowResults(rowResults);
const diagnostics = [
...rowResults.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors)),
...duplicateReport.summary.diagnostics,
];
return {
summary: {
format,
...summaryCounts,
duplicateGroups: duplicateReport.summary.duplicateGroups,
duplicateRows: duplicateReport.summary.duplicateRows,
diagnostics,
},
diagnostics,
duplicateGroups: duplicateReport.groups,
errorReport:
diagnostics.length > 0
? {
diagnostics,
rejectedRows: summaryCounts.rejectedRows,
}
: undefined,
};
}