Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,319 @@
|
||||
export * from "./dedupe.js";
|
||||
|
||||
export const IMPORT_FORMATS = ["csv", "json", "xlsx"] as const;
|
||||
export type ImportFormat = (typeof IMPORT_FORMATS)[number];
|
||||
|
||||
export type ImportSourceDescriptor = {
|
||||
tenantId: string;
|
||||
requesterId: string;
|
||||
sourceKey: string;
|
||||
fileName?: string;
|
||||
contentType?: string;
|
||||
byteLength?: number;
|
||||
requestId?: string;
|
||||
};
|
||||
|
||||
export type ColumnMapping = {
|
||||
sourceColumn: string;
|
||||
targetField: string;
|
||||
required?: boolean;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type NormalizedRowPrimitive = string | number | boolean | null;
|
||||
export type NormalizedRowValue = NormalizedRowPrimitive | readonly NormalizedRowPrimitive[];
|
||||
export type NormalizedImportRow = Record<string, NormalizedRowValue>;
|
||||
|
||||
export type ImportRowSuccessResult = {
|
||||
ok: true;
|
||||
rowNumber: number;
|
||||
normalized: NormalizedImportRow;
|
||||
warnings?: readonly ParserDiagnostic[];
|
||||
};
|
||||
|
||||
export type ImportRowErrorResult = {
|
||||
ok: false;
|
||||
rowNumber: number;
|
||||
errors: readonly ParserDiagnostic[];
|
||||
rejectedValues?: NormalizedImportRow;
|
||||
};
|
||||
|
||||
export type ImportRowResult = ImportRowSuccessResult | ImportRowErrorResult;
|
||||
|
||||
export type ParserDiagnosticSeverity = "info" | "warning" | "error";
|
||||
|
||||
export type ParserDiagnostic = {
|
||||
severity: ParserDiagnosticSeverity;
|
||||
code: string;
|
||||
message: string;
|
||||
rowNumber?: number;
|
||||
column?: string;
|
||||
valuePreview?: string;
|
||||
};
|
||||
|
||||
export type ParserErrorReport = {
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
rejectedRows: number;
|
||||
errorReportKey?: string;
|
||||
};
|
||||
|
||||
export type ImportDryRunSummary = {
|
||||
format: ImportFormat;
|
||||
totalRows: number;
|
||||
acceptedRows: number;
|
||||
rejectedRows: number;
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
};
|
||||
|
||||
export type ImportParserInput = {
|
||||
source: ImportSourceDescriptor;
|
||||
format: ImportFormat;
|
||||
mappings: readonly ColumnMapping[];
|
||||
text?: string;
|
||||
bytes?: Uint8Array;
|
||||
};
|
||||
|
||||
export type ImportParserOutput = {
|
||||
rows: readonly ImportRowResult[];
|
||||
summary: ImportDryRunSummary;
|
||||
errorReport?: ParserErrorReport;
|
||||
};
|
||||
|
||||
export type ImportParser = {
|
||||
readonly format: ImportFormat;
|
||||
parse(input: ImportParserInput): Promise<ImportParserOutput> | ImportParserOutput;
|
||||
};
|
||||
|
||||
export type ImportDryRunFieldDefinition = {
|
||||
targetField: string;
|
||||
required?: boolean;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type ImportPreviewInputRow = {
|
||||
rowNumber?: number;
|
||||
values: NormalizedImportRow;
|
||||
};
|
||||
|
||||
export type ImportMappedPreviewRow = {
|
||||
rowNumber: number;
|
||||
normalized: NormalizedImportRow;
|
||||
sourceColumns: readonly string[];
|
||||
};
|
||||
|
||||
export type ImportDryRunPreviewInput = {
|
||||
format: ImportFormat;
|
||||
fields: readonly ImportDryRunFieldDefinition[];
|
||||
mappings: readonly ColumnMapping[];
|
||||
rows: readonly ImportPreviewInputRow[];
|
||||
};
|
||||
|
||||
export type ImportDryRunPreviewOutput = ImportParserOutput & {
|
||||
mappedRows: readonly ImportMappedPreviewRow[];
|
||||
};
|
||||
|
||||
const EXTENSION_FORMATS: Readonly<Record<string, ImportFormat>> = {
|
||||
csv: "csv",
|
||||
json: "json",
|
||||
xlsx: "xlsx",
|
||||
};
|
||||
|
||||
const CONTENT_TYPE_FORMATS: Readonly<Record<string, ImportFormat>> = {
|
||||
"application/csv": "csv",
|
||||
"application/json": "json",
|
||||
"application/vnd.ms-excel": "csv",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
|
||||
"text/csv": "csv",
|
||||
};
|
||||
|
||||
const SECRET_PATTERN = /(?:body|payload|raw(?:file|body|payload|rows)?)\s*[:=]\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)|(?:api[_-]?key|authorization|bearer|password|secret|token)\s*[:=]\s*[^\s,;]+/gi;
|
||||
const REDACTED = "[REDACTED]";
|
||||
|
||||
export function isSupportedImportFormat(value: string): value is ImportFormat {
|
||||
return (IMPORT_FORMATS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
export function detectImportFormatFromExtension(fileNameOrExtension: string | undefined): ImportFormat | undefined {
|
||||
if (!fileNameOrExtension) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const extension = fileNameOrExtension.trim().toLowerCase().replace(/^.*\./, "");
|
||||
return EXTENSION_FORMATS[extension];
|
||||
}
|
||||
|
||||
export function detectImportFormatFromContentType(contentType: string | undefined): ImportFormat | undefined {
|
||||
if (!contentType) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase();
|
||||
return normalized ? CONTENT_TYPE_FORMATS[normalized] : undefined;
|
||||
}
|
||||
|
||||
export function summarizeRowResults(rows: readonly ImportRowResult[]): Pick<ImportDryRunSummary, "totalRows" | "acceptedRows" | "rejectedRows"> {
|
||||
const acceptedRows = rows.filter((row) => row.ok).length;
|
||||
|
||||
return {
|
||||
totalRows: rows.length,
|
||||
acceptedRows,
|
||||
rejectedRows: rows.length - acceptedRows,
|
||||
};
|
||||
}
|
||||
|
||||
function isBlankPreviewValue(value: NormalizedRowValue | undefined): boolean {
|
||||
if (value === undefined || value === null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
return value.trim().length === 0;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.length === 0 || value.every((item) => isBlankPreviewValue(item));
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function previewValue(value: NormalizedRowValue | undefined): string | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return redactDiagnosticMessage(String(value), 80);
|
||||
}
|
||||
|
||||
function buildRequiredFieldDefinitions(
|
||||
fields: readonly ImportDryRunFieldDefinition[],
|
||||
mappings: readonly ColumnMapping[],
|
||||
): Map<string, ImportDryRunFieldDefinition> {
|
||||
const requiredFields = new Map<string, ImportDryRunFieldDefinition>();
|
||||
|
||||
for (const field of fields) {
|
||||
if (field.required) {
|
||||
requiredFields.set(field.targetField, field);
|
||||
}
|
||||
}
|
||||
|
||||
for (const mapping of mappings) {
|
||||
if (mapping.required) {
|
||||
requiredFields.set(mapping.targetField, {
|
||||
targetField: mapping.targetField,
|
||||
required: true,
|
||||
label: mapping.label,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return requiredFields;
|
||||
}
|
||||
|
||||
export function createImportDryRunPreview(input: ImportDryRunPreviewInput): ImportDryRunPreviewOutput {
|
||||
const requiredFields = buildRequiredFieldDefinitions(input.fields, input.mappings);
|
||||
const rows = input.rows.map<ImportRowResult>((row, index) => {
|
||||
const rowNumber = row.rowNumber ?? index + 1;
|
||||
const normalized: NormalizedImportRow = {};
|
||||
const sourceColumns: string[] = [];
|
||||
|
||||
for (const mapping of input.mappings) {
|
||||
if (!mapping.sourceColumn || !mapping.targetField) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Object.prototype.hasOwnProperty.call(row.values, mapping.sourceColumn)) {
|
||||
normalized[mapping.targetField] = row.values[mapping.sourceColumn] ?? null;
|
||||
sourceColumns.push(mapping.sourceColumn);
|
||||
}
|
||||
}
|
||||
|
||||
const errors: ParserDiagnostic[] = [];
|
||||
|
||||
for (const [targetField, field] of requiredFields) {
|
||||
if (isBlankPreviewValue(normalized[targetField])) {
|
||||
errors.push(
|
||||
createRedactedParserDiagnostic(
|
||||
"missing_required_field",
|
||||
`${field.label ?? targetField} is required for import preview.`,
|
||||
{
|
||||
rowNumber,
|
||||
column: targetField,
|
||||
valuePreview: previewValue(normalized[targetField]),
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
rowNumber,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
rowNumber,
|
||||
normalized,
|
||||
warnings: undefined,
|
||||
};
|
||||
});
|
||||
|
||||
const mappedRows = rows
|
||||
.filter((row): row is ImportRowSuccessResult => row.ok)
|
||||
.map<ImportMappedPreviewRow>((row) => ({
|
||||
rowNumber: row.rowNumber,
|
||||
normalized: row.normalized,
|
||||
sourceColumns: input.mappings
|
||||
.filter((mapping) => Object.prototype.hasOwnProperty.call(row.normalized, mapping.targetField))
|
||||
.map((mapping) => mapping.sourceColumn),
|
||||
}));
|
||||
const diagnostics = rows.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors));
|
||||
const summaryCounts = summarizeRowResults(rows);
|
||||
|
||||
return {
|
||||
rows,
|
||||
mappedRows,
|
||||
summary: {
|
||||
format: input.format,
|
||||
...summaryCounts,
|
||||
diagnostics,
|
||||
},
|
||||
errorReport:
|
||||
summaryCounts.rejectedRows > 0
|
||||
? {
|
||||
diagnostics,
|
||||
rejectedRows: summaryCounts.rejectedRows,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function createRedactedParserDiagnostic(
|
||||
code: string,
|
||||
message: string,
|
||||
options: Omit<Partial<ParserDiagnostic>, "code" | "message"> = {},
|
||||
): ParserDiagnostic {
|
||||
return {
|
||||
severity: "error",
|
||||
...options,
|
||||
code,
|
||||
message: redactDiagnosticMessage(message),
|
||||
};
|
||||
}
|
||||
|
||||
export function redactDiagnosticMessage(message: string, maxLength = 300): string {
|
||||
const redacted = message.replace(SECRET_PATTERN, (match) => {
|
||||
const colonIndex = match.indexOf(":");
|
||||
const equalsIndex = match.indexOf("=");
|
||||
const separatorIndex =
|
||||
colonIndex === -1 ? equalsIndex : equalsIndex === -1 ? colonIndex : Math.min(colonIndex, equalsIndex);
|
||||
const prefix = separatorIndex >= 0 ? match.slice(0, separatorIndex + 1) : "secret:";
|
||||
return `${prefix} ${REDACTED}`;
|
||||
});
|
||||
|
||||
return redacted.length > maxLength ? `${redacted.slice(0, maxLength - 1)}…` : redacted;
|
||||
}
|
||||
Reference in new issue
Block a user