320 lines
8.7 KiB
TypeScript
320 lines
8.7 KiB
TypeScript
export * from "./dedupe.js";
|
|
|
|
export const IMPORT_FORMATS = ["csv", "json", "xlsx"] as const;
|
|
export type ImportFormat = (typeof IMPORT_FORMATS)[number];
|
|
|
|
export type ImportSourceDescriptor = {
|
|
tenantId: string;
|
|
requesterId: string;
|
|
sourceKey: string;
|
|
fileName?: string;
|
|
contentType?: string;
|
|
byteLength?: number;
|
|
requestId?: string;
|
|
};
|
|
|
|
export type ColumnMapping = {
|
|
sourceColumn: string;
|
|
targetField: string;
|
|
required?: boolean;
|
|
label?: string;
|
|
};
|
|
|
|
export type NormalizedRowPrimitive = string | number | boolean | null;
|
|
export type NormalizedRowValue = NormalizedRowPrimitive | readonly NormalizedRowPrimitive[];
|
|
export type NormalizedImportRow = Record<string, NormalizedRowValue>;
|
|
|
|
export type ImportRowSuccessResult = {
|
|
ok: true;
|
|
rowNumber: number;
|
|
normalized: NormalizedImportRow;
|
|
warnings?: readonly ParserDiagnostic[];
|
|
};
|
|
|
|
export type ImportRowErrorResult = {
|
|
ok: false;
|
|
rowNumber: number;
|
|
errors: readonly ParserDiagnostic[];
|
|
rejectedValues?: NormalizedImportRow;
|
|
};
|
|
|
|
export type ImportRowResult = ImportRowSuccessResult | ImportRowErrorResult;
|
|
|
|
export type ParserDiagnosticSeverity = "info" | "warning" | "error";
|
|
|
|
export type ParserDiagnostic = {
|
|
severity: ParserDiagnosticSeverity;
|
|
code: string;
|
|
message: string;
|
|
rowNumber?: number;
|
|
column?: string;
|
|
valuePreview?: string;
|
|
};
|
|
|
|
export type ParserErrorReport = {
|
|
diagnostics: readonly ParserDiagnostic[];
|
|
rejectedRows: number;
|
|
errorReportKey?: string;
|
|
};
|
|
|
|
export type ImportDryRunSummary = {
|
|
format: ImportFormat;
|
|
totalRows: number;
|
|
acceptedRows: number;
|
|
rejectedRows: number;
|
|
diagnostics: readonly ParserDiagnostic[];
|
|
};
|
|
|
|
export type ImportParserInput = {
|
|
source: ImportSourceDescriptor;
|
|
format: ImportFormat;
|
|
mappings: readonly ColumnMapping[];
|
|
text?: string;
|
|
bytes?: Uint8Array;
|
|
};
|
|
|
|
export type ImportParserOutput = {
|
|
rows: readonly ImportRowResult[];
|
|
summary: ImportDryRunSummary;
|
|
errorReport?: ParserErrorReport;
|
|
};
|
|
|
|
export type ImportParser = {
|
|
readonly format: ImportFormat;
|
|
parse(input: ImportParserInput): Promise<ImportParserOutput> | ImportParserOutput;
|
|
};
|
|
|
|
export type ImportDryRunFieldDefinition = {
|
|
targetField: string;
|
|
required?: boolean;
|
|
label?: string;
|
|
};
|
|
|
|
export type ImportPreviewInputRow = {
|
|
rowNumber?: number;
|
|
values: NormalizedImportRow;
|
|
};
|
|
|
|
export type ImportMappedPreviewRow = {
|
|
rowNumber: number;
|
|
normalized: NormalizedImportRow;
|
|
sourceColumns: readonly string[];
|
|
};
|
|
|
|
export type ImportDryRunPreviewInput = {
|
|
format: ImportFormat;
|
|
fields: readonly ImportDryRunFieldDefinition[];
|
|
mappings: readonly ColumnMapping[];
|
|
rows: readonly ImportPreviewInputRow[];
|
|
};
|
|
|
|
export type ImportDryRunPreviewOutput = ImportParserOutput & {
|
|
mappedRows: readonly ImportMappedPreviewRow[];
|
|
};
|
|
|
|
const EXTENSION_FORMATS: Readonly<Record<string, ImportFormat>> = {
|
|
csv: "csv",
|
|
json: "json",
|
|
xlsx: "xlsx",
|
|
};
|
|
|
|
const CONTENT_TYPE_FORMATS: Readonly<Record<string, ImportFormat>> = {
|
|
"application/csv": "csv",
|
|
"application/json": "json",
|
|
"application/vnd.ms-excel": "csv",
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
|
|
"text/csv": "csv",
|
|
};
|
|
|
|
const SECRET_PATTERN = /(?:body|payload|raw(?:file|body|payload|rows)?)\s*[:=]\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)|(?:api[_-]?key|authorization|bearer|password|secret|token)\s*[:=]\s*[^\s,;]+/gi;
|
|
const REDACTED = "[REDACTED]";
|
|
|
|
export function isSupportedImportFormat(value: string): value is ImportFormat {
|
|
return (IMPORT_FORMATS as readonly string[]).includes(value);
|
|
}
|
|
|
|
export function detectImportFormatFromExtension(fileNameOrExtension: string | undefined): ImportFormat | undefined {
|
|
if (!fileNameOrExtension) {
|
|
return undefined;
|
|
}
|
|
|
|
const extension = fileNameOrExtension.trim().toLowerCase().replace(/^.*\./, "");
|
|
return EXTENSION_FORMATS[extension];
|
|
}
|
|
|
|
export function detectImportFormatFromContentType(contentType: string | undefined): ImportFormat | undefined {
|
|
if (!contentType) {
|
|
return undefined;
|
|
}
|
|
|
|
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase();
|
|
return normalized ? CONTENT_TYPE_FORMATS[normalized] : undefined;
|
|
}
|
|
|
|
export function summarizeRowResults(rows: readonly ImportRowResult[]): Pick<ImportDryRunSummary, "totalRows" | "acceptedRows" | "rejectedRows"> {
|
|
const acceptedRows = rows.filter((row) => row.ok).length;
|
|
|
|
return {
|
|
totalRows: rows.length,
|
|
acceptedRows,
|
|
rejectedRows: rows.length - acceptedRows,
|
|
};
|
|
}
|
|
|
|
function isBlankPreviewValue(value: NormalizedRowValue | undefined): boolean {
|
|
if (value === undefined || value === null) {
|
|
return true;
|
|
}
|
|
|
|
if (typeof value === "string") {
|
|
return value.trim().length === 0;
|
|
}
|
|
|
|
if (Array.isArray(value)) {
|
|
return value.length === 0 || value.every((item) => isBlankPreviewValue(item));
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
function previewValue(value: NormalizedRowValue | undefined): string | undefined {
|
|
if (value === undefined) {
|
|
return undefined;
|
|
}
|
|
|
|
return redactDiagnosticMessage(String(value), 80);
|
|
}
|
|
|
|
function buildRequiredFieldDefinitions(
|
|
fields: readonly ImportDryRunFieldDefinition[],
|
|
mappings: readonly ColumnMapping[],
|
|
): Map<string, ImportDryRunFieldDefinition> {
|
|
const requiredFields = new Map<string, ImportDryRunFieldDefinition>();
|
|
|
|
for (const field of fields) {
|
|
if (field.required) {
|
|
requiredFields.set(field.targetField, field);
|
|
}
|
|
}
|
|
|
|
for (const mapping of mappings) {
|
|
if (mapping.required) {
|
|
requiredFields.set(mapping.targetField, {
|
|
targetField: mapping.targetField,
|
|
required: true,
|
|
label: mapping.label,
|
|
});
|
|
}
|
|
}
|
|
|
|
return requiredFields;
|
|
}
|
|
|
|
export function createImportDryRunPreview(input: ImportDryRunPreviewInput): ImportDryRunPreviewOutput {
|
|
const requiredFields = buildRequiredFieldDefinitions(input.fields, input.mappings);
|
|
const rows = input.rows.map<ImportRowResult>((row, index) => {
|
|
const rowNumber = row.rowNumber ?? index + 1;
|
|
const normalized: NormalizedImportRow = {};
|
|
const sourceColumns: string[] = [];
|
|
|
|
for (const mapping of input.mappings) {
|
|
if (!mapping.sourceColumn || !mapping.targetField) {
|
|
continue;
|
|
}
|
|
|
|
if (Object.prototype.hasOwnProperty.call(row.values, mapping.sourceColumn)) {
|
|
normalized[mapping.targetField] = row.values[mapping.sourceColumn] ?? null;
|
|
sourceColumns.push(mapping.sourceColumn);
|
|
}
|
|
}
|
|
|
|
const errors: ParserDiagnostic[] = [];
|
|
|
|
for (const [targetField, field] of requiredFields) {
|
|
if (isBlankPreviewValue(normalized[targetField])) {
|
|
errors.push(
|
|
createRedactedParserDiagnostic(
|
|
"missing_required_field",
|
|
`${field.label ?? targetField} is required for import preview.`,
|
|
{
|
|
rowNumber,
|
|
column: targetField,
|
|
valuePreview: previewValue(normalized[targetField]),
|
|
},
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
if (errors.length > 0) {
|
|
return {
|
|
ok: false,
|
|
rowNumber,
|
|
errors,
|
|
};
|
|
}
|
|
|
|
return {
|
|
ok: true,
|
|
rowNumber,
|
|
normalized,
|
|
warnings: undefined,
|
|
};
|
|
});
|
|
|
|
const mappedRows = rows
|
|
.filter((row): row is ImportRowSuccessResult => row.ok)
|
|
.map<ImportMappedPreviewRow>((row) => ({
|
|
rowNumber: row.rowNumber,
|
|
normalized: row.normalized,
|
|
sourceColumns: input.mappings
|
|
.filter((mapping) => Object.prototype.hasOwnProperty.call(row.normalized, mapping.targetField))
|
|
.map((mapping) => mapping.sourceColumn),
|
|
}));
|
|
const diagnostics = rows.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors));
|
|
const summaryCounts = summarizeRowResults(rows);
|
|
|
|
return {
|
|
rows,
|
|
mappedRows,
|
|
summary: {
|
|
format: input.format,
|
|
...summaryCounts,
|
|
diagnostics,
|
|
},
|
|
errorReport:
|
|
summaryCounts.rejectedRows > 0
|
|
? {
|
|
diagnostics,
|
|
rejectedRows: summaryCounts.rejectedRows,
|
|
}
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
export function createRedactedParserDiagnostic(
|
|
code: string,
|
|
message: string,
|
|
options: Omit<Partial<ParserDiagnostic>, "code" | "message"> = {},
|
|
): ParserDiagnostic {
|
|
return {
|
|
severity: "error",
|
|
...options,
|
|
code,
|
|
message: redactDiagnosticMessage(message),
|
|
};
|
|
}
|
|
|
|
export function redactDiagnosticMessage(message: string, maxLength = 300): string {
|
|
const redacted = message.replace(SECRET_PATTERN, (match) => {
|
|
const colonIndex = match.indexOf(":");
|
|
const equalsIndex = match.indexOf("=");
|
|
const separatorIndex =
|
|
colonIndex === -1 ? equalsIndex : equalsIndex === -1 ? colonIndex : Math.min(colonIndex, equalsIndex);
|
|
const prefix = separatorIndex >= 0 ? match.slice(0, separatorIndex + 1) : "secret:";
|
|
return `${prefix} ${REDACTED}`;
|
|
});
|
|
|
|
return redacted.length > maxLength ? `${redacted.slice(0, maxLength - 1)}…` : redacted;
|
|
}
|