Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/packages/imports/src/index.ts
T

320 lines
8.7 KiB
TypeScript

export * from "./dedupe.js";
export const IMPORT_FORMATS = ["csv", "json", "xlsx"] as const;
export type ImportFormat = (typeof IMPORT_FORMATS)[number];
export type ImportSourceDescriptor = {
tenantId: string;
requesterId: string;
sourceKey: string;
fileName?: string;
contentType?: string;
byteLength?: number;
requestId?: string;
};
export type ColumnMapping = {
sourceColumn: string;
targetField: string;
required?: boolean;
label?: string;
};
export type NormalizedRowPrimitive = string | number | boolean | null;
export type NormalizedRowValue = NormalizedRowPrimitive | readonly NormalizedRowPrimitive[];
export type NormalizedImportRow = Record<string, NormalizedRowValue>;
export type ImportRowSuccessResult = {
ok: true;
rowNumber: number;
normalized: NormalizedImportRow;
warnings?: readonly ParserDiagnostic[];
};
export type ImportRowErrorResult = {
ok: false;
rowNumber: number;
errors: readonly ParserDiagnostic[];
rejectedValues?: NormalizedImportRow;
};
export type ImportRowResult = ImportRowSuccessResult | ImportRowErrorResult;
export type ParserDiagnosticSeverity = "info" | "warning" | "error";
export type ParserDiagnostic = {
severity: ParserDiagnosticSeverity;
code: string;
message: string;
rowNumber?: number;
column?: string;
valuePreview?: string;
};
export type ParserErrorReport = {
diagnostics: readonly ParserDiagnostic[];
rejectedRows: number;
errorReportKey?: string;
};
export type ImportDryRunSummary = {
format: ImportFormat;
totalRows: number;
acceptedRows: number;
rejectedRows: number;
diagnostics: readonly ParserDiagnostic[];
};
export type ImportParserInput = {
source: ImportSourceDescriptor;
format: ImportFormat;
mappings: readonly ColumnMapping[];
text?: string;
bytes?: Uint8Array;
};
export type ImportParserOutput = {
rows: readonly ImportRowResult[];
summary: ImportDryRunSummary;
errorReport?: ParserErrorReport;
};
export type ImportParser = {
readonly format: ImportFormat;
parse(input: ImportParserInput): Promise<ImportParserOutput> | ImportParserOutput;
};
export type ImportDryRunFieldDefinition = {
targetField: string;
required?: boolean;
label?: string;
};
export type ImportPreviewInputRow = {
rowNumber?: number;
values: NormalizedImportRow;
};
export type ImportMappedPreviewRow = {
rowNumber: number;
normalized: NormalizedImportRow;
sourceColumns: readonly string[];
};
export type ImportDryRunPreviewInput = {
format: ImportFormat;
fields: readonly ImportDryRunFieldDefinition[];
mappings: readonly ColumnMapping[];
rows: readonly ImportPreviewInputRow[];
};
export type ImportDryRunPreviewOutput = ImportParserOutput & {
mappedRows: readonly ImportMappedPreviewRow[];
};
const EXTENSION_FORMATS: Readonly<Record<string, ImportFormat>> = {
csv: "csv",
json: "json",
xlsx: "xlsx",
};
const CONTENT_TYPE_FORMATS: Readonly<Record<string, ImportFormat>> = {
"application/csv": "csv",
"application/json": "json",
"application/vnd.ms-excel": "csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
"text/csv": "csv",
};
const SECRET_PATTERN = /(?:body|payload|raw(?:file|body|payload|rows)?)\s*[:=]\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)|(?:api[_-]?key|authorization|bearer|password|secret|token)\s*[:=]\s*[^\s,;]+/gi;
const REDACTED = "[REDACTED]";
export function isSupportedImportFormat(value: string): value is ImportFormat {
return (IMPORT_FORMATS as readonly string[]).includes(value);
}
export function detectImportFormatFromExtension(fileNameOrExtension: string | undefined): ImportFormat | undefined {
if (!fileNameOrExtension) {
return undefined;
}
const extension = fileNameOrExtension.trim().toLowerCase().replace(/^.*\./, "");
return EXTENSION_FORMATS[extension];
}
export function detectImportFormatFromContentType(contentType: string | undefined): ImportFormat | undefined {
if (!contentType) {
return undefined;
}
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase();
return normalized ? CONTENT_TYPE_FORMATS[normalized] : undefined;
}
export function summarizeRowResults(rows: readonly ImportRowResult[]): Pick<ImportDryRunSummary, "totalRows" | "acceptedRows" | "rejectedRows"> {
const acceptedRows = rows.filter((row) => row.ok).length;
return {
totalRows: rows.length,
acceptedRows,
rejectedRows: rows.length - acceptedRows,
};
}
function isBlankPreviewValue(value: NormalizedRowValue | undefined): boolean {
if (value === undefined || value === null) {
return true;
}
if (typeof value === "string") {
return value.trim().length === 0;
}
if (Array.isArray(value)) {
return value.length === 0 || value.every((item) => isBlankPreviewValue(item));
}
return false;
}
function previewValue(value: NormalizedRowValue | undefined): string | undefined {
if (value === undefined) {
return undefined;
}
return redactDiagnosticMessage(String(value), 80);
}
function buildRequiredFieldDefinitions(
fields: readonly ImportDryRunFieldDefinition[],
mappings: readonly ColumnMapping[],
): Map<string, ImportDryRunFieldDefinition> {
const requiredFields = new Map<string, ImportDryRunFieldDefinition>();
for (const field of fields) {
if (field.required) {
requiredFields.set(field.targetField, field);
}
}
for (const mapping of mappings) {
if (mapping.required) {
requiredFields.set(mapping.targetField, {
targetField: mapping.targetField,
required: true,
label: mapping.label,
});
}
}
return requiredFields;
}
export function createImportDryRunPreview(input: ImportDryRunPreviewInput): ImportDryRunPreviewOutput {
const requiredFields = buildRequiredFieldDefinitions(input.fields, input.mappings);
const rows = input.rows.map<ImportRowResult>((row, index) => {
const rowNumber = row.rowNumber ?? index + 1;
const normalized: NormalizedImportRow = {};
const sourceColumns: string[] = [];
for (const mapping of input.mappings) {
if (!mapping.sourceColumn || !mapping.targetField) {
continue;
}
if (Object.prototype.hasOwnProperty.call(row.values, mapping.sourceColumn)) {
normalized[mapping.targetField] = row.values[mapping.sourceColumn] ?? null;
sourceColumns.push(mapping.sourceColumn);
}
}
const errors: ParserDiagnostic[] = [];
for (const [targetField, field] of requiredFields) {
if (isBlankPreviewValue(normalized[targetField])) {
errors.push(
createRedactedParserDiagnostic(
"missing_required_field",
`${field.label ?? targetField} is required for import preview.`,
{
rowNumber,
column: targetField,
valuePreview: previewValue(normalized[targetField]),
},
),
);
}
}
if (errors.length > 0) {
return {
ok: false,
rowNumber,
errors,
};
}
return {
ok: true,
rowNumber,
normalized,
warnings: undefined,
};
});
const mappedRows = rows
.filter((row): row is ImportRowSuccessResult => row.ok)
.map<ImportMappedPreviewRow>((row) => ({
rowNumber: row.rowNumber,
normalized: row.normalized,
sourceColumns: input.mappings
.filter((mapping) => Object.prototype.hasOwnProperty.call(row.normalized, mapping.targetField))
.map((mapping) => mapping.sourceColumn),
}));
const diagnostics = rows.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors));
const summaryCounts = summarizeRowResults(rows);
return {
rows,
mappedRows,
summary: {
format: input.format,
...summaryCounts,
diagnostics,
},
errorReport:
summaryCounts.rejectedRows > 0
? {
diagnostics,
rejectedRows: summaryCounts.rejectedRows,
}
: undefined,
};
}
export function createRedactedParserDiagnostic(
code: string,
message: string,
options: Omit<Partial<ParserDiagnostic>, "code" | "message"> = {},
): ParserDiagnostic {
return {
severity: "error",
...options,
code,
message: redactDiagnosticMessage(message),
};
}
export function redactDiagnosticMessage(message: string, maxLength = 300): string {
const redacted = message.replace(SECRET_PATTERN, (match) => {
const colonIndex = match.indexOf(":");
const equalsIndex = match.indexOf("=");
const separatorIndex =
colonIndex === -1 ? equalsIndex : equalsIndex === -1 ? colonIndex : Math.min(colonIndex, equalsIndex);
const prefix = separatorIndex >= 0 ? match.slice(0, separatorIndex + 1) : "secret:";
return `${prefix} ${REDACTED}`;
});
return redacted.length > maxLength ? `${redacted.slice(0, maxLength - 1)}…` : redacted;
}