Track HiveGrid Cloud planning reference

This commit is contained in:
david committed 2026-06-26 20:48:01 +02:00
1 parent b3d5ea6764
commit e341c5938a
5784 files changed
+1104929 -3

No files matched your search

@@ -0,0 +1,13 @@
{
"name": "@toolctrl/imports",
"version": "0.0.0",
"private": true,
"type": "module",
"exports": {
".": "./src/index.ts"
},
"scripts": {
"test": "vitest run src",
"typecheck": "tsc -p tsconfig.json --noEmit"
}
}
@@ -0,0 +1,137 @@
import { describe, expect, it } from "vitest";
import { createValidationReport, detectDuplicateRows } from "./dedupe.js";
import type { ImportRowResult } from "./index.js";
describe("duplicate detection contracts", () => {
it("reports deterministic duplicate groups and row diagnostics", () => {
const rows = [
{ rowNumber: 10, normalized: { email: " ada@example.com ", name: "Ada" } },
{ rowNumber: 12, normalized: { email: "ben@example.com", name: "Ben" } },
{ rowNumber: 14, normalized: { email: "ada@example.com", name: "Ada Lovelace" } },
{ rowNumber: 16, normalized: { email: "ben@example.com", name: "Benjamin" } },
{ rowNumber: 18, normalized: { email: "case@example.com", name: "Case" } },
{ rowNumber: 20, normalized: { email: "CASE@example.com", name: "Case" } },
];
const first = detectDuplicateRows(rows, ["email"]);
const second = detectDuplicateRows(rows, ["email"]);
expect(first.groups).toEqual([
{
id: "duplicate-group-0001",
key: [{ field: "email", valuePreview: "ada@example.com" }],
rowNumbers: [10, 14],
},
{
id: "duplicate-group-0002",
key: [{ field: "email", valuePreview: "ben@example.com" }],
rowNumbers: [12, 16],
},
]);
expect(second).toEqual(first);
expect(first.summary).toMatchObject({ totalRows: 6, duplicateGroups: 2, duplicateRows: 4 });
expect(first.rowResults.filter((row) => !row.ok).map((row) => row.rowNumber)).toEqual([10, 12, 14, 16]);
expect(first.summary.diagnostics).toHaveLength(4);
expect(first.summary.diagnostics[0]).toMatchObject({
severity: "error",
code: "duplicate_key",
rowNumber: 10,
valuePreview: "duplicate-group-0001",
});
});
it("returns no duplicate errors for empty key configuration", () => {
const report = detectDuplicateRows(
[
{ rowNumber: 1, normalized: { email: "a@example.com" } },
{ rowNumber: 2, normalized: { email: "a@example.com" } },
],
[],
);
expect(report.groups).toEqual([]);
expect(report.summary).toEqual({ totalRows: 2, duplicateGroups: 0, duplicateRows: 0, diagnostics: [] });
expect(report.rowResults).toEqual([
{ ok: true, rowNumber: 1, normalized: { email: "a@example.com" }, warnings: undefined },
{ ok: true, rowNumber: 2, normalized: { email: "a@example.com" }, warnings: undefined },
]);
expect(report.errorReport).toBeUndefined();
});
it("ignores blank keys unless another configured key field is present", () => {
const report = detectDuplicateRows(
[
{ rowNumber: 1, normalized: { email: " ", accountId: "tenant-a" } },
{ rowNumber: 2, normalized: { email: null, accountId: "tenant-a" } },
{ rowNumber: 3, normalized: { email: "", accountId: "" } },
{ rowNumber: 4, normalized: { email: "", accountId: null } },
{ rowNumber: 5, normalized: { email: "", accountId: "tenant-b" } },
],
["email", "accountId"],
);
expect(report.groups).toEqual([
{
id: "duplicate-group-0001",
key: [{ field: "accountId", valuePreview: "tenant-a" }],
rowNumbers: [1, 2],
},
]);
expect(report.summary).toMatchObject({ duplicateGroups: 1, duplicateRows: 2 });
expect(report.rowResults.filter((row) => !row.ok).map((row) => row.rowNumber)).toEqual([1, 2]);
});
it("redacts secret-like values from duplicate diagnostics and reports", () => {
const report = detectDuplicateRows(
[
{ rowNumber: 1, normalized: { apiToken: "token=super-secret", email: "a@example.com" } },
{ rowNumber: 2, normalized: { apiToken: " token=super-secret ", email: "b@example.com" } },
],
["apiToken"],
);
const serialized = JSON.stringify(report);
expect(report.groups[0]).toMatchObject({
key: [{ field: "apiToken", valuePreview: "token= [REDACTED]" }],
rowNumbers: [1, 2],
});
expect(report.summary.diagnostics[0]?.message).toContain("apiToken= [REDACTED]");
expect(serialized).not.toContain("super-secret");
expect(serialized).not.toContain("rawFile");
expect(serialized).not.toContain("requestBody");
expect(serialized).not.toContain("buffer");
});
it("builds redacted validation report summaries without raw rejected rows", () => {
const rowResults: ImportRowResult[] = [
{ ok: true, rowNumber: 1, normalized: { email: "a@example.com" } },
{ ok: false, rowNumber: 2, errors: [{ severity: "error", code: "required", message: "password= [REDACTED]", rowNumber: 2 }] },
];
const duplicateReport = detectDuplicateRows(
[
{ rowNumber: 1, normalized: { email: "a@example.com" } },
{ rowNumber: 3, normalized: { email: "a@example.com" } },
],
["email"],
);
const report = createValidationReport("csv", rowResults, duplicateReport);
const serialized = JSON.stringify(report);
expect(report.summary).toMatchObject({
format: "csv",
totalRows: 2,
acceptedRows: 1,
rejectedRows: 1,
duplicateGroups: 1,
duplicateRows: 2,
});
expect(report.diagnostics).toHaveLength(3);
expect(report.errorReport).toMatchObject({ rejectedRows: 1 });
expect(serialized).not.toContain("rejectedValues");
expect(serialized).not.toContain("rawRows");
expect(serialized).not.toContain("requestBody");
expect(serialized).not.toContain("super-secret");
});
});
@@ -0,0 +1,272 @@
import {
createRedactedParserDiagnostic,
redactDiagnosticMessage,
summarizeRowResults,
type ImportDryRunSummary,
type ImportFormat,
type ImportRowErrorResult,
type ImportRowResult,
type NormalizedImportRow,
type NormalizedRowPrimitive,
type NormalizedRowValue,
type ParserDiagnostic,
type ParserErrorReport,
} from "./index.js";
export type DuplicateKeyField = {
field: string;
label?: string;
};
export type DuplicateDetectionRow = {
rowNumber?: number;
normalized: NormalizedImportRow;
};
export type DuplicateKeyPart = {
field: string;
valuePreview: string;
};
export type DuplicateGroup = {
id: string;
key: readonly DuplicateKeyPart[];
rowNumbers: readonly number[];
};
export type DuplicateDetectionSummary = {
totalRows: number;
duplicateGroups: number;
duplicateRows: number;
diagnostics: readonly ParserDiagnostic[];
};
export type DuplicateDetectionReport = {
groups: readonly DuplicateGroup[];
rowResults: readonly ImportRowResult[];
summary: DuplicateDetectionSummary;
errorReport?: ParserErrorReport;
};
export type ValidationReportSummary = ImportDryRunSummary & {
duplicateGroups: number;
duplicateRows: number;
};
export type ValidationReport = {
summary: ValidationReportSummary;
diagnostics: readonly ParserDiagnostic[];
duplicateGroups: readonly DuplicateGroup[];
errorReport?: ParserErrorReport;
};
type DuplicateKeySignature = {
signature: string;
parts: readonly DuplicateKeyPart[];
};
function normalizeDuplicateKeyValue(value: NormalizedRowValue | undefined): string | undefined {
if (value === undefined || value === null) {
return undefined;
}
if (typeof value === "string") {
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : undefined;
}
if (Array.isArray(value)) {
const normalizedItems = value
.map((item) => normalizeDuplicateKeyPrimitive(item))
.filter((item): item is string => item !== undefined);
return normalizedItems.length > 0 ? JSON.stringify(normalizedItems) : undefined;
}
return String(value);
}
function normalizeDuplicateKeyPrimitive(value: NormalizedRowPrimitive): string | undefined {
if (value === null) {
return undefined;
}
if (typeof value === "string") {
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : undefined;
}
return String(value);
}
function toDuplicateKeyField(field: string | DuplicateKeyField): DuplicateKeyField {
return typeof field === "string" ? { field } : field;
}
function createDuplicateKey(row: NormalizedImportRow, fields: readonly DuplicateKeyField[]): DuplicateKeySignature | undefined {
const parts: DuplicateKeyPart[] = [];
const signatureParts: string[] = [];
for (const field of fields) {
if (!field.field) {
continue;
}
const normalizedValue = normalizeDuplicateKeyValue(row[field.field]);
if (normalizedValue === undefined) {
continue;
}
parts.push({ field: field.field, valuePreview: redactDiagnosticMessage(normalizedValue, 80) });
signatureParts.push(`${JSON.stringify(field.field)}:${JSON.stringify(normalizedValue)}`);
}
if (signatureParts.length === 0) {
return undefined;
}
return {
signature: signatureParts.join("|"),
parts,
};
}
function duplicateGroupId(index: number): string {
return `duplicate-group-${String(index + 1).padStart(4, "0")}`;
}
function createDuplicateDiagnostic(group: DuplicateGroup, rowNumber: number): ParserDiagnostic {
return createRedactedParserDiagnostic(
"duplicate_key",
`Duplicate import row matches ${group.id} on ${group.key.map((part) => `${part.field}=${part.valuePreview}`).join(", ")}.`,
{
rowNumber,
valuePreview: group.id,
},
);
}
export function detectDuplicateRows(
rows: readonly DuplicateDetectionRow[],
keyFields: readonly (string | DuplicateKeyField)[],
): DuplicateDetectionReport {
const fields = keyFields.map(toDuplicateKeyField).filter((field) => field.field.trim().length > 0);
const rowResults: ImportRowResult[] = rows.map((row, index) => ({
ok: true,
rowNumber: row.rowNumber ?? index + 1,
normalized: row.normalized,
warnings: undefined,
}));
if (fields.length === 0) {
return {
groups: [],
rowResults,
summary: {
totalRows: rows.length,
duplicateGroups: 0,
duplicateRows: 0,
diagnostics: [],
},
errorReport: undefined,
};
}
const bySignature = new Map<string, { parts: readonly DuplicateKeyPart[]; rowNumbers: number[] }>();
for (let index = 0; index < rows.length; index += 1) {
const row = rows[index];
const key = createDuplicateKey(row.normalized, fields);
if (!key) {
continue;
}
const rowNumber = row.rowNumber ?? index + 1;
const existing = bySignature.get(key.signature);
if (existing) {
existing.rowNumbers.push(rowNumber);
} else {
bySignature.set(key.signature, { parts: key.parts, rowNumbers: [rowNumber] });
}
}
const groups = Array.from(bySignature.values())
.filter((group) => group.rowNumbers.length > 1)
.sort((left, right) => left.rowNumbers[0] - right.rowNumbers[0])
.map<DuplicateGroup>((group, index) => ({
id: duplicateGroupId(index),
key: group.parts,
rowNumbers: [...group.rowNumbers].sort((left, right) => left - right),
}));
const diagnostics = groups.flatMap((group) => group.rowNumbers.map((rowNumber) => createDuplicateDiagnostic(group, rowNumber)));
const diagnosticsByRowNumber = new Map<number, ParserDiagnostic[]>();
for (const diagnostic of diagnostics) {
if (diagnostic.rowNumber === undefined) {
continue;
}
diagnosticsByRowNumber.set(diagnostic.rowNumber, [...(diagnosticsByRowNumber.get(diagnostic.rowNumber) ?? []), diagnostic]);
}
const duplicateRowResults = rowResults.map<ImportRowResult>((row) => {
const errors = diagnosticsByRowNumber.get(row.rowNumber);
if (!errors) {
return row;
}
return {
ok: false,
rowNumber: row.rowNumber,
errors,
} satisfies ImportRowErrorResult;
});
return {
groups,
rowResults: duplicateRowResults,
summary: {
totalRows: rows.length,
duplicateGroups: groups.length,
duplicateRows: new Set(groups.flatMap((group) => group.rowNumbers)).size,
diagnostics,
},
errorReport:
diagnostics.length > 0
? {
diagnostics,
rejectedRows: diagnosticsByRowNumber.size,
}
: undefined,
};
}
export function createValidationReport(
format: ImportFormat,
rowResults: readonly ImportRowResult[],
duplicateReport: Pick<DuplicateDetectionReport, "groups" | "summary">,
): ValidationReport {
const summaryCounts = summarizeRowResults(rowResults);
const diagnostics = [
...rowResults.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors)),
...duplicateReport.summary.diagnostics,
];
return {
summary: {
format,
...summaryCounts,
duplicateGroups: duplicateReport.summary.duplicateGroups,
duplicateRows: duplicateReport.summary.duplicateRows,
diagnostics,
},
diagnostics,
duplicateGroups: duplicateReport.groups,
errorReport:
diagnostics.length > 0
? {
diagnostics,
rejectedRows: summaryCounts.rejectedRows,
}
: undefined,
};
}
@@ -0,0 +1,226 @@
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
createImportDryRunPreview,
createRedactedParserDiagnostic,
detectImportFormatFromContentType,
detectImportFormatFromExtension,
isSupportedImportFormat,
redactDiagnosticMessage,
summarizeRowResults,
type ImportParserOutput,
type ImportRowResult,
} from "./index.js";
describe("imports parser contracts", () => {
it("guards supported formats explicitly", () => {
expect(isSupportedImportFormat("csv")).toBe(true);
expect(isSupportedImportFormat("json")).toBe(true);
expect(isSupportedImportFormat("xlsx")).toBe(true);
expect(isSupportedImportFormat("xml")).toBe(false);
expect(isSupportedImportFormat("CSV")).toBe(false);
expect(isSupportedImportFormat("")).toBe(false);
});
it("detects csv/json/xlsx formats from extensions and content types", () => {
expect(detectImportFormatFromExtension("members.CSV")).toBe("csv");
expect(detectImportFormatFromExtension("members.json")).toBe("json");
expect(detectImportFormatFromExtension("exports/Members.XLSX")).toBe("xlsx");
expect(detectImportFormatFromExtension("xlsx")).toBe("xlsx");
expect(detectImportFormatFromContentType("text/csv; charset=utf-8")).toBe("csv");
expect(detectImportFormatFromContentType("application/json; charset=utf-8")).toBe("json");
expect(detectImportFormatFromContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")).toBe("xlsx");
});
it("rejects unknown and ambiguous formats", () => {
expect(detectImportFormatFromExtension("payload.unsupported")).toBeUndefined();
expect(detectImportFormatFromExtension("members.csv.zip")).toBeUndefined();
expect(detectImportFormatFromExtension(undefined)).toBeUndefined();
expect(detectImportFormatFromContentType("application/octet-stream")).toBeUndefined();
expect(detectImportFormatFromContentType("text/plain")).toBeUndefined();
expect(detectImportFormatFromContentType(undefined)).toBeUndefined();
});
it("summarizes accepted and rejected row results", () => {
const rows: ImportRowResult[] = [
{ ok: true, rowNumber: 1, normalized: { email: "a@example.com" } },
{ ok: true, rowNumber: 2, normalized: { email: "b@example.com", tags: ["member", "paid"] } },
{
ok: false,
rowNumber: 3,
errors: [{ severity: "error", code: "missing_email", message: "Email is required." }],
},
];
expect(summarizeRowResults(rows)).toEqual({ totalRows: 3, acceptedRows: 2, rejectedRows: 1 });
expect(summarizeRowResults([])).toEqual({ totalRows: 0, acceptedRows: 0, rejectedRows: 0 });
});
it("redacts secrets and body-like diagnostics without leaking raw values", () => {
const secretRedacted = redactDiagnosticMessage("token=abc123 password:super-secret authorization=BearerToken");
const bodyRedacted = redactDiagnosticMessage("body={raw:'payload'} rawRows=[secret]");
expect(secretRedacted).toContain("token= [REDACTED]");
expect(secretRedacted).toContain("password: [REDACTED]");
expect(secretRedacted).toContain("authorization= [REDACTED]");
expect(bodyRedacted).toContain("body= [REDACTED]");
expect(bodyRedacted).toContain("rawRows= [REDACTED]");
expect(secretRedacted).not.toContain("abc123");
expect(secretRedacted).not.toContain("super-secret");
expect(bodyRedacted).not.toContain("{raw:'payload'}");
expect(bodyRedacted).not.toContain("[secret]");
expect(redactDiagnosticMessage("x".repeat(10), 5)).toBe("xxxx…");
expect(createRedactedParserDiagnostic("parse_failed", "apiKey=secret-value", { rowNumber: 3 })).toEqual({
severity: "error",
code: "parse_failed",
message: "apiKey= [REDACTED]",
rowNumber: 3,
});
});
it("creates accepted mapped dry-run previews", () => {
const output = createImportDryRunPreview({
format: "csv",
fields: [{ targetField: "email", required: true, label: "Email" }],
mappings: [
{ sourceColumn: "Email Address", targetField: "email", required: true, label: "Email" },
{ sourceColumn: "Member Name", targetField: "name" },
],
rows: [
{ rowNumber: 10, values: { "Email Address": "a@example.com", "Member Name": "Ada", Ignored: "secret=not-mapped" } },
{ rowNumber: 11, values: { "Email Address": "b@example.com", "Member Name": "Ben" } },
],
});
expect(output.summary).toMatchObject({ format: "csv", totalRows: 2, acceptedRows: 2, rejectedRows: 0, diagnostics: [] });
expect(output.rows).toEqual([
{ ok: true, rowNumber: 10, normalized: { email: "a@example.com", name: "Ada" }, warnings: undefined },
{ ok: true, rowNumber: 11, normalized: { email: "b@example.com", name: "Ben" }, warnings: undefined },
]);
expect(output.mappedRows).toEqual([
{ rowNumber: 10, normalized: { email: "a@example.com", name: "Ada" }, sourceColumns: ["Email Address", "Member Name"] },
{ rowNumber: 11, normalized: { email: "b@example.com", name: "Ben" }, sourceColumns: ["Email Address", "Member Name"] },
]);
expect(JSON.stringify(output)).not.toContain("not-mapped");
});
it("rejects missing required mapped values deterministically", () => {
const output = createImportDryRunPreview({
format: "csv",
fields: [{ targetField: "email", required: true, label: "Email" }],
mappings: [{ sourceColumn: "Email Address", targetField: "email", required: true, label: "Email" }],
rows: [
{ rowNumber: 1, values: { "Email Address": "valid@example.com" } },
{ rowNumber: 2, values: { "Email Address": " " } },
{ rowNumber: 3, values: {} },
],
});
expect(output.summary).toMatchObject({ totalRows: 3, acceptedRows: 1, rejectedRows: 2 });
expect(output.rows[1]).toMatchObject({
ok: false,
rowNumber: 2,
errors: [{ severity: "error", code: "missing_required_field", message: "Email is required for import preview.", rowNumber: 2, column: "email" }],
});
expect(output.rows[2]).toMatchObject({
ok: false,
rowNumber: 3,
errors: [{ severity: "error", code: "missing_required_field", message: "Email is required for import preview.", rowNumber: 3, column: "email" }],
});
expect(output.errorReport).toMatchObject({ rejectedRows: 2 });
});
it("ignores unmapped columns and invalid mapping entries", () => {
const output = createImportDryRunPreview({
format: "json",
fields: [{ targetField: "email", required: true }],
mappings: [
{ sourceColumn: "email", targetField: "email" },
{ sourceColumn: "ignored", targetField: "" },
{ sourceColumn: "", targetField: "emptySource" },
],
rows: [{ values: { email: "a@example.com", ignored: "password:super-secret", extra: "rawBody={secret}" } }],
});
expect(output.summary).toMatchObject({ format: "json", totalRows: 1, acceptedRows: 1, rejectedRows: 0 });
expect(output.rows[0]).toMatchObject({ ok: true, rowNumber: 1, normalized: { email: "a@example.com" } });
expect(JSON.stringify(output)).not.toContain("super-secret");
expect(JSON.stringify(output)).not.toContain("rawBody");
});
it("redacts secret-like required values in preview diagnostics", () => {
const output = createImportDryRunPreview({
format: "csv",
fields: [{ targetField: "apiToken", required: true, label: "token=secret-value" }],
mappings: [{ sourceColumn: "Token", targetField: "apiToken" }],
rows: [{ rowNumber: 7, values: { Token: "" } }],
});
const serialized = JSON.stringify(output);
expect(output.summary.diagnostics[0]).toMatchObject({
code: "missing_required_field",
message: "token= [REDACTED] is required for import preview.",
rowNumber: 7,
column: "apiToken",
valuePreview: "",
});
expect(serialized).not.toContain("secret-value");
expect(serialized).not.toContain("body=");
expect(serialized).not.toContain("rawFile");
});
it("represents row-level validation errors without raw file buffers or request bodies", () => {
const output: ImportParserOutput = {
rows: [
{ ok: true, rowNumber: 1, normalized: { email: "valid@example.com", active: true, score: 7, tags: ["new"] } },
{
ok: false,
rowNumber: 2,
rejectedValues: { email: "", active: null },
errors: [
{
severity: "error",
code: "required",
message: "Email is required.",
rowNumber: 2,
column: "email",
valuePreview: "",
},
],
},
],
summary: {
format: "csv",
totalRows: 2,
acceptedRows: 1,
rejectedRows: 1,
diagnostics: [{ severity: "warning", code: "blank_row", message: "Skipped blank row.", rowNumber: 3 }],
},
errorReport: {
diagnostics: [{ severity: "error", code: "required", message: "Email is required.", rowNumber: 2, column: "email" }],
rejectedRows: 1,
errorReportKey: "imports/reports/import-job-1.json",
},
};
const serialized = JSON.stringify(output);
expect(output.rows[1]).toMatchObject({ ok: false, rowNumber: 2 });
expect(serialized).not.toContain("bytes");
expect(serialized).not.toContain("buffer");
expect(serialized).not.toContain("requestBody");
expect(serialized).not.toContain("rawFile");
});
it("keeps parser contracts free of database, storage, framework, and prisma imports", () => {
const sourcePath = path.join(path.dirname(fileURLToPath(import.meta.url)), "./index.ts");
const source = readFileSync(sourcePath, "utf8");
expect(source).not.toMatch(/from\s+["'](?:@toolctrl\/db|@toolctrl\/storage|react|react-router|@prisma\/client|\.prisma|prisma)["']/);
expect(source).not.toMatch(/import\s+["'](?:@toolctrl\/db|@toolctrl\/storage|react|react-router|@prisma\/client|\.prisma|prisma)["']/);
});
});
@@ -0,0 +1,319 @@
export * from "./dedupe.js";
export const IMPORT_FORMATS = ["csv", "json", "xlsx"] as const;
export type ImportFormat = (typeof IMPORT_FORMATS)[number];
export type ImportSourceDescriptor = {
tenantId: string;
requesterId: string;
sourceKey: string;
fileName?: string;
contentType?: string;
byteLength?: number;
requestId?: string;
};
export type ColumnMapping = {
sourceColumn: string;
targetField: string;
required?: boolean;
label?: string;
};
export type NormalizedRowPrimitive = string | number | boolean | null;
export type NormalizedRowValue = NormalizedRowPrimitive | readonly NormalizedRowPrimitive[];
export type NormalizedImportRow = Record<string, NormalizedRowValue>;
export type ImportRowSuccessResult = {
ok: true;
rowNumber: number;
normalized: NormalizedImportRow;
warnings?: readonly ParserDiagnostic[];
};
export type ImportRowErrorResult = {
ok: false;
rowNumber: number;
errors: readonly ParserDiagnostic[];
rejectedValues?: NormalizedImportRow;
};
export type ImportRowResult = ImportRowSuccessResult | ImportRowErrorResult;
export type ParserDiagnosticSeverity = "info" | "warning" | "error";
export type ParserDiagnostic = {
severity: ParserDiagnosticSeverity;
code: string;
message: string;
rowNumber?: number;
column?: string;
valuePreview?: string;
};
export type ParserErrorReport = {
diagnostics: readonly ParserDiagnostic[];
rejectedRows: number;
errorReportKey?: string;
};
export type ImportDryRunSummary = {
format: ImportFormat;
totalRows: number;
acceptedRows: number;
rejectedRows: number;
diagnostics: readonly ParserDiagnostic[];
};
export type ImportParserInput = {
source: ImportSourceDescriptor;
format: ImportFormat;
mappings: readonly ColumnMapping[];
text?: string;
bytes?: Uint8Array;
};
export type ImportParserOutput = {
rows: readonly ImportRowResult[];
summary: ImportDryRunSummary;
errorReport?: ParserErrorReport;
};
export type ImportParser = {
readonly format: ImportFormat;
parse(input: ImportParserInput): Promise<ImportParserOutput> | ImportParserOutput;
};
export type ImportDryRunFieldDefinition = {
targetField: string;
required?: boolean;
label?: string;
};
export type ImportPreviewInputRow = {
rowNumber?: number;
values: NormalizedImportRow;
};
export type ImportMappedPreviewRow = {
rowNumber: number;
normalized: NormalizedImportRow;
sourceColumns: readonly string[];
};
export type ImportDryRunPreviewInput = {
format: ImportFormat;
fields: readonly ImportDryRunFieldDefinition[];
mappings: readonly ColumnMapping[];
rows: readonly ImportPreviewInputRow[];
};
export type ImportDryRunPreviewOutput = ImportParserOutput & {
mappedRows: readonly ImportMappedPreviewRow[];
};
const EXTENSION_FORMATS: Readonly<Record<string, ImportFormat>> = {
csv: "csv",
json: "json",
xlsx: "xlsx",
};
const CONTENT_TYPE_FORMATS: Readonly<Record<string, ImportFormat>> = {
"application/csv": "csv",
"application/json": "json",
"application/vnd.ms-excel": "csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
"text/csv": "csv",
};
const SECRET_PATTERN = /(?:body|payload|raw(?:file|body|payload|rows)?)\s*[:=]\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)|(?:api[_-]?key|authorization|bearer|password|secret|token)\s*[:=]\s*[^\s,;]+/gi;
const REDACTED = "[REDACTED]";
export function isSupportedImportFormat(value: string): value is ImportFormat {
return (IMPORT_FORMATS as readonly string[]).includes(value);
}
export function detectImportFormatFromExtension(fileNameOrExtension: string | undefined): ImportFormat | undefined {
if (!fileNameOrExtension) {
return undefined;
}
const extension = fileNameOrExtension.trim().toLowerCase().replace(/^.*\./, "");
return EXTENSION_FORMATS[extension];
}
export function detectImportFormatFromContentType(contentType: string | undefined): ImportFormat | undefined {
if (!contentType) {
return undefined;
}
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase();
return normalized ? CONTENT_TYPE_FORMATS[normalized] : undefined;
}
export function summarizeRowResults(rows: readonly ImportRowResult[]): Pick<ImportDryRunSummary, "totalRows" | "acceptedRows" | "rejectedRows"> {
const acceptedRows = rows.filter((row) => row.ok).length;
return {
totalRows: rows.length,
acceptedRows,
rejectedRows: rows.length - acceptedRows,
};
}
function isBlankPreviewValue(value: NormalizedRowValue | undefined): boolean {
if (value === undefined || value === null) {
return true;
}
if (typeof value === "string") {
return value.trim().length === 0;
}
if (Array.isArray(value)) {
return value.length === 0 || value.every((item) => isBlankPreviewValue(item));
}
return false;
}
function previewValue(value: NormalizedRowValue | undefined): string | undefined {
if (value === undefined) {
return undefined;
}
return redactDiagnosticMessage(String(value), 80);
}
function buildRequiredFieldDefinitions(
fields: readonly ImportDryRunFieldDefinition[],
mappings: readonly ColumnMapping[],
): Map<string, ImportDryRunFieldDefinition> {
const requiredFields = new Map<string, ImportDryRunFieldDefinition>();
for (const field of fields) {
if (field.required) {
requiredFields.set(field.targetField, field);
}
}
for (const mapping of mappings) {
if (mapping.required) {
requiredFields.set(mapping.targetField, {
targetField: mapping.targetField,
required: true,
label: mapping.label,
});
}
}
return requiredFields;
}
export function createImportDryRunPreview(input: ImportDryRunPreviewInput): ImportDryRunPreviewOutput {
const requiredFields = buildRequiredFieldDefinitions(input.fields, input.mappings);
const rows = input.rows.map<ImportRowResult>((row, index) => {
const rowNumber = row.rowNumber ?? index + 1;
const normalized: NormalizedImportRow = {};
const sourceColumns: string[] = [];
for (const mapping of input.mappings) {
if (!mapping.sourceColumn || !mapping.targetField) {
continue;
}
if (Object.prototype.hasOwnProperty.call(row.values, mapping.sourceColumn)) {
normalized[mapping.targetField] = row.values[mapping.sourceColumn] ?? null;
sourceColumns.push(mapping.sourceColumn);
}
}
const errors: ParserDiagnostic[] = [];
for (const [targetField, field] of requiredFields) {
if (isBlankPreviewValue(normalized[targetField])) {
errors.push(
createRedactedParserDiagnostic(
"missing_required_field",
`${field.label ?? targetField} is required for import preview.`,
{
rowNumber,
column: targetField,
valuePreview: previewValue(normalized[targetField]),
},
),
);
}
}
if (errors.length > 0) {
return {
ok: false,
rowNumber,
errors,
};
}
return {
ok: true,
rowNumber,
normalized,
warnings: undefined,
};
});
const mappedRows = rows
.filter((row): row is ImportRowSuccessResult => row.ok)
.map<ImportMappedPreviewRow>((row) => ({
rowNumber: row.rowNumber,
normalized: row.normalized,
sourceColumns: input.mappings
.filter((mapping) => Object.prototype.hasOwnProperty.call(row.normalized, mapping.targetField))
.map((mapping) => mapping.sourceColumn),
}));
const diagnostics = rows.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors));
const summaryCounts = summarizeRowResults(rows);
return {
rows,
mappedRows,
summary: {
format: input.format,
...summaryCounts,
diagnostics,
},
errorReport:
summaryCounts.rejectedRows > 0
? {
diagnostics,
rejectedRows: summaryCounts.rejectedRows,
}
: undefined,
};
}
export function createRedactedParserDiagnostic(
code: string,
message: string,
options: Omit<Partial<ParserDiagnostic>, "code" | "message"> = {},
): ParserDiagnostic {
return {
severity: "error",
...options,
code,
message: redactDiagnosticMessage(message),
};
}
export function redactDiagnosticMessage(message: string, maxLength = 300): string {
const redacted = message.replace(SECRET_PATTERN, (match) => {
const colonIndex = match.indexOf(":");
const equalsIndex = match.indexOf("=");
const separatorIndex =
colonIndex === -1 ? equalsIndex : equalsIndex === -1 ? colonIndex : Math.min(colonIndex, equalsIndex);
const prefix = separatorIndex >= 0 ? match.slice(0, separatorIndex + 1) : "secret:";
return `${prefix} ${REDACTED}`;
});
return redacted.length > maxLength ? `${redacted.slice(0, maxLength - 1)}…` : redacted;
}
@@ -0,0 +1,13 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"allowJs": false,
"allowImportingTsExtensions": false,
"lib": ["ES2022"],
"module": "NodeNext",
"moduleResolution": "NodeNext",
"noEmit": true,
"types": []
},
"include": ["src/**/*.ts"]
}