Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "@toolctrl/imports",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "vitest run src",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { createValidationReport, detectDuplicateRows } from "./dedupe.js";
|
||||
import type { ImportRowResult } from "./index.js";
|
||||
|
||||
describe("duplicate detection contracts", () => {
|
||||
it("reports deterministic duplicate groups and row diagnostics", () => {
|
||||
const rows = [
|
||||
{ rowNumber: 10, normalized: { email: " ada@example.com ", name: "Ada" } },
|
||||
{ rowNumber: 12, normalized: { email: "ben@example.com", name: "Ben" } },
|
||||
{ rowNumber: 14, normalized: { email: "ada@example.com", name: "Ada Lovelace" } },
|
||||
{ rowNumber: 16, normalized: { email: "ben@example.com", name: "Benjamin" } },
|
||||
{ rowNumber: 18, normalized: { email: "case@example.com", name: "Case" } },
|
||||
{ rowNumber: 20, normalized: { email: "CASE@example.com", name: "Case" } },
|
||||
];
|
||||
|
||||
const first = detectDuplicateRows(rows, ["email"]);
|
||||
const second = detectDuplicateRows(rows, ["email"]);
|
||||
|
||||
expect(first.groups).toEqual([
|
||||
{
|
||||
id: "duplicate-group-0001",
|
||||
key: [{ field: "email", valuePreview: "ada@example.com" }],
|
||||
rowNumbers: [10, 14],
|
||||
},
|
||||
{
|
||||
id: "duplicate-group-0002",
|
||||
key: [{ field: "email", valuePreview: "ben@example.com" }],
|
||||
rowNumbers: [12, 16],
|
||||
},
|
||||
]);
|
||||
expect(second).toEqual(first);
|
||||
expect(first.summary).toMatchObject({ totalRows: 6, duplicateGroups: 2, duplicateRows: 4 });
|
||||
expect(first.rowResults.filter((row) => !row.ok).map((row) => row.rowNumber)).toEqual([10, 12, 14, 16]);
|
||||
expect(first.summary.diagnostics).toHaveLength(4);
|
||||
expect(first.summary.diagnostics[0]).toMatchObject({
|
||||
severity: "error",
|
||||
code: "duplicate_key",
|
||||
rowNumber: 10,
|
||||
valuePreview: "duplicate-group-0001",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns no duplicate errors for empty key configuration", () => {
|
||||
const report = detectDuplicateRows(
|
||||
[
|
||||
{ rowNumber: 1, normalized: { email: "a@example.com" } },
|
||||
{ rowNumber: 2, normalized: { email: "a@example.com" } },
|
||||
],
|
||||
[],
|
||||
);
|
||||
|
||||
expect(report.groups).toEqual([]);
|
||||
expect(report.summary).toEqual({ totalRows: 2, duplicateGroups: 0, duplicateRows: 0, diagnostics: [] });
|
||||
expect(report.rowResults).toEqual([
|
||||
{ ok: true, rowNumber: 1, normalized: { email: "a@example.com" }, warnings: undefined },
|
||||
{ ok: true, rowNumber: 2, normalized: { email: "a@example.com" }, warnings: undefined },
|
||||
]);
|
||||
expect(report.errorReport).toBeUndefined();
|
||||
});
|
||||
|
||||
it("ignores blank keys unless another configured key field is present", () => {
|
||||
const report = detectDuplicateRows(
|
||||
[
|
||||
{ rowNumber: 1, normalized: { email: " ", accountId: "tenant-a" } },
|
||||
{ rowNumber: 2, normalized: { email: null, accountId: "tenant-a" } },
|
||||
{ rowNumber: 3, normalized: { email: "", accountId: "" } },
|
||||
{ rowNumber: 4, normalized: { email: "", accountId: null } },
|
||||
{ rowNumber: 5, normalized: { email: "", accountId: "tenant-b" } },
|
||||
],
|
||||
["email", "accountId"],
|
||||
);
|
||||
|
||||
expect(report.groups).toEqual([
|
||||
{
|
||||
id: "duplicate-group-0001",
|
||||
key: [{ field: "accountId", valuePreview: "tenant-a" }],
|
||||
rowNumbers: [1, 2],
|
||||
},
|
||||
]);
|
||||
expect(report.summary).toMatchObject({ duplicateGroups: 1, duplicateRows: 2 });
|
||||
expect(report.rowResults.filter((row) => !row.ok).map((row) => row.rowNumber)).toEqual([1, 2]);
|
||||
});
|
||||
|
||||
it("redacts secret-like values from duplicate diagnostics and reports", () => {
|
||||
const report = detectDuplicateRows(
|
||||
[
|
||||
{ rowNumber: 1, normalized: { apiToken: "token=super-secret", email: "a@example.com" } },
|
||||
{ rowNumber: 2, normalized: { apiToken: " token=super-secret ", email: "b@example.com" } },
|
||||
],
|
||||
["apiToken"],
|
||||
);
|
||||
const serialized = JSON.stringify(report);
|
||||
|
||||
expect(report.groups[0]).toMatchObject({
|
||||
key: [{ field: "apiToken", valuePreview: "token= [REDACTED]" }],
|
||||
rowNumbers: [1, 2],
|
||||
});
|
||||
expect(report.summary.diagnostics[0]?.message).toContain("apiToken= [REDACTED]");
|
||||
expect(serialized).not.toContain("super-secret");
|
||||
expect(serialized).not.toContain("rawFile");
|
||||
expect(serialized).not.toContain("requestBody");
|
||||
expect(serialized).not.toContain("buffer");
|
||||
});
|
||||
|
||||
it("builds redacted validation report summaries without raw rejected rows", () => {
|
||||
const rowResults: ImportRowResult[] = [
|
||||
{ ok: true, rowNumber: 1, normalized: { email: "a@example.com" } },
|
||||
{ ok: false, rowNumber: 2, errors: [{ severity: "error", code: "required", message: "password= [REDACTED]", rowNumber: 2 }] },
|
||||
];
|
||||
const duplicateReport = detectDuplicateRows(
|
||||
[
|
||||
{ rowNumber: 1, normalized: { email: "a@example.com" } },
|
||||
{ rowNumber: 3, normalized: { email: "a@example.com" } },
|
||||
],
|
||||
["email"],
|
||||
);
|
||||
|
||||
const report = createValidationReport("csv", rowResults, duplicateReport);
|
||||
const serialized = JSON.stringify(report);
|
||||
|
||||
expect(report.summary).toMatchObject({
|
||||
format: "csv",
|
||||
totalRows: 2,
|
||||
acceptedRows: 1,
|
||||
rejectedRows: 1,
|
||||
duplicateGroups: 1,
|
||||
duplicateRows: 2,
|
||||
});
|
||||
expect(report.diagnostics).toHaveLength(3);
|
||||
expect(report.errorReport).toMatchObject({ rejectedRows: 1 });
|
||||
expect(serialized).not.toContain("rejectedValues");
|
||||
expect(serialized).not.toContain("rawRows");
|
||||
expect(serialized).not.toContain("requestBody");
|
||||
expect(serialized).not.toContain("super-secret");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,272 @@
|
||||
import {
|
||||
createRedactedParserDiagnostic,
|
||||
redactDiagnosticMessage,
|
||||
summarizeRowResults,
|
||||
type ImportDryRunSummary,
|
||||
type ImportFormat,
|
||||
type ImportRowErrorResult,
|
||||
type ImportRowResult,
|
||||
type NormalizedImportRow,
|
||||
type NormalizedRowPrimitive,
|
||||
type NormalizedRowValue,
|
||||
type ParserDiagnostic,
|
||||
type ParserErrorReport,
|
||||
} from "./index.js";
|
||||
|
||||
export type DuplicateKeyField = {
|
||||
field: string;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type DuplicateDetectionRow = {
|
||||
rowNumber?: number;
|
||||
normalized: NormalizedImportRow;
|
||||
};
|
||||
|
||||
export type DuplicateKeyPart = {
|
||||
field: string;
|
||||
valuePreview: string;
|
||||
};
|
||||
|
||||
export type DuplicateGroup = {
|
||||
id: string;
|
||||
key: readonly DuplicateKeyPart[];
|
||||
rowNumbers: readonly number[];
|
||||
};
|
||||
|
||||
export type DuplicateDetectionSummary = {
|
||||
totalRows: number;
|
||||
duplicateGroups: number;
|
||||
duplicateRows: number;
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
};
|
||||
|
||||
export type DuplicateDetectionReport = {
|
||||
groups: readonly DuplicateGroup[];
|
||||
rowResults: readonly ImportRowResult[];
|
||||
summary: DuplicateDetectionSummary;
|
||||
errorReport?: ParserErrorReport;
|
||||
};
|
||||
|
||||
export type ValidationReportSummary = ImportDryRunSummary & {
|
||||
duplicateGroups: number;
|
||||
duplicateRows: number;
|
||||
};
|
||||
|
||||
export type ValidationReport = {
|
||||
summary: ValidationReportSummary;
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
duplicateGroups: readonly DuplicateGroup[];
|
||||
errorReport?: ParserErrorReport;
|
||||
};
|
||||
|
||||
type DuplicateKeySignature = {
|
||||
signature: string;
|
||||
parts: readonly DuplicateKeyPart[];
|
||||
};
|
||||
|
||||
function normalizeDuplicateKeyValue(value: NormalizedRowValue | undefined): string | undefined {
|
||||
if (value === undefined || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
const normalizedItems = value
|
||||
.map((item) => normalizeDuplicateKeyPrimitive(item))
|
||||
.filter((item): item is string => item !== undefined);
|
||||
|
||||
return normalizedItems.length > 0 ? JSON.stringify(normalizedItems) : undefined;
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function normalizeDuplicateKeyPrimitive(value: NormalizedRowPrimitive): string | undefined {
|
||||
if (value === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function toDuplicateKeyField(field: string | DuplicateKeyField): DuplicateKeyField {
|
||||
return typeof field === "string" ? { field } : field;
|
||||
}
|
||||
|
||||
function createDuplicateKey(row: NormalizedImportRow, fields: readonly DuplicateKeyField[]): DuplicateKeySignature | undefined {
|
||||
const parts: DuplicateKeyPart[] = [];
|
||||
const signatureParts: string[] = [];
|
||||
|
||||
for (const field of fields) {
|
||||
if (!field.field) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalizedValue = normalizeDuplicateKeyValue(row[field.field]);
|
||||
if (normalizedValue === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
parts.push({ field: field.field, valuePreview: redactDiagnosticMessage(normalizedValue, 80) });
|
||||
signatureParts.push(`${JSON.stringify(field.field)}:${JSON.stringify(normalizedValue)}`);
|
||||
}
|
||||
|
||||
if (signatureParts.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
signature: signatureParts.join("|"),
|
||||
parts,
|
||||
};
|
||||
}
|
||||
|
||||
function duplicateGroupId(index: number): string {
|
||||
return `duplicate-group-${String(index + 1).padStart(4, "0")}`;
|
||||
}
|
||||
|
||||
function createDuplicateDiagnostic(group: DuplicateGroup, rowNumber: number): ParserDiagnostic {
|
||||
return createRedactedParserDiagnostic(
|
||||
"duplicate_key",
|
||||
`Duplicate import row matches ${group.id} on ${group.key.map((part) => `${part.field}=${part.valuePreview}`).join(", ")}.`,
|
||||
{
|
||||
rowNumber,
|
||||
valuePreview: group.id,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function detectDuplicateRows(
|
||||
rows: readonly DuplicateDetectionRow[],
|
||||
keyFields: readonly (string | DuplicateKeyField)[],
|
||||
): DuplicateDetectionReport {
|
||||
const fields = keyFields.map(toDuplicateKeyField).filter((field) => field.field.trim().length > 0);
|
||||
const rowResults: ImportRowResult[] = rows.map((row, index) => ({
|
||||
ok: true,
|
||||
rowNumber: row.rowNumber ?? index + 1,
|
||||
normalized: row.normalized,
|
||||
warnings: undefined,
|
||||
}));
|
||||
|
||||
if (fields.length === 0) {
|
||||
return {
|
||||
groups: [],
|
||||
rowResults,
|
||||
summary: {
|
||||
totalRows: rows.length,
|
||||
duplicateGroups: 0,
|
||||
duplicateRows: 0,
|
||||
diagnostics: [],
|
||||
},
|
||||
errorReport: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const bySignature = new Map<string, { parts: readonly DuplicateKeyPart[]; rowNumbers: number[] }>();
|
||||
|
||||
for (let index = 0; index < rows.length; index += 1) {
|
||||
const row = rows[index];
|
||||
const key = createDuplicateKey(row.normalized, fields);
|
||||
if (!key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const rowNumber = row.rowNumber ?? index + 1;
|
||||
const existing = bySignature.get(key.signature);
|
||||
if (existing) {
|
||||
existing.rowNumbers.push(rowNumber);
|
||||
} else {
|
||||
bySignature.set(key.signature, { parts: key.parts, rowNumbers: [rowNumber] });
|
||||
}
|
||||
}
|
||||
|
||||
const groups = Array.from(bySignature.values())
|
||||
.filter((group) => group.rowNumbers.length > 1)
|
||||
.sort((left, right) => left.rowNumbers[0] - right.rowNumbers[0])
|
||||
.map<DuplicateGroup>((group, index) => ({
|
||||
id: duplicateGroupId(index),
|
||||
key: group.parts,
|
||||
rowNumbers: [...group.rowNumbers].sort((left, right) => left - right),
|
||||
}));
|
||||
|
||||
const diagnostics = groups.flatMap((group) => group.rowNumbers.map((rowNumber) => createDuplicateDiagnostic(group, rowNumber)));
|
||||
const diagnosticsByRowNumber = new Map<number, ParserDiagnostic[]>();
|
||||
for (const diagnostic of diagnostics) {
|
||||
if (diagnostic.rowNumber === undefined) {
|
||||
continue;
|
||||
}
|
||||
diagnosticsByRowNumber.set(diagnostic.rowNumber, [...(diagnosticsByRowNumber.get(diagnostic.rowNumber) ?? []), diagnostic]);
|
||||
}
|
||||
|
||||
const duplicateRowResults = rowResults.map<ImportRowResult>((row) => {
|
||||
const errors = diagnosticsByRowNumber.get(row.rowNumber);
|
||||
if (!errors) {
|
||||
return row;
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
rowNumber: row.rowNumber,
|
||||
errors,
|
||||
} satisfies ImportRowErrorResult;
|
||||
});
|
||||
|
||||
return {
|
||||
groups,
|
||||
rowResults: duplicateRowResults,
|
||||
summary: {
|
||||
totalRows: rows.length,
|
||||
duplicateGroups: groups.length,
|
||||
duplicateRows: new Set(groups.flatMap((group) => group.rowNumbers)).size,
|
||||
diagnostics,
|
||||
},
|
||||
errorReport:
|
||||
diagnostics.length > 0
|
||||
? {
|
||||
diagnostics,
|
||||
rejectedRows: diagnosticsByRowNumber.size,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function createValidationReport(
|
||||
format: ImportFormat,
|
||||
rowResults: readonly ImportRowResult[],
|
||||
duplicateReport: Pick<DuplicateDetectionReport, "groups" | "summary">,
|
||||
): ValidationReport {
|
||||
const summaryCounts = summarizeRowResults(rowResults);
|
||||
const diagnostics = [
|
||||
...rowResults.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors)),
|
||||
...duplicateReport.summary.diagnostics,
|
||||
];
|
||||
|
||||
return {
|
||||
summary: {
|
||||
format,
|
||||
...summaryCounts,
|
||||
duplicateGroups: duplicateReport.summary.duplicateGroups,
|
||||
duplicateRows: duplicateReport.summary.duplicateRows,
|
||||
diagnostics,
|
||||
},
|
||||
diagnostics,
|
||||
duplicateGroups: duplicateReport.groups,
|
||||
errorReport:
|
||||
diagnostics.length > 0
|
||||
? {
|
||||
diagnostics,
|
||||
rejectedRows: summaryCounts.rejectedRows,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
createImportDryRunPreview,
|
||||
createRedactedParserDiagnostic,
|
||||
detectImportFormatFromContentType,
|
||||
detectImportFormatFromExtension,
|
||||
isSupportedImportFormat,
|
||||
redactDiagnosticMessage,
|
||||
summarizeRowResults,
|
||||
type ImportParserOutput,
|
||||
type ImportRowResult,
|
||||
} from "./index.js";
|
||||
|
||||
describe("imports parser contracts", () => {
|
||||
it("guards supported formats explicitly", () => {
|
||||
expect(isSupportedImportFormat("csv")).toBe(true);
|
||||
expect(isSupportedImportFormat("json")).toBe(true);
|
||||
expect(isSupportedImportFormat("xlsx")).toBe(true);
|
||||
expect(isSupportedImportFormat("xml")).toBe(false);
|
||||
expect(isSupportedImportFormat("CSV")).toBe(false);
|
||||
expect(isSupportedImportFormat("")).toBe(false);
|
||||
});
|
||||
|
||||
it("detects csv/json/xlsx formats from extensions and content types", () => {
|
||||
expect(detectImportFormatFromExtension("members.CSV")).toBe("csv");
|
||||
expect(detectImportFormatFromExtension("members.json")).toBe("json");
|
||||
expect(detectImportFormatFromExtension("exports/Members.XLSX")).toBe("xlsx");
|
||||
expect(detectImportFormatFromExtension("xlsx")).toBe("xlsx");
|
||||
expect(detectImportFormatFromContentType("text/csv; charset=utf-8")).toBe("csv");
|
||||
expect(detectImportFormatFromContentType("application/json; charset=utf-8")).toBe("json");
|
||||
expect(detectImportFormatFromContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")).toBe("xlsx");
|
||||
});
|
||||
|
||||
it("rejects unknown and ambiguous formats", () => {
|
||||
expect(detectImportFormatFromExtension("payload.unsupported")).toBeUndefined();
|
||||
expect(detectImportFormatFromExtension("members.csv.zip")).toBeUndefined();
|
||||
expect(detectImportFormatFromExtension(undefined)).toBeUndefined();
|
||||
expect(detectImportFormatFromContentType("application/octet-stream")).toBeUndefined();
|
||||
expect(detectImportFormatFromContentType("text/plain")).toBeUndefined();
|
||||
expect(detectImportFormatFromContentType(undefined)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("summarizes accepted and rejected row results", () => {
|
||||
const rows: ImportRowResult[] = [
|
||||
{ ok: true, rowNumber: 1, normalized: { email: "a@example.com" } },
|
||||
{ ok: true, rowNumber: 2, normalized: { email: "b@example.com", tags: ["member", "paid"] } },
|
||||
{
|
||||
ok: false,
|
||||
rowNumber: 3,
|
||||
errors: [{ severity: "error", code: "missing_email", message: "Email is required." }],
|
||||
},
|
||||
];
|
||||
|
||||
expect(summarizeRowResults(rows)).toEqual({ totalRows: 3, acceptedRows: 2, rejectedRows: 1 });
|
||||
expect(summarizeRowResults([])).toEqual({ totalRows: 0, acceptedRows: 0, rejectedRows: 0 });
|
||||
});
|
||||
|
||||
it("redacts secrets and body-like diagnostics without leaking raw values", () => {
|
||||
const secretRedacted = redactDiagnosticMessage("token=abc123 password:super-secret authorization=BearerToken");
|
||||
const bodyRedacted = redactDiagnosticMessage("body={raw:'payload'} rawRows=[secret]");
|
||||
|
||||
expect(secretRedacted).toContain("token= [REDACTED]");
|
||||
expect(secretRedacted).toContain("password: [REDACTED]");
|
||||
expect(secretRedacted).toContain("authorization= [REDACTED]");
|
||||
expect(bodyRedacted).toContain("body= [REDACTED]");
|
||||
expect(bodyRedacted).toContain("rawRows= [REDACTED]");
|
||||
expect(secretRedacted).not.toContain("abc123");
|
||||
expect(secretRedacted).not.toContain("super-secret");
|
||||
expect(bodyRedacted).not.toContain("{raw:'payload'}");
|
||||
expect(bodyRedacted).not.toContain("[secret]");
|
||||
expect(redactDiagnosticMessage("x".repeat(10), 5)).toBe("xxxx…");
|
||||
expect(createRedactedParserDiagnostic("parse_failed", "apiKey=secret-value", { rowNumber: 3 })).toEqual({
|
||||
severity: "error",
|
||||
code: "parse_failed",
|
||||
message: "apiKey= [REDACTED]",
|
||||
rowNumber: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("creates accepted mapped dry-run previews", () => {
|
||||
const output = createImportDryRunPreview({
|
||||
format: "csv",
|
||||
fields: [{ targetField: "email", required: true, label: "Email" }],
|
||||
mappings: [
|
||||
{ sourceColumn: "Email Address", targetField: "email", required: true, label: "Email" },
|
||||
{ sourceColumn: "Member Name", targetField: "name" },
|
||||
],
|
||||
rows: [
|
||||
{ rowNumber: 10, values: { "Email Address": "a@example.com", "Member Name": "Ada", Ignored: "secret=not-mapped" } },
|
||||
{ rowNumber: 11, values: { "Email Address": "b@example.com", "Member Name": "Ben" } },
|
||||
],
|
||||
});
|
||||
|
||||
expect(output.summary).toMatchObject({ format: "csv", totalRows: 2, acceptedRows: 2, rejectedRows: 0, diagnostics: [] });
|
||||
expect(output.rows).toEqual([
|
||||
{ ok: true, rowNumber: 10, normalized: { email: "a@example.com", name: "Ada" }, warnings: undefined },
|
||||
{ ok: true, rowNumber: 11, normalized: { email: "b@example.com", name: "Ben" }, warnings: undefined },
|
||||
]);
|
||||
expect(output.mappedRows).toEqual([
|
||||
{ rowNumber: 10, normalized: { email: "a@example.com", name: "Ada" }, sourceColumns: ["Email Address", "Member Name"] },
|
||||
{ rowNumber: 11, normalized: { email: "b@example.com", name: "Ben" }, sourceColumns: ["Email Address", "Member Name"] },
|
||||
]);
|
||||
expect(JSON.stringify(output)).not.toContain("not-mapped");
|
||||
});
|
||||
|
||||
it("rejects missing required mapped values deterministically", () => {
|
||||
const output = createImportDryRunPreview({
|
||||
format: "csv",
|
||||
fields: [{ targetField: "email", required: true, label: "Email" }],
|
||||
mappings: [{ sourceColumn: "Email Address", targetField: "email", required: true, label: "Email" }],
|
||||
rows: [
|
||||
{ rowNumber: 1, values: { "Email Address": "valid@example.com" } },
|
||||
{ rowNumber: 2, values: { "Email Address": " " } },
|
||||
{ rowNumber: 3, values: {} },
|
||||
],
|
||||
});
|
||||
|
||||
expect(output.summary).toMatchObject({ totalRows: 3, acceptedRows: 1, rejectedRows: 2 });
|
||||
expect(output.rows[1]).toMatchObject({
|
||||
ok: false,
|
||||
rowNumber: 2,
|
||||
errors: [{ severity: "error", code: "missing_required_field", message: "Email is required for import preview.", rowNumber: 2, column: "email" }],
|
||||
});
|
||||
expect(output.rows[2]).toMatchObject({
|
||||
ok: false,
|
||||
rowNumber: 3,
|
||||
errors: [{ severity: "error", code: "missing_required_field", message: "Email is required for import preview.", rowNumber: 3, column: "email" }],
|
||||
});
|
||||
expect(output.errorReport).toMatchObject({ rejectedRows: 2 });
|
||||
});
|
||||
|
||||
it("ignores unmapped columns and invalid mapping entries", () => {
|
||||
const output = createImportDryRunPreview({
|
||||
format: "json",
|
||||
fields: [{ targetField: "email", required: true }],
|
||||
mappings: [
|
||||
{ sourceColumn: "email", targetField: "email" },
|
||||
{ sourceColumn: "ignored", targetField: "" },
|
||||
{ sourceColumn: "", targetField: "emptySource" },
|
||||
],
|
||||
rows: [{ values: { email: "a@example.com", ignored: "password:super-secret", extra: "rawBody={secret}" } }],
|
||||
});
|
||||
|
||||
expect(output.summary).toMatchObject({ format: "json", totalRows: 1, acceptedRows: 1, rejectedRows: 0 });
|
||||
expect(output.rows[0]).toMatchObject({ ok: true, rowNumber: 1, normalized: { email: "a@example.com" } });
|
||||
expect(JSON.stringify(output)).not.toContain("super-secret");
|
||||
expect(JSON.stringify(output)).not.toContain("rawBody");
|
||||
});
|
||||
|
||||
it("redacts secret-like required values in preview diagnostics", () => {
|
||||
const output = createImportDryRunPreview({
|
||||
format: "csv",
|
||||
fields: [{ targetField: "apiToken", required: true, label: "token=secret-value" }],
|
||||
mappings: [{ sourceColumn: "Token", targetField: "apiToken" }],
|
||||
rows: [{ rowNumber: 7, values: { Token: "" } }],
|
||||
});
|
||||
|
||||
const serialized = JSON.stringify(output);
|
||||
|
||||
expect(output.summary.diagnostics[0]).toMatchObject({
|
||||
code: "missing_required_field",
|
||||
message: "token= [REDACTED] is required for import preview.",
|
||||
rowNumber: 7,
|
||||
column: "apiToken",
|
||||
valuePreview: "",
|
||||
});
|
||||
expect(serialized).not.toContain("secret-value");
|
||||
expect(serialized).not.toContain("body=");
|
||||
expect(serialized).not.toContain("rawFile");
|
||||
});
|
||||
|
||||
it("represents row-level validation errors without raw file buffers or request bodies", () => {
|
||||
const output: ImportParserOutput = {
|
||||
rows: [
|
||||
{ ok: true, rowNumber: 1, normalized: { email: "valid@example.com", active: true, score: 7, tags: ["new"] } },
|
||||
{
|
||||
ok: false,
|
||||
rowNumber: 2,
|
||||
rejectedValues: { email: "", active: null },
|
||||
errors: [
|
||||
{
|
||||
severity: "error",
|
||||
code: "required",
|
||||
message: "Email is required.",
|
||||
rowNumber: 2,
|
||||
column: "email",
|
||||
valuePreview: "",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
summary: {
|
||||
format: "csv",
|
||||
totalRows: 2,
|
||||
acceptedRows: 1,
|
||||
rejectedRows: 1,
|
||||
diagnostics: [{ severity: "warning", code: "blank_row", message: "Skipped blank row.", rowNumber: 3 }],
|
||||
},
|
||||
errorReport: {
|
||||
diagnostics: [{ severity: "error", code: "required", message: "Email is required.", rowNumber: 2, column: "email" }],
|
||||
rejectedRows: 1,
|
||||
errorReportKey: "imports/reports/import-job-1.json",
|
||||
},
|
||||
};
|
||||
|
||||
const serialized = JSON.stringify(output);
|
||||
|
||||
expect(output.rows[1]).toMatchObject({ ok: false, rowNumber: 2 });
|
||||
expect(serialized).not.toContain("bytes");
|
||||
expect(serialized).not.toContain("buffer");
|
||||
expect(serialized).not.toContain("requestBody");
|
||||
expect(serialized).not.toContain("rawFile");
|
||||
});
|
||||
|
||||
it("keeps parser contracts free of database, storage, framework, and prisma imports", () => {
|
||||
const sourcePath = path.join(path.dirname(fileURLToPath(import.meta.url)), "./index.ts");
|
||||
const source = readFileSync(sourcePath, "utf8");
|
||||
|
||||
expect(source).not.toMatch(/from\s+["'](?:@toolctrl\/db|@toolctrl\/storage|react|react-router|@prisma\/client|\.prisma|prisma)["']/);
|
||||
expect(source).not.toMatch(/import\s+["'](?:@toolctrl\/db|@toolctrl\/storage|react|react-router|@prisma\/client|\.prisma|prisma)["']/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,319 @@
|
||||
export * from "./dedupe.js";
|
||||
|
||||
export const IMPORT_FORMATS = ["csv", "json", "xlsx"] as const;
|
||||
export type ImportFormat = (typeof IMPORT_FORMATS)[number];
|
||||
|
||||
export type ImportSourceDescriptor = {
|
||||
tenantId: string;
|
||||
requesterId: string;
|
||||
sourceKey: string;
|
||||
fileName?: string;
|
||||
contentType?: string;
|
||||
byteLength?: number;
|
||||
requestId?: string;
|
||||
};
|
||||
|
||||
export type ColumnMapping = {
|
||||
sourceColumn: string;
|
||||
targetField: string;
|
||||
required?: boolean;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type NormalizedRowPrimitive = string | number | boolean | null;
|
||||
export type NormalizedRowValue = NormalizedRowPrimitive | readonly NormalizedRowPrimitive[];
|
||||
export type NormalizedImportRow = Record<string, NormalizedRowValue>;
|
||||
|
||||
export type ImportRowSuccessResult = {
|
||||
ok: true;
|
||||
rowNumber: number;
|
||||
normalized: NormalizedImportRow;
|
||||
warnings?: readonly ParserDiagnostic[];
|
||||
};
|
||||
|
||||
export type ImportRowErrorResult = {
|
||||
ok: false;
|
||||
rowNumber: number;
|
||||
errors: readonly ParserDiagnostic[];
|
||||
rejectedValues?: NormalizedImportRow;
|
||||
};
|
||||
|
||||
export type ImportRowResult = ImportRowSuccessResult | ImportRowErrorResult;
|
||||
|
||||
export type ParserDiagnosticSeverity = "info" | "warning" | "error";
|
||||
|
||||
export type ParserDiagnostic = {
|
||||
severity: ParserDiagnosticSeverity;
|
||||
code: string;
|
||||
message: string;
|
||||
rowNumber?: number;
|
||||
column?: string;
|
||||
valuePreview?: string;
|
||||
};
|
||||
|
||||
export type ParserErrorReport = {
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
rejectedRows: number;
|
||||
errorReportKey?: string;
|
||||
};
|
||||
|
||||
export type ImportDryRunSummary = {
|
||||
format: ImportFormat;
|
||||
totalRows: number;
|
||||
acceptedRows: number;
|
||||
rejectedRows: number;
|
||||
diagnostics: readonly ParserDiagnostic[];
|
||||
};
|
||||
|
||||
export type ImportParserInput = {
|
||||
source: ImportSourceDescriptor;
|
||||
format: ImportFormat;
|
||||
mappings: readonly ColumnMapping[];
|
||||
text?: string;
|
||||
bytes?: Uint8Array;
|
||||
};
|
||||
|
||||
export type ImportParserOutput = {
|
||||
rows: readonly ImportRowResult[];
|
||||
summary: ImportDryRunSummary;
|
||||
errorReport?: ParserErrorReport;
|
||||
};
|
||||
|
||||
export type ImportParser = {
|
||||
readonly format: ImportFormat;
|
||||
parse(input: ImportParserInput): Promise<ImportParserOutput> | ImportParserOutput;
|
||||
};
|
||||
|
||||
export type ImportDryRunFieldDefinition = {
|
||||
targetField: string;
|
||||
required?: boolean;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type ImportPreviewInputRow = {
|
||||
rowNumber?: number;
|
||||
values: NormalizedImportRow;
|
||||
};
|
||||
|
||||
export type ImportMappedPreviewRow = {
|
||||
rowNumber: number;
|
||||
normalized: NormalizedImportRow;
|
||||
sourceColumns: readonly string[];
|
||||
};
|
||||
|
||||
export type ImportDryRunPreviewInput = {
|
||||
format: ImportFormat;
|
||||
fields: readonly ImportDryRunFieldDefinition[];
|
||||
mappings: readonly ColumnMapping[];
|
||||
rows: readonly ImportPreviewInputRow[];
|
||||
};
|
||||
|
||||
export type ImportDryRunPreviewOutput = ImportParserOutput & {
|
||||
mappedRows: readonly ImportMappedPreviewRow[];
|
||||
};
|
||||
|
||||
const EXTENSION_FORMATS: Readonly<Record<string, ImportFormat>> = {
|
||||
csv: "csv",
|
||||
json: "json",
|
||||
xlsx: "xlsx",
|
||||
};
|
||||
|
||||
const CONTENT_TYPE_FORMATS: Readonly<Record<string, ImportFormat>> = {
|
||||
"application/csv": "csv",
|
||||
"application/json": "json",
|
||||
"application/vnd.ms-excel": "csv",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
|
||||
"text/csv": "csv",
|
||||
};
|
||||
|
||||
const SECRET_PATTERN = /(?:body|payload|raw(?:file|body|payload|rows)?)\s*[:=]\s*(?:\{[^}]*\}|\[[^\]]*\]|[^\s,;]+)|(?:api[_-]?key|authorization|bearer|password|secret|token)\s*[:=]\s*[^\s,;]+/gi;
|
||||
const REDACTED = "[REDACTED]";
|
||||
|
||||
export function isSupportedImportFormat(value: string): value is ImportFormat {
|
||||
return (IMPORT_FORMATS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
export function detectImportFormatFromExtension(fileNameOrExtension: string | undefined): ImportFormat | undefined {
|
||||
if (!fileNameOrExtension) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const extension = fileNameOrExtension.trim().toLowerCase().replace(/^.*\./, "");
|
||||
return EXTENSION_FORMATS[extension];
|
||||
}
|
||||
|
||||
export function detectImportFormatFromContentType(contentType: string | undefined): ImportFormat | undefined {
|
||||
if (!contentType) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase();
|
||||
return normalized ? CONTENT_TYPE_FORMATS[normalized] : undefined;
|
||||
}
|
||||
|
||||
export function summarizeRowResults(rows: readonly ImportRowResult[]): Pick<ImportDryRunSummary, "totalRows" | "acceptedRows" | "rejectedRows"> {
|
||||
const acceptedRows = rows.filter((row) => row.ok).length;
|
||||
|
||||
return {
|
||||
totalRows: rows.length,
|
||||
acceptedRows,
|
||||
rejectedRows: rows.length - acceptedRows,
|
||||
};
|
||||
}
|
||||
|
||||
function isBlankPreviewValue(value: NormalizedRowValue | undefined): boolean {
|
||||
if (value === undefined || value === null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
return value.trim().length === 0;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.length === 0 || value.every((item) => isBlankPreviewValue(item));
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function previewValue(value: NormalizedRowValue | undefined): string | undefined {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return redactDiagnosticMessage(String(value), 80);
|
||||
}
|
||||
|
||||
function buildRequiredFieldDefinitions(
|
||||
fields: readonly ImportDryRunFieldDefinition[],
|
||||
mappings: readonly ColumnMapping[],
|
||||
): Map<string, ImportDryRunFieldDefinition> {
|
||||
const requiredFields = new Map<string, ImportDryRunFieldDefinition>();
|
||||
|
||||
for (const field of fields) {
|
||||
if (field.required) {
|
||||
requiredFields.set(field.targetField, field);
|
||||
}
|
||||
}
|
||||
|
||||
for (const mapping of mappings) {
|
||||
if (mapping.required) {
|
||||
requiredFields.set(mapping.targetField, {
|
||||
targetField: mapping.targetField,
|
||||
required: true,
|
||||
label: mapping.label,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return requiredFields;
|
||||
}
|
||||
|
||||
export function createImportDryRunPreview(input: ImportDryRunPreviewInput): ImportDryRunPreviewOutput {
|
||||
const requiredFields = buildRequiredFieldDefinitions(input.fields, input.mappings);
|
||||
const rows = input.rows.map<ImportRowResult>((row, index) => {
|
||||
const rowNumber = row.rowNumber ?? index + 1;
|
||||
const normalized: NormalizedImportRow = {};
|
||||
const sourceColumns: string[] = [];
|
||||
|
||||
for (const mapping of input.mappings) {
|
||||
if (!mapping.sourceColumn || !mapping.targetField) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Object.prototype.hasOwnProperty.call(row.values, mapping.sourceColumn)) {
|
||||
normalized[mapping.targetField] = row.values[mapping.sourceColumn] ?? null;
|
||||
sourceColumns.push(mapping.sourceColumn);
|
||||
}
|
||||
}
|
||||
|
||||
const errors: ParserDiagnostic[] = [];
|
||||
|
||||
for (const [targetField, field] of requiredFields) {
|
||||
if (isBlankPreviewValue(normalized[targetField])) {
|
||||
errors.push(
|
||||
createRedactedParserDiagnostic(
|
||||
"missing_required_field",
|
||||
`${field.label ?? targetField} is required for import preview.`,
|
||||
{
|
||||
rowNumber,
|
||||
column: targetField,
|
||||
valuePreview: previewValue(normalized[targetField]),
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
rowNumber,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
rowNumber,
|
||||
normalized,
|
||||
warnings: undefined,
|
||||
};
|
||||
});
|
||||
|
||||
const mappedRows = rows
|
||||
.filter((row): row is ImportRowSuccessResult => row.ok)
|
||||
.map<ImportMappedPreviewRow>((row) => ({
|
||||
rowNumber: row.rowNumber,
|
||||
normalized: row.normalized,
|
||||
sourceColumns: input.mappings
|
||||
.filter((mapping) => Object.prototype.hasOwnProperty.call(row.normalized, mapping.targetField))
|
||||
.map((mapping) => mapping.sourceColumn),
|
||||
}));
|
||||
const diagnostics = rows.flatMap((row) => (row.ok ? (row.warnings ?? []) : row.errors));
|
||||
const summaryCounts = summarizeRowResults(rows);
|
||||
|
||||
return {
|
||||
rows,
|
||||
mappedRows,
|
||||
summary: {
|
||||
format: input.format,
|
||||
...summaryCounts,
|
||||
diagnostics,
|
||||
},
|
||||
errorReport:
|
||||
summaryCounts.rejectedRows > 0
|
||||
? {
|
||||
diagnostics,
|
||||
rejectedRows: summaryCounts.rejectedRows,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function createRedactedParserDiagnostic(
|
||||
code: string,
|
||||
message: string,
|
||||
options: Omit<Partial<ParserDiagnostic>, "code" | "message"> = {},
|
||||
): ParserDiagnostic {
|
||||
return {
|
||||
severity: "error",
|
||||
...options,
|
||||
code,
|
||||
message: redactDiagnosticMessage(message),
|
||||
};
|
||||
}
|
||||
|
||||
export function redactDiagnosticMessage(message: string, maxLength = 300): string {
|
||||
const redacted = message.replace(SECRET_PATTERN, (match) => {
|
||||
const colonIndex = match.indexOf(":");
|
||||
const equalsIndex = match.indexOf("=");
|
||||
const separatorIndex =
|
||||
colonIndex === -1 ? equalsIndex : equalsIndex === -1 ? colonIndex : Math.min(colonIndex, equalsIndex);
|
||||
const prefix = separatorIndex >= 0 ? match.slice(0, separatorIndex + 1) : "secret:";
|
||||
return `${prefix} ${REDACTED}`;
|
||||
});
|
||||
|
||||
return redacted.length > maxLength ? `${redacted.slice(0, maxLength - 1)}…` : redacted;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"allowJs": false,
|
||||
"allowImportingTsExtensions": false,
|
||||
"lib": ["ES2022"],
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"noEmit": true,
|
||||
"types": []
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in new issue
Block a user