Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,132 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportTableValidationError } from "../types.js";
|
||||
import { buildCsvExportArtifact, serializeExportTableToCsv } from "./index.js";
|
||||
|
||||
describe("@toolctrl/exports/csv public entrypoint", () => {
|
||||
it("exposes CSV-only runtime helpers without loading unrelated exporters", async () => {
|
||||
const exportsModule = await import("@toolctrl/exports/csv");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"ExportTableValidationError",
|
||||
"buildCsvExportArtifact",
|
||||
"serializeExportTableToCsv",
|
||||
"serializeExportTableToCsvStream",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToCsv", () => {
|
||||
it("uses explicit header labels and column order", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: [
|
||||
{ key: "email", label: "Email address" },
|
||||
{ key: "id", label: "Identifier" },
|
||||
],
|
||||
rows: [{ id: 7, email: "ada@example.com" }],
|
||||
});
|
||||
|
||||
expect(csv).toBe('"Email address","Identifier"\n"ada@example.com","7"');
|
||||
});
|
||||
|
||||
it("escapes commas, quotes, and newlines with RFC-style quoting", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: ["name", "note"],
|
||||
rows: [
|
||||
{
|
||||
name: 'Ada "Countess"',
|
||||
note: "line one,line two\nline three",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(csv).toBe('"name","note"\n"Ada ""Countess""","line one,line two\nline three"');
|
||||
});
|
||||
|
||||
it("neutralizes spreadsheet formula prefixes before CSV escaping", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: [
|
||||
{ key: "equals", label: "=header" },
|
||||
"plus",
|
||||
"minus",
|
||||
"at",
|
||||
"spaceFormula",
|
||||
"spaceTab",
|
||||
"spaceCr",
|
||||
"spaceLf",
|
||||
"safe",
|
||||
],
|
||||
rows: [
|
||||
{
|
||||
equals: "=1+1",
|
||||
plus: "+1",
|
||||
minus: "-1",
|
||||
at: "@cmd",
|
||||
spaceFormula: " =1+1",
|
||||
spaceTab: " \tvalue",
|
||||
spaceCr: " \rvalue",
|
||||
spaceLf: " \nvalue",
|
||||
safe: " value",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(csv).toBe([
|
||||
`"'=header","plus","minus","at","spaceFormula","spaceTab","spaceCr","spaceLf","safe"`,
|
||||
`"'=1+1","'+1","'-1","'@cmd","' =1+1","' \tvalue","' \rvalue","' \nvalue"," value"`,
|
||||
].join("\n"));
|
||||
});
|
||||
|
||||
it("serializes null and undefined values as blank cells", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: ["id", "empty", "missing"],
|
||||
rows: [{ id: 1, empty: null, missing: undefined }],
|
||||
});
|
||||
|
||||
expect(csv).toBe([
|
||||
`"id","empty","missing"`,
|
||||
`"1","",""`,
|
||||
].join("\n"));
|
||||
});
|
||||
|
||||
it("projects selected columns while preserving deterministic scalar strings", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: ["createdAt", "active", "amount"],
|
||||
rows: [
|
||||
{
|
||||
active: false,
|
||||
amount: 42.5,
|
||||
createdAt: new Date("2024-01-02T03:04:05.006Z"),
|
||||
ignored: "not exported",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(csv).toBe('"createdAt","active","amount"\n"2024-01-02T03:04:05.006Z","false","42.5"');
|
||||
});
|
||||
|
||||
it("rejects duplicate columns through the CSV API", () => {
|
||||
expect(() =>
|
||||
serializeExportTableToCsv({
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildCsvExportArtifact", () => {
|
||||
it("returns CSV bytes with export metadata", () => {
|
||||
const artifact = buildCsvExportArtifact({
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
});
|
||||
|
||||
expect(artifact).toEqual({
|
||||
bytes: '"id"\n"1"\n"2"',
|
||||
contentType: "text/csv; charset=utf-8",
|
||||
extension: "csv",
|
||||
rowCount: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import { type ExportColumn, type ExportRecord, type ExportTable, normalizeExportTable } from "../types.js";
|
||||
import { neutralizeSpreadsheetFormula } from "../spreadsheet-safety.js";
|
||||
|
||||
export { ExportTableValidationError } from "../types.js";
|
||||
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable, NormalizedExportColumn, NormalizedExportRow, NormalizedExportTable } from "../types.js";
|
||||
|
||||
export type CsvExportOptions = {
|
||||
eol?: "\n" | "\r\n";
|
||||
delimiter?: "," | "|" | ";";
|
||||
};
|
||||
|
||||
export type CsvExportArtifact = {
|
||||
bytes: string;
|
||||
contentType: "text/csv; charset=utf-8";
|
||||
extension: "csv";
|
||||
rowCount: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Streaming CSV writer. Iterates over `table.rows` row-by-row, writing each
|
||||
* encoded record directly to `writeStream`. Respects writable back-pressure
|
||||
* via the standard `write`/`drain` contract so worker memory stays flat for
|
||||
* tables of arbitrary size.
|
||||
*
|
||||
* The function is intentionally permissive about `table.rows` — it accepts
|
||||
* any sync or async iterable of {@link ExportRecord}, which lets callers
|
||||
* pipe a Prisma cursor (chunked async iterator) without buffering all rows.
|
||||
*/
|
||||
export async function serializeExportTableToCsvStream(
|
||||
writeStream: Writable,
|
||||
table: CsvStreamTable,
|
||||
options: CsvExportOptions = {},
|
||||
): Promise<{ rowCount: number }> {
|
||||
const eol = options.eol ?? "\n";
|
||||
const delimiter = options.delimiter ?? ",";
|
||||
const columns = resolveCsvColumns(table);
|
||||
const header = columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
|
||||
|
||||
await writeChunk(writeStream, header);
|
||||
|
||||
let rowCount = 0;
|
||||
let isFirstRow = true;
|
||||
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
if (row === null || typeof row !== "object" || Array.isArray(row)) {
|
||||
throw new Error("Export table row must be an object record.");
|
||||
}
|
||||
const encoded = columns.map((column) => escapeCsvCell(stringifyValue(row[column.key]))).join(delimiter);
|
||||
if (isFirstRow) {
|
||||
// header already emitted — every row is preceded by an eol so the
|
||||
// final byte of the stream is never an unterminated delimiter.
|
||||
isFirstRow = false;
|
||||
}
|
||||
await writeChunk(writeStream, `${eol}${encoded}`);
|
||||
rowCount += 1;
|
||||
}
|
||||
|
||||
return { rowCount };
|
||||
}
|
||||
|
||||
export function serializeExportTableToCsv(table: ExportTable, options: CsvExportOptions = {}): string {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const eol = options.eol ?? "\n";
|
||||
const delimiter = options.delimiter ?? ",";
|
||||
const header = normalizedTable.columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
|
||||
const rows = normalizedTable.rows.map((row) => normalizedTable.columns.map((column) => escapeCsvCell(row[column.key])).join(delimiter));
|
||||
|
||||
return [header, ...rows].join(eol);
|
||||
}
|
||||
|
||||
export function buildCsvExportArtifact(table: ExportTable, options: CsvExportOptions = {}): CsvExportArtifact {
|
||||
const bytes = serializeExportTableToCsv(table, options);
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
|
||||
return {
|
||||
bytes,
|
||||
contentType: "text/csv; charset=utf-8",
|
||||
extension: "csv",
|
||||
rowCount: normalizedTable.rowCount,
|
||||
};
|
||||
}
|
||||
|
||||
export type CsvStreamTable = {
|
||||
columns?: readonly (string | ExportColumn)[];
|
||||
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
|
||||
};
|
||||
|
||||
function resolveCsvColumns(table: CsvStreamTable): { key: string; label: string }[] {
|
||||
if (table.columns === undefined) {
|
||||
throw new Error("Streaming CSV writer requires explicit columns; cannot infer from an async row source.");
|
||||
}
|
||||
const seenKeys = new Set<string>();
|
||||
const columns = table.columns.map((column) => {
|
||||
const normalized = typeof column === "string" ? { key: column, label: column } : { key: column.key, label: column.label ?? column.key };
|
||||
if (seenKeys.has(normalized.key)) {
|
||||
throw new Error(`Export table columns contain duplicate key "${normalized.key}".`);
|
||||
}
|
||||
seenKeys.add(normalized.key);
|
||||
return normalized;
|
||||
});
|
||||
return columns;
|
||||
}
|
||||
|
||||
function stringifyValue(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
if (typeof value === "bigint") return value.toString();
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function escapeCsvCell(value: string): string {
|
||||
const safeValue = neutralizeSpreadsheetFormula(value);
|
||||
return `"${safeValue.replaceAll("\"", "\"\"")}"`;
|
||||
}
|
||||
|
||||
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
|
||||
if (!writeStream.write(chunk)) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
writeStream.off("drain", onDrain);
|
||||
writeStream.off("error", onError);
|
||||
};
|
||||
writeStream.once("drain", onDrain);
|
||||
writeStream.once("error", onError);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
|
||||
if (Symbol.asyncIterator in (source as object)) {
|
||||
return source as AsyncIterable<T>;
|
||||
}
|
||||
return (async function* () {
|
||||
for (const item of source as Iterable<T>) {
|
||||
yield item;
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { serializeExportTableToCsvStream } from "./index.js";
|
||||
|
||||
async function collectStream(stream: PassThrough): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
}
|
||||
|
||||
describe("serializeExportTableToCsvStream", () => {
|
||||
it("writes a header followed by encoded rows", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collectStream(sink);
|
||||
|
||||
const result = await serializeExportTableToCsvStream(
|
||||
sink,
|
||||
{
|
||||
columns: [
|
||||
{ key: "email", label: "Email address" },
|
||||
{ key: "id", label: "Identifier" },
|
||||
],
|
||||
rows: [
|
||||
{ id: 7, email: "ada@example.com" },
|
||||
{ id: 8, email: "grace@example.com" },
|
||||
],
|
||||
},
|
||||
);
|
||||
sink.end();
|
||||
|
||||
expect(result.rowCount).toBe(2);
|
||||
expect(await collected).toBe(
|
||||
[
|
||||
'"Email address","Identifier"',
|
||||
'"ada@example.com","7"',
|
||||
'"grace@example.com","8"',
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
it("supports async iterables as the row source (Prisma cursor friendly)", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collectStream(sink);
|
||||
|
||||
async function* rows() {
|
||||
yield { id: 1, name: "A" };
|
||||
yield { id: 2, name: "B" };
|
||||
yield { id: 3, name: "C" };
|
||||
}
|
||||
|
||||
const result = await serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id", "name"],
|
||||
rows: rows(),
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(result.rowCount).toBe(3);
|
||||
const text = await collected;
|
||||
expect(text.split("\n")).toEqual(['"id","name"', '"1","A"', '"2","B"', '"3","C"']);
|
||||
});
|
||||
|
||||
it("neutralises spreadsheet-formula prefixes the same as the buffered API", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collectStream(sink);
|
||||
|
||||
await serializeExportTableToCsvStream(sink, {
|
||||
columns: ["a"],
|
||||
rows: [{ a: "=1+1" }],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(await collected).toBe(['"a"', "\"'=1+1\""].join("\n"));
|
||||
});
|
||||
|
||||
it("rejects duplicate column keys eagerly", async () => {
|
||||
const sink = new PassThrough();
|
||||
await expect(
|
||||
serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow('Export table columns contain duplicate key "id".');
|
||||
});
|
||||
|
||||
it("keeps worker memory flat for 100k synthetic rows", async () => {
|
||||
const sink = new PassThrough();
|
||||
// Drain the sink as quickly as the producer writes so the PassThrough
|
||||
// buffer cannot mask a buffering regression in the streamer.
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
async function* rows() {
|
||||
for (let i = 0; i < 100_000; i += 1) {
|
||||
yield { id: i, value: `row-${i}` };
|
||||
}
|
||||
}
|
||||
|
||||
if (global.gc) global.gc();
|
||||
const before = process.memoryUsage().heapUsed;
|
||||
const result = await serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id", "value"],
|
||||
rows: rows(),
|
||||
});
|
||||
sink.end();
|
||||
if (global.gc) global.gc();
|
||||
const after = process.memoryUsage().heapUsed;
|
||||
|
||||
expect(result.rowCount).toBe(100_000);
|
||||
// Hard upper bound is generous to absorb GC jitter; a buffering
|
||||
// regression would inflate heap by tens of MB.
|
||||
expect(after - before).toBeLessThan(100 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user