Reorganize repo structure and standards references

This commit is contained in:
david committed 2026-06-29 05:03:42 +02:00
1 parent 3cb93a81be
commit be2054274d
5885 files changed
+2588 -195

No files matched your search

@@ -0,0 +1,132 @@
import { describe, expect, it } from "vitest";
import { ExportTableValidationError } from "../types.js";
import { buildCsvExportArtifact, serializeExportTableToCsv } from "./index.js";
describe("@toolctrl/exports/csv public entrypoint", () => {
it("exposes CSV-only runtime helpers without loading unrelated exporters", async () => {
const exportsModule = await import("@toolctrl/exports/csv");
expect(Object.keys(exportsModule).sort()).toEqual([
"ExportTableValidationError",
"buildCsvExportArtifact",
"serializeExportTableToCsv",
"serializeExportTableToCsvStream",
]);
});
});
describe("serializeExportTableToCsv", () => {
it("uses explicit header labels and column order", () => {
const csv = serializeExportTableToCsv({
columns: [
{ key: "email", label: "Email address" },
{ key: "id", label: "Identifier" },
],
rows: [{ id: 7, email: "ada@example.com" }],
});
expect(csv).toBe('"Email address","Identifier"\n"ada@example.com","7"');
});
it("escapes commas, quotes, and newlines with RFC-style quoting", () => {
const csv = serializeExportTableToCsv({
columns: ["name", "note"],
rows: [
{
name: 'Ada "Countess"',
note: "line one,line two\nline three",
},
],
});
expect(csv).toBe('"name","note"\n"Ada ""Countess""","line one,line two\nline three"');
});
it("neutralizes spreadsheet formula prefixes before CSV escaping", () => {
const csv = serializeExportTableToCsv({
columns: [
{ key: "equals", label: "=header" },
"plus",
"minus",
"at",
"spaceFormula",
"spaceTab",
"spaceCr",
"spaceLf",
"safe",
],
rows: [
{
equals: "=1+1",
plus: "+1",
minus: "-1",
at: "@cmd",
spaceFormula: " =1+1",
spaceTab: " \tvalue",
spaceCr: " \rvalue",
spaceLf: " \nvalue",
safe: " value",
},
],
});
expect(csv).toBe([
`"'=header","plus","minus","at","spaceFormula","spaceTab","spaceCr","spaceLf","safe"`,
`"'=1+1","'+1","'-1","'@cmd","' =1+1","' \tvalue","' \rvalue","' \nvalue"," value"`,
].join("\n"));
});
it("serializes null and undefined values as blank cells", () => {
const csv = serializeExportTableToCsv({
columns: ["id", "empty", "missing"],
rows: [{ id: 1, empty: null, missing: undefined }],
});
expect(csv).toBe([
`"id","empty","missing"`,
`"1","",""`,
].join("\n"));
});
it("projects selected columns while preserving deterministic scalar strings", () => {
const csv = serializeExportTableToCsv({
columns: ["createdAt", "active", "amount"],
rows: [
{
active: false,
amount: 42.5,
createdAt: new Date("2024-01-02T03:04:05.006Z"),
ignored: "not exported",
},
],
});
expect(csv).toBe('"createdAt","active","amount"\n"2024-01-02T03:04:05.006Z","false","42.5"');
});
it("rejects duplicate columns through the CSV API", () => {
expect(() =>
serializeExportTableToCsv({
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
});
});
describe("buildCsvExportArtifact", () => {
it("returns CSV bytes with export metadata", () => {
const artifact = buildCsvExportArtifact({
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
});
expect(artifact).toEqual({
bytes: '"id"\n"1"\n"2"',
contentType: "text/csv; charset=utf-8",
extension: "csv",
rowCount: 2,
});
});
});
@@ -0,0 +1,148 @@
import type { Writable } from "node:stream";
import { type ExportColumn, type ExportRecord, type ExportTable, normalizeExportTable } from "../types.js";
import { neutralizeSpreadsheetFormula } from "../spreadsheet-safety.js";
export { ExportTableValidationError } from "../types.js";
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable, NormalizedExportColumn, NormalizedExportRow, NormalizedExportTable } from "../types.js";
export type CsvExportOptions = {
eol?: "\n" | "\r\n";
delimiter?: "," | "|" | ";";
};
export type CsvExportArtifact = {
bytes: string;
contentType: "text/csv; charset=utf-8";
extension: "csv";
rowCount: number;
};
/**
* Streaming CSV writer. Iterates over `table.rows` row-by-row, writing each
* encoded record directly to `writeStream`. Respects writable back-pressure
* via the standard `write`/`drain` contract so worker memory stays flat for
* tables of arbitrary size.
*
* The function is intentionally permissive about `table.rows` — it accepts
* any sync or async iterable of {@link ExportRecord}, which lets callers
* pipe a Prisma cursor (chunked async iterator) without buffering all rows.
*/
export async function serializeExportTableToCsvStream(
writeStream: Writable,
table: CsvStreamTable,
options: CsvExportOptions = {},
): Promise<{ rowCount: number }> {
const eol = options.eol ?? "\n";
const delimiter = options.delimiter ?? ",";
const columns = resolveCsvColumns(table);
const header = columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
await writeChunk(writeStream, header);
let rowCount = 0;
let isFirstRow = true;
for await (const row of toAsyncIterable(table.rows)) {
if (row === null || typeof row !== "object" || Array.isArray(row)) {
throw new Error("Export table row must be an object record.");
}
const encoded = columns.map((column) => escapeCsvCell(stringifyValue(row[column.key]))).join(delimiter);
if (isFirstRow) {
// header already emitted — every row is preceded by an eol so the
// final byte of the stream is never an unterminated delimiter.
isFirstRow = false;
}
await writeChunk(writeStream, `${eol}${encoded}`);
rowCount += 1;
}
return { rowCount };
}
export function serializeExportTableToCsv(table: ExportTable, options: CsvExportOptions = {}): string {
const normalizedTable = normalizeExportTable(table);
const eol = options.eol ?? "\n";
const delimiter = options.delimiter ?? ",";
const header = normalizedTable.columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
const rows = normalizedTable.rows.map((row) => normalizedTable.columns.map((column) => escapeCsvCell(row[column.key])).join(delimiter));
return [header, ...rows].join(eol);
}
export function buildCsvExportArtifact(table: ExportTable, options: CsvExportOptions = {}): CsvExportArtifact {
const bytes = serializeExportTableToCsv(table, options);
const normalizedTable = normalizeExportTable(table);
return {
bytes,
contentType: "text/csv; charset=utf-8",
extension: "csv",
rowCount: normalizedTable.rowCount,
};
}
export type CsvStreamTable = {
columns?: readonly (string | ExportColumn)[];
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
};
function resolveCsvColumns(table: CsvStreamTable): { key: string; label: string }[] {
if (table.columns === undefined) {
throw new Error("Streaming CSV writer requires explicit columns; cannot infer from an async row source.");
}
const seenKeys = new Set<string>();
const columns = table.columns.map((column) => {
const normalized = typeof column === "string" ? { key: column, label: column } : { key: column.key, label: column.label ?? column.key };
if (seenKeys.has(normalized.key)) {
throw new Error(`Export table columns contain duplicate key "${normalized.key}".`);
}
seenKeys.add(normalized.key);
return normalized;
});
return columns;
}
function stringifyValue(value: unknown): string {
if (value === null || value === undefined) return "";
if (value instanceof Date) return value.toISOString();
if (typeof value === "bigint") return value.toString();
return String(value);
}
function escapeCsvCell(value: string): string {
const safeValue = neutralizeSpreadsheetFormula(value);
return `"${safeValue.replaceAll("\"", "\"\"")}"`;
}
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
if (Symbol.asyncIterator in (source as object)) {
return source as AsyncIterable<T>;
}
return (async function* () {
for (const item of source as Iterable<T>) {
yield item;
}
})();
}
@@ -0,0 +1,115 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import { serializeExportTableToCsvStream } from "./index.js";
async function collectStream(stream: PassThrough): Promise<string> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks).toString("utf8");
}
describe("serializeExportTableToCsvStream", () => {
it("writes a header followed by encoded rows", async () => {
const sink = new PassThrough();
const collected = collectStream(sink);
const result = await serializeExportTableToCsvStream(
sink,
{
columns: [
{ key: "email", label: "Email address" },
{ key: "id", label: "Identifier" },
],
rows: [
{ id: 7, email: "ada@example.com" },
{ id: 8, email: "grace@example.com" },
],
},
);
sink.end();
expect(result.rowCount).toBe(2);
expect(await collected).toBe(
[
'"Email address","Identifier"',
'"ada@example.com","7"',
'"grace@example.com","8"',
].join("\n"),
);
});
it("supports async iterables as the row source (Prisma cursor friendly)", async () => {
const sink = new PassThrough();
const collected = collectStream(sink);
async function* rows() {
yield { id: 1, name: "A" };
yield { id: 2, name: "B" };
yield { id: 3, name: "C" };
}
const result = await serializeExportTableToCsvStream(sink, {
columns: ["id", "name"],
rows: rows(),
});
sink.end();
expect(result.rowCount).toBe(3);
const text = await collected;
expect(text.split("\n")).toEqual(['"id","name"', '"1","A"', '"2","B"', '"3","C"']);
});
it("neutralises spreadsheet-formula prefixes the same as the buffered API", async () => {
const sink = new PassThrough();
const collected = collectStream(sink);
await serializeExportTableToCsvStream(sink, {
columns: ["a"],
rows: [{ a: "=1+1" }],
});
sink.end();
expect(await collected).toBe(['"a"', "\"'=1+1\""].join("\n"));
});
it("rejects duplicate column keys eagerly", async () => {
const sink = new PassThrough();
await expect(
serializeExportTableToCsvStream(sink, {
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).rejects.toThrow('Export table columns contain duplicate key "id".');
});
it("keeps worker memory flat for 100k synthetic rows", async () => {
const sink = new PassThrough();
// Drain the sink as quickly as the producer writes so the PassThrough
// buffer cannot mask a buffering regression in the streamer.
sink.on("data", () => undefined);
async function* rows() {
for (let i = 0; i < 100_000; i += 1) {
yield { id: i, value: `row-${i}` };
}
}
if (global.gc) global.gc();
const before = process.memoryUsage().heapUsed;
const result = await serializeExportTableToCsvStream(sink, {
columns: ["id", "value"],
rows: rows(),
});
sink.end();
if (global.gc) global.gc();
const after = process.memoryUsage().heapUsed;
expect(result.rowCount).toBe(100_000);
// Hard upper bound is generous to absorb GC jitter; a buffering
// regression would inflate heap by tens of MB.
expect(after - before).toBeLessThan(100 * 1024 * 1024);
});
});