Reorganize repo structure and standards references

This commit is contained in:
david committed 2026-06-29 05:03:42 +02:00
1 parent 3cb93a81be
commit be2054274d
5885 files changed
+2588 -195

No files matched your search

@@ -0,0 +1,144 @@
import { describe, expect, it } from "vitest";
import { ExportTableValidationError } from "../types.js";
import {
buildJsonExportArtifact,
serializeExportTableToJson,
serializeExportTableToJsonLines,
} from "./index.js";
describe("@toolctrl/exports/json public entrypoint", () => {
it("exposes JSON-only runtime helpers", async () => {
const exportsModule = await import("./index.js");
expect(Object.keys(exportsModule).sort()).toEqual([
"ExportPrettyJsonTooLargeError",
"ExportTableValidationError",
"PRETTY_JSON_STREAM_ROW_LIMIT",
"buildJsonExportArtifact",
"serializeExportTableToJson",
"serializeExportTableToJsonLines",
"serializeExportTableToJsonStream",
]);
});
});
describe("serializeExportTableToJson", () => {
it("projects fields in explicit column order", () => {
const json = serializeExportTableToJson({
columns: ["email", "id"],
rows: [{ id: 7, name: "Ada", email: "ada@example.com" }],
});
expect(JSON.parse(json)).toEqual([{ email: "ada@example.com", id: 7 }]);
expect(json).toBe('[\n {\n "email": "ada@example.com",\n "id": 7\n }\n]');
});
it("emits pretty JSON by default and compact JSON when disabled", () => {
const table = {
columns: ["id", "active"],
rows: [{ id: 1, active: true }],
};
expect(serializeExportTableToJson(table)).toBe('[\n {\n "id": 1,\n "active": true\n }\n]');
expect(serializeExportTableToJson(table, { pretty: false })).toBe('[{"id":1,"active":true}]');
});
it("preserves selected null values and projects away omitted fields", () => {
const json = serializeExportTableToJson({
columns: ["id", "deletedAt", "missing"],
rows: [{ id: 1, deletedAt: null, ignored: "not exported", missing: undefined }],
});
expect(JSON.parse(json)).toEqual([{ id: 1, deletedAt: null, missing: null }]);
expect(json).not.toContain("ignored");
});
it("serializes dates as deterministic ISO strings and bigints as JSON-safe strings", () => {
const json = serializeExportTableToJson({
columns: ["createdAt", "big"],
rows: [{ createdAt: new Date("2024-01-02T03:04:05.006Z"), big: 9007199254740993n }],
});
expect(JSON.parse(json)).toEqual([
{ createdAt: "2024-01-02T03:04:05.006Z", big: "9007199254740993" },
]);
});
it("rejects malformed row input through the JSON API", () => {
expect(() =>
serializeExportTableToJson({
rows: [{ id: 1 }, null],
} as never),
).toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
});
it("rejects duplicate columns through the JSON API", () => {
expect(() =>
serializeExportTableToJson({
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
});
});
describe("serializeExportTableToJsonLines", () => {
it("emits one compact JSON object per row without a trailing newline", () => {
const jsonl = serializeExportTableToJsonLines({
columns: ["id", "name"],
rows: [
{ id: 1, name: "Ada" },
{ id: 2, name: "Grace" },
],
});
expect(jsonl).toBe('{"id":1,"name":"Ada"}\n{"id":2,"name":"Grace"}');
expect(jsonl.endsWith("\n")).toBe(false);
});
it("supports CRLF separators for JSONL output", () => {
const jsonl = serializeExportTableToJsonLines(
{
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
},
{ eol: "\r\n" },
);
expect(jsonl).toBe('{"id":1}\r\n{"id":2}');
});
});
describe("buildJsonExportArtifact", () => {
it("returns JSON bytes with artifact metadata", () => {
const artifact = buildJsonExportArtifact({
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
});
expect(artifact).toEqual({
bytes: '[\n {\n "id": 1\n },\n {\n "id": 2\n }\n]',
contentType: "application/json; charset=utf-8",
extension: "json",
rowCount: 2,
});
});
it("returns JSONL bytes with artifact metadata", () => {
const artifact = buildJsonExportArtifact(
{
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
},
{ format: "jsonl" },
);
expect(artifact).toEqual({
bytes: '{"id":1}\n{"id":2}',
contentType: "application/x-ndjson; charset=utf-8",
extension: "jsonl",
rowCount: 2,
});
});
});
@@ -0,0 +1,203 @@
import type { Writable } from "node:stream";
import {
type ExportColumn,
type ExportPrimitive,
type ExportRecord,
type ExportTable,
normalizeExportTable,
} from "../types.js";
export { ExportTableValidationError } from "../types.js";
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
export type JsonExportOptions = {
pretty?: boolean;
eol?: "\n" | "\r\n";
};
export type JsonExportArtifactOptions = JsonExportOptions & {
format?: "json" | "jsonl";
};
export type JsonExportArtifact = {
bytes: string;
contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8";
extension: "json" | "jsonl";
rowCount: number;
};
type JsonExportValue = string | number | boolean | null;
type JsonExportRow = Record<string, JsonExportValue>;
/**
* Maximum row count permitted for the pretty-JSON streaming mode. Pretty
* mode requires building the full array in memory to emit a syntactically
* valid document; above this threshold callers must switch to NDJSON.
*/
export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000;
export class ExportPrettyJsonTooLargeError extends Error {
readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE";
constructor(rowCount: number) {
super(
`Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`,
);
this.name = "ExportPrettyJsonTooLargeError";
}
}
export type JsonStreamMode = "ndjson" | "pretty";
export type JsonStreamOptions = {
mode?: JsonStreamMode;
eol?: "\n" | "\r\n";
};
export type JsonStreamTable = {
columns?: readonly (string | ExportColumn)[];
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
};
export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string {
const rows = buildJsonRows(table);
return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0);
}
export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string {
const eol = options.eol ?? "\n";
return buildJsonRows(table)
.map((row) => JSON.stringify(row))
.join(eol);
}
/**
* Streaming JSON writer. NDJSON mode writes one JSON object per line and
* keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT}
* rows before flushing a single indented array; above the limit the writer
* throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON.
*/
export async function serializeExportTableToJsonStream(
writeStream: Writable,
table: JsonStreamTable,
options: JsonStreamOptions = {},
): Promise<{ rowCount: number; mode: JsonStreamMode }> {
const mode = options.mode ?? "ndjson";
const eol = options.eol ?? "\n";
const columnKeys = resolveJsonColumnKeys(table);
if (mode === "ndjson") {
let rowCount = 0;
for await (const row of toAsyncIterable(table.rows)) {
const projected = projectRow(row as ExportRecord, columnKeys);
const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`;
await writeChunk(writeStream, chunk);
rowCount += 1;
}
return { rowCount, mode };
}
const buffered: JsonExportRow[] = [];
for await (const row of toAsyncIterable(table.rows)) {
buffered.push(projectRow(row as ExportRecord, columnKeys));
if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) {
throw new ExportPrettyJsonTooLargeError(buffered.length);
}
}
await writeChunk(writeStream, JSON.stringify(buffered, null, 2));
return { rowCount: buffered.length, mode };
}
export function buildJsonExportArtifact(
table: ExportTable,
options: JsonExportArtifactOptions = {},
): JsonExportArtifact {
const normalizedTable = normalizeExportTable(table);
const format = options.format ?? "json";
const bytes =
format === "jsonl"
? serializeExportTableToJsonLines(table, options)
: serializeExportTableToJson(table, options);
return {
bytes,
contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8",
extension: format === "jsonl" ? "jsonl" : "json",
rowCount: normalizedTable.rowCount,
};
}
function buildJsonRows(table: ExportTable): JsonExportRow[] {
const normalizedTable = normalizeExportTable(table);
return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key)));
}
function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow {
return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow;
}
function normalizeJsonValue(value: ExportPrimitive): JsonExportValue {
if (value === null || value === undefined) {
return null;
}
if (value instanceof Date) {
return value.toISOString();
}
if (typeof value === "bigint") {
return value.toString();
}
return value;
}
function resolveJsonColumnKeys(table: JsonStreamTable): string[] {
if (table.columns === undefined) {
throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
}
const keys: string[] = [];
const seen = new Set<string>();
for (const column of table.columns) {
const key = typeof column === "string" ? column : column.key;
if (seen.has(key)) {
throw new Error(`Export table columns contain duplicate key "${key}".`);
}
seen.add(key);
keys.push(key);
}
return keys;
}
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
if (Symbol.asyncIterator in (source as object)) {
return source as AsyncIterable<T>;
}
return (async function* () {
for (const item of source as Iterable<T>) {
yield item;
}
})();
}
@@ -0,0 +1,103 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import {
ExportPrettyJsonTooLargeError,
PRETTY_JSON_STREAM_ROW_LIMIT,
serializeExportTableToJsonStream,
} from "./index.js";
async function collect(stream: PassThrough): Promise<string> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks).toString("utf8");
}
describe("serializeExportTableToJsonStream", () => {
it("emits one JSON object per line in ndjson mode (default)", async () => {
const sink = new PassThrough();
const collected = collect(sink);
const result = await serializeExportTableToJsonStream(sink, {
columns: ["id", "active"],
rows: [
{ id: 1, active: true },
{ id: 2, active: false },
],
});
sink.end();
expect(result).toEqual({ rowCount: 2, mode: "ndjson" });
expect(await collected).toBe('{"id":1,"active":true}\n{"id":2,"active":false}');
});
it("serialises Dates and BigInts predictably", async () => {
const sink = new PassThrough();
const collected = collect(sink);
await serializeExportTableToJsonStream(sink, {
columns: ["createdAt", "count"],
rows: [
{ createdAt: new Date("2024-01-02T03:04:05.006Z"), count: BigInt(42) },
],
});
sink.end();
expect(JSON.parse(await collected)).toEqual({
createdAt: "2024-01-02T03:04:05.006Z",
count: "42",
});
});
it("supports pretty mode for small tables", async () => {
const sink = new PassThrough();
const collected = collect(sink);
const result = await serializeExportTableToJsonStream(
sink,
{
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
},
{ mode: "pretty" },
);
sink.end();
expect(result).toEqual({ rowCount: 2, mode: "pretty" });
expect(JSON.parse(await collected)).toEqual([{ id: 1 }, { id: 2 }]);
});
it("rejects pretty mode above the 10k-row limit with a typed error", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
async function* rows() {
for (let i = 0; i <= PRETTY_JSON_STREAM_ROW_LIMIT; i += 1) {
yield { id: i };
}
}
await expect(
serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() }, { mode: "pretty" }),
).rejects.toBeInstanceOf(ExportPrettyJsonTooLargeError);
});
it("accepts async iterables (Prisma cursor friendly)", async () => {
const sink = new PassThrough();
const collected = collect(sink);
async function* rows() {
yield { id: 1 };
yield { id: 2 };
yield { id: 3 };
}
const result = await serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() });
sink.end();
expect(result.rowCount).toBe(3);
expect(await collected).toBe('{"id":1}\n{"id":2}\n{"id":3}');
});
});