Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,144 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportTableValidationError } from "../types.js";
|
||||
import {
|
||||
buildJsonExportArtifact,
|
||||
serializeExportTableToJson,
|
||||
serializeExportTableToJsonLines,
|
||||
} from "./index.js";
|
||||
|
||||
describe("@toolctrl/exports/json public entrypoint", () => {
|
||||
it("exposes JSON-only runtime helpers", async () => {
|
||||
const exportsModule = await import("./index.js");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"ExportPrettyJsonTooLargeError",
|
||||
"ExportTableValidationError",
|
||||
"PRETTY_JSON_STREAM_ROW_LIMIT",
|
||||
"buildJsonExportArtifact",
|
||||
"serializeExportTableToJson",
|
||||
"serializeExportTableToJsonLines",
|
||||
"serializeExportTableToJsonStream",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToJson", () => {
|
||||
it("projects fields in explicit column order", () => {
|
||||
const json = serializeExportTableToJson({
|
||||
columns: ["email", "id"],
|
||||
rows: [{ id: 7, name: "Ada", email: "ada@example.com" }],
|
||||
});
|
||||
|
||||
expect(JSON.parse(json)).toEqual([{ email: "ada@example.com", id: 7 }]);
|
||||
expect(json).toBe('[\n {\n "email": "ada@example.com",\n "id": 7\n }\n]');
|
||||
});
|
||||
|
||||
it("emits pretty JSON by default and compact JSON when disabled", () => {
|
||||
const table = {
|
||||
columns: ["id", "active"],
|
||||
rows: [{ id: 1, active: true }],
|
||||
};
|
||||
|
||||
expect(serializeExportTableToJson(table)).toBe('[\n {\n "id": 1,\n "active": true\n }\n]');
|
||||
expect(serializeExportTableToJson(table, { pretty: false })).toBe('[{"id":1,"active":true}]');
|
||||
});
|
||||
|
||||
it("preserves selected null values and projects away omitted fields", () => {
|
||||
const json = serializeExportTableToJson({
|
||||
columns: ["id", "deletedAt", "missing"],
|
||||
rows: [{ id: 1, deletedAt: null, ignored: "not exported", missing: undefined }],
|
||||
});
|
||||
|
||||
expect(JSON.parse(json)).toEqual([{ id: 1, deletedAt: null, missing: null }]);
|
||||
expect(json).not.toContain("ignored");
|
||||
});
|
||||
|
||||
it("serializes dates as deterministic ISO strings and bigints as JSON-safe strings", () => {
|
||||
const json = serializeExportTableToJson({
|
||||
columns: ["createdAt", "big"],
|
||||
rows: [{ createdAt: new Date("2024-01-02T03:04:05.006Z"), big: 9007199254740993n }],
|
||||
});
|
||||
|
||||
expect(JSON.parse(json)).toEqual([
|
||||
{ createdAt: "2024-01-02T03:04:05.006Z", big: "9007199254740993" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects malformed row input through the JSON API", () => {
|
||||
expect(() =>
|
||||
serializeExportTableToJson({
|
||||
rows: [{ id: 1 }, null],
|
||||
} as never),
|
||||
).toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
|
||||
});
|
||||
|
||||
it("rejects duplicate columns through the JSON API", () => {
|
||||
expect(() =>
|
||||
serializeExportTableToJson({
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToJsonLines", () => {
|
||||
it("emits one compact JSON object per row without a trailing newline", () => {
|
||||
const jsonl = serializeExportTableToJsonLines({
|
||||
columns: ["id", "name"],
|
||||
rows: [
|
||||
{ id: 1, name: "Ada" },
|
||||
{ id: 2, name: "Grace" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(jsonl).toBe('{"id":1,"name":"Ada"}\n{"id":2,"name":"Grace"}');
|
||||
expect(jsonl.endsWith("\n")).toBe(false);
|
||||
});
|
||||
|
||||
it("supports CRLF separators for JSONL output", () => {
|
||||
const jsonl = serializeExportTableToJsonLines(
|
||||
{
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
},
|
||||
{ eol: "\r\n" },
|
||||
);
|
||||
|
||||
expect(jsonl).toBe('{"id":1}\r\n{"id":2}');
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildJsonExportArtifact", () => {
|
||||
it("returns JSON bytes with artifact metadata", () => {
|
||||
const artifact = buildJsonExportArtifact({
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
});
|
||||
|
||||
expect(artifact).toEqual({
|
||||
bytes: '[\n {\n "id": 1\n },\n {\n "id": 2\n }\n]',
|
||||
contentType: "application/json; charset=utf-8",
|
||||
extension: "json",
|
||||
rowCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns JSONL bytes with artifact metadata", () => {
|
||||
const artifact = buildJsonExportArtifact(
|
||||
{
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
},
|
||||
{ format: "jsonl" },
|
||||
);
|
||||
|
||||
expect(artifact).toEqual({
|
||||
bytes: '{"id":1}\n{"id":2}',
|
||||
contentType: "application/x-ndjson; charset=utf-8",
|
||||
extension: "jsonl",
|
||||
rowCount: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,203 @@
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import {
|
||||
type ExportColumn,
|
||||
type ExportPrimitive,
|
||||
type ExportRecord,
|
||||
type ExportTable,
|
||||
normalizeExportTable,
|
||||
} from "../types.js";
|
||||
|
||||
export { ExportTableValidationError } from "../types.js";
|
||||
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
|
||||
|
||||
export type JsonExportOptions = {
|
||||
pretty?: boolean;
|
||||
eol?: "\n" | "\r\n";
|
||||
};
|
||||
|
||||
export type JsonExportArtifactOptions = JsonExportOptions & {
|
||||
format?: "json" | "jsonl";
|
||||
};
|
||||
|
||||
export type JsonExportArtifact = {
|
||||
bytes: string;
|
||||
contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8";
|
||||
extension: "json" | "jsonl";
|
||||
rowCount: number;
|
||||
};
|
||||
|
||||
type JsonExportValue = string | number | boolean | null;
|
||||
type JsonExportRow = Record<string, JsonExportValue>;
|
||||
|
||||
/**
|
||||
* Maximum row count permitted for the pretty-JSON streaming mode. Pretty
|
||||
* mode requires building the full array in memory to emit a syntactically
|
||||
* valid document; above this threshold callers must switch to NDJSON.
|
||||
*/
|
||||
export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000;
|
||||
|
||||
export class ExportPrettyJsonTooLargeError extends Error {
|
||||
readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE";
|
||||
constructor(rowCount: number) {
|
||||
super(
|
||||
`Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`,
|
||||
);
|
||||
this.name = "ExportPrettyJsonTooLargeError";
|
||||
}
|
||||
}
|
||||
|
||||
export type JsonStreamMode = "ndjson" | "pretty";
|
||||
|
||||
export type JsonStreamOptions = {
|
||||
mode?: JsonStreamMode;
|
||||
eol?: "\n" | "\r\n";
|
||||
};
|
||||
|
||||
export type JsonStreamTable = {
|
||||
columns?: readonly (string | ExportColumn)[];
|
||||
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
|
||||
};
|
||||
|
||||
export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string {
|
||||
const rows = buildJsonRows(table);
|
||||
return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0);
|
||||
}
|
||||
|
||||
export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string {
|
||||
const eol = options.eol ?? "\n";
|
||||
return buildJsonRows(table)
|
||||
.map((row) => JSON.stringify(row))
|
||||
.join(eol);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming JSON writer. NDJSON mode writes one JSON object per line and
|
||||
* keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT}
|
||||
* rows before flushing a single indented array; above the limit the writer
|
||||
* throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON.
|
||||
*/
|
||||
export async function serializeExportTableToJsonStream(
|
||||
writeStream: Writable,
|
||||
table: JsonStreamTable,
|
||||
options: JsonStreamOptions = {},
|
||||
): Promise<{ rowCount: number; mode: JsonStreamMode }> {
|
||||
const mode = options.mode ?? "ndjson";
|
||||
const eol = options.eol ?? "\n";
|
||||
const columnKeys = resolveJsonColumnKeys(table);
|
||||
|
||||
if (mode === "ndjson") {
|
||||
let rowCount = 0;
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
const projected = projectRow(row as ExportRecord, columnKeys);
|
||||
const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`;
|
||||
await writeChunk(writeStream, chunk);
|
||||
rowCount += 1;
|
||||
}
|
||||
return { rowCount, mode };
|
||||
}
|
||||
|
||||
const buffered: JsonExportRow[] = [];
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
buffered.push(projectRow(row as ExportRecord, columnKeys));
|
||||
if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) {
|
||||
throw new ExportPrettyJsonTooLargeError(buffered.length);
|
||||
}
|
||||
}
|
||||
await writeChunk(writeStream, JSON.stringify(buffered, null, 2));
|
||||
return { rowCount: buffered.length, mode };
|
||||
}
|
||||
|
||||
export function buildJsonExportArtifact(
|
||||
table: ExportTable,
|
||||
options: JsonExportArtifactOptions = {},
|
||||
): JsonExportArtifact {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const format = options.format ?? "json";
|
||||
const bytes =
|
||||
format === "jsonl"
|
||||
? serializeExportTableToJsonLines(table, options)
|
||||
: serializeExportTableToJson(table, options);
|
||||
|
||||
return {
|
||||
bytes,
|
||||
contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8",
|
||||
extension: format === "jsonl" ? "jsonl" : "json",
|
||||
rowCount: normalizedTable.rowCount,
|
||||
};
|
||||
}
|
||||
|
||||
function buildJsonRows(table: ExportTable): JsonExportRow[] {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
|
||||
return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key)));
|
||||
}
|
||||
|
||||
function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow {
|
||||
return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow;
|
||||
}
|
||||
|
||||
function normalizeJsonValue(value: ExportPrimitive): JsonExportValue {
|
||||
if (value === null || value === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
if (typeof value === "bigint") {
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function resolveJsonColumnKeys(table: JsonStreamTable): string[] {
|
||||
if (table.columns === undefined) {
|
||||
throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
|
||||
}
|
||||
const keys: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const column of table.columns) {
|
||||
const key = typeof column === "string" ? column : column.key;
|
||||
if (seen.has(key)) {
|
||||
throw new Error(`Export table columns contain duplicate key "${key}".`);
|
||||
}
|
||||
seen.add(key);
|
||||
keys.push(key);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
|
||||
if (!writeStream.write(chunk)) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
writeStream.off("drain", onDrain);
|
||||
writeStream.off("error", onError);
|
||||
};
|
||||
writeStream.once("drain", onDrain);
|
||||
writeStream.once("error", onError);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
|
||||
if (Symbol.asyncIterator in (source as object)) {
|
||||
return source as AsyncIterable<T>;
|
||||
}
|
||||
return (async function* () {
|
||||
for (const item of source as Iterable<T>) {
|
||||
yield item;
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
ExportPrettyJsonTooLargeError,
|
||||
PRETTY_JSON_STREAM_ROW_LIMIT,
|
||||
serializeExportTableToJsonStream,
|
||||
} from "./index.js";
|
||||
|
||||
async function collect(stream: PassThrough): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
}
|
||||
|
||||
describe("serializeExportTableToJsonStream", () => {
|
||||
it("emits one JSON object per line in ndjson mode (default)", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
const result = await serializeExportTableToJsonStream(sink, {
|
||||
columns: ["id", "active"],
|
||||
rows: [
|
||||
{ id: 1, active: true },
|
||||
{ id: 2, active: false },
|
||||
],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(result).toEqual({ rowCount: 2, mode: "ndjson" });
|
||||
expect(await collected).toBe('{"id":1,"active":true}\n{"id":2,"active":false}');
|
||||
});
|
||||
|
||||
it("serialises Dates and BigInts predictably", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
await serializeExportTableToJsonStream(sink, {
|
||||
columns: ["createdAt", "count"],
|
||||
rows: [
|
||||
{ createdAt: new Date("2024-01-02T03:04:05.006Z"), count: BigInt(42) },
|
||||
],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(JSON.parse(await collected)).toEqual({
|
||||
createdAt: "2024-01-02T03:04:05.006Z",
|
||||
count: "42",
|
||||
});
|
||||
});
|
||||
|
||||
it("supports pretty mode for small tables", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
const result = await serializeExportTableToJsonStream(
|
||||
sink,
|
||||
{
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
},
|
||||
{ mode: "pretty" },
|
||||
);
|
||||
sink.end();
|
||||
|
||||
expect(result).toEqual({ rowCount: 2, mode: "pretty" });
|
||||
expect(JSON.parse(await collected)).toEqual([{ id: 1 }, { id: 2 }]);
|
||||
});
|
||||
|
||||
it("rejects pretty mode above the 10k-row limit with a typed error", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
async function* rows() {
|
||||
for (let i = 0; i <= PRETTY_JSON_STREAM_ROW_LIMIT; i += 1) {
|
||||
yield { id: i };
|
||||
}
|
||||
}
|
||||
|
||||
await expect(
|
||||
serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() }, { mode: "pretty" }),
|
||||
).rejects.toBeInstanceOf(ExportPrettyJsonTooLargeError);
|
||||
});
|
||||
|
||||
it("accepts async iterables (Prisma cursor friendly)", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
async function* rows() {
|
||||
yield { id: 1 };
|
||||
yield { id: 2 };
|
||||
yield { id: 3 };
|
||||
}
|
||||
|
||||
const result = await serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() });
|
||||
sink.end();
|
||||
|
||||
expect(result.rowCount).toBe(3);
|
||||
expect(await collected).toBe('{"id":1}\n{"id":2}\n{"id":3}');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user