import type { Writable } from "node:stream"; import { type ExportColumn, type ExportPrimitive, type ExportRecord, type ExportTable, normalizeExportTable, } from "../types.js"; export { ExportTableValidationError } from "../types.js"; export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js"; export type JsonExportOptions = { pretty?: boolean; eol?: "\n" | "\r\n"; }; export type JsonExportArtifactOptions = JsonExportOptions & { format?: "json" | "jsonl"; }; export type JsonExportArtifact = { bytes: string; contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8"; extension: "json" | "jsonl"; rowCount: number; }; type JsonExportValue = string | number | boolean | null; type JsonExportRow = Record; /** * Maximum row count permitted for the pretty-JSON streaming mode. Pretty * mode requires building the full array in memory to emit a syntactically * valid document; above this threshold callers must switch to NDJSON. */ export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000; export class ExportPrettyJsonTooLargeError extends Error { readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE"; constructor(rowCount: number) { super( `Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`, ); this.name = "ExportPrettyJsonTooLargeError"; } } export type JsonStreamMode = "ndjson" | "pretty"; export type JsonStreamOptions = { mode?: JsonStreamMode; eol?: "\n" | "\r\n"; }; export type JsonStreamTable = { columns?: readonly (string | ExportColumn)[]; rows: Iterable | AsyncIterable; }; export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string { const rows = buildJsonRows(table); return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0); } export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string { const eol = options.eol ?? "\n"; return buildJsonRows(table) .map((row) => JSON.stringify(row)) .join(eol); } /** * Streaming JSON writer. NDJSON mode writes one JSON object per line and * keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT} * rows before flushing a single indented array; above the limit the writer * throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON. */ export async function serializeExportTableToJsonStream( writeStream: Writable, table: JsonStreamTable, options: JsonStreamOptions = {}, ): Promise<{ rowCount: number; mode: JsonStreamMode }> { const mode = options.mode ?? "ndjson"; const eol = options.eol ?? "\n"; const columnKeys = resolveJsonColumnKeys(table); if (mode === "ndjson") { let rowCount = 0; for await (const row of toAsyncIterable(table.rows)) { const projected = projectRow(row as ExportRecord, columnKeys); const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`; await writeChunk(writeStream, chunk); rowCount += 1; } return { rowCount, mode }; } const buffered: JsonExportRow[] = []; for await (const row of toAsyncIterable(table.rows)) { buffered.push(projectRow(row as ExportRecord, columnKeys)); if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) { throw new ExportPrettyJsonTooLargeError(buffered.length); } } await writeChunk(writeStream, JSON.stringify(buffered, null, 2)); return { rowCount: buffered.length, mode }; } export function buildJsonExportArtifact( table: ExportTable, options: JsonExportArtifactOptions = {}, ): JsonExportArtifact { const normalizedTable = normalizeExportTable(table); const format = options.format ?? "json"; const bytes = format === "jsonl" ? serializeExportTableToJsonLines(table, options) : serializeExportTableToJson(table, options); return { bytes, contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8", extension: format === "jsonl" ? "jsonl" : "json", rowCount: normalizedTable.rowCount, }; } function buildJsonRows(table: ExportTable): JsonExportRow[] { const normalizedTable = normalizeExportTable(table); return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key))); } function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow { return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow; } function normalizeJsonValue(value: ExportPrimitive): JsonExportValue { if (value === null || value === undefined) { return null; } if (value instanceof Date) { return value.toISOString(); } if (typeof value === "bigint") { return value.toString(); } return value; } function resolveJsonColumnKeys(table: JsonStreamTable): string[] { if (table.columns === undefined) { throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source."); } const keys: string[] = []; const seen = new Set(); for (const column of table.columns) { const key = typeof column === "string" ? column : column.key; if (seen.has(key)) { throw new Error(`Export table columns contain duplicate key "${key}".`); } seen.add(key); keys.push(key); } return keys; } async function writeChunk(writeStream: Writable, chunk: string): Promise { if (!writeStream.write(chunk)) { await new Promise((resolve, reject) => { const onDrain = () => { cleanup(); resolve(); }; const onError = (error: Error) => { cleanup(); reject(error); }; const cleanup = () => { writeStream.off("drain", onDrain); writeStream.off("error", onError); }; writeStream.once("drain", onDrain); writeStream.once("error", onError); }); } } function toAsyncIterable(source: Iterable | AsyncIterable): AsyncIterable { if (Symbol.asyncIterator in (source as object)) { return source as AsyncIterable; } return (async function* () { for (const item of source as Iterable) { yield item; } })(); }