204 lines
6.2 KiB
TypeScript
204 lines
6.2 KiB
TypeScript
import type { Writable } from "node:stream";
|
|
|
|
import {
|
|
type ExportColumn,
|
|
type ExportPrimitive,
|
|
type ExportRecord,
|
|
type ExportTable,
|
|
normalizeExportTable,
|
|
} from "../types.js";
|
|
|
|
export { ExportTableValidationError } from "../types.js";
|
|
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
|
|
|
|
export type JsonExportOptions = {
|
|
pretty?: boolean;
|
|
eol?: "\n" | "\r\n";
|
|
};
|
|
|
|
export type JsonExportArtifactOptions = JsonExportOptions & {
|
|
format?: "json" | "jsonl";
|
|
};
|
|
|
|
export type JsonExportArtifact = {
|
|
bytes: string;
|
|
contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8";
|
|
extension: "json" | "jsonl";
|
|
rowCount: number;
|
|
};
|
|
|
|
type JsonExportValue = string | number | boolean | null;
|
|
type JsonExportRow = Record<string, JsonExportValue>;
|
|
|
|
/**
|
|
* Maximum row count permitted for the pretty-JSON streaming mode. Pretty
|
|
* mode requires building the full array in memory to emit a syntactically
|
|
* valid document; above this threshold callers must switch to NDJSON.
|
|
*/
|
|
export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000;
|
|
|
|
export class ExportPrettyJsonTooLargeError extends Error {
|
|
readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE";
|
|
constructor(rowCount: number) {
|
|
super(
|
|
`Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`,
|
|
);
|
|
this.name = "ExportPrettyJsonTooLargeError";
|
|
}
|
|
}
|
|
|
|
export type JsonStreamMode = "ndjson" | "pretty";
|
|
|
|
export type JsonStreamOptions = {
|
|
mode?: JsonStreamMode;
|
|
eol?: "\n" | "\r\n";
|
|
};
|
|
|
|
export type JsonStreamTable = {
|
|
columns?: readonly (string | ExportColumn)[];
|
|
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
|
|
};
|
|
|
|
export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string {
|
|
const rows = buildJsonRows(table);
|
|
return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0);
|
|
}
|
|
|
|
export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string {
|
|
const eol = options.eol ?? "\n";
|
|
return buildJsonRows(table)
|
|
.map((row) => JSON.stringify(row))
|
|
.join(eol);
|
|
}
|
|
|
|
/**
|
|
* Streaming JSON writer. NDJSON mode writes one JSON object per line and
|
|
* keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT}
|
|
* rows before flushing a single indented array; above the limit the writer
|
|
* throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON.
|
|
*/
|
|
export async function serializeExportTableToJsonStream(
|
|
writeStream: Writable,
|
|
table: JsonStreamTable,
|
|
options: JsonStreamOptions = {},
|
|
): Promise<{ rowCount: number; mode: JsonStreamMode }> {
|
|
const mode = options.mode ?? "ndjson";
|
|
const eol = options.eol ?? "\n";
|
|
const columnKeys = resolveJsonColumnKeys(table);
|
|
|
|
if (mode === "ndjson") {
|
|
let rowCount = 0;
|
|
for await (const row of toAsyncIterable(table.rows)) {
|
|
const projected = projectRow(row as ExportRecord, columnKeys);
|
|
const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`;
|
|
await writeChunk(writeStream, chunk);
|
|
rowCount += 1;
|
|
}
|
|
return { rowCount, mode };
|
|
}
|
|
|
|
const buffered: JsonExportRow[] = [];
|
|
for await (const row of toAsyncIterable(table.rows)) {
|
|
buffered.push(projectRow(row as ExportRecord, columnKeys));
|
|
if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) {
|
|
throw new ExportPrettyJsonTooLargeError(buffered.length);
|
|
}
|
|
}
|
|
await writeChunk(writeStream, JSON.stringify(buffered, null, 2));
|
|
return { rowCount: buffered.length, mode };
|
|
}
|
|
|
|
export function buildJsonExportArtifact(
|
|
table: ExportTable,
|
|
options: JsonExportArtifactOptions = {},
|
|
): JsonExportArtifact {
|
|
const normalizedTable = normalizeExportTable(table);
|
|
const format = options.format ?? "json";
|
|
const bytes =
|
|
format === "jsonl"
|
|
? serializeExportTableToJsonLines(table, options)
|
|
: serializeExportTableToJson(table, options);
|
|
|
|
return {
|
|
bytes,
|
|
contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8",
|
|
extension: format === "jsonl" ? "jsonl" : "json",
|
|
rowCount: normalizedTable.rowCount,
|
|
};
|
|
}
|
|
|
|
function buildJsonRows(table: ExportTable): JsonExportRow[] {
|
|
const normalizedTable = normalizeExportTable(table);
|
|
|
|
return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key)));
|
|
}
|
|
|
|
function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow {
|
|
return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow;
|
|
}
|
|
|
|
function normalizeJsonValue(value: ExportPrimitive): JsonExportValue {
|
|
if (value === null || value === undefined) {
|
|
return null;
|
|
}
|
|
|
|
if (value instanceof Date) {
|
|
return value.toISOString();
|
|
}
|
|
|
|
if (typeof value === "bigint") {
|
|
return value.toString();
|
|
}
|
|
|
|
return value;
|
|
}
|
|
|
|
function resolveJsonColumnKeys(table: JsonStreamTable): string[] {
|
|
if (table.columns === undefined) {
|
|
throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
|
|
}
|
|
const keys: string[] = [];
|
|
const seen = new Set<string>();
|
|
for (const column of table.columns) {
|
|
const key = typeof column === "string" ? column : column.key;
|
|
if (seen.has(key)) {
|
|
throw new Error(`Export table columns contain duplicate key "${key}".`);
|
|
}
|
|
seen.add(key);
|
|
keys.push(key);
|
|
}
|
|
return keys;
|
|
}
|
|
|
|
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
|
|
if (!writeStream.write(chunk)) {
|
|
await new Promise<void>((resolve, reject) => {
|
|
const onDrain = () => {
|
|
cleanup();
|
|
resolve();
|
|
};
|
|
const onError = (error: Error) => {
|
|
cleanup();
|
|
reject(error);
|
|
};
|
|
const cleanup = () => {
|
|
writeStream.off("drain", onDrain);
|
|
writeStream.off("error", onError);
|
|
};
|
|
writeStream.once("drain", onDrain);
|
|
writeStream.once("error", onError);
|
|
});
|
|
}
|
|
}
|
|
|
|
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
|
|
if (Symbol.asyncIterator in (source as object)) {
|
|
return source as AsyncIterable<T>;
|
|
}
|
|
return (async function* () {
|
|
for (const item of source as Iterable<T>) {
|
|
yield item;
|
|
}
|
|
})();
|
|
}
|