Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/packages/exports/src/json/index.ts
T

204 lines
6.2 KiB
TypeScript

import type { Writable } from "node:stream";
import {
type ExportColumn,
type ExportPrimitive,
type ExportRecord,
type ExportTable,
normalizeExportTable,
} from "../types.js";
export { ExportTableValidationError } from "../types.js";
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
export type JsonExportOptions = {
pretty?: boolean;
eol?: "\n" | "\r\n";
};
export type JsonExportArtifactOptions = JsonExportOptions & {
format?: "json" | "jsonl";
};
export type JsonExportArtifact = {
bytes: string;
contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8";
extension: "json" | "jsonl";
rowCount: number;
};
type JsonExportValue = string | number | boolean | null;
type JsonExportRow = Record<string, JsonExportValue>;
/**
* Maximum row count permitted for the pretty-JSON streaming mode. Pretty
* mode requires building the full array in memory to emit a syntactically
* valid document; above this threshold callers must switch to NDJSON.
*/
export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000;
export class ExportPrettyJsonTooLargeError extends Error {
readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE";
constructor(rowCount: number) {
super(
`Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`,
);
this.name = "ExportPrettyJsonTooLargeError";
}
}
export type JsonStreamMode = "ndjson" | "pretty";
export type JsonStreamOptions = {
mode?: JsonStreamMode;
eol?: "\n" | "\r\n";
};
export type JsonStreamTable = {
columns?: readonly (string | ExportColumn)[];
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
};
export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string {
const rows = buildJsonRows(table);
return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0);
}
export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string {
const eol = options.eol ?? "\n";
return buildJsonRows(table)
.map((row) => JSON.stringify(row))
.join(eol);
}
/**
* Streaming JSON writer. NDJSON mode writes one JSON object per line and
* keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT}
* rows before flushing a single indented array; above the limit the writer
* throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON.
*/
export async function serializeExportTableToJsonStream(
writeStream: Writable,
table: JsonStreamTable,
options: JsonStreamOptions = {},
): Promise<{ rowCount: number; mode: JsonStreamMode }> {
const mode = options.mode ?? "ndjson";
const eol = options.eol ?? "\n";
const columnKeys = resolveJsonColumnKeys(table);
if (mode === "ndjson") {
let rowCount = 0;
for await (const row of toAsyncIterable(table.rows)) {
const projected = projectRow(row as ExportRecord, columnKeys);
const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`;
await writeChunk(writeStream, chunk);
rowCount += 1;
}
return { rowCount, mode };
}
const buffered: JsonExportRow[] = [];
for await (const row of toAsyncIterable(table.rows)) {
buffered.push(projectRow(row as ExportRecord, columnKeys));
if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) {
throw new ExportPrettyJsonTooLargeError(buffered.length);
}
}
await writeChunk(writeStream, JSON.stringify(buffered, null, 2));
return { rowCount: buffered.length, mode };
}
export function buildJsonExportArtifact(
table: ExportTable,
options: JsonExportArtifactOptions = {},
): JsonExportArtifact {
const normalizedTable = normalizeExportTable(table);
const format = options.format ?? "json";
const bytes =
format === "jsonl"
? serializeExportTableToJsonLines(table, options)
: serializeExportTableToJson(table, options);
return {
bytes,
contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8",
extension: format === "jsonl" ? "jsonl" : "json",
rowCount: normalizedTable.rowCount,
};
}
function buildJsonRows(table: ExportTable): JsonExportRow[] {
const normalizedTable = normalizeExportTable(table);
return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key)));
}
function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow {
return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow;
}
function normalizeJsonValue(value: ExportPrimitive): JsonExportValue {
if (value === null || value === undefined) {
return null;
}
if (value instanceof Date) {
return value.toISOString();
}
if (typeof value === "bigint") {
return value.toString();
}
return value;
}
function resolveJsonColumnKeys(table: JsonStreamTable): string[] {
if (table.columns === undefined) {
throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
}
const keys: string[] = [];
const seen = new Set<string>();
for (const column of table.columns) {
const key = typeof column === "string" ? column : column.key;
if (seen.has(key)) {
throw new Error(`Export table columns contain duplicate key "${key}".`);
}
seen.add(key);
keys.push(key);
}
return keys;
}
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
if (Symbol.asyncIterator in (source as object)) {
return source as AsyncIterable<T>;
}
return (async function* () {
for (const item of source as Iterable<T>) {
yield item;
}
})();
}