Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,203 @@
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import {
|
||||
type ExportColumn,
|
||||
type ExportPrimitive,
|
||||
type ExportRecord,
|
||||
type ExportTable,
|
||||
normalizeExportTable,
|
||||
} from "../types.js";
|
||||
|
||||
export { ExportTableValidationError } from "../types.js";
|
||||
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
|
||||
|
||||
export type JsonExportOptions = {
|
||||
pretty?: boolean;
|
||||
eol?: "\n" | "\r\n";
|
||||
};
|
||||
|
||||
export type JsonExportArtifactOptions = JsonExportOptions & {
|
||||
format?: "json" | "jsonl";
|
||||
};
|
||||
|
||||
export type JsonExportArtifact = {
|
||||
bytes: string;
|
||||
contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8";
|
||||
extension: "json" | "jsonl";
|
||||
rowCount: number;
|
||||
};
|
||||
|
||||
type JsonExportValue = string | number | boolean | null;
|
||||
type JsonExportRow = Record<string, JsonExportValue>;
|
||||
|
||||
/**
|
||||
* Maximum row count permitted for the pretty-JSON streaming mode. Pretty
|
||||
* mode requires building the full array in memory to emit a syntactically
|
||||
* valid document; above this threshold callers must switch to NDJSON.
|
||||
*/
|
||||
export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000;
|
||||
|
||||
export class ExportPrettyJsonTooLargeError extends Error {
|
||||
readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE";
|
||||
constructor(rowCount: number) {
|
||||
super(
|
||||
`Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`,
|
||||
);
|
||||
this.name = "ExportPrettyJsonTooLargeError";
|
||||
}
|
||||
}
|
||||
|
||||
export type JsonStreamMode = "ndjson" | "pretty";
|
||||
|
||||
export type JsonStreamOptions = {
|
||||
mode?: JsonStreamMode;
|
||||
eol?: "\n" | "\r\n";
|
||||
};
|
||||
|
||||
export type JsonStreamTable = {
|
||||
columns?: readonly (string | ExportColumn)[];
|
||||
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
|
||||
};
|
||||
|
||||
export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string {
|
||||
const rows = buildJsonRows(table);
|
||||
return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0);
|
||||
}
|
||||
|
||||
export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string {
|
||||
const eol = options.eol ?? "\n";
|
||||
return buildJsonRows(table)
|
||||
.map((row) => JSON.stringify(row))
|
||||
.join(eol);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming JSON writer. NDJSON mode writes one JSON object per line and
|
||||
* keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT}
|
||||
* rows before flushing a single indented array; above the limit the writer
|
||||
* throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON.
|
||||
*/
|
||||
export async function serializeExportTableToJsonStream(
|
||||
writeStream: Writable,
|
||||
table: JsonStreamTable,
|
||||
options: JsonStreamOptions = {},
|
||||
): Promise<{ rowCount: number; mode: JsonStreamMode }> {
|
||||
const mode = options.mode ?? "ndjson";
|
||||
const eol = options.eol ?? "\n";
|
||||
const columnKeys = resolveJsonColumnKeys(table);
|
||||
|
||||
if (mode === "ndjson") {
|
||||
let rowCount = 0;
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
const projected = projectRow(row as ExportRecord, columnKeys);
|
||||
const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`;
|
||||
await writeChunk(writeStream, chunk);
|
||||
rowCount += 1;
|
||||
}
|
||||
return { rowCount, mode };
|
||||
}
|
||||
|
||||
const buffered: JsonExportRow[] = [];
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
buffered.push(projectRow(row as ExportRecord, columnKeys));
|
||||
if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) {
|
||||
throw new ExportPrettyJsonTooLargeError(buffered.length);
|
||||
}
|
||||
}
|
||||
await writeChunk(writeStream, JSON.stringify(buffered, null, 2));
|
||||
return { rowCount: buffered.length, mode };
|
||||
}
|
||||
|
||||
export function buildJsonExportArtifact(
|
||||
table: ExportTable,
|
||||
options: JsonExportArtifactOptions = {},
|
||||
): JsonExportArtifact {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const format = options.format ?? "json";
|
||||
const bytes =
|
||||
format === "jsonl"
|
||||
? serializeExportTableToJsonLines(table, options)
|
||||
: serializeExportTableToJson(table, options);
|
||||
|
||||
return {
|
||||
bytes,
|
||||
contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8",
|
||||
extension: format === "jsonl" ? "jsonl" : "json",
|
||||
rowCount: normalizedTable.rowCount,
|
||||
};
|
||||
}
|
||||
|
||||
function buildJsonRows(table: ExportTable): JsonExportRow[] {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
|
||||
return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key)));
|
||||
}
|
||||
|
||||
function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow {
|
||||
return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow;
|
||||
}
|
||||
|
||||
function normalizeJsonValue(value: ExportPrimitive): JsonExportValue {
|
||||
if (value === null || value === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
if (typeof value === "bigint") {
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function resolveJsonColumnKeys(table: JsonStreamTable): string[] {
|
||||
if (table.columns === undefined) {
|
||||
throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
|
||||
}
|
||||
const keys: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const column of table.columns) {
|
||||
const key = typeof column === "string" ? column : column.key;
|
||||
if (seen.has(key)) {
|
||||
throw new Error(`Export table columns contain duplicate key "${key}".`);
|
||||
}
|
||||
seen.add(key);
|
||||
keys.push(key);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
|
||||
if (!writeStream.write(chunk)) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
writeStream.off("drain", onDrain);
|
||||
writeStream.off("error", onError);
|
||||
};
|
||||
writeStream.once("drain", onDrain);
|
||||
writeStream.once("error", onError);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
|
||||
if (Symbol.asyncIterator in (source as object)) {
|
||||
return source as AsyncIterable<T>;
|
||||
}
|
||||
return (async function* () {
|
||||
for (const item of source as Iterable<T>) {
|
||||
yield item;
|
||||
}
|
||||
})();
|
||||
}
|
||||
Reference in new issue
Block a user