Track HiveGrid Cloud planning reference

This commit is contained in:
david committed 2026-06-26 20:48:01 +02:00
1 parent b3d5ea6764
commit e341c5938a
5784 files changed
+1104929 -3

No files matched your search

@@ -0,0 +1,30 @@
{
"name": "@toolctrl/exports",
"version": "0.0.0",
"private": true,
"type": "module",
"exports": {
".": "./src/index.ts",
"./audit-pdf": "./src/audit-pdf.ts",
"./csv": "./src/csv/index.ts",
"./json": "./src/json/index.ts",
"./limits": "./src/limits.ts",
"./pdf": "./src/pdf/index.ts",
"./s3-multipart": "./src/sinks/s3-multipart.ts",
"./types": "./src/types.ts",
"./xlsx": "./src/xlsx/index.ts"
},
"scripts": {
"test": "vitest run src",
"typecheck": "tsc -p tsconfig.json --noEmit"
},
"dependencies": {
"@aws-sdk/client-s3": "3.1045.0",
"@aws-sdk/lib-storage": "3.1045.0",
"@react-pdf/renderer": "^4.3.2",
"exceljs": "^4.4.0"
},
"devDependencies": {
"aws-sdk-client-mock": "^4.1.0"
}
}
@@ -0,0 +1,130 @@
import { describe, expect, it } from "vitest";
import {
AUDIT_PDF_HARD_ROW_LIMIT,
buildAuditPdfDocument,
serializeAuditEventsToPdf,
summarizeAuditPdfFilters,
type AuditPdfEvent,
type AuditPdfReportInput,
} from "./audit-pdf.js";
import { ExportPdfRowLimitExceededError } from "./pdf/index.js";
function makeEvent(over: Partial<AuditPdfEvent> = {}): AuditPdfEvent {
return {
id: "audit_1",
createdAt: new Date("2026-05-17T00:00:00.000Z"),
tenantId: "tenant_1",
actorType: "user",
actorId: "user_1",
actorEmail: null,
actorIp: null,
actorUserAgent: null,
action: "widget.created",
targetType: "widget",
targetId: "widget_1",
requestId: null,
sourceModule: "crm",
outcome: "success",
dataSensitivity: "internal",
regulatoryScope: ["soc2"],
metadata: null,
previousHash: null,
eventHash: "deadbeef0123456789abcdef0123456789abcdef0123456789abcdef01234567",
...over,
};
}
function makeInput(over: Partial<AuditPdfReportInput> = {}): AuditPdfReportInput {
return {
tenantId: "tenant_1",
from: "2026-05-01",
to: "2026-05-17",
filterSummary: "regulatoryScope=soc2",
integrity: { ok: true, inspected: 1, firstBreakId: null, reason: null },
events: [makeEvent()],
generatedAt: new Date("2026-05-17T12:34:56.000Z"),
generatedBy: "ops-bot",
...over,
};
}
function collectText(node: unknown): string {
if (typeof node === "string" || typeof node === "number") {
return String(node);
}
if (!node || typeof node !== "object") {
return "";
}
const props = (node as { props?: { children?: unknown } }).props;
const children = props?.children;
if (Array.isArray(children)) {
return children.map(collectText).join(" ");
}
return collectText(children);
}
describe("summarizeAuditPdfFilters", () => {
it("joins regulatoryScope, actorType and outcome", () => {
expect(summarizeAuditPdfFilters({ regulatoryScope: ["soc2", "gdpr"], actorType: "admin", outcome: "blocked" })).toBe(
"regulatoryScope=soc2|gdpr actorType=admin outcome=blocked"
);
});
it("returns empty when nothing is set", () => {
expect(summarizeAuditPdfFilters({})).toBe("");
});
});
describe("buildAuditPdfDocument", () => {
it("returns a React element with the expected title", () => {
const doc = buildAuditPdfDocument(makeInput({ title: "Custom title" }));
expect(doc).toBeTruthy();
expect((doc as unknown as { props: { title: string } }).props.title).toBe("Custom title");
});
it("accepts localized labels for visible report copy", () => {
const doc = buildAuditPdfDocument(makeInput({
labels: {
title: "Audit-Compliance-Bericht",
tenant: "Konto",
platformTenant: "Plattform",
range: "Zeitraum",
filters: "Filter",
generatedBy: "Erstellt von",
generatedAt: "Erstellt am",
hashChainIntegrity: "Hash-Chain-Integritaet",
events: (count) => `Ereignisse (${count})`,
integrityOk: (inspected) => `verifyAuditChain OK - ${inspected} Ereignisse geprueft.`,
columns: {
actorType: "Akteur",
outcome: "Ergebnis",
regulatoryScope: "Geltungsbereich",
},
},
}));
const text = collectText(doc);
expect((doc as unknown as { props: { title: string } }).props.title).toBe("Audit-Compliance-Bericht");
expect(text).toContain("Konto: tenant_1");
expect(text).toContain("Hash-Chain-Integritaet");
expect(text).toContain("Ereignisse (1)");
expect(text).toContain("Akteur");
expect(text).toContain("Ergebnis");
});
});
describe("serializeAuditEventsToPdf", () => {
it("renders a real PDF byte stream", async () => {
const bytes = await serializeAuditEventsToPdf(makeInput());
expect(bytes).toBeInstanceOf(Uint8Array);
expect(bytes.length).toBeGreaterThan(0);
// PDFs start with %PDF
const header = Buffer.from(bytes.slice(0, 4)).toString("utf8");
expect(header).toBe("%PDF");
});
it("throws when event count exceeds the hard cap", async () => {
const tooMany: AuditPdfEvent[] = Array.from({ length: AUDIT_PDF_HARD_ROW_LIMIT + 1 }, (_, i) => makeEvent({ id: `e_${i}` }));
await expect(serializeAuditEventsToPdf(makeInput({ events: tooMany }))).rejects.toBeInstanceOf(ExportPdfRowLimitExceededError);
});
});
@@ -0,0 +1,285 @@
/**
* Compliance-Audit PDF report renderer.
*
* Renders an AuditEvent slice into a tamper-evident PDF report suitable
* for SOC2 / HIPAA / GDPR auditor evidence packs. The first page is a
* cover page with tenant + range + filter summary + verifyAuditChain
* integrity outcome; following pages list events as a tabular log.
*
* The renderer is host-agnostic and async — both the operator CLI
* (`toolctrl audit export-pdf …`) and the admin web export route use
* it via `serializeAuditEventsToPdf` / `serializeAuditEventsToPdfStream`.
*
* See docs/improvement-plan/10-compliance-polish/compliance-audit-export.md.
*/
import type { Writable } from "node:stream";
import React from "react";
import { Document, Page, StyleSheet, Text, View, renderToBuffer, renderToStream } from "@react-pdf/renderer";
import { PDF_STREAM_HARD_ROW_LIMIT, ExportPdfRowLimitExceededError } from "./pdf/index.js";
export const AUDIT_PDF_CONTENT_TYPE = "application/pdf" as const;
export const AUDIT_PDF_HARD_ROW_LIMIT = PDF_STREAM_HARD_ROW_LIMIT;
export type AuditPdfEvent = {
id: string;
createdAt: Date | string;
tenantId: string | null;
actorType: string;
actorId: string | null;
actorEmail: string | null;
actorIp: string | null;
actorUserAgent: string | null;
action: string;
targetType: string | null;
targetId: string | null;
requestId: string | null;
sourceModule: string | null;
outcome: string;
dataSensitivity: string;
regulatoryScope: ReadonlyArray<string>;
metadata: unknown;
previousHash: string | null;
eventHash: string | null;
};
export type AuditPdfIntegrity = {
ok: boolean;
inspected: number;
firstBreakId: string | null;
reason: string | null;
};
export type AuditPdfReportInput = {
/** Tenant identifier; null = platform-wide. */
tenantId: string | null;
/** ISO-8601 lower bound used by the export query. */
from: string | null;
/** ISO-8601 upper bound used by the export query. */
to: string | null;
/** Free-text filter summary (regulatoryScope/actorType/outcome). */
filterSummary: string;
/** Pre-computed verifyAuditChain integrity result. */
integrity: AuditPdfIntegrity;
/** Events in chronological order. */
events: ReadonlyArray<AuditPdfEvent>;
/** Operator-facing report title (defaults to "Audit-Compliance Report"). */
title?: string;
/** Human display name of the operator who generated the report. */
generatedBy?: string;
/** Timestamp of report generation; defaults to new Date(). */
generatedAt?: Date;
/** Visible report labels; pass localized labels from the caller when needed. */
labels?: AuditPdfLabelOverrides;
};
type AuditPdfColumnKey = "createdAt" | "actorType" | "actorId" | "action" | "targetType" | "outcome" | "regulatoryScope" | "eventHash";
export type AuditPdfLabels = {
title: string;
tenant: string;
platformTenant: string;
range: string;
openRange: string;
filters: string;
noFilters: string;
generatedBy: string;
generatedAt: string;
hashChainIntegrity: string;
events: (count: number) => string;
integrityOk: (inspected: number) => string;
integrityBroken: (args: { inspected: number; firstBreakId: string; reason: string }) => string;
columns: Record<AuditPdfColumnKey, string>;
};
export type AuditPdfLabelOverrides = Partial<Omit<AuditPdfLabels, "columns">> & {
columns?: Partial<AuditPdfLabels["columns"]>;
};
const labelText = (value: string) => value;
const DEFAULT_AUDIT_PDF_LABELS: AuditPdfLabels = {
title: labelText("Audit compliance report"),
tenant: "Tenant",
platformTenant: "platform",
range: "Range",
openRange: "open",
filters: "Filters",
noFilters: "none",
generatedBy: "Generated by",
generatedAt: "Generated at",
hashChainIntegrity: "Hash-chain integrity",
events: (count) => `Events (${count})`,
integrityOk: (inspected) => `verifyAuditChain OK - inspected ${inspected} events.`,
integrityBroken: ({ inspected, firstBreakId, reason }) => `verifyAuditChain BROKEN - inspected ${inspected} events. First-break id=${firstBreakId} reason=${reason}.`,
columns: {
createdAt: "Time",
actorType: "Actor",
actorId: "Actor ID",
action: "Action",
targetType: "Target",
outcome: "Outcome",
regulatoryScope: "Scope",
eventHash: "Hash",
},
};
const styles = StyleSheet.create({
page: { padding: 32, fontSize: 9, fontFamily: "Helvetica" },
title: { fontSize: 18, fontWeight: 700, marginBottom: 12 },
subtitle: { fontSize: 12, fontWeight: 700, marginTop: 12, marginBottom: 6 },
meta: { marginBottom: 4 },
integrityOk: { color: "#047857", fontWeight: 700 },
integrityBroken: { color: "#b91c1c", fontWeight: 700 },
table: {
display: "flex",
flexDirection: "column",
width: "100%",
borderTopWidth: 1,
borderLeftWidth: 1,
borderColor: "#d1d5db",
marginTop: 8,
},
row: { display: "flex", flexDirection: "row" },
headerCell: {
flexGrow: 1,
flexBasis: 0,
padding: 4,
fontWeight: 700,
backgroundColor: "#f3f4f6",
borderRightWidth: 1,
borderBottomWidth: 1,
borderColor: "#d1d5db",
},
cell: {
flexGrow: 1,
flexBasis: 0,
padding: 4,
borderRightWidth: 1,
borderBottomWidth: 1,
borderColor: "#d1d5db",
},
});
const COLUMNS: ReadonlyArray<{ key: AuditPdfColumnKey; render?: (event: AuditPdfEvent) => string }> = [
{ key: "createdAt", render: (e) => (e.createdAt instanceof Date ? e.createdAt.toISOString() : String(e.createdAt)) },
{ key: "actorType" },
{ key: "actorId", render: (e) => e.actorId ?? "-" },
{ key: "action" },
{ key: "targetType", render: (e) => `${e.targetType ?? "-"}/${e.targetId ?? "-"}` },
{ key: "outcome" },
{ key: "regulatoryScope", render: (e) => e.regulatoryScope.join(",") || "-" },
{ key: "eventHash", render: (e) => (e.eventHash ? e.eventHash.slice(0, 16) : "-") },
];
function resolveAuditPdfLabels(input?: AuditPdfLabelOverrides): AuditPdfLabels {
return {
...DEFAULT_AUDIT_PDF_LABELS,
...input,
columns: {
...DEFAULT_AUDIT_PDF_LABELS.columns,
...input?.columns,
},
};
}
function buildIntegrityLine(integrity: AuditPdfIntegrity, labels: AuditPdfLabels): string {
if (integrity.ok) {
return labels.integrityOk(integrity.inspected);
}
return labels.integrityBroken({
inspected: integrity.inspected,
firstBreakId: integrity.firstBreakId ?? "?",
reason: integrity.reason ?? "unknown",
});
}
export function buildAuditPdfDocument(input: AuditPdfReportInput): ReturnType<typeof React.createElement> {
const labels = resolveAuditPdfLabels(input.labels);
const generatedAt = (input.generatedAt ?? new Date()).toISOString();
const integrityStyle = input.integrity.ok ? styles.integrityOk : styles.integrityBroken;
const eventRows = input.events.map((event, index) =>
React.createElement(
View,
{ key: `row-${event.id}-${index}`, style: styles.row },
...COLUMNS.map((col) => {
const raw = col.render ? col.render(event) : (event[col.key] as unknown);
const text = raw === null || raw === undefined ? "-" : String(raw);
return React.createElement(Text, { key: `${event.id}-${col.key}`, style: styles.cell }, text);
})
)
);
return React.createElement(
Document,
{ title: input.title ?? labels.title },
React.createElement(
Page,
{ size: "A4", style: styles.page },
React.createElement(Text, { style: styles.title }, input.title ?? labels.title),
React.createElement(Text, { style: styles.meta }, `${labels.tenant}: ${input.tenantId ?? `(${labels.platformTenant})`}`),
React.createElement(Text, { style: styles.meta }, `${labels.range}: ${input.from ?? `(${labels.openRange})`} ... ${input.to ?? `(${labels.openRange})`}`),
React.createElement(Text, { style: styles.meta }, `${labels.filters}: ${input.filterSummary || labels.noFilters}`),
input.generatedBy ? React.createElement(Text, { style: styles.meta }, `${labels.generatedBy}: ${input.generatedBy}`) : null,
React.createElement(Text, { style: styles.meta }, `${labels.generatedAt}: ${generatedAt}`),
React.createElement(Text, { style: styles.subtitle }, labels.hashChainIntegrity),
React.createElement(Text, { style: integrityStyle }, buildIntegrityLine(input.integrity, labels)),
React.createElement(Text, { style: styles.subtitle }, labels.events(input.events.length)),
React.createElement(
View,
{ style: styles.table },
React.createElement(
View,
{ style: styles.row },
...COLUMNS.map((col) =>
React.createElement(Text, { key: `head-${col.key}`, style: styles.headerCell }, labels.columns[col.key])
)
),
...eventRows
)
)
);
}
export async function serializeAuditEventsToPdf(input: AuditPdfReportInput): Promise<Uint8Array> {
if (input.events.length > AUDIT_PDF_HARD_ROW_LIMIT) {
throw new ExportPdfRowLimitExceededError(input.events.length);
}
const doc = buildAuditPdfDocument(input);
return new Uint8Array(await renderToBuffer(doc));
}
export async function serializeAuditEventsToPdfStream(writeStream: Writable, input: AuditPdfReportInput): Promise<{ rowCount: number }> {
if (input.events.length > AUDIT_PDF_HARD_ROW_LIMIT) {
throw new ExportPdfRowLimitExceededError(input.events.length);
}
const doc = buildAuditPdfDocument(input);
const readable = await renderToStream(doc);
await new Promise<void>((resolve, reject) => {
readable.on("error", reject);
writeStream.on("error", reject);
readable.on("end", () => resolve());
readable.pipe(writeStream, { end: false });
});
return { rowCount: input.events.length };
}
/**
* Format the active filters into a compact one-liner suitable for the
* cover-page metadata. Shared between CLI and web call-sites so the
* report self-describes the operator inputs identically.
*/
export function summarizeAuditPdfFilters(filters: {
regulatoryScope?: ReadonlyArray<string>;
actorType?: string | null;
outcome?: string | null;
}): string {
const parts: string[] = [];
if (filters.regulatoryScope && filters.regulatoryScope.length > 0) {
parts.push(`regulatoryScope=${filters.regulatoryScope.join("|")}`);
}
if (filters.actorType) parts.push(`actorType=${filters.actorType}`);
if (filters.outcome) parts.push(`outcome=${filters.outcome}`);
return parts.join(" ");
}
@@ -0,0 +1,132 @@
import { describe, expect, it } from "vitest";
import { ExportTableValidationError } from "../types.js";
import { buildCsvExportArtifact, serializeExportTableToCsv } from "./index.js";
describe("@toolctrl/exports/csv public entrypoint", () => {
it("exposes CSV-only runtime helpers without loading unrelated exporters", async () => {
const exportsModule = await import("@toolctrl/exports/csv");
expect(Object.keys(exportsModule).sort()).toEqual([
"ExportTableValidationError",
"buildCsvExportArtifact",
"serializeExportTableToCsv",
"serializeExportTableToCsvStream",
]);
});
});
describe("serializeExportTableToCsv", () => {
it("uses explicit header labels and column order", () => {
const csv = serializeExportTableToCsv({
columns: [
{ key: "email", label: "Email address" },
{ key: "id", label: "Identifier" },
],
rows: [{ id: 7, email: "ada@example.com" }],
});
expect(csv).toBe('"Email address","Identifier"\n"ada@example.com","7"');
});
it("escapes commas, quotes, and newlines with RFC-style quoting", () => {
const csv = serializeExportTableToCsv({
columns: ["name", "note"],
rows: [
{
name: 'Ada "Countess"',
note: "line one,line two\nline three",
},
],
});
expect(csv).toBe('"name","note"\n"Ada ""Countess""","line one,line two\nline three"');
});
it("neutralizes spreadsheet formula prefixes before CSV escaping", () => {
const csv = serializeExportTableToCsv({
columns: [
{ key: "equals", label: "=header" },
"plus",
"minus",
"at",
"spaceFormula",
"spaceTab",
"spaceCr",
"spaceLf",
"safe",
],
rows: [
{
equals: "=1+1",
plus: "+1",
minus: "-1",
at: "@cmd",
spaceFormula: " =1+1",
spaceTab: " \tvalue",
spaceCr: " \rvalue",
spaceLf: " \nvalue",
safe: " value",
},
],
});
expect(csv).toBe([
`"'=header","plus","minus","at","spaceFormula","spaceTab","spaceCr","spaceLf","safe"`,
`"'=1+1","'+1","'-1","'@cmd","' =1+1","' \tvalue","' \rvalue","' \nvalue"," value"`,
].join("\n"));
});
it("serializes null and undefined values as blank cells", () => {
const csv = serializeExportTableToCsv({
columns: ["id", "empty", "missing"],
rows: [{ id: 1, empty: null, missing: undefined }],
});
expect(csv).toBe([
`"id","empty","missing"`,
`"1","",""`,
].join("\n"));
});
it("projects selected columns while preserving deterministic scalar strings", () => {
const csv = serializeExportTableToCsv({
columns: ["createdAt", "active", "amount"],
rows: [
{
active: false,
amount: 42.5,
createdAt: new Date("2024-01-02T03:04:05.006Z"),
ignored: "not exported",
},
],
});
expect(csv).toBe('"createdAt","active","amount"\n"2024-01-02T03:04:05.006Z","false","42.5"');
});
it("rejects duplicate columns through the CSV API", () => {
expect(() =>
serializeExportTableToCsv({
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
});
});
describe("buildCsvExportArtifact", () => {
it("returns CSV bytes with export metadata", () => {
const artifact = buildCsvExportArtifact({
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
});
expect(artifact).toEqual({
bytes: '"id"\n"1"\n"2"',
contentType: "text/csv; charset=utf-8",
extension: "csv",
rowCount: 2,
});
});
});
@@ -0,0 +1,148 @@
import type { Writable } from "node:stream";
import { type ExportColumn, type ExportRecord, type ExportTable, normalizeExportTable } from "../types.js";
import { neutralizeSpreadsheetFormula } from "../spreadsheet-safety.js";
export { ExportTableValidationError } from "../types.js";
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable, NormalizedExportColumn, NormalizedExportRow, NormalizedExportTable } from "../types.js";
export type CsvExportOptions = {
eol?: "\n" | "\r\n";
delimiter?: "," | "|" | ";";
};
export type CsvExportArtifact = {
bytes: string;
contentType: "text/csv; charset=utf-8";
extension: "csv";
rowCount: number;
};
/**
* Streaming CSV writer. Iterates over `table.rows` row-by-row, writing each
* encoded record directly to `writeStream`. Respects writable back-pressure
* via the standard `write`/`drain` contract so worker memory stays flat for
* tables of arbitrary size.
*
* The function is intentionally permissive about `table.rows` — it accepts
* any sync or async iterable of {@link ExportRecord}, which lets callers
* pipe a Prisma cursor (chunked async iterator) without buffering all rows.
*/
export async function serializeExportTableToCsvStream(
writeStream: Writable,
table: CsvStreamTable,
options: CsvExportOptions = {},
): Promise<{ rowCount: number }> {
const eol = options.eol ?? "\n";
const delimiter = options.delimiter ?? ",";
const columns = resolveCsvColumns(table);
const header = columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
await writeChunk(writeStream, header);
let rowCount = 0;
let isFirstRow = true;
for await (const row of toAsyncIterable(table.rows)) {
if (row === null || typeof row !== "object" || Array.isArray(row)) {
throw new Error("Export table row must be an object record.");
}
const encoded = columns.map((column) => escapeCsvCell(stringifyValue(row[column.key]))).join(delimiter);
if (isFirstRow) {
// header already emitted — every row is preceded by an eol so the
// final byte of the stream is never an unterminated delimiter.
isFirstRow = false;
}
await writeChunk(writeStream, `${eol}${encoded}`);
rowCount += 1;
}
return { rowCount };
}
export function serializeExportTableToCsv(table: ExportTable, options: CsvExportOptions = {}): string {
const normalizedTable = normalizeExportTable(table);
const eol = options.eol ?? "\n";
const delimiter = options.delimiter ?? ",";
const header = normalizedTable.columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
const rows = normalizedTable.rows.map((row) => normalizedTable.columns.map((column) => escapeCsvCell(row[column.key])).join(delimiter));
return [header, ...rows].join(eol);
}
export function buildCsvExportArtifact(table: ExportTable, options: CsvExportOptions = {}): CsvExportArtifact {
const bytes = serializeExportTableToCsv(table, options);
const normalizedTable = normalizeExportTable(table);
return {
bytes,
contentType: "text/csv; charset=utf-8",
extension: "csv",
rowCount: normalizedTable.rowCount,
};
}
export type CsvStreamTable = {
columns?: readonly (string | ExportColumn)[];
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
};
function resolveCsvColumns(table: CsvStreamTable): { key: string; label: string }[] {
if (table.columns === undefined) {
throw new Error("Streaming CSV writer requires explicit columns; cannot infer from an async row source.");
}
const seenKeys = new Set<string>();
const columns = table.columns.map((column) => {
const normalized = typeof column === "string" ? { key: column, label: column } : { key: column.key, label: column.label ?? column.key };
if (seenKeys.has(normalized.key)) {
throw new Error(`Export table columns contain duplicate key "${normalized.key}".`);
}
seenKeys.add(normalized.key);
return normalized;
});
return columns;
}
function stringifyValue(value: unknown): string {
if (value === null || value === undefined) return "";
if (value instanceof Date) return value.toISOString();
if (typeof value === "bigint") return value.toString();
return String(value);
}
function escapeCsvCell(value: string): string {
const safeValue = neutralizeSpreadsheetFormula(value);
return `"${safeValue.replaceAll("\"", "\"\"")}"`;
}
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
if (Symbol.asyncIterator in (source as object)) {
return source as AsyncIterable<T>;
}
return (async function* () {
for (const item of source as Iterable<T>) {
yield item;
}
})();
}
@@ -0,0 +1,115 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import { serializeExportTableToCsvStream } from "./index.js";
async function collectStream(stream: PassThrough): Promise<string> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks).toString("utf8");
}
describe("serializeExportTableToCsvStream", () => {
it("writes a header followed by encoded rows", async () => {
const sink = new PassThrough();
const collected = collectStream(sink);
const result = await serializeExportTableToCsvStream(
sink,
{
columns: [
{ key: "email", label: "Email address" },
{ key: "id", label: "Identifier" },
],
rows: [
{ id: 7, email: "ada@example.com" },
{ id: 8, email: "grace@example.com" },
],
},
);
sink.end();
expect(result.rowCount).toBe(2);
expect(await collected).toBe(
[
'"Email address","Identifier"',
'"ada@example.com","7"',
'"grace@example.com","8"',
].join("\n"),
);
});
it("supports async iterables as the row source (Prisma cursor friendly)", async () => {
const sink = new PassThrough();
const collected = collectStream(sink);
async function* rows() {
yield { id: 1, name: "A" };
yield { id: 2, name: "B" };
yield { id: 3, name: "C" };
}
const result = await serializeExportTableToCsvStream(sink, {
columns: ["id", "name"],
rows: rows(),
});
sink.end();
expect(result.rowCount).toBe(3);
const text = await collected;
expect(text.split("\n")).toEqual(['"id","name"', '"1","A"', '"2","B"', '"3","C"']);
});
it("neutralises spreadsheet-formula prefixes the same as the buffered API", async () => {
const sink = new PassThrough();
const collected = collectStream(sink);
await serializeExportTableToCsvStream(sink, {
columns: ["a"],
rows: [{ a: "=1+1" }],
});
sink.end();
expect(await collected).toBe(['"a"', "\"'=1+1\""].join("\n"));
});
it("rejects duplicate column keys eagerly", async () => {
const sink = new PassThrough();
await expect(
serializeExportTableToCsvStream(sink, {
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).rejects.toThrow('Export table columns contain duplicate key "id".');
});
it("keeps worker memory flat for 100k synthetic rows", async () => {
const sink = new PassThrough();
// Drain the sink as quickly as the producer writes so the PassThrough
// buffer cannot mask a buffering regression in the streamer.
sink.on("data", () => undefined);
async function* rows() {
for (let i = 0; i < 100_000; i += 1) {
yield { id: i, value: `row-${i}` };
}
}
if (global.gc) global.gc();
const before = process.memoryUsage().heapUsed;
const result = await serializeExportTableToCsvStream(sink, {
columns: ["id", "value"],
rows: rows(),
});
sink.end();
if (global.gc) global.gc();
const after = process.memoryUsage().heapUsed;
expect(result.rowCount).toBe(100_000);
// Hard upper bound is generous to absorb GC jitter; a buffering
// regression would inflate heap by tens of MB.
expect(after - before).toBeLessThan(100 * 1024 * 1024);
});
});
@@ -0,0 +1,3 @@
export const PDF_STREAM_HARD_ROW_LIMIT = 5_000;
export const XLSX_STREAM_HARD_ROW_LIMIT = 500_000;
export const XLSX_STREAM_THRESHOLD = 50_000;
@@ -0,0 +1,182 @@
import { describe, expect, it } from "vitest";
import { readFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
ExportTableValidationError,
buildCsvExportArtifact,
buildJsonExportArtifact,
buildPdfExportArtifact,
buildXlsxExportArtifact,
normalizeExportTable,
serializeExportTableToCsv,
serializeExportTableToJson,
serializeExportTableToJsonLines,
serializeExportTableToPdf,
serializeExportTableToXlsx,
} from "@toolctrl/exports";
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
describe("@toolctrl/exports public entrypoint", () => {
it("publishes dedicated subpaths so lightweight consumers can avoid the aggregate entrypoint", async () => {
const packageJson = JSON.parse(
await readFile(join(packageRoot, "package.json"), "utf8"),
) as { exports: Record<string, string> };
expect(packageJson.exports).toMatchObject({
".": "./src/index.ts",
"./audit-pdf": "./src/audit-pdf.ts",
"./csv": "./src/csv/index.ts",
"./json": "./src/json/index.ts",
"./limits": "./src/limits.ts",
"./pdf": "./src/pdf/index.ts",
"./s3-multipart": "./src/sinks/s3-multipart.ts",
"./types": "./src/types.ts",
"./xlsx": "./src/xlsx/index.ts",
});
});
it("exposes CSV, JSON, shared validation, XLSX, and PDF APIs exactly once", async () => {
const exportsModule = await import("@toolctrl/exports");
const expectedExports = [
"AUDIT_PDF_CONTENT_TYPE",
"AUDIT_PDF_HARD_ROW_LIMIT",
"EXPORT_PLAN_ROW_CAPS",
"ExportPdfRowLimitExceededError",
"ExportPrettyJsonTooLargeError",
"ExportRowCapExceededError",
"ExportTableValidationError",
"ExportXlsxRowLimitExceededError",
"PDF_STREAM_HARD_ROW_LIMIT",
"PRETTY_JSON_STREAM_ROW_LIMIT",
"XLSX_STREAM_HARD_ROW_LIMIT",
"XLSX_STREAM_THRESHOLD",
"assertExportRowCap",
"buildAuditPdfDocument",
"buildCsvExportArtifact",
"buildJsonExportArtifact",
"buildPdfExportArtifact",
"buildXlsxExportArtifact",
"createS3MultipartUploadSink",
"normalizeExportTable",
"resolveExportRowCap",
"serializeAuditEventsToPdf",
"serializeAuditEventsToPdfStream",
"serializeExportTableToCsv",
"serializeExportTableToCsvStream",
"serializeExportTableToJson",
"serializeExportTableToJsonLines",
"serializeExportTableToJsonStream",
"serializeExportTableToPdf",
"serializeExportTableToPdfStream",
"serializeExportTableToXlsx",
"serializeExportTableToXlsxStream",
"summarizeAuditPdfFilters",
];
expect(Object.keys(exportsModule).sort()).toEqual(expectedExports);
expect(new Set(Object.keys(exportsModule))).toHaveLength(expectedExports.length);
expect(exportsModule.ExportTableValidationError).toBe(ExportTableValidationError);
expect(exportsModule.buildCsvExportArtifact).toBe(buildCsvExportArtifact);
expect(exportsModule.buildJsonExportArtifact).toBe(buildJsonExportArtifact);
expect(exportsModule.buildPdfExportArtifact).toBe(buildPdfExportArtifact);
expect(exportsModule.buildXlsxExportArtifact).toBe(buildXlsxExportArtifact);
expect(exportsModule.normalizeExportTable).toBe(normalizeExportTable);
expect(exportsModule.serializeExportTableToCsv).toBe(serializeExportTableToCsv);
expect(exportsModule.serializeExportTableToJson).toBe(serializeExportTableToJson);
expect(exportsModule.serializeExportTableToJsonLines).toBe(serializeExportTableToJsonLines);
expect(exportsModule.serializeExportTableToPdf).toBe(serializeExportTableToPdf);
expect(exportsModule.serializeExportTableToXlsx).toBe(serializeExportTableToXlsx);
});
it("builds CSV, JSON, XLSX, and PDF artifacts through the package entrypoint", async () => {
const table = {
columns: [
{ key: "email", label: "Email" },
{ key: "id", label: "Identifier" },
"active",
],
rows: [
{ id: 7, email: "ada@example.com", active: true, ignored: "not exported" },
{ id: 8, email: null, active: false },
],
};
const expectedRowCount = normalizeExportTable(table).rowCount;
expect(expectedRowCount).toBe(2);
expect(serializeExportTableToCsv(table)).toBe(
[
'"Email","Identifier","active"',
'"ada@example.com","7","true"',
'"","8","false"',
].join("\n"),
);
expect(buildCsvExportArtifact(table)).toEqual({
bytes: [
'"Email","Identifier","active"',
'"ada@example.com","7","true"',
'"","8","false"',
].join("\n"),
contentType: "text/csv; charset=utf-8",
extension: "csv",
rowCount: expectedRowCount,
});
expect(serializeExportTableToJson(table)).toBe(
'[\n {\n "email": "ada@example.com",\n "id": 7,\n "active": true\n },\n {\n "email": null,\n "id": 8,\n "active": false\n }\n]',
);
expect(serializeExportTableToJsonLines(table)).toBe(
'{"email":"ada@example.com","id":7,"active":true}\n{"email":null,"id":8,"active":false}',
);
expect(buildJsonExportArtifact(table, { format: "jsonl" })).toEqual({
bytes: '{"email":"ada@example.com","id":7,"active":true}\n{"email":null,"id":8,"active":false}',
contentType: "application/x-ndjson; charset=utf-8",
extension: "jsonl",
rowCount: expectedRowCount,
});
const xlsxBytes = await serializeExportTableToXlsx(table);
expect(xlsxBytes.byteLength).toBeGreaterThan(0);
const xlsxArtifact = await buildXlsxExportArtifact(table);
expect(xlsxArtifact).toMatchObject({
contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
extension: "xlsx",
rowCount: expectedRowCount,
byteLength: xlsxArtifact.bytes.byteLength,
});
expect(xlsxArtifact.byteLength).toBeGreaterThan(0);
const pdfBytes = await serializeExportTableToPdf(table, { title: "Member export" });
expect(new TextDecoder().decode(pdfBytes.slice(0, 5))).toBe("%PDF-");
const pdfArtifact = await buildPdfExportArtifact(table);
expect(pdfArtifact).toMatchObject({
contentType: "application/pdf",
extension: "pdf",
rowCount: expectedRowCount,
byteLength: pdfArtifact.bytes.byteLength,
});
expect(pdfArtifact.byteLength).toBeGreaterThan(0);
expect([xlsxArtifact, pdfArtifact]).toEqual([
expect.objectContaining({
rowCount: expectedRowCount,
contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
extension: "xlsx",
byteLength: expect.any(Number),
}),
expect.objectContaining({
rowCount: expectedRowCount,
contentType: "application/pdf",
extension: "pdf",
byteLength: expect.any(Number),
}),
]);
});
});
@@ -0,0 +1,8 @@
export * from "./csv/index.js";
export * from "./json/index.js";
export * from "./pdf/index.js";
export * from "./xlsx/index.js";
export * from "./types.js";
export * from "./sinks/s3-multipart.js";
export * from "./limits.js";
export * from "./audit-pdf.js";
@@ -0,0 +1,144 @@
import { describe, expect, it } from "vitest";
import { ExportTableValidationError } from "../types.js";
import {
buildJsonExportArtifact,
serializeExportTableToJson,
serializeExportTableToJsonLines,
} from "./index.js";
describe("@toolctrl/exports/json public entrypoint", () => {
it("exposes JSON-only runtime helpers", async () => {
const exportsModule = await import("./index.js");
expect(Object.keys(exportsModule).sort()).toEqual([
"ExportPrettyJsonTooLargeError",
"ExportTableValidationError",
"PRETTY_JSON_STREAM_ROW_LIMIT",
"buildJsonExportArtifact",
"serializeExportTableToJson",
"serializeExportTableToJsonLines",
"serializeExportTableToJsonStream",
]);
});
});
describe("serializeExportTableToJson", () => {
it("projects fields in explicit column order", () => {
const json = serializeExportTableToJson({
columns: ["email", "id"],
rows: [{ id: 7, name: "Ada", email: "ada@example.com" }],
});
expect(JSON.parse(json)).toEqual([{ email: "ada@example.com", id: 7 }]);
expect(json).toBe('[\n {\n "email": "ada@example.com",\n "id": 7\n }\n]');
});
it("emits pretty JSON by default and compact JSON when disabled", () => {
const table = {
columns: ["id", "active"],
rows: [{ id: 1, active: true }],
};
expect(serializeExportTableToJson(table)).toBe('[\n {\n "id": 1,\n "active": true\n }\n]');
expect(serializeExportTableToJson(table, { pretty: false })).toBe('[{"id":1,"active":true}]');
});
it("preserves selected null values and projects away omitted fields", () => {
const json = serializeExportTableToJson({
columns: ["id", "deletedAt", "missing"],
rows: [{ id: 1, deletedAt: null, ignored: "not exported", missing: undefined }],
});
expect(JSON.parse(json)).toEqual([{ id: 1, deletedAt: null, missing: null }]);
expect(json).not.toContain("ignored");
});
it("serializes dates as deterministic ISO strings and bigints as JSON-safe strings", () => {
const json = serializeExportTableToJson({
columns: ["createdAt", "big"],
rows: [{ createdAt: new Date("2024-01-02T03:04:05.006Z"), big: 9007199254740993n }],
});
expect(JSON.parse(json)).toEqual([
{ createdAt: "2024-01-02T03:04:05.006Z", big: "9007199254740993" },
]);
});
it("rejects malformed row input through the JSON API", () => {
expect(() =>
serializeExportTableToJson({
rows: [{ id: 1 }, null],
} as never),
).toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
});
it("rejects duplicate columns through the JSON API", () => {
expect(() =>
serializeExportTableToJson({
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
});
});
describe("serializeExportTableToJsonLines", () => {
it("emits one compact JSON object per row without a trailing newline", () => {
const jsonl = serializeExportTableToJsonLines({
columns: ["id", "name"],
rows: [
{ id: 1, name: "Ada" },
{ id: 2, name: "Grace" },
],
});
expect(jsonl).toBe('{"id":1,"name":"Ada"}\n{"id":2,"name":"Grace"}');
expect(jsonl.endsWith("\n")).toBe(false);
});
it("supports CRLF separators for JSONL output", () => {
const jsonl = serializeExportTableToJsonLines(
{
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
},
{ eol: "\r\n" },
);
expect(jsonl).toBe('{"id":1}\r\n{"id":2}');
});
});
describe("buildJsonExportArtifact", () => {
it("returns JSON bytes with artifact metadata", () => {
const artifact = buildJsonExportArtifact({
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
});
expect(artifact).toEqual({
bytes: '[\n {\n "id": 1\n },\n {\n "id": 2\n }\n]',
contentType: "application/json; charset=utf-8",
extension: "json",
rowCount: 2,
});
});
it("returns JSONL bytes with artifact metadata", () => {
const artifact = buildJsonExportArtifact(
{
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
},
{ format: "jsonl" },
);
expect(artifact).toEqual({
bytes: '{"id":1}\n{"id":2}',
contentType: "application/x-ndjson; charset=utf-8",
extension: "jsonl",
rowCount: 2,
});
});
});
@@ -0,0 +1,203 @@
import type { Writable } from "node:stream";
import {
type ExportColumn,
type ExportPrimitive,
type ExportRecord,
type ExportTable,
normalizeExportTable,
} from "../types.js";
export { ExportTableValidationError } from "../types.js";
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
export type JsonExportOptions = {
pretty?: boolean;
eol?: "\n" | "\r\n";
};
export type JsonExportArtifactOptions = JsonExportOptions & {
format?: "json" | "jsonl";
};
export type JsonExportArtifact = {
bytes: string;
contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8";
extension: "json" | "jsonl";
rowCount: number;
};
type JsonExportValue = string | number | boolean | null;
type JsonExportRow = Record<string, JsonExportValue>;
/**
* Maximum row count permitted for the pretty-JSON streaming mode. Pretty
* mode requires building the full array in memory to emit a syntactically
* valid document; above this threshold callers must switch to NDJSON.
*/
export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000;
export class ExportPrettyJsonTooLargeError extends Error {
readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE";
constructor(rowCount: number) {
super(
`Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`,
);
this.name = "ExportPrettyJsonTooLargeError";
}
}
export type JsonStreamMode = "ndjson" | "pretty";
export type JsonStreamOptions = {
mode?: JsonStreamMode;
eol?: "\n" | "\r\n";
};
export type JsonStreamTable = {
columns?: readonly (string | ExportColumn)[];
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
};
export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string {
const rows = buildJsonRows(table);
return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0);
}
export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string {
const eol = options.eol ?? "\n";
return buildJsonRows(table)
.map((row) => JSON.stringify(row))
.join(eol);
}
/**
* Streaming JSON writer. NDJSON mode writes one JSON object per line and
* keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT}
* rows before flushing a single indented array; above the limit the writer
* throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON.
*/
export async function serializeExportTableToJsonStream(
writeStream: Writable,
table: JsonStreamTable,
options: JsonStreamOptions = {},
): Promise<{ rowCount: number; mode: JsonStreamMode }> {
const mode = options.mode ?? "ndjson";
const eol = options.eol ?? "\n";
const columnKeys = resolveJsonColumnKeys(table);
if (mode === "ndjson") {
let rowCount = 0;
for await (const row of toAsyncIterable(table.rows)) {
const projected = projectRow(row as ExportRecord, columnKeys);
const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`;
await writeChunk(writeStream, chunk);
rowCount += 1;
}
return { rowCount, mode };
}
const buffered: JsonExportRow[] = [];
for await (const row of toAsyncIterable(table.rows)) {
buffered.push(projectRow(row as ExportRecord, columnKeys));
if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) {
throw new ExportPrettyJsonTooLargeError(buffered.length);
}
}
await writeChunk(writeStream, JSON.stringify(buffered, null, 2));
return { rowCount: buffered.length, mode };
}
export function buildJsonExportArtifact(
table: ExportTable,
options: JsonExportArtifactOptions = {},
): JsonExportArtifact {
const normalizedTable = normalizeExportTable(table);
const format = options.format ?? "json";
const bytes =
format === "jsonl"
? serializeExportTableToJsonLines(table, options)
: serializeExportTableToJson(table, options);
return {
bytes,
contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8",
extension: format === "jsonl" ? "jsonl" : "json",
rowCount: normalizedTable.rowCount,
};
}
function buildJsonRows(table: ExportTable): JsonExportRow[] {
const normalizedTable = normalizeExportTable(table);
return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key)));
}
function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow {
return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow;
}
function normalizeJsonValue(value: ExportPrimitive): JsonExportValue {
if (value === null || value === undefined) {
return null;
}
if (value instanceof Date) {
return value.toISOString();
}
if (typeof value === "bigint") {
return value.toString();
}
return value;
}
function resolveJsonColumnKeys(table: JsonStreamTable): string[] {
if (table.columns === undefined) {
throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
}
const keys: string[] = [];
const seen = new Set<string>();
for (const column of table.columns) {
const key = typeof column === "string" ? column : column.key;
if (seen.has(key)) {
throw new Error(`Export table columns contain duplicate key "${key}".`);
}
seen.add(key);
keys.push(key);
}
return keys;
}
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
if (Symbol.asyncIterator in (source as object)) {
return source as AsyncIterable<T>;
}
return (async function* () {
for (const item of source as Iterable<T>) {
yield item;
}
})();
}
@@ -0,0 +1,103 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import {
ExportPrettyJsonTooLargeError,
PRETTY_JSON_STREAM_ROW_LIMIT,
serializeExportTableToJsonStream,
} from "./index.js";
async function collect(stream: PassThrough): Promise<string> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks).toString("utf8");
}
describe("serializeExportTableToJsonStream", () => {
it("emits one JSON object per line in ndjson mode (default)", async () => {
const sink = new PassThrough();
const collected = collect(sink);
const result = await serializeExportTableToJsonStream(sink, {
columns: ["id", "active"],
rows: [
{ id: 1, active: true },
{ id: 2, active: false },
],
});
sink.end();
expect(result).toEqual({ rowCount: 2, mode: "ndjson" });
expect(await collected).toBe('{"id":1,"active":true}\n{"id":2,"active":false}');
});
it("serialises Dates and BigInts predictably", async () => {
const sink = new PassThrough();
const collected = collect(sink);
await serializeExportTableToJsonStream(sink, {
columns: ["createdAt", "count"],
rows: [
{ createdAt: new Date("2024-01-02T03:04:05.006Z"), count: BigInt(42) },
],
});
sink.end();
expect(JSON.parse(await collected)).toEqual({
createdAt: "2024-01-02T03:04:05.006Z",
count: "42",
});
});
it("supports pretty mode for small tables", async () => {
const sink = new PassThrough();
const collected = collect(sink);
const result = await serializeExportTableToJsonStream(
sink,
{
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
},
{ mode: "pretty" },
);
sink.end();
expect(result).toEqual({ rowCount: 2, mode: "pretty" });
expect(JSON.parse(await collected)).toEqual([{ id: 1 }, { id: 2 }]);
});
it("rejects pretty mode above the 10k-row limit with a typed error", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
async function* rows() {
for (let i = 0; i <= PRETTY_JSON_STREAM_ROW_LIMIT; i += 1) {
yield { id: i };
}
}
await expect(
serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() }, { mode: "pretty" }),
).rejects.toBeInstanceOf(ExportPrettyJsonTooLargeError);
});
it("accepts async iterables (Prisma cursor friendly)", async () => {
const sink = new PassThrough();
const collected = collect(sink);
async function* rows() {
yield { id: 1 };
yield { id: 2 };
yield { id: 3 };
}
const result = await serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() });
sink.end();
expect(result.rowCount).toBe(3);
expect(await collected).toBe('{"id":1}\n{"id":2}\n{"id":3}');
});
});
@@ -0,0 +1,84 @@
import { describe, expect, it } from "vitest";
import { readFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
EXPORT_PLAN_ROW_CAPS,
ExportRowCapExceededError,
assertExportRowCap,
resolveExportRowCap,
} from "./limits.js";
import { PDF_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_HARD_ROW_LIMIT } from "./format-caps.js";
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
describe("@toolctrl/exports/limits public entrypoint", () => {
it("exposes row-cap helpers without importing heavy PDF/XLSX runtimes", async () => {
const exportsModule = await import("./limits.js");
expect(Object.keys(exportsModule).sort()).toEqual([
"EXPORT_PLAN_ROW_CAPS",
"ExportRowCapExceededError",
"assertExportRowCap",
"resolveExportRowCap",
]);
});
it("keeps limits wired to lightweight cap constants", async () => {
const source = await readFile(join(packageRoot, "src/limits.ts"), "utf8");
expect(source).toContain("./format-caps.js");
expect(source).not.toMatch(/["']\.\/(?:pdf|xlsx)\/index\.js["']/);
});
});
describe("resolveExportRowCap", () => {
it("applies the plan cap to unbounded formats (csv, json)", () => {
expect(resolveExportRowCap("free", "csv")).toBe(EXPORT_PLAN_ROW_CAPS.free);
expect(resolveExportRowCap("pro", "json")).toBe(EXPORT_PLAN_ROW_CAPS.pro);
expect(resolveExportRowCap("enterprise", "csv")).toBe(Number.POSITIVE_INFINITY);
});
it("respects the per-format hard cap when stricter than the plan cap", () => {
expect(resolveExportRowCap("enterprise", "xlsx")).toBe(XLSX_STREAM_HARD_ROW_LIMIT);
expect(resolveExportRowCap("enterprise", "pdf")).toBe(PDF_STREAM_HARD_ROW_LIMIT);
// free plan cap (10k) is lower than pdf hard cap (5k), so pdf cap wins.
expect(resolveExportRowCap("free", "pdf")).toBe(PDF_STREAM_HARD_ROW_LIMIT);
});
it("falls back to the free-plan cap for unknown plan tiers", () => {
expect(resolveExportRowCap("vip" as never, "csv")).toBe(EXPORT_PLAN_ROW_CAPS.free);
});
});
describe("assertExportRowCap", () => {
it("returns silently when the requested count fits the cap", () => {
expect(() => assertExportRowCap(5_000, "free", "csv")).not.toThrow();
expect(() => assertExportRowCap(50_000, "pro", "json")).not.toThrow();
});
it("throws ExportRowCapExceededError with the resolved cap", () => {
let caught: ExportRowCapExceededError | undefined;
try {
assertExportRowCap(20_000, "free", "csv");
} catch (error) {
caught = error as ExportRowCapExceededError;
}
expect(caught).toBeInstanceOf(ExportRowCapExceededError);
expect(caught?.code).toBe("EXPORT_ROW_LIMIT_PLAN");
expect(caught?.effectiveCap).toBe(EXPORT_PLAN_ROW_CAPS.free);
expect(caught?.requestedRowCount).toBe(20_000);
expect(caught?.plan).toBe("free");
expect(caught?.format).toBe("csv");
});
it("enforces format hard caps even for enterprise plans", () => {
expect(() => assertExportRowCap(PDF_STREAM_HARD_ROW_LIMIT + 1, "enterprise", "pdf")).toThrow(
ExportRowCapExceededError,
);
expect(() => assertExportRowCap(XLSX_STREAM_HARD_ROW_LIMIT + 1, "enterprise", "xlsx")).toThrow(
ExportRowCapExceededError,
);
});
});
@@ -0,0 +1,72 @@
import { PDF_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_HARD_ROW_LIMIT } from "./format-caps.js";
/**
* Tenant plan tier used to resolve export row caps. Matches the
* billing-plan vocabulary used elsewhere in the repo. Callers that
* encounter an unknown plan should fall back to "free" — the most
* conservative cap — rather than throwing, because the row cap is a
* safety net, not an authorisation primitive.
*/
export type TenantPlanTier = "free" | "pro" | "enterprise";
export type ExportFormat = "csv" | "json" | "xlsx" | "pdf";
/**
* Plan-based soft caps applied on top of the per-format hard caps. The
* plan cap protects shared worker capacity from a single noisy tenant; the
* format cap protects against format-specific failure modes.
*/
export const EXPORT_PLAN_ROW_CAPS: Record<TenantPlanTier, number> = {
free: 10_000,
pro: 100_000,
enterprise: Number.POSITIVE_INFINITY,
};
const FORMAT_HARD_CAPS: Record<ExportFormat, number> = {
csv: Number.POSITIVE_INFINITY,
json: Number.POSITIVE_INFINITY,
xlsx: XLSX_STREAM_HARD_ROW_LIMIT,
pdf: PDF_STREAM_HARD_ROW_LIMIT,
};
export class ExportRowCapExceededError extends Error {
readonly code = "EXPORT_ROW_LIMIT_PLAN";
constructor(
readonly requestedRowCount: number,
readonly effectiveCap: number,
readonly plan: TenantPlanTier,
readonly format: ExportFormat,
) {
super(
`Export request of ${requestedRowCount} rows exceeds the ${plan} plan / ${format} format cap of ${Number.isFinite(effectiveCap) ? effectiveCap : "unlimited"}.`,
);
this.name = "ExportRowCapExceededError";
}
}
/**
* Resolves the effective row cap for a given plan and format. Returns
* `Infinity` only when both the plan and the format are unbounded; in
* practice all PDF and XLSX exports still hit a finite ceiling.
*/
export function resolveExportRowCap(plan: TenantPlanTier, format: ExportFormat): number {
const planCap = EXPORT_PLAN_ROW_CAPS[plan] ?? EXPORT_PLAN_ROW_CAPS.free;
const formatCap = FORMAT_HARD_CAPS[format] ?? Number.POSITIVE_INFINITY;
return Math.min(planCap, formatCap);
}
/**
* Convenience guard: throws {@link ExportRowCapExceededError} when the
* caller knows the row count up front. Streamers also enforce caps
* mid-stream so this is a fast-path check, not the sole guard.
*/
export function assertExportRowCap(
rowCount: number,
plan: TenantPlanTier,
format: ExportFormat,
): void {
const cap = resolveExportRowCap(plan, format);
if (rowCount > cap) {
throw new ExportRowCapExceededError(rowCount, cap, plan, format);
}
}
@@ -0,0 +1,77 @@
import { describe, expect, it } from "vitest";
import { ExportTableValidationError } from "../types.js";
import { buildPdfExportArtifact, serializeExportTableToPdf } from "./index.js";
const PDF_HEADER = "%PDF-";
function decodeHeader(bytes: Uint8Array): string {
return new TextDecoder().decode(bytes.slice(0, PDF_HEADER.length));
}
describe("@toolctrl/exports/pdf public entrypoint", () => {
it("exposes PDF helpers through the dedicated subpath", async () => {
const exportsModule = await import("./index.js");
expect(Object.keys(exportsModule).sort()).toEqual([
"ExportPdfRowLimitExceededError",
"ExportTableValidationError",
"PDF_STREAM_HARD_ROW_LIMIT",
"buildPdfExportArtifact",
"serializeExportTableToPdf",
"serializeExportTableToPdfStream",
]);
});
});
describe("serializeExportTableToPdf", () => {
it("renders PDF bytes with explicit title, header labels, and column order", async () => {
const bytes = await serializeExportTableToPdf(
{
columns: [
{ key: "email", label: "Email address" },
{ key: "id", label: "Identifier" },
"active",
],
rows: [{ id: 7, email: "ada@example.com", active: true, ignored: "not exported" }],
},
{ title: "Member export" },
);
expect(decodeHeader(bytes)).toBe(PDF_HEADER);
expect(bytes.byteLength).toBeGreaterThan(0);
});
it("rejects malformed row input through the PDF API", async () => {
await expect(
serializeExportTableToPdf({
rows: [{ id: 1 }, null],
} as never),
).rejects.toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
});
it("rejects duplicate columns through the PDF API", async () => {
await expect(
serializeExportTableToPdf({
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).rejects.toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
});
});
describe("buildPdfExportArtifact", () => {
it("returns PDF bytes with binary export metadata", async () => {
const artifact = await buildPdfExportArtifact({
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
});
expect(decodeHeader(artifact.bytes)).toBe(PDF_HEADER);
expect(artifact.contentType).toBe("application/pdf");
expect(artifact.extension).toBe("pdf");
expect(artifact.rowCount).toBe(2);
expect(artifact.byteLength).toBe(artifact.bytes.byteLength);
expect(artifact.byteLength).toBeGreaterThan(0);
});
});
@@ -0,0 +1,194 @@
import type { Writable } from "node:stream";
import React from "react";
import { Document, Page, StyleSheet, Text, View, renderToBuffer, renderToStream } from "@react-pdf/renderer";
import { PDF_STREAM_HARD_ROW_LIMIT } from "../format-caps.js";
import { type ExportTable, normalizeExportTable } from "../types.js";
export { ExportTableValidationError } from "../types.js";
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
const PDF_CONTENT_TYPE = "application/pdf";
/**
* PDF rendering cost grows super-linearly with row count and produces
* unusable >500-page documents above this threshold. The cap protects
* both the worker and downstream UX.
*/
export { PDF_STREAM_HARD_ROW_LIMIT };
export class ExportPdfRowLimitExceededError extends Error {
readonly code = "EXPORT_PDF_ROW_LIMIT_EXCEEDED";
constructor(rowCount: number) {
super(
`PDF export exceeds the ${PDF_STREAM_HARD_ROW_LIMIT}-row hard cap (got ${rowCount}); use CSV or XLSX for larger datasets.`,
);
this.name = "ExportPdfRowLimitExceededError";
}
}
export type PdfExportOptions = {
title?: string;
};
export type PdfExportArtifact = {
bytes: Uint8Array;
contentType: typeof PDF_CONTENT_TYPE;
extension: "pdf";
rowCount: number;
byteLength: number;
};
const styles = StyleSheet.create({
page: {
padding: 32,
fontSize: 10,
fontFamily: "Helvetica",
},
title: {
fontSize: 16,
fontWeight: 700,
marginBottom: 16,
},
table: {
display: "flex",
flexDirection: "column",
width: "100%",
borderTopWidth: 1,
borderLeftWidth: 1,
borderColor: "#d1d5db",
},
row: {
display: "flex",
flexDirection: "row",
},
headerCell: {
flexGrow: 1,
flexBasis: 0,
padding: 6,
fontWeight: 700,
backgroundColor: "#f3f4f6",
borderRightWidth: 1,
borderBottomWidth: 1,
borderColor: "#d1d5db",
},
cell: {
flexGrow: 1,
flexBasis: 0,
padding: 6,
borderRightWidth: 1,
borderBottomWidth: 1,
borderColor: "#d1d5db",
},
});
export async function serializeExportTableToPdf(
table: ExportTable,
options: PdfExportOptions = {},
): Promise<Uint8Array> {
const normalizedTable = normalizeExportTable(table);
const document = React.createElement(
Document,
{ title: options.title },
React.createElement(
Page,
{ size: "A4", style: styles.page },
options.title === undefined ? null : React.createElement(Text, { style: styles.title }, options.title),
React.createElement(
View,
{ style: styles.table },
React.createElement(
View,
{ style: styles.row },
...normalizedTable.columns.map((column) =>
React.createElement(Text, { key: column.key, style: styles.headerCell }, column.label),
),
),
...normalizedTable.rows.map((row, rowIndex) =>
React.createElement(
View,
{ key: `row-${rowIndex}`, style: styles.row },
...normalizedTable.columns.map((column) =>
React.createElement(Text, { key: column.key, style: styles.cell }, row[column.key]),
),
),
),
),
),
);
return new Uint8Array(await renderToBuffer(document));
}
export async function buildPdfExportArtifact(
table: ExportTable,
options: PdfExportOptions = {},
): Promise<PdfExportArtifact> {
const normalizedTable = normalizeExportTable(table);
const bytes = await serializeExportTableToPdf(table, options);
return {
bytes,
contentType: PDF_CONTENT_TYPE,
extension: "pdf",
rowCount: normalizedTable.rowCount,
byteLength: bytes.byteLength,
};
}
/**
* Streaming PDF writer. Uses `@react-pdf/renderer`'s renderToStream and
* pipes the resulting Readable into `writeStream`. Enforces a hard cap of
* {@link PDF_STREAM_HARD_ROW_LIMIT} rows up front — PDF cost is
* super-linear and unbounded exports produce unusable artefacts.
*/
export async function serializeExportTableToPdfStream(
writeStream: Writable,
table: ExportTable,
options: PdfExportOptions = {},
): Promise<{ rowCount: number }> {
const normalizedTable = normalizeExportTable(table);
if (normalizedTable.rowCount > PDF_STREAM_HARD_ROW_LIMIT) {
throw new ExportPdfRowLimitExceededError(normalizedTable.rowCount);
}
const document = React.createElement(
Document,
{ title: options.title },
React.createElement(
Page,
{ size: "A4", style: styles.page },
options.title === undefined ? null : React.createElement(Text, { style: styles.title }, options.title),
React.createElement(
View,
{ style: styles.table },
React.createElement(
View,
{ style: styles.row },
...normalizedTable.columns.map((column) =>
React.createElement(Text, { key: column.key, style: styles.headerCell }, column.label),
),
),
...normalizedTable.rows.map((row, rowIndex) =>
React.createElement(
View,
{ key: `row-${rowIndex}`, style: styles.row },
...normalizedTable.columns.map((column) =>
React.createElement(Text, { key: column.key, style: styles.cell }, row[column.key]),
),
),
),
),
),
);
const readable = await renderToStream(document);
await new Promise<void>((resolve, reject) => {
readable.on("error", reject);
writeStream.on("error", reject);
readable.on("end", () => resolve());
readable.pipe(writeStream, { end: false });
});
return { rowCount: normalizedTable.rowCount };
}
@@ -0,0 +1,49 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import {
ExportPdfRowLimitExceededError,
PDF_STREAM_HARD_ROW_LIMIT,
serializeExportTableToPdfStream,
} from "./index.js";
async function collect(stream: PassThrough): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
describe("serializeExportTableToPdfStream", () => {
it("renders a valid PDF prelude for small tables", async () => {
const sink = new PassThrough();
const collected = collect(sink);
const rows = Array.from({ length: 10 }, (_, i) => ({ id: i, name: `name-${i}` }));
const result = await serializeExportTableToPdfStream(sink, {
columns: [
{ key: "id", label: "Identifier" },
{ key: "name", label: "Name" },
],
rows,
}, { title: "Member export" });
sink.end();
expect(result).toEqual({ rowCount: 10 });
const buffer = await collected;
expect(buffer.slice(0, 5).toString("utf8")).toBe("%PDF-");
expect(buffer.byteLength).toBeGreaterThan(100);
});
it("rejects exports above the 5k-row hard cap", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
const rows = Array.from({ length: PDF_STREAM_HARD_ROW_LIMIT + 1 }, (_, i) => ({ id: i }));
await expect(
serializeExportTableToPdfStream(sink, { columns: ["id"], rows }),
).rejects.toBeInstanceOf(ExportPdfRowLimitExceededError);
});
});
@@ -0,0 +1,96 @@
import { describe, expect, it, vi } from "vitest";
import { createS3MultipartUploadSink } from "./s3-multipart.js";
describe("createS3MultipartUploadSink", () => {
it("invokes the Upload constructor with sensible multipart defaults", async () => {
const done = vi.fn().mockResolvedValue({ ETag: '"deadbeef"' });
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
const fakeClient = {} as never;
const { writeStream, completionPromise } = createS3MultipartUploadSink({
client: fakeClient,
bucket: "exports",
key: "tenant/abc/users.csv",
contentType: "text/csv; charset=utf-8",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
uploadCtor: UploadStub as any,
});
writeStream.end("hello");
const result = await completionPromise;
expect(UploadStub).toHaveBeenCalledTimes(1);
const ctorArg = UploadStub.mock.calls[0]![0] as {
partSize: number;
queueSize: number;
params: { Bucket: string; Key: string; ContentType?: string };
};
expect(ctorArg.partSize).toBe(8 * 1024 * 1024);
expect(ctorArg.queueSize).toBe(4);
expect(ctorArg.params.Bucket).toBe("exports");
expect(ctorArg.params.Key).toBe("tenant/abc/users.csv");
expect(ctorArg.params.ContentType).toBe("text/csv; charset=utf-8");
expect(result).toEqual({ bucket: "exports", key: "tenant/abc/users.csv", etag: '"deadbeef"' });
});
it("returns etag: null when S3 omits an ETag", async () => {
const done = vi.fn().mockResolvedValue({});
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
const { writeStream, completionPromise } = createS3MultipartUploadSink({
client: {} as never,
bucket: "exports",
key: "x.csv",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
uploadCtor: UploadStub as any,
});
writeStream.end();
await expect(completionPromise).resolves.toEqual({ bucket: "exports", key: "x.csv", etag: null });
});
it("propagates Upload failures by destroying the write stream", async () => {
const failure = new Error("multipart aborted");
const done = vi.fn().mockRejectedValue(failure);
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
const { writeStream, completionPromise } = createS3MultipartUploadSink({
client: {} as never,
bucket: "exports",
key: "x.csv",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
uploadCtor: UploadStub as any,
});
// Stream errors propagate via the standard 'error' event; attach a
// listener so it does not crash the test runner as an unhandled emit.
const errorListener = vi.fn();
writeStream.on("error", errorListener);
await expect(completionPromise).rejects.toBe(failure);
await new Promise((resolve) => setImmediate(resolve));
expect(writeStream.destroyed).toBe(true);
expect(errorListener).toHaveBeenCalledWith(failure);
});
it("honours partSizeBytes and queueSize overrides", async () => {
const done = vi.fn().mockResolvedValue({ ETag: "x" });
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
const { writeStream, completionPromise } = createS3MultipartUploadSink({
client: {} as never,
bucket: "b",
key: "k",
partSizeBytes: 5 * 1024 * 1024,
queueSize: 2,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
uploadCtor: UploadStub as any,
});
writeStream.end();
await completionPromise;
const ctorArg = UploadStub.mock.calls[0]![0] as { partSize: number; queueSize: number };
expect(ctorArg.partSize).toBe(5 * 1024 * 1024);
expect(ctorArg.queueSize).toBe(2);
});
});
@@ -0,0 +1,77 @@
import { PassThrough } from "node:stream";
import type { S3Client } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
/**
* S3 multipart-upload sink for streaming exports. Returns a `writeStream`
* that callers pipe their serialised output into and a `completionPromise`
* that resolves with the S3 ETag once the multipart upload finalises.
*
* The underlying `@aws-sdk/lib-storage` `Upload` class handles part
* batching, retries, and `AbortMultipartUpload` on failure, so callers do
* not need to track orphaned multipart uploads themselves.
*/
export interface CreateS3MultipartUploadSinkOptions {
client: S3Client;
bucket: string;
key: string;
contentType?: string;
metadata?: Record<string, string>;
/** Override the part size; defaults to 8 MB (S3 minimum is 5 MB). */
partSizeBytes?: number;
/** Override concurrent part-upload concurrency; defaults to 4. */
queueSize?: number;
/** Test seam: inject an alternative Upload implementation. */
uploadCtor?: typeof Upload;
}
export interface S3MultipartUploadSink {
writeStream: PassThrough;
completionPromise: Promise<{ bucket: string; key: string; etag: string | null }>;
}
const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
const DEFAULT_QUEUE_SIZE = 4;
export function createS3MultipartUploadSink(options: CreateS3MultipartUploadSinkOptions): S3MultipartUploadSink {
const writeStream = new PassThrough();
// PassThrough emits 'error' synchronously on destroy(err). Producers
// surface failures through completionPromise, not the stream, so attach
// a noop listener to keep Node from treating the destroy-triggered error
// as an uncaught exception.
writeStream.on("error", () => undefined);
const Ctor = options.uploadCtor ?? Upload;
const upload = new Ctor({
client: options.client,
partSize: options.partSizeBytes ?? DEFAULT_PART_SIZE_BYTES,
queueSize: options.queueSize ?? DEFAULT_QUEUE_SIZE,
params: {
Bucket: options.bucket,
Key: options.key,
Body: writeStream,
ContentType: options.contentType,
Metadata: options.metadata,
},
});
const completionPromise = (async () => {
const result = (await upload.done()) as { ETag?: string };
return {
bucket: options.bucket,
key: options.key,
etag: result.ETag ?? null,
};
})();
// Surface upload errors on the write stream so producers see them via
// standard 'error' propagation instead of an unhandled rejection.
completionPromise.catch((error: unknown) => {
if (!writeStream.destroyed) {
writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
}
});
return { writeStream, completionPromise };
}
@@ -0,0 +1,23 @@
const FORMULA_NEUTRALIZER = "'";
const FORMULA_PREFIXES = new Set(["=", "+", "-", "@"]);
const CONTROL_PREFIXES = new Set(["\t", "\r", "\n"]);
export function neutralizeSpreadsheetFormula(value: string): string {
return hasUnsafeSpreadsheetPrefix(value) ? `${FORMULA_NEUTRALIZER}${value}` : value;
}
function hasUnsafeSpreadsheetPrefix(value: string): boolean {
for (const char of value) {
if (FORMULA_PREFIXES.has(char) || CONTROL_PREFIXES.has(char)) {
return true;
}
if (char.trim() === "") {
continue;
}
return false;
}
return false;
}
@@ -0,0 +1,101 @@
import { describe, expect, it } from "vitest";
import { ExportTableValidationError, normalizeExportTable } from "./types.js";
describe("normalizeExportTable", () => {
it("infers deterministic columns from row key insertion order", () => {
const table = normalizeExportTable({
rows: [
{ id: 1, name: "Ada" },
{ email: "ada@example.com", id: 2 },
],
});
expect(table.columns).toEqual([
{ key: "id", label: "id" },
{ key: "name", label: "name" },
{ key: "email", label: "email" },
]);
expect(table.rows).toEqual([
{ id: "1", name: "Ada", email: "" },
{ id: "2", name: "", email: "ada@example.com" },
]);
});
it("preserves explicit column labels and order while projecting fields", () => {
const table = normalizeExportTable({
columns: [
{ key: "email", label: "Email address" },
"id",
],
rows: [{ id: 7, name: "Grace", email: "grace@example.com" }],
});
expect(table.columns).toEqual([
{ key: "email", label: "Email address" },
{ key: "id", label: "id" },
]);
expect(table.rows).toEqual([{ email: "grace@example.com", id: "7" }]);
});
it("rejects duplicate column keys before rows are normalized", () => {
expect(() =>
normalizeExportTable({
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
});
it("rejects malformed non-array rows", () => {
expect(() =>
normalizeExportTable({
rows: { id: 1 },
} as never),
).toThrow(new ExportTableValidationError("Export table rows must be an array."));
});
it("rejects malformed row entries", () => {
expect(() =>
normalizeExportTable({
rows: [{ id: 1 }, null],
} as never),
).toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
});
it("normalizes dates and scalar values into stable strings", () => {
const table = normalizeExportTable({
rows: [
{
active: true,
amount: 42.5,
big: 9007199254740993n,
createdAt: new Date("2024-01-02T03:04:05.006Z"),
empty: null,
missing: undefined,
},
],
});
expect(table.rows).toEqual([
{
active: "true",
amount: "42.5",
big: "9007199254740993",
createdAt: "2024-01-02T03:04:05.006Z",
empty: "",
missing: "",
},
]);
});
it("preserves row counts after projection", () => {
const table = normalizeExportTable({
columns: ["id"],
rows: [{ id: 1 }, { id: 2, ignored: "value" }, { id: 3 }],
});
expect(table.rowCount).toBe(3);
expect(table.rows).toHaveLength(3);
});
});
@@ -0,0 +1,120 @@
export type ExportPrimitive = string | number | boolean | bigint | Date | null | undefined;
export type ExportRecord = Record<string, ExportPrimitive>;
export type ExportColumn = {
key: string;
label?: string;
};
export type ExportTable = {
columns?: readonly (string | ExportColumn)[];
rows: readonly ExportRecord[];
};
export type NormalizedExportColumn = {
key: string;
label: string;
};
export type NormalizedExportRow = Record<string, string>;
export type NormalizedExportTable = {
columns: readonly NormalizedExportColumn[];
rows: readonly NormalizedExportRow[];
rowCount: number;
};
export class ExportTableValidationError extends Error {
constructor(message: string) {
super(message);
this.name = "ExportTableValidationError";
}
}
export function normalizeExportTable(table: ExportTable): NormalizedExportTable {
if (!Array.isArray(table.rows)) {
throw new ExportTableValidationError("Export table rows must be an array.");
}
assertRowsAreRecords(table.rows);
const columns = normalizeColumns(table);
const rows = table.rows.map((row, index) => normalizeRow(row, index, columns));
return {
columns,
rows,
rowCount: table.rows.length,
};
}
function normalizeColumns(table: ExportTable): NormalizedExportColumn[] {
const columns = table.columns === undefined ? inferColumns(table.rows) : table.columns.map(normalizeColumn);
const seenKeys = new Set<string>();
for (const column of columns) {
if (seenKeys.has(column.key)) {
throw new ExportTableValidationError(`Export table columns contain duplicate key "${column.key}".`);
}
seenKeys.add(column.key);
}
return columns;
}
function inferColumns(rows: readonly ExportRecord[]): NormalizedExportColumn[] {
const seenKeys = new Set<string>();
const columns: NormalizedExportColumn[] = [];
for (const row of rows) {
for (const key of Object.keys(row)) {
if (!seenKeys.has(key)) {
seenKeys.add(key);
columns.push({ key, label: key });
}
}
}
return columns;
}
function assertRowsAreRecords(rows: readonly ExportRecord[]): void {
rows.forEach((row, index) => {
if (row === null || typeof row !== "object" || Array.isArray(row)) {
throw new ExportTableValidationError(`Export table row at index ${index} must be an object record.`);
}
});
}
function normalizeColumn(column: string | ExportColumn): NormalizedExportColumn {
if (typeof column === "string") {
return { key: column, label: column };
}
return { key: column.key, label: column.label ?? column.key };
}
function normalizeRow(
row: ExportRecord,
index: number,
columns: readonly NormalizedExportColumn[],
): NormalizedExportRow {
if (row === null || typeof row !== "object" || Array.isArray(row)) {
throw new ExportTableValidationError(`Export table row at index ${index} must be an object record.`);
}
return Object.fromEntries(columns.map((column) => [column.key, normalizePrimitive(row[column.key])]));
}
function normalizePrimitive(value: ExportPrimitive): string {
if (value === null || value === undefined) {
return "";
}
if (value instanceof Date) {
return value.toISOString();
}
return String(value);
}
@@ -0,0 +1,153 @@
import ExcelJS from "exceljs";
import { describe, expect, it } from "vitest";
import { ExportTableValidationError } from "../types.js";
import { buildXlsxExportArtifact, serializeExportTableToXlsx } from "./index.js";
async function readWorkbook(bytes: Uint8Array): Promise<ExcelJS.Workbook> {
const workbook = new ExcelJS.Workbook();
const buffer = Buffer.from(bytes);
await workbook.xlsx.load(buffer as unknown as ArrayBuffer);
return workbook;
}
describe("@toolctrl/exports/xlsx public entrypoint", () => {
it("exposes XLSX helpers through the dedicated subpath", async () => {
const exportsModule = await import("./index.js");
expect(Object.keys(exportsModule).sort()).toEqual([
"ExportTableValidationError",
"ExportXlsxRowLimitExceededError",
"XLSX_STREAM_HARD_ROW_LIMIT",
"XLSX_STREAM_THRESHOLD",
"buildXlsxExportArtifact",
"serializeExportTableToXlsx",
"serializeExportTableToXlsxStream",
]);
});
});
describe("serializeExportTableToXlsx", () => {
it("uses explicit header labels and column order", async () => {
const bytes = await serializeExportTableToXlsx({
columns: [
{ key: "email", label: "Email address" },
{ key: "id", label: "Identifier" },
"active",
],
rows: [{ id: 7, email: "ada@example.com", active: true, ignored: "not exported" }],
});
const worksheet = (await readWorkbook(bytes)).getWorksheet("Export");
expect(worksheet?.getRow(1).values).toEqual([undefined, "Email address", "Identifier", "active"]);
expect(worksheet?.getRow(2).values).toEqual([undefined, "ada@example.com", "7", "true"]);
});
it("serializes null and missing values as blank workbook cells", async () => {
const bytes = await serializeExportTableToXlsx({
columns: ["id", "empty", "missing"],
rows: [{ id: 1, empty: null, missing: undefined }],
});
const worksheet = (await readWorkbook(bytes)).getWorksheet("Export");
expect(worksheet?.getRow(2).getCell(1).value).toBe("1");
expect(worksheet?.getRow(2).getCell(2).value).toBe("");
expect(worksheet?.getRow(2).getCell(3).value).toBe("");
});
it("stores spreadsheet formula prefixes as neutralized literal strings", async () => {
const bytes = await serializeExportTableToXlsx({
columns: [
{ key: "equals", label: "=header" },
"plus",
"minus",
"at",
"spaceFormula",
"spaceTab",
"spaceCr",
"spaceLf",
"safe",
],
rows: [
{
equals: "=1+1",
plus: "+1",
minus: "-1",
at: "@cmd",
spaceFormula: " =1+1",
spaceTab: " \tvalue",
spaceCr: " \rvalue",
spaceLf: " \nvalue",
safe: " value",
},
],
});
const worksheet = (await readWorkbook(bytes)).getWorksheet("Export");
expect(worksheet?.getRow(1).values).toEqual([
undefined,
"'=header",
"plus",
"minus",
"at",
"spaceFormula",
"spaceTab",
"spaceCr",
"spaceLf",
"safe",
]);
expect(worksheet?.getRow(2).values).toEqual([
undefined,
"'=1+1",
"'+1",
"'-1",
"'@cmd",
"' =1+1",
"' \tvalue",
"' \nvalue",
"' \nvalue",
" value",
]);
expect(worksheet?.getRow(2).getCell(1).value).toBe("'=1+1");
});
it("rejects malformed row input through the XLSX API", async () => {
await expect(
serializeExportTableToXlsx({
rows: [{ id: 1 }, null],
} as never),
).rejects.toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
});
it("rejects duplicate columns through the XLSX API", async () => {
await expect(
serializeExportTableToXlsx({
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).rejects.toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
});
});
describe("buildXlsxExportArtifact", () => {
it("returns parseable workbook bytes with binary export metadata", async () => {
const artifact = await buildXlsxExportArtifact({
columns: ["id"],
rows: [{ id: 1 }, { id: 2 }],
});
expect(artifact.contentType).toBe("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
expect(artifact.extension).toBe("xlsx");
expect(artifact.rowCount).toBe(2);
expect(artifact.byteLength).toBe(artifact.bytes.byteLength);
expect(artifact.byteLength).toBeGreaterThan(0);
const worksheet = (await readWorkbook(artifact.bytes)).getWorksheet("Export");
expect(worksheet?.getRow(1).values).toEqual([undefined, "id"]);
expect(worksheet?.getRow(2).values).toEqual([undefined, "1"]);
expect(worksheet?.getRow(3).values).toEqual([undefined, "2"]);
});
});
@@ -0,0 +1,227 @@
import type { Writable } from "node:stream";
import ExcelJS from "exceljs";
import { XLSX_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_THRESHOLD } from "../format-caps.js";
import { neutralizeSpreadsheetFormula } from "../spreadsheet-safety.js";
import { type ExportColumn, type ExportRecord, type ExportTable, normalizeExportTable } from "../types.js";
export { ExportTableValidationError } from "../types.js";
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
const XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
/**
* Hard cap (per spec) — even an Enterprise tenant cannot stream more than
* 500k rows into a single XLSX artifact. ExcelJS' stream-writer has
* limitations that make larger workbooks unreliable.
*/
export { XLSX_STREAM_HARD_ROW_LIMIT };
/**
* Row count above which the streamer switches from the in-memory ExcelJS
* Workbook to the stream-writer. Stream-writer cells cannot be re-read
* after commit, so we keep the simpler in-memory path for small exports.
*/
export { XLSX_STREAM_THRESHOLD };
export class ExportXlsxRowLimitExceededError extends Error {
readonly code = "EXPORT_ROW_LIMIT_EXCEEDED";
constructor(rowCount: number, limit: number) {
super(`XLSX export exceeds the ${limit}-row hard cap (got ${rowCount}); split the export or use CSV/NDJSON.`);
this.name = "ExportXlsxRowLimitExceededError";
}
}
export type XlsxExportOptions = {
sheetName?: string;
};
export type XlsxExportArtifact = {
bytes: Uint8Array;
contentType: typeof XLSX_CONTENT_TYPE;
extension: "xlsx";
rowCount: number;
byteLength: number;
};
export type XlsxStreamTable = {
columns?: readonly (string | ExportColumn)[];
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
estimatedRowCount?: number;
};
export type XlsxStreamOptions = XlsxExportOptions & {
strategy?: "auto" | "stream" | "memory";
hardRowLimit?: number;
};
export async function serializeExportTableToXlsx(
table: ExportTable,
options: XlsxExportOptions = {},
): Promise<Uint8Array> {
const normalizedTable = normalizeExportTable(table);
const workbook = new ExcelJS.Workbook();
const worksheet = workbook.addWorksheet(options.sheetName ?? "Export");
worksheet.addRow(normalizedTable.columns.map((column) => neutralizeSpreadsheetFormula(column.label)));
for (const row of normalizedTable.rows) {
worksheet.addRow(normalizedTable.columns.map((column) => neutralizeSpreadsheetFormula(row[column.key])));
}
const buffer = await workbook.xlsx.writeBuffer();
return buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
}
/**
* Streaming XLSX writer. Picks ExcelJS' stream-writer for large exports
* (above {@link XLSX_STREAM_THRESHOLD}) and the in-memory workbook for
* small ones. Always enforces the {@link XLSX_STREAM_HARD_ROW_LIMIT} hard
* cap — independent of tenant plan — because the format is unreliable
* above 500k rows.
*/
export async function serializeExportTableToXlsxStream(
writeStream: Writable,
table: XlsxStreamTable,
options: XlsxStreamOptions = {},
): Promise<{ rowCount: number; strategy: "stream" | "memory" }> {
const hardLimit = options.hardRowLimit ?? XLSX_STREAM_HARD_ROW_LIMIT;
if (table.estimatedRowCount !== undefined && table.estimatedRowCount > hardLimit) {
throw new ExportXlsxRowLimitExceededError(table.estimatedRowCount, hardLimit);
}
const columns = resolveColumns(table);
const strategy = pickStrategy(table.estimatedRowCount, options.strategy);
if (strategy === "stream") {
return runStreamWriter(writeStream, table, columns, options, hardLimit);
}
return runMemoryWriter(writeStream, table, columns, options, hardLimit);
}
export async function buildXlsxExportArtifact(
table: ExportTable,
options: XlsxExportOptions = {},
): Promise<XlsxExportArtifact> {
const normalizedTable = normalizeExportTable(table);
const bytes = await serializeExportTableToXlsx(table, options);
return {
bytes,
contentType: XLSX_CONTENT_TYPE,
extension: "xlsx",
rowCount: normalizedTable.rowCount,
byteLength: bytes.byteLength,
};
}
function pickStrategy(estimated: number | undefined, override: XlsxStreamOptions["strategy"]): "stream" | "memory" {
if (override === "stream" || override === "memory") return override;
if (estimated !== undefined && estimated > XLSX_STREAM_THRESHOLD) return "stream";
return "memory";
}
function resolveColumns(table: XlsxStreamTable): { key: string; label: string }[] {
if (table.columns === undefined) {
throw new Error("Streaming XLSX writer requires explicit columns; cannot infer from an async row source.");
}
const seen = new Set<string>();
return table.columns.map((column) => {
const normalized = typeof column === "string" ? { key: column, label: column } : { key: column.key, label: column.label ?? column.key };
if (seen.has(normalized.key)) {
throw new Error(`Export table columns contain duplicate key "${normalized.key}".`);
}
seen.add(normalized.key);
return normalized;
});
}
async function runStreamWriter(
writeStream: Writable,
table: XlsxStreamTable,
columns: { key: string; label: string }[],
options: XlsxStreamOptions,
hardLimit: number,
): Promise<{ rowCount: number; strategy: "stream" | "memory" }> {
const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: writeStream });
const worksheet = workbook.addWorksheet(options.sheetName ?? "Export");
worksheet.addRow(columns.map((column) => neutralizeSpreadsheetFormula(column.label))).commit();
let rowCount = 0;
for await (const row of toAsyncIterable(table.rows)) {
if (rowCount >= hardLimit) {
throw new ExportXlsxRowLimitExceededError(rowCount + 1, hardLimit);
}
worksheet
.addRow(columns.map((column) => neutralizeSpreadsheetFormula(stringifyValue((row as ExportRecord)[column.key]))))
.commit();
rowCount += 1;
}
worksheet.commit();
await workbook.commit();
return { rowCount, strategy: "stream" };
}
async function runMemoryWriter(
writeStream: Writable,
table: XlsxStreamTable,
columns: { key: string; label: string }[],
options: XlsxStreamOptions,
hardLimit: number,
): Promise<{ rowCount: number; strategy: "stream" | "memory" }> {
const workbook = new ExcelJS.Workbook();
const worksheet = workbook.addWorksheet(options.sheetName ?? "Export");
worksheet.addRow(columns.map((column) => neutralizeSpreadsheetFormula(column.label)));
let rowCount = 0;
for await (const row of toAsyncIterable(table.rows)) {
if (rowCount >= hardLimit) {
throw new ExportXlsxRowLimitExceededError(rowCount + 1, hardLimit);
}
worksheet.addRow(columns.map((column) => neutralizeSpreadsheetFormula(stringifyValue((row as ExportRecord)[column.key]))));
rowCount += 1;
}
const buffer = await workbook.xlsx.writeBuffer();
const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
await writeBuffered(writeStream, Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
return { rowCount, strategy: "memory" };
}
function stringifyValue(value: unknown): string {
if (value === null || value === undefined) return "";
if (value instanceof Date) return value.toISOString();
if (typeof value === "bigint") return value.toString();
return String(value);
}
async function writeBuffered(writeStream: Writable, chunk: Buffer): Promise<void> {
if (!writeStream.write(chunk)) {
await new Promise<void>((resolve, reject) => {
const onDrain = () => {
cleanup();
resolve();
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const cleanup = () => {
writeStream.off("drain", onDrain);
writeStream.off("error", onError);
};
writeStream.once("drain", onDrain);
writeStream.once("error", onError);
});
}
}
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
if (Symbol.asyncIterator in (source as object)) {
return source as AsyncIterable<T>;
}
return (async function* () {
for (const item of source as Iterable<T>) {
yield item;
}
})();
}
@@ -0,0 +1,123 @@
import { PassThrough } from "node:stream";
import ExcelJS from "exceljs";
import { describe, expect, it } from "vitest";
import {
ExportXlsxRowLimitExceededError,
XLSX_STREAM_THRESHOLD,
serializeExportTableToXlsxStream,
} from "./index.js";
async function collect(stream: PassThrough): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
async function readBack(buffer: Buffer): Promise<{ rows: unknown[][] }> {
const workbook = new ExcelJS.Workbook();
// ExcelJS' .d.ts declares `Buffer` from an old @types/node and is
// incompatible with the modern Buffer<ArrayBufferLike>; runtime accepts
// either, so cast through `unknown` to satisfy the type-checker.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await workbook.xlsx.load(buffer as any);
const sheet = workbook.worksheets[0]!;
const rows: unknown[][] = [];
sheet.eachRow({ includeEmpty: false }, (row) => {
const values = row.values as unknown[];
rows.push(values.slice(1));
});
return { rows };
}
describe("serializeExportTableToXlsxStream", () => {
it("writes a small workbook via the in-memory strategy and round-trips", async () => {
const sink = new PassThrough();
const collected = collect(sink);
const result = await serializeExportTableToXlsxStream(sink, {
columns: [
{ key: "id", label: "Identifier" },
{ key: "name", label: "Name" },
],
rows: [
{ id: 1, name: "Ada" },
{ id: 2, name: "Grace" },
],
});
sink.end();
expect(result).toEqual({ rowCount: 2, strategy: "memory" });
const parsed = await readBack(await collected);
expect(parsed.rows[0]).toEqual(["Identifier", "Name"]);
expect(parsed.rows[1]).toEqual(["1", "Ada"]);
expect(parsed.rows[2]).toEqual(["2", "Grace"]);
});
it("switches to stream-writer when estimatedRowCount exceeds the threshold", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
async function* rows(count: number) {
for (let i = 0; i < count; i += 1) {
yield { id: i, value: `row-${i}` };
}
}
const result = await serializeExportTableToXlsxStream(sink, {
columns: ["id", "value"],
estimatedRowCount: XLSX_STREAM_THRESHOLD + 1,
rows: rows(50),
});
sink.end();
expect(result.strategy).toBe("stream");
expect(result.rowCount).toBe(50);
});
it("rejects estimatedRowCount above the hard cap before doing any work", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
await expect(
serializeExportTableToXlsxStream(sink, {
columns: ["id"],
rows: [],
estimatedRowCount: 600_000,
}),
).rejects.toBeInstanceOf(ExportXlsxRowLimitExceededError);
});
it("enforces the hard cap mid-stream when the iterator overshoots the estimate", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
async function* rows() {
for (let i = 0; i < 10; i += 1) {
yield { id: i };
}
}
await expect(
serializeExportTableToXlsxStream(
sink,
{ columns: ["id"], rows: rows() },
{ strategy: "memory", hardRowLimit: 3 },
),
).rejects.toBeInstanceOf(ExportXlsxRowLimitExceededError);
});
it("rejects duplicate column keys", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
await expect(
serializeExportTableToXlsxStream(sink, {
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).rejects.toThrow('Export table columns contain duplicate key "id".');
});
});
@@ -0,0 +1,13 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"allowJs": false,
"allowImportingTsExtensions": false,
"lib": ["ES2022"],
"module": "NodeNext",
"moduleResolution": "NodeNext",
"noEmit": true,
"types": ["node"]
},
"include": ["src/**/*.ts"]
}