Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "@toolctrl/exports",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./audit-pdf": "./src/audit-pdf.ts",
|
||||
"./csv": "./src/csv/index.ts",
|
||||
"./json": "./src/json/index.ts",
|
||||
"./limits": "./src/limits.ts",
|
||||
"./pdf": "./src/pdf/index.ts",
|
||||
"./s3-multipart": "./src/sinks/s3-multipart.ts",
|
||||
"./types": "./src/types.ts",
|
||||
"./xlsx": "./src/xlsx/index.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "vitest run src",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.1045.0",
|
||||
"@aws-sdk/lib-storage": "3.1045.0",
|
||||
"@react-pdf/renderer": "^4.3.2",
|
||||
"exceljs": "^4.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"aws-sdk-client-mock": "^4.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
AUDIT_PDF_HARD_ROW_LIMIT,
|
||||
buildAuditPdfDocument,
|
||||
serializeAuditEventsToPdf,
|
||||
summarizeAuditPdfFilters,
|
||||
type AuditPdfEvent,
|
||||
type AuditPdfReportInput,
|
||||
} from "./audit-pdf.js";
|
||||
import { ExportPdfRowLimitExceededError } from "./pdf/index.js";
|
||||
|
||||
function makeEvent(over: Partial<AuditPdfEvent> = {}): AuditPdfEvent {
|
||||
return {
|
||||
id: "audit_1",
|
||||
createdAt: new Date("2026-05-17T00:00:00.000Z"),
|
||||
tenantId: "tenant_1",
|
||||
actorType: "user",
|
||||
actorId: "user_1",
|
||||
actorEmail: null,
|
||||
actorIp: null,
|
||||
actorUserAgent: null,
|
||||
action: "widget.created",
|
||||
targetType: "widget",
|
||||
targetId: "widget_1",
|
||||
requestId: null,
|
||||
sourceModule: "crm",
|
||||
outcome: "success",
|
||||
dataSensitivity: "internal",
|
||||
regulatoryScope: ["soc2"],
|
||||
metadata: null,
|
||||
previousHash: null,
|
||||
eventHash: "deadbeef0123456789abcdef0123456789abcdef0123456789abcdef01234567",
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
function makeInput(over: Partial<AuditPdfReportInput> = {}): AuditPdfReportInput {
|
||||
return {
|
||||
tenantId: "tenant_1",
|
||||
from: "2026-05-01",
|
||||
to: "2026-05-17",
|
||||
filterSummary: "regulatoryScope=soc2",
|
||||
integrity: { ok: true, inspected: 1, firstBreakId: null, reason: null },
|
||||
events: [makeEvent()],
|
||||
generatedAt: new Date("2026-05-17T12:34:56.000Z"),
|
||||
generatedBy: "ops-bot",
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
function collectText(node: unknown): string {
|
||||
if (typeof node === "string" || typeof node === "number") {
|
||||
return String(node);
|
||||
}
|
||||
if (!node || typeof node !== "object") {
|
||||
return "";
|
||||
}
|
||||
const props = (node as { props?: { children?: unknown } }).props;
|
||||
const children = props?.children;
|
||||
if (Array.isArray(children)) {
|
||||
return children.map(collectText).join(" ");
|
||||
}
|
||||
return collectText(children);
|
||||
}
|
||||
|
||||
describe("summarizeAuditPdfFilters", () => {
|
||||
it("joins regulatoryScope, actorType and outcome", () => {
|
||||
expect(summarizeAuditPdfFilters({ regulatoryScope: ["soc2", "gdpr"], actorType: "admin", outcome: "blocked" })).toBe(
|
||||
"regulatoryScope=soc2|gdpr actorType=admin outcome=blocked"
|
||||
);
|
||||
});
|
||||
it("returns empty when nothing is set", () => {
|
||||
expect(summarizeAuditPdfFilters({})).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildAuditPdfDocument", () => {
|
||||
it("returns a React element with the expected title", () => {
|
||||
const doc = buildAuditPdfDocument(makeInput({ title: "Custom title" }));
|
||||
expect(doc).toBeTruthy();
|
||||
expect((doc as unknown as { props: { title: string } }).props.title).toBe("Custom title");
|
||||
});
|
||||
|
||||
it("accepts localized labels for visible report copy", () => {
|
||||
const doc = buildAuditPdfDocument(makeInput({
|
||||
labels: {
|
||||
title: "Audit-Compliance-Bericht",
|
||||
tenant: "Konto",
|
||||
platformTenant: "Plattform",
|
||||
range: "Zeitraum",
|
||||
filters: "Filter",
|
||||
generatedBy: "Erstellt von",
|
||||
generatedAt: "Erstellt am",
|
||||
hashChainIntegrity: "Hash-Chain-Integritaet",
|
||||
events: (count) => `Ereignisse (${count})`,
|
||||
integrityOk: (inspected) => `verifyAuditChain OK - ${inspected} Ereignisse geprueft.`,
|
||||
columns: {
|
||||
actorType: "Akteur",
|
||||
outcome: "Ergebnis",
|
||||
regulatoryScope: "Geltungsbereich",
|
||||
},
|
||||
},
|
||||
}));
|
||||
const text = collectText(doc);
|
||||
|
||||
expect((doc as unknown as { props: { title: string } }).props.title).toBe("Audit-Compliance-Bericht");
|
||||
expect(text).toContain("Konto: tenant_1");
|
||||
expect(text).toContain("Hash-Chain-Integritaet");
|
||||
expect(text).toContain("Ereignisse (1)");
|
||||
expect(text).toContain("Akteur");
|
||||
expect(text).toContain("Ergebnis");
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeAuditEventsToPdf", () => {
|
||||
it("renders a real PDF byte stream", async () => {
|
||||
const bytes = await serializeAuditEventsToPdf(makeInput());
|
||||
expect(bytes).toBeInstanceOf(Uint8Array);
|
||||
expect(bytes.length).toBeGreaterThan(0);
|
||||
// PDFs start with %PDF
|
||||
const header = Buffer.from(bytes.slice(0, 4)).toString("utf8");
|
||||
expect(header).toBe("%PDF");
|
||||
});
|
||||
|
||||
it("throws when event count exceeds the hard cap", async () => {
|
||||
const tooMany: AuditPdfEvent[] = Array.from({ length: AUDIT_PDF_HARD_ROW_LIMIT + 1 }, (_, i) => makeEvent({ id: `e_${i}` }));
|
||||
await expect(serializeAuditEventsToPdf(makeInput({ events: tooMany }))).rejects.toBeInstanceOf(ExportPdfRowLimitExceededError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* Compliance-Audit PDF report renderer.
|
||||
*
|
||||
* Renders an AuditEvent slice into a tamper-evident PDF report suitable
|
||||
* for SOC2 / HIPAA / GDPR auditor evidence packs. The first page is a
|
||||
* cover page with tenant + range + filter summary + verifyAuditChain
|
||||
* integrity outcome; following pages list events as a tabular log.
|
||||
*
|
||||
* The renderer is host-agnostic and async — both the operator CLI
|
||||
* (`toolctrl audit export-pdf …`) and the admin web export route use
|
||||
* it via `serializeAuditEventsToPdf` / `serializeAuditEventsToPdfStream`.
|
||||
*
|
||||
* See docs/improvement-plan/10-compliance-polish/compliance-audit-export.md.
|
||||
*/
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import React from "react";
|
||||
import { Document, Page, StyleSheet, Text, View, renderToBuffer, renderToStream } from "@react-pdf/renderer";
|
||||
|
||||
import { PDF_STREAM_HARD_ROW_LIMIT, ExportPdfRowLimitExceededError } from "./pdf/index.js";
|
||||
|
||||
export const AUDIT_PDF_CONTENT_TYPE = "application/pdf" as const;
|
||||
export const AUDIT_PDF_HARD_ROW_LIMIT = PDF_STREAM_HARD_ROW_LIMIT;
|
||||
|
||||
export type AuditPdfEvent = {
|
||||
id: string;
|
||||
createdAt: Date | string;
|
||||
tenantId: string | null;
|
||||
actorType: string;
|
||||
actorId: string | null;
|
||||
actorEmail: string | null;
|
||||
actorIp: string | null;
|
||||
actorUserAgent: string | null;
|
||||
action: string;
|
||||
targetType: string | null;
|
||||
targetId: string | null;
|
||||
requestId: string | null;
|
||||
sourceModule: string | null;
|
||||
outcome: string;
|
||||
dataSensitivity: string;
|
||||
regulatoryScope: ReadonlyArray<string>;
|
||||
metadata: unknown;
|
||||
previousHash: string | null;
|
||||
eventHash: string | null;
|
||||
};
|
||||
|
||||
export type AuditPdfIntegrity = {
|
||||
ok: boolean;
|
||||
inspected: number;
|
||||
firstBreakId: string | null;
|
||||
reason: string | null;
|
||||
};
|
||||
|
||||
export type AuditPdfReportInput = {
|
||||
/** Tenant identifier; null = platform-wide. */
|
||||
tenantId: string | null;
|
||||
/** ISO-8601 lower bound used by the export query. */
|
||||
from: string | null;
|
||||
/** ISO-8601 upper bound used by the export query. */
|
||||
to: string | null;
|
||||
/** Free-text filter summary (regulatoryScope/actorType/outcome). */
|
||||
filterSummary: string;
|
||||
/** Pre-computed verifyAuditChain integrity result. */
|
||||
integrity: AuditPdfIntegrity;
|
||||
/** Events in chronological order. */
|
||||
events: ReadonlyArray<AuditPdfEvent>;
|
||||
/** Operator-facing report title (defaults to "Audit-Compliance Report"). */
|
||||
title?: string;
|
||||
/** Human display name of the operator who generated the report. */
|
||||
generatedBy?: string;
|
||||
/** Timestamp of report generation; defaults to new Date(). */
|
||||
generatedAt?: Date;
|
||||
/** Visible report labels; pass localized labels from the caller when needed. */
|
||||
labels?: AuditPdfLabelOverrides;
|
||||
};
|
||||
|
||||
type AuditPdfColumnKey = "createdAt" | "actorType" | "actorId" | "action" | "targetType" | "outcome" | "regulatoryScope" | "eventHash";
|
||||
|
||||
export type AuditPdfLabels = {
|
||||
title: string;
|
||||
tenant: string;
|
||||
platformTenant: string;
|
||||
range: string;
|
||||
openRange: string;
|
||||
filters: string;
|
||||
noFilters: string;
|
||||
generatedBy: string;
|
||||
generatedAt: string;
|
||||
hashChainIntegrity: string;
|
||||
events: (count: number) => string;
|
||||
integrityOk: (inspected: number) => string;
|
||||
integrityBroken: (args: { inspected: number; firstBreakId: string; reason: string }) => string;
|
||||
columns: Record<AuditPdfColumnKey, string>;
|
||||
};
|
||||
|
||||
export type AuditPdfLabelOverrides = Partial<Omit<AuditPdfLabels, "columns">> & {
|
||||
columns?: Partial<AuditPdfLabels["columns"]>;
|
||||
};
|
||||
|
||||
const labelText = (value: string) => value;
|
||||
|
||||
const DEFAULT_AUDIT_PDF_LABELS: AuditPdfLabels = {
|
||||
title: labelText("Audit compliance report"),
|
||||
tenant: "Tenant",
|
||||
platformTenant: "platform",
|
||||
range: "Range",
|
||||
openRange: "open",
|
||||
filters: "Filters",
|
||||
noFilters: "none",
|
||||
generatedBy: "Generated by",
|
||||
generatedAt: "Generated at",
|
||||
hashChainIntegrity: "Hash-chain integrity",
|
||||
events: (count) => `Events (${count})`,
|
||||
integrityOk: (inspected) => `verifyAuditChain OK - inspected ${inspected} events.`,
|
||||
integrityBroken: ({ inspected, firstBreakId, reason }) => `verifyAuditChain BROKEN - inspected ${inspected} events. First-break id=${firstBreakId} reason=${reason}.`,
|
||||
columns: {
|
||||
createdAt: "Time",
|
||||
actorType: "Actor",
|
||||
actorId: "Actor ID",
|
||||
action: "Action",
|
||||
targetType: "Target",
|
||||
outcome: "Outcome",
|
||||
regulatoryScope: "Scope",
|
||||
eventHash: "Hash",
|
||||
},
|
||||
};
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
page: { padding: 32, fontSize: 9, fontFamily: "Helvetica" },
|
||||
title: { fontSize: 18, fontWeight: 700, marginBottom: 12 },
|
||||
subtitle: { fontSize: 12, fontWeight: 700, marginTop: 12, marginBottom: 6 },
|
||||
meta: { marginBottom: 4 },
|
||||
integrityOk: { color: "#047857", fontWeight: 700 },
|
||||
integrityBroken: { color: "#b91c1c", fontWeight: 700 },
|
||||
table: {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
width: "100%",
|
||||
borderTopWidth: 1,
|
||||
borderLeftWidth: 1,
|
||||
borderColor: "#d1d5db",
|
||||
marginTop: 8,
|
||||
},
|
||||
row: { display: "flex", flexDirection: "row" },
|
||||
headerCell: {
|
||||
flexGrow: 1,
|
||||
flexBasis: 0,
|
||||
padding: 4,
|
||||
fontWeight: 700,
|
||||
backgroundColor: "#f3f4f6",
|
||||
borderRightWidth: 1,
|
||||
borderBottomWidth: 1,
|
||||
borderColor: "#d1d5db",
|
||||
},
|
||||
cell: {
|
||||
flexGrow: 1,
|
||||
flexBasis: 0,
|
||||
padding: 4,
|
||||
borderRightWidth: 1,
|
||||
borderBottomWidth: 1,
|
||||
borderColor: "#d1d5db",
|
||||
},
|
||||
});
|
||||
|
||||
const COLUMNS: ReadonlyArray<{ key: AuditPdfColumnKey; render?: (event: AuditPdfEvent) => string }> = [
|
||||
{ key: "createdAt", render: (e) => (e.createdAt instanceof Date ? e.createdAt.toISOString() : String(e.createdAt)) },
|
||||
{ key: "actorType" },
|
||||
{ key: "actorId", render: (e) => e.actorId ?? "-" },
|
||||
{ key: "action" },
|
||||
{ key: "targetType", render: (e) => `${e.targetType ?? "-"}/${e.targetId ?? "-"}` },
|
||||
{ key: "outcome" },
|
||||
{ key: "regulatoryScope", render: (e) => e.regulatoryScope.join(",") || "-" },
|
||||
{ key: "eventHash", render: (e) => (e.eventHash ? e.eventHash.slice(0, 16) : "-") },
|
||||
];
|
||||
|
||||
function resolveAuditPdfLabels(input?: AuditPdfLabelOverrides): AuditPdfLabels {
|
||||
return {
|
||||
...DEFAULT_AUDIT_PDF_LABELS,
|
||||
...input,
|
||||
columns: {
|
||||
...DEFAULT_AUDIT_PDF_LABELS.columns,
|
||||
...input?.columns,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildIntegrityLine(integrity: AuditPdfIntegrity, labels: AuditPdfLabels): string {
|
||||
if (integrity.ok) {
|
||||
return labels.integrityOk(integrity.inspected);
|
||||
}
|
||||
return labels.integrityBroken({
|
||||
inspected: integrity.inspected,
|
||||
firstBreakId: integrity.firstBreakId ?? "?",
|
||||
reason: integrity.reason ?? "unknown",
|
||||
});
|
||||
}
|
||||
|
||||
export function buildAuditPdfDocument(input: AuditPdfReportInput): ReturnType<typeof React.createElement> {
|
||||
const labels = resolveAuditPdfLabels(input.labels);
|
||||
const generatedAt = (input.generatedAt ?? new Date()).toISOString();
|
||||
const integrityStyle = input.integrity.ok ? styles.integrityOk : styles.integrityBroken;
|
||||
const eventRows = input.events.map((event, index) =>
|
||||
React.createElement(
|
||||
View,
|
||||
{ key: `row-${event.id}-${index}`, style: styles.row },
|
||||
...COLUMNS.map((col) => {
|
||||
const raw = col.render ? col.render(event) : (event[col.key] as unknown);
|
||||
const text = raw === null || raw === undefined ? "-" : String(raw);
|
||||
return React.createElement(Text, { key: `${event.id}-${col.key}`, style: styles.cell }, text);
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
return React.createElement(
|
||||
Document,
|
||||
{ title: input.title ?? labels.title },
|
||||
React.createElement(
|
||||
Page,
|
||||
{ size: "A4", style: styles.page },
|
||||
React.createElement(Text, { style: styles.title }, input.title ?? labels.title),
|
||||
React.createElement(Text, { style: styles.meta }, `${labels.tenant}: ${input.tenantId ?? `(${labels.platformTenant})`}`),
|
||||
React.createElement(Text, { style: styles.meta }, `${labels.range}: ${input.from ?? `(${labels.openRange})`} ... ${input.to ?? `(${labels.openRange})`}`),
|
||||
React.createElement(Text, { style: styles.meta }, `${labels.filters}: ${input.filterSummary || labels.noFilters}`),
|
||||
input.generatedBy ? React.createElement(Text, { style: styles.meta }, `${labels.generatedBy}: ${input.generatedBy}`) : null,
|
||||
React.createElement(Text, { style: styles.meta }, `${labels.generatedAt}: ${generatedAt}`),
|
||||
React.createElement(Text, { style: styles.subtitle }, labels.hashChainIntegrity),
|
||||
React.createElement(Text, { style: integrityStyle }, buildIntegrityLine(input.integrity, labels)),
|
||||
React.createElement(Text, { style: styles.subtitle }, labels.events(input.events.length)),
|
||||
React.createElement(
|
||||
View,
|
||||
{ style: styles.table },
|
||||
React.createElement(
|
||||
View,
|
||||
{ style: styles.row },
|
||||
...COLUMNS.map((col) =>
|
||||
React.createElement(Text, { key: `head-${col.key}`, style: styles.headerCell }, labels.columns[col.key])
|
||||
)
|
||||
),
|
||||
...eventRows
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export async function serializeAuditEventsToPdf(input: AuditPdfReportInput): Promise<Uint8Array> {
|
||||
if (input.events.length > AUDIT_PDF_HARD_ROW_LIMIT) {
|
||||
throw new ExportPdfRowLimitExceededError(input.events.length);
|
||||
}
|
||||
const doc = buildAuditPdfDocument(input);
|
||||
return new Uint8Array(await renderToBuffer(doc));
|
||||
}
|
||||
|
||||
export async function serializeAuditEventsToPdfStream(writeStream: Writable, input: AuditPdfReportInput): Promise<{ rowCount: number }> {
|
||||
if (input.events.length > AUDIT_PDF_HARD_ROW_LIMIT) {
|
||||
throw new ExportPdfRowLimitExceededError(input.events.length);
|
||||
}
|
||||
const doc = buildAuditPdfDocument(input);
|
||||
const readable = await renderToStream(doc);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
readable.on("error", reject);
|
||||
writeStream.on("error", reject);
|
||||
readable.on("end", () => resolve());
|
||||
readable.pipe(writeStream, { end: false });
|
||||
});
|
||||
return { rowCount: input.events.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Format the active filters into a compact one-liner suitable for the
|
||||
* cover-page metadata. Shared between CLI and web call-sites so the
|
||||
* report self-describes the operator inputs identically.
|
||||
*/
|
||||
export function summarizeAuditPdfFilters(filters: {
|
||||
regulatoryScope?: ReadonlyArray<string>;
|
||||
actorType?: string | null;
|
||||
outcome?: string | null;
|
||||
}): string {
|
||||
const parts: string[] = [];
|
||||
if (filters.regulatoryScope && filters.regulatoryScope.length > 0) {
|
||||
parts.push(`regulatoryScope=${filters.regulatoryScope.join("|")}`);
|
||||
}
|
||||
if (filters.actorType) parts.push(`actorType=${filters.actorType}`);
|
||||
if (filters.outcome) parts.push(`outcome=${filters.outcome}`);
|
||||
return parts.join(" ");
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportTableValidationError } from "../types.js";
|
||||
import { buildCsvExportArtifact, serializeExportTableToCsv } from "./index.js";
|
||||
|
||||
describe("@toolctrl/exports/csv public entrypoint", () => {
|
||||
it("exposes CSV-only runtime helpers without loading unrelated exporters", async () => {
|
||||
const exportsModule = await import("@toolctrl/exports/csv");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"ExportTableValidationError",
|
||||
"buildCsvExportArtifact",
|
||||
"serializeExportTableToCsv",
|
||||
"serializeExportTableToCsvStream",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToCsv", () => {
|
||||
it("uses explicit header labels and column order", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: [
|
||||
{ key: "email", label: "Email address" },
|
||||
{ key: "id", label: "Identifier" },
|
||||
],
|
||||
rows: [{ id: 7, email: "ada@example.com" }],
|
||||
});
|
||||
|
||||
expect(csv).toBe('"Email address","Identifier"\n"ada@example.com","7"');
|
||||
});
|
||||
|
||||
it("escapes commas, quotes, and newlines with RFC-style quoting", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: ["name", "note"],
|
||||
rows: [
|
||||
{
|
||||
name: 'Ada "Countess"',
|
||||
note: "line one,line two\nline three",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(csv).toBe('"name","note"\n"Ada ""Countess""","line one,line two\nline three"');
|
||||
});
|
||||
|
||||
it("neutralizes spreadsheet formula prefixes before CSV escaping", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: [
|
||||
{ key: "equals", label: "=header" },
|
||||
"plus",
|
||||
"minus",
|
||||
"at",
|
||||
"spaceFormula",
|
||||
"spaceTab",
|
||||
"spaceCr",
|
||||
"spaceLf",
|
||||
"safe",
|
||||
],
|
||||
rows: [
|
||||
{
|
||||
equals: "=1+1",
|
||||
plus: "+1",
|
||||
minus: "-1",
|
||||
at: "@cmd",
|
||||
spaceFormula: " =1+1",
|
||||
spaceTab: " \tvalue",
|
||||
spaceCr: " \rvalue",
|
||||
spaceLf: " \nvalue",
|
||||
safe: " value",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(csv).toBe([
|
||||
`"'=header","plus","minus","at","spaceFormula","spaceTab","spaceCr","spaceLf","safe"`,
|
||||
`"'=1+1","'+1","'-1","'@cmd","' =1+1","' \tvalue","' \rvalue","' \nvalue"," value"`,
|
||||
].join("\n"));
|
||||
});
|
||||
|
||||
it("serializes null and undefined values as blank cells", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: ["id", "empty", "missing"],
|
||||
rows: [{ id: 1, empty: null, missing: undefined }],
|
||||
});
|
||||
|
||||
expect(csv).toBe([
|
||||
`"id","empty","missing"`,
|
||||
`"1","",""`,
|
||||
].join("\n"));
|
||||
});
|
||||
|
||||
it("projects selected columns while preserving deterministic scalar strings", () => {
|
||||
const csv = serializeExportTableToCsv({
|
||||
columns: ["createdAt", "active", "amount"],
|
||||
rows: [
|
||||
{
|
||||
active: false,
|
||||
amount: 42.5,
|
||||
createdAt: new Date("2024-01-02T03:04:05.006Z"),
|
||||
ignored: "not exported",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(csv).toBe('"createdAt","active","amount"\n"2024-01-02T03:04:05.006Z","false","42.5"');
|
||||
});
|
||||
|
||||
it("rejects duplicate columns through the CSV API", () => {
|
||||
expect(() =>
|
||||
serializeExportTableToCsv({
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildCsvExportArtifact", () => {
|
||||
it("returns CSV bytes with export metadata", () => {
|
||||
const artifact = buildCsvExportArtifact({
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
});
|
||||
|
||||
expect(artifact).toEqual({
|
||||
bytes: '"id"\n"1"\n"2"',
|
||||
contentType: "text/csv; charset=utf-8",
|
||||
extension: "csv",
|
||||
rowCount: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,148 @@
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import { type ExportColumn, type ExportRecord, type ExportTable, normalizeExportTable } from "../types.js";
|
||||
import { neutralizeSpreadsheetFormula } from "../spreadsheet-safety.js";
|
||||
|
||||
export { ExportTableValidationError } from "../types.js";
|
||||
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable, NormalizedExportColumn, NormalizedExportRow, NormalizedExportTable } from "../types.js";
|
||||
|
||||
export type CsvExportOptions = {
|
||||
eol?: "\n" | "\r\n";
|
||||
delimiter?: "," | "|" | ";";
|
||||
};
|
||||
|
||||
export type CsvExportArtifact = {
|
||||
bytes: string;
|
||||
contentType: "text/csv; charset=utf-8";
|
||||
extension: "csv";
|
||||
rowCount: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Streaming CSV writer. Iterates over `table.rows` row-by-row, writing each
|
||||
* encoded record directly to `writeStream`. Respects writable back-pressure
|
||||
* via the standard `write`/`drain` contract so worker memory stays flat for
|
||||
* tables of arbitrary size.
|
||||
*
|
||||
* The function is intentionally permissive about `table.rows` — it accepts
|
||||
* any sync or async iterable of {@link ExportRecord}, which lets callers
|
||||
* pipe a Prisma cursor (chunked async iterator) without buffering all rows.
|
||||
*/
|
||||
export async function serializeExportTableToCsvStream(
|
||||
writeStream: Writable,
|
||||
table: CsvStreamTable,
|
||||
options: CsvExportOptions = {},
|
||||
): Promise<{ rowCount: number }> {
|
||||
const eol = options.eol ?? "\n";
|
||||
const delimiter = options.delimiter ?? ",";
|
||||
const columns = resolveCsvColumns(table);
|
||||
const header = columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
|
||||
|
||||
await writeChunk(writeStream, header);
|
||||
|
||||
let rowCount = 0;
|
||||
let isFirstRow = true;
|
||||
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
if (row === null || typeof row !== "object" || Array.isArray(row)) {
|
||||
throw new Error("Export table row must be an object record.");
|
||||
}
|
||||
const encoded = columns.map((column) => escapeCsvCell(stringifyValue(row[column.key]))).join(delimiter);
|
||||
if (isFirstRow) {
|
||||
// header already emitted — every row is preceded by an eol so the
|
||||
// final byte of the stream is never an unterminated delimiter.
|
||||
isFirstRow = false;
|
||||
}
|
||||
await writeChunk(writeStream, `${eol}${encoded}`);
|
||||
rowCount += 1;
|
||||
}
|
||||
|
||||
return { rowCount };
|
||||
}
|
||||
|
||||
export function serializeExportTableToCsv(table: ExportTable, options: CsvExportOptions = {}): string {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const eol = options.eol ?? "\n";
|
||||
const delimiter = options.delimiter ?? ",";
|
||||
const header = normalizedTable.columns.map((column) => escapeCsvCell(column.label)).join(delimiter);
|
||||
const rows = normalizedTable.rows.map((row) => normalizedTable.columns.map((column) => escapeCsvCell(row[column.key])).join(delimiter));
|
||||
|
||||
return [header, ...rows].join(eol);
|
||||
}
|
||||
|
||||
export function buildCsvExportArtifact(table: ExportTable, options: CsvExportOptions = {}): CsvExportArtifact {
|
||||
const bytes = serializeExportTableToCsv(table, options);
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
|
||||
return {
|
||||
bytes,
|
||||
contentType: "text/csv; charset=utf-8",
|
||||
extension: "csv",
|
||||
rowCount: normalizedTable.rowCount,
|
||||
};
|
||||
}
|
||||
|
||||
export type CsvStreamTable = {
|
||||
columns?: readonly (string | ExportColumn)[];
|
||||
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
|
||||
};
|
||||
|
||||
function resolveCsvColumns(table: CsvStreamTable): { key: string; label: string }[] {
|
||||
if (table.columns === undefined) {
|
||||
throw new Error("Streaming CSV writer requires explicit columns; cannot infer from an async row source.");
|
||||
}
|
||||
const seenKeys = new Set<string>();
|
||||
const columns = table.columns.map((column) => {
|
||||
const normalized = typeof column === "string" ? { key: column, label: column } : { key: column.key, label: column.label ?? column.key };
|
||||
if (seenKeys.has(normalized.key)) {
|
||||
throw new Error(`Export table columns contain duplicate key "${normalized.key}".`);
|
||||
}
|
||||
seenKeys.add(normalized.key);
|
||||
return normalized;
|
||||
});
|
||||
return columns;
|
||||
}
|
||||
|
||||
function stringifyValue(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
if (typeof value === "bigint") return value.toString();
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function escapeCsvCell(value: string): string {
|
||||
const safeValue = neutralizeSpreadsheetFormula(value);
|
||||
return `"${safeValue.replaceAll("\"", "\"\"")}"`;
|
||||
}
|
||||
|
||||
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
|
||||
if (!writeStream.write(chunk)) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
writeStream.off("drain", onDrain);
|
||||
writeStream.off("error", onError);
|
||||
};
|
||||
writeStream.once("drain", onDrain);
|
||||
writeStream.once("error", onError);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
|
||||
if (Symbol.asyncIterator in (source as object)) {
|
||||
return source as AsyncIterable<T>;
|
||||
}
|
||||
return (async function* () {
|
||||
for (const item of source as Iterable<T>) {
|
||||
yield item;
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { serializeExportTableToCsvStream } from "./index.js";
|
||||
|
||||
async function collectStream(stream: PassThrough): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
}
|
||||
|
||||
describe("serializeExportTableToCsvStream", () => {
|
||||
it("writes a header followed by encoded rows", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collectStream(sink);
|
||||
|
||||
const result = await serializeExportTableToCsvStream(
|
||||
sink,
|
||||
{
|
||||
columns: [
|
||||
{ key: "email", label: "Email address" },
|
||||
{ key: "id", label: "Identifier" },
|
||||
],
|
||||
rows: [
|
||||
{ id: 7, email: "ada@example.com" },
|
||||
{ id: 8, email: "grace@example.com" },
|
||||
],
|
||||
},
|
||||
);
|
||||
sink.end();
|
||||
|
||||
expect(result.rowCount).toBe(2);
|
||||
expect(await collected).toBe(
|
||||
[
|
||||
'"Email address","Identifier"',
|
||||
'"ada@example.com","7"',
|
||||
'"grace@example.com","8"',
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
it("supports async iterables as the row source (Prisma cursor friendly)", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collectStream(sink);
|
||||
|
||||
async function* rows() {
|
||||
yield { id: 1, name: "A" };
|
||||
yield { id: 2, name: "B" };
|
||||
yield { id: 3, name: "C" };
|
||||
}
|
||||
|
||||
const result = await serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id", "name"],
|
||||
rows: rows(),
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(result.rowCount).toBe(3);
|
||||
const text = await collected;
|
||||
expect(text.split("\n")).toEqual(['"id","name"', '"1","A"', '"2","B"', '"3","C"']);
|
||||
});
|
||||
|
||||
it("neutralises spreadsheet-formula prefixes the same as the buffered API", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collectStream(sink);
|
||||
|
||||
await serializeExportTableToCsvStream(sink, {
|
||||
columns: ["a"],
|
||||
rows: [{ a: "=1+1" }],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(await collected).toBe(['"a"', "\"'=1+1\""].join("\n"));
|
||||
});
|
||||
|
||||
it("rejects duplicate column keys eagerly", async () => {
|
||||
const sink = new PassThrough();
|
||||
await expect(
|
||||
serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow('Export table columns contain duplicate key "id".');
|
||||
});
|
||||
|
||||
it("keeps worker memory flat for 100k synthetic rows", async () => {
|
||||
const sink = new PassThrough();
|
||||
// Drain the sink as quickly as the producer writes so the PassThrough
|
||||
// buffer cannot mask a buffering regression in the streamer.
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
async function* rows() {
|
||||
for (let i = 0; i < 100_000; i += 1) {
|
||||
yield { id: i, value: `row-${i}` };
|
||||
}
|
||||
}
|
||||
|
||||
if (global.gc) global.gc();
|
||||
const before = process.memoryUsage().heapUsed;
|
||||
const result = await serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id", "value"],
|
||||
rows: rows(),
|
||||
});
|
||||
sink.end();
|
||||
if (global.gc) global.gc();
|
||||
const after = process.memoryUsage().heapUsed;
|
||||
|
||||
expect(result.rowCount).toBe(100_000);
|
||||
// Hard upper bound is generous to absorb GC jitter; a buffering
|
||||
// regression would inflate heap by tens of MB.
|
||||
expect(after - before).toBeLessThan(100 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,3 @@
|
||||
export const PDF_STREAM_HARD_ROW_LIMIT = 5_000;
|
||||
export const XLSX_STREAM_HARD_ROW_LIMIT = 500_000;
|
||||
export const XLSX_STREAM_THRESHOLD = 50_000;
|
||||
@@ -0,0 +1,182 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
ExportTableValidationError,
|
||||
buildCsvExportArtifact,
|
||||
buildJsonExportArtifact,
|
||||
buildPdfExportArtifact,
|
||||
buildXlsxExportArtifact,
|
||||
normalizeExportTable,
|
||||
serializeExportTableToCsv,
|
||||
serializeExportTableToJson,
|
||||
serializeExportTableToJsonLines,
|
||||
serializeExportTableToPdf,
|
||||
serializeExportTableToXlsx,
|
||||
} from "@toolctrl/exports";
|
||||
|
||||
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
|
||||
describe("@toolctrl/exports public entrypoint", () => {
|
||||
it("publishes dedicated subpaths so lightweight consumers can avoid the aggregate entrypoint", async () => {
|
||||
const packageJson = JSON.parse(
|
||||
await readFile(join(packageRoot, "package.json"), "utf8"),
|
||||
) as { exports: Record<string, string> };
|
||||
|
||||
expect(packageJson.exports).toMatchObject({
|
||||
".": "./src/index.ts",
|
||||
"./audit-pdf": "./src/audit-pdf.ts",
|
||||
"./csv": "./src/csv/index.ts",
|
||||
"./json": "./src/json/index.ts",
|
||||
"./limits": "./src/limits.ts",
|
||||
"./pdf": "./src/pdf/index.ts",
|
||||
"./s3-multipart": "./src/sinks/s3-multipart.ts",
|
||||
"./types": "./src/types.ts",
|
||||
"./xlsx": "./src/xlsx/index.ts",
|
||||
});
|
||||
});
|
||||
|
||||
it("exposes CSV, JSON, shared validation, XLSX, and PDF APIs exactly once", async () => {
|
||||
const exportsModule = await import("@toolctrl/exports");
|
||||
const expectedExports = [
|
||||
"AUDIT_PDF_CONTENT_TYPE",
|
||||
"AUDIT_PDF_HARD_ROW_LIMIT",
|
||||
"EXPORT_PLAN_ROW_CAPS",
|
||||
"ExportPdfRowLimitExceededError",
|
||||
"ExportPrettyJsonTooLargeError",
|
||||
"ExportRowCapExceededError",
|
||||
"ExportTableValidationError",
|
||||
"ExportXlsxRowLimitExceededError",
|
||||
"PDF_STREAM_HARD_ROW_LIMIT",
|
||||
"PRETTY_JSON_STREAM_ROW_LIMIT",
|
||||
"XLSX_STREAM_HARD_ROW_LIMIT",
|
||||
"XLSX_STREAM_THRESHOLD",
|
||||
"assertExportRowCap",
|
||||
"buildAuditPdfDocument",
|
||||
"buildCsvExportArtifact",
|
||||
"buildJsonExportArtifact",
|
||||
"buildPdfExportArtifact",
|
||||
"buildXlsxExportArtifact",
|
||||
"createS3MultipartUploadSink",
|
||||
"normalizeExportTable",
|
||||
"resolveExportRowCap",
|
||||
"serializeAuditEventsToPdf",
|
||||
"serializeAuditEventsToPdfStream",
|
||||
"serializeExportTableToCsv",
|
||||
"serializeExportTableToCsvStream",
|
||||
"serializeExportTableToJson",
|
||||
"serializeExportTableToJsonLines",
|
||||
"serializeExportTableToJsonStream",
|
||||
"serializeExportTableToPdf",
|
||||
"serializeExportTableToPdfStream",
|
||||
"serializeExportTableToXlsx",
|
||||
"serializeExportTableToXlsxStream",
|
||||
"summarizeAuditPdfFilters",
|
||||
];
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual(expectedExports);
|
||||
expect(new Set(Object.keys(exportsModule))).toHaveLength(expectedExports.length);
|
||||
expect(exportsModule.ExportTableValidationError).toBe(ExportTableValidationError);
|
||||
expect(exportsModule.buildCsvExportArtifact).toBe(buildCsvExportArtifact);
|
||||
expect(exportsModule.buildJsonExportArtifact).toBe(buildJsonExportArtifact);
|
||||
expect(exportsModule.buildPdfExportArtifact).toBe(buildPdfExportArtifact);
|
||||
expect(exportsModule.buildXlsxExportArtifact).toBe(buildXlsxExportArtifact);
|
||||
expect(exportsModule.normalizeExportTable).toBe(normalizeExportTable);
|
||||
expect(exportsModule.serializeExportTableToCsv).toBe(serializeExportTableToCsv);
|
||||
expect(exportsModule.serializeExportTableToJson).toBe(serializeExportTableToJson);
|
||||
expect(exportsModule.serializeExportTableToJsonLines).toBe(serializeExportTableToJsonLines);
|
||||
expect(exportsModule.serializeExportTableToPdf).toBe(serializeExportTableToPdf);
|
||||
expect(exportsModule.serializeExportTableToXlsx).toBe(serializeExportTableToXlsx);
|
||||
});
|
||||
|
||||
it("builds CSV, JSON, XLSX, and PDF artifacts through the package entrypoint", async () => {
|
||||
const table = {
|
||||
columns: [
|
||||
{ key: "email", label: "Email" },
|
||||
{ key: "id", label: "Identifier" },
|
||||
"active",
|
||||
],
|
||||
rows: [
|
||||
{ id: 7, email: "ada@example.com", active: true, ignored: "not exported" },
|
||||
{ id: 8, email: null, active: false },
|
||||
],
|
||||
};
|
||||
const expectedRowCount = normalizeExportTable(table).rowCount;
|
||||
|
||||
expect(expectedRowCount).toBe(2);
|
||||
|
||||
expect(serializeExportTableToCsv(table)).toBe(
|
||||
[
|
||||
'"Email","Identifier","active"',
|
||||
'"ada@example.com","7","true"',
|
||||
'"","8","false"',
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(buildCsvExportArtifact(table)).toEqual({
|
||||
bytes: [
|
||||
'"Email","Identifier","active"',
|
||||
'"ada@example.com","7","true"',
|
||||
'"","8","false"',
|
||||
].join("\n"),
|
||||
contentType: "text/csv; charset=utf-8",
|
||||
extension: "csv",
|
||||
rowCount: expectedRowCount,
|
||||
});
|
||||
|
||||
expect(serializeExportTableToJson(table)).toBe(
|
||||
'[\n {\n "email": "ada@example.com",\n "id": 7,\n "active": true\n },\n {\n "email": null,\n "id": 8,\n "active": false\n }\n]',
|
||||
);
|
||||
expect(serializeExportTableToJsonLines(table)).toBe(
|
||||
'{"email":"ada@example.com","id":7,"active":true}\n{"email":null,"id":8,"active":false}',
|
||||
);
|
||||
|
||||
expect(buildJsonExportArtifact(table, { format: "jsonl" })).toEqual({
|
||||
bytes: '{"email":"ada@example.com","id":7,"active":true}\n{"email":null,"id":8,"active":false}',
|
||||
contentType: "application/x-ndjson; charset=utf-8",
|
||||
extension: "jsonl",
|
||||
rowCount: expectedRowCount,
|
||||
});
|
||||
|
||||
const xlsxBytes = await serializeExportTableToXlsx(table);
|
||||
expect(xlsxBytes.byteLength).toBeGreaterThan(0);
|
||||
|
||||
const xlsxArtifact = await buildXlsxExportArtifact(table);
|
||||
expect(xlsxArtifact).toMatchObject({
|
||||
contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
extension: "xlsx",
|
||||
rowCount: expectedRowCount,
|
||||
byteLength: xlsxArtifact.bytes.byteLength,
|
||||
});
|
||||
expect(xlsxArtifact.byteLength).toBeGreaterThan(0);
|
||||
|
||||
const pdfBytes = await serializeExportTableToPdf(table, { title: "Member export" });
|
||||
expect(new TextDecoder().decode(pdfBytes.slice(0, 5))).toBe("%PDF-");
|
||||
|
||||
const pdfArtifact = await buildPdfExportArtifact(table);
|
||||
expect(pdfArtifact).toMatchObject({
|
||||
contentType: "application/pdf",
|
||||
extension: "pdf",
|
||||
rowCount: expectedRowCount,
|
||||
byteLength: pdfArtifact.bytes.byteLength,
|
||||
});
|
||||
expect(pdfArtifact.byteLength).toBeGreaterThan(0);
|
||||
|
||||
expect([xlsxArtifact, pdfArtifact]).toEqual([
|
||||
expect.objectContaining({
|
||||
rowCount: expectedRowCount,
|
||||
contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
extension: "xlsx",
|
||||
byteLength: expect.any(Number),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
rowCount: expectedRowCount,
|
||||
contentType: "application/pdf",
|
||||
extension: "pdf",
|
||||
byteLength: expect.any(Number),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
export * from "./csv/index.js";
|
||||
export * from "./json/index.js";
|
||||
export * from "./pdf/index.js";
|
||||
export * from "./xlsx/index.js";
|
||||
export * from "./types.js";
|
||||
export * from "./sinks/s3-multipart.js";
|
||||
export * from "./limits.js";
|
||||
export * from "./audit-pdf.js";
|
||||
@@ -0,0 +1,144 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportTableValidationError } from "../types.js";
|
||||
import {
|
||||
buildJsonExportArtifact,
|
||||
serializeExportTableToJson,
|
||||
serializeExportTableToJsonLines,
|
||||
} from "./index.js";
|
||||
|
||||
describe("@toolctrl/exports/json public entrypoint", () => {
|
||||
it("exposes JSON-only runtime helpers", async () => {
|
||||
const exportsModule = await import("./index.js");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"ExportPrettyJsonTooLargeError",
|
||||
"ExportTableValidationError",
|
||||
"PRETTY_JSON_STREAM_ROW_LIMIT",
|
||||
"buildJsonExportArtifact",
|
||||
"serializeExportTableToJson",
|
||||
"serializeExportTableToJsonLines",
|
||||
"serializeExportTableToJsonStream",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToJson", () => {
|
||||
it("projects fields in explicit column order", () => {
|
||||
const json = serializeExportTableToJson({
|
||||
columns: ["email", "id"],
|
||||
rows: [{ id: 7, name: "Ada", email: "ada@example.com" }],
|
||||
});
|
||||
|
||||
expect(JSON.parse(json)).toEqual([{ email: "ada@example.com", id: 7 }]);
|
||||
expect(json).toBe('[\n {\n "email": "ada@example.com",\n "id": 7\n }\n]');
|
||||
});
|
||||
|
||||
it("emits pretty JSON by default and compact JSON when disabled", () => {
|
||||
const table = {
|
||||
columns: ["id", "active"],
|
||||
rows: [{ id: 1, active: true }],
|
||||
};
|
||||
|
||||
expect(serializeExportTableToJson(table)).toBe('[\n {\n "id": 1,\n "active": true\n }\n]');
|
||||
expect(serializeExportTableToJson(table, { pretty: false })).toBe('[{"id":1,"active":true}]');
|
||||
});
|
||||
|
||||
it("preserves selected null values and projects away omitted fields", () => {
|
||||
const json = serializeExportTableToJson({
|
||||
columns: ["id", "deletedAt", "missing"],
|
||||
rows: [{ id: 1, deletedAt: null, ignored: "not exported", missing: undefined }],
|
||||
});
|
||||
|
||||
expect(JSON.parse(json)).toEqual([{ id: 1, deletedAt: null, missing: null }]);
|
||||
expect(json).not.toContain("ignored");
|
||||
});
|
||||
|
||||
it("serializes dates as deterministic ISO strings and bigints as JSON-safe strings", () => {
|
||||
const json = serializeExportTableToJson({
|
||||
columns: ["createdAt", "big"],
|
||||
rows: [{ createdAt: new Date("2024-01-02T03:04:05.006Z"), big: 9007199254740993n }],
|
||||
});
|
||||
|
||||
expect(JSON.parse(json)).toEqual([
|
||||
{ createdAt: "2024-01-02T03:04:05.006Z", big: "9007199254740993" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects malformed row input through the JSON API", () => {
|
||||
expect(() =>
|
||||
serializeExportTableToJson({
|
||||
rows: [{ id: 1 }, null],
|
||||
} as never),
|
||||
).toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
|
||||
});
|
||||
|
||||
it("rejects duplicate columns through the JSON API", () => {
|
||||
expect(() =>
|
||||
serializeExportTableToJson({
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToJsonLines", () => {
|
||||
it("emits one compact JSON object per row without a trailing newline", () => {
|
||||
const jsonl = serializeExportTableToJsonLines({
|
||||
columns: ["id", "name"],
|
||||
rows: [
|
||||
{ id: 1, name: "Ada" },
|
||||
{ id: 2, name: "Grace" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(jsonl).toBe('{"id":1,"name":"Ada"}\n{"id":2,"name":"Grace"}');
|
||||
expect(jsonl.endsWith("\n")).toBe(false);
|
||||
});
|
||||
|
||||
it("supports CRLF separators for JSONL output", () => {
|
||||
const jsonl = serializeExportTableToJsonLines(
|
||||
{
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
},
|
||||
{ eol: "\r\n" },
|
||||
);
|
||||
|
||||
expect(jsonl).toBe('{"id":1}\r\n{"id":2}');
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildJsonExportArtifact", () => {
|
||||
it("returns JSON bytes with artifact metadata", () => {
|
||||
const artifact = buildJsonExportArtifact({
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
});
|
||||
|
||||
expect(artifact).toEqual({
|
||||
bytes: '[\n {\n "id": 1\n },\n {\n "id": 2\n }\n]',
|
||||
contentType: "application/json; charset=utf-8",
|
||||
extension: "json",
|
||||
rowCount: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns JSONL bytes with artifact metadata", () => {
|
||||
const artifact = buildJsonExportArtifact(
|
||||
{
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
},
|
||||
{ format: "jsonl" },
|
||||
);
|
||||
|
||||
expect(artifact).toEqual({
|
||||
bytes: '{"id":1}\n{"id":2}',
|
||||
contentType: "application/x-ndjson; charset=utf-8",
|
||||
extension: "jsonl",
|
||||
rowCount: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,203 @@
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import {
|
||||
type ExportColumn,
|
||||
type ExportPrimitive,
|
||||
type ExportRecord,
|
||||
type ExportTable,
|
||||
normalizeExportTable,
|
||||
} from "../types.js";
|
||||
|
||||
export { ExportTableValidationError } from "../types.js";
|
||||
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
|
||||
|
||||
export type JsonExportOptions = {
|
||||
pretty?: boolean;
|
||||
eol?: "\n" | "\r\n";
|
||||
};
|
||||
|
||||
export type JsonExportArtifactOptions = JsonExportOptions & {
|
||||
format?: "json" | "jsonl";
|
||||
};
|
||||
|
||||
export type JsonExportArtifact = {
|
||||
bytes: string;
|
||||
contentType: "application/json; charset=utf-8" | "application/x-ndjson; charset=utf-8";
|
||||
extension: "json" | "jsonl";
|
||||
rowCount: number;
|
||||
};
|
||||
|
||||
type JsonExportValue = string | number | boolean | null;
|
||||
type JsonExportRow = Record<string, JsonExportValue>;
|
||||
|
||||
/**
|
||||
* Maximum row count permitted for the pretty-JSON streaming mode. Pretty
|
||||
* mode requires building the full array in memory to emit a syntactically
|
||||
* valid document; above this threshold callers must switch to NDJSON.
|
||||
*/
|
||||
export const PRETTY_JSON_STREAM_ROW_LIMIT = 10_000;
|
||||
|
||||
export class ExportPrettyJsonTooLargeError extends Error {
|
||||
readonly code = "EXPORT_PRETTY_JSON_TOO_LARGE";
|
||||
constructor(rowCount: number) {
|
||||
super(
|
||||
`Pretty-JSON streaming mode is limited to ${PRETTY_JSON_STREAM_ROW_LIMIT} rows (got ${rowCount}); use mode: 'ndjson' for larger exports.`,
|
||||
);
|
||||
this.name = "ExportPrettyJsonTooLargeError";
|
||||
}
|
||||
}
|
||||
|
||||
export type JsonStreamMode = "ndjson" | "pretty";
|
||||
|
||||
export type JsonStreamOptions = {
|
||||
mode?: JsonStreamMode;
|
||||
eol?: "\n" | "\r\n";
|
||||
};
|
||||
|
||||
export type JsonStreamTable = {
|
||||
columns?: readonly (string | ExportColumn)[];
|
||||
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
|
||||
};
|
||||
|
||||
export function serializeExportTableToJson(table: ExportTable, options: JsonExportOptions = {}): string {
|
||||
const rows = buildJsonRows(table);
|
||||
return JSON.stringify(rows, null, options.pretty ?? true ? 2 : 0);
|
||||
}
|
||||
|
||||
export function serializeExportTableToJsonLines(table: ExportTable, options: JsonExportOptions = {}): string {
|
||||
const eol = options.eol ?? "\n";
|
||||
return buildJsonRows(table)
|
||||
.map((row) => JSON.stringify(row))
|
||||
.join(eol);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming JSON writer. NDJSON mode writes one JSON object per line and
|
||||
* keeps memory flat. Pretty mode buffers up to {@link PRETTY_JSON_STREAM_ROW_LIMIT}
|
||||
* rows before flushing a single indented array; above the limit the writer
|
||||
* throws {@link ExportPrettyJsonTooLargeError} with a hint to switch to NDJSON.
|
||||
*/
|
||||
export async function serializeExportTableToJsonStream(
|
||||
writeStream: Writable,
|
||||
table: JsonStreamTable,
|
||||
options: JsonStreamOptions = {},
|
||||
): Promise<{ rowCount: number; mode: JsonStreamMode }> {
|
||||
const mode = options.mode ?? "ndjson";
|
||||
const eol = options.eol ?? "\n";
|
||||
const columnKeys = resolveJsonColumnKeys(table);
|
||||
|
||||
if (mode === "ndjson") {
|
||||
let rowCount = 0;
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
const projected = projectRow(row as ExportRecord, columnKeys);
|
||||
const chunk = rowCount === 0 ? JSON.stringify(projected) : `${eol}${JSON.stringify(projected)}`;
|
||||
await writeChunk(writeStream, chunk);
|
||||
rowCount += 1;
|
||||
}
|
||||
return { rowCount, mode };
|
||||
}
|
||||
|
||||
const buffered: JsonExportRow[] = [];
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
buffered.push(projectRow(row as ExportRecord, columnKeys));
|
||||
if (buffered.length > PRETTY_JSON_STREAM_ROW_LIMIT) {
|
||||
throw new ExportPrettyJsonTooLargeError(buffered.length);
|
||||
}
|
||||
}
|
||||
await writeChunk(writeStream, JSON.stringify(buffered, null, 2));
|
||||
return { rowCount: buffered.length, mode };
|
||||
}
|
||||
|
||||
export function buildJsonExportArtifact(
|
||||
table: ExportTable,
|
||||
options: JsonExportArtifactOptions = {},
|
||||
): JsonExportArtifact {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const format = options.format ?? "json";
|
||||
const bytes =
|
||||
format === "jsonl"
|
||||
? serializeExportTableToJsonLines(table, options)
|
||||
: serializeExportTableToJson(table, options);
|
||||
|
||||
return {
|
||||
bytes,
|
||||
contentType: format === "jsonl" ? "application/x-ndjson; charset=utf-8" : "application/json; charset=utf-8",
|
||||
extension: format === "jsonl" ? "jsonl" : "json",
|
||||
rowCount: normalizedTable.rowCount,
|
||||
};
|
||||
}
|
||||
|
||||
function buildJsonRows(table: ExportTable): JsonExportRow[] {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
|
||||
return table.rows.map((row) => projectRow(row, normalizedTable.columns.map((column) => column.key)));
|
||||
}
|
||||
|
||||
function projectRow(row: ExportRecord, columnKeys: readonly string[]): JsonExportRow {
|
||||
return Object.fromEntries(columnKeys.map((key) => [key, normalizeJsonValue(row[key])])) as JsonExportRow;
|
||||
}
|
||||
|
||||
function normalizeJsonValue(value: ExportPrimitive): JsonExportValue {
|
||||
if (value === null || value === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
if (typeof value === "bigint") {
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function resolveJsonColumnKeys(table: JsonStreamTable): string[] {
|
||||
if (table.columns === undefined) {
|
||||
throw new Error("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
|
||||
}
|
||||
const keys: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const column of table.columns) {
|
||||
const key = typeof column === "string" ? column : column.key;
|
||||
if (seen.has(key)) {
|
||||
throw new Error(`Export table columns contain duplicate key "${key}".`);
|
||||
}
|
||||
seen.add(key);
|
||||
keys.push(key);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
async function writeChunk(writeStream: Writable, chunk: string): Promise<void> {
|
||||
if (!writeStream.write(chunk)) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
writeStream.off("drain", onDrain);
|
||||
writeStream.off("error", onError);
|
||||
};
|
||||
writeStream.once("drain", onDrain);
|
||||
writeStream.once("error", onError);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
|
||||
if (Symbol.asyncIterator in (source as object)) {
|
||||
return source as AsyncIterable<T>;
|
||||
}
|
||||
return (async function* () {
|
||||
for (const item of source as Iterable<T>) {
|
||||
yield item;
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
ExportPrettyJsonTooLargeError,
|
||||
PRETTY_JSON_STREAM_ROW_LIMIT,
|
||||
serializeExportTableToJsonStream,
|
||||
} from "./index.js";
|
||||
|
||||
async function collect(stream: PassThrough): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
}
|
||||
|
||||
describe("serializeExportTableToJsonStream", () => {
|
||||
it("emits one JSON object per line in ndjson mode (default)", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
const result = await serializeExportTableToJsonStream(sink, {
|
||||
columns: ["id", "active"],
|
||||
rows: [
|
||||
{ id: 1, active: true },
|
||||
{ id: 2, active: false },
|
||||
],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(result).toEqual({ rowCount: 2, mode: "ndjson" });
|
||||
expect(await collected).toBe('{"id":1,"active":true}\n{"id":2,"active":false}');
|
||||
});
|
||||
|
||||
it("serialises Dates and BigInts predictably", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
await serializeExportTableToJsonStream(sink, {
|
||||
columns: ["createdAt", "count"],
|
||||
rows: [
|
||||
{ createdAt: new Date("2024-01-02T03:04:05.006Z"), count: BigInt(42) },
|
||||
],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(JSON.parse(await collected)).toEqual({
|
||||
createdAt: "2024-01-02T03:04:05.006Z",
|
||||
count: "42",
|
||||
});
|
||||
});
|
||||
|
||||
it("supports pretty mode for small tables", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
const result = await serializeExportTableToJsonStream(
|
||||
sink,
|
||||
{
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
},
|
||||
{ mode: "pretty" },
|
||||
);
|
||||
sink.end();
|
||||
|
||||
expect(result).toEqual({ rowCount: 2, mode: "pretty" });
|
||||
expect(JSON.parse(await collected)).toEqual([{ id: 1 }, { id: 2 }]);
|
||||
});
|
||||
|
||||
it("rejects pretty mode above the 10k-row limit with a typed error", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
async function* rows() {
|
||||
for (let i = 0; i <= PRETTY_JSON_STREAM_ROW_LIMIT; i += 1) {
|
||||
yield { id: i };
|
||||
}
|
||||
}
|
||||
|
||||
await expect(
|
||||
serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() }, { mode: "pretty" }),
|
||||
).rejects.toBeInstanceOf(ExportPrettyJsonTooLargeError);
|
||||
});
|
||||
|
||||
it("accepts async iterables (Prisma cursor friendly)", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
async function* rows() {
|
||||
yield { id: 1 };
|
||||
yield { id: 2 };
|
||||
yield { id: 3 };
|
||||
}
|
||||
|
||||
const result = await serializeExportTableToJsonStream(sink, { columns: ["id"], rows: rows() });
|
||||
sink.end();
|
||||
|
||||
expect(result.rowCount).toBe(3);
|
||||
expect(await collected).toBe('{"id":1}\n{"id":2}\n{"id":3}');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
EXPORT_PLAN_ROW_CAPS,
|
||||
ExportRowCapExceededError,
|
||||
assertExportRowCap,
|
||||
resolveExportRowCap,
|
||||
} from "./limits.js";
|
||||
import { PDF_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_HARD_ROW_LIMIT } from "./format-caps.js";
|
||||
|
||||
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
|
||||
describe("@toolctrl/exports/limits public entrypoint", () => {
|
||||
it("exposes row-cap helpers without importing heavy PDF/XLSX runtimes", async () => {
|
||||
const exportsModule = await import("./limits.js");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"EXPORT_PLAN_ROW_CAPS",
|
||||
"ExportRowCapExceededError",
|
||||
"assertExportRowCap",
|
||||
"resolveExportRowCap",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps limits wired to lightweight cap constants", async () => {
|
||||
const source = await readFile(join(packageRoot, "src/limits.ts"), "utf8");
|
||||
|
||||
expect(source).toContain("./format-caps.js");
|
||||
expect(source).not.toMatch(/["']\.\/(?:pdf|xlsx)\/index\.js["']/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveExportRowCap", () => {
|
||||
it("applies the plan cap to unbounded formats (csv, json)", () => {
|
||||
expect(resolveExportRowCap("free", "csv")).toBe(EXPORT_PLAN_ROW_CAPS.free);
|
||||
expect(resolveExportRowCap("pro", "json")).toBe(EXPORT_PLAN_ROW_CAPS.pro);
|
||||
expect(resolveExportRowCap("enterprise", "csv")).toBe(Number.POSITIVE_INFINITY);
|
||||
});
|
||||
|
||||
it("respects the per-format hard cap when stricter than the plan cap", () => {
|
||||
expect(resolveExportRowCap("enterprise", "xlsx")).toBe(XLSX_STREAM_HARD_ROW_LIMIT);
|
||||
expect(resolveExportRowCap("enterprise", "pdf")).toBe(PDF_STREAM_HARD_ROW_LIMIT);
|
||||
// free plan cap (10k) is lower than pdf hard cap (5k), so pdf cap wins.
|
||||
expect(resolveExportRowCap("free", "pdf")).toBe(PDF_STREAM_HARD_ROW_LIMIT);
|
||||
});
|
||||
|
||||
it("falls back to the free-plan cap for unknown plan tiers", () => {
|
||||
expect(resolveExportRowCap("vip" as never, "csv")).toBe(EXPORT_PLAN_ROW_CAPS.free);
|
||||
});
|
||||
});
|
||||
|
||||
describe("assertExportRowCap", () => {
|
||||
it("returns silently when the requested count fits the cap", () => {
|
||||
expect(() => assertExportRowCap(5_000, "free", "csv")).not.toThrow();
|
||||
expect(() => assertExportRowCap(50_000, "pro", "json")).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws ExportRowCapExceededError with the resolved cap", () => {
|
||||
let caught: ExportRowCapExceededError | undefined;
|
||||
try {
|
||||
assertExportRowCap(20_000, "free", "csv");
|
||||
} catch (error) {
|
||||
caught = error as ExportRowCapExceededError;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(ExportRowCapExceededError);
|
||||
expect(caught?.code).toBe("EXPORT_ROW_LIMIT_PLAN");
|
||||
expect(caught?.effectiveCap).toBe(EXPORT_PLAN_ROW_CAPS.free);
|
||||
expect(caught?.requestedRowCount).toBe(20_000);
|
||||
expect(caught?.plan).toBe("free");
|
||||
expect(caught?.format).toBe("csv");
|
||||
});
|
||||
|
||||
it("enforces format hard caps even for enterprise plans", () => {
|
||||
expect(() => assertExportRowCap(PDF_STREAM_HARD_ROW_LIMIT + 1, "enterprise", "pdf")).toThrow(
|
||||
ExportRowCapExceededError,
|
||||
);
|
||||
expect(() => assertExportRowCap(XLSX_STREAM_HARD_ROW_LIMIT + 1, "enterprise", "xlsx")).toThrow(
|
||||
ExportRowCapExceededError,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
import { PDF_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_HARD_ROW_LIMIT } from "./format-caps.js";
|
||||
|
||||
/**
|
||||
* Tenant plan tier used to resolve export row caps. Matches the
|
||||
* billing-plan vocabulary used elsewhere in the repo. Callers that
|
||||
* encounter an unknown plan should fall back to "free" — the most
|
||||
* conservative cap — rather than throwing, because the row cap is a
|
||||
* safety net, not an authorisation primitive.
|
||||
*/
|
||||
export type TenantPlanTier = "free" | "pro" | "enterprise";
|
||||
|
||||
export type ExportFormat = "csv" | "json" | "xlsx" | "pdf";
|
||||
|
||||
/**
|
||||
* Plan-based soft caps applied on top of the per-format hard caps. The
|
||||
* plan cap protects shared worker capacity from a single noisy tenant; the
|
||||
* format cap protects against format-specific failure modes.
|
||||
*/
|
||||
export const EXPORT_PLAN_ROW_CAPS: Record<TenantPlanTier, number> = {
|
||||
free: 10_000,
|
||||
pro: 100_000,
|
||||
enterprise: Number.POSITIVE_INFINITY,
|
||||
};
|
||||
|
||||
const FORMAT_HARD_CAPS: Record<ExportFormat, number> = {
|
||||
csv: Number.POSITIVE_INFINITY,
|
||||
json: Number.POSITIVE_INFINITY,
|
||||
xlsx: XLSX_STREAM_HARD_ROW_LIMIT,
|
||||
pdf: PDF_STREAM_HARD_ROW_LIMIT,
|
||||
};
|
||||
|
||||
export class ExportRowCapExceededError extends Error {
|
||||
readonly code = "EXPORT_ROW_LIMIT_PLAN";
|
||||
constructor(
|
||||
readonly requestedRowCount: number,
|
||||
readonly effectiveCap: number,
|
||||
readonly plan: TenantPlanTier,
|
||||
readonly format: ExportFormat,
|
||||
) {
|
||||
super(
|
||||
`Export request of ${requestedRowCount} rows exceeds the ${plan} plan / ${format} format cap of ${Number.isFinite(effectiveCap) ? effectiveCap : "unlimited"}.`,
|
||||
);
|
||||
this.name = "ExportRowCapExceededError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the effective row cap for a given plan and format. Returns
|
||||
* `Infinity` only when both the plan and the format are unbounded; in
|
||||
* practice all PDF and XLSX exports still hit a finite ceiling.
|
||||
*/
|
||||
export function resolveExportRowCap(plan: TenantPlanTier, format: ExportFormat): number {
|
||||
const planCap = EXPORT_PLAN_ROW_CAPS[plan] ?? EXPORT_PLAN_ROW_CAPS.free;
|
||||
const formatCap = FORMAT_HARD_CAPS[format] ?? Number.POSITIVE_INFINITY;
|
||||
return Math.min(planCap, formatCap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience guard: throws {@link ExportRowCapExceededError} when the
|
||||
* caller knows the row count up front. Streamers also enforce caps
|
||||
* mid-stream so this is a fast-path check, not the sole guard.
|
||||
*/
|
||||
export function assertExportRowCap(
|
||||
rowCount: number,
|
||||
plan: TenantPlanTier,
|
||||
format: ExportFormat,
|
||||
): void {
|
||||
const cap = resolveExportRowCap(plan, format);
|
||||
if (rowCount > cap) {
|
||||
throw new ExportRowCapExceededError(rowCount, cap, plan, format);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportTableValidationError } from "../types.js";
|
||||
import { buildPdfExportArtifact, serializeExportTableToPdf } from "./index.js";
|
||||
|
||||
const PDF_HEADER = "%PDF-";
|
||||
|
||||
function decodeHeader(bytes: Uint8Array): string {
|
||||
return new TextDecoder().decode(bytes.slice(0, PDF_HEADER.length));
|
||||
}
|
||||
|
||||
describe("@toolctrl/exports/pdf public entrypoint", () => {
|
||||
it("exposes PDF helpers through the dedicated subpath", async () => {
|
||||
const exportsModule = await import("./index.js");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"ExportPdfRowLimitExceededError",
|
||||
"ExportTableValidationError",
|
||||
"PDF_STREAM_HARD_ROW_LIMIT",
|
||||
"buildPdfExportArtifact",
|
||||
"serializeExportTableToPdf",
|
||||
"serializeExportTableToPdfStream",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToPdf", () => {
|
||||
it("renders PDF bytes with explicit title, header labels, and column order", async () => {
|
||||
const bytes = await serializeExportTableToPdf(
|
||||
{
|
||||
columns: [
|
||||
{ key: "email", label: "Email address" },
|
||||
{ key: "id", label: "Identifier" },
|
||||
"active",
|
||||
],
|
||||
rows: [{ id: 7, email: "ada@example.com", active: true, ignored: "not exported" }],
|
||||
},
|
||||
{ title: "Member export" },
|
||||
);
|
||||
|
||||
expect(decodeHeader(bytes)).toBe(PDF_HEADER);
|
||||
expect(bytes.byteLength).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("rejects malformed row input through the PDF API", async () => {
|
||||
await expect(
|
||||
serializeExportTableToPdf({
|
||||
rows: [{ id: 1 }, null],
|
||||
} as never),
|
||||
).rejects.toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
|
||||
});
|
||||
|
||||
it("rejects duplicate columns through the PDF API", async () => {
|
||||
await expect(
|
||||
serializeExportTableToPdf({
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildPdfExportArtifact", () => {
|
||||
it("returns PDF bytes with binary export metadata", async () => {
|
||||
const artifact = await buildPdfExportArtifact({
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
});
|
||||
|
||||
expect(decodeHeader(artifact.bytes)).toBe(PDF_HEADER);
|
||||
expect(artifact.contentType).toBe("application/pdf");
|
||||
expect(artifact.extension).toBe("pdf");
|
||||
expect(artifact.rowCount).toBe(2);
|
||||
expect(artifact.byteLength).toBe(artifact.bytes.byteLength);
|
||||
expect(artifact.byteLength).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import React from "react";
|
||||
import { Document, Page, StyleSheet, Text, View, renderToBuffer, renderToStream } from "@react-pdf/renderer";
|
||||
|
||||
import { PDF_STREAM_HARD_ROW_LIMIT } from "../format-caps.js";
|
||||
import { type ExportTable, normalizeExportTable } from "../types.js";
|
||||
|
||||
export { ExportTableValidationError } from "../types.js";
|
||||
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
|
||||
|
||||
const PDF_CONTENT_TYPE = "application/pdf";
|
||||
|
||||
/**
|
||||
* PDF rendering cost grows super-linearly with row count and produces
|
||||
* unusable >500-page documents above this threshold. The cap protects
|
||||
* both the worker and downstream UX.
|
||||
*/
|
||||
export { PDF_STREAM_HARD_ROW_LIMIT };
|
||||
|
||||
export class ExportPdfRowLimitExceededError extends Error {
|
||||
readonly code = "EXPORT_PDF_ROW_LIMIT_EXCEEDED";
|
||||
constructor(rowCount: number) {
|
||||
super(
|
||||
`PDF export exceeds the ${PDF_STREAM_HARD_ROW_LIMIT}-row hard cap (got ${rowCount}); use CSV or XLSX for larger datasets.`,
|
||||
);
|
||||
this.name = "ExportPdfRowLimitExceededError";
|
||||
}
|
||||
}
|
||||
|
||||
export type PdfExportOptions = {
|
||||
title?: string;
|
||||
};
|
||||
|
||||
export type PdfExportArtifact = {
|
||||
bytes: Uint8Array;
|
||||
contentType: typeof PDF_CONTENT_TYPE;
|
||||
extension: "pdf";
|
||||
rowCount: number;
|
||||
byteLength: number;
|
||||
};
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
page: {
|
||||
padding: 32,
|
||||
fontSize: 10,
|
||||
fontFamily: "Helvetica",
|
||||
},
|
||||
title: {
|
||||
fontSize: 16,
|
||||
fontWeight: 700,
|
||||
marginBottom: 16,
|
||||
},
|
||||
table: {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
width: "100%",
|
||||
borderTopWidth: 1,
|
||||
borderLeftWidth: 1,
|
||||
borderColor: "#d1d5db",
|
||||
},
|
||||
row: {
|
||||
display: "flex",
|
||||
flexDirection: "row",
|
||||
},
|
||||
headerCell: {
|
||||
flexGrow: 1,
|
||||
flexBasis: 0,
|
||||
padding: 6,
|
||||
fontWeight: 700,
|
||||
backgroundColor: "#f3f4f6",
|
||||
borderRightWidth: 1,
|
||||
borderBottomWidth: 1,
|
||||
borderColor: "#d1d5db",
|
||||
},
|
||||
cell: {
|
||||
flexGrow: 1,
|
||||
flexBasis: 0,
|
||||
padding: 6,
|
||||
borderRightWidth: 1,
|
||||
borderBottomWidth: 1,
|
||||
borderColor: "#d1d5db",
|
||||
},
|
||||
});
|
||||
|
||||
export async function serializeExportTableToPdf(
|
||||
table: ExportTable,
|
||||
options: PdfExportOptions = {},
|
||||
): Promise<Uint8Array> {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const document = React.createElement(
|
||||
Document,
|
||||
{ title: options.title },
|
||||
React.createElement(
|
||||
Page,
|
||||
{ size: "A4", style: styles.page },
|
||||
options.title === undefined ? null : React.createElement(Text, { style: styles.title }, options.title),
|
||||
React.createElement(
|
||||
View,
|
||||
{ style: styles.table },
|
||||
React.createElement(
|
||||
View,
|
||||
{ style: styles.row },
|
||||
...normalizedTable.columns.map((column) =>
|
||||
React.createElement(Text, { key: column.key, style: styles.headerCell }, column.label),
|
||||
),
|
||||
),
|
||||
...normalizedTable.rows.map((row, rowIndex) =>
|
||||
React.createElement(
|
||||
View,
|
||||
{ key: `row-${rowIndex}`, style: styles.row },
|
||||
...normalizedTable.columns.map((column) =>
|
||||
React.createElement(Text, { key: column.key, style: styles.cell }, row[column.key]),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
return new Uint8Array(await renderToBuffer(document));
|
||||
}
|
||||
|
||||
export async function buildPdfExportArtifact(
|
||||
table: ExportTable,
|
||||
options: PdfExportOptions = {},
|
||||
): Promise<PdfExportArtifact> {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const bytes = await serializeExportTableToPdf(table, options);
|
||||
|
||||
return {
|
||||
bytes,
|
||||
contentType: PDF_CONTENT_TYPE,
|
||||
extension: "pdf",
|
||||
rowCount: normalizedTable.rowCount,
|
||||
byteLength: bytes.byteLength,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming PDF writer. Uses `@react-pdf/renderer`'s renderToStream and
|
||||
* pipes the resulting Readable into `writeStream`. Enforces a hard cap of
|
||||
* {@link PDF_STREAM_HARD_ROW_LIMIT} rows up front — PDF cost is
|
||||
* super-linear and unbounded exports produce unusable artefacts.
|
||||
*/
|
||||
export async function serializeExportTableToPdfStream(
|
||||
writeStream: Writable,
|
||||
table: ExportTable,
|
||||
options: PdfExportOptions = {},
|
||||
): Promise<{ rowCount: number }> {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
if (normalizedTable.rowCount > PDF_STREAM_HARD_ROW_LIMIT) {
|
||||
throw new ExportPdfRowLimitExceededError(normalizedTable.rowCount);
|
||||
}
|
||||
|
||||
const document = React.createElement(
|
||||
Document,
|
||||
{ title: options.title },
|
||||
React.createElement(
|
||||
Page,
|
||||
{ size: "A4", style: styles.page },
|
||||
options.title === undefined ? null : React.createElement(Text, { style: styles.title }, options.title),
|
||||
React.createElement(
|
||||
View,
|
||||
{ style: styles.table },
|
||||
React.createElement(
|
||||
View,
|
||||
{ style: styles.row },
|
||||
...normalizedTable.columns.map((column) =>
|
||||
React.createElement(Text, { key: column.key, style: styles.headerCell }, column.label),
|
||||
),
|
||||
),
|
||||
...normalizedTable.rows.map((row, rowIndex) =>
|
||||
React.createElement(
|
||||
View,
|
||||
{ key: `row-${rowIndex}`, style: styles.row },
|
||||
...normalizedTable.columns.map((column) =>
|
||||
React.createElement(Text, { key: column.key, style: styles.cell }, row[column.key]),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const readable = await renderToStream(document);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
readable.on("error", reject);
|
||||
writeStream.on("error", reject);
|
||||
readable.on("end", () => resolve());
|
||||
readable.pipe(writeStream, { end: false });
|
||||
});
|
||||
return { rowCount: normalizedTable.rowCount };
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
ExportPdfRowLimitExceededError,
|
||||
PDF_STREAM_HARD_ROW_LIMIT,
|
||||
serializeExportTableToPdfStream,
|
||||
} from "./index.js";
|
||||
|
||||
async function collect(stream: PassThrough): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
describe("serializeExportTableToPdfStream", () => {
|
||||
it("renders a valid PDF prelude for small tables", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
const rows = Array.from({ length: 10 }, (_, i) => ({ id: i, name: `name-${i}` }));
|
||||
const result = await serializeExportTableToPdfStream(sink, {
|
||||
columns: [
|
||||
{ key: "id", label: "Identifier" },
|
||||
{ key: "name", label: "Name" },
|
||||
],
|
||||
rows,
|
||||
}, { title: "Member export" });
|
||||
sink.end();
|
||||
|
||||
expect(result).toEqual({ rowCount: 10 });
|
||||
const buffer = await collected;
|
||||
expect(buffer.slice(0, 5).toString("utf8")).toBe("%PDF-");
|
||||
expect(buffer.byteLength).toBeGreaterThan(100);
|
||||
});
|
||||
|
||||
it("rejects exports above the 5k-row hard cap", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
const rows = Array.from({ length: PDF_STREAM_HARD_ROW_LIMIT + 1 }, (_, i) => ({ id: i }));
|
||||
|
||||
await expect(
|
||||
serializeExportTableToPdfStream(sink, { columns: ["id"], rows }),
|
||||
).rejects.toBeInstanceOf(ExportPdfRowLimitExceededError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createS3MultipartUploadSink } from "./s3-multipart.js";
|
||||
|
||||
describe("createS3MultipartUploadSink", () => {
|
||||
it("invokes the Upload constructor with sensible multipart defaults", async () => {
|
||||
const done = vi.fn().mockResolvedValue({ ETag: '"deadbeef"' });
|
||||
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
|
||||
const fakeClient = {} as never;
|
||||
|
||||
const { writeStream, completionPromise } = createS3MultipartUploadSink({
|
||||
client: fakeClient,
|
||||
bucket: "exports",
|
||||
key: "tenant/abc/users.csv",
|
||||
contentType: "text/csv; charset=utf-8",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
uploadCtor: UploadStub as any,
|
||||
});
|
||||
|
||||
writeStream.end("hello");
|
||||
const result = await completionPromise;
|
||||
|
||||
expect(UploadStub).toHaveBeenCalledTimes(1);
|
||||
const ctorArg = UploadStub.mock.calls[0]![0] as {
|
||||
partSize: number;
|
||||
queueSize: number;
|
||||
params: { Bucket: string; Key: string; ContentType?: string };
|
||||
};
|
||||
expect(ctorArg.partSize).toBe(8 * 1024 * 1024);
|
||||
expect(ctorArg.queueSize).toBe(4);
|
||||
expect(ctorArg.params.Bucket).toBe("exports");
|
||||
expect(ctorArg.params.Key).toBe("tenant/abc/users.csv");
|
||||
expect(ctorArg.params.ContentType).toBe("text/csv; charset=utf-8");
|
||||
expect(result).toEqual({ bucket: "exports", key: "tenant/abc/users.csv", etag: '"deadbeef"' });
|
||||
});
|
||||
|
||||
it("returns etag: null when S3 omits an ETag", async () => {
|
||||
const done = vi.fn().mockResolvedValue({});
|
||||
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
|
||||
|
||||
const { writeStream, completionPromise } = createS3MultipartUploadSink({
|
||||
client: {} as never,
|
||||
bucket: "exports",
|
||||
key: "x.csv",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
uploadCtor: UploadStub as any,
|
||||
});
|
||||
writeStream.end();
|
||||
|
||||
await expect(completionPromise).resolves.toEqual({ bucket: "exports", key: "x.csv", etag: null });
|
||||
});
|
||||
|
||||
it("propagates Upload failures by destroying the write stream", async () => {
|
||||
const failure = new Error("multipart aborted");
|
||||
const done = vi.fn().mockRejectedValue(failure);
|
||||
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
|
||||
|
||||
const { writeStream, completionPromise } = createS3MultipartUploadSink({
|
||||
client: {} as never,
|
||||
bucket: "exports",
|
||||
key: "x.csv",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
uploadCtor: UploadStub as any,
|
||||
});
|
||||
// Stream errors propagate via the standard 'error' event; attach a
|
||||
// listener so it does not crash the test runner as an unhandled emit.
|
||||
const errorListener = vi.fn();
|
||||
writeStream.on("error", errorListener);
|
||||
|
||||
await expect(completionPromise).rejects.toBe(failure);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(writeStream.destroyed).toBe(true);
|
||||
expect(errorListener).toHaveBeenCalledWith(failure);
|
||||
});
|
||||
|
||||
it("honours partSizeBytes and queueSize overrides", async () => {
|
||||
const done = vi.fn().mockResolvedValue({ ETag: "x" });
|
||||
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
|
||||
|
||||
const { writeStream, completionPromise } = createS3MultipartUploadSink({
|
||||
client: {} as never,
|
||||
bucket: "b",
|
||||
key: "k",
|
||||
partSizeBytes: 5 * 1024 * 1024,
|
||||
queueSize: 2,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
uploadCtor: UploadStub as any,
|
||||
});
|
||||
writeStream.end();
|
||||
await completionPromise;
|
||||
|
||||
const ctorArg = UploadStub.mock.calls[0]![0] as { partSize: number; queueSize: number };
|
||||
expect(ctorArg.partSize).toBe(5 * 1024 * 1024);
|
||||
expect(ctorArg.queueSize).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
|
||||
/**
|
||||
* S3 multipart-upload sink for streaming exports. Returns a `writeStream`
|
||||
* that callers pipe their serialised output into and a `completionPromise`
|
||||
* that resolves with the S3 ETag once the multipart upload finalises.
|
||||
*
|
||||
* The underlying `@aws-sdk/lib-storage` `Upload` class handles part
|
||||
* batching, retries, and `AbortMultipartUpload` on failure, so callers do
|
||||
* not need to track orphaned multipart uploads themselves.
|
||||
*/
|
||||
export interface CreateS3MultipartUploadSinkOptions {
|
||||
client: S3Client;
|
||||
bucket: string;
|
||||
key: string;
|
||||
contentType?: string;
|
||||
metadata?: Record<string, string>;
|
||||
/** Override the part size; defaults to 8 MB (S3 minimum is 5 MB). */
|
||||
partSizeBytes?: number;
|
||||
/** Override concurrent part-upload concurrency; defaults to 4. */
|
||||
queueSize?: number;
|
||||
/** Test seam: inject an alternative Upload implementation. */
|
||||
uploadCtor?: typeof Upload;
|
||||
}
|
||||
|
||||
export interface S3MultipartUploadSink {
|
||||
writeStream: PassThrough;
|
||||
completionPromise: Promise<{ bucket: string; key: string; etag: string | null }>;
|
||||
}
|
||||
|
||||
const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
|
||||
const DEFAULT_QUEUE_SIZE = 4;
|
||||
|
||||
export function createS3MultipartUploadSink(options: CreateS3MultipartUploadSinkOptions): S3MultipartUploadSink {
|
||||
const writeStream = new PassThrough();
|
||||
// PassThrough emits 'error' synchronously on destroy(err). Producers
|
||||
// surface failures through completionPromise, not the stream, so attach
|
||||
// a noop listener to keep Node from treating the destroy-triggered error
|
||||
// as an uncaught exception.
|
||||
writeStream.on("error", () => undefined);
|
||||
const Ctor = options.uploadCtor ?? Upload;
|
||||
|
||||
const upload = new Ctor({
|
||||
client: options.client,
|
||||
partSize: options.partSizeBytes ?? DEFAULT_PART_SIZE_BYTES,
|
||||
queueSize: options.queueSize ?? DEFAULT_QUEUE_SIZE,
|
||||
params: {
|
||||
Bucket: options.bucket,
|
||||
Key: options.key,
|
||||
Body: writeStream,
|
||||
ContentType: options.contentType,
|
||||
Metadata: options.metadata,
|
||||
},
|
||||
});
|
||||
|
||||
const completionPromise = (async () => {
|
||||
const result = (await upload.done()) as { ETag?: string };
|
||||
return {
|
||||
bucket: options.bucket,
|
||||
key: options.key,
|
||||
etag: result.ETag ?? null,
|
||||
};
|
||||
})();
|
||||
|
||||
// Surface upload errors on the write stream so producers see them via
|
||||
// standard 'error' propagation instead of an unhandled rejection.
|
||||
completionPromise.catch((error: unknown) => {
|
||||
if (!writeStream.destroyed) {
|
||||
writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
});
|
||||
|
||||
return { writeStream, completionPromise };
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
const FORMULA_NEUTRALIZER = "'";
|
||||
const FORMULA_PREFIXES = new Set(["=", "+", "-", "@"]);
|
||||
const CONTROL_PREFIXES = new Set(["\t", "\r", "\n"]);
|
||||
|
||||
export function neutralizeSpreadsheetFormula(value: string): string {
|
||||
return hasUnsafeSpreadsheetPrefix(value) ? `${FORMULA_NEUTRALIZER}${value}` : value;
|
||||
}
|
||||
|
||||
function hasUnsafeSpreadsheetPrefix(value: string): boolean {
|
||||
for (const char of value) {
|
||||
if (FORMULA_PREFIXES.has(char) || CONTROL_PREFIXES.has(char)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (char.trim() === "") {
|
||||
continue;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportTableValidationError, normalizeExportTable } from "./types.js";
|
||||
|
||||
describe("normalizeExportTable", () => {
|
||||
it("infers deterministic columns from row key insertion order", () => {
|
||||
const table = normalizeExportTable({
|
||||
rows: [
|
||||
{ id: 1, name: "Ada" },
|
||||
{ email: "ada@example.com", id: 2 },
|
||||
],
|
||||
});
|
||||
|
||||
expect(table.columns).toEqual([
|
||||
{ key: "id", label: "id" },
|
||||
{ key: "name", label: "name" },
|
||||
{ key: "email", label: "email" },
|
||||
]);
|
||||
expect(table.rows).toEqual([
|
||||
{ id: "1", name: "Ada", email: "" },
|
||||
{ id: "2", name: "", email: "ada@example.com" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves explicit column labels and order while projecting fields", () => {
|
||||
const table = normalizeExportTable({
|
||||
columns: [
|
||||
{ key: "email", label: "Email address" },
|
||||
"id",
|
||||
],
|
||||
rows: [{ id: 7, name: "Grace", email: "grace@example.com" }],
|
||||
});
|
||||
|
||||
expect(table.columns).toEqual([
|
||||
{ key: "email", label: "Email address" },
|
||||
{ key: "id", label: "id" },
|
||||
]);
|
||||
expect(table.rows).toEqual([{ email: "grace@example.com", id: "7" }]);
|
||||
});
|
||||
|
||||
it("rejects duplicate column keys before rows are normalized", () => {
|
||||
expect(() =>
|
||||
normalizeExportTable({
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
|
||||
});
|
||||
|
||||
it("rejects malformed non-array rows", () => {
|
||||
expect(() =>
|
||||
normalizeExportTable({
|
||||
rows: { id: 1 },
|
||||
} as never),
|
||||
).toThrow(new ExportTableValidationError("Export table rows must be an array."));
|
||||
});
|
||||
|
||||
it("rejects malformed row entries", () => {
|
||||
expect(() =>
|
||||
normalizeExportTable({
|
||||
rows: [{ id: 1 }, null],
|
||||
} as never),
|
||||
).toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
|
||||
});
|
||||
|
||||
it("normalizes dates and scalar values into stable strings", () => {
|
||||
const table = normalizeExportTable({
|
||||
rows: [
|
||||
{
|
||||
active: true,
|
||||
amount: 42.5,
|
||||
big: 9007199254740993n,
|
||||
createdAt: new Date("2024-01-02T03:04:05.006Z"),
|
||||
empty: null,
|
||||
missing: undefined,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(table.rows).toEqual([
|
||||
{
|
||||
active: "true",
|
||||
amount: "42.5",
|
||||
big: "9007199254740993",
|
||||
createdAt: "2024-01-02T03:04:05.006Z",
|
||||
empty: "",
|
||||
missing: "",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves row counts after projection", () => {
|
||||
const table = normalizeExportTable({
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2, ignored: "value" }, { id: 3 }],
|
||||
});
|
||||
|
||||
expect(table.rowCount).toBe(3);
|
||||
expect(table.rows).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
export type ExportPrimitive = string | number | boolean | bigint | Date | null | undefined;
|
||||
export type ExportRecord = Record<string, ExportPrimitive>;
|
||||
|
||||
export type ExportColumn = {
|
||||
key: string;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type ExportTable = {
|
||||
columns?: readonly (string | ExportColumn)[];
|
||||
rows: readonly ExportRecord[];
|
||||
};
|
||||
|
||||
export type NormalizedExportColumn = {
|
||||
key: string;
|
||||
label: string;
|
||||
};
|
||||
|
||||
export type NormalizedExportRow = Record<string, string>;
|
||||
|
||||
export type NormalizedExportTable = {
|
||||
columns: readonly NormalizedExportColumn[];
|
||||
rows: readonly NormalizedExportRow[];
|
||||
rowCount: number;
|
||||
};
|
||||
|
||||
export class ExportTableValidationError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ExportTableValidationError";
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeExportTable(table: ExportTable): NormalizedExportTable {
|
||||
if (!Array.isArray(table.rows)) {
|
||||
throw new ExportTableValidationError("Export table rows must be an array.");
|
||||
}
|
||||
|
||||
assertRowsAreRecords(table.rows);
|
||||
|
||||
const columns = normalizeColumns(table);
|
||||
const rows = table.rows.map((row, index) => normalizeRow(row, index, columns));
|
||||
|
||||
return {
|
||||
columns,
|
||||
rows,
|
||||
rowCount: table.rows.length,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeColumns(table: ExportTable): NormalizedExportColumn[] {
|
||||
const columns = table.columns === undefined ? inferColumns(table.rows) : table.columns.map(normalizeColumn);
|
||||
const seenKeys = new Set<string>();
|
||||
|
||||
for (const column of columns) {
|
||||
if (seenKeys.has(column.key)) {
|
||||
throw new ExportTableValidationError(`Export table columns contain duplicate key "${column.key}".`);
|
||||
}
|
||||
|
||||
seenKeys.add(column.key);
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
function inferColumns(rows: readonly ExportRecord[]): NormalizedExportColumn[] {
|
||||
const seenKeys = new Set<string>();
|
||||
const columns: NormalizedExportColumn[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
for (const key of Object.keys(row)) {
|
||||
if (!seenKeys.has(key)) {
|
||||
seenKeys.add(key);
|
||||
columns.push({ key, label: key });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
function assertRowsAreRecords(rows: readonly ExportRecord[]): void {
|
||||
rows.forEach((row, index) => {
|
||||
if (row === null || typeof row !== "object" || Array.isArray(row)) {
|
||||
throw new ExportTableValidationError(`Export table row at index ${index} must be an object record.`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeColumn(column: string | ExportColumn): NormalizedExportColumn {
|
||||
if (typeof column === "string") {
|
||||
return { key: column, label: column };
|
||||
}
|
||||
|
||||
return { key: column.key, label: column.label ?? column.key };
|
||||
}
|
||||
|
||||
function normalizeRow(
|
||||
row: ExportRecord,
|
||||
index: number,
|
||||
columns: readonly NormalizedExportColumn[],
|
||||
): NormalizedExportRow {
|
||||
if (row === null || typeof row !== "object" || Array.isArray(row)) {
|
||||
throw new ExportTableValidationError(`Export table row at index ${index} must be an object record.`);
|
||||
}
|
||||
|
||||
return Object.fromEntries(columns.map((column) => [column.key, normalizePrimitive(row[column.key])]));
|
||||
}
|
||||
|
||||
function normalizePrimitive(value: ExportPrimitive): string {
|
||||
if (value === null || value === undefined) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import ExcelJS from "exceljs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportTableValidationError } from "../types.js";
|
||||
import { buildXlsxExportArtifact, serializeExportTableToXlsx } from "./index.js";
|
||||
|
||||
async function readWorkbook(bytes: Uint8Array): Promise<ExcelJS.Workbook> {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const buffer = Buffer.from(bytes);
|
||||
await workbook.xlsx.load(buffer as unknown as ArrayBuffer);
|
||||
return workbook;
|
||||
}
|
||||
|
||||
describe("@toolctrl/exports/xlsx public entrypoint", () => {
|
||||
it("exposes XLSX helpers through the dedicated subpath", async () => {
|
||||
const exportsModule = await import("./index.js");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"ExportTableValidationError",
|
||||
"ExportXlsxRowLimitExceededError",
|
||||
"XLSX_STREAM_HARD_ROW_LIMIT",
|
||||
"XLSX_STREAM_THRESHOLD",
|
||||
"buildXlsxExportArtifact",
|
||||
"serializeExportTableToXlsx",
|
||||
"serializeExportTableToXlsxStream",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToXlsx", () => {
|
||||
it("uses explicit header labels and column order", async () => {
|
||||
const bytes = await serializeExportTableToXlsx({
|
||||
columns: [
|
||||
{ key: "email", label: "Email address" },
|
||||
{ key: "id", label: "Identifier" },
|
||||
"active",
|
||||
],
|
||||
rows: [{ id: 7, email: "ada@example.com", active: true, ignored: "not exported" }],
|
||||
});
|
||||
|
||||
const worksheet = (await readWorkbook(bytes)).getWorksheet("Export");
|
||||
|
||||
expect(worksheet?.getRow(1).values).toEqual([undefined, "Email address", "Identifier", "active"]);
|
||||
expect(worksheet?.getRow(2).values).toEqual([undefined, "ada@example.com", "7", "true"]);
|
||||
});
|
||||
|
||||
it("serializes null and missing values as blank workbook cells", async () => {
|
||||
const bytes = await serializeExportTableToXlsx({
|
||||
columns: ["id", "empty", "missing"],
|
||||
rows: [{ id: 1, empty: null, missing: undefined }],
|
||||
});
|
||||
|
||||
const worksheet = (await readWorkbook(bytes)).getWorksheet("Export");
|
||||
|
||||
expect(worksheet?.getRow(2).getCell(1).value).toBe("1");
|
||||
expect(worksheet?.getRow(2).getCell(2).value).toBe("");
|
||||
expect(worksheet?.getRow(2).getCell(3).value).toBe("");
|
||||
});
|
||||
|
||||
it("stores spreadsheet formula prefixes as neutralized literal strings", async () => {
|
||||
const bytes = await serializeExportTableToXlsx({
|
||||
columns: [
|
||||
{ key: "equals", label: "=header" },
|
||||
"plus",
|
||||
"minus",
|
||||
"at",
|
||||
"spaceFormula",
|
||||
"spaceTab",
|
||||
"spaceCr",
|
||||
"spaceLf",
|
||||
"safe",
|
||||
],
|
||||
rows: [
|
||||
{
|
||||
equals: "=1+1",
|
||||
plus: "+1",
|
||||
minus: "-1",
|
||||
at: "@cmd",
|
||||
spaceFormula: " =1+1",
|
||||
spaceTab: " \tvalue",
|
||||
spaceCr: " \rvalue",
|
||||
spaceLf: " \nvalue",
|
||||
safe: " value",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const worksheet = (await readWorkbook(bytes)).getWorksheet("Export");
|
||||
|
||||
expect(worksheet?.getRow(1).values).toEqual([
|
||||
undefined,
|
||||
"'=header",
|
||||
"plus",
|
||||
"minus",
|
||||
"at",
|
||||
"spaceFormula",
|
||||
"spaceTab",
|
||||
"spaceCr",
|
||||
"spaceLf",
|
||||
"safe",
|
||||
]);
|
||||
expect(worksheet?.getRow(2).values).toEqual([
|
||||
undefined,
|
||||
"'=1+1",
|
||||
"'+1",
|
||||
"'-1",
|
||||
"'@cmd",
|
||||
"' =1+1",
|
||||
"' \tvalue",
|
||||
"' \nvalue",
|
||||
"' \nvalue",
|
||||
" value",
|
||||
]);
|
||||
expect(worksheet?.getRow(2).getCell(1).value).toBe("'=1+1");
|
||||
});
|
||||
|
||||
it("rejects malformed row input through the XLSX API", async () => {
|
||||
await expect(
|
||||
serializeExportTableToXlsx({
|
||||
rows: [{ id: 1 }, null],
|
||||
} as never),
|
||||
).rejects.toThrow(new ExportTableValidationError("Export table row at index 1 must be an object record."));
|
||||
});
|
||||
|
||||
it("rejects duplicate columns through the XLSX API", async () => {
|
||||
await expect(
|
||||
serializeExportTableToXlsx({
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow(new ExportTableValidationError('Export table columns contain duplicate key "id".'));
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildXlsxExportArtifact", () => {
|
||||
it("returns parseable workbook bytes with binary export metadata", async () => {
|
||||
const artifact = await buildXlsxExportArtifact({
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
});
|
||||
|
||||
expect(artifact.contentType).toBe("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
expect(artifact.extension).toBe("xlsx");
|
||||
expect(artifact.rowCount).toBe(2);
|
||||
expect(artifact.byteLength).toBe(artifact.bytes.byteLength);
|
||||
expect(artifact.byteLength).toBeGreaterThan(0);
|
||||
|
||||
const worksheet = (await readWorkbook(artifact.bytes)).getWorksheet("Export");
|
||||
expect(worksheet?.getRow(1).values).toEqual([undefined, "id"]);
|
||||
expect(worksheet?.getRow(2).values).toEqual([undefined, "1"]);
|
||||
expect(worksheet?.getRow(3).values).toEqual([undefined, "2"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,227 @@
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import ExcelJS from "exceljs";
|
||||
|
||||
import { XLSX_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_THRESHOLD } from "../format-caps.js";
|
||||
import { neutralizeSpreadsheetFormula } from "../spreadsheet-safety.js";
|
||||
import { type ExportColumn, type ExportRecord, type ExportTable, normalizeExportTable } from "../types.js";
|
||||
|
||||
export { ExportTableValidationError } from "../types.js";
|
||||
export type { ExportColumn, ExportPrimitive, ExportRecord, ExportTable } from "../types.js";
|
||||
|
||||
const XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
|
||||
/**
|
||||
* Hard cap (per spec) — even an Enterprise tenant cannot stream more than
|
||||
* 500k rows into a single XLSX artifact. ExcelJS' stream-writer has
|
||||
* limitations that make larger workbooks unreliable.
|
||||
*/
|
||||
export { XLSX_STREAM_HARD_ROW_LIMIT };
|
||||
|
||||
/**
|
||||
* Row count above which the streamer switches from the in-memory ExcelJS
|
||||
* Workbook to the stream-writer. Stream-writer cells cannot be re-read
|
||||
* after commit, so we keep the simpler in-memory path for small exports.
|
||||
*/
|
||||
export { XLSX_STREAM_THRESHOLD };
|
||||
|
||||
export class ExportXlsxRowLimitExceededError extends Error {
|
||||
readonly code = "EXPORT_ROW_LIMIT_EXCEEDED";
|
||||
constructor(rowCount: number, limit: number) {
|
||||
super(`XLSX export exceeds the ${limit}-row hard cap (got ${rowCount}); split the export or use CSV/NDJSON.`);
|
||||
this.name = "ExportXlsxRowLimitExceededError";
|
||||
}
|
||||
}
|
||||
|
||||
export type XlsxExportOptions = {
|
||||
sheetName?: string;
|
||||
};
|
||||
|
||||
export type XlsxExportArtifact = {
|
||||
bytes: Uint8Array;
|
||||
contentType: typeof XLSX_CONTENT_TYPE;
|
||||
extension: "xlsx";
|
||||
rowCount: number;
|
||||
byteLength: number;
|
||||
};
|
||||
|
||||
export type XlsxStreamTable = {
|
||||
columns?: readonly (string | ExportColumn)[];
|
||||
rows: Iterable<ExportRecord> | AsyncIterable<ExportRecord>;
|
||||
estimatedRowCount?: number;
|
||||
};
|
||||
|
||||
export type XlsxStreamOptions = XlsxExportOptions & {
|
||||
strategy?: "auto" | "stream" | "memory";
|
||||
hardRowLimit?: number;
|
||||
};
|
||||
|
||||
export async function serializeExportTableToXlsx(
|
||||
table: ExportTable,
|
||||
options: XlsxExportOptions = {},
|
||||
): Promise<Uint8Array> {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const worksheet = workbook.addWorksheet(options.sheetName ?? "Export");
|
||||
|
||||
worksheet.addRow(normalizedTable.columns.map((column) => neutralizeSpreadsheetFormula(column.label)));
|
||||
|
||||
for (const row of normalizedTable.rows) {
|
||||
worksheet.addRow(normalizedTable.columns.map((column) => neutralizeSpreadsheetFormula(row[column.key])));
|
||||
}
|
||||
|
||||
const buffer = await workbook.xlsx.writeBuffer();
|
||||
return buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming XLSX writer. Picks ExcelJS' stream-writer for large exports
|
||||
* (above {@link XLSX_STREAM_THRESHOLD}) and the in-memory workbook for
|
||||
* small ones. Always enforces the {@link XLSX_STREAM_HARD_ROW_LIMIT} hard
|
||||
* cap — independent of tenant plan — because the format is unreliable
|
||||
* above 500k rows.
|
||||
*/
|
||||
export async function serializeExportTableToXlsxStream(
|
||||
writeStream: Writable,
|
||||
table: XlsxStreamTable,
|
||||
options: XlsxStreamOptions = {},
|
||||
): Promise<{ rowCount: number; strategy: "stream" | "memory" }> {
|
||||
const hardLimit = options.hardRowLimit ?? XLSX_STREAM_HARD_ROW_LIMIT;
|
||||
if (table.estimatedRowCount !== undefined && table.estimatedRowCount > hardLimit) {
|
||||
throw new ExportXlsxRowLimitExceededError(table.estimatedRowCount, hardLimit);
|
||||
}
|
||||
const columns = resolveColumns(table);
|
||||
const strategy = pickStrategy(table.estimatedRowCount, options.strategy);
|
||||
|
||||
if (strategy === "stream") {
|
||||
return runStreamWriter(writeStream, table, columns, options, hardLimit);
|
||||
}
|
||||
return runMemoryWriter(writeStream, table, columns, options, hardLimit);
|
||||
}
|
||||
|
||||
export async function buildXlsxExportArtifact(
|
||||
table: ExportTable,
|
||||
options: XlsxExportOptions = {},
|
||||
): Promise<XlsxExportArtifact> {
|
||||
const normalizedTable = normalizeExportTable(table);
|
||||
const bytes = await serializeExportTableToXlsx(table, options);
|
||||
|
||||
return {
|
||||
bytes,
|
||||
contentType: XLSX_CONTENT_TYPE,
|
||||
extension: "xlsx",
|
||||
rowCount: normalizedTable.rowCount,
|
||||
byteLength: bytes.byteLength,
|
||||
};
|
||||
}
|
||||
|
||||
function pickStrategy(estimated: number | undefined, override: XlsxStreamOptions["strategy"]): "stream" | "memory" {
|
||||
if (override === "stream" || override === "memory") return override;
|
||||
if (estimated !== undefined && estimated > XLSX_STREAM_THRESHOLD) return "stream";
|
||||
return "memory";
|
||||
}
|
||||
|
||||
function resolveColumns(table: XlsxStreamTable): { key: string; label: string }[] {
|
||||
if (table.columns === undefined) {
|
||||
throw new Error("Streaming XLSX writer requires explicit columns; cannot infer from an async row source.");
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
return table.columns.map((column) => {
|
||||
const normalized = typeof column === "string" ? { key: column, label: column } : { key: column.key, label: column.label ?? column.key };
|
||||
if (seen.has(normalized.key)) {
|
||||
throw new Error(`Export table columns contain duplicate key "${normalized.key}".`);
|
||||
}
|
||||
seen.add(normalized.key);
|
||||
return normalized;
|
||||
});
|
||||
}
|
||||
|
||||
async function runStreamWriter(
|
||||
writeStream: Writable,
|
||||
table: XlsxStreamTable,
|
||||
columns: { key: string; label: string }[],
|
||||
options: XlsxStreamOptions,
|
||||
hardLimit: number,
|
||||
): Promise<{ rowCount: number; strategy: "stream" | "memory" }> {
|
||||
const workbook = new ExcelJS.stream.xlsx.WorkbookWriter({ stream: writeStream });
|
||||
const worksheet = workbook.addWorksheet(options.sheetName ?? "Export");
|
||||
worksheet.addRow(columns.map((column) => neutralizeSpreadsheetFormula(column.label))).commit();
|
||||
|
||||
let rowCount = 0;
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
if (rowCount >= hardLimit) {
|
||||
throw new ExportXlsxRowLimitExceededError(rowCount + 1, hardLimit);
|
||||
}
|
||||
worksheet
|
||||
.addRow(columns.map((column) => neutralizeSpreadsheetFormula(stringifyValue((row as ExportRecord)[column.key]))))
|
||||
.commit();
|
||||
rowCount += 1;
|
||||
}
|
||||
worksheet.commit();
|
||||
await workbook.commit();
|
||||
return { rowCount, strategy: "stream" };
|
||||
}
|
||||
|
||||
async function runMemoryWriter(
|
||||
writeStream: Writable,
|
||||
table: XlsxStreamTable,
|
||||
columns: { key: string; label: string }[],
|
||||
options: XlsxStreamOptions,
|
||||
hardLimit: number,
|
||||
): Promise<{ rowCount: number; strategy: "stream" | "memory" }> {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
const worksheet = workbook.addWorksheet(options.sheetName ?? "Export");
|
||||
worksheet.addRow(columns.map((column) => neutralizeSpreadsheetFormula(column.label)));
|
||||
|
||||
let rowCount = 0;
|
||||
for await (const row of toAsyncIterable(table.rows)) {
|
||||
if (rowCount >= hardLimit) {
|
||||
throw new ExportXlsxRowLimitExceededError(rowCount + 1, hardLimit);
|
||||
}
|
||||
worksheet.addRow(columns.map((column) => neutralizeSpreadsheetFormula(stringifyValue((row as ExportRecord)[column.key]))));
|
||||
rowCount += 1;
|
||||
}
|
||||
const buffer = await workbook.xlsx.writeBuffer();
|
||||
const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
|
||||
await writeBuffered(writeStream, Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
||||
return { rowCount, strategy: "memory" };
|
||||
}
|
||||
|
||||
function stringifyValue(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
if (typeof value === "bigint") return value.toString();
|
||||
return String(value);
|
||||
}
|
||||
|
||||
async function writeBuffered(writeStream: Writable, chunk: Buffer): Promise<void> {
|
||||
if (!writeStream.write(chunk)) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onDrain = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (error: Error) => {
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
writeStream.off("drain", onDrain);
|
||||
writeStream.off("error", onError);
|
||||
};
|
||||
writeStream.once("drain", onDrain);
|
||||
writeStream.once("error", onError);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAsyncIterable<T>(source: Iterable<T> | AsyncIterable<T>): AsyncIterable<T> {
|
||||
if (Symbol.asyncIterator in (source as object)) {
|
||||
return source as AsyncIterable<T>;
|
||||
}
|
||||
return (async function* () {
|
||||
for (const item of source as Iterable<T>) {
|
||||
yield item;
|
||||
}
|
||||
})();
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import ExcelJS from "exceljs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
ExportXlsxRowLimitExceededError,
|
||||
XLSX_STREAM_THRESHOLD,
|
||||
serializeExportTableToXlsxStream,
|
||||
} from "./index.js";
|
||||
|
||||
async function collect(stream: PassThrough): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async function readBack(buffer: Buffer): Promise<{ rows: unknown[][] }> {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
// ExcelJS' .d.ts declares `Buffer` from an old @types/node and is
|
||||
// incompatible with the modern Buffer<ArrayBufferLike>; runtime accepts
|
||||
// either, so cast through `unknown` to satisfy the type-checker.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await workbook.xlsx.load(buffer as any);
|
||||
const sheet = workbook.worksheets[0]!;
|
||||
const rows: unknown[][] = [];
|
||||
sheet.eachRow({ includeEmpty: false }, (row) => {
|
||||
const values = row.values as unknown[];
|
||||
rows.push(values.slice(1));
|
||||
});
|
||||
return { rows };
|
||||
}
|
||||
|
||||
describe("serializeExportTableToXlsxStream", () => {
|
||||
it("writes a small workbook via the in-memory strategy and round-trips", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
const result = await serializeExportTableToXlsxStream(sink, {
|
||||
columns: [
|
||||
{ key: "id", label: "Identifier" },
|
||||
{ key: "name", label: "Name" },
|
||||
],
|
||||
rows: [
|
||||
{ id: 1, name: "Ada" },
|
||||
{ id: 2, name: "Grace" },
|
||||
],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(result).toEqual({ rowCount: 2, strategy: "memory" });
|
||||
const parsed = await readBack(await collected);
|
||||
expect(parsed.rows[0]).toEqual(["Identifier", "Name"]);
|
||||
expect(parsed.rows[1]).toEqual(["1", "Ada"]);
|
||||
expect(parsed.rows[2]).toEqual(["2", "Grace"]);
|
||||
});
|
||||
|
||||
it("switches to stream-writer when estimatedRowCount exceeds the threshold", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
async function* rows(count: number) {
|
||||
for (let i = 0; i < count; i += 1) {
|
||||
yield { id: i, value: `row-${i}` };
|
||||
}
|
||||
}
|
||||
|
||||
const result = await serializeExportTableToXlsxStream(sink, {
|
||||
columns: ["id", "value"],
|
||||
estimatedRowCount: XLSX_STREAM_THRESHOLD + 1,
|
||||
rows: rows(50),
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(result.strategy).toBe("stream");
|
||||
expect(result.rowCount).toBe(50);
|
||||
});
|
||||
|
||||
it("rejects estimatedRowCount above the hard cap before doing any work", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
await expect(
|
||||
serializeExportTableToXlsxStream(sink, {
|
||||
columns: ["id"],
|
||||
rows: [],
|
||||
estimatedRowCount: 600_000,
|
||||
}),
|
||||
).rejects.toBeInstanceOf(ExportXlsxRowLimitExceededError);
|
||||
});
|
||||
|
||||
it("enforces the hard cap mid-stream when the iterator overshoots the estimate", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
async function* rows() {
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
yield { id: i };
|
||||
}
|
||||
}
|
||||
|
||||
await expect(
|
||||
serializeExportTableToXlsxStream(
|
||||
sink,
|
||||
{ columns: ["id"], rows: rows() },
|
||||
{ strategy: "memory", hardRowLimit: 3 },
|
||||
),
|
||||
).rejects.toBeInstanceOf(ExportXlsxRowLimitExceededError);
|
||||
});
|
||||
|
||||
it("rejects duplicate column keys", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
await expect(
|
||||
serializeExportTableToXlsxStream(sink, {
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow('Export table columns contain duplicate key "id".');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"allowJs": false,
|
||||
"allowImportingTsExtensions": false,
|
||||
"lib": ["ES2022"],
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"noEmit": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in new issue
Block a user