Update hivegrid-cloud reference to latest main

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
davidandClaude Fable 5 committed 2026-07-08 16:15:37 +02:00
1 parent e98d38e716
commit 7b21524c86
3781 files changed
+373484 -225607

No files matched your search

@@ -1,5 +1,5 @@
{
"name": "@toolctrl/exports",
"name": "@hivegrid/exports",
"version": "0.0.0",
"private": true,
"type": "module",
@@ -12,6 +12,7 @@
"./pdf": "./src/pdf/index.ts",
"./s3-multipart": "./src/sinks/s3-multipart.ts",
"./types": "./src/types.ts",
"./zip": "./src/zip.ts",
"./xlsx": "./src/xlsx/index.ts"
},
"scripts": {
@@ -25,6 +26,7 @@
"exceljs": "^4.4.0"
},
"devDependencies": {
"@types/node": "^25.0.0",
"aws-sdk-client-mock": "^4.1.0"
}
}
@@ -0,0 +1,148 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import {
AUDIT_PDF_HARD_ROW_LIMIT,
buildAuditPdfDocument,
serializeAuditEventsToPdfStream,
type AuditPdfEvent,
type AuditPdfReportInput,
} from "./audit-pdf.js";
import { ExportPdfRowLimitExceededError } from "./pdf/index.js";
function makeEvent(over: Partial<AuditPdfEvent> = {}): AuditPdfEvent {
return {
id: "audit_1",
createdAt: new Date("2026-05-17T00:00:00.000Z"),
tenantId: "tenant_1",
actorType: "user",
actorId: "user_1",
actorEmail: null,
actorIp: null,
actorUserAgent: null,
action: "widget.created",
targetType: "widget",
targetId: "widget_1",
requestId: null,
sourceModule: "crm",
outcome: "success",
dataSensitivity: "internal",
regulatoryScope: ["soc2"],
metadata: null,
previousHash: null,
eventHash: "deadbeef0123456789abcdef0123456789abcdef0123456789abcdef01234567",
...over,
};
}
function makeInput(over: Partial<AuditPdfReportInput> = {}): AuditPdfReportInput {
return {
tenantId: "tenant_1",
from: "2026-05-01",
to: "2026-05-17",
filterSummary: "regulatoryScope=soc2",
integrity: { ok: true, inspected: 1, firstBreakId: null, reason: null },
events: [makeEvent()],
generatedAt: new Date("2026-05-17T12:34:56.000Z"),
generatedBy: "ops-bot",
...over,
};
}
function collectText(node: unknown): string {
if (typeof node === "string" || typeof node === "number") {
return String(node);
}
if (!node || typeof node !== "object") {
return "";
}
const children = (node as { props?: { children?: unknown } }).props?.children;
if (Array.isArray(children)) {
return children.map(collectText).join(" ");
}
return collectText(children);
}
describe("buildAuditPdfDocument — broken integrity and null-heavy events", () => {
it("renders a broken hash-chain line with the offending id and reason", () => {
const doc = buildAuditPdfDocument(
makeInput({
integrity: { ok: false, inspected: 9, firstBreakId: "audit_7", reason: "hash mismatch" },
}),
);
const text = collectText(doc);
expect(text).toContain("verifyAuditChain BROKEN");
expect(text).toContain("inspected 9 events");
expect(text).toContain("id=audit_7");
expect(text).toContain("reason=hash mismatch");
});
it("falls back to placeholders when the break id and reason are null", () => {
const doc = buildAuditPdfDocument(
makeInput({
integrity: { ok: false, inspected: 3, firstBreakId: null, reason: null },
}),
);
const text = collectText(doc);
expect(text).toContain("id=?");
expect(text).toContain("reason=unknown");
});
it("renders placeholder dashes and platform/open metadata for sparse inputs", () => {
const doc = buildAuditPdfDocument({
tenantId: null,
from: null,
to: null,
filterSummary: "",
integrity: { ok: true, inspected: 0, firstBreakId: null, reason: null },
events: [
makeEvent({
createdAt: "2026-05-17T00:00:00.000Z", // string branch, not a Date
actorId: null,
targetType: null,
targetId: null,
regulatoryScope: [],
eventHash: null,
}),
],
// generatedBy intentionally omitted, generatedAt defaults to now
});
const text = collectText(doc);
// tenant null -> "(platform)", range null -> "(open)", empty filter -> "none"
expect(text).toContain("(platform)");
expect(text).toContain("(open)");
expect(text).toContain("none");
// regulatoryScope [] and eventHash null both collapse to "-"
expect(text).toContain("-");
});
});
describe("serializeAuditEventsToPdfStream", () => {
it("pipes a real PDF byte stream into the sink and reports the row count", async () => {
const sink = new PassThrough();
const chunks: Buffer[] = [];
sink.on("data", (chunk: Buffer) => chunks.push(chunk));
const result = await serializeAuditEventsToPdfStream(sink, makeInput({ events: [makeEvent(), makeEvent({ id: "audit_2" })] }));
expect(result).toEqual({ rowCount: 2 });
const buffer = Buffer.concat(chunks);
expect(buffer.byteLength).toBeGreaterThan(100);
expect(buffer.slice(0, 4).toString("utf8")).toBe("%PDF");
});
it("rejects when the event count exceeds the hard cap before rendering", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
const tooMany: AuditPdfEvent[] = Array.from({ length: AUDIT_PDF_HARD_ROW_LIMIT + 1 }, (_, i) =>
makeEvent({ id: `e_${i}` }),
);
await expect(
serializeAuditEventsToPdfStream(sink, makeInput({ events: tooMany })),
).rejects.toBeInstanceOf(ExportPdfRowLimitExceededError);
});
});
@@ -7,7 +7,7 @@
* integrity outcome; following pages list events as a tabular log.
*
* The renderer is host-agnostic and async — both the operator CLI
* (`toolctrl audit export-pdf …`) and the admin web export route use
* (`hivegrid audit export-pdf …`) and the admin web export route use
* it via `serializeAuditEventsToPdf` / `serializeAuditEventsToPdfStream`.
*
* See docs/improvement-plan/10-compliance-polish/compliance-audit-export.md.
@@ -3,9 +3,9 @@ import { describe, expect, it } from "vitest";
import { ExportTableValidationError } from "../types.js";
import { buildCsvExportArtifact, serializeExportTableToCsv } from "./index.js";
describe("@toolctrl/exports/csv public entrypoint", () => {
describe("@hivegrid/exports/csv public entrypoint", () => {
it("exposes CSV-only runtime helpers without loading unrelated exporters", async () => {
const exportsModule = await import("@toolctrl/exports/csv");
const exportsModule = await import("@hivegrid/exports/csv");
expect(Object.keys(exportsModule).sort()).toEqual([
"ExportTableValidationError",
@@ -0,0 +1,131 @@
import { Writable } from "node:stream";
import { describe, expect, it } from "vitest";
import { serializeExportTableToCsvStream } from "./index.js";
/**
* Writable that always reports back-pressure (highWaterMark 1) and defers its
* write callback to the next event-loop tick. Forces every `writeStream.write`
* to return `false`, driving the producer through the drain-wait branch of
* `writeChunk` that a fast PassThrough sink never exercises.
*/
class BackpressureSink extends Writable {
public readonly chunks: Buffer[] = [];
constructor() {
super({ highWaterMark: 1 });
}
_write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
this.chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
setImmediate(callback);
}
text(): string {
return Buffer.concat(this.chunks).toString("utf8");
}
}
/** Writable whose deferred write callback fails, so writeChunk hits the error branch. */
class ExplodingSink extends Writable {
constructor(private readonly failure: Error) {
super({ highWaterMark: 1 });
}
_write(_chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
setImmediate(() => callback(this.failure));
}
}
describe("serializeExportTableToCsvStream — back-pressure and error propagation", () => {
it("waits for drain on a back-pressured sink without corrupting output", async () => {
const sink = new BackpressureSink();
const result = await serializeExportTableToCsvStream(sink, {
columns: ["id", "name"],
rows: [
{ id: 1, name: "Ada" },
{ id: 2, name: "Grace" },
],
});
expect(result.rowCount).toBe(2);
expect(sink.text()).toBe(['"id","name"', '"1","Ada"', '"2","Grace"'].join("\n"));
});
it("rejects when the sink emits an error mid-write", async () => {
const sink = new ExplodingSink(new Error("sink exploded"));
await expect(
serializeExportTableToCsvStream(sink, {
columns: ["id"],
rows: [{ id: 1 }],
}),
).rejects.toThrow("sink exploded");
});
});
describe("serializeExportTableToCsvStream — input validation", () => {
it("throws when a row is not an object record", async () => {
const sink = new BackpressureSink();
await expect(
serializeExportTableToCsvStream(sink, {
columns: ["id"],
rows: [42 as never],
}),
).rejects.toThrow("Export table row must be an object record.");
});
it("treats arrays and null as non-record rows", async () => {
const sink = new BackpressureSink();
await expect(
serializeExportTableToCsvStream(sink, {
columns: ["id"],
rows: [[1, 2] as never],
}),
).rejects.toThrow("Export table row must be an object record.");
const nullSink = new BackpressureSink();
await expect(
serializeExportTableToCsvStream(nullSink, {
columns: ["id"],
rows: [null as never],
}),
).rejects.toThrow("Export table row must be an object record.");
});
it("throws when columns are omitted for an async row source", async () => {
const sink = new BackpressureSink();
await expect(
serializeExportTableToCsvStream(sink, {
rows: [{ id: 1 }],
} as never),
).rejects.toThrow("Streaming CSV writer requires explicit columns; cannot infer from an async row source.");
});
});
describe("serializeExportTableToCsvStream — value stringification", () => {
it("renders dates, bigints and null/undefined deterministically and labels default to the key", async () => {
const sink = new BackpressureSink();
const result = await serializeExportTableToCsvStream(sink, {
// object column without a label -> label falls back to the key
columns: [{ key: "createdAt" }, "big", "missing", "blank"],
rows: [
{
createdAt: new Date("2024-01-02T03:04:05.006Z"),
big: 9007199254740993n,
missing: undefined,
blank: null,
},
],
});
expect(result.rowCount).toBe(1);
expect(sink.text()).toBe(
[
'"createdAt","big","missing","blank"',
'"2024-01-02T03:04:05.006Z","9007199254740993","",""',
].join("\n"),
);
});
});
@@ -9,17 +9,18 @@ import {
buildJsonExportArtifact,
buildPdfExportArtifact,
buildXlsxExportArtifact,
createStoredZipBuffer,
normalizeExportTable,
serializeExportTableToCsv,
serializeExportTableToJson,
serializeExportTableToJsonLines,
serializeExportTableToPdf,
serializeExportTableToXlsx,
} from "@toolctrl/exports";
} from "@hivegrid/exports";
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
describe("@toolctrl/exports public entrypoint", () => {
describe("@hivegrid/exports public entrypoint", () => {
it("publishes dedicated subpaths so lightweight consumers can avoid the aggregate entrypoint", async () => {
const packageJson = JSON.parse(
await readFile(join(packageRoot, "package.json"), "utf8"),
@@ -34,12 +35,13 @@ describe("@toolctrl/exports public entrypoint", () => {
"./pdf": "./src/pdf/index.ts",
"./s3-multipart": "./src/sinks/s3-multipart.ts",
"./types": "./src/types.ts",
"./zip": "./src/zip.ts",
"./xlsx": "./src/xlsx/index.ts",
});
});
it("exposes CSV, JSON, shared validation, XLSX, and PDF APIs exactly once", async () => {
const exportsModule = await import("@toolctrl/exports");
it("exposes CSV, JSON, shared validation, ZIP, XLSX, and PDF APIs exactly once", async () => {
const exportsModule = await import("@hivegrid/exports");
const expectedExports = [
"AUDIT_PDF_CONTENT_TYPE",
"AUDIT_PDF_HARD_ROW_LIMIT",
@@ -60,6 +62,7 @@ describe("@toolctrl/exports public entrypoint", () => {
"buildPdfExportArtifact",
"buildXlsxExportArtifact",
"createS3MultipartUploadSink",
"createStoredZipBuffer",
"normalizeExportTable",
"resolveExportRowCap",
"serializeAuditEventsToPdf",
@@ -83,6 +86,7 @@ describe("@toolctrl/exports public entrypoint", () => {
expect(exportsModule.buildJsonExportArtifact).toBe(buildJsonExportArtifact);
expect(exportsModule.buildPdfExportArtifact).toBe(buildPdfExportArtifact);
expect(exportsModule.buildXlsxExportArtifact).toBe(buildXlsxExportArtifact);
expect(exportsModule.createStoredZipBuffer).toBe(createStoredZipBuffer);
expect(exportsModule.normalizeExportTable).toBe(normalizeExportTable);
expect(exportsModule.serializeExportTableToCsv).toBe(serializeExportTableToCsv);
expect(exportsModule.serializeExportTableToJson).toBe(serializeExportTableToJson);
@@ -6,3 +6,4 @@ export * from "./types.js";
export * from "./sinks/s3-multipart.js";
export * from "./limits.js";
export * from "./audit-pdf.js";
export * from "./zip.js";
@@ -7,7 +7,7 @@ import {
serializeExportTableToJsonLines,
} from "./index.js";
describe("@toolctrl/exports/json public entrypoint", () => {
describe("@hivegrid/exports/json public entrypoint", () => {
it("exposes JSON-only runtime helpers", async () => {
const exportsModule = await import("./index.js");
@@ -0,0 +1,103 @@
import { Writable } from "node:stream";
import { describe, expect, it } from "vitest";
import { PRETTY_JSON_STREAM_ROW_LIMIT, serializeExportTableToJsonStream } from "./index.js";
/** Back-pressured sink (highWaterMark 1, deferred callback) for the drain branch. */
class BackpressureSink extends Writable {
public readonly chunks: Buffer[] = [];
constructor() {
super({ highWaterMark: 1 });
}
_write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
this.chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
setImmediate(callback);
}
text(): string {
return Buffer.concat(this.chunks).toString("utf8");
}
}
/** Sink whose deferred callback fails, so writeChunk takes the error branch. */
class ExplodingSink extends Writable {
constructor(private readonly failure: Error) {
super({ highWaterMark: 1 });
}
_write(_chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
setImmediate(() => callback(this.failure));
}
}
describe("serializeExportTableToJsonStream — back-pressure and error propagation", () => {
it("drains a back-pressured sink and still emits well-formed ndjson", async () => {
const sink = new BackpressureSink();
const result = await serializeExportTableToJsonStream(sink, {
// object columns drive the non-string branch of resolveJsonColumnKeys
columns: [{ key: "id" }, { key: "name" }],
rows: [
{ id: 1, name: "Ada" },
{ id: 2, name: "Grace" },
],
});
expect(result).toEqual({ rowCount: 2, mode: "ndjson" });
expect(sink.text()).toBe('{"id":1,"name":"Ada"}\n{"id":2,"name":"Grace"}');
});
it("rejects when the sink errors mid-write", async () => {
const sink = new ExplodingSink(new Error("json sink down"));
await expect(
serializeExportTableToJsonStream(sink, {
columns: ["id"],
rows: [{ id: 1 }],
}),
).rejects.toThrow("json sink down");
});
});
describe("serializeExportTableToJsonStream — column resolution", () => {
it("throws when columns are omitted for an async row source", async () => {
const sink = new BackpressureSink();
await expect(
serializeExportTableToJsonStream(sink, { rows: [{ id: 1 }] } as never),
).rejects.toThrow("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
});
it("rejects duplicate column keys eagerly", async () => {
const sink = new BackpressureSink();
await expect(
serializeExportTableToJsonStream(sink, {
columns: ["id", { key: "id", label: "Identifier" }],
rows: [{ id: 1 }],
}),
).rejects.toThrow('Export table columns contain duplicate key "id".');
});
});
describe("serializeExportTableToJsonStream — pretty mode boundary", () => {
it("accepts exactly the row limit without throwing", async () => {
const sink = new BackpressureSink();
async function* rows() {
for (let i = 0; i < PRETTY_JSON_STREAM_ROW_LIMIT; i += 1) {
yield { id: i };
}
}
const result = await serializeExportTableToJsonStream(
sink,
{ columns: ["id"], rows: rows() },
{ mode: "pretty" },
);
expect(result).toEqual({ rowCount: PRETTY_JSON_STREAM_ROW_LIMIT, mode: "pretty" });
const parsed = JSON.parse(sink.text()) as { id: number }[];
expect(parsed).toHaveLength(PRETTY_JSON_STREAM_ROW_LIMIT);
expect(parsed[0]).toEqual({ id: 0 });
expect(parsed.at(-1)).toEqual({ id: PRETTY_JSON_STREAM_ROW_LIMIT - 1 });
});
});
@@ -0,0 +1,32 @@
import { describe, expect, it } from "vitest";
import { EXPORT_PLAN_ROW_CAPS, ExportRowCapExceededError, resolveExportRowCap } from "./limits.js";
describe("resolveExportRowCap — unknown format fallback", () => {
it("falls back to an unbounded format cap for unrecognised formats", () => {
// Unknown format -> FORMAT_HARD_CAPS lookup misses -> Infinity fallback;
// the enterprise plan is also unbounded, so the effective cap is Infinity.
expect(resolveExportRowCap("enterprise", "parquet" as never)).toBe(Number.POSITIVE_INFINITY);
});
it("still applies the plan cap when only the format is unknown", () => {
expect(resolveExportRowCap("free", "parquet" as never)).toBe(EXPORT_PLAN_ROW_CAPS.free);
});
});
describe("ExportRowCapExceededError message", () => {
it("describes an unbounded cap as 'unlimited' rather than 'Infinity'", () => {
const error = new ExportRowCapExceededError(100, Number.POSITIVE_INFINITY, "enterprise", "csv");
expect(error.message).toContain("unlimited");
expect(error.message).not.toContain("Infinity");
expect(error.code).toBe("EXPORT_ROW_LIMIT_PLAN");
});
it("prints the numeric cap when it is finite", () => {
const error = new ExportRowCapExceededError(20_000, 10_000, "free", "csv");
expect(error.message).toContain("10000");
expect(error.message).not.toContain("unlimited");
});
});
@@ -13,7 +13,7 @@ import { PDF_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_HARD_ROW_LIMIT } from "./format-
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
describe("@toolctrl/exports/limits public entrypoint", () => {
describe("@hivegrid/exports/limits public entrypoint", () => {
it("exposes row-cap helpers without importing heavy PDF/XLSX runtimes", async () => {
const exportsModule = await import("./limits.js");
@@ -9,7 +9,7 @@ function decodeHeader(bytes: Uint8Array): string {
return new TextDecoder().decode(bytes.slice(0, PDF_HEADER.length));
}
describe("@toolctrl/exports/pdf public entrypoint", () => {
describe("@hivegrid/exports/pdf public entrypoint", () => {
it("exposes PDF helpers through the dedicated subpath", async () => {
const exportsModule = await import("./index.js");
@@ -0,0 +1,31 @@
import { PassThrough } from "node:stream";
import { describe, expect, it } from "vitest";
import { serializeExportTableToPdfStream } from "./index.js";
async function collect(stream: PassThrough): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
describe("serializeExportTableToPdfStream — title omitted", () => {
it("renders a valid PDF without a title heading", async () => {
const sink = new PassThrough();
const collected = collect(sink);
// No `title` option -> the title <Text> node is skipped (null branch).
const result = await serializeExportTableToPdfStream(sink, {
columns: [{ key: "id", label: "Identifier" }],
rows: [{ id: 1 }, { id: 2 }],
});
sink.end();
expect(result).toEqual({ rowCount: 2 });
const buffer = await collected;
expect(buffer.slice(0, 5).toString("utf8")).toBe("%PDF-");
expect(buffer.byteLength).toBeGreaterThan(100);
});
});
@@ -0,0 +1,91 @@
import { describe, expect, it, vi } from "vitest";
import { mockClient } from "aws-sdk-client-mock";
import {
CompleteMultipartUploadCommand,
CreateMultipartUploadCommand,
PutObjectCommand,
S3Client,
UploadPartCommand,
} from "@aws-sdk/client-s3";
import { createS3MultipartUploadSink } from "./s3-multipart.js";
describe("createS3MultipartUploadSink — default Upload implementation", () => {
it("uses the real lib-storage Upload when no uploadCtor is injected", async () => {
const s3Mock = mockClient(S3Client);
// Cover both the single-part (PutObject) and multipart code paths so the
// assertion does not depend on lib-storage's body-size heuristic.
s3Mock.on(PutObjectCommand).resolves({ ETag: '"single-part-etag"' });
s3Mock.on(CreateMultipartUploadCommand).resolves({ UploadId: "mpu-1" });
s3Mock.on(UploadPartCommand).resolves({ ETag: '"part-etag"' });
s3Mock.on(CompleteMultipartUploadCommand).resolves({ ETag: '"multipart-etag"' });
const client = new S3Client({ region: "eu-central-1" });
const { writeStream, completionPromise } = createS3MultipartUploadSink({
client,
bucket: "exports",
key: "tenant/abc/users.csv",
contentType: "text/csv; charset=utf-8",
});
writeStream.end("id\n1\n2\n");
const result = await completionPromise;
expect(result.bucket).toBe("exports");
expect(result.key).toBe("tenant/abc/users.csv");
expect(typeof result.etag === "string" || result.etag === null).toBe(true);
s3Mock.restore();
});
});
describe("createS3MultipartUploadSink — failure when stream already destroyed", () => {
it("does not double-destroy a write stream that the caller already destroyed", async () => {
const failure = new Error("multipart aborted");
const done = vi.fn().mockRejectedValue(failure);
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
const { writeStream, completionPromise } = createS3MultipartUploadSink({
client: {} as never,
bucket: "exports",
key: "x.csv",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
uploadCtor: UploadStub as any,
});
const destroySpy = vi.spyOn(writeStream, "destroy");
// Destroy before the rejection handler runs so the `!destroyed` guard
// takes its false branch and skips the redundant destroy.
writeStream.destroy();
expect(writeStream.destroyed).toBe(true);
const callsAfterManualDestroy = destroySpy.mock.calls.length;
await expect(completionPromise).rejects.toBe(failure);
await new Promise((resolve) => setImmediate(resolve));
expect(destroySpy.mock.calls.length).toBe(callsAfterManualDestroy);
});
it("wraps a non-Error rejection value into an Error before destroying the stream", async () => {
const done = vi.fn().mockRejectedValue("plain string failure");
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
const { writeStream, completionPromise } = createS3MultipartUploadSink({
client: {} as never,
bucket: "b",
key: "k",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
uploadCtor: UploadStub as any,
});
const errorListener = vi.fn();
writeStream.on("error", errorListener);
await expect(completionPromise).rejects.toBe("plain string failure");
await new Promise((resolve) => setImmediate(resolve));
expect(writeStream.destroyed).toBe(true);
const wrapped = errorListener.mock.calls[0]?.[0] as Error;
expect(wrapped).toBeInstanceOf(Error);
expect(wrapped.message).toBe("plain string failure");
});
});
@@ -0,0 +1,18 @@
import { describe, expect, it } from "vitest";
import { normalizeExportTable } from "./types.js";
describe("normalizeExportTable — object columns without an explicit label", () => {
it("falls back to the column key as the label", () => {
const table = normalizeExportTable({
columns: [{ key: "id" }, { key: "email", label: "Email address" }],
rows: [{ id: 1, email: "ada@example.com" }],
});
expect(table.columns).toEqual([
{ key: "id", label: "id" },
{ key: "email", label: "Email address" },
]);
expect(table.rows).toEqual([{ id: "1", email: "ada@example.com" }]);
});
});
@@ -11,7 +11,7 @@ async function readWorkbook(bytes: Uint8Array): Promise<ExcelJS.Workbook> {
return workbook;
}
describe("@toolctrl/exports/xlsx public entrypoint", () => {
describe("@hivegrid/exports/xlsx public entrypoint", () => {
it("exposes XLSX helpers through the dedicated subpath", async () => {
const exportsModule = await import("./index.js");
@@ -0,0 +1,111 @@
import { PassThrough, Writable } from "node:stream";
import ExcelJS from "exceljs";
import { describe, expect, it } from "vitest";
import { ExportXlsxRowLimitExceededError, serializeExportTableToXlsxStream } from "./index.js";
async function readBack(buffer: Buffer): Promise<unknown[][]> {
const workbook = new ExcelJS.Workbook();
// ExcelJS' bundled @types/node Buffer is incompatible with the modern
// Buffer type; runtime accepts either, so cast through `any`.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await workbook.xlsx.load(buffer as any);
const sheet = workbook.worksheets[0]!;
const rows: unknown[][] = [];
sheet.eachRow({ includeEmpty: false }, (row) => {
rows.push((row.values as unknown[]).slice(1));
});
return rows;
}
/** Back-pressured sink (highWaterMark 1, deferred callback) for writeBuffered's drain branch. */
class BackpressureSink extends Writable {
public readonly chunks: Buffer[] = [];
constructor() {
super({ highWaterMark: 1 });
}
_write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
this.chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
setImmediate(callback);
}
buffer(): Buffer {
return Buffer.concat(this.chunks);
}
}
/** Sink whose deferred callback fails so writeBuffered hits the error branch. */
class ExplodingSink extends Writable {
constructor(private readonly failure: Error) {
super({ highWaterMark: 1 });
}
_write(_chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
setImmediate(() => callback(this.failure));
}
}
describe("serializeExportTableToXlsxStream — column resolution", () => {
it("throws when columns are omitted for an async row source", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
await expect(
serializeExportTableToXlsxStream(sink, { rows: [{ id: 1 }] } as never),
).rejects.toThrow("Streaming XLSX writer requires explicit columns; cannot infer from an async row source.");
});
});
describe("serializeExportTableToXlsxStream — hard cap in stream strategy", () => {
it("enforces the hard cap mid-stream when forced onto the stream-writer", async () => {
const sink = new PassThrough();
sink.on("data", () => undefined);
async function* rows() {
for (let i = 0; i < 10; i += 1) {
yield { id: i };
}
}
await expect(
serializeExportTableToXlsxStream(
sink,
{ columns: ["id"], rows: rows() },
{ strategy: "stream", hardRowLimit: 3 },
),
).rejects.toBeInstanceOf(ExportXlsxRowLimitExceededError);
});
});
describe("serializeExportTableToXlsxStream — memory writer flush", () => {
it("flushes the buffered workbook through a back-pressured sink and round-trips", async () => {
const sink = new BackpressureSink();
const result = await serializeExportTableToXlsxStream(sink, {
// object column without label -> label falls back to the key
columns: [{ key: "id" }, "createdAt", "big", "blank"],
rows: [
{
id: 1,
createdAt: new Date("2024-01-02T03:04:05.006Z"),
big: 9007199254740993n,
blank: null,
},
],
});
expect(result).toEqual({ rowCount: 1, strategy: "memory" });
const rows = await readBack(sink.buffer());
expect(rows[0]).toEqual(["id", "createdAt", "big", "blank"]);
expect(rows[1]).toEqual(["1", "2024-01-02T03:04:05.006Z", "9007199254740993", ""]);
});
it("rejects when the sink errors while flushing the workbook buffer", async () => {
const sink = new ExplodingSink(new Error("xlsx sink down"));
await expect(
serializeExportTableToXlsxStream(sink, {
columns: ["id"],
rows: [{ id: 1 }],
}),
).rejects.toThrow("xlsx sink down");
});
});
@@ -0,0 +1,42 @@
import { describe, expect, it } from "vitest";
import { createStoredZipBuffer } from "./zip.js";
function listZipEntries(zip: Buffer): string[] {
const names: string[] = [];
let offset = zip.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
while (offset >= 0 && zip.readUInt32LE(offset) === 0x02014b50) {
const nameLength = zip.readUInt16LE(offset + 28);
const extraLength = zip.readUInt16LE(offset + 30);
const commentLength = zip.readUInt16LE(offset + 32);
names.push(zip.subarray(offset + 46, offset + 46 + nameLength).toString("utf8"));
offset += 46 + nameLength + extraLength + commentLength;
}
return names;
}
describe("createStoredZipBuffer", () => {
it("creates a standards-shaped ZIP with local files and central directory", () => {
const zip = createStoredZipBuffer([
{ path: "README.md", data: "hello", modifiedAt: new Date("2026-07-02T12:00:00.000Z") },
{ path: "manifest.json", data: JSON.stringify({ ok: true }) },
]);
expect(zip.readUInt32LE(0)).toBe(0x04034b50);
expect(zip.includes(Buffer.from("README.md"))).toBe(true);
expect(zip.includes(Buffer.from("manifest.json"))).toBe(true);
expect(listZipEntries(zip)).toEqual(["README.md", "manifest.json"]);
expect(zip.readUInt32LE(zip.byteLength - 22)).toBe(0x06054b50);
});
it("rejects empty, unsafe, and duplicate entries", () => {
expect(() => createStoredZipBuffer([])).toThrow("at least one entry");
expect(() => createStoredZipBuffer([{ path: "../secret.txt", data: "x" }])).toThrow("Invalid ZIP entry path");
expect(() =>
createStoredZipBuffer([
{ path: "same.txt", data: "1" },
{ path: "same.txt", data: "2" },
]),
).toThrow("Duplicate ZIP entry path");
});
});
@@ -0,0 +1,152 @@
export type ZipEntryInput = {
path: string;
data: string | Uint8Array;
modifiedAt?: Date;
contentType?: string;
};
type NormalizedZipEntry = {
path: string;
data: Uint8Array;
crc32: number;
modifiedAt: Date;
};
const UTF8_FLAG = 1 << 11;
const STORE_METHOD = 0;
const ZIP_VERSION_NEEDED = 20;
const ZIP_VERSION_MADE_BY = 20;
export function createStoredZipBuffer(entries: readonly ZipEntryInput[]): Buffer {
if (entries.length === 0) {
throw new Error("ZIP bundle requires at least one entry.");
}
const normalized = entries.map(normalizeEntry);
const seen = new Set<string>();
for (const entry of normalized) {
if (seen.has(entry.path)) {
throw new Error(`Duplicate ZIP entry path: ${entry.path}`);
}
seen.add(entry.path);
}
const localParts: Buffer[] = [];
const centralParts: Buffer[] = [];
let offset = 0;
for (const entry of normalized) {
const fileName = Buffer.from(entry.path, "utf8");
const localHeader = buildLocalFileHeader(entry, fileName);
localParts.push(localHeader, Buffer.from(entry.data));
centralParts.push(buildCentralDirectoryHeader(entry, fileName, offset));
offset += localHeader.byteLength + entry.data.byteLength;
}
const centralDirectory = Buffer.concat(centralParts);
const end = buildEndOfCentralDirectory(normalized.length, centralDirectory.byteLength, offset);
return Buffer.concat([...localParts, centralDirectory, end]);
}
function normalizeEntry(entry: ZipEntryInput): NormalizedZipEntry {
const path = normalizeZipPath(entry.path);
const data = typeof entry.data === "string" ? new TextEncoder().encode(entry.data) : entry.data;
return {
path,
data,
crc32: crc32(data),
modifiedAt: entry.modifiedAt ?? new Date(),
};
}
function normalizeZipPath(path: string): string {
const normalized = path.replace(/\\/g, "/").replace(/^\/+/, "").replace(/\/+/g, "/").trim();
if (!normalized || normalized.endsWith("/") || normalized.split("/").some((part) => part === "." || part === ".." || part.length === 0)) {
throw new Error(`Invalid ZIP entry path: ${path}`);
}
return normalized;
}
function buildLocalFileHeader(entry: NormalizedZipEntry, fileName: Buffer): Buffer {
const header = Buffer.alloc(30);
const { dosTime, dosDate } = toDosDateTime(entry.modifiedAt);
header.writeUInt32LE(0x04034b50, 0);
header.writeUInt16LE(ZIP_VERSION_NEEDED, 4);
header.writeUInt16LE(UTF8_FLAG, 6);
header.writeUInt16LE(STORE_METHOD, 8);
header.writeUInt16LE(dosTime, 10);
header.writeUInt16LE(dosDate, 12);
header.writeUInt32LE(entry.crc32, 14);
header.writeUInt32LE(entry.data.byteLength, 18);
header.writeUInt32LE(entry.data.byteLength, 22);
header.writeUInt16LE(fileName.byteLength, 26);
header.writeUInt16LE(0, 28);
return Buffer.concat([header, fileName]);
}
function buildCentralDirectoryHeader(entry: NormalizedZipEntry, fileName: Buffer, localHeaderOffset: number): Buffer {
const header = Buffer.alloc(46);
const { dosTime, dosDate } = toDosDateTime(entry.modifiedAt);
header.writeUInt32LE(0x02014b50, 0);
header.writeUInt16LE(ZIP_VERSION_MADE_BY, 4);
header.writeUInt16LE(ZIP_VERSION_NEEDED, 6);
header.writeUInt16LE(UTF8_FLAG, 8);
header.writeUInt16LE(STORE_METHOD, 10);
header.writeUInt16LE(dosTime, 12);
header.writeUInt16LE(dosDate, 14);
header.writeUInt32LE(entry.crc32, 16);
header.writeUInt32LE(entry.data.byteLength, 20);
header.writeUInt32LE(entry.data.byteLength, 24);
header.writeUInt16LE(fileName.byteLength, 28);
header.writeUInt16LE(0, 30);
header.writeUInt16LE(0, 32);
header.writeUInt16LE(0, 34);
header.writeUInt16LE(0, 36);
header.writeUInt32LE(0, 38);
header.writeUInt32LE(localHeaderOffset, 42);
return Buffer.concat([header, fileName]);
}
function buildEndOfCentralDirectory(entryCount: number, centralDirectorySize: number, centralDirectoryOffset: number): Buffer {
const end = Buffer.alloc(22);
end.writeUInt32LE(0x06054b50, 0);
end.writeUInt16LE(0, 4);
end.writeUInt16LE(0, 6);
end.writeUInt16LE(entryCount, 8);
end.writeUInt16LE(entryCount, 10);
end.writeUInt32LE(centralDirectorySize, 12);
end.writeUInt32LE(centralDirectoryOffset, 16);
end.writeUInt16LE(0, 20);
return end;
}
function toDosDateTime(date: Date): { dosTime: number; dosDate: number } {
const year = Math.max(1980, date.getUTCFullYear());
const month = date.getUTCMonth() + 1;
const day = date.getUTCDate();
const hours = date.getUTCHours();
const minutes = date.getUTCMinutes();
const seconds = Math.floor(date.getUTCSeconds() / 2);
return {
dosTime: (hours << 11) | (minutes << 5) | seconds,
dosDate: ((year - 1980) << 9) | (month << 5) | day,
};
}
function crc32(data: Uint8Array): number {
let crc = 0xffffffff;
for (const byte of data) {
crc = (crc >>> 8) ^ CRC32_TABLE[(crc ^ byte) & 0xff];
}
return (crc ^ 0xffffffff) >>> 0;
}
const CRC32_TABLE = new Uint32Array(
Array.from({ length: 256 }, (_, index) => {
let value = index;
for (let bit = 0; bit < 8; bit += 1) {
value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
}
return value >>> 0;
}),
);