Update hivegrid-cloud reference to latest main
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e98d38e716
commit
7b21524c86
3781 files changed
+373484
-225607
No files matched your search
+148
@@ -0,0 +1,148 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
AUDIT_PDF_HARD_ROW_LIMIT,
|
||||
buildAuditPdfDocument,
|
||||
serializeAuditEventsToPdfStream,
|
||||
type AuditPdfEvent,
|
||||
type AuditPdfReportInput,
|
||||
} from "./audit-pdf.js";
|
||||
import { ExportPdfRowLimitExceededError } from "./pdf/index.js";
|
||||
|
||||
function makeEvent(over: Partial<AuditPdfEvent> = {}): AuditPdfEvent {
|
||||
return {
|
||||
id: "audit_1",
|
||||
createdAt: new Date("2026-05-17T00:00:00.000Z"),
|
||||
tenantId: "tenant_1",
|
||||
actorType: "user",
|
||||
actorId: "user_1",
|
||||
actorEmail: null,
|
||||
actorIp: null,
|
||||
actorUserAgent: null,
|
||||
action: "widget.created",
|
||||
targetType: "widget",
|
||||
targetId: "widget_1",
|
||||
requestId: null,
|
||||
sourceModule: "crm",
|
||||
outcome: "success",
|
||||
dataSensitivity: "internal",
|
||||
regulatoryScope: ["soc2"],
|
||||
metadata: null,
|
||||
previousHash: null,
|
||||
eventHash: "deadbeef0123456789abcdef0123456789abcdef0123456789abcdef01234567",
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
function makeInput(over: Partial<AuditPdfReportInput> = {}): AuditPdfReportInput {
|
||||
return {
|
||||
tenantId: "tenant_1",
|
||||
from: "2026-05-01",
|
||||
to: "2026-05-17",
|
||||
filterSummary: "regulatoryScope=soc2",
|
||||
integrity: { ok: true, inspected: 1, firstBreakId: null, reason: null },
|
||||
events: [makeEvent()],
|
||||
generatedAt: new Date("2026-05-17T12:34:56.000Z"),
|
||||
generatedBy: "ops-bot",
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
function collectText(node: unknown): string {
|
||||
if (typeof node === "string" || typeof node === "number") {
|
||||
return String(node);
|
||||
}
|
||||
if (!node || typeof node !== "object") {
|
||||
return "";
|
||||
}
|
||||
const children = (node as { props?: { children?: unknown } }).props?.children;
|
||||
if (Array.isArray(children)) {
|
||||
return children.map(collectText).join(" ");
|
||||
}
|
||||
return collectText(children);
|
||||
}
|
||||
|
||||
describe("buildAuditPdfDocument — broken integrity and null-heavy events", () => {
|
||||
it("renders a broken hash-chain line with the offending id and reason", () => {
|
||||
const doc = buildAuditPdfDocument(
|
||||
makeInput({
|
||||
integrity: { ok: false, inspected: 9, firstBreakId: "audit_7", reason: "hash mismatch" },
|
||||
}),
|
||||
);
|
||||
|
||||
const text = collectText(doc);
|
||||
expect(text).toContain("verifyAuditChain BROKEN");
|
||||
expect(text).toContain("inspected 9 events");
|
||||
expect(text).toContain("id=audit_7");
|
||||
expect(text).toContain("reason=hash mismatch");
|
||||
});
|
||||
|
||||
it("falls back to placeholders when the break id and reason are null", () => {
|
||||
const doc = buildAuditPdfDocument(
|
||||
makeInput({
|
||||
integrity: { ok: false, inspected: 3, firstBreakId: null, reason: null },
|
||||
}),
|
||||
);
|
||||
|
||||
const text = collectText(doc);
|
||||
expect(text).toContain("id=?");
|
||||
expect(text).toContain("reason=unknown");
|
||||
});
|
||||
|
||||
it("renders placeholder dashes and platform/open metadata for sparse inputs", () => {
|
||||
const doc = buildAuditPdfDocument({
|
||||
tenantId: null,
|
||||
from: null,
|
||||
to: null,
|
||||
filterSummary: "",
|
||||
integrity: { ok: true, inspected: 0, firstBreakId: null, reason: null },
|
||||
events: [
|
||||
makeEvent({
|
||||
createdAt: "2026-05-17T00:00:00.000Z", // string branch, not a Date
|
||||
actorId: null,
|
||||
targetType: null,
|
||||
targetId: null,
|
||||
regulatoryScope: [],
|
||||
eventHash: null,
|
||||
}),
|
||||
],
|
||||
// generatedBy intentionally omitted, generatedAt defaults to now
|
||||
});
|
||||
|
||||
const text = collectText(doc);
|
||||
// tenant null -> "(platform)", range null -> "(open)", empty filter -> "none"
|
||||
expect(text).toContain("(platform)");
|
||||
expect(text).toContain("(open)");
|
||||
expect(text).toContain("none");
|
||||
// regulatoryScope [] and eventHash null both collapse to "-"
|
||||
expect(text).toContain("-");
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeAuditEventsToPdfStream", () => {
|
||||
it("pipes a real PDF byte stream into the sink and reports the row count", async () => {
|
||||
const sink = new PassThrough();
|
||||
const chunks: Buffer[] = [];
|
||||
sink.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
|
||||
const result = await serializeAuditEventsToPdfStream(sink, makeInput({ events: [makeEvent(), makeEvent({ id: "audit_2" })] }));
|
||||
|
||||
expect(result).toEqual({ rowCount: 2 });
|
||||
const buffer = Buffer.concat(chunks);
|
||||
expect(buffer.byteLength).toBeGreaterThan(100);
|
||||
expect(buffer.slice(0, 4).toString("utf8")).toBe("%PDF");
|
||||
});
|
||||
|
||||
it("rejects when the event count exceeds the hard cap before rendering", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
const tooMany: AuditPdfEvent[] = Array.from({ length: AUDIT_PDF_HARD_ROW_LIMIT + 1 }, (_, i) =>
|
||||
makeEvent({ id: `e_${i}` }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
serializeAuditEventsToPdfStream(sink, makeInput({ events: tooMany })),
|
||||
).rejects.toBeInstanceOf(ExportPdfRowLimitExceededError);
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,7 @@
|
||||
* integrity outcome; following pages list events as a tabular log.
|
||||
*
|
||||
* The renderer is host-agnostic and async — both the operator CLI
|
||||
* (`toolctrl audit export-pdf …`) and the admin web export route use
|
||||
* (`hivegrid audit export-pdf …`) and the admin web export route use
|
||||
* it via `serializeAuditEventsToPdf` / `serializeAuditEventsToPdfStream`.
|
||||
*
|
||||
* See docs/improvement-plan/10-compliance-polish/compliance-audit-export.md.
|
||||
|
||||
@@ -3,9 +3,9 @@ import { describe, expect, it } from "vitest";
|
||||
import { ExportTableValidationError } from "../types.js";
|
||||
import { buildCsvExportArtifact, serializeExportTableToCsv } from "./index.js";
|
||||
|
||||
describe("@toolctrl/exports/csv public entrypoint", () => {
|
||||
describe("@hivegrid/exports/csv public entrypoint", () => {
|
||||
it("exposes CSV-only runtime helpers without loading unrelated exporters", async () => {
|
||||
const exportsModule = await import("@toolctrl/exports/csv");
|
||||
const exportsModule = await import("@hivegrid/exports/csv");
|
||||
|
||||
expect(Object.keys(exportsModule).sort()).toEqual([
|
||||
"ExportTableValidationError",
|
||||
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
import { Writable } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { serializeExportTableToCsvStream } from "./index.js";
|
||||
|
||||
/**
|
||||
* Writable that always reports back-pressure (highWaterMark 1) and defers its
|
||||
* write callback to the next event-loop tick. Forces every `writeStream.write`
|
||||
* to return `false`, driving the producer through the drain-wait branch of
|
||||
* `writeChunk` that a fast PassThrough sink never exercises.
|
||||
*/
|
||||
class BackpressureSink extends Writable {
|
||||
public readonly chunks: Buffer[] = [];
|
||||
constructor() {
|
||||
super({ highWaterMark: 1 });
|
||||
}
|
||||
_write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
|
||||
this.chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
setImmediate(callback);
|
||||
}
|
||||
text(): string {
|
||||
return Buffer.concat(this.chunks).toString("utf8");
|
||||
}
|
||||
}
|
||||
|
||||
/** Writable whose deferred write callback fails, so writeChunk hits the error branch. */
|
||||
class ExplodingSink extends Writable {
|
||||
constructor(private readonly failure: Error) {
|
||||
super({ highWaterMark: 1 });
|
||||
}
|
||||
_write(_chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
|
||||
setImmediate(() => callback(this.failure));
|
||||
}
|
||||
}
|
||||
|
||||
describe("serializeExportTableToCsvStream — back-pressure and error propagation", () => {
|
||||
it("waits for drain on a back-pressured sink without corrupting output", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
const result = await serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id", "name"],
|
||||
rows: [
|
||||
{ id: 1, name: "Ada" },
|
||||
{ id: 2, name: "Grace" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.rowCount).toBe(2);
|
||||
expect(sink.text()).toBe(['"id","name"', '"1","Ada"', '"2","Grace"'].join("\n"));
|
||||
});
|
||||
|
||||
it("rejects when the sink emits an error mid-write", async () => {
|
||||
const sink = new ExplodingSink(new Error("sink exploded"));
|
||||
|
||||
await expect(
|
||||
serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow("sink exploded");
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToCsvStream — input validation", () => {
|
||||
it("throws when a row is not an object record", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
await expect(
|
||||
serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id"],
|
||||
rows: [42 as never],
|
||||
}),
|
||||
).rejects.toThrow("Export table row must be an object record.");
|
||||
});
|
||||
|
||||
it("treats arrays and null as non-record rows", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
await expect(
|
||||
serializeExportTableToCsvStream(sink, {
|
||||
columns: ["id"],
|
||||
rows: [[1, 2] as never],
|
||||
}),
|
||||
).rejects.toThrow("Export table row must be an object record.");
|
||||
|
||||
const nullSink = new BackpressureSink();
|
||||
await expect(
|
||||
serializeExportTableToCsvStream(nullSink, {
|
||||
columns: ["id"],
|
||||
rows: [null as never],
|
||||
}),
|
||||
).rejects.toThrow("Export table row must be an object record.");
|
||||
});
|
||||
|
||||
it("throws when columns are omitted for an async row source", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
await expect(
|
||||
serializeExportTableToCsvStream(sink, {
|
||||
rows: [{ id: 1 }],
|
||||
} as never),
|
||||
).rejects.toThrow("Streaming CSV writer requires explicit columns; cannot infer from an async row source.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToCsvStream — value stringification", () => {
|
||||
it("renders dates, bigints and null/undefined deterministically and labels default to the key", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
const result = await serializeExportTableToCsvStream(sink, {
|
||||
// object column without a label -> label falls back to the key
|
||||
columns: [{ key: "createdAt" }, "big", "missing", "blank"],
|
||||
rows: [
|
||||
{
|
||||
createdAt: new Date("2024-01-02T03:04:05.006Z"),
|
||||
big: 9007199254740993n,
|
||||
missing: undefined,
|
||||
blank: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.rowCount).toBe(1);
|
||||
expect(sink.text()).toBe(
|
||||
[
|
||||
'"createdAt","big","missing","blank"',
|
||||
'"2024-01-02T03:04:05.006Z","9007199254740993","",""',
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -9,17 +9,18 @@ import {
|
||||
buildJsonExportArtifact,
|
||||
buildPdfExportArtifact,
|
||||
buildXlsxExportArtifact,
|
||||
createStoredZipBuffer,
|
||||
normalizeExportTable,
|
||||
serializeExportTableToCsv,
|
||||
serializeExportTableToJson,
|
||||
serializeExportTableToJsonLines,
|
||||
serializeExportTableToPdf,
|
||||
serializeExportTableToXlsx,
|
||||
} from "@toolctrl/exports";
|
||||
} from "@hivegrid/exports";
|
||||
|
||||
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
|
||||
describe("@toolctrl/exports public entrypoint", () => {
|
||||
describe("@hivegrid/exports public entrypoint", () => {
|
||||
it("publishes dedicated subpaths so lightweight consumers can avoid the aggregate entrypoint", async () => {
|
||||
const packageJson = JSON.parse(
|
||||
await readFile(join(packageRoot, "package.json"), "utf8"),
|
||||
@@ -34,12 +35,13 @@ describe("@toolctrl/exports public entrypoint", () => {
|
||||
"./pdf": "./src/pdf/index.ts",
|
||||
"./s3-multipart": "./src/sinks/s3-multipart.ts",
|
||||
"./types": "./src/types.ts",
|
||||
"./zip": "./src/zip.ts",
|
||||
"./xlsx": "./src/xlsx/index.ts",
|
||||
});
|
||||
});
|
||||
|
||||
it("exposes CSV, JSON, shared validation, XLSX, and PDF APIs exactly once", async () => {
|
||||
const exportsModule = await import("@toolctrl/exports");
|
||||
it("exposes CSV, JSON, shared validation, ZIP, XLSX, and PDF APIs exactly once", async () => {
|
||||
const exportsModule = await import("@hivegrid/exports");
|
||||
const expectedExports = [
|
||||
"AUDIT_PDF_CONTENT_TYPE",
|
||||
"AUDIT_PDF_HARD_ROW_LIMIT",
|
||||
@@ -60,6 +62,7 @@ describe("@toolctrl/exports public entrypoint", () => {
|
||||
"buildPdfExportArtifact",
|
||||
"buildXlsxExportArtifact",
|
||||
"createS3MultipartUploadSink",
|
||||
"createStoredZipBuffer",
|
||||
"normalizeExportTable",
|
||||
"resolveExportRowCap",
|
||||
"serializeAuditEventsToPdf",
|
||||
@@ -83,6 +86,7 @@ describe("@toolctrl/exports public entrypoint", () => {
|
||||
expect(exportsModule.buildJsonExportArtifact).toBe(buildJsonExportArtifact);
|
||||
expect(exportsModule.buildPdfExportArtifact).toBe(buildPdfExportArtifact);
|
||||
expect(exportsModule.buildXlsxExportArtifact).toBe(buildXlsxExportArtifact);
|
||||
expect(exportsModule.createStoredZipBuffer).toBe(createStoredZipBuffer);
|
||||
expect(exportsModule.normalizeExportTable).toBe(normalizeExportTable);
|
||||
expect(exportsModule.serializeExportTableToCsv).toBe(serializeExportTableToCsv);
|
||||
expect(exportsModule.serializeExportTableToJson).toBe(serializeExportTableToJson);
|
||||
|
||||
@@ -6,3 +6,4 @@ export * from "./types.js";
|
||||
export * from "./sinks/s3-multipart.js";
|
||||
export * from "./limits.js";
|
||||
export * from "./audit-pdf.js";
|
||||
export * from "./zip.js";
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
serializeExportTableToJsonLines,
|
||||
} from "./index.js";
|
||||
|
||||
describe("@toolctrl/exports/json public entrypoint", () => {
|
||||
describe("@hivegrid/exports/json public entrypoint", () => {
|
||||
it("exposes JSON-only runtime helpers", async () => {
|
||||
const exportsModule = await import("./index.js");
|
||||
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
import { Writable } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { PRETTY_JSON_STREAM_ROW_LIMIT, serializeExportTableToJsonStream } from "./index.js";
|
||||
|
||||
/** Back-pressured sink (highWaterMark 1, deferred callback) for the drain branch. */
|
||||
class BackpressureSink extends Writable {
|
||||
public readonly chunks: Buffer[] = [];
|
||||
constructor() {
|
||||
super({ highWaterMark: 1 });
|
||||
}
|
||||
_write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
|
||||
this.chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
setImmediate(callback);
|
||||
}
|
||||
text(): string {
|
||||
return Buffer.concat(this.chunks).toString("utf8");
|
||||
}
|
||||
}
|
||||
|
||||
/** Sink whose deferred callback fails, so writeChunk takes the error branch. */
|
||||
class ExplodingSink extends Writable {
|
||||
constructor(private readonly failure: Error) {
|
||||
super({ highWaterMark: 1 });
|
||||
}
|
||||
_write(_chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
|
||||
setImmediate(() => callback(this.failure));
|
||||
}
|
||||
}
|
||||
|
||||
describe("serializeExportTableToJsonStream — back-pressure and error propagation", () => {
|
||||
it("drains a back-pressured sink and still emits well-formed ndjson", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
const result = await serializeExportTableToJsonStream(sink, {
|
||||
// object columns drive the non-string branch of resolveJsonColumnKeys
|
||||
columns: [{ key: "id" }, { key: "name" }],
|
||||
rows: [
|
||||
{ id: 1, name: "Ada" },
|
||||
{ id: 2, name: "Grace" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({ rowCount: 2, mode: "ndjson" });
|
||||
expect(sink.text()).toBe('{"id":1,"name":"Ada"}\n{"id":2,"name":"Grace"}');
|
||||
});
|
||||
|
||||
it("rejects when the sink errors mid-write", async () => {
|
||||
const sink = new ExplodingSink(new Error("json sink down"));
|
||||
|
||||
await expect(
|
||||
serializeExportTableToJsonStream(sink, {
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow("json sink down");
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToJsonStream — column resolution", () => {
|
||||
it("throws when columns are omitted for an async row source", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
await expect(
|
||||
serializeExportTableToJsonStream(sink, { rows: [{ id: 1 }] } as never),
|
||||
).rejects.toThrow("Streaming JSON writer requires explicit columns; cannot infer from an async row source.");
|
||||
});
|
||||
|
||||
it("rejects duplicate column keys eagerly", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
await expect(
|
||||
serializeExportTableToJsonStream(sink, {
|
||||
columns: ["id", { key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow('Export table columns contain duplicate key "id".');
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToJsonStream — pretty mode boundary", () => {
|
||||
it("accepts exactly the row limit without throwing", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
async function* rows() {
|
||||
for (let i = 0; i < PRETTY_JSON_STREAM_ROW_LIMIT; i += 1) {
|
||||
yield { id: i };
|
||||
}
|
||||
}
|
||||
|
||||
const result = await serializeExportTableToJsonStream(
|
||||
sink,
|
||||
{ columns: ["id"], rows: rows() },
|
||||
{ mode: "pretty" },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ rowCount: PRETTY_JSON_STREAM_ROW_LIMIT, mode: "pretty" });
|
||||
const parsed = JSON.parse(sink.text()) as { id: number }[];
|
||||
expect(parsed).toHaveLength(PRETTY_JSON_STREAM_ROW_LIMIT);
|
||||
expect(parsed[0]).toEqual({ id: 0 });
|
||||
expect(parsed.at(-1)).toEqual({ id: PRETTY_JSON_STREAM_ROW_LIMIT - 1 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { EXPORT_PLAN_ROW_CAPS, ExportRowCapExceededError, resolveExportRowCap } from "./limits.js";
|
||||
|
||||
describe("resolveExportRowCap — unknown format fallback", () => {
|
||||
it("falls back to an unbounded format cap for unrecognised formats", () => {
|
||||
// Unknown format -> FORMAT_HARD_CAPS lookup misses -> Infinity fallback;
|
||||
// the enterprise plan is also unbounded, so the effective cap is Infinity.
|
||||
expect(resolveExportRowCap("enterprise", "parquet" as never)).toBe(Number.POSITIVE_INFINITY);
|
||||
});
|
||||
|
||||
it("still applies the plan cap when only the format is unknown", () => {
|
||||
expect(resolveExportRowCap("free", "parquet" as never)).toBe(EXPORT_PLAN_ROW_CAPS.free);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ExportRowCapExceededError message", () => {
|
||||
it("describes an unbounded cap as 'unlimited' rather than 'Infinity'", () => {
|
||||
const error = new ExportRowCapExceededError(100, Number.POSITIVE_INFINITY, "enterprise", "csv");
|
||||
|
||||
expect(error.message).toContain("unlimited");
|
||||
expect(error.message).not.toContain("Infinity");
|
||||
expect(error.code).toBe("EXPORT_ROW_LIMIT_PLAN");
|
||||
});
|
||||
|
||||
it("prints the numeric cap when it is finite", () => {
|
||||
const error = new ExportRowCapExceededError(20_000, 10_000, "free", "csv");
|
||||
|
||||
expect(error.message).toContain("10000");
|
||||
expect(error.message).not.toContain("unlimited");
|
||||
});
|
||||
});
|
||||
@@ -13,7 +13,7 @@ import { PDF_STREAM_HARD_ROW_LIMIT, XLSX_STREAM_HARD_ROW_LIMIT } from "./format-
|
||||
|
||||
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
|
||||
describe("@toolctrl/exports/limits public entrypoint", () => {
|
||||
describe("@hivegrid/exports/limits public entrypoint", () => {
|
||||
it("exposes row-cap helpers without importing heavy PDF/XLSX runtimes", async () => {
|
||||
const exportsModule = await import("./limits.js");
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ function decodeHeader(bytes: Uint8Array): string {
|
||||
return new TextDecoder().decode(bytes.slice(0, PDF_HEADER.length));
|
||||
}
|
||||
|
||||
describe("@toolctrl/exports/pdf public entrypoint", () => {
|
||||
describe("@hivegrid/exports/pdf public entrypoint", () => {
|
||||
it("exposes PDF helpers through the dedicated subpath", async () => {
|
||||
const exportsModule = await import("./index.js");
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { serializeExportTableToPdfStream } from "./index.js";
|
||||
|
||||
async function collect(stream: PassThrough): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
describe("serializeExportTableToPdfStream — title omitted", () => {
|
||||
it("renders a valid PDF without a title heading", async () => {
|
||||
const sink = new PassThrough();
|
||||
const collected = collect(sink);
|
||||
|
||||
// No `title` option -> the title <Text> node is skipped (null branch).
|
||||
const result = await serializeExportTableToPdfStream(sink, {
|
||||
columns: [{ key: "id", label: "Identifier" }],
|
||||
rows: [{ id: 1 }, { id: 2 }],
|
||||
});
|
||||
sink.end();
|
||||
|
||||
expect(result).toEqual({ rowCount: 2 });
|
||||
const buffer = await collected;
|
||||
expect(buffer.slice(0, 5).toString("utf8")).toBe("%PDF-");
|
||||
expect(buffer.byteLength).toBeGreaterThan(100);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { mockClient } from "aws-sdk-client-mock";
|
||||
import {
|
||||
CompleteMultipartUploadCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
UploadPartCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
|
||||
import { createS3MultipartUploadSink } from "./s3-multipart.js";
|
||||
|
||||
describe("createS3MultipartUploadSink — default Upload implementation", () => {
|
||||
it("uses the real lib-storage Upload when no uploadCtor is injected", async () => {
|
||||
const s3Mock = mockClient(S3Client);
|
||||
// Cover both the single-part (PutObject) and multipart code paths so the
|
||||
// assertion does not depend on lib-storage's body-size heuristic.
|
||||
s3Mock.on(PutObjectCommand).resolves({ ETag: '"single-part-etag"' });
|
||||
s3Mock.on(CreateMultipartUploadCommand).resolves({ UploadId: "mpu-1" });
|
||||
s3Mock.on(UploadPartCommand).resolves({ ETag: '"part-etag"' });
|
||||
s3Mock.on(CompleteMultipartUploadCommand).resolves({ ETag: '"multipart-etag"' });
|
||||
|
||||
const client = new S3Client({ region: "eu-central-1" });
|
||||
const { writeStream, completionPromise } = createS3MultipartUploadSink({
|
||||
client,
|
||||
bucket: "exports",
|
||||
key: "tenant/abc/users.csv",
|
||||
contentType: "text/csv; charset=utf-8",
|
||||
});
|
||||
|
||||
writeStream.end("id\n1\n2\n");
|
||||
const result = await completionPromise;
|
||||
|
||||
expect(result.bucket).toBe("exports");
|
||||
expect(result.key).toBe("tenant/abc/users.csv");
|
||||
expect(typeof result.etag === "string" || result.etag === null).toBe(true);
|
||||
s3Mock.restore();
|
||||
});
|
||||
});
|
||||
|
||||
describe("createS3MultipartUploadSink — failure when stream already destroyed", () => {
|
||||
it("does not double-destroy a write stream that the caller already destroyed", async () => {
|
||||
const failure = new Error("multipart aborted");
|
||||
const done = vi.fn().mockRejectedValue(failure);
|
||||
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
|
||||
|
||||
const { writeStream, completionPromise } = createS3MultipartUploadSink({
|
||||
client: {} as never,
|
||||
bucket: "exports",
|
||||
key: "x.csv",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
uploadCtor: UploadStub as any,
|
||||
});
|
||||
|
||||
const destroySpy = vi.spyOn(writeStream, "destroy");
|
||||
// Destroy before the rejection handler runs so the `!destroyed` guard
|
||||
// takes its false branch and skips the redundant destroy.
|
||||
writeStream.destroy();
|
||||
expect(writeStream.destroyed).toBe(true);
|
||||
const callsAfterManualDestroy = destroySpy.mock.calls.length;
|
||||
|
||||
await expect(completionPromise).rejects.toBe(failure);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(destroySpy.mock.calls.length).toBe(callsAfterManualDestroy);
|
||||
});
|
||||
|
||||
it("wraps a non-Error rejection value into an Error before destroying the stream", async () => {
|
||||
const done = vi.fn().mockRejectedValue("plain string failure");
|
||||
const UploadStub = vi.fn().mockImplementation(() => ({ done }));
|
||||
|
||||
const { writeStream, completionPromise } = createS3MultipartUploadSink({
|
||||
client: {} as never,
|
||||
bucket: "b",
|
||||
key: "k",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
uploadCtor: UploadStub as any,
|
||||
});
|
||||
|
||||
const errorListener = vi.fn();
|
||||
writeStream.on("error", errorListener);
|
||||
|
||||
await expect(completionPromise).rejects.toBe("plain string failure");
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(writeStream.destroyed).toBe(true);
|
||||
const wrapped = errorListener.mock.calls[0]?.[0] as Error;
|
||||
expect(wrapped).toBeInstanceOf(Error);
|
||||
expect(wrapped.message).toBe("plain string failure");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { normalizeExportTable } from "./types.js";
|
||||
|
||||
describe("normalizeExportTable — object columns without an explicit label", () => {
|
||||
it("falls back to the column key as the label", () => {
|
||||
const table = normalizeExportTable({
|
||||
columns: [{ key: "id" }, { key: "email", label: "Email address" }],
|
||||
rows: [{ id: 1, email: "ada@example.com" }],
|
||||
});
|
||||
|
||||
expect(table.columns).toEqual([
|
||||
{ key: "id", label: "id" },
|
||||
{ key: "email", label: "Email address" },
|
||||
]);
|
||||
expect(table.rows).toEqual([{ id: "1", email: "ada@example.com" }]);
|
||||
});
|
||||
});
|
||||
@@ -11,7 +11,7 @@ async function readWorkbook(bytes: Uint8Array): Promise<ExcelJS.Workbook> {
|
||||
return workbook;
|
||||
}
|
||||
|
||||
describe("@toolctrl/exports/xlsx public entrypoint", () => {
|
||||
describe("@hivegrid/exports/xlsx public entrypoint", () => {
|
||||
it("exposes XLSX helpers through the dedicated subpath", async () => {
|
||||
const exportsModule = await import("./index.js");
|
||||
|
||||
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
import { PassThrough, Writable } from "node:stream";
|
||||
import ExcelJS from "exceljs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { ExportXlsxRowLimitExceededError, serializeExportTableToXlsxStream } from "./index.js";
|
||||
|
||||
async function readBack(buffer: Buffer): Promise<unknown[][]> {
|
||||
const workbook = new ExcelJS.Workbook();
|
||||
// ExcelJS' bundled @types/node Buffer is incompatible with the modern
|
||||
// Buffer type; runtime accepts either, so cast through `any`.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
await workbook.xlsx.load(buffer as any);
|
||||
const sheet = workbook.worksheets[0]!;
|
||||
const rows: unknown[][] = [];
|
||||
sheet.eachRow({ includeEmpty: false }, (row) => {
|
||||
rows.push((row.values as unknown[]).slice(1));
|
||||
});
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Back-pressured sink (highWaterMark 1, deferred callback) for writeBuffered's drain branch. */
|
||||
class BackpressureSink extends Writable {
|
||||
public readonly chunks: Buffer[] = [];
|
||||
constructor() {
|
||||
super({ highWaterMark: 1 });
|
||||
}
|
||||
_write(chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
|
||||
this.chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
setImmediate(callback);
|
||||
}
|
||||
buffer(): Buffer {
|
||||
return Buffer.concat(this.chunks);
|
||||
}
|
||||
}
|
||||
|
||||
/** Sink whose deferred callback fails so writeBuffered hits the error branch. */
|
||||
class ExplodingSink extends Writable {
|
||||
constructor(private readonly failure: Error) {
|
||||
super({ highWaterMark: 1 });
|
||||
}
|
||||
_write(_chunk: Buffer | string, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
|
||||
setImmediate(() => callback(this.failure));
|
||||
}
|
||||
}
|
||||
|
||||
describe("serializeExportTableToXlsxStream — column resolution", () => {
|
||||
it("throws when columns are omitted for an async row source", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
await expect(
|
||||
serializeExportTableToXlsxStream(sink, { rows: [{ id: 1 }] } as never),
|
||||
).rejects.toThrow("Streaming XLSX writer requires explicit columns; cannot infer from an async row source.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToXlsxStream — hard cap in stream strategy", () => {
|
||||
it("enforces the hard cap mid-stream when forced onto the stream-writer", async () => {
|
||||
const sink = new PassThrough();
|
||||
sink.on("data", () => undefined);
|
||||
|
||||
async function* rows() {
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
yield { id: i };
|
||||
}
|
||||
}
|
||||
|
||||
await expect(
|
||||
serializeExportTableToXlsxStream(
|
||||
sink,
|
||||
{ columns: ["id"], rows: rows() },
|
||||
{ strategy: "stream", hardRowLimit: 3 },
|
||||
),
|
||||
).rejects.toBeInstanceOf(ExportXlsxRowLimitExceededError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeExportTableToXlsxStream — memory writer flush", () => {
|
||||
it("flushes the buffered workbook through a back-pressured sink and round-trips", async () => {
|
||||
const sink = new BackpressureSink();
|
||||
|
||||
const result = await serializeExportTableToXlsxStream(sink, {
|
||||
// object column without label -> label falls back to the key
|
||||
columns: [{ key: "id" }, "createdAt", "big", "blank"],
|
||||
rows: [
|
||||
{
|
||||
id: 1,
|
||||
createdAt: new Date("2024-01-02T03:04:05.006Z"),
|
||||
big: 9007199254740993n,
|
||||
blank: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toEqual({ rowCount: 1, strategy: "memory" });
|
||||
const rows = await readBack(sink.buffer());
|
||||
expect(rows[0]).toEqual(["id", "createdAt", "big", "blank"]);
|
||||
expect(rows[1]).toEqual(["1", "2024-01-02T03:04:05.006Z", "9007199254740993", ""]);
|
||||
});
|
||||
|
||||
it("rejects when the sink errors while flushing the workbook buffer", async () => {
|
||||
const sink = new ExplodingSink(new Error("xlsx sink down"));
|
||||
|
||||
await expect(
|
||||
serializeExportTableToXlsxStream(sink, {
|
||||
columns: ["id"],
|
||||
rows: [{ id: 1 }],
|
||||
}),
|
||||
).rejects.toThrow("xlsx sink down");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { createStoredZipBuffer } from "./zip.js";
|
||||
|
||||
function listZipEntries(zip: Buffer): string[] {
|
||||
const names: string[] = [];
|
||||
let offset = zip.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
|
||||
while (offset >= 0 && zip.readUInt32LE(offset) === 0x02014b50) {
|
||||
const nameLength = zip.readUInt16LE(offset + 28);
|
||||
const extraLength = zip.readUInt16LE(offset + 30);
|
||||
const commentLength = zip.readUInt16LE(offset + 32);
|
||||
names.push(zip.subarray(offset + 46, offset + 46 + nameLength).toString("utf8"));
|
||||
offset += 46 + nameLength + extraLength + commentLength;
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
describe("createStoredZipBuffer", () => {
|
||||
it("creates a standards-shaped ZIP with local files and central directory", () => {
|
||||
const zip = createStoredZipBuffer([
|
||||
{ path: "README.md", data: "hello", modifiedAt: new Date("2026-07-02T12:00:00.000Z") },
|
||||
{ path: "manifest.json", data: JSON.stringify({ ok: true }) },
|
||||
]);
|
||||
|
||||
expect(zip.readUInt32LE(0)).toBe(0x04034b50);
|
||||
expect(zip.includes(Buffer.from("README.md"))).toBe(true);
|
||||
expect(zip.includes(Buffer.from("manifest.json"))).toBe(true);
|
||||
expect(listZipEntries(zip)).toEqual(["README.md", "manifest.json"]);
|
||||
expect(zip.readUInt32LE(zip.byteLength - 22)).toBe(0x06054b50);
|
||||
});
|
||||
|
||||
it("rejects empty, unsafe, and duplicate entries", () => {
|
||||
expect(() => createStoredZipBuffer([])).toThrow("at least one entry");
|
||||
expect(() => createStoredZipBuffer([{ path: "../secret.txt", data: "x" }])).toThrow("Invalid ZIP entry path");
|
||||
expect(() =>
|
||||
createStoredZipBuffer([
|
||||
{ path: "same.txt", data: "1" },
|
||||
{ path: "same.txt", data: "2" },
|
||||
]),
|
||||
).toThrow("Duplicate ZIP entry path");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,152 @@
|
||||
export type ZipEntryInput = {
|
||||
path: string;
|
||||
data: string | Uint8Array;
|
||||
modifiedAt?: Date;
|
||||
contentType?: string;
|
||||
};
|
||||
|
||||
type NormalizedZipEntry = {
|
||||
path: string;
|
||||
data: Uint8Array;
|
||||
crc32: number;
|
||||
modifiedAt: Date;
|
||||
};
|
||||
|
||||
const UTF8_FLAG = 1 << 11;
|
||||
const STORE_METHOD = 0;
|
||||
const ZIP_VERSION_NEEDED = 20;
|
||||
const ZIP_VERSION_MADE_BY = 20;
|
||||
|
||||
export function createStoredZipBuffer(entries: readonly ZipEntryInput[]): Buffer {
|
||||
if (entries.length === 0) {
|
||||
throw new Error("ZIP bundle requires at least one entry.");
|
||||
}
|
||||
|
||||
const normalized = entries.map(normalizeEntry);
|
||||
const seen = new Set<string>();
|
||||
for (const entry of normalized) {
|
||||
if (seen.has(entry.path)) {
|
||||
throw new Error(`Duplicate ZIP entry path: ${entry.path}`);
|
||||
}
|
||||
seen.add(entry.path);
|
||||
}
|
||||
|
||||
const localParts: Buffer[] = [];
|
||||
const centralParts: Buffer[] = [];
|
||||
let offset = 0;
|
||||
|
||||
for (const entry of normalized) {
|
||||
const fileName = Buffer.from(entry.path, "utf8");
|
||||
const localHeader = buildLocalFileHeader(entry, fileName);
|
||||
localParts.push(localHeader, Buffer.from(entry.data));
|
||||
centralParts.push(buildCentralDirectoryHeader(entry, fileName, offset));
|
||||
offset += localHeader.byteLength + entry.data.byteLength;
|
||||
}
|
||||
|
||||
const centralDirectory = Buffer.concat(centralParts);
|
||||
const end = buildEndOfCentralDirectory(normalized.length, centralDirectory.byteLength, offset);
|
||||
return Buffer.concat([...localParts, centralDirectory, end]);
|
||||
}
|
||||
|
||||
function normalizeEntry(entry: ZipEntryInput): NormalizedZipEntry {
|
||||
const path = normalizeZipPath(entry.path);
|
||||
const data = typeof entry.data === "string" ? new TextEncoder().encode(entry.data) : entry.data;
|
||||
return {
|
||||
path,
|
||||
data,
|
||||
crc32: crc32(data),
|
||||
modifiedAt: entry.modifiedAt ?? new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeZipPath(path: string): string {
|
||||
const normalized = path.replace(/\\/g, "/").replace(/^\/+/, "").replace(/\/+/g, "/").trim();
|
||||
if (!normalized || normalized.endsWith("/") || normalized.split("/").some((part) => part === "." || part === ".." || part.length === 0)) {
|
||||
throw new Error(`Invalid ZIP entry path: ${path}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function buildLocalFileHeader(entry: NormalizedZipEntry, fileName: Buffer): Buffer {
|
||||
const header = Buffer.alloc(30);
|
||||
const { dosTime, dosDate } = toDosDateTime(entry.modifiedAt);
|
||||
header.writeUInt32LE(0x04034b50, 0);
|
||||
header.writeUInt16LE(ZIP_VERSION_NEEDED, 4);
|
||||
header.writeUInt16LE(UTF8_FLAG, 6);
|
||||
header.writeUInt16LE(STORE_METHOD, 8);
|
||||
header.writeUInt16LE(dosTime, 10);
|
||||
header.writeUInt16LE(dosDate, 12);
|
||||
header.writeUInt32LE(entry.crc32, 14);
|
||||
header.writeUInt32LE(entry.data.byteLength, 18);
|
||||
header.writeUInt32LE(entry.data.byteLength, 22);
|
||||
header.writeUInt16LE(fileName.byteLength, 26);
|
||||
header.writeUInt16LE(0, 28);
|
||||
return Buffer.concat([header, fileName]);
|
||||
}
|
||||
|
||||
function buildCentralDirectoryHeader(entry: NormalizedZipEntry, fileName: Buffer, localHeaderOffset: number): Buffer {
|
||||
const header = Buffer.alloc(46);
|
||||
const { dosTime, dosDate } = toDosDateTime(entry.modifiedAt);
|
||||
header.writeUInt32LE(0x02014b50, 0);
|
||||
header.writeUInt16LE(ZIP_VERSION_MADE_BY, 4);
|
||||
header.writeUInt16LE(ZIP_VERSION_NEEDED, 6);
|
||||
header.writeUInt16LE(UTF8_FLAG, 8);
|
||||
header.writeUInt16LE(STORE_METHOD, 10);
|
||||
header.writeUInt16LE(dosTime, 12);
|
||||
header.writeUInt16LE(dosDate, 14);
|
||||
header.writeUInt32LE(entry.crc32, 16);
|
||||
header.writeUInt32LE(entry.data.byteLength, 20);
|
||||
header.writeUInt32LE(entry.data.byteLength, 24);
|
||||
header.writeUInt16LE(fileName.byteLength, 28);
|
||||
header.writeUInt16LE(0, 30);
|
||||
header.writeUInt16LE(0, 32);
|
||||
header.writeUInt16LE(0, 34);
|
||||
header.writeUInt16LE(0, 36);
|
||||
header.writeUInt32LE(0, 38);
|
||||
header.writeUInt32LE(localHeaderOffset, 42);
|
||||
return Buffer.concat([header, fileName]);
|
||||
}
|
||||
|
||||
function buildEndOfCentralDirectory(entryCount: number, centralDirectorySize: number, centralDirectoryOffset: number): Buffer {
|
||||
const end = Buffer.alloc(22);
|
||||
end.writeUInt32LE(0x06054b50, 0);
|
||||
end.writeUInt16LE(0, 4);
|
||||
end.writeUInt16LE(0, 6);
|
||||
end.writeUInt16LE(entryCount, 8);
|
||||
end.writeUInt16LE(entryCount, 10);
|
||||
end.writeUInt32LE(centralDirectorySize, 12);
|
||||
end.writeUInt32LE(centralDirectoryOffset, 16);
|
||||
end.writeUInt16LE(0, 20);
|
||||
return end;
|
||||
}
|
||||
|
||||
function toDosDateTime(date: Date): { dosTime: number; dosDate: number } {
|
||||
const year = Math.max(1980, date.getUTCFullYear());
|
||||
const month = date.getUTCMonth() + 1;
|
||||
const day = date.getUTCDate();
|
||||
const hours = date.getUTCHours();
|
||||
const minutes = date.getUTCMinutes();
|
||||
const seconds = Math.floor(date.getUTCSeconds() / 2);
|
||||
return {
|
||||
dosTime: (hours << 11) | (minutes << 5) | seconds,
|
||||
dosDate: ((year - 1980) << 9) | (month << 5) | day,
|
||||
};
|
||||
}
|
||||
|
||||
function crc32(data: Uint8Array): number {
|
||||
let crc = 0xffffffff;
|
||||
for (const byte of data) {
|
||||
crc = (crc >>> 8) ^ CRC32_TABLE[(crc ^ byte) & 0xff];
|
||||
}
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
|
||||
const CRC32_TABLE = new Uint32Array(
|
||||
Array.from({ length: 256 }, (_, index) => {
|
||||
let value = index;
|
||||
for (let bit = 0; bit < 8; bit += 1) {
|
||||
value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
||||
}
|
||||
return value >>> 0;
|
||||
}),
|
||||
);
|
||||
Reference in new issue
Block a user