200 lines
7.1 KiB
TypeScript
200 lines
7.1 KiB
TypeScript
import { PassThrough } from "node:stream";
|
|
|
|
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type { TenantPlanTier } from "@toolctrl/exports/limits";
|
|
import type { ExportRecord } from "@toolctrl/exports/types";
|
|
import type { ExportJob } from "@toolctrl/queue";
|
|
import type { SharedStore } from "@toolctrl/shared-store";
|
|
|
|
import { createExportJobRunner, type ExportRowSource, type ExportStreamerRegistry } from "./export.js";
|
|
|
|
function buildPayload(overrides: Partial<ExportJob> = {}): ExportJob {
|
|
return {
|
|
kind: "export.run",
|
|
exportJobId: "job-1",
|
|
tenantId: "tenant-1",
|
|
requestedByUserId: "user-1",
|
|
exportType: "users",
|
|
format: "csv",
|
|
...overrides,
|
|
} as ExportJob;
|
|
}
|
|
|
|
function buildRowSource(rows: ExportRecord[], columns: readonly string[] = ["id", "name"], estimated?: number): ExportRowSource {
|
|
return {
|
|
estimateRowCount: vi.fn().mockResolvedValue(estimated),
|
|
resolveColumns: vi.fn().mockResolvedValue(columns),
|
|
iterateRows: () => (async function* () {
|
|
for (const row of rows) yield row;
|
|
})(),
|
|
} satisfies ExportRowSource;
|
|
}
|
|
|
|
interface SharedStoreStub {
|
|
store: SharedStore;
|
|
set: ReturnType<typeof vi.fn>;
|
|
}
|
|
|
|
function buildSharedStoreStub(): SharedStoreStub {
|
|
const set = vi.fn().mockResolvedValue(true);
|
|
const stub = {
|
|
set,
|
|
get: vi.fn(),
|
|
del: vi.fn(),
|
|
incr: vi.fn(),
|
|
expire: vi.fn(),
|
|
mget: vi.fn(),
|
|
scan: vi.fn(),
|
|
publish: vi.fn(),
|
|
subscribe: vi.fn(),
|
|
zAdd: vi.fn(),
|
|
zRemRangeByScore: vi.fn(),
|
|
zCard: vi.fn(),
|
|
eval: vi.fn(),
|
|
close: vi.fn(),
|
|
key: vi.fn(),
|
|
namespace: "x",
|
|
ttlDefault: undefined,
|
|
adapter: {} as never,
|
|
};
|
|
return { store: stub as unknown as SharedStore, set };
|
|
}
|
|
|
|
function buildCountingRegistry(): { registry: ExportStreamerRegistry; written: PassThrough; calls: { csv: number; json: number; xlsx: number; pdf: number } } {
|
|
const written = new PassThrough();
|
|
written.on("data", () => undefined);
|
|
const calls = { csv: 0, json: 0, xlsx: 0, pdf: 0 };
|
|
|
|
const consume: ExportStreamerRegistry["csv"] = async (writeStream, _columns, rows) => {
|
|
let count = 0;
|
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
for await (const _row of rows) {
|
|
count += 1;
|
|
}
|
|
writeStream.end();
|
|
return { rowCount: count };
|
|
};
|
|
|
|
return {
|
|
written,
|
|
calls,
|
|
registry: {
|
|
csv: async (...args) => { calls.csv += 1; return consume(...args); },
|
|
json: async (...args) => { calls.json += 1; return consume(...args); },
|
|
xlsx: async (...args) => { calls.xlsx += 1; return consume(...args); },
|
|
pdf: async (...args) => { calls.pdf += 1; return consume(...args); },
|
|
},
|
|
};
|
|
}
|
|
|
|
function buildUploadStub(written: PassThrough): { Upload: unknown; done: ReturnType<typeof vi.fn> } {
|
|
const done = vi.fn().mockResolvedValue({ ETag: '"abc"' });
|
|
const Upload = vi.fn().mockImplementation((args: { params: { Body: PassThrough } }) => {
|
|
args.params.Body.pipe(written);
|
|
return { done };
|
|
});
|
|
return { Upload, done };
|
|
}
|
|
|
|
describe("createExportJobRunner", () => {
|
|
it("returns the resolved row count and artifact metadata on success", async () => {
|
|
const { registry, written } = buildCountingRegistry();
|
|
const upload = buildUploadStub(written);
|
|
|
|
const runner = createExportJobRunner({
|
|
s3Client: {} as never,
|
|
rowSource: buildRowSource([{ id: 1 }, { id: 2 }, { id: 3 }], ["id"]),
|
|
resolveTenantPlan: async (): Promise<TenantPlanTier> => "pro",
|
|
buildArtifactKey: () => ({ bucket: "exports", key: "tenants/t/job.csv", contentType: "text/csv", expiresAt: new Date("2026-01-01T00:00:00Z") }),
|
|
streamerRegistry: registry,
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
uploadCtor: upload.Upload as any,
|
|
});
|
|
|
|
const result = await runner(buildPayload());
|
|
expect(result).toEqual({
|
|
artifactBucket: "exports",
|
|
artifactKey: "tenants/t/job.csv",
|
|
rowCount: 3,
|
|
expiresAt: new Date("2026-01-01T00:00:00Z"),
|
|
});
|
|
expect(upload.done).toHaveBeenCalled();
|
|
});
|
|
|
|
it("rejects up front when estimateRowCount exceeds the resolved plan cap", async () => {
|
|
const { registry, written } = buildCountingRegistry();
|
|
const upload = buildUploadStub(written);
|
|
const runner = createExportJobRunner({
|
|
s3Client: {} as never,
|
|
rowSource: buildRowSource([{ id: 1 }], ["id"], 20_000),
|
|
resolveTenantPlan: async () => "free",
|
|
buildArtifactKey: () => ({ bucket: "b", key: "k", contentType: "text/csv", expiresAt: new Date() }),
|
|
streamerRegistry: registry,
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
uploadCtor: upload.Upload as any,
|
|
});
|
|
|
|
await expect(runner(buildPayload())).rejects.toThrow(/EXPORT_ROW_LIMIT_PLAN|exceeds the free/);
|
|
expect(registry.csv).not.toHaveBeenCalled;
|
|
});
|
|
|
|
it("publishes progress to the shared store on every flushInterval rows", async () => {
|
|
const rows: ExportRecord[] = Array.from({ length: 12 }, (_, i) => ({ id: i }));
|
|
const { registry, written } = buildCountingRegistry();
|
|
const upload = buildUploadStub(written);
|
|
const sharedStore = buildSharedStoreStub();
|
|
|
|
const runner = createExportJobRunner({
|
|
s3Client: {} as never,
|
|
sharedStore: sharedStore.store,
|
|
rowSource: buildRowSource(rows, ["id"], 12),
|
|
resolveTenantPlan: async () => "pro",
|
|
buildArtifactKey: () => ({ bucket: "b", key: "k", contentType: "text/csv", expiresAt: new Date("2026-01-01") }),
|
|
streamerRegistry: registry,
|
|
progressFlushInterval: 5,
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
uploadCtor: upload.Upload as any,
|
|
});
|
|
|
|
const result = await runner(buildPayload());
|
|
expect(result.rowCount).toBe(12);
|
|
// Expect at least the row-5, row-10 progress flushes plus the final
|
|
// success flush.
|
|
expect(sharedStore.set).toHaveBeenCalledWith(
|
|
"export:progress:job-1",
|
|
expect.stringContaining('"processedRows":5'),
|
|
expect.objectContaining({ ttlSeconds: 3600 }),
|
|
);
|
|
expect(sharedStore.set).toHaveBeenCalledWith(
|
|
"export:progress:job-1",
|
|
expect.stringContaining('"processedRows":12'),
|
|
expect.objectContaining({ ttlSeconds: 3600 }),
|
|
);
|
|
});
|
|
|
|
it("destroys the write stream when serialisation throws", async () => {
|
|
const written = new PassThrough();
|
|
written.on("data", () => undefined);
|
|
const failingRegistry: ExportStreamerRegistry = {
|
|
csv: async () => { throw new Error("boom"); },
|
|
json: async () => { throw new Error("boom"); },
|
|
xlsx: async () => { throw new Error("boom"); },
|
|
pdf: async () => { throw new Error("boom"); },
|
|
};
|
|
const upload = buildUploadStub(written);
|
|
|
|
const runner = createExportJobRunner({
|
|
s3Client: {} as never,
|
|
rowSource: buildRowSource([{ id: 1 }], ["id"]),
|
|
resolveTenantPlan: async () => "pro",
|
|
buildArtifactKey: () => ({ bucket: "b", key: "k", contentType: "text/csv", expiresAt: new Date() }),
|
|
streamerRegistry: failingRegistry,
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
uploadCtor: upload.Upload as any,
|
|
});
|
|
|
|
await expect(runner(buildPayload())).rejects.toThrow("boom");
|
|
});
|
|
});
|