import { PassThrough } from "node:stream"; import type { S3Client } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; /** * S3 multipart-upload sink for streaming exports. Returns a `writeStream` * that callers pipe their serialised output into and a `completionPromise` * that resolves with the S3 ETag once the multipart upload finalises. * * The underlying `@aws-sdk/lib-storage` `Upload` class handles part * batching, retries, and `AbortMultipartUpload` on failure, so callers do * not need to track orphaned multipart uploads themselves. */ export interface CreateS3MultipartUploadSinkOptions { client: S3Client; bucket: string; key: string; contentType?: string; metadata?: Record; /** Override the part size; defaults to 8 MB (S3 minimum is 5 MB). */ partSizeBytes?: number; /** Override concurrent part-upload concurrency; defaults to 4. */ queueSize?: number; /** Test seam: inject an alternative Upload implementation. */ uploadCtor?: typeof Upload; } export interface S3MultipartUploadSink { writeStream: PassThrough; completionPromise: Promise<{ bucket: string; key: string; etag: string | null }>; } const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB const DEFAULT_QUEUE_SIZE = 4; export function createS3MultipartUploadSink(options: CreateS3MultipartUploadSinkOptions): S3MultipartUploadSink { const writeStream = new PassThrough(); // PassThrough emits 'error' synchronously on destroy(err). Producers // surface failures through completionPromise, not the stream, so attach // a noop listener to keep Node from treating the destroy-triggered error // as an uncaught exception. writeStream.on("error", () => undefined); const Ctor = options.uploadCtor ?? Upload; const upload = new Ctor({ client: options.client, partSize: options.partSizeBytes ?? DEFAULT_PART_SIZE_BYTES, queueSize: options.queueSize ?? DEFAULT_QUEUE_SIZE, params: { Bucket: options.bucket, Key: options.key, Body: writeStream, ContentType: options.contentType, Metadata: options.metadata, }, }); const completionPromise = (async () => { const result = (await upload.done()) as { ETag?: string }; return { bucket: options.bucket, key: options.key, etag: result.ETag ?? null, }; })(); // Surface upload errors on the write stream so producers see them via // standard 'error' propagation instead of an unhandled rejection. completionPromise.catch((error: unknown) => { if (!writeStream.destroyed) { writeStream.destroy(error instanceof Error ? error : new Error(String(error))); } }); return { writeStream, completionPromise }; }