Files
HiveGrid-Wireframes/hivegrid-cloud/packages/exports/src/sinks/s3-multipart.ts
T

78 lines
2.7 KiB
TypeScript

import { PassThrough } from "node:stream";
import type { S3Client } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
/**
* S3 multipart-upload sink for streaming exports. Returns a `writeStream`
* that callers pipe their serialised output into and a `completionPromise`
* that resolves with the S3 ETag once the multipart upload finalises.
*
* The underlying `@aws-sdk/lib-storage` `Upload` class handles part
* batching, retries, and `AbortMultipartUpload` on failure, so callers do
* not need to track orphaned multipart uploads themselves.
*/
export interface CreateS3MultipartUploadSinkOptions {
client: S3Client;
bucket: string;
key: string;
contentType?: string;
metadata?: Record<string, string>;
/** Override the part size; defaults to 8 MB (S3 minimum is 5 MB). */
partSizeBytes?: number;
/** Override concurrent part-upload concurrency; defaults to 4. */
queueSize?: number;
/** Test seam: inject an alternative Upload implementation. */
uploadCtor?: typeof Upload;
}
export interface S3MultipartUploadSink {
writeStream: PassThrough;
completionPromise: Promise<{ bucket: string; key: string; etag: string | null }>;
}
const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
const DEFAULT_QUEUE_SIZE = 4;
export function createS3MultipartUploadSink(options: CreateS3MultipartUploadSinkOptions): S3MultipartUploadSink {
const writeStream = new PassThrough();
// PassThrough emits 'error' synchronously on destroy(err). Producers
// surface failures through completionPromise, not the stream, so attach
// a noop listener to keep Node from treating the destroy-triggered error
// as an uncaught exception.
writeStream.on("error", () => undefined);
const Ctor = options.uploadCtor ?? Upload;
const upload = new Ctor({
client: options.client,
partSize: options.partSizeBytes ?? DEFAULT_PART_SIZE_BYTES,
queueSize: options.queueSize ?? DEFAULT_QUEUE_SIZE,
params: {
Bucket: options.bucket,
Key: options.key,
Body: writeStream,
ContentType: options.contentType,
Metadata: options.metadata,
},
});
const completionPromise = (async () => {
const result = (await upload.done()) as { ETag?: string };
return {
bucket: options.bucket,
key: options.key,
etag: result.ETag ?? null,
};
})();
// Surface upload errors on the write stream so producers see them via
// standard 'error' propagation instead of an unhandled rejection.
completionPromise.catch((error: unknown) => {
if (!writeStream.destroyed) {
writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
}
});
return { writeStream, completionPromise };
}