78 lines
2.7 KiB
TypeScript
78 lines
2.7 KiB
TypeScript
import { PassThrough } from "node:stream";
|
|
|
|
import type { S3Client } from "@aws-sdk/client-s3";
|
|
import { Upload } from "@aws-sdk/lib-storage";
|
|
|
|
/**
|
|
* S3 multipart-upload sink for streaming exports. Returns a `writeStream`
|
|
* that callers pipe their serialised output into and a `completionPromise`
|
|
* that resolves with the S3 ETag once the multipart upload finalises.
|
|
*
|
|
* The underlying `@aws-sdk/lib-storage` `Upload` class handles part
|
|
* batching, retries, and `AbortMultipartUpload` on failure, so callers do
|
|
* not need to track orphaned multipart uploads themselves.
|
|
*/
|
|
export interface CreateS3MultipartUploadSinkOptions {
|
|
client: S3Client;
|
|
bucket: string;
|
|
key: string;
|
|
contentType?: string;
|
|
metadata?: Record<string, string>;
|
|
/** Override the part size; defaults to 8 MB (S3 minimum is 5 MB). */
|
|
partSizeBytes?: number;
|
|
/** Override concurrent part-upload concurrency; defaults to 4. */
|
|
queueSize?: number;
|
|
/** Test seam: inject an alternative Upload implementation. */
|
|
uploadCtor?: typeof Upload;
|
|
}
|
|
|
|
export interface S3MultipartUploadSink {
|
|
writeStream: PassThrough;
|
|
completionPromise: Promise<{ bucket: string; key: string; etag: string | null }>;
|
|
}
|
|
|
|
const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
|
|
const DEFAULT_QUEUE_SIZE = 4;
|
|
|
|
export function createS3MultipartUploadSink(options: CreateS3MultipartUploadSinkOptions): S3MultipartUploadSink {
|
|
const writeStream = new PassThrough();
|
|
// PassThrough emits 'error' synchronously on destroy(err). Producers
|
|
// surface failures through completionPromise, not the stream, so attach
|
|
// a noop listener to keep Node from treating the destroy-triggered error
|
|
// as an uncaught exception.
|
|
writeStream.on("error", () => undefined);
|
|
const Ctor = options.uploadCtor ?? Upload;
|
|
|
|
const upload = new Ctor({
|
|
client: options.client,
|
|
partSize: options.partSizeBytes ?? DEFAULT_PART_SIZE_BYTES,
|
|
queueSize: options.queueSize ?? DEFAULT_QUEUE_SIZE,
|
|
params: {
|
|
Bucket: options.bucket,
|
|
Key: options.key,
|
|
Body: writeStream,
|
|
ContentType: options.contentType,
|
|
Metadata: options.metadata,
|
|
},
|
|
});
|
|
|
|
const completionPromise = (async () => {
|
|
const result = (await upload.done()) as { ETag?: string };
|
|
return {
|
|
bucket: options.bucket,
|
|
key: options.key,
|
|
etag: result.ETag ?? null,
|
|
};
|
|
})();
|
|
|
|
// Surface upload errors on the write stream so producers see them via
|
|
// standard 'error' propagation instead of an unhandled rejection.
|
|
completionPromise.catch((error: unknown) => {
|
|
if (!writeStream.destroyed) {
|
|
writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
|
}
|
|
});
|
|
|
|
return { writeStream, completionPromise };
|
|
}
|