Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,77 @@
|
||||
import { PassThrough } from "node:stream";
|
||||
|
||||
import type { S3Client } from "@aws-sdk/client-s3";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
|
||||
/**
|
||||
* S3 multipart-upload sink for streaming exports. Returns a `writeStream`
|
||||
* that callers pipe their serialised output into and a `completionPromise`
|
||||
* that resolves with the S3 ETag once the multipart upload finalises.
|
||||
*
|
||||
* The underlying `@aws-sdk/lib-storage` `Upload` class handles part
|
||||
* batching, retries, and `AbortMultipartUpload` on failure, so callers do
|
||||
* not need to track orphaned multipart uploads themselves.
|
||||
*/
|
||||
export interface CreateS3MultipartUploadSinkOptions {
|
||||
client: S3Client;
|
||||
bucket: string;
|
||||
key: string;
|
||||
contentType?: string;
|
||||
metadata?: Record<string, string>;
|
||||
/** Override the part size; defaults to 8 MB (S3 minimum is 5 MB). */
|
||||
partSizeBytes?: number;
|
||||
/** Override concurrent part-upload concurrency; defaults to 4. */
|
||||
queueSize?: number;
|
||||
/** Test seam: inject an alternative Upload implementation. */
|
||||
uploadCtor?: typeof Upload;
|
||||
}
|
||||
|
||||
export interface S3MultipartUploadSink {
|
||||
writeStream: PassThrough;
|
||||
completionPromise: Promise<{ bucket: string; key: string; etag: string | null }>;
|
||||
}
|
||||
|
||||
const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
|
||||
const DEFAULT_QUEUE_SIZE = 4;
|
||||
|
||||
export function createS3MultipartUploadSink(options: CreateS3MultipartUploadSinkOptions): S3MultipartUploadSink {
|
||||
const writeStream = new PassThrough();
|
||||
// PassThrough emits 'error' synchronously on destroy(err). Producers
|
||||
// surface failures through completionPromise, not the stream, so attach
|
||||
// a noop listener to keep Node from treating the destroy-triggered error
|
||||
// as an uncaught exception.
|
||||
writeStream.on("error", () => undefined);
|
||||
const Ctor = options.uploadCtor ?? Upload;
|
||||
|
||||
const upload = new Ctor({
|
||||
client: options.client,
|
||||
partSize: options.partSizeBytes ?? DEFAULT_PART_SIZE_BYTES,
|
||||
queueSize: options.queueSize ?? DEFAULT_QUEUE_SIZE,
|
||||
params: {
|
||||
Bucket: options.bucket,
|
||||
Key: options.key,
|
||||
Body: writeStream,
|
||||
ContentType: options.contentType,
|
||||
Metadata: options.metadata,
|
||||
},
|
||||
});
|
||||
|
||||
const completionPromise = (async () => {
|
||||
const result = (await upload.done()) as { ETag?: string };
|
||||
return {
|
||||
bucket: options.bucket,
|
||||
key: options.key,
|
||||
etag: result.ETag ?? null,
|
||||
};
|
||||
})();
|
||||
|
||||
// Surface upload errors on the write stream so producers see them via
|
||||
// standard 'error' propagation instead of an unhandled rejection.
|
||||
completionPromise.catch((error: unknown) => {
|
||||
if (!writeStream.destroyed) {
|
||||
writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
});
|
||||
|
||||
return { writeStream, completionPromise };
|
||||
}
|
||||
Reference in new issue
Block a user