import { AbortMultipartUploadCommand as AwsAbortMultipartUploadCommand, CompleteMultipartUploadCommand as AwsCompleteMultipartUploadCommand, CreateMultipartUploadCommand as AwsCreateMultipartUploadCommand, DeleteObjectCommand as AwsDeleteObjectCommand, PutObjectCommand as AwsPutObjectCommand, S3Client as AwsS3Client, UploadPartCopyCommand as AwsUploadPartCopyCommand, } from "@aws-sdk/client-s3"; import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { dirname, resolve, sep } from "node:path"; export * from "./access/index.js"; export * from "./paths/index.js"; export * from "./scan/index.js"; export * from "./signed/index.js"; export type UploadBody = string | Uint8Array | ArrayBuffer | ReadableStream; export interface StorageObjectResult { bucket: string; key: string; publicUrl: string | null; } export interface DeleteObjectResult { bucket: string; key: string; } /** Handle for an in-flight multipart upload (S3 `UploadId` + target key). */ export interface MultipartUploadHandle { bucket: string; key: string; uploadId: string; } /** One completed part of a multipart upload (1-based `partNumber` + ETag). */ export interface UploadedPart { partNumber: number; etag: string; } export type StorageAdapterErrorCode = | "config_missing" | "put_failed" | "delete_failed" | "signed_upload_failed" | "signed_download_failed" | "invalid_key" | "invalid_bucket" | "invalid_expiry"; export interface StorageAdapterErrorDetails { operation: string; bucket?: string; key?: string; missingFields?: string[]; upstream?: { name?: string; message?: string; status?: number; requestId?: string; }; } export class StorageAdapterError extends Error { readonly code: StorageAdapterErrorCode; readonly details: StorageAdapterErrorDetails; override readonly cause?: unknown; constructor( code: StorageAdapterErrorCode, message: string, details: StorageAdapterErrorDetails, options: { cause?: unknown } = {}, ) { super(message); this.name = "StorageAdapterError"; this.code = code; this.details = details; this.cause = options.cause; } } export interface PutObjectInput { key: string; body: UploadBody; contentType?: string; cacheControl?: string; metadata?: Record; } export interface DeleteObjectInput { bucket?: string; key: string; } export interface CreateMultipartUploadInput { key: string; contentType?: string; metadata?: Record; } export interface UploadPartCopyInput { /** Target object key of the multipart upload. */ key: string; uploadId: string; /** 1-based part number. */ partNumber: number; /** Source object key (already stored in the same bucket) to copy as this part. */ sourceKey: string; } export interface CompleteMultipartUploadInput { key: string; uploadId: string; parts: readonly UploadedPart[]; } export interface AbortMultipartUploadInput { key: string; uploadId: string; } export interface StorageAdapter { putObject(input: PutObjectInput): Promise; deleteObject(input: DeleteObjectInput): Promise; /** * Server-side chunk assembly (§3). Chunks are first stored as temp objects in * the same bucket, then assembled into the final object via multipart * UploadPartCopy — memory-safe for files up to the 500 MB limit. The whole * create/copy/complete sequence runs within a single `completeUpload` request, * so the S3 `UploadId` never needs to be persisted across requests. */ createMultipartUpload(input: CreateMultipartUploadInput): Promise; uploadPartCopy(input: UploadPartCopyInput): Promise; completeMultipartUpload(input: CompleteMultipartUploadInput): Promise; abortMultipartUpload(input: AbortMultipartUploadInput): Promise; } export interface PutObjectCommandInput { Bucket: string; Key: string; Body: UploadBody; ContentType?: string; CacheControl?: string; Metadata?: Record; } export class PutObjectCommand { readonly input: PutObjectCommandInput; constructor(input: PutObjectCommandInput) { this.input = input; } } export interface DeleteObjectCommandInput { Bucket: string; Key: string; } export class DeleteObjectCommand { readonly input: DeleteObjectCommandInput; constructor(input: DeleteObjectCommandInput) { this.input = input; } } export interface CreateMultipartUploadCommandInput { Bucket: string; Key: string; ContentType?: string; Metadata?: Record; } export class CreateMultipartUploadCommand { readonly input: CreateMultipartUploadCommandInput; constructor(input: CreateMultipartUploadCommandInput) { this.input = input; } } export interface UploadPartCopyCommandInput { Bucket: string; Key: string; UploadId: string; PartNumber: number; CopySource: string; } export class UploadPartCopyCommand { readonly input: UploadPartCopyCommandInput; constructor(input: UploadPartCopyCommandInput) { this.input = input; } } export interface CompleteMultipartUploadCommandInput { Bucket: string; Key: string; UploadId: string; MultipartUpload: { Parts: { ETag: string; PartNumber: number }[] }; } export class CompleteMultipartUploadCommand { readonly input: CompleteMultipartUploadCommandInput; constructor(input: CompleteMultipartUploadCommandInput) { this.input = input; } } export interface AbortMultipartUploadCommandInput { Bucket: string; Key: string; UploadId: string; } export class AbortMultipartUploadCommand { readonly input: AbortMultipartUploadCommandInput; constructor(input: AbortMultipartUploadCommandInput) { this.input = input; } } export type StorageCommand = | PutObjectCommand | DeleteObjectCommand | CreateMultipartUploadCommand | UploadPartCopyCommand | CompleteMultipartUploadCommand | AbortMultipartUploadCommand; export interface S3StorageClient { send(command: StorageCommand): Promise; } export class S3Client implements S3StorageClient { readonly config: { endpoint: string; region: string; forcePathStyle?: boolean; credentials: { accessKeyId: string; secretAccessKey: string; sessionToken?: string; }; }; private readonly client: AwsS3Client; constructor(config: S3Client["config"]) { this.config = config; this.client = new AwsS3Client(config); } async send(command: StorageCommand): Promise { if (command instanceof PutObjectCommand) { return this.client.send(new AwsPutObjectCommand(toAwsPutObjectInput(command.input))); } if (command instanceof CreateMultipartUploadCommand) { return this.client.send(new AwsCreateMultipartUploadCommand(command.input)); } if (command instanceof UploadPartCopyCommand) { return this.client.send(new AwsUploadPartCopyCommand(command.input)); } if (command instanceof CompleteMultipartUploadCommand) { return this.client.send(new AwsCompleteMultipartUploadCommand(command.input)); } if (command instanceof AbortMultipartUploadCommand) { return this.client.send(new AwsAbortMultipartUploadCommand(command.input)); } return this.client.send(new AwsDeleteObjectCommand(command.input)); } } function toAwsPutObjectInput(input: PutObjectCommandInput): ConstructorParameters[0] { return { ...input, Body: normalizeAwsUploadBody(input.Body) as ConstructorParameters[0]["Body"], }; } function normalizeAwsUploadBody(body: UploadBody) { return body instanceof ArrayBuffer ? new Uint8Array(body) : body; } export interface S3StorageAdapterConfig { endpoint: string; region: string; bucket: string; accessKeyId: string; secretAccessKey: string; sessionToken?: string; publicBaseUrl?: string; forcePathStyle?: boolean; client?: S3StorageClient; } export interface LocalStorageAdapterConfig { rootDir: string; bucket: string; publicBaseUrl?: string; } const REQUIRED_CONFIG_FIELDS = [ "endpoint", "region", "bucket", "accessKeyId", "secretAccessKey", ] as const satisfies readonly (keyof S3StorageAdapterConfig)[]; export function createS3StorageAdapter( config: S3StorageAdapterConfig, ): StorageAdapter { const missingFields = REQUIRED_CONFIG_FIELDS.filter( (field) => !isPresent(config[field]), ); if (missingFields.length > 0) { throw new StorageAdapterError( "config_missing", `Missing S3 storage configuration: ${missingFields.join(", ")}`, { operation: "createS3StorageAdapter", bucket: isPresent(config.bucket) ? config.bucket : undefined, missingFields: [...missingFields], }, ); } const client = config.client ?? new S3Client({ endpoint: config.endpoint, region: config.region, forcePathStyle: config.forcePathStyle, credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey, sessionToken: config.sessionToken, }, }); return { async putObject(input) { const command = new PutObjectCommand({ Bucket: config.bucket, Key: input.key, Body: input.body, ContentType: input.contentType, CacheControl: input.cacheControl, Metadata: input.metadata, }); try { await client.send(command); } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to put object in S3 storage.", { operation: "putObject", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause), }, { cause }, ); } return { bucket: config.bucket, key: input.key, publicUrl: buildPublicUrl(config.publicBaseUrl, input.key), }; }, async deleteObject(input) { const bucket = input.bucket ?? config.bucket; if (!isPresent(bucket)) { throw new StorageAdapterError( "invalid_bucket", "S3 object deletion requires a non-empty bucket.", { operation: "deleteObject", bucket: input.bucket, key: input.key, }, ); } if (!isPresent(input.key)) { throw new StorageAdapterError( "invalid_key", "S3 object deletion requires a non-empty key.", { operation: "deleteObject", bucket, key: input.key, }, ); } const command = new DeleteObjectCommand({ Bucket: bucket, Key: input.key, }); try { await client.send(command); } catch (cause) { throw new StorageAdapterError( "delete_failed", "Failed to delete object from S3 storage.", { operation: "deleteObject", bucket, key: input.key, upstream: extractUpstreamDetails(cause), }, { cause }, ); } return { bucket, key: input.key, }; }, async createMultipartUpload(input) { try { const response = (await client.send( new CreateMultipartUploadCommand({ Bucket: config.bucket, Key: input.key, ContentType: input.contentType, Metadata: input.metadata, }), )) as { UploadId?: string }; if (!isPresent(response.UploadId)) { throw new Error("S3 CreateMultipartUpload returned no UploadId."); } return { bucket: config.bucket, key: input.key, uploadId: response.UploadId }; } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to create multipart upload in S3 storage.", { operation: "createMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) }, { cause }, ); } }, async uploadPartCopy(input) { try { const response = (await client.send( new UploadPartCopyCommand({ Bucket: config.bucket, Key: input.key, UploadId: input.uploadId, PartNumber: input.partNumber, // CopySource must be URL-encoded "bucket/key". CopySource: `${config.bucket}/${input.sourceKey}` .split("/") .map((segment) => encodeURIComponent(segment)) .join("/"), }), )) as { CopyPartResult?: { ETag?: string } }; const etag = response.CopyPartResult?.ETag; if (!isPresent(etag)) { throw new Error("S3 UploadPartCopy returned no ETag."); } return { partNumber: input.partNumber, etag }; } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to copy part in S3 multipart upload.", { operation: "uploadPartCopy", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) }, { cause }, ); } }, async completeMultipartUpload(input) { try { await client.send( new CompleteMultipartUploadCommand({ Bucket: config.bucket, Key: input.key, UploadId: input.uploadId, MultipartUpload: { Parts: [...input.parts] .sort((a, b) => a.partNumber - b.partNumber) .map((part) => ({ ETag: part.etag, PartNumber: part.partNumber })), }, }), ); } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to complete S3 multipart upload.", { operation: "completeMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) }, { cause }, ); } return { bucket: config.bucket, key: input.key, publicUrl: buildPublicUrl(config.publicBaseUrl, input.key) }; }, async abortMultipartUpload(input) { try { await client.send( new AbortMultipartUploadCommand({ Bucket: config.bucket, Key: input.key, UploadId: input.uploadId }), ); } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to abort S3 multipart upload.", { operation: "abortMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) }, { cause }, ); } }, }; } export function createLocalStorageAdapter(config: LocalStorageAdapterConfig): StorageAdapter { const missingFields = [ ...(!isPresent(config.rootDir) ? ["rootDir"] : []), ...(!isPresent(config.bucket) ? ["bucket"] : []), ]; if (missingFields.length > 0) { throw new StorageAdapterError("config_missing", `Missing local storage configuration: ${missingFields.join(", ")}`, { operation: "createLocalStorageAdapter", bucket: isPresent(config.bucket) ? config.bucket : undefined, missingFields, }); } const rootDir = resolve(config.rootDir); const bucketRoot = resolve(rootDir, config.bucket); return { async putObject(input) { const key = normalizeLocalObjectKey(input.key); const target = resolve(bucketRoot, key); ensureInsideRoot(bucketRoot, target, key); try { await mkdir(dirname(target), { recursive: true }); await writeFile(target, await toLocalStorageBuffer(input.body)); } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to put object in local storage.", { operation: "putObject", bucket: config.bucket, key, upstream: extractUpstreamDetails(cause), }, { cause }, ); } return { bucket: config.bucket, key, publicUrl: buildPublicUrl(config.publicBaseUrl, `${config.bucket}/${key}`), }; }, async deleteObject(input) { const bucket = input.bucket ?? config.bucket; if (bucket !== config.bucket) { throw new StorageAdapterError("invalid_bucket", "Local object deletion requires the configured bucket.", { operation: "deleteObject", bucket, key: input.key, }); } const key = normalizeLocalObjectKey(input.key); const target = resolve(bucketRoot, key); ensureInsideRoot(bucketRoot, target, key); try { await rm(target, { force: true }); } catch (cause) { throw new StorageAdapterError( "delete_failed", "Failed to delete object from local storage.", { operation: "deleteObject", bucket, key, upstream: extractUpstreamDetails(cause), }, { cause }, ); } return { bucket, key }; }, // Local multipart: parts are staged as files under ".multipart/" and // concatenated on complete. The whole sequence runs in one process, so the // returned uploadId is just the key. async createMultipartUpload(input) { const key = normalizeLocalObjectKey(input.key); return { bucket: config.bucket, key, uploadId: key }; }, async uploadPartCopy(input) { const key = normalizeLocalObjectKey(input.key); const sourceKey = normalizeLocalObjectKey(input.sourceKey); const source = resolve(bucketRoot, sourceKey); ensureInsideRoot(bucketRoot, source, sourceKey); const partPath = resolve(bucketRoot, localMultipartPartKey(key, input.partNumber)); ensureInsideRoot(bucketRoot, partPath, key); try { const bytes = await readFile(source); await mkdir(dirname(partPath), { recursive: true }); await writeFile(partPath, bytes); } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to copy part in local multipart upload.", { operation: "uploadPartCopy", bucket: config.bucket, key, upstream: extractUpstreamDetails(cause) }, { cause }, ); } return { partNumber: input.partNumber, etag: `local-${input.partNumber}` }; }, async completeMultipartUpload(input) { const key = normalizeLocalObjectKey(input.key); const target = resolve(bucketRoot, key); ensureInsideRoot(bucketRoot, target, key); const ordered = [...input.parts].sort((a, b) => a.partNumber - b.partNumber); try { const buffers: Buffer[] = []; for (const part of ordered) { const partPath = resolve(bucketRoot, localMultipartPartKey(key, part.partNumber)); ensureInsideRoot(bucketRoot, partPath, key); buffers.push(await readFile(partPath)); } await mkdir(dirname(target), { recursive: true }); await writeFile(target, Buffer.concat(buffers)); await rm(resolve(bucketRoot, localMultipartDir(key)), { recursive: true, force: true }); } catch (cause) { throw new StorageAdapterError( "put_failed", "Failed to complete local multipart upload.", { operation: "completeMultipartUpload", bucket: config.bucket, key, upstream: extractUpstreamDetails(cause) }, { cause }, ); } return { bucket: config.bucket, key, publicUrl: buildPublicUrl(config.publicBaseUrl, `${config.bucket}/${key}`) }; }, async abortMultipartUpload(input) { const key = normalizeLocalObjectKey(input.key); const dir = resolve(bucketRoot, localMultipartDir(key)); ensureInsideRoot(bucketRoot, dir, key); await rm(dir, { recursive: true, force: true }).catch(() => {}); }, }; } function localMultipartDir(key: string): string { return `${key}.multipart`; } function localMultipartPartKey(key: string, partNumber: number): string { return `${localMultipartDir(key)}/part-${String(partNumber).padStart(6, "0")}`; } async function toLocalStorageBuffer(body: UploadBody): Promise { if (typeof body === "string") { return Buffer.from(body); } if (body instanceof ArrayBuffer) { return Buffer.from(body); } if (body instanceof Uint8Array) { return Buffer.from(body); } const reader = body.getReader(); const chunks: Uint8Array[] = []; while (true) { const chunk = await reader.read(); if (chunk.done) { break; } chunks.push(chunk.value); } return Buffer.concat(chunks); } function normalizeLocalObjectKey(key: string): string { if (!isPresent(key) || key.startsWith("/") || key.includes("\\") || key.split("/").some((part) => part === "..")) { throw new StorageAdapterError("invalid_key", "Storage object key is invalid.", { operation: "normalizeLocalObjectKey", key, }); } return key.replace(/^\.\/+/, ""); } function ensureInsideRoot(rootDir: string, target: string, key: string): void { if (target !== rootDir && !target.startsWith(`${rootDir}${sep}`)) { throw new StorageAdapterError("invalid_key", "Storage object key escapes the configured storage root.", { operation: "resolveLocalObjectPath", key, }); } } function isPresent(value: unknown): value is string { return typeof value === "string" && value.trim().length > 0; } function buildPublicUrl(publicBaseUrl: string | undefined, key: string): string | null { if (!isPresent(publicBaseUrl)) { return null; } const base = publicBaseUrl.replace(/\/+$/, ""); const encodedKey = key .split("/") .map((segment) => encodeURIComponent(segment)) .join("/"); return `${base}/${encodedKey}`; } function extractUpstreamDetails(cause: unknown): StorageAdapterErrorDetails["upstream"] { if (!isRecord(cause)) { return undefined; } const metadata = isRecord(cause.$metadata) ? cause.$metadata : undefined; return { name: typeof cause.name === "string" ? cause.name : undefined, message: typeof cause.message === "string" ? redactSecretLikeText(cause.message) : undefined, status: typeof metadata?.httpStatusCode === "number" ? metadata.httpStatusCode : undefined, requestId: typeof metadata?.requestId === "string" ? metadata.requestId : undefined, }; } function redactSecretLikeText(value: string): string { return value.replace(/(token|secret|credential)([\s:=]+)([^\s,;]+)/gi, "$1$2[redacted]"); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; }