Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/packages/storage/src/index.ts
T

766 lines
22 KiB
TypeScript

import {
AbortMultipartUploadCommand as AwsAbortMultipartUploadCommand,
CompleteMultipartUploadCommand as AwsCompleteMultipartUploadCommand,
CreateMultipartUploadCommand as AwsCreateMultipartUploadCommand,
DeleteObjectCommand as AwsDeleteObjectCommand,
PutObjectCommand as AwsPutObjectCommand,
S3Client as AwsS3Client,
UploadPartCopyCommand as AwsUploadPartCopyCommand,
} from "@aws-sdk/client-s3";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { dirname, resolve, sep } from "node:path";
export * from "./access/index.js";
export * from "./paths/index.js";
export * from "./scan/index.js";
export * from "./signed/index.js";
export type UploadBody = string | Uint8Array | ArrayBuffer | ReadableStream<Uint8Array>;
export interface StorageObjectResult {
bucket: string;
key: string;
publicUrl: string | null;
}
export interface DeleteObjectResult {
bucket: string;
key: string;
}
/** Handle for an in-flight multipart upload (S3 `UploadId` + target key). */
export interface MultipartUploadHandle {
bucket: string;
key: string;
uploadId: string;
}
/** One completed part of a multipart upload (1-based `partNumber` + ETag). */
export interface UploadedPart {
partNumber: number;
etag: string;
}
export type StorageAdapterErrorCode =
| "config_missing"
| "put_failed"
| "delete_failed"
| "signed_upload_failed"
| "signed_download_failed"
| "invalid_key"
| "invalid_bucket"
| "invalid_expiry";
export interface StorageAdapterErrorDetails {
operation: string;
bucket?: string;
key?: string;
missingFields?: string[];
upstream?: {
name?: string;
message?: string;
status?: number;
requestId?: string;
};
}
export class StorageAdapterError extends Error {
readonly code: StorageAdapterErrorCode;
readonly details: StorageAdapterErrorDetails;
override readonly cause?: unknown;
constructor(
code: StorageAdapterErrorCode,
message: string,
details: StorageAdapterErrorDetails,
options: { cause?: unknown } = {},
) {
super(message);
this.name = "StorageAdapterError";
this.code = code;
this.details = details;
this.cause = options.cause;
}
}
export interface PutObjectInput {
key: string;
body: UploadBody;
contentType?: string;
cacheControl?: string;
metadata?: Record<string, string>;
}
export interface DeleteObjectInput {
bucket?: string;
key: string;
}
export interface CreateMultipartUploadInput {
key: string;
contentType?: string;
metadata?: Record<string, string>;
}
export interface UploadPartCopyInput {
/** Target object key of the multipart upload. */
key: string;
uploadId: string;
/** 1-based part number. */
partNumber: number;
/** Source object key (already stored in the same bucket) to copy as this part. */
sourceKey: string;
}
export interface CompleteMultipartUploadInput {
key: string;
uploadId: string;
parts: readonly UploadedPart[];
}
export interface AbortMultipartUploadInput {
key: string;
uploadId: string;
}
export interface StorageAdapter {
putObject(input: PutObjectInput): Promise<StorageObjectResult>;
deleteObject(input: DeleteObjectInput): Promise<DeleteObjectResult>;
/**
* Server-side chunk assembly (§3). Chunks are first stored as temp objects in
* the same bucket, then assembled into the final object via multipart
* UploadPartCopy — memory-safe for files up to the 500 MB limit. The whole
* create/copy/complete sequence runs within a single `completeUpload` request,
* so the S3 `UploadId` never needs to be persisted across requests.
*/
createMultipartUpload(input: CreateMultipartUploadInput): Promise<MultipartUploadHandle>;
uploadPartCopy(input: UploadPartCopyInput): Promise<UploadedPart>;
completeMultipartUpload(input: CompleteMultipartUploadInput): Promise<StorageObjectResult>;
abortMultipartUpload(input: AbortMultipartUploadInput): Promise<void>;
}
export interface PutObjectCommandInput {
Bucket: string;
Key: string;
Body: UploadBody;
ContentType?: string;
CacheControl?: string;
Metadata?: Record<string, string>;
}
export class PutObjectCommand {
readonly input: PutObjectCommandInput;
constructor(input: PutObjectCommandInput) {
this.input = input;
}
}
export interface DeleteObjectCommandInput {
Bucket: string;
Key: string;
}
export class DeleteObjectCommand {
readonly input: DeleteObjectCommandInput;
constructor(input: DeleteObjectCommandInput) {
this.input = input;
}
}
export interface CreateMultipartUploadCommandInput {
Bucket: string;
Key: string;
ContentType?: string;
Metadata?: Record<string, string>;
}
export class CreateMultipartUploadCommand {
readonly input: CreateMultipartUploadCommandInput;
constructor(input: CreateMultipartUploadCommandInput) {
this.input = input;
}
}
export interface UploadPartCopyCommandInput {
Bucket: string;
Key: string;
UploadId: string;
PartNumber: number;
CopySource: string;
}
export class UploadPartCopyCommand {
readonly input: UploadPartCopyCommandInput;
constructor(input: UploadPartCopyCommandInput) {
this.input = input;
}
}
export interface CompleteMultipartUploadCommandInput {
Bucket: string;
Key: string;
UploadId: string;
MultipartUpload: { Parts: { ETag: string; PartNumber: number }[] };
}
export class CompleteMultipartUploadCommand {
readonly input: CompleteMultipartUploadCommandInput;
constructor(input: CompleteMultipartUploadCommandInput) {
this.input = input;
}
}
export interface AbortMultipartUploadCommandInput {
Bucket: string;
Key: string;
UploadId: string;
}
export class AbortMultipartUploadCommand {
readonly input: AbortMultipartUploadCommandInput;
constructor(input: AbortMultipartUploadCommandInput) {
this.input = input;
}
}
export type StorageCommand =
| PutObjectCommand
| DeleteObjectCommand
| CreateMultipartUploadCommand
| UploadPartCopyCommand
| CompleteMultipartUploadCommand
| AbortMultipartUploadCommand;
export interface S3StorageClient {
send(command: StorageCommand): Promise<unknown>;
}
export class S3Client implements S3StorageClient {
readonly config: {
endpoint: string;
region: string;
forcePathStyle?: boolean;
credentials: {
accessKeyId: string;
secretAccessKey: string;
sessionToken?: string;
};
};
private readonly client: AwsS3Client;
constructor(config: S3Client["config"]) {
this.config = config;
this.client = new AwsS3Client(config);
}
async send(command: StorageCommand): Promise<unknown> {
if (command instanceof PutObjectCommand) {
return this.client.send(new AwsPutObjectCommand(toAwsPutObjectInput(command.input)));
}
if (command instanceof CreateMultipartUploadCommand) {
return this.client.send(new AwsCreateMultipartUploadCommand(command.input));
}
if (command instanceof UploadPartCopyCommand) {
return this.client.send(new AwsUploadPartCopyCommand(command.input));
}
if (command instanceof CompleteMultipartUploadCommand) {
return this.client.send(new AwsCompleteMultipartUploadCommand(command.input));
}
if (command instanceof AbortMultipartUploadCommand) {
return this.client.send(new AwsAbortMultipartUploadCommand(command.input));
}
return this.client.send(new AwsDeleteObjectCommand(command.input));
}
}
function toAwsPutObjectInput(input: PutObjectCommandInput): ConstructorParameters<typeof AwsPutObjectCommand>[0] {
return {
...input,
Body: normalizeAwsUploadBody(input.Body) as ConstructorParameters<typeof AwsPutObjectCommand>[0]["Body"],
};
}
function normalizeAwsUploadBody(body: UploadBody) {
return body instanceof ArrayBuffer ? new Uint8Array(body) : body;
}
export interface S3StorageAdapterConfig {
endpoint: string;
region: string;
bucket: string;
accessKeyId: string;
secretAccessKey: string;
sessionToken?: string;
publicBaseUrl?: string;
forcePathStyle?: boolean;
client?: S3StorageClient;
}
export interface LocalStorageAdapterConfig {
rootDir: string;
bucket: string;
publicBaseUrl?: string;
}
const REQUIRED_CONFIG_FIELDS = [
"endpoint",
"region",
"bucket",
"accessKeyId",
"secretAccessKey",
] as const satisfies readonly (keyof S3StorageAdapterConfig)[];
export function createS3StorageAdapter(
config: S3StorageAdapterConfig,
): StorageAdapter {
const missingFields = REQUIRED_CONFIG_FIELDS.filter(
(field) => !isPresent(config[field]),
);
if (missingFields.length > 0) {
throw new StorageAdapterError(
"config_missing",
`Missing S3 storage configuration: ${missingFields.join(", ")}`,
{
operation: "createS3StorageAdapter",
bucket: isPresent(config.bucket) ? config.bucket : undefined,
missingFields: [...missingFields],
},
);
}
const client =
config.client ??
new S3Client({
endpoint: config.endpoint,
region: config.region,
forcePathStyle: config.forcePathStyle,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
sessionToken: config.sessionToken,
},
});
return {
async putObject(input) {
const command = new PutObjectCommand({
Bucket: config.bucket,
Key: input.key,
Body: input.body,
ContentType: input.contentType,
CacheControl: input.cacheControl,
Metadata: input.metadata,
});
try {
await client.send(command);
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to put object in S3 storage.",
{
operation: "putObject",
bucket: config.bucket,
key: input.key,
upstream: extractUpstreamDetails(cause),
},
{ cause },
);
}
return {
bucket: config.bucket,
key: input.key,
publicUrl: buildPublicUrl(config.publicBaseUrl, input.key),
};
},
async deleteObject(input) {
const bucket = input.bucket ?? config.bucket;
if (!isPresent(bucket)) {
throw new StorageAdapterError(
"invalid_bucket",
"S3 object deletion requires a non-empty bucket.",
{
operation: "deleteObject",
bucket: input.bucket,
key: input.key,
},
);
}
if (!isPresent(input.key)) {
throw new StorageAdapterError(
"invalid_key",
"S3 object deletion requires a non-empty key.",
{
operation: "deleteObject",
bucket,
key: input.key,
},
);
}
const command = new DeleteObjectCommand({
Bucket: bucket,
Key: input.key,
});
try {
await client.send(command);
} catch (cause) {
throw new StorageAdapterError(
"delete_failed",
"Failed to delete object from S3 storage.",
{
operation: "deleteObject",
bucket,
key: input.key,
upstream: extractUpstreamDetails(cause),
},
{ cause },
);
}
return {
bucket,
key: input.key,
};
},
async createMultipartUpload(input) {
try {
const response = (await client.send(
new CreateMultipartUploadCommand({
Bucket: config.bucket,
Key: input.key,
ContentType: input.contentType,
Metadata: input.metadata,
}),
)) as { UploadId?: string };
if (!isPresent(response.UploadId)) {
throw new Error("S3 CreateMultipartUpload returned no UploadId.");
}
return { bucket: config.bucket, key: input.key, uploadId: response.UploadId };
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to create multipart upload in S3 storage.",
{ operation: "createMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
{ cause },
);
}
},
async uploadPartCopy(input) {
try {
const response = (await client.send(
new UploadPartCopyCommand({
Bucket: config.bucket,
Key: input.key,
UploadId: input.uploadId,
PartNumber: input.partNumber,
// CopySource must be URL-encoded "bucket/key".
CopySource: `${config.bucket}/${input.sourceKey}`
.split("/")
.map((segment) => encodeURIComponent(segment))
.join("/"),
}),
)) as { CopyPartResult?: { ETag?: string } };
const etag = response.CopyPartResult?.ETag;
if (!isPresent(etag)) {
throw new Error("S3 UploadPartCopy returned no ETag.");
}
return { partNumber: input.partNumber, etag };
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to copy part in S3 multipart upload.",
{ operation: "uploadPartCopy", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
{ cause },
);
}
},
async completeMultipartUpload(input) {
try {
await client.send(
new CompleteMultipartUploadCommand({
Bucket: config.bucket,
Key: input.key,
UploadId: input.uploadId,
MultipartUpload: {
Parts: [...input.parts]
.sort((a, b) => a.partNumber - b.partNumber)
.map((part) => ({ ETag: part.etag, PartNumber: part.partNumber })),
},
}),
);
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to complete S3 multipart upload.",
{ operation: "completeMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
{ cause },
);
}
return { bucket: config.bucket, key: input.key, publicUrl: buildPublicUrl(config.publicBaseUrl, input.key) };
},
async abortMultipartUpload(input) {
try {
await client.send(
new AbortMultipartUploadCommand({ Bucket: config.bucket, Key: input.key, UploadId: input.uploadId }),
);
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to abort S3 multipart upload.",
{ operation: "abortMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
{ cause },
);
}
},
};
}
export function createLocalStorageAdapter(config: LocalStorageAdapterConfig): StorageAdapter {
const missingFields = [
...(!isPresent(config.rootDir) ? ["rootDir"] : []),
...(!isPresent(config.bucket) ? ["bucket"] : []),
];
if (missingFields.length > 0) {
throw new StorageAdapterError("config_missing", `Missing local storage configuration: ${missingFields.join(", ")}`, {
operation: "createLocalStorageAdapter",
bucket: isPresent(config.bucket) ? config.bucket : undefined,
missingFields,
});
}
const rootDir = resolve(config.rootDir);
const bucketRoot = resolve(rootDir, config.bucket);
return {
async putObject(input) {
const key = normalizeLocalObjectKey(input.key);
const target = resolve(bucketRoot, key);
ensureInsideRoot(bucketRoot, target, key);
try {
await mkdir(dirname(target), { recursive: true });
await writeFile(target, await toLocalStorageBuffer(input.body));
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to put object in local storage.",
{
operation: "putObject",
bucket: config.bucket,
key,
upstream: extractUpstreamDetails(cause),
},
{ cause },
);
}
return {
bucket: config.bucket,
key,
publicUrl: buildPublicUrl(config.publicBaseUrl, `${config.bucket}/${key}`),
};
},
async deleteObject(input) {
const bucket = input.bucket ?? config.bucket;
if (bucket !== config.bucket) {
throw new StorageAdapterError("invalid_bucket", "Local object deletion requires the configured bucket.", {
operation: "deleteObject",
bucket,
key: input.key,
});
}
const key = normalizeLocalObjectKey(input.key);
const target = resolve(bucketRoot, key);
ensureInsideRoot(bucketRoot, target, key);
try {
await rm(target, { force: true });
} catch (cause) {
throw new StorageAdapterError(
"delete_failed",
"Failed to delete object from local storage.",
{
operation: "deleteObject",
bucket,
key,
upstream: extractUpstreamDetails(cause),
},
{ cause },
);
}
return { bucket, key };
},
// Local multipart: parts are staged as files under "<key>.multipart/" and
// concatenated on complete. The whole sequence runs in one process, so the
// returned uploadId is just the key.
async createMultipartUpload(input) {
const key = normalizeLocalObjectKey(input.key);
return { bucket: config.bucket, key, uploadId: key };
},
async uploadPartCopy(input) {
const key = normalizeLocalObjectKey(input.key);
const sourceKey = normalizeLocalObjectKey(input.sourceKey);
const source = resolve(bucketRoot, sourceKey);
ensureInsideRoot(bucketRoot, source, sourceKey);
const partPath = resolve(bucketRoot, localMultipartPartKey(key, input.partNumber));
ensureInsideRoot(bucketRoot, partPath, key);
try {
const bytes = await readFile(source);
await mkdir(dirname(partPath), { recursive: true });
await writeFile(partPath, bytes);
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to copy part in local multipart upload.",
{ operation: "uploadPartCopy", bucket: config.bucket, key, upstream: extractUpstreamDetails(cause) },
{ cause },
);
}
return { partNumber: input.partNumber, etag: `local-${input.partNumber}` };
},
async completeMultipartUpload(input) {
const key = normalizeLocalObjectKey(input.key);
const target = resolve(bucketRoot, key);
ensureInsideRoot(bucketRoot, target, key);
const ordered = [...input.parts].sort((a, b) => a.partNumber - b.partNumber);
try {
const buffers: Buffer[] = [];
for (const part of ordered) {
const partPath = resolve(bucketRoot, localMultipartPartKey(key, part.partNumber));
ensureInsideRoot(bucketRoot, partPath, key);
buffers.push(await readFile(partPath));
}
await mkdir(dirname(target), { recursive: true });
await writeFile(target, Buffer.concat(buffers));
await rm(resolve(bucketRoot, localMultipartDir(key)), { recursive: true, force: true });
} catch (cause) {
throw new StorageAdapterError(
"put_failed",
"Failed to complete local multipart upload.",
{ operation: "completeMultipartUpload", bucket: config.bucket, key, upstream: extractUpstreamDetails(cause) },
{ cause },
);
}
return { bucket: config.bucket, key, publicUrl: buildPublicUrl(config.publicBaseUrl, `${config.bucket}/${key}`) };
},
async abortMultipartUpload(input) {
const key = normalizeLocalObjectKey(input.key);
const dir = resolve(bucketRoot, localMultipartDir(key));
ensureInsideRoot(bucketRoot, dir, key);
await rm(dir, { recursive: true, force: true }).catch(() => {});
},
};
}
function localMultipartDir(key: string): string {
return `${key}.multipart`;
}
function localMultipartPartKey(key: string, partNumber: number): string {
return `${localMultipartDir(key)}/part-${String(partNumber).padStart(6, "0")}`;
}
async function toLocalStorageBuffer(body: UploadBody): Promise<Buffer> {
if (typeof body === "string") {
return Buffer.from(body);
}
if (body instanceof ArrayBuffer) {
return Buffer.from(body);
}
if (body instanceof Uint8Array) {
return Buffer.from(body);
}
const reader = body.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const chunk = await reader.read();
if (chunk.done) {
break;
}
chunks.push(chunk.value);
}
return Buffer.concat(chunks);
}
function normalizeLocalObjectKey(key: string): string {
if (!isPresent(key) || key.startsWith("/") || key.includes("\\") || key.split("/").some((part) => part === "..")) {
throw new StorageAdapterError("invalid_key", "Storage object key is invalid.", {
operation: "normalizeLocalObjectKey",
key,
});
}
return key.replace(/^\.\/+/, "");
}
function ensureInsideRoot(rootDir: string, target: string, key: string): void {
if (target !== rootDir && !target.startsWith(`${rootDir}${sep}`)) {
throw new StorageAdapterError("invalid_key", "Storage object key escapes the configured storage root.", {
operation: "resolveLocalObjectPath",
key,
});
}
}
function isPresent(value: unknown): value is string {
return typeof value === "string" && value.trim().length > 0;
}
function buildPublicUrl(publicBaseUrl: string | undefined, key: string): string | null {
if (!isPresent(publicBaseUrl)) {
return null;
}
const base = publicBaseUrl.replace(/\/+$/, "");
const encodedKey = key
.split("/")
.map((segment) => encodeURIComponent(segment))
.join("/");
return `${base}/${encodedKey}`;
}
function extractUpstreamDetails(cause: unknown): StorageAdapterErrorDetails["upstream"] {
if (!isRecord(cause)) {
return undefined;
}
const metadata = isRecord(cause.$metadata) ? cause.$metadata : undefined;
return {
name: typeof cause.name === "string" ? cause.name : undefined,
message: typeof cause.message === "string" ? redactSecretLikeText(cause.message) : undefined,
status: typeof metadata?.httpStatusCode === "number" ? metadata.httpStatusCode : undefined,
requestId: typeof metadata?.requestId === "string" ? metadata.requestId : undefined,
};
}
function redactSecretLikeText(value: string): string {
return value.replace(/(token|secret|credential)([\s:=]+)([^\s,;]+)/gi, "$1$2[redacted]");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}