import { loadRequiredWebConfig } from "@toolctrl/config"; import { createLocalStorageAdapter, createS3StorageAdapter, createMagicBytesScanAdapter, resolveAllowList, StorageAdapterError, type PreUploadScanAdapter, type PreUploadScanResult, type ScanContext, type StorageAdapter, } from "@toolctrl/storage"; import { fetchOutboundHttpWithRedirects } from "~/utils/outboundHttp.server"; type StorageConfig = ReturnType["storage"]; type RuntimeProvider = "s3" | "local"; // Upload "content" can be caller-supplied. Only inline data: URIs are fetched // directly; any remote URL is routed through the SSRF-safe helper, which // rejects private/loopback/link-local/metadata addresses and validates the // scheme and every redirect hop. A bare fetch() here would let an authenticated // user reach cloud metadata or internal services (e.g. via "HTTP://169.254..." // which the prior case-sensitive "http" check missed). async function fetchUploadContent(content: string): Promise { const trimmed = (content ?? "").trim(); if (trimmed.toLowerCase().startsWith("data:")) { return await (await fetch(trimmed)).blob(); } const response = await fetchOutboundHttpWithRedirects(trimmed, {}); return await response.blob(); } export class StorageScanRejectedError extends Error { readonly bucket: string; readonly id: string; readonly filename: string; readonly scanResult: PreUploadScanResult; constructor(bucket: string, id: string, filename: string, scanResult: PreUploadScanResult) { super(`Upload rejected by pre-upload scanner: ${scanResult.reason ?? "unknown"} (${scanResult.detail ?? ""})`); this.name = "StorageScanRejectedError"; this.bucket = bucket; this.id = id; this.filename = filename; this.scanResult = scanResult; } } export type StoreFileInput = { bucket: string; content: string; id: string; replace?: boolean; throwOnFailure?: boolean; /** * When set, the upload buffer is run through MagicBytesScanAdapter before * the storage PUT. A rejected scan throws StorageScanRejectedError (always, * regardless of throwOnFailure) so callers cannot accidentally swallow a * security verdict. When unset the scan is skipped — that's only safe for * server-internal uploads where the bytes never came from a user. */ scan?: { context: ScanContext; declaredMimeType?: string; filename?: string; /** * Side-effect callback fired exactly once when the scanner returns a * rejected verdict, BEFORE the StorageScanRejectedError is thrown. The * callback may write an audit event or telemetry; storeFile awaits it so * the audit record is persisted before the route gets a chance to * swallow the error. */ onReject?: (verdict: PreUploadScanResult) => void | Promise; /** * When true AND the layer-1 scan passes AND the storage PUT succeeds, * storeFile enqueues a toolctrl.file-scan job so the worker runs ClamAV * (layer 2) over the stored object. Requires SCAN_QUEUE_ENABLED=true * on the runtime; otherwise the enqueue is a no-op. Failures to enqueue * are logged but never throw — the synchronous user-visible upload * should not block on a queue-write hiccup. */ enqueuePostScan?: boolean; /** * Optional tenant attribution for the post-upload scan job (carried * through to the worker's audit row). */ tenantId?: string; }; }; const defaultScanAdapter: PreUploadScanAdapter = createMagicBytesScanAdapter({ allowList: resolveAllowList(typeof process !== "undefined" ? (process.env as Record) : {}), enabled: typeof process !== "undefined" ? process.env.STORAGE_SCAN_LAYER1_ENABLED !== "false" : true, }); export async function storeFile({ bucket, content, id, replace, throwOnFailure = false, scan }: StoreFileInput): Promise { if (content.trim().toLowerCase().startsWith("http") && !replace) { if (throwOnFailure) { throw new Error("Strict uploads require upload content, not an existing URL."); } return content; } const config = loadRequiredWebConfig(); const primaryProvider = resolvePrimaryStorageProvider(config.storage); if (!primaryProvider) { if (throwOnFailure) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "storeFile", bucket, }); } return content; } if (scan) { const blob = await fetchUploadContent(content); const body = new Uint8Array(await blob.arrayBuffer()); const declared = scan.declaredMimeType ?? blob.type ?? "application/octet-stream"; const filename = scan.filename ?? id; const verdict = await defaultScanAdapter.scan({ buffer: body, declaredMimeType: declared, filename, context: scan.context, }); if (verdict.status === "rejected") { if (scan.onReject) { try { await scan.onReject(verdict); } catch { // never let an audit-write failure mask the security verdict } } throw new StorageScanRejectedError(bucket, id, filename, verdict); } let stored: string; try { stored = await putBufferViaProvider(primaryProvider, bucket, id, body, blob.type || undefined, config.storage); } catch (error) { const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider); if (fallbackProvider) { try { stored = await putBufferViaProvider(fallbackProvider, bucket, id, body, blob.type || undefined, config.storage); } catch (fallbackError) { if (throwOnFailure) throw fallbackError; return content; } } else { if (throwOnFailure) throw error; return content; } } if (scan.enqueuePostScan) { try { const { enqueueFileScan } = await import("./fileScanQueue.server.js"); await enqueueFileScan({ bucket, key: id, tenantId: scan.tenantId, contentType: blob.type || scan.declaredMimeType, }); } catch { // never block a successful upload on a queue write } } return stored; } try { return await putFileViaProvider(primaryProvider, bucket, id, content, config.storage); } catch (error) { const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider); if (fallbackProvider) { try { return await putFileViaProvider(fallbackProvider, bucket, id, content, config.storage); } catch (fallbackError) { if (throwOnFailure) { throw fallbackError; } return content; } } if (throwOnFailure) { throw error; } return content; } } export async function putBufferAsFile({ bucket, key, body, contentType, }: { bucket: string; key: string; body: Uint8Array; contentType?: string; }): Promise<{ publicUrl: string | null }> { const config = loadRequiredWebConfig(); const primaryProvider = resolvePrimaryStorageProvider(config.storage); if (!primaryProvider) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "putBufferAsFile", bucket, }); } const adapter = createRuntimeStorageAdapter(primaryProvider, bucket, config.storage); const result = await adapter.putObject({ key, body, contentType }); return { publicUrl: result.publicUrl ?? null }; } export async function deleteFile({ bucket, key }: { bucket: string; key: string }): Promise { const config = loadRequiredWebConfig(); const primaryProvider = resolvePrimaryStorageProvider(config.storage); if (!primaryProvider) { return; } const adapter = createRuntimeStorageAdapter(primaryProvider, bucket, config.storage); await adapter.deleteObject({ key }); } export function hasStorageCredentials(): boolean { const config = loadRequiredWebConfig(); return resolvePrimaryStorageProvider(config.storage) !== undefined; } /** * Default files bucket (STORAGE_BUCKET). The Files module (AK-P4a) stores all * objects in this single private bucket; tenant isolation is carried by the * `tenants/{tenantId}/...` key prefix (§2) plus app-level scoping/RLS. */ export function resolveStorageBucket(): string { const config = loadRequiredWebConfig(); const bucket = config.storage.bucket; if (!bucket || bucket.trim().length === 0) { throw new StorageAdapterError("config_missing", "STORAGE_BUCKET is not configured.", { operation: "resolveStorageBucket", missingFields: ["bucket"], }); } return bucket; } /** * Assemble previously-stored temp chunk objects into a single object via * multipart UploadPartCopy (§3). `chunkSourceKeys` MUST be ordered by chunk * index. Aborts the multipart upload on any failure so no dangling parts remain. */ export async function assembleChunkedUpload({ bucket, targetKey, chunkSourceKeys, contentType, }: { bucket: string; targetKey: string; chunkSourceKeys: readonly string[]; contentType?: string; }): Promise<{ publicUrl: string | null }> { const config = loadRequiredWebConfig(); const provider = resolvePrimaryStorageProvider(config.storage); if (!provider) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "assembleChunkedUpload", bucket, }); } const adapter = createRuntimeStorageAdapter(provider, bucket, config.storage); const handle = await adapter.createMultipartUpload({ key: targetKey, contentType }); try { const parts = []; for (let index = 0; index < chunkSourceKeys.length; index += 1) { parts.push( await adapter.uploadPartCopy({ key: targetKey, uploadId: handle.uploadId, partNumber: index + 1, sourceKey: chunkSourceKeys[index], }), ); } const result = await adapter.completeMultipartUpload({ key: targetKey, uploadId: handle.uploadId, parts }); return { publicUrl: result.publicUrl ?? null }; } catch (error) { await adapter.abortMultipartUpload({ key: targetKey, uploadId: handle.uploadId }).catch(() => {}); throw error; } } async function putFileViaProvider(provider: RuntimeProvider, bucket: string, id: string, content: string, storage: StorageConfig): Promise { const blob = await fetchUploadContent(content); const body = new Uint8Array(await blob.arrayBuffer()); const contentType = blob.type || undefined; return putBufferViaProvider(provider, bucket, id, body, contentType, storage); } async function putBufferViaProvider( provider: RuntimeProvider, bucket: string, id: string, body: Uint8Array, contentType: string | undefined, storage: StorageConfig ): Promise { const adapter = createRuntimeStorageAdapter(provider, bucket, storage); const result = await adapter.putObject({ key: id, body, contentType }); return result.publicUrl ?? ""; } function resolvePrimaryStorageProvider(storage: StorageConfig): RuntimeProvider | undefined { if (storage.provider === "s3" || storage.provider === "local") { return storage.provider; } if (hasCompleteS3StorageConfig(storage)) { return "s3"; } if (storage.localRoot) { return "local"; } return undefined; } function resolveFallbackProvider(storage: StorageConfig, primary: RuntimeProvider): RuntimeProvider | undefined { const fallback = storage.fallbackProvider; if ((fallback === "s3" || fallback === "local") && fallback !== primary) { return fallback; } return undefined; } function createRuntimeStorageAdapter(provider: RuntimeProvider, bucket: string, storage: StorageConfig): StorageAdapter { if (provider === "s3") { if (!hasCompleteS3StorageConfig(storage)) { throw new StorageAdapterError("config_missing", "S3 storage configuration is incomplete.", { operation: "createRuntimeStorageAdapter", bucket, missingFields: ["endpoint", "region", "accessKeyId", "secretAccessKey"].filter((field) => !storage[field as keyof typeof storage]), }); } return createS3StorageAdapter({ endpoint: storage.endpoint, region: storage.region, bucket, accessKeyId: storage.accessKeyId, secretAccessKey: storage.secretAccessKey, sessionToken: storage.sessionToken, publicBaseUrl: storage.publicBaseUrl, forcePathStyle: storage.forcePathStyle === undefined ? undefined : storage.forcePathStyle.toLowerCase() === "true", }); } if (!storage.localRoot) { throw new StorageAdapterError("config_missing", "Local storage root is not configured.", { operation: "createRuntimeStorageAdapter", bucket, missingFields: ["localRoot"], }); } return createLocalStorageAdapter({ rootDir: storage.localRoot, bucket, publicBaseUrl: storage.localPublicBaseUrl, }); } function hasCompleteS3StorageConfig(storage: StorageConfig): storage is StorageConfig & { endpoint: string; region: string; accessKeyId: string; secretAccessKey: string; } { return Boolean(storage.endpoint && storage.region && storage.accessKeyId && storage.secretAccessKey); }