import { loadRequiredWebConfig } from "@hivegrid/config"; import { createLocalStorageAdapter, createS3StorageAdapter, createMagicBytesScanAdapter, createClamAVBufferScanner, ClamAVUnreachableError, resolveAllowList, StorageAdapterError, type PreUploadScanAdapter, type PreUploadScanResult, type ScanContext, type StorageAdapter, type ClamAVBufferScanner, } from "@hivegrid/storage"; import { fetchOutboundHttpWithRedirects } from "~/utils/outboundHttp.server"; type StorageConfig = ReturnType["storage"]; type RuntimeProvider = "s3" | "local"; export const DEFAULT_UPLOAD_CONTENT_MAX_BYTES = 100 * 1024 * 1024; export const AVATAR_UPLOAD_CONTENT_MAX_BYTES = 5 * 1024 * 1024; export const DEFAULT_UPLOAD_BODY_TIMEOUT_MS = 10_000; export class UploadContentReadError extends Error { readonly code: "too_large" | "timeout" | "upstream_failed"; constructor(code: "too_large" | "timeout" | "upstream_failed", message: string) { super(message); this.name = "UploadContentReadError"; this.code = code; } } type FetchUploadContentOptions = { maxBytes?: number; bodyTimeoutMs?: number; }; // Upload "content" can be caller-supplied. Only inline data: URIs are fetched // directly; any remote URL is routed through the SSRF-safe helper, which // rejects private/loopback/link-local/metadata addresses and validates the // scheme and every redirect hop. A bare fetch() here would let an authenticated // user reach cloud metadata or internal services (e.g. via "HTTP://169.254..." // which the prior case-sensitive "http" check missed). export async function fetchUploadContent(content: string, options: FetchUploadContentOptions = {}): Promise { const trimmed = (content ?? "").trim(); const maxBytes = options.maxBytes ?? DEFAULT_UPLOAD_CONTENT_MAX_BYTES; const bodyTimeoutMs = options.bodyTimeoutMs ?? DEFAULT_UPLOAD_BODY_TIMEOUT_MS; assertValidReadLimits(maxBytes, bodyTimeoutMs); let response: Response; if (trimmed.toLowerCase().startsWith("data:")) { assertDataUriWithinLimit(trimmed, maxBytes); response = await fetch(trimmed); } else { response = await fetchOutboundHttpWithRedirects(trimmed, {}, { timeoutMs: bodyTimeoutMs }); } if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new UploadContentReadError("upstream_failed", "Upload content could not be fetched."); } return readLimitedResponseBlob(response, { maxBytes, bodyTimeoutMs }); } async function readLimitedResponseBlob(response: Response, { maxBytes, bodyTimeoutMs }: Required): Promise { const contentLength = response.headers.get("content-length"); if (contentLength && /^\d+$/.test(contentLength) && Number(contentLength) > maxBytes) { await response.body?.cancel().catch(() => undefined); throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`); } if (!response.body) { return new Blob([], { type: response.headers.get("content-type") ?? "" }); } const reader = response.body.getReader(); const chunks: Uint8Array[] = []; let bytesRead = 0; let timeout: ReturnType | undefined; const timeoutPromise = new Promise((_resolve, reject) => { timeout = setTimeout(() => { reject(new UploadContentReadError("timeout", "Upload content body timed out.")); void reader.cancel().catch(() => undefined); }, bodyTimeoutMs); }); try { while (true) { const chunk = await Promise.race([reader.read(), timeoutPromise]); if (chunk.done) { break; } bytesRead += chunk.value.byteLength; if (bytesRead > maxBytes) { await reader.cancel().catch(() => undefined); throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`); } chunks.push(chunk.value); } } finally { if (timeout) { clearTimeout(timeout); } } const blobParts = chunks.map((chunk) => { const copy = new Uint8Array(chunk.byteLength); copy.set(chunk); return copy.buffer; }); return new Blob(blobParts, { type: response.headers.get("content-type") ?? "" }); } function assertValidReadLimits(maxBytes: number, bodyTimeoutMs: number) { if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || !Number.isSafeInteger(bodyTimeoutMs) || bodyTimeoutMs <= 0) { throw new TypeError("Upload read limits must be positive safe integers."); } } function assertDataUriWithinLimit(dataUri: string, maxBytes: number) { const commaIndex = dataUri.indexOf(","); if (commaIndex < 0) { throw new UploadContentReadError("upstream_failed", "Upload data URI is malformed."); } const metadata = dataUri.slice(0, commaIndex).toLowerCase(); const payloadLength = dataUri.length - commaIndex - 1; const maximumEncodedLength = metadata.includes(";base64") ? Math.ceil((maxBytes * 4) / 3) + 4 : maxBytes; if (payloadLength > maximumEncodedLength) { throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`); } } function uploadContentLimitForScan(scan: StoreFileInput["scan"]): number { return scan?.context === "avatar" ? AVATAR_UPLOAD_CONTENT_MAX_BYTES : DEFAULT_UPLOAD_CONTENT_MAX_BYTES; } export class StorageScanRejectedError extends Error { readonly bucket: string; readonly id: string; readonly filename: string; readonly scanResult: PreUploadScanResult; constructor(bucket: string, id: string, filename: string, scanResult: PreUploadScanResult) { super(`Upload rejected by pre-upload scanner: ${scanResult.reason ?? "unknown"} (${scanResult.detail ?? ""})`); this.name = "StorageScanRejectedError"; this.bucket = bucket; this.id = id; this.filename = filename; this.scanResult = scanResult; } } type ClamAvRuntimeEnv = { CLAMD_HOST?: string; CLAMD_PORT?: string; CLAMD_TIMEOUT_MS?: string; CLAMAV_HOST?: string; CLAMAV_PORT?: string; CLAMAV_TIMEOUT_MS?: string; }; let cachedClamAvScanner: { key: string; scanner: ClamAVBufferScanner } | undefined; function getClamAvBufferScanner(env: ClamAvRuntimeEnv = process.env): ClamAVBufferScanner { const host = env.CLAMD_HOST?.trim() || env.CLAMAV_HOST?.trim(); if (!host) { throw new ClamAVUnreachableError("clamd host is not configured"); } const port = parsePositiveInteger(env.CLAMD_PORT ?? env.CLAMAV_PORT, 3310, "clamd port"); const timeoutMs = parsePositiveInteger(env.CLAMD_TIMEOUT_MS ?? env.CLAMAV_TIMEOUT_MS, 30_000, "clamd timeout"); const key = `${host}:${port}:${timeoutMs}`; if (cachedClamAvScanner?.key !== key) { cachedClamAvScanner = { key, scanner: createClamAVBufferScanner({ host, port, timeoutMs }), }; } return cachedClamAvScanner.scanner; } function parsePositiveInteger(rawValue: string | undefined, fallback: number, label: string): number { if (rawValue === undefined || rawValue.trim() === "") { return fallback; } const value = Number(rawValue); if (!Number.isSafeInteger(value) || value <= 0) { throw new ClamAVUnreachableError(`${label} is invalid`); } return value; } export async function requireCleanMalwareScan({ body, bucket, key, filename, onReject, scanner, }: { body: Uint8Array; bucket: string; key: string; filename: string; onReject?: (verdict: PreUploadScanResult) => void | Promise; scanner?: ClamAVBufferScanner; }): Promise { let verdict: Awaited>; try { const buffer = Buffer.from(body.buffer, body.byteOffset, body.byteLength); verdict = await (scanner ?? getClamAvBufferScanner()).scanBuffer(buffer); } catch { const rejected: PreUploadScanResult = { status: "rejected", reason: "scanner_error", detail: "The malware scanner did not return a clean verdict.", }; await persistRejectedScan(onReject, rejected); throw new StorageScanRejectedError(bucket, key, filename, rejected); } if (verdict.verdict !== "clean") { const rejected: PreUploadScanResult = { status: "rejected", reason: "malware_detected", detail: "The malware scanner rejected the upload.", }; await persistRejectedScan(onReject, rejected); throw new StorageScanRejectedError(bucket, key, filename, rejected); } } async function persistRejectedScan( onReject: ((verdict: PreUploadScanResult) => void | Promise) | undefined, verdict: PreUploadScanResult, ) { if (!onReject) return; try { await onReject(verdict); } catch { // An audit-write failure must never mask or reverse a security verdict. } } export type StoreFileInput = { bucket: string; content: string; id: string; replace?: boolean; throwOnFailure?: boolean; /** * When set, the upload buffer is run through MagicBytesScanAdapter before * the storage PUT. A rejected scan throws StorageScanRejectedError (always, * regardless of throwOnFailure) so callers cannot accidentally swallow a * security verdict. When unset the scan is skipped — that's only safe for * server-internal uploads where the bytes never came from a user. */ scan?: { context: ScanContext; declaredMimeType?: string; filename?: string; /** * Side-effect callback fired exactly once when the scanner returns a * rejected verdict, BEFORE the StorageScanRejectedError is thrown. The * callback may write an audit event or telemetry; storeFile awaits it so * the audit record is persisted before the route gets a chance to * swallow the error. */ onReject?: (verdict: PreUploadScanResult) => void | Promise; /** * When true, ClamAV must return a clean verdict for the in-memory upload * before any storage PUT. Missing configuration, transport failures and * indeterminate verdicts all reject the upload. */ requireMalwareScan?: boolean; /** * Optional tenant attribution for the post-upload scan job (carried * through to the worker's audit row). */ tenantId?: string; }; }; export type StoreFileBytesInput = { bucket: string; body: Uint8Array; contentType?: string; id: string; scan?: StoreFileInput["scan"]; }; const defaultScanAdapter: PreUploadScanAdapter = createMagicBytesScanAdapter({ allowList: resolveAllowList(typeof process !== "undefined" ? (process.env as Record) : {}), enabled: typeof process !== "undefined" ? process.env.STORAGE_SCAN_LAYER1_ENABLED !== "false" : true, }); export async function storeFileBytes({ bucket, body, contentType, id, scan, }: StoreFileBytesInput): Promise { const config = loadRequiredWebConfig(); const primaryProvider = resolvePrimaryStorageProvider(config.storage); if (!primaryProvider) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "storeFileBytes", bucket, }); } if (scan) { const maxBytes = uploadContentLimitForScan(scan); if (body.byteLength > maxBytes) { throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`); } const declared = scan.declaredMimeType ?? contentType ?? "application/octet-stream"; const filename = scan.filename ?? id; const verdict = await defaultScanAdapter.scan({ buffer: body, declaredMimeType: declared, filename, context: scan.context, }); if (verdict.status === "rejected") { await persistRejectedScan(scan.onReject, verdict); throw new StorageScanRejectedError(bucket, id, filename, verdict); } if (scan.requireMalwareScan) { await requireCleanMalwareScan({ body, bucket, key: id, filename, onReject: scan.onReject }); } } try { return await putBufferViaProvider(primaryProvider, bucket, id, body, contentType, config.storage); } catch (error) { const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider); if (fallbackProvider) { try { return await putBufferViaProvider(fallbackProvider, bucket, id, body, contentType, config.storage); } catch (fallbackError) { throw fallbackError; } } throw error; } } export async function storeFile({ bucket, content, id, replace, throwOnFailure = false, scan }: StoreFileInput): Promise { // A scan marks the bytes as untrusted input. Never fall back to persisting a // data URI or remote URL when storage is unavailable, even when a legacy // caller did not opt into the general throwOnFailure behavior. const failClosed = throwOnFailure || Boolean(scan); // The legacy no-op is safe only when no scan was requested. Treating an // arbitrary remote URL as an already-stored object must never bypass a // caller's magic-byte or malware gate. if (content.trim().toLowerCase().startsWith("http") && !replace && !scan) { if (failClosed) { throw new Error("Strict uploads require upload content, not an existing URL."); } return content; } const config = loadRequiredWebConfig(); const primaryProvider = resolvePrimaryStorageProvider(config.storage); if (!primaryProvider) { if (failClosed) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "storeFile", bucket, }); } return content; } if (scan) { const blob = await fetchUploadContent(content, { maxBytes: uploadContentLimitForScan(scan) }); const body = new Uint8Array(await blob.arrayBuffer()); const declared = scan.declaredMimeType ?? blob.type ?? "application/octet-stream"; const filename = scan.filename ?? id; const verdict = await defaultScanAdapter.scan({ buffer: body, declaredMimeType: declared, filename, context: scan.context, }); if (verdict.status === "rejected") { if (scan.onReject) { try { await scan.onReject(verdict); } catch { // never let an audit-write failure mask the security verdict } } throw new StorageScanRejectedError(bucket, id, filename, verdict); } if (scan.requireMalwareScan) { await requireCleanMalwareScan({ body, bucket, key: id, filename, onReject: scan.onReject }); } let stored: string; try { stored = await putBufferViaProvider(primaryProvider, bucket, id, body, blob.type || undefined, config.storage); } catch (error) { const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider); if (fallbackProvider) { try { stored = await putBufferViaProvider(fallbackProvider, bucket, id, body, blob.type || undefined, config.storage); } catch (fallbackError) { if (failClosed) throw fallbackError; return content; } } else { if (failClosed) throw error; return content; } } return stored; } try { return await putFileViaProvider(primaryProvider, bucket, id, content, config.storage); } catch (error) { const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider); if (fallbackProvider) { try { return await putFileViaProvider(fallbackProvider, bucket, id, content, config.storage); } catch (fallbackError) { if (failClosed) { throw fallbackError; } return content; } } if (failClosed) { throw error; } return content; } } export async function putBufferAsFile({ bucket, key, body, contentType, }: { bucket: string; key: string; body: Uint8Array; contentType?: string; }): Promise<{ publicUrl: string | null }> { const config = loadRequiredWebConfig(); const primaryProvider = resolvePrimaryStorageProvider(config.storage); if (!primaryProvider) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "putBufferAsFile", bucket, }); } const adapter = createRuntimeStorageAdapter(primaryProvider, bucket, config.storage); const result = await adapter.putObject({ key, body, contentType }); return { publicUrl: result.publicUrl ?? null }; } export async function deleteFile({ bucket, key }: { bucket: string; key: string }): Promise { const config = loadRequiredWebConfig(); const primaryProvider = resolvePrimaryStorageProvider(config.storage); if (!primaryProvider) { return; } const adapter = createRuntimeStorageAdapter(primaryProvider, bucket, config.storage); await adapter.deleteObject({ key }); } export function hasStorageCredentials(): boolean { const config = loadRequiredWebConfig(); return resolvePrimaryStorageProvider(config.storage) !== undefined; } /** * Default files bucket (STORAGE_BUCKET). The Files module (AK-P4a) stores all * objects in this single private bucket; tenant isolation is carried by the * `tenants/{tenantId}/...` key prefix (§2) plus app-level scoping/RLS. */ export function resolveStorageBucket(): string { const config = loadRequiredWebConfig(); const bucket = config.storage.bucket; if (!bucket || bucket.trim().length === 0) { throw new StorageAdapterError("config_missing", "STORAGE_BUCKET is not configured.", { operation: "resolveStorageBucket", missingFields: ["bucket"], }); } return bucket; } export function resolveStorageBinding(): { provider: RuntimeProvider; bucket: string; endpoint: string } { const config = loadRequiredWebConfig(); const provider = resolvePrimaryStorageProvider(config.storage); const bucket = resolveStorageBucket(); if (!provider) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "resolveStorageBinding", bucket, }); } const endpoint = provider === "s3" ? config.storage.endpoint : config.storage.localRoot; if (!endpoint) { throw new StorageAdapterError("config_missing", "Storage endpoint is not configured.", { operation: "resolveStorageBinding", bucket, missingFields: [provider === "s3" ? "endpoint" : "localRoot"], }); } return { provider, bucket, endpoint }; } /** * Assemble previously-stored temp chunk objects into a single object via * multipart UploadPartCopy (§3). `chunkSourceKeys` MUST be ordered by chunk * index. Aborts the multipart upload on any failure so no dangling parts remain. */ export async function assembleChunkedUpload({ bucket, targetKey, chunkSourceKeys, contentType, }: { bucket: string; targetKey: string; chunkSourceKeys: readonly string[]; contentType?: string; }): Promise<{ publicUrl: string | null }> { const config = loadRequiredWebConfig(); const provider = resolvePrimaryStorageProvider(config.storage); if (!provider) { throw new StorageAdapterError("config_missing", "No storage provider configured.", { operation: "assembleChunkedUpload", bucket, }); } const adapter = createRuntimeStorageAdapter(provider, bucket, config.storage); const handle = await adapter.createMultipartUpload({ key: targetKey, contentType }); try { const parts = []; for (let index = 0; index < chunkSourceKeys.length; index += 1) { parts.push( await adapter.uploadPartCopy({ key: targetKey, uploadId: handle.uploadId, partNumber: index + 1, sourceKey: chunkSourceKeys[index], }), ); } const result = await adapter.completeMultipartUpload({ key: targetKey, uploadId: handle.uploadId, parts }); return { publicUrl: result.publicUrl ?? null }; } catch (error) { await adapter.abortMultipartUpload({ key: targetKey, uploadId: handle.uploadId }).catch(() => {}); throw error; } } async function putFileViaProvider(provider: RuntimeProvider, bucket: string, id: string, content: string, storage: StorageConfig): Promise { const blob = await fetchUploadContent(content); const body = new Uint8Array(await blob.arrayBuffer()); const contentType = blob.type || undefined; return putBufferViaProvider(provider, bucket, id, body, contentType, storage); } async function putBufferViaProvider( provider: RuntimeProvider, bucket: string, id: string, body: Uint8Array, contentType: string | undefined, storage: StorageConfig ): Promise { const adapter = createRuntimeStorageAdapter(provider, bucket, storage); const result = await adapter.putObject({ key: id, body, contentType }); return result.publicUrl ?? ""; } function resolvePrimaryStorageProvider(storage: StorageConfig): RuntimeProvider | undefined { if (storage.provider === "s3" || storage.provider === "local") { return storage.provider; } if (hasCompleteS3StorageConfig(storage)) { return "s3"; } if (storage.localRoot) { return "local"; } return undefined; } function resolveFallbackProvider(storage: StorageConfig, primary: RuntimeProvider): RuntimeProvider | undefined { const fallback = storage.fallbackProvider; if ((fallback === "s3" || fallback === "local") && fallback !== primary) { return fallback; } return undefined; } function createRuntimeStorageAdapter(provider: RuntimeProvider, bucket: string, storage: StorageConfig): StorageAdapter { if (provider === "s3") { if (!hasCompleteS3StorageConfig(storage)) { throw new StorageAdapterError("config_missing", "S3 storage configuration is incomplete.", { operation: "createRuntimeStorageAdapter", bucket, missingFields: ["endpoint", "region", "accessKeyId", "secretAccessKey"].filter((field) => !storage[field as keyof typeof storage]), }); } return createS3StorageAdapter({ endpoint: storage.endpoint, region: storage.region, bucket, accessKeyId: storage.accessKeyId, secretAccessKey: storage.secretAccessKey, sessionToken: storage.sessionToken, publicBaseUrl: storage.publicBaseUrl, forcePathStyle: storage.forcePathStyle === undefined ? undefined : storage.forcePathStyle.toLowerCase() === "true", }); } if (!storage.localRoot) { throw new StorageAdapterError("config_missing", "Local storage root is not configured.", { operation: "createRuntimeStorageAdapter", bucket, missingFields: ["localRoot"], }); } return createLocalStorageAdapter({ rootDir: storage.localRoot, bucket, publicBaseUrl: storage.localPublicBaseUrl, }); } function hasCompleteS3StorageConfig(storage: StorageConfig): storage is StorageConfig & { endpoint: string; region: string; accessKeyId: string; secretAccessKey: string; } { return Boolean(storage.endpoint && storage.region && storage.accessKeyId && storage.secretAccessKey); }