Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -3,31 +3,135 @@ import {
createLocalStorageAdapter,
createS3StorageAdapter,
createMagicBytesScanAdapter,
createClamAVBufferScanner,
ClamAVUnreachableError,
resolveAllowList,
StorageAdapterError,
type PreUploadScanAdapter,
type PreUploadScanResult,
type ScanContext,
type StorageAdapter,
type ClamAVBufferScanner,
} from "@hivegrid/storage";
import { fetchOutboundHttpWithRedirects } from "~/utils/outboundHttp.server";
type StorageConfig = ReturnType<typeof loadRequiredWebConfig>["storage"];
type RuntimeProvider = "s3" | "local";
export const DEFAULT_UPLOAD_CONTENT_MAX_BYTES = 100 * 1024 * 1024;
export const AVATAR_UPLOAD_CONTENT_MAX_BYTES = 5 * 1024 * 1024;
export const DEFAULT_UPLOAD_BODY_TIMEOUT_MS = 10_000;
export class UploadContentReadError extends Error {
readonly code: "too_large" | "timeout" | "upstream_failed";
constructor(code: "too_large" | "timeout" | "upstream_failed", message: string) {
super(message);
this.name = "UploadContentReadError";
this.code = code;
}
}
type FetchUploadContentOptions = {
maxBytes?: number;
bodyTimeoutMs?: number;
};
// Upload "content" can be caller-supplied. Only inline data: URIs are fetched
// directly; any remote URL is routed through the SSRF-safe helper, which
// rejects private/loopback/link-local/metadata addresses and validates the
// scheme and every redirect hop. A bare fetch() here would let an authenticated
// user reach cloud metadata or internal services (e.g. via "HTTP://169.254..."
// which the prior case-sensitive "http" check missed).
async function fetchUploadContent(content: string): Promise<Blob> {
export async function fetchUploadContent(content: string, options: FetchUploadContentOptions = {}): Promise<Blob> {
const trimmed = (content ?? "").trim();
const maxBytes = options.maxBytes ?? DEFAULT_UPLOAD_CONTENT_MAX_BYTES;
const bodyTimeoutMs = options.bodyTimeoutMs ?? DEFAULT_UPLOAD_BODY_TIMEOUT_MS;
assertValidReadLimits(maxBytes, bodyTimeoutMs);
let response: Response;
if (trimmed.toLowerCase().startsWith("data:")) {
return await (await fetch(trimmed)).blob();
assertDataUriWithinLimit(trimmed, maxBytes);
response = await fetch(trimmed);
} else {
response = await fetchOutboundHttpWithRedirects(trimmed, {}, { timeoutMs: bodyTimeoutMs });
}
const response = await fetchOutboundHttpWithRedirects(trimmed, {});
return await response.blob();
if (!response.ok) {
await response.body?.cancel().catch(() => undefined);
throw new UploadContentReadError("upstream_failed", "Upload content could not be fetched.");
}
return readLimitedResponseBlob(response, { maxBytes, bodyTimeoutMs });
}
async function readLimitedResponseBlob(response: Response, { maxBytes, bodyTimeoutMs }: Required<FetchUploadContentOptions>): Promise<Blob> {
const contentLength = response.headers.get("content-length");
if (contentLength && /^\d+$/.test(contentLength) && Number(contentLength) > maxBytes) {
await response.body?.cancel().catch(() => undefined);
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
}
if (!response.body) {
return new Blob([], { type: response.headers.get("content-type") ?? "" });
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let bytesRead = 0;
let timeout: ReturnType<typeof setTimeout> | undefined;
const timeoutPromise = new Promise<never>((_resolve, reject) => {
timeout = setTimeout(() => {
reject(new UploadContentReadError("timeout", "Upload content body timed out."));
void reader.cancel().catch(() => undefined);
}, bodyTimeoutMs);
});
try {
while (true) {
const chunk = await Promise.race([reader.read(), timeoutPromise]);
if (chunk.done) {
break;
}
bytesRead += chunk.value.byteLength;
if (bytesRead > maxBytes) {
await reader.cancel().catch(() => undefined);
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
}
chunks.push(chunk.value);
}
} finally {
if (timeout) {
clearTimeout(timeout);
}
}
const blobParts = chunks.map((chunk) => {
const copy = new Uint8Array(chunk.byteLength);
copy.set(chunk);
return copy.buffer;
});
return new Blob(blobParts, { type: response.headers.get("content-type") ?? "" });
}
function assertValidReadLimits(maxBytes: number, bodyTimeoutMs: number) {
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || !Number.isSafeInteger(bodyTimeoutMs) || bodyTimeoutMs <= 0) {
throw new TypeError("Upload read limits must be positive safe integers.");
}
}
function assertDataUriWithinLimit(dataUri: string, maxBytes: number) {
const commaIndex = dataUri.indexOf(",");
if (commaIndex < 0) {
throw new UploadContentReadError("upstream_failed", "Upload data URI is malformed.");
}
const metadata = dataUri.slice(0, commaIndex).toLowerCase();
const payloadLength = dataUri.length - commaIndex - 1;
const maximumEncodedLength = metadata.includes(";base64") ? Math.ceil((maxBytes * 4) / 3) + 4 : maxBytes;
if (payloadLength > maximumEncodedLength) {
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
}
}
function uploadContentLimitForScan(scan: StoreFileInput["scan"]): number {
return scan?.context === "avatar" ? AVATAR_UPLOAD_CONTENT_MAX_BYTES : DEFAULT_UPLOAD_CONTENT_MAX_BYTES;
}
export class StorageScanRejectedError extends Error {
@@ -45,6 +149,97 @@ export class StorageScanRejectedError extends Error {
}
}
type ClamAvRuntimeEnv = {
CLAMD_HOST?: string;
CLAMD_PORT?: string;
CLAMD_TIMEOUT_MS?: string;
CLAMAV_HOST?: string;
CLAMAV_PORT?: string;
CLAMAV_TIMEOUT_MS?: string;
};
let cachedClamAvScanner: { key: string; scanner: ClamAVBufferScanner } | undefined;
function getClamAvBufferScanner(env: ClamAvRuntimeEnv = process.env): ClamAVBufferScanner {
const host = env.CLAMD_HOST?.trim() || env.CLAMAV_HOST?.trim();
if (!host) {
throw new ClamAVUnreachableError("clamd host is not configured");
}
const port = parsePositiveInteger(env.CLAMD_PORT ?? env.CLAMAV_PORT, 3310, "clamd port");
const timeoutMs = parsePositiveInteger(env.CLAMD_TIMEOUT_MS ?? env.CLAMAV_TIMEOUT_MS, 30_000, "clamd timeout");
const key = `${host}:${port}:${timeoutMs}`;
if (cachedClamAvScanner?.key !== key) {
cachedClamAvScanner = {
key,
scanner: createClamAVBufferScanner({ host, port, timeoutMs }),
};
}
return cachedClamAvScanner.scanner;
}
function parsePositiveInteger(rawValue: string | undefined, fallback: number, label: string): number {
if (rawValue === undefined || rawValue.trim() === "") {
return fallback;
}
const value = Number(rawValue);
if (!Number.isSafeInteger(value) || value <= 0) {
throw new ClamAVUnreachableError(`${label} is invalid`);
}
return value;
}
export async function requireCleanMalwareScan({
body,
bucket,
key,
filename,
onReject,
scanner,
}: {
body: Uint8Array;
bucket: string;
key: string;
filename: string;
onReject?: (verdict: PreUploadScanResult) => void | Promise<void>;
scanner?: ClamAVBufferScanner;
}): Promise<void> {
let verdict: Awaited<ReturnType<ClamAVBufferScanner["scanBuffer"]>>;
try {
const buffer = Buffer.from(body.buffer, body.byteOffset, body.byteLength);
verdict = await (scanner ?? getClamAvBufferScanner()).scanBuffer(buffer);
} catch {
const rejected: PreUploadScanResult = {
status: "rejected",
reason: "scanner_error",
detail: "The malware scanner did not return a clean verdict.",
};
await persistRejectedScan(onReject, rejected);
throw new StorageScanRejectedError(bucket, key, filename, rejected);
}
if (verdict.verdict !== "clean") {
const rejected: PreUploadScanResult = {
status: "rejected",
reason: "malware_detected",
detail: "The malware scanner rejected the upload.",
};
await persistRejectedScan(onReject, rejected);
throw new StorageScanRejectedError(bucket, key, filename, rejected);
}
}
async function persistRejectedScan(
onReject: ((verdict: PreUploadScanResult) => void | Promise<void>) | undefined,
verdict: PreUploadScanResult,
) {
if (!onReject) return;
try {
await onReject(verdict);
} catch {
// An audit-write failure must never mask or reverse a security verdict.
}
}
export type StoreFileInput = {
bucket: string;
content: string;
@@ -71,14 +266,11 @@ export type StoreFileInput = {
*/
onReject?: (verdict: PreUploadScanResult) => void | Promise<void>;
/**
* When true AND the layer-1 scan passes AND the storage PUT succeeds,
* storeFile enqueues a hivegrid.file-scan job so the worker runs ClamAV
* (layer 2) over the stored object. Requires SCAN_QUEUE_ENABLED=true
* on the runtime; otherwise the enqueue is a no-op. Failures to enqueue
* are logged but never throw — the synchronous user-visible upload
* should not block on a queue-write hiccup.
* When true, ClamAV must return a clean verdict for the in-memory upload
* before any storage PUT. Missing configuration, transport failures and
* indeterminate verdicts all reject the upload.
*/
enqueuePostScan?: boolean;
requireMalwareScan?: boolean;
/**
* Optional tenant attribution for the post-upload scan job (carried
* through to the worker's audit row).
@@ -87,14 +279,82 @@ export type StoreFileInput = {
};
};
export type StoreFileBytesInput = {
bucket: string;
body: Uint8Array;
contentType?: string;
id: string;
scan?: StoreFileInput["scan"];
};
const defaultScanAdapter: PreUploadScanAdapter = createMagicBytesScanAdapter({
allowList: resolveAllowList(typeof process !== "undefined" ? (process.env as Record<string, string | undefined>) : {}),
enabled: typeof process !== "undefined" ? process.env.STORAGE_SCAN_LAYER1_ENABLED !== "false" : true,
});
export async function storeFileBytes({
bucket,
body,
contentType,
id,
scan,
}: StoreFileBytesInput): Promise<string> {
const config = loadRequiredWebConfig();
const primaryProvider = resolvePrimaryStorageProvider(config.storage);
if (!primaryProvider) {
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
operation: "storeFileBytes",
bucket,
});
}
if (scan) {
const maxBytes = uploadContentLimitForScan(scan);
if (body.byteLength > maxBytes) {
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
}
const declared = scan.declaredMimeType ?? contentType ?? "application/octet-stream";
const filename = scan.filename ?? id;
const verdict = await defaultScanAdapter.scan({
buffer: body,
declaredMimeType: declared,
filename,
context: scan.context,
});
if (verdict.status === "rejected") {
await persistRejectedScan(scan.onReject, verdict);
throw new StorageScanRejectedError(bucket, id, filename, verdict);
}
if (scan.requireMalwareScan) {
await requireCleanMalwareScan({ body, bucket, key: id, filename, onReject: scan.onReject });
}
}
try {
return await putBufferViaProvider(primaryProvider, bucket, id, body, contentType, config.storage);
} catch (error) {
const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider);
if (fallbackProvider) {
try {
return await putBufferViaProvider(fallbackProvider, bucket, id, body, contentType, config.storage);
} catch (fallbackError) {
throw fallbackError;
}
}
throw error;
}
}
export async function storeFile({ bucket, content, id, replace, throwOnFailure = false, scan }: StoreFileInput): Promise<string> {
if (content.trim().toLowerCase().startsWith("http") && !replace) {
if (throwOnFailure) {
// A scan marks the bytes as untrusted input. Never fall back to persisting a
// data URI or remote URL when storage is unavailable, even when a legacy
// caller did not opt into the general throwOnFailure behavior.
const failClosed = throwOnFailure || Boolean(scan);
// The legacy no-op is safe only when no scan was requested. Treating an
// arbitrary remote URL as an already-stored object must never bypass a
// caller's magic-byte or malware gate.
if (content.trim().toLowerCase().startsWith("http") && !replace && !scan) {
if (failClosed) {
throw new Error("Strict uploads require upload content, not an existing URL.");
}
return content;
@@ -103,7 +363,7 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
const config = loadRequiredWebConfig();
const primaryProvider = resolvePrimaryStorageProvider(config.storage);
if (!primaryProvider) {
if (throwOnFailure) {
if (failClosed) {
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
operation: "storeFile",
bucket,
@@ -113,7 +373,7 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
}
if (scan) {
const blob = await fetchUploadContent(content);
const blob = await fetchUploadContent(content, { maxBytes: uploadContentLimitForScan(scan) });
const body = new Uint8Array(await blob.arrayBuffer());
const declared = scan.declaredMimeType ?? blob.type ?? "application/octet-stream";
const filename = scan.filename ?? id;
@@ -133,6 +393,9 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
}
throw new StorageScanRejectedError(bucket, id, filename, verdict);
}
if (scan.requireMalwareScan) {
await requireCleanMalwareScan({ body, bucket, key: id, filename, onReject: scan.onReject });
}
let stored: string;
try {
stored = await putBufferViaProvider(primaryProvider, bucket, id, body, blob.type || undefined, config.storage);
@@ -142,27 +405,14 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
try {
stored = await putBufferViaProvider(fallbackProvider, bucket, id, body, blob.type || undefined, config.storage);
} catch (fallbackError) {
if (throwOnFailure) throw fallbackError;
if (failClosed) throw fallbackError;
return content;
}
} else {
if (throwOnFailure) throw error;
if (failClosed) throw error;
return content;
}
}
if (scan.enqueuePostScan) {
try {
const { enqueueFileScan } = await import("./fileScanQueue.server.js");
await enqueueFileScan({
bucket,
key: id,
tenantId: scan.tenantId,
contentType: blob.type || scan.declaredMimeType,
});
} catch {
// never block a successful upload on a queue write
}
}
return stored;
}
@@ -174,13 +424,13 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
try {
return await putFileViaProvider(fallbackProvider, bucket, id, content, config.storage);
} catch (fallbackError) {
if (throwOnFailure) {
if (failClosed) {
throw fallbackError;
}
return content;
}
}
if (throwOnFailure) {
if (failClosed) {
throw error;
}
return content;
@@ -243,6 +493,27 @@ export function resolveStorageBucket(): string {
return bucket;
}
export function resolveStorageBinding(): { provider: RuntimeProvider; bucket: string; endpoint: string } {
const config = loadRequiredWebConfig();
const provider = resolvePrimaryStorageProvider(config.storage);
const bucket = resolveStorageBucket();
if (!provider) {
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
operation: "resolveStorageBinding",
bucket,
});
}
const endpoint = provider === "s3" ? config.storage.endpoint : config.storage.localRoot;
if (!endpoint) {
throw new StorageAdapterError("config_missing", "Storage endpoint is not configured.", {
operation: "resolveStorageBinding",
bucket,
missingFields: [provider === "s3" ? "endpoint" : "localRoot"],
});
}
return { provider, bucket, endpoint };
}
/**
* Assemble previously-stored temp chunk objects into a single object via
* multipart UploadPartCopy (§3). `chunkSourceKeys` MUST be ordered by chunk