Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+303
-32
@@ -3,31 +3,135 @@ import {
|
||||
createLocalStorageAdapter,
|
||||
createS3StorageAdapter,
|
||||
createMagicBytesScanAdapter,
|
||||
createClamAVBufferScanner,
|
||||
ClamAVUnreachableError,
|
||||
resolveAllowList,
|
||||
StorageAdapterError,
|
||||
type PreUploadScanAdapter,
|
||||
type PreUploadScanResult,
|
||||
type ScanContext,
|
||||
type StorageAdapter,
|
||||
type ClamAVBufferScanner,
|
||||
} from "@hivegrid/storage";
|
||||
import { fetchOutboundHttpWithRedirects } from "~/utils/outboundHttp.server";
|
||||
|
||||
type StorageConfig = ReturnType<typeof loadRequiredWebConfig>["storage"];
|
||||
type RuntimeProvider = "s3" | "local";
|
||||
|
||||
export const DEFAULT_UPLOAD_CONTENT_MAX_BYTES = 100 * 1024 * 1024;
|
||||
export const AVATAR_UPLOAD_CONTENT_MAX_BYTES = 5 * 1024 * 1024;
|
||||
export const DEFAULT_UPLOAD_BODY_TIMEOUT_MS = 10_000;
|
||||
|
||||
export class UploadContentReadError extends Error {
|
||||
readonly code: "too_large" | "timeout" | "upstream_failed";
|
||||
|
||||
constructor(code: "too_large" | "timeout" | "upstream_failed", message: string) {
|
||||
super(message);
|
||||
this.name = "UploadContentReadError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
type FetchUploadContentOptions = {
|
||||
maxBytes?: number;
|
||||
bodyTimeoutMs?: number;
|
||||
};
|
||||
|
||||
// Upload "content" can be caller-supplied. Only inline data: URIs are fetched
|
||||
// directly; any remote URL is routed through the SSRF-safe helper, which
|
||||
// rejects private/loopback/link-local/metadata addresses and validates the
|
||||
// scheme and every redirect hop. A bare fetch() here would let an authenticated
|
||||
// user reach cloud metadata or internal services (e.g. via "HTTP://169.254..."
|
||||
// which the prior case-sensitive "http" check missed).
|
||||
async function fetchUploadContent(content: string): Promise<Blob> {
|
||||
export async function fetchUploadContent(content: string, options: FetchUploadContentOptions = {}): Promise<Blob> {
|
||||
const trimmed = (content ?? "").trim();
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_UPLOAD_CONTENT_MAX_BYTES;
|
||||
const bodyTimeoutMs = options.bodyTimeoutMs ?? DEFAULT_UPLOAD_BODY_TIMEOUT_MS;
|
||||
assertValidReadLimits(maxBytes, bodyTimeoutMs);
|
||||
|
||||
let response: Response;
|
||||
if (trimmed.toLowerCase().startsWith("data:")) {
|
||||
return await (await fetch(trimmed)).blob();
|
||||
assertDataUriWithinLimit(trimmed, maxBytes);
|
||||
response = await fetch(trimmed);
|
||||
} else {
|
||||
response = await fetchOutboundHttpWithRedirects(trimmed, {}, { timeoutMs: bodyTimeoutMs });
|
||||
}
|
||||
const response = await fetchOutboundHttpWithRedirects(trimmed, {});
|
||||
return await response.blob();
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
throw new UploadContentReadError("upstream_failed", "Upload content could not be fetched.");
|
||||
}
|
||||
return readLimitedResponseBlob(response, { maxBytes, bodyTimeoutMs });
|
||||
}
|
||||
|
||||
async function readLimitedResponseBlob(response: Response, { maxBytes, bodyTimeoutMs }: Required<FetchUploadContentOptions>): Promise<Blob> {
|
||||
const contentLength = response.headers.get("content-length");
|
||||
if (contentLength && /^\d+$/.test(contentLength) && Number(contentLength) > maxBytes) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
|
||||
}
|
||||
if (!response.body) {
|
||||
return new Blob([], { type: response.headers.get("content-type") ?? "" });
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let bytesRead = 0;
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeoutPromise = new Promise<never>((_resolve, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
reject(new UploadContentReadError("timeout", "Upload content body timed out."));
|
||||
void reader.cancel().catch(() => undefined);
|
||||
}, bodyTimeoutMs);
|
||||
});
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const chunk = await Promise.race([reader.read(), timeoutPromise]);
|
||||
if (chunk.done) {
|
||||
break;
|
||||
}
|
||||
bytesRead += chunk.value.byteLength;
|
||||
if (bytesRead > maxBytes) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
|
||||
}
|
||||
chunks.push(chunk.value);
|
||||
}
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
const blobParts = chunks.map((chunk) => {
|
||||
const copy = new Uint8Array(chunk.byteLength);
|
||||
copy.set(chunk);
|
||||
return copy.buffer;
|
||||
});
|
||||
return new Blob(blobParts, { type: response.headers.get("content-type") ?? "" });
|
||||
}
|
||||
|
||||
function assertValidReadLimits(maxBytes: number, bodyTimeoutMs: number) {
|
||||
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || !Number.isSafeInteger(bodyTimeoutMs) || bodyTimeoutMs <= 0) {
|
||||
throw new TypeError("Upload read limits must be positive safe integers.");
|
||||
}
|
||||
}
|
||||
|
||||
function assertDataUriWithinLimit(dataUri: string, maxBytes: number) {
|
||||
const commaIndex = dataUri.indexOf(",");
|
||||
if (commaIndex < 0) {
|
||||
throw new UploadContentReadError("upstream_failed", "Upload data URI is malformed.");
|
||||
}
|
||||
const metadata = dataUri.slice(0, commaIndex).toLowerCase();
|
||||
const payloadLength = dataUri.length - commaIndex - 1;
|
||||
const maximumEncodedLength = metadata.includes(";base64") ? Math.ceil((maxBytes * 4) / 3) + 4 : maxBytes;
|
||||
if (payloadLength > maximumEncodedLength) {
|
||||
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
|
||||
}
|
||||
}
|
||||
|
||||
function uploadContentLimitForScan(scan: StoreFileInput["scan"]): number {
|
||||
return scan?.context === "avatar" ? AVATAR_UPLOAD_CONTENT_MAX_BYTES : DEFAULT_UPLOAD_CONTENT_MAX_BYTES;
|
||||
}
|
||||
|
||||
export class StorageScanRejectedError extends Error {
|
||||
@@ -45,6 +149,97 @@ export class StorageScanRejectedError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
type ClamAvRuntimeEnv = {
|
||||
CLAMD_HOST?: string;
|
||||
CLAMD_PORT?: string;
|
||||
CLAMD_TIMEOUT_MS?: string;
|
||||
CLAMAV_HOST?: string;
|
||||
CLAMAV_PORT?: string;
|
||||
CLAMAV_TIMEOUT_MS?: string;
|
||||
};
|
||||
|
||||
let cachedClamAvScanner: { key: string; scanner: ClamAVBufferScanner } | undefined;
|
||||
|
||||
function getClamAvBufferScanner(env: ClamAvRuntimeEnv = process.env): ClamAVBufferScanner {
|
||||
const host = env.CLAMD_HOST?.trim() || env.CLAMAV_HOST?.trim();
|
||||
if (!host) {
|
||||
throw new ClamAVUnreachableError("clamd host is not configured");
|
||||
}
|
||||
const port = parsePositiveInteger(env.CLAMD_PORT ?? env.CLAMAV_PORT, 3310, "clamd port");
|
||||
const timeoutMs = parsePositiveInteger(env.CLAMD_TIMEOUT_MS ?? env.CLAMAV_TIMEOUT_MS, 30_000, "clamd timeout");
|
||||
const key = `${host}:${port}:${timeoutMs}`;
|
||||
if (cachedClamAvScanner?.key !== key) {
|
||||
cachedClamAvScanner = {
|
||||
key,
|
||||
scanner: createClamAVBufferScanner({ host, port, timeoutMs }),
|
||||
};
|
||||
}
|
||||
return cachedClamAvScanner.scanner;
|
||||
}
|
||||
|
||||
function parsePositiveInteger(rawValue: string | undefined, fallback: number, label: string): number {
|
||||
if (rawValue === undefined || rawValue.trim() === "") {
|
||||
return fallback;
|
||||
}
|
||||
const value = Number(rawValue);
|
||||
if (!Number.isSafeInteger(value) || value <= 0) {
|
||||
throw new ClamAVUnreachableError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export async function requireCleanMalwareScan({
|
||||
body,
|
||||
bucket,
|
||||
key,
|
||||
filename,
|
||||
onReject,
|
||||
scanner,
|
||||
}: {
|
||||
body: Uint8Array;
|
||||
bucket: string;
|
||||
key: string;
|
||||
filename: string;
|
||||
onReject?: (verdict: PreUploadScanResult) => void | Promise<void>;
|
||||
scanner?: ClamAVBufferScanner;
|
||||
}): Promise<void> {
|
||||
let verdict: Awaited<ReturnType<ClamAVBufferScanner["scanBuffer"]>>;
|
||||
try {
|
||||
const buffer = Buffer.from(body.buffer, body.byteOffset, body.byteLength);
|
||||
verdict = await (scanner ?? getClamAvBufferScanner()).scanBuffer(buffer);
|
||||
} catch {
|
||||
const rejected: PreUploadScanResult = {
|
||||
status: "rejected",
|
||||
reason: "scanner_error",
|
||||
detail: "The malware scanner did not return a clean verdict.",
|
||||
};
|
||||
await persistRejectedScan(onReject, rejected);
|
||||
throw new StorageScanRejectedError(bucket, key, filename, rejected);
|
||||
}
|
||||
|
||||
if (verdict.verdict !== "clean") {
|
||||
const rejected: PreUploadScanResult = {
|
||||
status: "rejected",
|
||||
reason: "malware_detected",
|
||||
detail: "The malware scanner rejected the upload.",
|
||||
};
|
||||
await persistRejectedScan(onReject, rejected);
|
||||
throw new StorageScanRejectedError(bucket, key, filename, rejected);
|
||||
}
|
||||
}
|
||||
|
||||
async function persistRejectedScan(
|
||||
onReject: ((verdict: PreUploadScanResult) => void | Promise<void>) | undefined,
|
||||
verdict: PreUploadScanResult,
|
||||
) {
|
||||
if (!onReject) return;
|
||||
try {
|
||||
await onReject(verdict);
|
||||
} catch {
|
||||
// An audit-write failure must never mask or reverse a security verdict.
|
||||
}
|
||||
}
|
||||
|
||||
export type StoreFileInput = {
|
||||
bucket: string;
|
||||
content: string;
|
||||
@@ -71,14 +266,11 @@ export type StoreFileInput = {
|
||||
*/
|
||||
onReject?: (verdict: PreUploadScanResult) => void | Promise<void>;
|
||||
/**
|
||||
* When true AND the layer-1 scan passes AND the storage PUT succeeds,
|
||||
* storeFile enqueues a hivegrid.file-scan job so the worker runs ClamAV
|
||||
* (layer 2) over the stored object. Requires SCAN_QUEUE_ENABLED=true
|
||||
* on the runtime; otherwise the enqueue is a no-op. Failures to enqueue
|
||||
* are logged but never throw — the synchronous user-visible upload
|
||||
* should not block on a queue-write hiccup.
|
||||
* When true, ClamAV must return a clean verdict for the in-memory upload
|
||||
* before any storage PUT. Missing configuration, transport failures and
|
||||
* indeterminate verdicts all reject the upload.
|
||||
*/
|
||||
enqueuePostScan?: boolean;
|
||||
requireMalwareScan?: boolean;
|
||||
/**
|
||||
* Optional tenant attribution for the post-upload scan job (carried
|
||||
* through to the worker's audit row).
|
||||
@@ -87,14 +279,82 @@ export type StoreFileInput = {
|
||||
};
|
||||
};
|
||||
|
||||
export type StoreFileBytesInput = {
|
||||
bucket: string;
|
||||
body: Uint8Array;
|
||||
contentType?: string;
|
||||
id: string;
|
||||
scan?: StoreFileInput["scan"];
|
||||
};
|
||||
|
||||
const defaultScanAdapter: PreUploadScanAdapter = createMagicBytesScanAdapter({
|
||||
allowList: resolveAllowList(typeof process !== "undefined" ? (process.env as Record<string, string | undefined>) : {}),
|
||||
enabled: typeof process !== "undefined" ? process.env.STORAGE_SCAN_LAYER1_ENABLED !== "false" : true,
|
||||
});
|
||||
|
||||
export async function storeFileBytes({
|
||||
bucket,
|
||||
body,
|
||||
contentType,
|
||||
id,
|
||||
scan,
|
||||
}: StoreFileBytesInput): Promise<string> {
|
||||
const config = loadRequiredWebConfig();
|
||||
const primaryProvider = resolvePrimaryStorageProvider(config.storage);
|
||||
if (!primaryProvider) {
|
||||
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
|
||||
operation: "storeFileBytes",
|
||||
bucket,
|
||||
});
|
||||
}
|
||||
|
||||
if (scan) {
|
||||
const maxBytes = uploadContentLimitForScan(scan);
|
||||
if (body.byteLength > maxBytes) {
|
||||
throw new UploadContentReadError("too_large", `Upload content exceeds the ${maxBytes}-byte limit.`);
|
||||
}
|
||||
const declared = scan.declaredMimeType ?? contentType ?? "application/octet-stream";
|
||||
const filename = scan.filename ?? id;
|
||||
const verdict = await defaultScanAdapter.scan({
|
||||
buffer: body,
|
||||
declaredMimeType: declared,
|
||||
filename,
|
||||
context: scan.context,
|
||||
});
|
||||
if (verdict.status === "rejected") {
|
||||
await persistRejectedScan(scan.onReject, verdict);
|
||||
throw new StorageScanRejectedError(bucket, id, filename, verdict);
|
||||
}
|
||||
if (scan.requireMalwareScan) {
|
||||
await requireCleanMalwareScan({ body, bucket, key: id, filename, onReject: scan.onReject });
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return await putBufferViaProvider(primaryProvider, bucket, id, body, contentType, config.storage);
|
||||
} catch (error) {
|
||||
const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider);
|
||||
if (fallbackProvider) {
|
||||
try {
|
||||
return await putBufferViaProvider(fallbackProvider, bucket, id, body, contentType, config.storage);
|
||||
} catch (fallbackError) {
|
||||
throw fallbackError;
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function storeFile({ bucket, content, id, replace, throwOnFailure = false, scan }: StoreFileInput): Promise<string> {
|
||||
if (content.trim().toLowerCase().startsWith("http") && !replace) {
|
||||
if (throwOnFailure) {
|
||||
// A scan marks the bytes as untrusted input. Never fall back to persisting a
|
||||
// data URI or remote URL when storage is unavailable, even when a legacy
|
||||
// caller did not opt into the general throwOnFailure behavior.
|
||||
const failClosed = throwOnFailure || Boolean(scan);
|
||||
// The legacy no-op is safe only when no scan was requested. Treating an
|
||||
// arbitrary remote URL as an already-stored object must never bypass a
|
||||
// caller's magic-byte or malware gate.
|
||||
if (content.trim().toLowerCase().startsWith("http") && !replace && !scan) {
|
||||
if (failClosed) {
|
||||
throw new Error("Strict uploads require upload content, not an existing URL.");
|
||||
}
|
||||
return content;
|
||||
@@ -103,7 +363,7 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
|
||||
const config = loadRequiredWebConfig();
|
||||
const primaryProvider = resolvePrimaryStorageProvider(config.storage);
|
||||
if (!primaryProvider) {
|
||||
if (throwOnFailure) {
|
||||
if (failClosed) {
|
||||
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
|
||||
operation: "storeFile",
|
||||
bucket,
|
||||
@@ -113,7 +373,7 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
|
||||
}
|
||||
|
||||
if (scan) {
|
||||
const blob = await fetchUploadContent(content);
|
||||
const blob = await fetchUploadContent(content, { maxBytes: uploadContentLimitForScan(scan) });
|
||||
const body = new Uint8Array(await blob.arrayBuffer());
|
||||
const declared = scan.declaredMimeType ?? blob.type ?? "application/octet-stream";
|
||||
const filename = scan.filename ?? id;
|
||||
@@ -133,6 +393,9 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
|
||||
}
|
||||
throw new StorageScanRejectedError(bucket, id, filename, verdict);
|
||||
}
|
||||
if (scan.requireMalwareScan) {
|
||||
await requireCleanMalwareScan({ body, bucket, key: id, filename, onReject: scan.onReject });
|
||||
}
|
||||
let stored: string;
|
||||
try {
|
||||
stored = await putBufferViaProvider(primaryProvider, bucket, id, body, blob.type || undefined, config.storage);
|
||||
@@ -142,27 +405,14 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
|
||||
try {
|
||||
stored = await putBufferViaProvider(fallbackProvider, bucket, id, body, blob.type || undefined, config.storage);
|
||||
} catch (fallbackError) {
|
||||
if (throwOnFailure) throw fallbackError;
|
||||
if (failClosed) throw fallbackError;
|
||||
return content;
|
||||
}
|
||||
} else {
|
||||
if (throwOnFailure) throw error;
|
||||
if (failClosed) throw error;
|
||||
return content;
|
||||
}
|
||||
}
|
||||
if (scan.enqueuePostScan) {
|
||||
try {
|
||||
const { enqueueFileScan } = await import("./fileScanQueue.server.js");
|
||||
await enqueueFileScan({
|
||||
bucket,
|
||||
key: id,
|
||||
tenantId: scan.tenantId,
|
||||
contentType: blob.type || scan.declaredMimeType,
|
||||
});
|
||||
} catch {
|
||||
// never block a successful upload on a queue write
|
||||
}
|
||||
}
|
||||
return stored;
|
||||
}
|
||||
|
||||
@@ -174,13 +424,13 @@ export async function storeFile({ bucket, content, id, replace, throwOnFailure =
|
||||
try {
|
||||
return await putFileViaProvider(fallbackProvider, bucket, id, content, config.storage);
|
||||
} catch (fallbackError) {
|
||||
if (throwOnFailure) {
|
||||
if (failClosed) {
|
||||
throw fallbackError;
|
||||
}
|
||||
return content;
|
||||
}
|
||||
}
|
||||
if (throwOnFailure) {
|
||||
if (failClosed) {
|
||||
throw error;
|
||||
}
|
||||
return content;
|
||||
@@ -243,6 +493,27 @@ export function resolveStorageBucket(): string {
|
||||
return bucket;
|
||||
}
|
||||
|
||||
export function resolveStorageBinding(): { provider: RuntimeProvider; bucket: string; endpoint: string } {
|
||||
const config = loadRequiredWebConfig();
|
||||
const provider = resolvePrimaryStorageProvider(config.storage);
|
||||
const bucket = resolveStorageBucket();
|
||||
if (!provider) {
|
||||
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
|
||||
operation: "resolveStorageBinding",
|
||||
bucket,
|
||||
});
|
||||
}
|
||||
const endpoint = provider === "s3" ? config.storage.endpoint : config.storage.localRoot;
|
||||
if (!endpoint) {
|
||||
throw new StorageAdapterError("config_missing", "Storage endpoint is not configured.", {
|
||||
operation: "resolveStorageBinding",
|
||||
bucket,
|
||||
missingFields: [provider === "s3" ? "endpoint" : "localRoot"],
|
||||
});
|
||||
}
|
||||
return { provider, bucket, endpoint };
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble previously-stored temp chunk objects into a single object via
|
||||
* multipart UploadPartCopy (§3). `chunkSourceKeys` MUST be ordered by chunk
|
||||
|
||||
Reference in new issue
Block a user