Track HiveGrid Cloud planning reference
This commit is contained in:
david committed
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,375 @@
|
||||
import { loadRequiredWebConfig } from "@toolctrl/config";
|
||||
import {
|
||||
createLocalStorageAdapter,
|
||||
createS3StorageAdapter,
|
||||
createMagicBytesScanAdapter,
|
||||
resolveAllowList,
|
||||
StorageAdapterError,
|
||||
type PreUploadScanAdapter,
|
||||
type PreUploadScanResult,
|
||||
type ScanContext,
|
||||
type StorageAdapter,
|
||||
} from "@toolctrl/storage";
|
||||
import { fetchOutboundHttpWithRedirects } from "~/utils/outboundHttp.server";
|
||||
|
||||
type StorageConfig = ReturnType<typeof loadRequiredWebConfig>["storage"];
|
||||
type RuntimeProvider = "s3" | "local";
|
||||
|
||||
// Upload "content" can be caller-supplied. Only inline data: URIs are fetched
|
||||
// directly; any remote URL is routed through the SSRF-safe helper, which
|
||||
// rejects private/loopback/link-local/metadata addresses and validates the
|
||||
// scheme and every redirect hop. A bare fetch() here would let an authenticated
|
||||
// user reach cloud metadata or internal services (e.g. via "HTTP://169.254..."
|
||||
// which the prior case-sensitive "http" check missed).
|
||||
async function fetchUploadContent(content: string): Promise<Blob> {
|
||||
const trimmed = (content ?? "").trim();
|
||||
if (trimmed.toLowerCase().startsWith("data:")) {
|
||||
return await (await fetch(trimmed)).blob();
|
||||
}
|
||||
const response = await fetchOutboundHttpWithRedirects(trimmed, {});
|
||||
return await response.blob();
|
||||
}
|
||||
|
||||
export class StorageScanRejectedError extends Error {
|
||||
readonly bucket: string;
|
||||
readonly id: string;
|
||||
readonly filename: string;
|
||||
readonly scanResult: PreUploadScanResult;
|
||||
constructor(bucket: string, id: string, filename: string, scanResult: PreUploadScanResult) {
|
||||
super(`Upload rejected by pre-upload scanner: ${scanResult.reason ?? "unknown"} (${scanResult.detail ?? ""})`);
|
||||
this.name = "StorageScanRejectedError";
|
||||
this.bucket = bucket;
|
||||
this.id = id;
|
||||
this.filename = filename;
|
||||
this.scanResult = scanResult;
|
||||
}
|
||||
}
|
||||
|
||||
export type StoreFileInput = {
|
||||
bucket: string;
|
||||
content: string;
|
||||
id: string;
|
||||
replace?: boolean;
|
||||
throwOnFailure?: boolean;
|
||||
/**
|
||||
* When set, the upload buffer is run through MagicBytesScanAdapter before
|
||||
* the storage PUT. A rejected scan throws StorageScanRejectedError (always,
|
||||
* regardless of throwOnFailure) so callers cannot accidentally swallow a
|
||||
* security verdict. When unset the scan is skipped — that's only safe for
|
||||
* server-internal uploads where the bytes never came from a user.
|
||||
*/
|
||||
scan?: {
|
||||
context: ScanContext;
|
||||
declaredMimeType?: string;
|
||||
filename?: string;
|
||||
/**
|
||||
* Side-effect callback fired exactly once when the scanner returns a
|
||||
* rejected verdict, BEFORE the StorageScanRejectedError is thrown. The
|
||||
* callback may write an audit event or telemetry; storeFile awaits it so
|
||||
* the audit record is persisted before the route gets a chance to
|
||||
* swallow the error.
|
||||
*/
|
||||
onReject?: (verdict: PreUploadScanResult) => void | Promise<void>;
|
||||
/**
|
||||
* When true AND the layer-1 scan passes AND the storage PUT succeeds,
|
||||
* storeFile enqueues a toolctrl.file-scan job so the worker runs ClamAV
|
||||
* (layer 2) over the stored object. Requires SCAN_QUEUE_ENABLED=true
|
||||
* on the runtime; otherwise the enqueue is a no-op. Failures to enqueue
|
||||
* are logged but never throw — the synchronous user-visible upload
|
||||
* should not block on a queue-write hiccup.
|
||||
*/
|
||||
enqueuePostScan?: boolean;
|
||||
/**
|
||||
* Optional tenant attribution for the post-upload scan job (carried
|
||||
* through to the worker's audit row).
|
||||
*/
|
||||
tenantId?: string;
|
||||
};
|
||||
};
|
||||
|
||||
const defaultScanAdapter: PreUploadScanAdapter = createMagicBytesScanAdapter({
|
||||
allowList: resolveAllowList(typeof process !== "undefined" ? (process.env as Record<string, string | undefined>) : {}),
|
||||
enabled: typeof process !== "undefined" ? process.env.STORAGE_SCAN_LAYER1_ENABLED !== "false" : true,
|
||||
});
|
||||
|
||||
export async function storeFile({ bucket, content, id, replace, throwOnFailure = false, scan }: StoreFileInput): Promise<string> {
|
||||
if (content.trim().toLowerCase().startsWith("http") && !replace) {
|
||||
if (throwOnFailure) {
|
||||
throw new Error("Strict uploads require upload content, not an existing URL.");
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
const config = loadRequiredWebConfig();
|
||||
const primaryProvider = resolvePrimaryStorageProvider(config.storage);
|
||||
if (!primaryProvider) {
|
||||
if (throwOnFailure) {
|
||||
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
|
||||
operation: "storeFile",
|
||||
bucket,
|
||||
});
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
if (scan) {
|
||||
const blob = await fetchUploadContent(content);
|
||||
const body = new Uint8Array(await blob.arrayBuffer());
|
||||
const declared = scan.declaredMimeType ?? blob.type ?? "application/octet-stream";
|
||||
const filename = scan.filename ?? id;
|
||||
const verdict = await defaultScanAdapter.scan({
|
||||
buffer: body,
|
||||
declaredMimeType: declared,
|
||||
filename,
|
||||
context: scan.context,
|
||||
});
|
||||
if (verdict.status === "rejected") {
|
||||
if (scan.onReject) {
|
||||
try {
|
||||
await scan.onReject(verdict);
|
||||
} catch {
|
||||
// never let an audit-write failure mask the security verdict
|
||||
}
|
||||
}
|
||||
throw new StorageScanRejectedError(bucket, id, filename, verdict);
|
||||
}
|
||||
let stored: string;
|
||||
try {
|
||||
stored = await putBufferViaProvider(primaryProvider, bucket, id, body, blob.type || undefined, config.storage);
|
||||
} catch (error) {
|
||||
const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider);
|
||||
if (fallbackProvider) {
|
||||
try {
|
||||
stored = await putBufferViaProvider(fallbackProvider, bucket, id, body, blob.type || undefined, config.storage);
|
||||
} catch (fallbackError) {
|
||||
if (throwOnFailure) throw fallbackError;
|
||||
return content;
|
||||
}
|
||||
} else {
|
||||
if (throwOnFailure) throw error;
|
||||
return content;
|
||||
}
|
||||
}
|
||||
if (scan.enqueuePostScan) {
|
||||
try {
|
||||
const { enqueueFileScan } = await import("./fileScanQueue.server.js");
|
||||
await enqueueFileScan({
|
||||
bucket,
|
||||
key: id,
|
||||
tenantId: scan.tenantId,
|
||||
contentType: blob.type || scan.declaredMimeType,
|
||||
});
|
||||
} catch {
|
||||
// never block a successful upload on a queue write
|
||||
}
|
||||
}
|
||||
return stored;
|
||||
}
|
||||
|
||||
try {
|
||||
return await putFileViaProvider(primaryProvider, bucket, id, content, config.storage);
|
||||
} catch (error) {
|
||||
const fallbackProvider = resolveFallbackProvider(config.storage, primaryProvider);
|
||||
if (fallbackProvider) {
|
||||
try {
|
||||
return await putFileViaProvider(fallbackProvider, bucket, id, content, config.storage);
|
||||
} catch (fallbackError) {
|
||||
if (throwOnFailure) {
|
||||
throw fallbackError;
|
||||
}
|
||||
return content;
|
||||
}
|
||||
}
|
||||
if (throwOnFailure) {
|
||||
throw error;
|
||||
}
|
||||
return content;
|
||||
}
|
||||
}
|
||||
|
||||
export async function putBufferAsFile({
|
||||
bucket,
|
||||
key,
|
||||
body,
|
||||
contentType,
|
||||
}: {
|
||||
bucket: string;
|
||||
key: string;
|
||||
body: Uint8Array;
|
||||
contentType?: string;
|
||||
}): Promise<{ publicUrl: string | null }> {
|
||||
const config = loadRequiredWebConfig();
|
||||
const primaryProvider = resolvePrimaryStorageProvider(config.storage);
|
||||
if (!primaryProvider) {
|
||||
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
|
||||
operation: "putBufferAsFile",
|
||||
bucket,
|
||||
});
|
||||
}
|
||||
const adapter = createRuntimeStorageAdapter(primaryProvider, bucket, config.storage);
|
||||
const result = await adapter.putObject({ key, body, contentType });
|
||||
return { publicUrl: result.publicUrl ?? null };
|
||||
}
|
||||
|
||||
export async function deleteFile({ bucket, key }: { bucket: string; key: string }): Promise<void> {
|
||||
const config = loadRequiredWebConfig();
|
||||
const primaryProvider = resolvePrimaryStorageProvider(config.storage);
|
||||
if (!primaryProvider) {
|
||||
return;
|
||||
}
|
||||
const adapter = createRuntimeStorageAdapter(primaryProvider, bucket, config.storage);
|
||||
await adapter.deleteObject({ key });
|
||||
}
|
||||
|
||||
export function hasStorageCredentials(): boolean {
|
||||
const config = loadRequiredWebConfig();
|
||||
return resolvePrimaryStorageProvider(config.storage) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default files bucket (STORAGE_BUCKET). The Files module (AK-P4a) stores all
|
||||
* objects in this single private bucket; tenant isolation is carried by the
|
||||
* `tenants/{tenantId}/...` key prefix (§2) plus app-level scoping/RLS.
|
||||
*/
|
||||
export function resolveStorageBucket(): string {
|
||||
const config = loadRequiredWebConfig();
|
||||
const bucket = config.storage.bucket;
|
||||
if (!bucket || bucket.trim().length === 0) {
|
||||
throw new StorageAdapterError("config_missing", "STORAGE_BUCKET is not configured.", {
|
||||
operation: "resolveStorageBucket",
|
||||
missingFields: ["bucket"],
|
||||
});
|
||||
}
|
||||
return bucket;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble previously-stored temp chunk objects into a single object via
|
||||
* multipart UploadPartCopy (§3). `chunkSourceKeys` MUST be ordered by chunk
|
||||
* index. Aborts the multipart upload on any failure so no dangling parts remain.
|
||||
*/
|
||||
export async function assembleChunkedUpload({
|
||||
bucket,
|
||||
targetKey,
|
||||
chunkSourceKeys,
|
||||
contentType,
|
||||
}: {
|
||||
bucket: string;
|
||||
targetKey: string;
|
||||
chunkSourceKeys: readonly string[];
|
||||
contentType?: string;
|
||||
}): Promise<{ publicUrl: string | null }> {
|
||||
const config = loadRequiredWebConfig();
|
||||
const provider = resolvePrimaryStorageProvider(config.storage);
|
||||
if (!provider) {
|
||||
throw new StorageAdapterError("config_missing", "No storage provider configured.", {
|
||||
operation: "assembleChunkedUpload",
|
||||
bucket,
|
||||
});
|
||||
}
|
||||
const adapter = createRuntimeStorageAdapter(provider, bucket, config.storage);
|
||||
const handle = await adapter.createMultipartUpload({ key: targetKey, contentType });
|
||||
try {
|
||||
const parts = [];
|
||||
for (let index = 0; index < chunkSourceKeys.length; index += 1) {
|
||||
parts.push(
|
||||
await adapter.uploadPartCopy({
|
||||
key: targetKey,
|
||||
uploadId: handle.uploadId,
|
||||
partNumber: index + 1,
|
||||
sourceKey: chunkSourceKeys[index],
|
||||
}),
|
||||
);
|
||||
}
|
||||
const result = await adapter.completeMultipartUpload({ key: targetKey, uploadId: handle.uploadId, parts });
|
||||
return { publicUrl: result.publicUrl ?? null };
|
||||
} catch (error) {
|
||||
await adapter.abortMultipartUpload({ key: targetKey, uploadId: handle.uploadId }).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function putFileViaProvider(provider: RuntimeProvider, bucket: string, id: string, content: string, storage: StorageConfig): Promise<string> {
|
||||
const blob = await fetchUploadContent(content);
|
||||
const body = new Uint8Array(await blob.arrayBuffer());
|
||||
const contentType = blob.type || undefined;
|
||||
return putBufferViaProvider(provider, bucket, id, body, contentType, storage);
|
||||
}
|
||||
|
||||
async function putBufferViaProvider(
|
||||
provider: RuntimeProvider,
|
||||
bucket: string,
|
||||
id: string,
|
||||
body: Uint8Array,
|
||||
contentType: string | undefined,
|
||||
storage: StorageConfig
|
||||
): Promise<string> {
|
||||
const adapter = createRuntimeStorageAdapter(provider, bucket, storage);
|
||||
const result = await adapter.putObject({ key: id, body, contentType });
|
||||
return result.publicUrl ?? "";
|
||||
}
|
||||
|
||||
function resolvePrimaryStorageProvider(storage: StorageConfig): RuntimeProvider | undefined {
|
||||
if (storage.provider === "s3" || storage.provider === "local") {
|
||||
return storage.provider;
|
||||
}
|
||||
if (hasCompleteS3StorageConfig(storage)) {
|
||||
return "s3";
|
||||
}
|
||||
if (storage.localRoot) {
|
||||
return "local";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveFallbackProvider(storage: StorageConfig, primary: RuntimeProvider): RuntimeProvider | undefined {
|
||||
const fallback = storage.fallbackProvider;
|
||||
if ((fallback === "s3" || fallback === "local") && fallback !== primary) {
|
||||
return fallback;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function createRuntimeStorageAdapter(provider: RuntimeProvider, bucket: string, storage: StorageConfig): StorageAdapter {
|
||||
if (provider === "s3") {
|
||||
if (!hasCompleteS3StorageConfig(storage)) {
|
||||
throw new StorageAdapterError("config_missing", "S3 storage configuration is incomplete.", {
|
||||
operation: "createRuntimeStorageAdapter",
|
||||
bucket,
|
||||
missingFields: ["endpoint", "region", "accessKeyId", "secretAccessKey"].filter((field) => !storage[field as keyof typeof storage]),
|
||||
});
|
||||
}
|
||||
return createS3StorageAdapter({
|
||||
endpoint: storage.endpoint,
|
||||
region: storage.region,
|
||||
bucket,
|
||||
accessKeyId: storage.accessKeyId,
|
||||
secretAccessKey: storage.secretAccessKey,
|
||||
sessionToken: storage.sessionToken,
|
||||
publicBaseUrl: storage.publicBaseUrl,
|
||||
forcePathStyle: storage.forcePathStyle === undefined ? undefined : storage.forcePathStyle.toLowerCase() === "true",
|
||||
});
|
||||
}
|
||||
|
||||
if (!storage.localRoot) {
|
||||
throw new StorageAdapterError("config_missing", "Local storage root is not configured.", {
|
||||
operation: "createRuntimeStorageAdapter",
|
||||
bucket,
|
||||
missingFields: ["localRoot"],
|
||||
});
|
||||
}
|
||||
return createLocalStorageAdapter({
|
||||
rootDir: storage.localRoot,
|
||||
bucket,
|
||||
publicBaseUrl: storage.localPublicBaseUrl,
|
||||
});
|
||||
}
|
||||
|
||||
function hasCompleteS3StorageConfig(storage: StorageConfig): storage is StorageConfig & {
|
||||
endpoint: string;
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
} {
|
||||
return Boolean(storage.endpoint && storage.region && storage.accessKeyId && storage.secretAccessKey);
|
||||
}
|
||||
Reference in new issue
Block a user