Reorganize repo structure and standards references

This commit is contained in:
david committed 2026-06-29 05:03:42 +02:00
1 parent 3cb93a81be
commit be2054274d
5885 files changed
+2588 -195

No files matched your search

@@ -0,0 +1,36 @@
import { describe, expect, it } from "vitest";
import { DEFAULT_ALLOW_LIST, resolveAllowList } from "./allow-list.js";
describe("resolveAllowList", () => {
it("returns restrictive defaults when no environment overrides are supplied", () => {
expect(resolveAllowList()).toEqual(DEFAULT_ALLOW_LIST);
});
it("replaces a context default with the operator supplied env allow-list", () => {
const allowList = resolveAllowList({
STORAGE_SCAN_ALLOWLIST_EMAIL_ATTACHMENT: "application/pdf,text/calendar",
});
expect(allowList.emailAttachment).toEqual(["application/pdf", "text/calendar"]);
expect(allowList.avatar).toEqual(DEFAULT_ALLOW_LIST.avatar);
expect(allowList.mediaLibrary).toEqual(DEFAULT_ALLOW_LIST.mediaLibrary);
});
it("normalizes override values before they are used by upload validators", () => {
const allowList = resolveAllowList({
STORAGE_SCAN_ALLOWLIST_AVATAR: " Image/PNG, IMAGE/WEBP ,, image/jpeg ",
});
expect(allowList.avatar).toEqual(["image/png", "image/webp", "image/jpeg"]);
});
it("allows an explicit empty override to deny every mime type for a context", () => {
const allowList = resolveAllowList({
STORAGE_SCAN_ALLOWLIST_FORM_UPLOAD: " ",
});
expect(allowList.formUpload).toEqual([]);
expect(allowList.kbAsset).toEqual(DEFAULT_ALLOW_LIST.kbAsset);
});
});
@@ -0,0 +1,54 @@
import type { PreUploadAllowList, ScanContext } from "./pre-upload-types.js";
const IMAGE_TYPES = ["image/png", "image/jpeg", "image/webp"] as const;
const IMAGE_TYPES_WITH_GIF = [...IMAGE_TYPES, "image/gif"] as const;
const OFFICE_TYPES = [
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
] as const;
/**
* Restrictive defaults per upload context. Operator can extend per-context via the
* STORAGE_SCAN_ALLOWLIST_<CONTEXT> env var (parsed as a comma-separated list and
* REPLACES the default — not merged, so the operator's list is authoritative).
*/
export const DEFAULT_ALLOW_LIST: PreUploadAllowList = {
avatar: IMAGE_TYPES,
mediaLibrary: [...IMAGE_TYPES_WITH_GIF, "image/svg+xml", "image/x-icon", "image/vnd.microsoft.icon"],
emailAttachment: [...IMAGE_TYPES_WITH_GIF, "application/pdf", "text/plain", "text/csv", "application/zip", ...OFFICE_TYPES],
formUpload: [...IMAGE_TYPES, "application/pdf"],
kbAsset: [...IMAGE_TYPES, "application/pdf"],
};
const ENV_PREFIX = "STORAGE_SCAN_ALLOWLIST_";
const ENV_KEYS: Record<ScanContext, string> = {
avatar: `${ENV_PREFIX}AVATAR`,
mediaLibrary: `${ENV_PREFIX}MEDIA_LIBRARY`,
emailAttachment: `${ENV_PREFIX}EMAIL_ATTACHMENT`,
formUpload: `${ENV_PREFIX}FORM_UPLOAD`,
kbAsset: `${ENV_PREFIX}KB_ASSET`,
};
/**
* Build the effective allow-list by merging the static defaults with any
* `STORAGE_SCAN_ALLOWLIST_<CONTEXT>` overrides in the supplied env. If the env
* var is set (even to an empty string), it REPLACES the default for that
* context. Unset env vars leave the default untouched.
*
* Caller responsibility: pass the runtime env explicitly so the function stays
* pure and testable. Don't read process.env inside this module.
*/
export function resolveAllowList(env: Record<string, string | undefined> = {}): PreUploadAllowList {
const next: Record<ScanContext, readonly string[]> = { ...DEFAULT_ALLOW_LIST };
for (const context of Object.keys(ENV_KEYS) as ScanContext[]) {
const raw = env[ENV_KEYS[context]];
if (raw === undefined) continue;
next[context] = raw
.split(",")
.map((value) => value.trim().toLowerCase())
.filter((value) => value.length > 0);
}
return next;
}
@@ -0,0 +1,128 @@
// @vitest-environment node
import { Buffer } from "node:buffer";
import { describe, expect, test, vi } from "vitest";
import { ClamAVUnreachableError, createClamAVStorageScanner, isClamAVUnreachableError, type ClamScanClient } from "./clamav-scanner.js";
const EICAR_BUFFER = Buffer.from("X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*", "utf8");
const CLEAN_BUFFER = Buffer.from("hello world", "utf8");
function buildClient(overrides: Partial<ClamScanClient> = {}): ClamScanClient {
return {
isReachable: vi.fn(async () => true),
scanBuffer: vi.fn(async () => ({ isInfected: false })),
scanStream: vi.fn(async () => ({ isInfected: false })),
...overrides,
};
}
const baseConfig = {
host: "clamav",
port: 3310,
timeoutMs: 30000,
};
describe("ClamAVStorageScanner", () => {
test("returns 'clean' when clamd reports no infection", async () => {
const client = buildClient();
const scanner = createClamAVStorageScanner({
...baseConfig,
loadObject: async () => CLEAN_BUFFER,
initClient: async () => client,
});
const result = await scanner.scanObject({ bucket: "uploads", key: "user/1/avatar.png" });
expect(result.verdict).toBe("clean");
expect((result as { engine?: string }).engine).toBe("clamav");
expect(client.scanBuffer).toHaveBeenCalledWith(CLEAN_BUFFER);
});
test("returns 'infected' with the first signature when clamd hits", async () => {
const client = buildClient({
scanBuffer: vi.fn(async () => ({ isInfected: true, viruses: ["Win.Test.EICAR_HDB-1", "extra-signature"] })),
});
const scanner = createClamAVStorageScanner({
...baseConfig,
loadObject: async () => EICAR_BUFFER,
initClient: async () => client,
});
const result = await scanner.scanObject({ bucket: "uploads", key: "user/1/eicar.txt" });
expect(result.verdict).toBe("infected");
if (result.verdict === "infected") {
expect(result.signature).toBe("Win.Test.EICAR_HDB-1");
expect(result.engine).toBe("clamav");
}
});
test("falls back to signature 'unknown' when clamd reports infected with no viruses array", async () => {
const client = buildClient({ scanBuffer: vi.fn(async () => ({ isInfected: true })) });
const scanner = createClamAVStorageScanner({
...baseConfig,
loadObject: async () => EICAR_BUFFER,
initClient: async () => client,
});
const result = await scanner.scanObject({ bucket: "uploads", key: "x" });
expect(result.verdict).toBe("infected");
if (result.verdict === "infected") expect(result.signature).toBe("unknown");
});
test("throws ClamAVUnreachableError when isReachable returns false", async () => {
const client = buildClient({ isReachable: vi.fn(async () => false) });
const scanner = createClamAVStorageScanner({
...baseConfig,
loadObject: async () => CLEAN_BUFFER,
initClient: async () => client,
});
await expect(scanner.scanObject({ bucket: "uploads", key: "x" })).rejects.toThrow(ClamAVUnreachableError);
expect(client.scanBuffer).not.toHaveBeenCalled();
});
test("treats isReachable rejection as unreachable instead of leaking the error", async () => {
const client = buildClient({
isReachable: vi.fn(async () => {
throw new Error("ENOTFOUND");
}),
});
const scanner = createClamAVStorageScanner({
...baseConfig,
loadObject: async () => CLEAN_BUFFER,
initClient: async () => client,
});
await expect(scanner.scanObject({ bucket: "uploads", key: "x" })).rejects.toSatisfy(isClamAVUnreachableError);
});
test("scans a Readable stream when loadObject returns one", async () => {
const client = buildClient({
scanStream: vi.fn(async () => ({ isInfected: false })),
});
const { Readable } = await import("node:stream");
const stream = Readable.from([Buffer.from("payload")]);
const scanner = createClamAVStorageScanner({
...baseConfig,
loadObject: async () => stream,
initClient: async () => client,
});
const result = await scanner.scanObject({ bucket: "uploads", key: "x" });
expect(result.verdict).toBe("clean");
expect(client.scanStream).toHaveBeenCalledWith(stream);
expect(client.scanBuffer).not.toHaveBeenCalled();
});
test("caches the client between scans (single init)", async () => {
const client = buildClient();
const initClient = vi.fn(async () => client);
const scanner = createClamAVStorageScanner({
...baseConfig,
loadObject: async () => CLEAN_BUFFER,
initClient,
});
await scanner.scanObject({ bucket: "uploads", key: "a" });
await scanner.scanObject({ bucket: "uploads", key: "b" });
expect(initClient).toHaveBeenCalledTimes(1);
});
test("isClamAVUnreachableError narrows arbitrary errors", () => {
expect(isClamAVUnreachableError(new ClamAVUnreachableError("nope"))).toBe(true);
expect(isClamAVUnreachableError(new Error("regular"))).toBe(false);
expect(isClamAVUnreachableError("string")).toBe(false);
expect(isClamAVUnreachableError(null)).toBe(false);
});
});
@@ -0,0 +1,144 @@
// ClamAV-backed implementation of the StorageScanner contract from
// ./index.ts. Talks to a clamd daemon via the TCP clamscan protocol
// (default port 3310). Used by the worker runtime's scan-file job handler.
/// <reference path="./clamscan.d.ts" />
import type { StorageScanner, StorageScannerInput, StorageScannerResult } from "./index.js";
export interface ClamAVScannerConfig {
host: string;
port: number;
/**
* Connect / scan timeout in ms. clamd can take several seconds on the very
* first request after boot while it warms up; 30 s is the operational
* default.
*/
timeoutMs: number;
/**
* Loader for the upload payload. Returning a Buffer or Readable is fine;
* clamscan accepts both via scanStream/scanBuffer. The scanner doesn't read
* the object itself so callers (typically the worker job handler) can plug
* in any storage backend.
*/
loadObject: (input: StorageScannerInput) => Promise<Buffer | NodeJS.ReadableStream>;
/**
* Override for the clamscan client constructor. Used by tests; production
* leaves it unset so the real npm package is loaded lazily.
*/
initClient?: (options: { host: string; port: number; timeoutMs: number }) => Promise<ClamScanClient>;
/**
* Override for the wall clock. Tests inject a stub to make scannedAt
* deterministic.
*/
now?: () => Date;
}
export interface ClamScanResult {
isInfected: boolean | null;
viruses?: readonly string[];
}
export interface ClamScanClient {
/** Returns true when clamd answers a PING within the configured timeout. */
isReachable(): Promise<boolean>;
/** Scan a single in-memory buffer. Throws on transport errors. */
scanBuffer(buffer: Buffer): Promise<ClamScanResult>;
/** Scan a stream end-to-end. */
scanStream(stream: NodeJS.ReadableStream): Promise<ClamScanResult>;
}
const ENGINE = "clamav";
export function createClamAVStorageScanner(config: ClamAVScannerConfig): StorageScanner {
const initClient = config.initClient ?? defaultInitClient;
let cachedClient: ClamScanClient | undefined;
async function getClient(): Promise<ClamScanClient> {
if (!cachedClient) {
cachedClient = await initClient({ host: config.host, port: config.port, timeoutMs: config.timeoutMs });
}
return cachedClient;
}
return {
async scanObject(target: StorageScannerInput): Promise<StorageScannerResult> {
const client = await getClient();
const reachable = await client.isReachable().catch(() => false);
if (!reachable) {
throw new ClamAVUnreachableError(`clamd at ${config.host}:${config.port} is unreachable`);
}
const payload = await config.loadObject(target);
const result = await scanWithClient(client, payload);
if (result.isInfected) {
const signature = result.viruses?.[0] ?? "unknown";
return { verdict: "infected", signature, engine: ENGINE };
}
return { verdict: "clean", engine: ENGINE };
},
};
}
async function scanWithClient(client: ClamScanClient, payload: Buffer | NodeJS.ReadableStream): Promise<ClamScanResult> {
if (Buffer.isBuffer(payload)) {
return client.scanBuffer(payload);
}
return client.scanStream(payload);
}
async function defaultInitClient({ host, port, timeoutMs }: { host: string; port: number; timeoutMs: number }): Promise<ClamScanClient> {
// Lazy import so non-worker bundles (the web app, the API) don't try to
// resolve the native clamscan dependency. The clamscan package exports a
// factory whose init() returns the client. See clamscan.d.ts in this
// directory for the (intentionally narrow) module ambient typing.
const mod = await import("clamscan");
const Factory = (mod as { default?: unknown }).default ?? mod;
const factory = typeof Factory === "function" ? new (Factory as new () => { init: (opts: unknown) => Promise<unknown> })() : (Factory as { init: (opts: unknown) => Promise<unknown> });
const client = (await factory.init({
removeInfected: false,
debugMode: false,
clamdscan: {
host,
port,
timeout: timeoutMs,
localFallback: false,
},
preference: "clamdscan",
})) as ClamScanLib;
return {
async isReachable() {
const status = await client.ping();
return Boolean(status);
},
async scanBuffer(buffer) {
const res = await client.scanBuffer(buffer);
return { isInfected: res.isInfected, viruses: res.viruses };
},
async scanStream(stream) {
const res = await client.scanStream(stream);
return { isInfected: res.isInfected, viruses: res.viruses };
},
};
}
// Subset of the clamscan npm package surface that we actually use. Avoids a
// dependency on @types/clamscan and lets the lazy import stay loose.
interface ClamScanLib {
ping(): Promise<unknown>;
scanBuffer(buffer: Buffer): Promise<{ isInfected: boolean | null; viruses?: string[] }>;
scanStream(stream: NodeJS.ReadableStream): Promise<{ isInfected: boolean | null; viruses?: string[] }>;
}
export class ClamAVUnreachableError extends Error {
constructor(message: string) {
super(message);
this.name = "ClamAVUnreachableError";
}
}
export function isClamAVUnreachableError(error: unknown): error is ClamAVUnreachableError {
return error instanceof Error && error.name === "ClamAVUnreachableError";
}
@@ -0,0 +1,27 @@
// Narrow ambient typing for the `clamscan` npm package, which ships no
// .d.ts of its own. We only touch ping/scanBuffer/scanStream; anything else
// stays untyped on purpose.
declare module "clamscan" {
export interface ClamScanInitOptions {
removeInfected?: boolean;
debugMode?: boolean;
clamdscan?: {
host?: string;
port?: number;
timeout?: number;
localFallback?: boolean;
};
preference?: "clamdscan" | "clamscan";
}
export interface ClamScanInstance {
ping(): Promise<unknown>;
scanBuffer(buffer: Buffer): Promise<{ isInfected: boolean | null; viruses?: string[] }>;
scanStream(stream: NodeJS.ReadableStream): Promise<{ isInfected: boolean | null; viruses?: string[] }>;
}
export default class NodeClam {
init(options: ClamScanInitOptions): Promise<ClamScanInstance>;
}
}
@@ -0,0 +1,194 @@
import { describe, expect, it, vi } from "vitest";
import {
createStorageScanService,
type StorageQuarantineAdapter,
type StorageScanner,
} from "./index.js";
const fixedNow = new Date("2026-02-03T04:05:06.000Z");
function createService(overrides: {
scanner?: StorageScanner;
quarantine?: StorageQuarantineAdapter;
} = {}) {
return createStorageScanService({
scanner: overrides.scanner ?? {
scanObject: vi.fn().mockResolvedValue({ verdict: "clean", engine: "clamav-1.4" }),
},
quarantine: overrides.quarantine,
now: () => fixedNow,
});
}
describe("createStorageScanService", () => {
it("returns clean scan verdicts with scannedAt and invokes the scanner once", async () => {
const scanObject = vi.fn().mockResolvedValue({ verdict: "clean", engine: "clamav-1.4" });
const service = createService({ scanner: { scanObject } });
await expect(
service.scanObject({ bucket: "uploads", key: "avatars/user.png" }),
).resolves.toEqual({
verdict: "clean",
bucket: "uploads",
key: "avatars/user.png",
scannedAt: fixedNow,
engine: "clamav-1.4",
});
expect(scanObject).toHaveBeenCalledOnce();
expect(scanObject).toHaveBeenCalledWith({ bucket: "uploads", key: "avatars/user.png" });
});
it("normalizes infected scanner verdicts with signature and quarantine metadata", async () => {
const scanObject = vi.fn().mockResolvedValue({
verdict: "infected",
engine: "clamav-1.4",
signature: "Eicar-Test-Signature",
quarantineKey: "quarantine/uploads/avatar.png",
});
const quarantineObject = vi.fn().mockResolvedValue({
bucket: "quarantine",
key: "uploads/avatar.png",
quarantineKey: "quarantine/uploads/avatar.png",
action: "moved",
});
const service = createService({
scanner: { scanObject },
quarantine: { quarantineObject },
});
await expect(
service.scanObject({ bucket: "uploads", key: "avatar.png" }),
).resolves.toEqual({
verdict: "infected",
bucket: "uploads",
key: "avatar.png",
scannedAt: fixedNow,
engine: "clamav-1.4",
signature: "Eicar-Test-Signature",
quarantineKey: "quarantine/uploads/avatar.png",
quarantine: {
bucket: "quarantine",
key: "uploads/avatar.png",
quarantineKey: "quarantine/uploads/avatar.png",
action: "moved",
},
});
expect(scanObject).toHaveBeenCalledOnce();
expect(quarantineObject).toHaveBeenCalledWith({
bucket: "uploads",
key: "avatar.png",
quarantineKey: "quarantine/uploads/avatar.png",
signature: "Eicar-Test-Signature",
});
});
it("returns redacted scanner error verdicts without copying arbitrary thrown fields", async () => {
const scannerError = Object.assign(
new Error("scanner failed credential=secret-token body=malware password=hunter2"),
{
name: "ClamAvUnavailable",
accessKeyId: "access-key",
secretAccessKey: "secret-key",
body: "file-body-secret",
$metadata: { httpStatusCode: 503, requestId: "scan-req-1" },
},
);
const service = createService({
scanner: { scanObject: vi.fn().mockRejectedValue(scannerError) },
});
const result = await service.scanObject({ bucket: "uploads", key: "avatar.png" });
expect(result).toEqual({
verdict: "error",
bucket: "uploads",
key: "avatar.png",
scannedAt: fixedNow,
error: {
code: "scanner_failed",
message: "Storage scanner failed while scanning object.",
upstream: {
name: "ClamAvUnavailable",
message: "scanner failed credential=[redacted] body=[redacted] password=[redacted]",
status: 503,
requestId: "scan-req-1",
},
},
});
const serialized = JSON.stringify(result);
expect(serialized).not.toContain("secret-token");
expect(serialized).not.toContain("file-body-secret");
expect(serialized).not.toContain("secret-key");
expect(serialized).not.toContain("hunter2");
});
it("rejects invalid empty bucket or key before invoking the scanner", async () => {
const scanObject = vi.fn();
const service = createService({ scanner: { scanObject } });
await expect(service.scanObject({ bucket: "", key: "avatar.png" })).resolves.toMatchObject({
verdict: "error",
key: "avatar.png",
error: {
code: "invalid_target",
message: "Storage scan target bucket and key must be non-empty strings.",
},
});
await expect(service.scanObject({ bucket: "uploads", key: " " })).resolves.toMatchObject({
verdict: "error",
bucket: "uploads",
error: { code: "invalid_target" },
});
expect(scanObject).not.toHaveBeenCalled();
});
it("returns redacted quarantine failure diagnostics for infected objects", async () => {
const quarantineError = Object.assign(
new Error("copy failed authorization=BearerSecret body=infected-bytes"),
{
name: "QuarantineWriteFailed",
token: "BearerSecret",
responseBody: "infected-bytes",
$metadata: { httpStatusCode: 500, requestId: "quarantine-req-1" },
},
);
const service = createService({
scanner: {
scanObject: vi.fn().mockResolvedValue({
verdict: "infected",
signature: "Eicar-Test-Signature",
quarantineKey: "quarantine/avatar.png",
}),
},
quarantine: { quarantineObject: vi.fn().mockRejectedValue(quarantineError) },
});
const result = await service.scanObject({ bucket: "uploads", key: "avatar.png" });
expect(result).toEqual({
verdict: "error",
bucket: "uploads",
key: "avatar.png",
scannedAt: fixedNow,
error: {
code: "quarantine_failed",
message: "Storage quarantine failed for infected object.",
upstream: {
name: "QuarantineWriteFailed",
message: "copy failed authorization=[redacted] body=[redacted]",
status: 500,
requestId: "quarantine-req-1",
},
},
});
const serialized = JSON.stringify(result);
expect(serialized).not.toContain("BearerSecret");
expect(serialized).not.toContain("infected-bytes");
expect(serialized).not.toContain("responseBody");
});
});
@@ -0,0 +1,280 @@
export * from "./pre-upload-types.js";
export { createMagicBytesScanAdapter, type MagicBytesScanAdapterOptions } from "./magic-bytes-adapter.js";
export { DEFAULT_ALLOW_LIST, resolveAllowList } from "./allow-list.js";
export {
createClamAVStorageScanner,
ClamAVUnreachableError,
isClamAVUnreachableError,
type ClamAVScannerConfig,
type ClamScanClient,
type ClamScanResult,
} from "./clamav-scanner.js";
export { createS3StorageQuarantineAdapter, type S3QuarantineConfig } from "./s3-quarantine.js";
export type ScanVerdict = "clean" | "infected" | "error";
export interface StorageScanTarget {
bucket: string;
key: string;
requestId?: string;
metadata?: Record<string, unknown>;
}
export interface StorageScannerInput {
bucket: string;
key: string;
}
export type StorageScannerResult =
| {
verdict: "clean";
engine?: string;
}
| {
verdict: "infected";
signature: string;
engine?: string;
quarantineKey?: string;
};
export interface StorageScanner {
scanObject(target: StorageScannerInput): Promise<StorageScannerResult> | StorageScannerResult;
}
export interface StorageQuarantineInput {
bucket: string;
key: string;
quarantineKey?: string;
signature: string;
}
export interface StorageQuarantineResult {
bucket?: string;
key?: string;
quarantineKey?: string;
action?: "copied" | "moved" | "tagged" | "skipped";
}
export interface StorageQuarantineAdapter {
quarantineObject(input: StorageQuarantineInput): Promise<StorageQuarantineResult> | StorageQuarantineResult;
}
export interface StorageScanDiagnostic {
code: StorageScanErrorCode;
message: string;
upstream?: {
name?: string;
message?: string;
status?: number;
requestId?: string;
};
}
export type StorageScanErrorCode =
| "invalid_target"
| "scanner_failed"
| "quarantine_failed";
export type StorageScanResult =
| {
verdict: "clean";
bucket: string;
key: string;
scannedAt: Date;
engine?: string;
}
| {
verdict: "infected";
bucket: string;
key: string;
scannedAt: Date;
engine?: string;
signature: string;
quarantineKey?: string;
quarantine?: StorageQuarantineResult;
}
| {
verdict: "error";
bucket?: string;
key?: string;
scannedAt: Date;
error: StorageScanDiagnostic;
};
export interface StorageScanService {
scanObject(target: StorageScanTarget): Promise<StorageScanResult>;
}
export interface CreateStorageScanServiceInput {
scanner: StorageScanner;
quarantine?: StorageQuarantineAdapter;
now?: () => Date;
}
export class StorageScanError extends Error {
readonly code: StorageScanErrorCode;
readonly details: StorageScanDiagnostic;
override readonly cause?: unknown;
constructor(
code: StorageScanErrorCode,
message: string,
details: StorageScanDiagnostic,
options: { cause?: unknown } = {},
) {
super(message);
this.name = "StorageScanError";
this.code = code;
this.details = details;
this.cause = options.cause;
}
}
export function createStorageScanService({
scanner,
quarantine,
now = () => new Date(),
}: CreateStorageScanServiceInput): StorageScanService {
return {
async scanObject(target) {
const scannedAt = now();
const normalizedTarget = normalizeTarget(target);
if (!normalizedTarget) {
return {
verdict: "error",
bucket: isPresent(target.bucket) ? target.bucket : undefined,
key: isPresent(target.key) ? target.key : undefined,
scannedAt,
error: createDiagnostic(
"invalid_target",
"Storage scan target bucket and key must be non-empty strings.",
),
};
}
let scanResult: StorageScannerResult;
try {
scanResult = await scanner.scanObject(normalizedTarget);
} catch (cause) {
return {
verdict: "error",
bucket: normalizedTarget.bucket,
key: normalizedTarget.key,
scannedAt,
error: createDiagnostic(
"scanner_failed",
"Storage scanner failed while scanning object.",
cause,
),
};
}
if (scanResult.verdict === "clean") {
return {
verdict: "clean",
bucket: normalizedTarget.bucket,
key: normalizedTarget.key,
scannedAt,
engine: scanResult.engine,
};
}
const quarantineKey = scanResult.quarantineKey;
let quarantineResult: StorageQuarantineResult | undefined;
if (quarantine) {
try {
quarantineResult = await quarantine.quarantineObject({
bucket: normalizedTarget.bucket,
key: normalizedTarget.key,
quarantineKey,
signature: scanResult.signature,
});
} catch (cause) {
return {
verdict: "error",
bucket: normalizedTarget.bucket,
key: normalizedTarget.key,
scannedAt,
error: createDiagnostic(
"quarantine_failed",
"Storage quarantine failed for infected object.",
cause,
),
};
}
}
return {
verdict: "infected",
bucket: normalizedTarget.bucket,
key: normalizedTarget.key,
scannedAt,
engine: scanResult.engine,
signature: scanResult.signature,
quarantineKey: quarantineResult?.quarantineKey ?? quarantineKey,
quarantine: quarantineResult,
};
},
};
}
function normalizeTarget(target: StorageScanTarget): StorageScannerInput | null {
if (!isPresent(target.bucket) || !isPresent(target.key)) {
return null;
}
return {
bucket: target.bucket.trim(),
key: target.key.trim(),
};
}
function createDiagnostic(
code: StorageScanErrorCode,
message: string,
cause?: unknown,
): StorageScanDiagnostic {
const upstream = extractRedactedUpstreamDetails(cause);
return {
code,
message,
...(upstream ? { upstream } : {}),
};
}
function extractRedactedUpstreamDetails(
cause: unknown,
): StorageScanDiagnostic["upstream"] {
if (!isRecord(cause)) {
return undefined;
}
const metadata = isRecord(cause.$metadata) ? cause.$metadata : undefined;
const status = typeof metadata?.httpStatusCode === "number" ? metadata.httpStatusCode : undefined;
const requestId = typeof metadata?.requestId === "string" ? metadata.requestId : undefined;
const message = typeof cause.message === "string" ? redactSecretLikeText(cause.message) : undefined;
return {
name: typeof cause.name === "string" ? cause.name : undefined,
message,
status,
requestId,
};
}
function redactSecretLikeText(message: string): string {
return message
.replace(/(access[_-]?key(?:id)?|secret[_-]?access[_-]?key|session[_-]?token|credential|authorization|token|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
.replace(/(body)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]");
}
function isPresent(value: unknown): value is string {
return typeof value === "string" && value.trim().length > 0;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
@@ -0,0 +1,131 @@
import { fileTypeFromBuffer } from "file-type";
import { DEFAULT_ALLOW_LIST } from "./allow-list.js";
import type { PreUploadAllowList, PreUploadScanAdapter, PreUploadScanInput, PreUploadScanResult } from "./pre-upload-types.js";
// Reject control characters (\x00 null, \x01-\x1f, \x7f DEL) and path-traversal
// sequences (../ or ..\\) which would let an attacker escape the bucket prefix.
// eslint-disable-next-line no-control-regex
const FILENAME_BAD_PATTERN = /[\x00-\x1f\x7f]|\.\.[\\/]/;
const FILENAME_MAX_LENGTH = 255;
const SVG_SIGNATURE = /^<\?xml[\s\S]*?<svg\b|^<svg\b/i;
const TEXT_PLAIN_FALLBACK_MIMES = new Set(["text/plain", "text/csv"]);
export interface MagicBytesScanAdapterOptions {
/**
* Per-context MIME allow-list. Defaults to DEFAULT_ALLOW_LIST.
*/
allowList?: PreUploadAllowList;
/**
* Env-style toggle. When false, scan() returns `{ status: "skipped",
* reason: "scanner_disabled" }`. Intended as a Notfall-Bypass — never
* default to false.
*/
enabled?: boolean;
}
/**
* Synchronous pre-upload scanner. Uses file-type's magic-bytes signature
* database to derive the actual MIME type from the leading bytes of the buffer
* and validates it against (a) the client-declared MIME type and (b) the
* per-context allow-list.
*
* Defense-in-depth: the adapter does NOT log the buffer contents. Only the
* filename, sizes, and detected/declared MIME types are surfaced via the
* `detail` field so the caller can decide what to write to the audit log.
*/
export function createMagicBytesScanAdapter(options: MagicBytesScanAdapterOptions = {}): PreUploadScanAdapter {
const allowList = options.allowList ?? DEFAULT_ALLOW_LIST;
const enabled = options.enabled !== false;
return {
async scan(input: PreUploadScanInput): Promise<PreUploadScanResult> {
if (!enabled) {
return { status: "skipped", reason: "scanner_disabled" };
}
const filenameVerdict = validateFilename(input.filename);
if (filenameVerdict) return filenameVerdict;
if (!input.buffer || input.buffer.byteLength === 0) {
return { status: "rejected", reason: "empty", detail: "buffer was empty" };
}
const declared = input.declaredMimeType.trim().toLowerCase();
const contextAllowList = allowList[input.context] ?? [];
const detected = await detectMimeType(input.buffer);
// SVG-special-case: file-type reports SVG (which is just XML) as
// application/xml. Promote it to image/svg+xml only when the declared
// type asks for SVG, the content looks like SVG, and the context
// allow-list permits it. Otherwise XML stays XML and falls through
// to the normal mismatch path below.
if (detected === "application/xml" && declared === "image/svg+xml" && contextAllowList.includes("image/svg+xml") && looksLikeSvg(input.buffer)) {
return { status: "clean", detectedMimeType: "image/svg+xml" };
}
if (!detected) {
// file-type returns undefined for plain text / csv and a handful of
// long-tail formats. Allow it through only when the declared MIME is
// in our text fallback set AND the allow-list permits it.
if (TEXT_PLAIN_FALLBACK_MIMES.has(declared) && contextAllowList.includes(declared)) {
return { status: "clean" };
}
return {
status: "rejected",
reason: "mime_undetectable",
detail: `unable to detect MIME for declared="${declared}"`,
};
}
if (detected !== declared) {
return {
status: "rejected",
reason: "mime_mismatch",
detectedMimeType: detected,
detail: `declared="${declared}", detected="${detected}"`,
};
}
if (!contextAllowList.includes(detected)) {
return {
status: "rejected",
reason: "mime_not_allowed",
detectedMimeType: detected,
detail: `context="${input.context}" does not permit "${detected}"`,
};
}
return { status: "clean", detectedMimeType: detected };
},
};
}
async function detectMimeType(buffer: Uint8Array): Promise<string | undefined> {
const head = buffer.byteLength > 4100 ? buffer.subarray(0, 4100) : buffer;
try {
const result = await fileTypeFromBuffer(head);
return result?.mime;
} catch {
return undefined;
}
}
function validateFilename(filename: string): PreUploadScanResult | undefined {
if (typeof filename !== "string" || filename.length === 0) {
return { status: "rejected", reason: "filename_invalid", detail: "filename was empty" };
}
if (filename.length > FILENAME_MAX_LENGTH) {
return { status: "rejected", reason: "filename_invalid", detail: `filename exceeded ${FILENAME_MAX_LENGTH} chars` };
}
if (FILENAME_BAD_PATTERN.test(filename)) {
return { status: "rejected", reason: "filename_invalid", detail: "filename contained control chars or path traversal" };
}
return undefined;
}
function looksLikeSvg(buffer: Uint8Array): boolean {
const head = new TextDecoder("utf-8", { fatal: false }).decode(buffer.subarray(0, 512)).trimStart();
return SVG_SIGNATURE.test(head);
}
@@ -0,0 +1,53 @@
// Pre-upload content-scan types. Distinct from the post-upload
// StorageScanner / StorageQuarantineAdapter pipeline in index.ts —
// those run after an object is already in the bucket. The types
// here run synchronously against an in-memory buffer before the
// commit, primarily to validate the declared MIME type against
// the actual magic-bytes signature (layer 1) and, in a follow-up,
// to hand the same buffer off to a content scanner like ClamAV
// (layer 2).
/** Logical upload contexts. Each one carries its own allow-list. */
export type ScanContext = "avatar" | "mediaLibrary" | "emailAttachment" | "formUpload" | "kbAsset";
export const SCAN_CONTEXTS: readonly ScanContext[] = ["avatar", "mediaLibrary", "emailAttachment", "formUpload", "kbAsset"] as const;
/** Verdict of a single pre-upload scan call. */
export type PreUploadScanStatus = "clean" | "rejected" | "skipped";
export type PreUploadScanReason =
| "empty"
| "mime_mismatch"
| "mime_not_allowed"
| "mime_undetectable"
| "filename_invalid"
| "scanner_disabled"
| "scanner_error";
export interface PreUploadScanInput {
/** The bytes about to be uploaded. Adapters MUST NOT log this content. */
buffer: Uint8Array;
/** The MIME type the client declared (Content-Type header or form metadata). */
declaredMimeType: string;
/** The original filename the client supplied. May be untrusted. */
filename: string;
/** Logical context that drives the allow-list. */
context: ScanContext;
}
export interface PreUploadScanResult {
status: PreUploadScanStatus;
/** Set on `rejected` / `skipped`. Concise enum, not a user-facing string. */
reason?: PreUploadScanReason;
/** MIME type the adapter detected from the buffer (e.g. by magic-bytes), if any. */
detectedMimeType?: string;
/** Adapter-level diagnostic detail. Safe to log and audit. */
detail?: string;
}
export interface PreUploadScanAdapter {
scan(input: PreUploadScanInput): Promise<PreUploadScanResult>;
}
/** Per-context allow-list configuration. Empty array = nothing accepted in that context. */
export type PreUploadAllowList = Record<ScanContext, readonly string[]>;
@@ -0,0 +1,100 @@
// @vitest-environment node
import { describe, expect, test, vi } from "vitest";
import { CopyObjectCommand, DeleteObjectCommand, S3Client as AwsS3Client } from "@aws-sdk/client-s3";
import { createS3StorageQuarantineAdapter } from "./s3-quarantine.js";
const baseConfig = {
region: "us-east-1",
accessKeyId: "ak",
secretAccessKey: "sk",
};
function buildFakeClient() {
return { send: vi.fn(async () => ({})) } as unknown as AwsS3Client;
}
describe("createS3StorageQuarantineAdapter", () => {
test("server-side copies into quarantine/<key> and deletes the source", async () => {
const client = buildFakeClient();
const adapter = createS3StorageQuarantineAdapter({ ...baseConfig, clientFactory: () => client });
const result = await adapter.quarantineObject({
bucket: "uploads",
key: "users/123/avatar.png",
signature: "Win.Test.EICAR_HDB-1",
});
expect(result.bucket).toBe("uploads");
expect(result.key).toBe("quarantine/users/123/avatar.png");
expect(result.quarantineKey).toBe("quarantine/users/123/avatar.png");
expect(result.action).toBe("moved");
const sendMock = vi.mocked(client.send);
expect(sendMock).toHaveBeenCalledTimes(2);
const copyCall = sendMock.mock.calls[0]?.[0];
expect(copyCall).toBeInstanceOf(CopyObjectCommand);
const copyInput = (copyCall as CopyObjectCommand).input;
expect(copyInput.Bucket).toBe("uploads");
expect(copyInput.Key).toBe("quarantine/users/123/avatar.png");
expect(copyInput.CopySource).toBe(`uploads/${encodeURIComponent("users/123/avatar.png")}`);
expect(copyInput.MetadataDirective).toBe("REPLACE");
expect(copyInput.Metadata?.["scan-signature"]).toBe("Win.Test.EICAR_HDB-1");
expect(copyInput.Metadata?.["scan-source-bucket"]).toBe("uploads");
expect(copyInput.Metadata?.["scan-source-key"]).toBe("users/123/avatar.png");
const deleteCall = sendMock.mock.calls[1]?.[0];
expect(deleteCall).toBeInstanceOf(DeleteObjectCommand);
const deleteInput = (deleteCall as DeleteObjectCommand).input;
expect(deleteInput.Bucket).toBe("uploads");
expect(deleteInput.Key).toBe("users/123/avatar.png");
});
test("uses an explicit quarantineKey when the input provides one", async () => {
const client = buildFakeClient();
const adapter = createS3StorageQuarantineAdapter({ ...baseConfig, clientFactory: () => client });
const result = await adapter.quarantineObject({
bucket: "uploads",
key: "foo",
quarantineKey: "isolated/bar",
signature: "sig",
});
expect(result.key).toBe("isolated/bar");
const copyCall = vi.mocked(client.send).mock.calls[0]?.[0] as CopyObjectCommand;
expect(copyCall.input.Key).toBe("isolated/bar");
});
test("targets a dedicated quarantineBucket when configured", async () => {
const client = buildFakeClient();
const adapter = createS3StorageQuarantineAdapter({ ...baseConfig, quarantineBucket: "infected", clientFactory: () => client });
const result = await adapter.quarantineObject({ bucket: "uploads", key: "a", signature: "sig" });
expect(result.bucket).toBe("infected");
const copyCall = vi.mocked(client.send).mock.calls[0]?.[0] as CopyObjectCommand;
expect(copyCall.input.Bucket).toBe("infected");
});
test("honours a custom buildQuarantineKey", async () => {
const client = buildFakeClient();
const adapter = createS3StorageQuarantineAdapter({
...baseConfig,
clientFactory: () => client,
buildQuarantineKey: (input) => `viral/${input.signature}/${input.key}`,
});
const result = await adapter.quarantineObject({ bucket: "uploads", key: "f.png", signature: "Foo.Bar-1" });
expect(result.key).toBe("viral/Foo.Bar-1/f.png");
});
test("caches the S3 client across calls", async () => {
const client = buildFakeClient();
const factory = vi.fn(() => client);
const adapter = createS3StorageQuarantineAdapter({ ...baseConfig, clientFactory: factory });
await adapter.quarantineObject({ bucket: "u", key: "a", signature: "s" });
await adapter.quarantineObject({ bucket: "u", key: "b", signature: "s" });
expect(factory).toHaveBeenCalledTimes(1);
});
test("surfaces SDK errors instead of swallowing them", async () => {
const client = { send: vi.fn(async () => { throw new Error("AccessDenied"); }) } as unknown as AwsS3Client;
const adapter = createS3StorageQuarantineAdapter({ ...baseConfig, clientFactory: () => client });
await expect(adapter.quarantineObject({ bucket: "u", key: "k", signature: "s" })).rejects.toThrow("AccessDenied");
});
});
@@ -0,0 +1,93 @@
// S3-backed implementation of the StorageQuarantineAdapter contract from
// ./index.ts. Server-side copies an infected object into a `quarantine/`
// prefix and deletes the original, so the file is gone from any public
// listing of the source key without ever round-tripping through the worker
// process memory.
import { CopyObjectCommand, DeleteObjectCommand, S3Client as AwsS3Client } from "@aws-sdk/client-s3";
import type { StorageQuarantineAdapter, StorageQuarantineInput, StorageQuarantineResult } from "./index.js";
export interface S3QuarantineConfig {
endpoint?: string;
region: string;
accessKeyId: string;
secretAccessKey: string;
sessionToken?: string;
forcePathStyle?: boolean;
/**
* Bucket the quarantined object lives in. Defaults to the same bucket as
* the source. Pass a dedicated quarantine bucket if your S3 ACL strategy
* keeps it isolated from public listing.
*/
quarantineBucket?: string;
/**
* Override the key derivation. Defaults to prefixing the source key with
* `quarantine/` so the bucket layout looks like quarantine/<originalKey>.
*/
buildQuarantineKey?: (input: StorageQuarantineInput) => string;
/**
* Test seam. Defaults to a real AwsS3Client constructed from the config.
*/
clientFactory?: () => AwsS3Client;
}
const DEFAULT_PREFIX = "quarantine/";
export function createS3StorageQuarantineAdapter(config: S3QuarantineConfig): StorageQuarantineAdapter {
const buildKey = config.buildQuarantineKey ?? defaultBuildQuarantineKey;
const clientFactory =
config.clientFactory ??
(() =>
new AwsS3Client({
endpoint: config.endpoint,
region: config.region,
forcePathStyle: config.forcePathStyle,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
sessionToken: config.sessionToken,
},
}));
let client: AwsS3Client | undefined;
return {
async quarantineObject(input: StorageQuarantineInput): Promise<StorageQuarantineResult> {
const targetBucket = config.quarantineBucket ?? input.bucket;
const targetKey = buildKey(input);
if (!client) client = clientFactory();
await client.send(
new CopyObjectCommand({
Bucket: targetBucket,
Key: targetKey,
CopySource: `${input.bucket}/${encodeURIComponent(input.key)}`,
MetadataDirective: "REPLACE",
Metadata: {
"scan-signature": input.signature,
"scan-quarantined-at": new Date().toISOString(),
"scan-source-bucket": input.bucket,
"scan-source-key": input.key,
},
})
);
await client.send(
new DeleteObjectCommand({
Bucket: input.bucket,
Key: input.key,
})
);
return {
bucket: targetBucket,
key: targetKey,
quarantineKey: targetKey,
action: "moved",
};
},
};
}
function defaultBuildQuarantineKey(input: StorageQuarantineInput): string {
if (input.quarantineKey) return input.quarantineKey;
return `${DEFAULT_PREFIX}${input.key}`;
}