Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
@@ -12,7 +12,7 @@
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.958.0",
|
||||
"@aws-sdk/client-s3": "3.1048.0",
|
||||
"clamscan": "^2.4.0",
|
||||
"file-type": "^22.0.1"
|
||||
},
|
||||
|
||||
@@ -194,6 +194,16 @@ describe("createS3StorageAdapter", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards an abort signal to bounded S3 deletions", async () => {
|
||||
const send = vi.fn().mockResolvedValue({});
|
||||
const adapter = createS3StorageAdapter({ ...baseConfig, client: { send } });
|
||||
const controller = new AbortController();
|
||||
|
||||
await adapter.deleteObject({ key: "exports/job-123.csv", signal: controller.signal });
|
||||
|
||||
expect(send).toHaveBeenCalledWith(expect.any(DeleteObjectCommand), { abortSignal: controller.signal });
|
||||
});
|
||||
|
||||
it("rejects invalid delete keys and buckets before sending to S3", async () => {
|
||||
const send = vi.fn().mockResolvedValue({});
|
||||
const adapter = createS3StorageAdapter({ ...baseConfig, client: { send } });
|
||||
|
||||
@@ -1,12 +1,3 @@
|
||||
import {
|
||||
AbortMultipartUploadCommand as AwsAbortMultipartUploadCommand,
|
||||
CompleteMultipartUploadCommand as AwsCompleteMultipartUploadCommand,
|
||||
CreateMultipartUploadCommand as AwsCreateMultipartUploadCommand,
|
||||
DeleteObjectCommand as AwsDeleteObjectCommand,
|
||||
PutObjectCommand as AwsPutObjectCommand,
|
||||
S3Client as AwsS3Client,
|
||||
UploadPartCopyCommand as AwsUploadPartCopyCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve, sep } from "node:path";
|
||||
|
||||
@@ -14,6 +5,7 @@ export * from "./access/index.js";
|
||||
export * from "./paths/index.js";
|
||||
export * from "./scan/index.js";
|
||||
export * from "./signed/index.js";
|
||||
export * from "./sse-c.js";
|
||||
|
||||
export type UploadBody = string | Uint8Array | ArrayBuffer | ReadableStream<Uint8Array>;
|
||||
|
||||
@@ -69,12 +61,7 @@ export class StorageAdapterError extends Error {
|
||||
readonly details: StorageAdapterErrorDetails;
|
||||
override readonly cause?: unknown;
|
||||
|
||||
constructor(
|
||||
code: StorageAdapterErrorCode,
|
||||
message: string,
|
||||
details: StorageAdapterErrorDetails,
|
||||
options: { cause?: unknown } = {},
|
||||
) {
|
||||
constructor(code: StorageAdapterErrorCode, message: string, details: StorageAdapterErrorDetails, options: { cause?: unknown } = {}) {
|
||||
super(message);
|
||||
this.name = "StorageAdapterError";
|
||||
this.code = code;
|
||||
@@ -94,6 +81,7 @@ export interface PutObjectInput {
|
||||
export interface DeleteObjectInput {
|
||||
bucket?: string;
|
||||
key: string;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface CreateMultipartUploadInput {
|
||||
@@ -129,7 +117,7 @@ export interface StorageAdapter {
|
||||
/**
|
||||
* Server-side chunk assembly (§3). Chunks are first stored as temp objects in
|
||||
* the same bucket, then assembled into the final object via multipart
|
||||
* UploadPartCopy — memory-safe for files up to the 500 MB limit. The whole
|
||||
* UploadPartCopy — memory-safe for files up to the 100 MB limit. The whole
|
||||
* create/copy/complete sequence runs within a single `completeUpload` request,
|
||||
* so the S3 `UploadId` never needs to be persisted across requests.
|
||||
*/
|
||||
@@ -234,7 +222,21 @@ export type StorageCommand =
|
||||
| AbortMultipartUploadCommand;
|
||||
|
||||
export interface S3StorageClient {
|
||||
send(command: StorageCommand): Promise<unknown>;
|
||||
send(command: StorageCommand, options?: { abortSignal?: AbortSignal }): Promise<unknown>;
|
||||
}
|
||||
|
||||
type AwsS3Module = typeof import("@aws-sdk/client-s3");
|
||||
type AwsS3Client = InstanceType<AwsS3Module["S3Client"]>;
|
||||
type AwsPutObjectCommandInput = ConstructorParameters<AwsS3Module["PutObjectCommand"]>[0];
|
||||
|
||||
let awsS3ModulePromise: Promise<AwsS3Module> | undefined;
|
||||
|
||||
function loadAwsS3Module(): Promise<AwsS3Module> {
|
||||
// Most web requests only need key validation or the local adapter. Keep the
|
||||
// large AWS SDK out of process startup and load one shared copy on the first
|
||||
// real S3 operation.
|
||||
awsS3ModulePromise ??= import("@aws-sdk/client-s3");
|
||||
return awsS3ModulePromise;
|
||||
}
|
||||
|
||||
export class S3Client implements S3StorageClient {
|
||||
@@ -248,37 +250,40 @@ export class S3Client implements S3StorageClient {
|
||||
sessionToken?: string;
|
||||
};
|
||||
};
|
||||
private readonly client: AwsS3Client;
|
||||
private clientPromise: Promise<AwsS3Client> | undefined;
|
||||
|
||||
constructor(config: S3Client["config"]) {
|
||||
this.config = config;
|
||||
this.client = new AwsS3Client(config);
|
||||
}
|
||||
|
||||
async send(command: StorageCommand): Promise<unknown> {
|
||||
async send(command: StorageCommand, options?: { abortSignal?: AbortSignal }): Promise<unknown> {
|
||||
const aws = await loadAwsS3Module();
|
||||
this.clientPromise ??= Promise.resolve(new aws.S3Client(this.config));
|
||||
const client = await this.clientPromise;
|
||||
|
||||
if (command instanceof PutObjectCommand) {
|
||||
return this.client.send(new AwsPutObjectCommand(toAwsPutObjectInput(command.input)));
|
||||
return client.send(new aws.PutObjectCommand(toAwsPutObjectInput(command.input)));
|
||||
}
|
||||
if (command instanceof CreateMultipartUploadCommand) {
|
||||
return this.client.send(new AwsCreateMultipartUploadCommand(command.input));
|
||||
return client.send(new aws.CreateMultipartUploadCommand(command.input));
|
||||
}
|
||||
if (command instanceof UploadPartCopyCommand) {
|
||||
return this.client.send(new AwsUploadPartCopyCommand(command.input));
|
||||
return client.send(new aws.UploadPartCopyCommand(command.input));
|
||||
}
|
||||
if (command instanceof CompleteMultipartUploadCommand) {
|
||||
return this.client.send(new AwsCompleteMultipartUploadCommand(command.input));
|
||||
return client.send(new aws.CompleteMultipartUploadCommand(command.input));
|
||||
}
|
||||
if (command instanceof AbortMultipartUploadCommand) {
|
||||
return this.client.send(new AwsAbortMultipartUploadCommand(command.input));
|
||||
return client.send(new aws.AbortMultipartUploadCommand(command.input));
|
||||
}
|
||||
return this.client.send(new AwsDeleteObjectCommand(command.input));
|
||||
return client.send(new aws.DeleteObjectCommand(command.input), options?.abortSignal ? { abortSignal: options.abortSignal } : undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function toAwsPutObjectInput(input: PutObjectCommandInput): ConstructorParameters<typeof AwsPutObjectCommand>[0] {
|
||||
function toAwsPutObjectInput(input: PutObjectCommandInput): AwsPutObjectCommandInput {
|
||||
return {
|
||||
...input,
|
||||
Body: normalizeAwsUploadBody(input.Body) as ConstructorParameters<typeof AwsPutObjectCommand>[0]["Body"],
|
||||
Body: normalizeAwsUploadBody(input.Body) as AwsPutObjectCommandInput["Body"],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -304,31 +309,17 @@ export interface LocalStorageAdapterConfig {
|
||||
publicBaseUrl?: string;
|
||||
}
|
||||
|
||||
const REQUIRED_CONFIG_FIELDS = [
|
||||
"endpoint",
|
||||
"region",
|
||||
"bucket",
|
||||
"accessKeyId",
|
||||
"secretAccessKey",
|
||||
] as const satisfies readonly (keyof S3StorageAdapterConfig)[];
|
||||
const REQUIRED_CONFIG_FIELDS = ["endpoint", "region", "bucket", "accessKeyId", "secretAccessKey"] as const satisfies readonly (keyof S3StorageAdapterConfig)[];
|
||||
|
||||
export function createS3StorageAdapter(
|
||||
config: S3StorageAdapterConfig,
|
||||
): StorageAdapter {
|
||||
const missingFields = REQUIRED_CONFIG_FIELDS.filter(
|
||||
(field) => !isPresent(config[field]),
|
||||
);
|
||||
export function createS3StorageAdapter(config: S3StorageAdapterConfig): StorageAdapter {
|
||||
const missingFields = REQUIRED_CONFIG_FIELDS.filter((field) => !isPresent(config[field]));
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
throw new StorageAdapterError(
|
||||
"config_missing",
|
||||
`Missing S3 storage configuration: ${missingFields.join(", ")}`,
|
||||
{
|
||||
operation: "createS3StorageAdapter",
|
||||
bucket: isPresent(config.bucket) ? config.bucket : undefined,
|
||||
missingFields: [...missingFields],
|
||||
},
|
||||
);
|
||||
throw new StorageAdapterError("config_missing", `Missing S3 storage configuration: ${missingFields.join(", ")}`, {
|
||||
operation: "createS3StorageAdapter",
|
||||
bucket: isPresent(config.bucket) ? config.bucket : undefined,
|
||||
missingFields: [...missingFields],
|
||||
});
|
||||
}
|
||||
|
||||
const client =
|
||||
@@ -367,7 +358,7 @@ export function createS3StorageAdapter(
|
||||
key: input.key,
|
||||
upstream: extractUpstreamDetails(cause),
|
||||
},
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -382,27 +373,19 @@ export function createS3StorageAdapter(
|
||||
const bucket = input.bucket ?? config.bucket;
|
||||
|
||||
if (!isPresent(bucket)) {
|
||||
throw new StorageAdapterError(
|
||||
"invalid_bucket",
|
||||
"S3 object deletion requires a non-empty bucket.",
|
||||
{
|
||||
operation: "deleteObject",
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
},
|
||||
);
|
||||
throw new StorageAdapterError("invalid_bucket", "S3 object deletion requires a non-empty bucket.", {
|
||||
operation: "deleteObject",
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
});
|
||||
}
|
||||
|
||||
if (!isPresent(input.key)) {
|
||||
throw new StorageAdapterError(
|
||||
"invalid_key",
|
||||
"S3 object deletion requires a non-empty key.",
|
||||
{
|
||||
operation: "deleteObject",
|
||||
bucket,
|
||||
key: input.key,
|
||||
},
|
||||
);
|
||||
throw new StorageAdapterError("invalid_key", "S3 object deletion requires a non-empty key.", {
|
||||
operation: "deleteObject",
|
||||
bucket,
|
||||
key: input.key,
|
||||
});
|
||||
}
|
||||
|
||||
const command = new DeleteObjectCommand({
|
||||
@@ -411,7 +394,7 @@ export function createS3StorageAdapter(
|
||||
});
|
||||
|
||||
try {
|
||||
await client.send(command);
|
||||
await client.send(command, input.signal ? { abortSignal: input.signal } : undefined);
|
||||
} catch (cause) {
|
||||
throw new StorageAdapterError(
|
||||
"delete_failed",
|
||||
@@ -422,7 +405,7 @@ export function createS3StorageAdapter(
|
||||
key: input.key,
|
||||
upstream: extractUpstreamDetails(cause),
|
||||
},
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -440,7 +423,7 @@ export function createS3StorageAdapter(
|
||||
Key: input.key,
|
||||
ContentType: input.contentType,
|
||||
Metadata: input.metadata,
|
||||
}),
|
||||
})
|
||||
)) as { UploadId?: string };
|
||||
if (!isPresent(response.UploadId)) {
|
||||
throw new Error("S3 CreateMultipartUpload returned no UploadId.");
|
||||
@@ -451,7 +434,7 @@ export function createS3StorageAdapter(
|
||||
"put_failed",
|
||||
"Failed to create multipart upload in S3 storage.",
|
||||
{ operation: "createMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
},
|
||||
@@ -469,7 +452,7 @@ export function createS3StorageAdapter(
|
||||
.split("/")
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join("/"),
|
||||
}),
|
||||
})
|
||||
)) as { CopyPartResult?: { ETag?: string } };
|
||||
const etag = response.CopyPartResult?.ETag;
|
||||
if (!isPresent(etag)) {
|
||||
@@ -481,7 +464,7 @@ export function createS3StorageAdapter(
|
||||
"put_failed",
|
||||
"Failed to copy part in S3 multipart upload.",
|
||||
{ operation: "uploadPartCopy", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
},
|
||||
@@ -494,18 +477,16 @@ export function createS3StorageAdapter(
|
||||
Key: input.key,
|
||||
UploadId: input.uploadId,
|
||||
MultipartUpload: {
|
||||
Parts: [...input.parts]
|
||||
.sort((a, b) => a.partNumber - b.partNumber)
|
||||
.map((part) => ({ ETag: part.etag, PartNumber: part.partNumber })),
|
||||
Parts: [...input.parts].sort((a, b) => a.partNumber - b.partNumber).map((part) => ({ ETag: part.etag, PartNumber: part.partNumber })),
|
||||
},
|
||||
}),
|
||||
})
|
||||
);
|
||||
} catch (cause) {
|
||||
throw new StorageAdapterError(
|
||||
"put_failed",
|
||||
"Failed to complete S3 multipart upload.",
|
||||
{ operation: "completeMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
return { bucket: config.bucket, key: input.key, publicUrl: buildPublicUrl(config.publicBaseUrl, input.key) };
|
||||
@@ -513,15 +494,13 @@ export function createS3StorageAdapter(
|
||||
|
||||
async abortMultipartUpload(input) {
|
||||
try {
|
||||
await client.send(
|
||||
new AbortMultipartUploadCommand({ Bucket: config.bucket, Key: input.key, UploadId: input.uploadId }),
|
||||
);
|
||||
await client.send(new AbortMultipartUploadCommand({ Bucket: config.bucket, Key: input.key, UploadId: input.uploadId }));
|
||||
} catch (cause) {
|
||||
throw new StorageAdapterError(
|
||||
"put_failed",
|
||||
"Failed to abort S3 multipart upload.",
|
||||
{ operation: "abortMultipartUpload", bucket: config.bucket, key: input.key, upstream: extractUpstreamDetails(cause) },
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
},
|
||||
@@ -529,10 +508,7 @@ export function createS3StorageAdapter(
|
||||
}
|
||||
|
||||
export function createLocalStorageAdapter(config: LocalStorageAdapterConfig): StorageAdapter {
|
||||
const missingFields = [
|
||||
...(!isPresent(config.rootDir) ? ["rootDir"] : []),
|
||||
...(!isPresent(config.bucket) ? ["bucket"] : []),
|
||||
];
|
||||
const missingFields = [...(!isPresent(config.rootDir) ? ["rootDir"] : []), ...(!isPresent(config.bucket) ? ["bucket"] : [])];
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
throw new StorageAdapterError("config_missing", `Missing local storage configuration: ${missingFields.join(", ")}`, {
|
||||
@@ -564,7 +540,7 @@ export function createLocalStorageAdapter(config: LocalStorageAdapterConfig): St
|
||||
key,
|
||||
upstream: extractUpstreamDetails(cause),
|
||||
},
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -601,7 +577,7 @@ export function createLocalStorageAdapter(config: LocalStorageAdapterConfig): St
|
||||
key,
|
||||
upstream: extractUpstreamDetails(cause),
|
||||
},
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -632,7 +608,7 @@ export function createLocalStorageAdapter(config: LocalStorageAdapterConfig): St
|
||||
"put_failed",
|
||||
"Failed to copy part in local multipart upload.",
|
||||
{ operation: "uploadPartCopy", bucket: config.bucket, key, upstream: extractUpstreamDetails(cause) },
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
return { partNumber: input.partNumber, etag: `local-${input.partNumber}` };
|
||||
@@ -658,7 +634,7 @@ export function createLocalStorageAdapter(config: LocalStorageAdapterConfig): St
|
||||
"put_failed",
|
||||
"Failed to complete local multipart upload.",
|
||||
{ operation: "completeMultipartUpload", bucket: config.bucket, key, upstream: extractUpstreamDetails(cause) },
|
||||
{ cause },
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
return { bucket: config.bucket, key, publicUrl: buildPublicUrl(config.publicBaseUrl, `${config.bucket}/${key}`) };
|
||||
|
||||
@@ -2,6 +2,9 @@ import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
assertFileKeyBelongsToTenant,
|
||||
assertGdprExportArtifactKey,
|
||||
buildGdprExportArtifactKey,
|
||||
buildGdprExportTenantPrefix,
|
||||
buildFilePreviewKey,
|
||||
buildFileStorageKeys,
|
||||
buildUploadChunkKey,
|
||||
@@ -40,4 +43,35 @@ describe("file storage paths", () => {
|
||||
expect(assertFileKeyBelongsToTenant("acme", key)).toBe(key);
|
||||
expect(() => assertFileKeyBelongsToTenant("other", key)).toThrow();
|
||||
});
|
||||
|
||||
it("accepts only exact tenant-scoped GDPR export keys in the export-first lifecycle prefix", () => {
|
||||
const privacy = buildGdprExportArtifactKey({ tenantId: "acme", exportJobId: "job_123", attemptId: "attempt_123", scope: "privacy" });
|
||||
const portal = buildGdprExportArtifactKey({ tenantId: "acme", exportJobId: "job_123", attemptId: "attempt_123", scope: "portal" });
|
||||
expect(assertGdprExportArtifactKey({ tenantId: "acme", exportJobId: "job_123", key: privacy, scope: "privacy" })).toBe(privacy);
|
||||
expect(assertGdprExportArtifactKey({ tenantId: "acme", exportJobId: "job_123", key: portal, scope: "portal" })).toBe(portal);
|
||||
expect(() => assertGdprExportArtifactKey({ tenantId: "other", exportJobId: "job_123", key: privacy, scope: "privacy" })).toThrow();
|
||||
expect(() => assertGdprExportArtifactKey({ tenantId: "acme", exportJobId: "other", key: privacy, scope: "privacy" })).toThrow();
|
||||
expect(() => assertGdprExportArtifactKey({ tenantId: "acme", exportJobId: "job_123", key: portal, scope: "privacy" })).toThrow();
|
||||
expect(() =>
|
||||
assertGdprExportArtifactKey({
|
||||
tenantId: "acme",
|
||||
exportJobId: "job_123",
|
||||
key: "gdpr-exports/privacy/tenants/acme/../../files/x.zip",
|
||||
scope: "privacy",
|
||||
})
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
assertGdprExportArtifactKey({ tenantId: "acme", exportJobId: "job_123", key: "tenants/acme/files/file_1/original", scope: "privacy" })
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("builds guarded tenant export prefixes for privacy and portal hard-delete", () => {
|
||||
expect(buildGdprExportTenantPrefix({ tenantId: "acme", scope: "privacy" })).toBe(
|
||||
"gdpr-exports/privacy/tenants/acme"
|
||||
);
|
||||
expect(buildGdprExportTenantPrefix({ tenantId: "acme", scope: "portal" })).toBe(
|
||||
"gdpr-exports/portal/tenants/acme"
|
||||
);
|
||||
expect(() => buildGdprExportTenantPrefix({ tenantId: "../other", scope: "privacy" })).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,8 @@ export interface FileStorageKeys {
|
||||
readonly thumbnail: string;
|
||||
}
|
||||
|
||||
export type GdprExportArtifactScope = "privacy" | "portal";
|
||||
|
||||
function requireSegment(value: string, field: string): string {
|
||||
const normalized = value.trim();
|
||||
if (normalized.length === 0 || normalized.includes("/") || normalized.includes("\\")) {
|
||||
@@ -77,3 +79,69 @@ export function assertFileKeyBelongsToTenant(
|
||||
}
|
||||
return decision.key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the dedicated export-first keyspace used by the SSE-C writer,
|
||||
* authenticated proxy and retention worker. This deliberately does not accept
|
||||
* arbitrary `tenants/{id}/...` media keys because retention credentials may
|
||||
* delete only GDPR export artifacts.
|
||||
*/
|
||||
export function buildGdprExportArtifactKey(input: {
|
||||
tenantId: string;
|
||||
exportJobId: string;
|
||||
attemptId: string;
|
||||
scope: GdprExportArtifactScope;
|
||||
}): string {
|
||||
const extension = input.scope === "privacy" ? "zip" : "json";
|
||||
return `${buildGdprExportTenantPrefix(input)}/${requireOpaqueExportSegment(
|
||||
input.exportJobId,
|
||||
"exportJobId"
|
||||
)}/${requireOpaqueExportSegment(input.attemptId, "attemptId")}.${extension}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tenant-owned root of the export-first keyspace. Hard-delete uses this
|
||||
* prefix so it also removes upload attempts that were never committed to a
|
||||
* database row.
|
||||
*/
|
||||
export function buildGdprExportTenantPrefix(input: {
|
||||
tenantId: string;
|
||||
scope: GdprExportArtifactScope;
|
||||
}): string {
|
||||
return `gdpr-exports/${input.scope}/tenants/${requireOpaqueExportSegment(input.tenantId, "tenantId")}`;
|
||||
}
|
||||
|
||||
export function assertGdprExportArtifactKey(input: {
|
||||
tenantId: string;
|
||||
exportJobId: string;
|
||||
key: string;
|
||||
scope: GdprExportArtifactScope;
|
||||
}): string {
|
||||
const normalizedTenantId = requireOpaqueExportSegment(input.tenantId, "tenantId");
|
||||
const normalizedExportJobId = requireOpaqueExportSegment(input.exportJobId, "exportJobId");
|
||||
const parts = input.key.split("/");
|
||||
const extension = input.scope === "privacy" ? ".zip" : ".json";
|
||||
if (
|
||||
parts.length !== 6 ||
|
||||
parts[0] !== "gdpr-exports" ||
|
||||
parts[1] !== input.scope ||
|
||||
parts[2] !== "tenants" ||
|
||||
parts[3] !== normalizedTenantId ||
|
||||
parts[4] !== normalizedExportJobId ||
|
||||
!isOpaqueExportSegment((parts[5] ?? "").slice(0, -extension.length)) ||
|
||||
!parts[5]?.endsWith(extension)
|
||||
) {
|
||||
throw new Error("GDPR export artifact key is outside the permitted tenant export prefix.");
|
||||
}
|
||||
return input.key;
|
||||
}
|
||||
|
||||
function requireOpaqueExportSegment(value: string, field: string): string {
|
||||
const normalized = value.trim();
|
||||
if (!isOpaqueExportSegment(normalized)) throw new Error(`Invalid GDPR export artifact segment for ${field}.`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function isOpaqueExportSegment(value: string): boolean {
|
||||
return /^[A-Za-z0-9_-]{1,191}$/u.test(value);
|
||||
}
|
||||
@@ -5,6 +5,14 @@ import { DEFAULT_ALLOW_LIST, resolveAllowList } from "./allow-list.js";
|
||||
describe("resolveAllowList", () => {
|
||||
it("returns restrictive defaults when no environment overrides are supplied", () => {
|
||||
expect(resolveAllowList()).toEqual(DEFAULT_ALLOW_LIST);
|
||||
expect(DEFAULT_ALLOW_LIST.formUpload).toEqual(
|
||||
expect.arrayContaining([
|
||||
"image/jpeg",
|
||||
"application/pdf",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"audio/mpeg",
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it("replaces a context default with the operator supplied env allow-list", () => {
|
||||
|
||||
@@ -7,6 +7,7 @@ const OFFICE_TYPES = [
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
] as const;
|
||||
const AUDIO_TYPES = ["audio/mpeg", "audio/wav", "audio/x-wav", "audio/ogg", "audio/mp4", "audio/aac", "audio/flac"] as const;
|
||||
|
||||
/**
|
||||
* Restrictive defaults per upload context. Operator can extend per-context via the
|
||||
@@ -17,7 +18,7 @@ export const DEFAULT_ALLOW_LIST: PreUploadAllowList = {
|
||||
avatar: IMAGE_TYPES,
|
||||
mediaLibrary: [...IMAGE_TYPES_WITH_GIF, "image/svg+xml", "image/x-icon", "image/vnd.microsoft.icon"],
|
||||
emailAttachment: [...IMAGE_TYPES_WITH_GIF, "application/pdf", "text/plain", "text/csv", "application/zip", ...OFFICE_TYPES],
|
||||
formUpload: [...IMAGE_TYPES, "application/pdf"],
|
||||
formUpload: [...IMAGE_TYPES, "image/heic", "image/heif", "application/pdf", ...OFFICE_TYPES, ...AUDIO_TYPES],
|
||||
kbAsset: [...IMAGE_TYPES, "application/pdf"],
|
||||
};
|
||||
|
||||
|
||||
+12
-4
@@ -6,10 +6,18 @@ import { describe, expect, test, vi } from "vitest";
|
||||
// where `clamscan` exposes `init` directly on the module object with NO default
|
||||
// export. This exercises the `?? mod` fallback and the non-constructor branch
|
||||
// of defaultInitClient (`typeof Factory === "function" ? new ... : Factory`).
|
||||
// Mirrors the real clamscan surface: ping + scanStream, no scanBuffer.
|
||||
const ping = vi.fn(async () => "PONG");
|
||||
const scanBuffer = vi.fn(async () => ({ isInfected: false as boolean, viruses: [] as string[] }));
|
||||
const scanStream = vi.fn(async () => ({ isInfected: false as boolean, viruses: [] as string[] }));
|
||||
const init = vi.fn(async () => ({ ping, scanBuffer, scanStream }));
|
||||
const scanStream = vi.fn(async (stream: NodeJS.ReadableStream) => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.from(chunk as Buffer));
|
||||
}
|
||||
scannedPayloads.push(Buffer.concat(chunks));
|
||||
return { isInfected: false as boolean, viruses: [] as string[] };
|
||||
});
|
||||
const scannedPayloads: Buffer[] = [];
|
||||
const init = vi.fn(async () => ({ ping, scanStream }));
|
||||
|
||||
// `default: undefined` keeps the property declared (so the source's `.default`
|
||||
// access is legal under vitest's strict ESM namespace) while still being
|
||||
@@ -33,7 +41,7 @@ describe("ClamAVStorageScanner with a default-less clamscan module", () => {
|
||||
// init() is called directly on the module export, never via `new`.
|
||||
expect(init).toHaveBeenCalledTimes(1);
|
||||
expect(ping).toHaveBeenCalled();
|
||||
expect(scanBuffer).toHaveBeenCalledWith(Buffer.from("hello"));
|
||||
expect(scannedPayloads.at(-1)).toEqual(Buffer.from("hello"));
|
||||
});
|
||||
|
||||
test("reuses the cached client across scans (init runs once per scanner)", async () => {
|
||||
|
||||
+20
-9
@@ -3,12 +3,21 @@ import { Buffer } from "node:buffer";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
|
||||
// Exercises the production defaultInitClient path (no initClient override) by
|
||||
// mocking the lazily-imported `clamscan` package. The factory exposes init(),
|
||||
// which returns the ping/scanBuffer/scanStream surface the wrapper adapts.
|
||||
// mocking the lazily-imported `clamscan` package. The mock MUST mirror the real
|
||||
// client surface: NodeClam.init() returns ping/scanStream/scanFile/scanDir and
|
||||
// has NO scanBuffer. An earlier mock invented one, which hid a TypeError on
|
||||
// every buffer scan in production (fail-closed => every avatar upload rejected).
|
||||
const ping = vi.fn(async () => "PONG");
|
||||
const scanBuffer = vi.fn(async () => ({ isInfected: false as boolean, viruses: [] as string[] }));
|
||||
const scanStream = vi.fn(async () => ({ isInfected: false as boolean, viruses: [] as string[] }));
|
||||
const init = vi.fn(async () => ({ ping, scanBuffer, scanStream }));
|
||||
const scanStream = vi.fn(async (stream: NodeJS.ReadableStream) => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.from(chunk as Buffer));
|
||||
}
|
||||
scannedPayloads.push(Buffer.concat(chunks));
|
||||
return { isInfected: false as boolean, viruses: [] as string[] };
|
||||
});
|
||||
const scannedPayloads: Buffer[] = [];
|
||||
const init = vi.fn(async () => ({ ping, scanStream }));
|
||||
|
||||
vi.mock("clamscan", () => {
|
||||
class NodeClam {
|
||||
@@ -22,7 +31,7 @@ import { createClamAVStorageScanner } from "./clamav-scanner.js";
|
||||
const baseConfig = { host: "clamav", port: 3310, timeoutMs: 30000 };
|
||||
|
||||
describe("ClamAVStorageScanner with the real (mocked) clamscan client", () => {
|
||||
test("initializes the clamscan factory and reports clean for a buffer payload", async () => {
|
||||
test("initializes the clamscan factory and streams a buffer payload to clamd", async () => {
|
||||
const scanner = createClamAVStorageScanner({
|
||||
...baseConfig,
|
||||
loadObject: async () => Buffer.from("hello world"),
|
||||
@@ -33,11 +42,13 @@ describe("ClamAVStorageScanner with the real (mocked) clamscan client", () => {
|
||||
expect(result.verdict).toBe("clean");
|
||||
expect(init).toHaveBeenCalledTimes(1);
|
||||
expect(ping).toHaveBeenCalled();
|
||||
expect(scanBuffer).toHaveBeenCalledWith(Buffer.from("hello world"));
|
||||
// The bytes must arrive intact over INSTREAM, not as the integers a Buffer
|
||||
// iterates into (the Readable.from(buffer) trap).
|
||||
expect(scannedPayloads.at(-1)).toEqual(Buffer.from("hello world"));
|
||||
});
|
||||
|
||||
test("adapts an infected scanBuffer verdict into the scanner result", async () => {
|
||||
scanBuffer.mockResolvedValueOnce({ isInfected: true, viruses: ["Win.Test.EICAR_HDB-1"] });
|
||||
test("adapts an infected verdict into the scanner result", async () => {
|
||||
scanStream.mockResolvedValueOnce({ isInfected: true, viruses: ["Win.Test.EICAR_HDB-1"] });
|
||||
const scanner = createClamAVStorageScanner({
|
||||
...baseConfig,
|
||||
loadObject: async () => Buffer.from("infected"),
|
||||
|
||||
+30
-1
@@ -1,7 +1,13 @@
|
||||
// @vitest-environment node
|
||||
import { Buffer } from "node:buffer";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { ClamAVUnreachableError, createClamAVStorageScanner, isClamAVUnreachableError, type ClamScanClient } from "./clamav-scanner.js";
|
||||
import {
|
||||
ClamAVUnreachableError,
|
||||
createClamAVBufferScanner,
|
||||
createClamAVStorageScanner,
|
||||
isClamAVUnreachableError,
|
||||
type ClamScanClient,
|
||||
} from "./clamav-scanner.js";
|
||||
|
||||
const EICAR_BUFFER = Buffer.from("X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*", "utf8");
|
||||
const CLEAN_BUFFER = Buffer.from("hello world", "utf8");
|
||||
@@ -64,6 +70,17 @@ describe("ClamAVStorageScanner", () => {
|
||||
if (result.verdict === "infected") expect(result.signature).toBe("unknown");
|
||||
});
|
||||
|
||||
test("fails closed when clamd returns an indeterminate verdict", async () => {
|
||||
const client = buildClient({ scanBuffer: vi.fn(async () => ({ isInfected: null })) });
|
||||
const scanner = createClamAVStorageScanner({
|
||||
...baseConfig,
|
||||
loadObject: async () => CLEAN_BUFFER,
|
||||
initClient: async () => client,
|
||||
});
|
||||
|
||||
await expect(scanner.scanObject({ bucket: "uploads", key: "x" })).rejects.toThrow(ClamAVUnreachableError);
|
||||
});
|
||||
|
||||
test("throws ClamAVUnreachableError when isReachable returns false", async () => {
|
||||
const client = buildClient({ isReachable: vi.fn(async () => false) });
|
||||
const scanner = createClamAVStorageScanner({
|
||||
@@ -119,6 +136,18 @@ describe("ClamAVStorageScanner", () => {
|
||||
expect(initClient).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("reuses one client for synchronous pre-upload buffer scans", async () => {
|
||||
const client = buildClient();
|
||||
const initClient = vi.fn(async () => client);
|
||||
const scanner = createClamAVBufferScanner({ ...baseConfig, initClient });
|
||||
|
||||
await expect(scanner.scanBuffer(CLEAN_BUFFER)).resolves.toMatchObject({ verdict: "clean" });
|
||||
await expect(scanner.scanBuffer(CLEAN_BUFFER)).resolves.toMatchObject({ verdict: "clean" });
|
||||
|
||||
expect(initClient).toHaveBeenCalledTimes(1);
|
||||
expect(client.scanBuffer).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("isClamAVUnreachableError narrows arbitrary errors", () => {
|
||||
expect(isClamAVUnreachableError(new ClamAVUnreachableError("nope"))).toBe(true);
|
||||
expect(isClamAVUnreachableError(new Error("regular"))).toBe(false);
|
||||
|
||||
+50
-8
@@ -4,6 +4,7 @@
|
||||
|
||||
/// <reference path="./clamscan.d.ts" />
|
||||
|
||||
import { Readable } from "node:stream";
|
||||
import type { StorageScanner, StorageScannerInput, StorageScannerResult } from "./index.js";
|
||||
|
||||
export interface ClamAVScannerConfig {
|
||||
@@ -34,6 +35,12 @@ export interface ClamAVScannerConfig {
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
export type ClamAVBufferScannerConfig = Omit<ClamAVScannerConfig, "loadObject">;
|
||||
|
||||
export interface ClamAVBufferScanner {
|
||||
scanBuffer(buffer: Buffer): Promise<StorageScannerResult>;
|
||||
}
|
||||
|
||||
export interface ClamScanResult {
|
||||
isInfected: boolean | null;
|
||||
viruses?: readonly string[];
|
||||
@@ -51,6 +58,26 @@ export interface ClamScanClient {
|
||||
const ENGINE = "clamav";
|
||||
|
||||
export function createClamAVStorageScanner(config: ClamAVScannerConfig): StorageScanner {
|
||||
const payloadScanner = createClamAVPayloadScanner(config);
|
||||
|
||||
return {
|
||||
async scanObject(target: StorageScannerInput): Promise<StorageScannerResult> {
|
||||
const payload = await config.loadObject(target);
|
||||
return payloadScanner.scan(payload);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createClamAVBufferScanner(config: ClamAVBufferScannerConfig): ClamAVBufferScanner {
|
||||
const payloadScanner = createClamAVPayloadScanner(config);
|
||||
return {
|
||||
scanBuffer(buffer) {
|
||||
return payloadScanner.scan(buffer);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createClamAVPayloadScanner(config: ClamAVBufferScannerConfig) {
|
||||
const initClient = config.initClient ?? defaultInitClient;
|
||||
let cachedClient: ClamScanClient | undefined;
|
||||
|
||||
@@ -62,19 +89,19 @@ export function createClamAVStorageScanner(config: ClamAVScannerConfig): Storage
|
||||
}
|
||||
|
||||
return {
|
||||
async scanObject(target: StorageScannerInput): Promise<StorageScannerResult> {
|
||||
async scan(payload: Buffer | NodeJS.ReadableStream): Promise<StorageScannerResult> {
|
||||
const client = await getClient();
|
||||
const reachable = await client.isReachable().catch(() => false);
|
||||
if (!reachable) {
|
||||
throw new ClamAVUnreachableError(`clamd at ${config.host}:${config.port} is unreachable`);
|
||||
}
|
||||
|
||||
const payload = await config.loadObject(target);
|
||||
const result = await scanWithClient(client, payload);
|
||||
|
||||
if (result.isInfected) {
|
||||
const signature = result.viruses?.[0] ?? "unknown";
|
||||
return { verdict: "infected", signature, engine: ENGINE };
|
||||
return { verdict: "infected", signature: result.viruses?.[0] ?? "unknown", engine: ENGINE };
|
||||
}
|
||||
if (result.isInfected !== false) {
|
||||
throw new ClamAVUnreachableError("clamd returned an indeterminate scan verdict");
|
||||
}
|
||||
return { verdict: "clean", engine: ENGINE };
|
||||
},
|
||||
@@ -114,7 +141,11 @@ async function defaultInitClient({ host, port, timeoutMs }: { host: string; port
|
||||
return Boolean(status);
|
||||
},
|
||||
async scanBuffer(buffer) {
|
||||
const res = await client.scanBuffer(buffer);
|
||||
// clamscan has no scanBuffer — the daemon-backed client only exposes
|
||||
// ping/scanStream/scanFile/scanDir. Feed the in-memory bytes through
|
||||
// scanStream (clamd INSTREAM), which is what STANDARDS §13 mandates
|
||||
// anyway: no path scan, no shared filesystem with the AV host.
|
||||
const res = await client.scanStream(bufferToStream(buffer));
|
||||
return { isInfected: res.isInfected, viruses: res.viruses };
|
||||
},
|
||||
async scanStream(stream) {
|
||||
@@ -124,11 +155,22 @@ async function defaultInitClient({ host, port, timeoutMs }: { host: string; port
|
||||
};
|
||||
}
|
||||
|
||||
// A plain byte stream, not Readable.from(buffer) — a Buffer iterates as single
|
||||
// integers, so Readable.from would emit numbers instead of the payload.
|
||||
function bufferToStream(buffer: Buffer): NodeJS.ReadableStream {
|
||||
return new Readable({
|
||||
read() {
|
||||
this.push(buffer);
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Subset of the clamscan npm package surface that we actually use. Avoids a
|
||||
// dependency on @types/clamscan and lets the lazy import stay loose.
|
||||
// dependency on @types/clamscan and lets the lazy import stay loose. Keep this
|
||||
// in sync with the real client: it does NOT have scanBuffer.
|
||||
interface ClamScanLib {
|
||||
ping(): Promise<unknown>;
|
||||
scanBuffer(buffer: Buffer): Promise<{ isInfected: boolean | null; viruses?: string[] }>;
|
||||
scanStream(stream: NodeJS.ReadableStream): Promise<{ isInfected: boolean | null; viruses?: string[] }>;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,9 +3,12 @@ export { createMagicBytesScanAdapter, type MagicBytesScanAdapterOptions } from "
|
||||
export { DEFAULT_ALLOW_LIST, resolveAllowList } from "./allow-list.js";
|
||||
export {
|
||||
createClamAVStorageScanner,
|
||||
createClamAVBufferScanner,
|
||||
ClamAVUnreachableError,
|
||||
isClamAVUnreachableError,
|
||||
type ClamAVScannerConfig,
|
||||
type ClamAVBufferScanner,
|
||||
type ClamAVBufferScannerConfig,
|
||||
type ClamScanClient,
|
||||
type ClamScanResult,
|
||||
} from "./clamav-scanner.js";
|
||||
|
||||
+18
-4
@@ -2,16 +2,30 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { DEFAULT_ALLOW_LIST } from "./allow-list.js";
|
||||
import { createMagicBytesScanAdapter } from "./magic-bytes-adapter.js";
|
||||
import type { PreUploadAllowList } from "./pre-upload-types.js";
|
||||
|
||||
const REAL_PNG = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
);
|
||||
const REAL_PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=", "base64");
|
||||
const asUint8 = (buf: Buffer): Uint8Array => new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
||||
|
||||
describe("magic-bytes-adapter filename guard edge cases", () => {
|
||||
test("accepts plain markdown when the context explicitly allows it", async () => {
|
||||
const adapter = createMagicBytesScanAdapter({
|
||||
allowList: {
|
||||
...DEFAULT_ALLOW_LIST,
|
||||
formUpload: [...DEFAULT_ALLOW_LIST.formUpload, "text/markdown"],
|
||||
},
|
||||
});
|
||||
const result = await adapter.scan({
|
||||
buffer: new TextEncoder().encode("# Pilot notes\n"),
|
||||
declaredMimeType: "text/markdown",
|
||||
filename: "pilot.md",
|
||||
context: "formUpload",
|
||||
});
|
||||
expect(result.status).toBe("clean");
|
||||
});
|
||||
|
||||
test("rejects an empty filename before inspecting the buffer", async () => {
|
||||
const adapter = createMagicBytesScanAdapter();
|
||||
const result = await adapter.scan({
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ const FILENAME_BAD_PATTERN = /[\x00-\x1f\x7f]|\.\.[\\/]/;
|
||||
const FILENAME_MAX_LENGTH = 255;
|
||||
|
||||
const SVG_SIGNATURE = /^<\?xml[\s\S]*?<svg\b|^<svg\b/i;
|
||||
const TEXT_PLAIN_FALLBACK_MIMES = new Set(["text/plain", "text/csv"]);
|
||||
const TEXT_PLAIN_FALLBACK_MIMES = new Set(["text/plain", "text/csv", "text/markdown"]);
|
||||
|
||||
export interface MagicBytesScanAdapterOptions {
|
||||
/**
|
||||
|
||||
@@ -22,6 +22,7 @@ export type PreUploadScanReason =
|
||||
| "mime_undetectable"
|
||||
| "filename_invalid"
|
||||
| "scanner_disabled"
|
||||
| "malware_detected"
|
||||
| "scanner_error";
|
||||
|
||||
export interface PreUploadScanInput {
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createHash, timingSafeEqual } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
|
||||
|
||||
import { createS3SseCObjectReader, type S3SseCGetObjectInput } from "./sse-c.js";
|
||||
|
||||
const CUSTOMER_KEY = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
|
||||
const config = {
|
||||
endpoint: "https://storage.example.test",
|
||||
region: "eu-central-1",
|
||||
accessKeyId: "access-key",
|
||||
secretAccessKey: "secret-key",
|
||||
customerKeyBase64: CUSTOMER_KEY,
|
||||
};
|
||||
|
||||
function body(chunks: Uint8Array[], cancel = vi.fn()) {
|
||||
return {
|
||||
cancel,
|
||||
transformToWebStream: () =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
for (const chunk of chunks) controller.enqueue(chunk);
|
||||
controller.close();
|
||||
},
|
||||
cancel,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
describe("S3 SSE-C object reader", () => {
|
||||
it("sends the customer-key contract and streams a size-matched object", async () => {
|
||||
const sendGetObject = vi.fn(async (_input: S3SseCGetObjectInput) => ({
|
||||
Body: body([new Uint8Array([1, 2]), new Uint8Array([3])]),
|
||||
ContentLength: 3,
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
}));
|
||||
const reader = createS3SseCObjectReader(config, { sendGetObject });
|
||||
|
||||
const result = await reader.readObject({ bucket: "exports", key: "tenants/t1/gdpr/job.zip", maxBytes: 10, expectedSizeBytes: 3 });
|
||||
await expect(new Response(result.body).arrayBuffer()).resolves.toEqual(new Uint8Array([1, 2, 3]).buffer);
|
||||
expect(result.contentLength).toBe(3);
|
||||
expect(sendGetObject).toHaveBeenCalledWith({
|
||||
Bucket: "exports",
|
||||
Key: "tenants/t1/gdpr/job.zip",
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: CUSTOMER_KEY,
|
||||
});
|
||||
expect(sendGetObject.mock.calls[0]?.[0]).not.toHaveProperty("ServerSideEncryption");
|
||||
});
|
||||
|
||||
it("fails before streaming when the provider length exceeds the bound", async () => {
|
||||
const destroy = vi.fn();
|
||||
const reader = createS3SseCObjectReader(config, {
|
||||
sendGetObject: vi.fn(async () => ({ Body: { ...body([]), destroy }, ContentLength: 11, SSECustomerAlgorithm: "AES256" })),
|
||||
});
|
||||
|
||||
await expect(reader.readObject({ bucket: "exports", key: "too-large", maxBytes: 10 })).rejects.toMatchObject({ code: "read_failed" });
|
||||
expect(destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("bounds unknown-length streams and propagates browser cancellation", async () => {
|
||||
const oversized = createS3SseCObjectReader(config, {
|
||||
sendGetObject: vi.fn(async () => ({ Body: body([new Uint8Array(6), new Uint8Array(6)]), SSECustomerAlgorithm: "AES256" })),
|
||||
});
|
||||
const result = await oversized.readObject({ bucket: "exports", key: "bounded", maxBytes: 10 });
|
||||
await expect(new Response(result.body).arrayBuffer()).rejects.toMatchObject({ code: "read_failed" });
|
||||
|
||||
const browserCancel = vi.fn();
|
||||
const cancellable = createS3SseCObjectReader(config, {
|
||||
sendGetObject: vi.fn(async () => ({
|
||||
Body: {
|
||||
transformToWebStream: () =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
controller.enqueue(new Uint8Array([1]));
|
||||
},
|
||||
cancel: browserCancel,
|
||||
}),
|
||||
},
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
})),
|
||||
});
|
||||
const cancellableResult = await cancellable.readObject({ bucket: "exports", key: "cancel", maxBytes: 10 });
|
||||
await cancellableResult.body.cancel("browser-abort");
|
||||
expect(browserCancel).toHaveBeenCalledWith("browser-abort");
|
||||
});
|
||||
|
||||
it("destroys and rejects a body when the provider does not acknowledge SSE-C", async () => {
|
||||
const destroy = vi.fn();
|
||||
const reader = createS3SseCObjectReader(config, {
|
||||
sendGetObject: vi.fn(async () => ({ Body: { ...body([]), destroy }, ContentLength: 0 })),
|
||||
});
|
||||
|
||||
await expect(reader.readObject({ bucket: "exports", key: "legacy-plaintext", maxBytes: 10 })).rejects.toMatchObject({ code: "read_failed" });
|
||||
expect(destroy).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("serializes the exact SSE-C GET and PUT wire headers without an SSE-S3 fallback", async () => {
|
||||
const capturedHeaders: Array<Record<string, string>> = [];
|
||||
const requestHandler = {
|
||||
handle: vi.fn(async (request: { method?: string; headers?: Record<string, string> }) => {
|
||||
capturedHeaders.push({ ...(request.headers ?? {}) });
|
||||
return {
|
||||
response: {
|
||||
statusCode: 200,
|
||||
headers: { "x-amz-server-side-encryption-customer-algorithm": "AES256", "content-length": "1" },
|
||||
...(request.method === "GET" ? { body: Readable.from([Buffer.from("x")]) } : {}),
|
||||
},
|
||||
};
|
||||
}),
|
||||
};
|
||||
const client = new S3Client({
|
||||
endpoint: "https://storage.example.test",
|
||||
region: "eu-central-1",
|
||||
credentials: { accessKeyId: "test-access", secretAccessKey: "test-secret" },
|
||||
requestHandler,
|
||||
});
|
||||
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: "exports",
|
||||
Key: "tenants/t1/gdpr/job.zip",
|
||||
Body: new Uint8Array([1]),
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: CUSTOMER_KEY,
|
||||
})
|
||||
);
|
||||
const getResult = await client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: "exports",
|
||||
Key: "tenants/t1/gdpr/job.zip",
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: CUSTOMER_KEY,
|
||||
})
|
||||
);
|
||||
(getResult.Body as { destroy?: () => void } | undefined)?.destroy?.();
|
||||
|
||||
expect(capturedHeaders).toHaveLength(2);
|
||||
for (const headers of capturedHeaders) {
|
||||
const algorithm = headers["x-amz-server-side-encryption-customer-algorithm"];
|
||||
const serializedKey = headers["x-amz-server-side-encryption-customer-key"];
|
||||
const serializedMd5 = headers["x-amz-server-side-encryption-customer-key-md5"];
|
||||
const expectedMd5 = createHash("md5").update(Buffer.from(CUSTOMER_KEY, "base64")).digest();
|
||||
const actualMd5 = typeof serializedMd5 === "string" ? Buffer.from(serializedMd5, "base64") : Buffer.alloc(0);
|
||||
expect(algorithm).toBe("AES256");
|
||||
expect(typeof serializedKey === "string" && Buffer.from(serializedKey, "base64").byteLength === 32).toBe(true);
|
||||
expect(actualMd5.byteLength === expectedMd5.byteLength && timingSafeEqual(actualMd5, expectedMd5)).toBe(true);
|
||||
expect(Object.keys(headers)).not.toContain("x-amz-server-side-encryption");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,220 @@
|
||||
import { Buffer } from "node:buffer";
|
||||
import type { GetObjectCommandOutput, S3Client } from "@aws-sdk/client-s3";
|
||||
|
||||
const AES_256_KEY_BYTES = 32;
|
||||
const CANONICAL_BASE64_32_BYTES = /^[A-Za-z0-9+/]{43}=$/u;
|
||||
|
||||
export type S3SseCReaderConfig = {
|
||||
endpoint: string;
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
sessionToken?: string;
|
||||
forcePathStyle?: boolean;
|
||||
customerKeyBase64: string;
|
||||
};
|
||||
|
||||
export type S3SseCReadInput = {
|
||||
bucket: string;
|
||||
key: string;
|
||||
maxBytes: number;
|
||||
expectedSizeBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
export type S3SseCGetObjectInput = {
|
||||
Bucket: string;
|
||||
Key: string;
|
||||
SSECustomerAlgorithm: "AES256";
|
||||
SSECustomerKey: string;
|
||||
};
|
||||
|
||||
type S3SseCBody = {
|
||||
transformToWebStream(): ReadableStream<Uint8Array>;
|
||||
destroy?: () => void;
|
||||
};
|
||||
|
||||
type S3SseCGetObjectOutput = Pick<GetObjectCommandOutput, "ContentLength" | "SSECustomerAlgorithm"> & {
|
||||
Body?: S3SseCBody;
|
||||
};
|
||||
|
||||
export type S3SseCReaderDependencies = {
|
||||
sendGetObject?: (input: S3SseCGetObjectInput, options?: { abortSignal?: AbortSignal }) => Promise<S3SseCGetObjectOutput>;
|
||||
};
|
||||
|
||||
export type S3SseCObjectReader = {
|
||||
readObject(input: S3SseCReadInput): Promise<{ body: ReadableStream<Uint8Array>; contentLength?: number }>;
|
||||
};
|
||||
|
||||
export class S3SseCError extends Error {
|
||||
readonly code: "config_invalid" | "read_failed";
|
||||
readonly upstreamCode?: string;
|
||||
|
||||
constructor(code: S3SseCError["code"], message: string, details: { upstreamCode?: string } = {}) {
|
||||
super(message);
|
||||
this.name = "S3SseCError";
|
||||
this.code = code;
|
||||
this.upstreamCode = details.upstreamCode;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a server-only reader for SSE-C objects. The customer key is sent only
|
||||
* to the object store over the signed request and is never returned to callers.
|
||||
*/
|
||||
export function createS3SseCObjectReader(config: S3SseCReaderConfig, dependencies: S3SseCReaderDependencies = {}): S3SseCObjectReader {
|
||||
const customerKey = requireCanonicalCustomerKey(config.customerKeyBase64);
|
||||
const missingFields = (["endpoint", "region", "accessKeyId", "secretAccessKey"] as const).filter((field) => !config[field]?.trim());
|
||||
if (missingFields.length > 0) {
|
||||
throw new S3SseCError("config_invalid", `Missing S3 SSE-C reader configuration: ${missingFields.join(", ")}`);
|
||||
}
|
||||
|
||||
const sendGetObject = dependencies.sendGetObject ?? createAwsGetObjectSender(config);
|
||||
return {
|
||||
async readObject(input) {
|
||||
assertReadInput(input);
|
||||
let response: S3SseCGetObjectOutput;
|
||||
try {
|
||||
const request = {
|
||||
Bucket: input.bucket,
|
||||
Key: input.key,
|
||||
SSECustomerAlgorithm: "AES256" as const,
|
||||
SSECustomerKey: customerKey,
|
||||
};
|
||||
response = input.signal ? await sendGetObject(request, { abortSignal: input.signal }) : await sendGetObject(request);
|
||||
} catch (cause) {
|
||||
throw readFailure(input, "Failed to read encrypted GDPR export object.", cause);
|
||||
}
|
||||
|
||||
const body = response.Body;
|
||||
if (response.SSECustomerAlgorithm !== "AES256") {
|
||||
body?.destroy?.();
|
||||
throw readFailure(input, "Object storage did not acknowledge SSE-C encryption for the GDPR export object.");
|
||||
}
|
||||
if (!body || typeof body.transformToWebStream !== "function") {
|
||||
throw readFailure(input, "Encrypted GDPR export object returned no readable body.");
|
||||
}
|
||||
if (response.ContentLength !== undefined && response.ContentLength > input.maxBytes) {
|
||||
body.destroy?.();
|
||||
throw readFailure(input, "Encrypted GDPR export object exceeds the download limit.");
|
||||
}
|
||||
if (input.expectedSizeBytes !== undefined && response.ContentLength !== undefined && response.ContentLength !== input.expectedSizeBytes) {
|
||||
body.destroy?.();
|
||||
throw readFailure(input, "Encrypted GDPR export object size does not match its database record.");
|
||||
}
|
||||
|
||||
let source: ReadableStream<Uint8Array>;
|
||||
try {
|
||||
source = body.transformToWebStream();
|
||||
} catch (cause) {
|
||||
throw readFailure(input, "Encrypted GDPR export object body could not be read.", cause);
|
||||
}
|
||||
return {
|
||||
body: createBoundedStream(source, input),
|
||||
...(response.ContentLength !== undefined
|
||||
? { contentLength: response.ContentLength }
|
||||
: input.expectedSizeBytes !== undefined
|
||||
? { contentLength: input.expectedSizeBytes }
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function assertS3SseCAcknowledgement(output: { SSECustomerAlgorithm?: string } | undefined): void {
|
||||
if (output?.SSECustomerAlgorithm !== "AES256") {
|
||||
throw new S3SseCError("read_failed", "Object storage did not acknowledge SSE-C encryption for the GDPR export object.");
|
||||
}
|
||||
}
|
||||
|
||||
function createAwsGetObjectSender(
|
||||
config: S3SseCReaderConfig
|
||||
): (input: S3SseCGetObjectInput, options?: { abortSignal?: AbortSignal }) => Promise<S3SseCGetObjectOutput> {
|
||||
let clientPromise: Promise<S3Client> | undefined;
|
||||
return async (input, options) => {
|
||||
const aws = await import("@aws-sdk/client-s3");
|
||||
clientPromise ??= Promise.resolve(
|
||||
new aws.S3Client({
|
||||
endpoint: config.endpoint,
|
||||
region: config.region,
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
credentials: {
|
||||
accessKeyId: config.accessKeyId,
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
...(config.sessionToken ? { sessionToken: config.sessionToken } : {}),
|
||||
},
|
||||
})
|
||||
);
|
||||
const client = await clientPromise;
|
||||
return (await client.send(new aws.GetObjectCommand(input), options)) as S3SseCGetObjectOutput;
|
||||
};
|
||||
}
|
||||
|
||||
function requireCanonicalCustomerKey(value: string): string {
|
||||
const normalized = value.trim();
|
||||
if (!CANONICAL_BASE64_32_BYTES.test(normalized)) {
|
||||
throw new S3SseCError("config_invalid", "GDPR export SSE-C key must encode exactly 32 random bytes.");
|
||||
}
|
||||
const decoded = Buffer.from(normalized, "base64");
|
||||
if (decoded.byteLength !== AES_256_KEY_BYTES || decoded.toString("base64") !== normalized) {
|
||||
throw new S3SseCError("config_invalid", "GDPR export SSE-C key must encode exactly 32 random bytes.");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function assertReadInput(input: S3SseCReadInput): void {
|
||||
if (!input.bucket.trim() || !input.key.trim()) {
|
||||
throw readFailure(input, "Encrypted GDPR export object location is invalid.");
|
||||
}
|
||||
if (!Number.isSafeInteger(input.maxBytes) || input.maxBytes <= 0) {
|
||||
throw readFailure(input, "Encrypted GDPR export download limit is invalid.");
|
||||
}
|
||||
if (
|
||||
input.expectedSizeBytes !== undefined &&
|
||||
(!Number.isSafeInteger(input.expectedSizeBytes) || input.expectedSizeBytes < 0 || input.expectedSizeBytes > input.maxBytes)
|
||||
) {
|
||||
throw readFailure(input, "Encrypted GDPR export object size is invalid.");
|
||||
}
|
||||
}
|
||||
|
||||
function readFailure(_input: Pick<S3SseCReadInput, "bucket" | "key">, message: string, cause?: unknown): S3SseCError {
|
||||
return new S3SseCError("read_failed", message, { upstreamCode: safeUpstreamCode(cause) });
|
||||
}
|
||||
|
||||
function safeUpstreamCode(error: unknown): string | undefined {
|
||||
const candidate = error as { code?: unknown; name?: unknown } | null;
|
||||
const value = typeof candidate?.code === "string" ? candidate.code : typeof candidate?.name === "string" ? candidate.name : undefined;
|
||||
return value && /^[A-Za-z0-9._-]{1,64}$/u.test(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function createBoundedStream(source: ReadableStream<Uint8Array>, input: S3SseCReadInput): ReadableStream<Uint8Array> {
|
||||
const reader = source.getReader();
|
||||
let bytesRead = 0;
|
||||
return new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
try {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) {
|
||||
if (input.expectedSizeBytes !== undefined && bytesRead !== input.expectedSizeBytes) {
|
||||
controller.error(readFailure(input, "Encrypted GDPR export object size does not match its database record."));
|
||||
return;
|
||||
}
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
bytesRead += chunk.value.byteLength;
|
||||
if (bytesRead > input.maxBytes || (input.expectedSizeBytes !== undefined && bytesRead > input.expectedSizeBytes)) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
controller.error(readFailure(input, "Encrypted GDPR export object exceeds its permitted size."));
|
||||
return;
|
||||
}
|
||||
controller.enqueue(chunk.value);
|
||||
} catch (cause) {
|
||||
controller.error(readFailure(input, "Encrypted GDPR export object body could not be read.", cause));
|
||||
}
|
||||
},
|
||||
async cancel(reason) {
|
||||
await reader.cancel(reason).catch(() => undefined);
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user