Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+121
@@ -0,0 +1,121 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
loadConfig: vi.fn(),
|
||||
createReader: vi.fn(),
|
||||
readObject: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@hivegrid/config", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("@hivegrid/config")>()),
|
||||
loadRequiredWebConfig: () => mocks.loadConfig(),
|
||||
}));
|
||||
vi.mock("@hivegrid/storage", () => ({ createS3SseCObjectReader: (...args: unknown[]) => mocks.createReader(...args) }));
|
||||
|
||||
import { createGdprExportDownloadResponse, readGdprExportObject } from "./gdprExportDownload.server";
|
||||
|
||||
describe("GDPR export server-side download", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.loadConfig.mockReturnValue({
|
||||
storage: {
|
||||
endpoint: "https://storage.example.test",
|
||||
region: "eu-central-1",
|
||||
bucket: "private-exports",
|
||||
accessKeyId: "access-key",
|
||||
secretAccessKey: "secret-key",
|
||||
gdprExportSseCKey: "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=",
|
||||
forcePathStyle: "false",
|
||||
},
|
||||
});
|
||||
mocks.createReader.mockReturnValue({ readObject: mocks.readObject });
|
||||
mocks.readObject.mockResolvedValue({
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("payload"));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
contentLength: 7,
|
||||
});
|
||||
});
|
||||
|
||||
it("builds a server-only SSE-C reader and enforces the configured bucket", async () => {
|
||||
await expect(
|
||||
readGdprExportObject({ bucket: "private-exports", key: "tenants/t1/gdpr/job.zip", maxBytes: 32, expectedSizeBytes: 7 })
|
||||
).resolves.toMatchObject({ contentLength: 7 });
|
||||
expect(mocks.createReader).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
endpoint: "https://storage.example.test",
|
||||
customerKeyBase64: "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=",
|
||||
})
|
||||
);
|
||||
expect(mocks.readObject).toHaveBeenCalledWith({
|
||||
bucket: "private-exports",
|
||||
key: "tenants/t1/gdpr/job.zip",
|
||||
maxBytes: 32,
|
||||
expectedSizeBytes: 7,
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
|
||||
await expect(readGdprExportObject({ bucket: "foreign", key: "tenants/t1/gdpr/job.zip", maxBytes: 32 })).rejects.toThrow(/configured private bucket/u);
|
||||
expect(mocks.readObject).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("propagates a provider stream failure without buffering the object in the web heap", async () => {
|
||||
mocks.readObject.mockResolvedValueOnce({
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("partial"));
|
||||
controller.error(new Error("provider stream failed"));
|
||||
},
|
||||
}),
|
||||
contentLength: 7,
|
||||
});
|
||||
|
||||
const object = await readGdprExportObject({ bucket: "private-exports", key: "gdpr-exports/privacy/tenants/t1/job/attempt.zip", maxBytes: 32 });
|
||||
await expect(new Response(object.body).arrayBuffer()).rejects.toThrow(/provider stream failed/u);
|
||||
});
|
||||
|
||||
it("propagates a request that was already aborted before listener registration", async () => {
|
||||
const requestAbort = new AbortController();
|
||||
const reason = new Error("client disconnected before storage read");
|
||||
requestAbort.abort(reason);
|
||||
|
||||
await expect(
|
||||
readGdprExportObject({
|
||||
bucket: "private-exports",
|
||||
key: "tenants/t1/gdpr/job.zip",
|
||||
maxBytes: 32,
|
||||
signal: requestAbort.signal,
|
||||
}),
|
||||
).rejects.toBe(reason);
|
||||
const storageSignal = mocks.readObject.mock.calls[0]?.[0]?.signal as AbortSignal | undefined;
|
||||
expect(storageSignal?.aborted).toBe(true);
|
||||
expect(storageSignal?.reason).toBe(reason);
|
||||
});
|
||||
|
||||
it("returns private attachment headers without a redirect", async () => {
|
||||
const response = createGdprExportDownloadResponse({
|
||||
object: {
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode("payload"));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
contentLength: 7,
|
||||
},
|
||||
filename: 'export"\r\n.zip',
|
||||
contentType: "application/zip",
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("Location")).toBeNull();
|
||||
expect(response.headers.get("Content-Disposition")).toBe('attachment; filename="export.zip"');
|
||||
expect(response.headers.get("Content-Length")).toBe("7");
|
||||
expect(response.headers.get("Cache-Control")).toBe("private, no-store");
|
||||
expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff");
|
||||
await expect(response.text()).resolves.toBe("payload");
|
||||
});
|
||||
});
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
import { loadRequiredWebConfig, PORTAL_GDPR_EXPORT_MAX_BYTES, PRIVACY_GDPR_EXPORT_MAX_BYTES } from "@hivegrid/config";
|
||||
import { createS3SseCObjectReader, type S3SseCObjectReader, type S3SseCReaderConfig } from "@hivegrid/storage";
|
||||
|
||||
export const PORTAL_GDPR_EXPORT_DOWNLOAD_MAX_BYTES = PORTAL_GDPR_EXPORT_MAX_BYTES;
|
||||
export const PRIVACY_GDPR_EXPORT_DOWNLOAD_MAX_BYTES = PRIVACY_GDPR_EXPORT_MAX_BYTES;
|
||||
|
||||
export type GdprExportObject = {
|
||||
body: ReadableStream<Uint8Array>;
|
||||
contentLength: number;
|
||||
};
|
||||
|
||||
const GDPR_EXPORT_READ_TIMEOUT_MS = 60_000;
|
||||
let cachedReader: { config: S3SseCReaderConfig; reader: S3SseCObjectReader } | undefined;
|
||||
|
||||
export async function readGdprExportObject({
|
||||
bucket,
|
||||
key,
|
||||
maxBytes,
|
||||
expectedSizeBytes,
|
||||
signal,
|
||||
}: {
|
||||
bucket: string;
|
||||
key: string;
|
||||
maxBytes: number;
|
||||
expectedSizeBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<GdprExportObject> {
|
||||
const { storage } = loadRequiredWebConfig();
|
||||
if (!storage.bucket || bucket !== storage.bucket) {
|
||||
throw new Error("GDPR export bucket does not match the configured private bucket.");
|
||||
}
|
||||
|
||||
const readerConfig: S3SseCReaderConfig = {
|
||||
endpoint: storage.endpoint ?? "",
|
||||
region: storage.region ?? "",
|
||||
accessKeyId: storage.accessKeyId ?? "",
|
||||
secretAccessKey: storage.secretAccessKey ?? "",
|
||||
...(storage.sessionToken ? { sessionToken: storage.sessionToken } : {}),
|
||||
forcePathStyle: storage.forcePathStyle?.toLowerCase() === "true",
|
||||
customerKeyBase64: storage.gdprExportSseCKey ?? "",
|
||||
};
|
||||
const reader = getCachedReader(readerConfig);
|
||||
const timeoutController = new AbortController();
|
||||
const abortFromRequest = () => timeoutController.abort(signal?.reason);
|
||||
if (signal?.aborted) abortFromRequest();
|
||||
else signal?.addEventListener("abort", abortFromRequest, { once: true });
|
||||
const timeout = setTimeout(() => timeoutController.abort(new Error("GDPR export object read timed out.")), GDPR_EXPORT_READ_TIMEOUT_MS);
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
signal?.removeEventListener("abort", abortFromRequest);
|
||||
};
|
||||
let handedOff = false;
|
||||
try {
|
||||
const object = await reader.readObject({
|
||||
bucket,
|
||||
key,
|
||||
maxBytes,
|
||||
...(expectedSizeBytes !== undefined ? { expectedSizeBytes } : {}),
|
||||
signal: timeoutController.signal,
|
||||
});
|
||||
if (timeoutController.signal.aborted) {
|
||||
await object.body.cancel(timeoutController.signal.reason).catch(() => undefined);
|
||||
throw timeoutController.signal.reason instanceof Error
|
||||
? timeoutController.signal.reason
|
||||
: new Error("GDPR export object read aborted.");
|
||||
}
|
||||
if (object.contentLength === undefined || !Number.isSafeInteger(object.contentLength) || object.contentLength < 0 || object.contentLength > maxBytes) {
|
||||
await object.body.cancel().catch(() => undefined);
|
||||
throw new Error("GDPR export object returned no valid bounded length.");
|
||||
}
|
||||
const body = keepAbortAndTimeoutUntilStreamFinished(object.body, timeoutController.signal, cleanup);
|
||||
handedOff = true;
|
||||
return { body, contentLength: object.contentLength };
|
||||
} finally {
|
||||
if (!handedOff) cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
export function createGdprExportDownloadResponse({
|
||||
object,
|
||||
filename,
|
||||
contentType,
|
||||
headers: initialHeaders,
|
||||
}: {
|
||||
object: GdprExportObject;
|
||||
filename: string;
|
||||
contentType: "application/json" | "application/zip";
|
||||
headers?: HeadersInit;
|
||||
}): Response {
|
||||
const headers = new Headers(initialHeaders);
|
||||
headers.set("Cache-Control", "private, no-store");
|
||||
headers.set("Content-Disposition", `attachment; filename="${sanitizeFilename(filename)}"`);
|
||||
headers.set("Content-Type", contentType);
|
||||
headers.set("X-Content-Type-Options", "nosniff");
|
||||
if (object.contentLength !== undefined) headers.set("Content-Length", String(object.contentLength));
|
||||
return new Response(object.body, { status: 200, headers });
|
||||
}
|
||||
|
||||
function getCachedReader(config: S3SseCReaderConfig): S3SseCObjectReader {
|
||||
if (!cachedReader || !sameReaderConfig(cachedReader.config, config)) {
|
||||
cachedReader = { config, reader: createS3SseCObjectReader(config) };
|
||||
}
|
||||
return cachedReader.reader;
|
||||
}
|
||||
|
||||
function sameReaderConfig(left: S3SseCReaderConfig, right: S3SseCReaderConfig): boolean {
|
||||
return (
|
||||
left.endpoint === right.endpoint &&
|
||||
left.region === right.region &&
|
||||
left.accessKeyId === right.accessKeyId &&
|
||||
left.secretAccessKey === right.secretAccessKey &&
|
||||
left.sessionToken === right.sessionToken &&
|
||||
left.forcePathStyle === right.forcePathStyle &&
|
||||
left.customerKeyBase64 === right.customerKeyBase64
|
||||
);
|
||||
}
|
||||
|
||||
function keepAbortAndTimeoutUntilStreamFinished(
|
||||
stream: ReadableStream<Uint8Array>,
|
||||
signal: AbortSignal,
|
||||
cleanup: () => void
|
||||
): ReadableStream<Uint8Array> {
|
||||
const reader = stream.getReader();
|
||||
let finished = false;
|
||||
const finish = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
cleanup();
|
||||
};
|
||||
const abort = () => {
|
||||
void reader.cancel(signal.reason).catch(() => undefined);
|
||||
finish();
|
||||
};
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
return new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
try {
|
||||
if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("GDPR export object read aborted.");
|
||||
const chunk = await reader.read();
|
||||
if (signal.aborted) throw signal.reason instanceof Error ? signal.reason : new Error("GDPR export object read aborted.");
|
||||
if (chunk.done) {
|
||||
finish();
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(chunk.value);
|
||||
} catch (error) {
|
||||
finish();
|
||||
controller.error(error);
|
||||
}
|
||||
},
|
||||
async cancel(reason) {
|
||||
try {
|
||||
await reader.cancel(reason);
|
||||
} finally {
|
||||
finish();
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function sanitizeFilename(filename: string): string {
|
||||
return filename.replace(/["\\\r\n]/gu, "").slice(0, 200);
|
||||
}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createSharedStore, MemoryAdapter, type SharedStore } from "@hivegrid/shared-store";
|
||||
|
||||
import {
|
||||
__resetGdprExportDownloadGateStoreForTests,
|
||||
__setGdprExportDownloadGateStoreForTests,
|
||||
acquireGdprExportDownloadGate,
|
||||
GDPR_EXPORT_DOWNLOAD_LEASE_RENEW_SECONDS,
|
||||
GDPR_EXPORT_DOWNLOAD_LEASE_TTL_SECONDS,
|
||||
GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_MAX,
|
||||
holdGdprExportDownloadGateUntilResponseFinished,
|
||||
} from "./gdprExportDownloadGate.server";
|
||||
|
||||
const secret = "gdpr-download-gate-secret-with-at-least-32-bytes";
|
||||
const base = {
|
||||
scope: "portal",
|
||||
tenantId: "tenant-a1",
|
||||
principalId: "portal-user-a",
|
||||
artifactId: "export-a1",
|
||||
secret,
|
||||
} as const;
|
||||
|
||||
let store: SharedStore;
|
||||
|
||||
describe("GDPR export distributed download gate", () => {
|
||||
beforeEach(() => {
|
||||
store = createSharedStore({ namespace: "gdpr-download-gate-test", adapter: new MemoryAdapter() });
|
||||
__setGdprExportDownloadGateStoreForTests(store);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
__resetGdprExportDownloadGateStoreForTests();
|
||||
});
|
||||
|
||||
it("allows only one active lease per principal and per artifact", async () => {
|
||||
const first = await acquireGdprExportDownloadGate(base);
|
||||
expect(first).not.toBeNull();
|
||||
|
||||
await expect(acquireGdprExportDownloadGate({ ...base, artifactId: "export-b1" })).resolves.toBeNull();
|
||||
await expect(acquireGdprExportDownloadGate({ ...base, principalId: "portal-user-b" })).resolves.toBeNull();
|
||||
const independent = await acquireGdprExportDownloadGate({
|
||||
...base,
|
||||
principalId: "portal-user-b",
|
||||
artifactId: "export-b1",
|
||||
});
|
||||
expect(independent).not.toBeNull();
|
||||
|
||||
await first?.release();
|
||||
const afterRelease = await acquireGdprExportDownloadGate(base);
|
||||
expect(afterRelease).not.toBeNull();
|
||||
await independent?.release();
|
||||
await afterRelease?.release();
|
||||
});
|
||||
|
||||
it("does not expose tenant, principal or artifact identifiers in shared-store keys", async () => {
|
||||
const lease = await acquireGdprExportDownloadGate(base);
|
||||
const keys: string[] = [];
|
||||
for await (const key of store.scan("*")) keys.push(key);
|
||||
|
||||
expect(keys).toHaveLength(4);
|
||||
expect(keys.join("\n")).not.toContain(base.tenantId);
|
||||
expect(keys.join("\n")).not.toContain(base.principalId);
|
||||
expect(keys.join("\n")).not.toContain(base.artifactId);
|
||||
await lease?.release();
|
||||
});
|
||||
|
||||
it("lets an expired owner release without deleting a newer lease", async () => {
|
||||
let now = 1_000;
|
||||
store = createSharedStore({
|
||||
namespace: "gdpr-download-gate-expiry-test",
|
||||
adapter: new MemoryAdapter({ now: () => now }),
|
||||
});
|
||||
__setGdprExportDownloadGateStoreForTests(store);
|
||||
const expiredOwner = await acquireGdprExportDownloadGate(base);
|
||||
expect(expiredOwner).not.toBeNull();
|
||||
|
||||
now += GDPR_EXPORT_DOWNLOAD_LEASE_TTL_SECONDS * 1_000 + 1;
|
||||
const newOwner = await acquireGdprExportDownloadGate(base);
|
||||
expect(newOwner).not.toBeNull();
|
||||
await expiredOwner?.release();
|
||||
|
||||
await expect(acquireGdprExportDownloadGate(base)).resolves.toBeNull();
|
||||
await newOwner?.release();
|
||||
const afterRelease = await acquireGdprExportDownloadGate(base);
|
||||
expect(afterRelease).not.toBeNull();
|
||||
await afterRelease?.release();
|
||||
});
|
||||
|
||||
it("rate-limits quick sequential reads per principal and artifact", async () => {
|
||||
for (let index = 0; index < GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_MAX; index += 1) {
|
||||
const lease = await acquireGdprExportDownloadGate(base);
|
||||
expect(lease).not.toBeNull();
|
||||
await lease?.release();
|
||||
}
|
||||
await expect(acquireGdprExportDownloadGate(base)).resolves.toBeNull();
|
||||
|
||||
const independent = await acquireGdprExportDownloadGate({
|
||||
...base,
|
||||
principalId: "portal-user-b",
|
||||
artifactId: "export-b1",
|
||||
});
|
||||
expect(independent).not.toBeNull();
|
||||
await independent?.release();
|
||||
});
|
||||
|
||||
it("does not allocate attacker-controlled artifact rate keys after the principal limit is exhausted", async () => {
|
||||
for (let index = 0; index < GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_MAX; index += 1) {
|
||||
const lease = await acquireGdprExportDownloadGate(base);
|
||||
expect(lease).not.toBeNull();
|
||||
await lease?.release();
|
||||
}
|
||||
const keysBefore: string[] = [];
|
||||
for await (const key of store.scan("*")) keysBefore.push(key);
|
||||
|
||||
for (let index = 0; index < 20; index += 1) {
|
||||
await expect(acquireGdprExportDownloadGate({ ...base, artifactId: `export-random-${index}` })).resolves.toBeNull();
|
||||
}
|
||||
|
||||
const keysAfter: string[] = [];
|
||||
for await (const key of store.scan("*")) keysAfter.push(key);
|
||||
expect(keysAfter.sort()).toEqual(keysBefore.sort());
|
||||
});
|
||||
|
||||
it("renews an active response lease before its crash-recovery TTL", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
store = createSharedStore({
|
||||
namespace: "gdpr-download-gate-renew-test",
|
||||
adapter: new MemoryAdapter({ now: () => Date.now() }),
|
||||
});
|
||||
__setGdprExportDownloadGateStoreForTests(store);
|
||||
const lease = await acquireGdprExportDownloadGate(base);
|
||||
expect(lease).not.toBeNull();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(GDPR_EXPORT_DOWNLOAD_LEASE_RENEW_SECONDS * 1_000);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
(GDPR_EXPORT_DOWNLOAD_LEASE_TTL_SECONDS - GDPR_EXPORT_DOWNLOAD_LEASE_RENEW_SECONDS + 1) * 1_000,
|
||||
);
|
||||
await expect(acquireGdprExportDownloadGate(base)).resolves.toBeNull();
|
||||
await lease?.release();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails closed for weak identifiers, weak secrets and shared-store failures", async () => {
|
||||
await expect(acquireGdprExportDownloadGate({ ...base, artifactId: "bad" })).resolves.toBeNull();
|
||||
await expect(acquireGdprExportDownloadGate({ ...base, secret: "short" })).resolves.toBeNull();
|
||||
|
||||
vi.spyOn(store, "eval").mockRejectedValueOnce(new Error("redis unavailable"));
|
||||
await expect(acquireGdprExportDownloadGate(base)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it("holds the lease until the response body finishes or is cancelled", async () => {
|
||||
const released = vi.fn().mockResolvedValue(undefined);
|
||||
const response = holdGdprExportDownloadGateUntilResponseFinished(new Response("export-body"), { release: released });
|
||||
expect(released).not.toHaveBeenCalled();
|
||||
await expect(response.text()).resolves.toBe("export-body");
|
||||
expect(released).toHaveBeenCalledTimes(1);
|
||||
|
||||
const cancelled = vi.fn().mockResolvedValue(undefined);
|
||||
const cancelledResponse = holdGdprExportDownloadGateUntilResponseFinished(new Response("cancelled-body"), { release: cancelled });
|
||||
await cancelledResponse.body?.cancel("client disconnected");
|
||||
expect(cancelled).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
+284
@@ -0,0 +1,284 @@
|
||||
import { createHmac, randomUUID } from "node:crypto";
|
||||
|
||||
import { createFixedWindowRateLimiter, MemoryAdapter, type SharedStore } from "@hivegrid/shared-store";
|
||||
|
||||
import { getSharedStore } from "~/shared-store.server";
|
||||
import { createScopedLogger } from "~/utils/logger.server";
|
||||
|
||||
export type GdprExportDownloadGateScope = "portal" | "privacy";
|
||||
|
||||
export type GdprExportDownloadGateLease = {
|
||||
/**
|
||||
* Releases both distributed lease keys when this caller still owns them.
|
||||
* Store failures are logged and left to the bounded lease TTL.
|
||||
*/
|
||||
release(): Promise<void>;
|
||||
};
|
||||
|
||||
export type AcquireGdprExportDownloadGateInput = {
|
||||
readonly scope: GdprExportDownloadGateScope;
|
||||
readonly tenantId: string;
|
||||
readonly principalId: string;
|
||||
readonly artifactId: string;
|
||||
readonly secret: string;
|
||||
};
|
||||
|
||||
export const GDPR_EXPORT_DOWNLOAD_LEASE_TTL_SECONDS = 90;
|
||||
export const GDPR_EXPORT_DOWNLOAD_LEASE_RENEW_SECONDS = 30;
|
||||
export const GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_MAX = 6;
|
||||
export const GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_WINDOW_SECONDS = 5 * 60;
|
||||
|
||||
const ACQUIRE_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT = `
|
||||
local owner = ARGV[1]
|
||||
local ttl = tonumber(ARGV[2])
|
||||
|
||||
if redis.call('GET', KEYS[1]) then
|
||||
return { 0, 1 }
|
||||
end
|
||||
if redis.call('GET', KEYS[2]) then
|
||||
return { 0, 2 }
|
||||
end
|
||||
|
||||
redis.call('SET', KEYS[1], owner, 'EX', ttl)
|
||||
redis.call('SET', KEYS[2], owner, 'EX', ttl)
|
||||
return { 1, 0 }
|
||||
`;
|
||||
|
||||
const RELEASE_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT = `
|
||||
local owner = ARGV[1]
|
||||
local released = 0
|
||||
|
||||
if redis.call('GET', KEYS[1]) == owner then
|
||||
released = released + redis.call('DEL', KEYS[1])
|
||||
end
|
||||
if redis.call('GET', KEYS[2]) == owner then
|
||||
released = released + redis.call('DEL', KEYS[2])
|
||||
end
|
||||
return released
|
||||
`;
|
||||
|
||||
const RENEW_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT = `
|
||||
local owner = ARGV[1]
|
||||
local ttl = tonumber(ARGV[2])
|
||||
|
||||
if redis.call('GET', KEYS[1]) ~= owner then
|
||||
return 0
|
||||
end
|
||||
if redis.call('GET', KEYS[2]) ~= owner then
|
||||
return 0
|
||||
end
|
||||
|
||||
redis.call('EXPIRE', KEYS[1], ttl)
|
||||
redis.call('EXPIRE', KEYS[2], ttl)
|
||||
return 1
|
||||
`;
|
||||
|
||||
MemoryAdapter.registerScript(ACQUIRE_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT, async (adapter, keys, args) => {
|
||||
const owner = args[0] ?? "";
|
||||
const ttlSeconds = Number.parseInt(args[1] ?? "0", 10);
|
||||
if (await adapter.get(keys[0])) return [0, 1];
|
||||
if (await adapter.get(keys[1])) return [0, 2];
|
||||
await adapter.set(keys[0], owner, { ttlSeconds });
|
||||
await adapter.set(keys[1], owner, { ttlSeconds });
|
||||
return [1, 0];
|
||||
});
|
||||
|
||||
MemoryAdapter.registerScript(RELEASE_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT, async (adapter, keys, args) => {
|
||||
const owner = args[0] ?? "";
|
||||
let released = 0;
|
||||
if ((await adapter.get(keys[0])) === owner) released += await adapter.del(keys[0]);
|
||||
if ((await adapter.get(keys[1])) === owner) released += await adapter.del(keys[1]);
|
||||
return released;
|
||||
});
|
||||
|
||||
MemoryAdapter.registerScript(RENEW_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT, async (adapter, keys, args) => {
|
||||
const owner = args[0] ?? "";
|
||||
const ttlSeconds = Number.parseInt(args[1] ?? "0", 10);
|
||||
if ((await adapter.get(keys[0])) !== owner || (await adapter.get(keys[1])) !== owner) return 0;
|
||||
await adapter.expire(keys[0], ttlSeconds);
|
||||
await adapter.expire(keys[1], ttlSeconds);
|
||||
return 1;
|
||||
});
|
||||
|
||||
const logger = createScopedLogger("gdpr-export-download-gate");
|
||||
let storeFactory: () => SharedStore = () => getSharedStore();
|
||||
|
||||
/**
|
||||
* Reserves one distributed buffered download for both the authenticated
|
||||
* principal and the artifact. Redis keys contain only HMAC digests. Any
|
||||
* validation/store failure denies the read, so a lost shared store cannot
|
||||
* turn into an unbounded per-replica memory path.
|
||||
*/
|
||||
export async function acquireGdprExportDownloadGate(
|
||||
input: AcquireGdprExportDownloadGateInput,
|
||||
): Promise<GdprExportDownloadGateLease | null> {
|
||||
try {
|
||||
const keys = gdprExportDownloadGateKeys(input);
|
||||
const store = storeFactory();
|
||||
const rateAllowed = await reserveRateLimit(store, input.scope, keys);
|
||||
if (!rateAllowed) return null;
|
||||
const owner = randomUUID();
|
||||
const result = (await store.eval(ACQUIRE_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT, keys, [
|
||||
owner,
|
||||
String(GDPR_EXPORT_DOWNLOAD_LEASE_TTL_SECONDS),
|
||||
])) as [number | string, number | string] | undefined;
|
||||
if (Number(result?.[0]) !== 1) return null;
|
||||
|
||||
let releasePromise: Promise<void> | undefined;
|
||||
const renewalTimer = setInterval(() => {
|
||||
void renewLease(store, keys, owner, input.scope);
|
||||
}, GDPR_EXPORT_DOWNLOAD_LEASE_RENEW_SECONDS * 1_000);
|
||||
renewalTimer.unref?.();
|
||||
return {
|
||||
release() {
|
||||
if (!releasePromise) {
|
||||
clearInterval(renewalTimer);
|
||||
releasePromise = releaseLease(store, keys, owner, input.scope);
|
||||
}
|
||||
return releasePromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
logGateFailure("acquire", input.scope, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function reserveRateLimit(
|
||||
store: SharedStore,
|
||||
scope: GdprExportDownloadGateScope,
|
||||
keys: [string, string],
|
||||
): Promise<boolean> {
|
||||
const principal = await createFixedWindowRateLimiter({
|
||||
store,
|
||||
prefix: `gdpr-export-download-v1:${scope}:principal`,
|
||||
max: GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_MAX,
|
||||
windowSeconds: GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_WINDOW_SECONDS,
|
||||
}).consume(keys[0]);
|
||||
if (!principal.allowed) return false;
|
||||
const artifact = await createFixedWindowRateLimiter({
|
||||
store,
|
||||
prefix: `gdpr-export-download-v1:${scope}:artifact`,
|
||||
max: GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_MAX,
|
||||
windowSeconds: GDPR_EXPORT_DOWNLOAD_RATE_LIMIT_WINDOW_SECONDS,
|
||||
}).consume(keys[1]);
|
||||
return artifact.allowed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps the lease until the HTTP runtime has consumed or cancelled the body,
|
||||
* rather than releasing it as soon as the fully buffered Response is built.
|
||||
*/
|
||||
export function holdGdprExportDownloadGateUntilResponseFinished(
|
||||
response: Response,
|
||||
lease: GdprExportDownloadGateLease,
|
||||
): Response {
|
||||
if (!response.body || response.bodyUsed) throw new Error("GDPR export download response body is unavailable.");
|
||||
const reader = response.body.getReader();
|
||||
let releasePromise: Promise<void> | undefined;
|
||||
const release = () => {
|
||||
releasePromise ??= lease.release();
|
||||
return releasePromise;
|
||||
};
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
try {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) {
|
||||
controller.close();
|
||||
await release();
|
||||
return;
|
||||
}
|
||||
controller.enqueue(chunk.value);
|
||||
} catch (error) {
|
||||
controller.error(error);
|
||||
await release();
|
||||
}
|
||||
},
|
||||
async cancel(reason) {
|
||||
try {
|
||||
await reader.cancel(reason);
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
},
|
||||
});
|
||||
return new Response(body, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: response.headers,
|
||||
});
|
||||
}
|
||||
|
||||
function gdprExportDownloadGateKeys(input: AcquireGdprExportDownloadGateInput): [string, string] {
|
||||
if (input.scope !== "portal" && input.scope !== "privacy") throw new Error("GDPR export download gate scope is invalid.");
|
||||
const tenantId = normalizeIdentifier(input.tenantId, "tenant");
|
||||
const principalId = normalizeIdentifier(input.principalId, "principal");
|
||||
const artifactId = normalizeIdentifier(input.artifactId, "artifact");
|
||||
if (Buffer.byteLength(input.secret, "utf8") < 32) throw new Error("GDPR export download gate secret is invalid.");
|
||||
|
||||
const digest = (dimension: "principal" | "artifact", value: string) =>
|
||||
createHmac("sha256", input.secret)
|
||||
.update(`gdpr-export-download-v1\0${input.scope}\0${dimension}\0${tenantId}\0${value}`)
|
||||
.digest("hex");
|
||||
return [
|
||||
`gdpr-export-download-v1:${input.scope}:principal:${digest("principal", principalId)}`,
|
||||
`gdpr-export-download-v1:${input.scope}:artifact:${digest("artifact", artifactId)}`,
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeIdentifier(value: string, name: "tenant" | "principal" | "artifact"): string {
|
||||
const normalized = value.trim();
|
||||
if (!/^[A-Za-z0-9_-]{8,191}$/u.test(normalized)) throw new Error(`GDPR export download ${name} identifier is invalid.`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
async function releaseLease(
|
||||
store: SharedStore,
|
||||
keys: [string, string],
|
||||
owner: string,
|
||||
scope: GdprExportDownloadGateScope,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await store.eval(RELEASE_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT, keys, [owner]);
|
||||
} catch (error) {
|
||||
logGateFailure("release", scope, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function renewLease(
|
||||
store: SharedStore,
|
||||
keys: [string, string],
|
||||
owner: string,
|
||||
scope: GdprExportDownloadGateScope,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const renewed = await store.eval(RENEW_GDPR_EXPORT_DOWNLOAD_LEASE_SCRIPT, keys, [
|
||||
owner,
|
||||
String(GDPR_EXPORT_DOWNLOAD_LEASE_TTL_SECONDS),
|
||||
]);
|
||||
if (Number(renewed) !== 1) logGateFailure("renew", scope, new Error("Lease ownership was lost."));
|
||||
} catch (error) {
|
||||
logGateFailure("renew", scope, error);
|
||||
}
|
||||
}
|
||||
|
||||
function logGateFailure(operation: "acquire" | "renew" | "release", scope: GdprExportDownloadGateScope, error: unknown): void {
|
||||
logger.error(
|
||||
{
|
||||
event: "gdpr_export.download_gate_unavailable",
|
||||
operation,
|
||||
downloadScope: scope,
|
||||
errorName: error instanceof Error ? error.name : "UnknownError",
|
||||
},
|
||||
"GDPR export download gate unavailable",
|
||||
);
|
||||
}
|
||||
|
||||
export function __setGdprExportDownloadGateStoreForTests(store: SharedStore): void {
|
||||
storeFactory = () => store;
|
||||
}
|
||||
|
||||
export function __resetGdprExportDownloadGateStoreForTests(): void {
|
||||
storeFactory = () => getSharedStore();
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import {
|
||||
buildNotificationJob,
|
||||
enqueueJob,
|
||||
queueNames,
|
||||
resolveRedisConnectionOptions,
|
||||
resolveRedisConnectionOptionsFromProcessEnv,
|
||||
type RedisConnectionEnv,
|
||||
type TypedQueue,
|
||||
} from "@hivegrid/queue";
|
||||
|
||||
/**
|
||||
* AK-P21 (AGENTS.md DoD C15) — privacy notices go through the M011
|
||||
* notification queue instead of calling a mail provider from the route. The
|
||||
* notification handler owns preference evaluation, the NotificationDelivery
|
||||
* record and the provider call.
|
||||
*
|
||||
* These notices ride the locked "critical" category on purpose: a GDPR
|
||||
* deletion confirmation carries a one-time link with a hard deadline and must
|
||||
* not be suppressible by a notification preference.
|
||||
*/
|
||||
export const PRIVACY_NOTIFICATION_CATEGORY = "critical";
|
||||
|
||||
export type EnqueuePrivacyNotificationInput = {
|
||||
tenantId: string;
|
||||
userId: string;
|
||||
email: string;
|
||||
messageKey: string;
|
||||
sourceKey: string;
|
||||
subject: string;
|
||||
text: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type EnqueuePrivacyNotificationResult = { enqueued: true; jobId?: string } | { enqueued: false; reason: "queue-unavailable" };
|
||||
|
||||
type Dependencies = {
|
||||
env?: RedisConnectionEnv;
|
||||
queue?: TypedQueue<typeof queueNames.notifications>;
|
||||
};
|
||||
|
||||
export async function enqueuePrivacyNotification(
|
||||
input: EnqueuePrivacyNotificationInput,
|
||||
dependencies: Dependencies = {}
|
||||
): Promise<EnqueuePrivacyNotificationResult> {
|
||||
let queue = dependencies.queue;
|
||||
let shouldCloseQueue = false;
|
||||
const connection = queue
|
||||
? undefined
|
||||
: dependencies.env
|
||||
? resolveRedisConnectionOptions({ env: dependencies.env })
|
||||
: resolveRedisConnectionOptionsFromProcessEnv();
|
||||
|
||||
if (!queue && !connection) {
|
||||
return { enqueued: false, reason: "queue-unavailable" };
|
||||
}
|
||||
|
||||
if (!queue && connection) {
|
||||
const queueRuntimeModule = "@hivegrid/queue/runtime";
|
||||
const { createQueue } = (await import(/* @vite-ignore */ queueRuntimeModule)) as typeof import("@hivegrid/queue/runtime");
|
||||
queue = createQueue(queueNames.notifications, { connection });
|
||||
shouldCloseQueue = true;
|
||||
}
|
||||
|
||||
if (!queue) {
|
||||
return { enqueued: false, reason: "queue-unavailable" };
|
||||
}
|
||||
|
||||
try {
|
||||
const descriptor = buildNotificationJob({
|
||||
tenantId: input.tenantId,
|
||||
userId: input.userId,
|
||||
email: input.email,
|
||||
channel: "email",
|
||||
category: PRIVACY_NOTIFICATION_CATEGORY,
|
||||
messageKey: input.messageKey,
|
||||
sourceKey: input.sourceKey,
|
||||
subject: input.subject,
|
||||
text: input.text,
|
||||
...(input.metadata ? { metadata: input.metadata } : {}),
|
||||
});
|
||||
const job = await enqueueJob(queue, descriptor, { tenantId: input.tenantId });
|
||||
return { enqueued: true, jobId: typeof job.id === "string" ? job.id : undefined };
|
||||
} finally {
|
||||
if (shouldCloseQueue && queue) {
|
||||
await queue.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user