Reorganize repo structure and standards references

This commit is contained in:
david committed 2026-06-29 05:03:42 +02:00
1 parent 3cb93a81be
commit be2054274d
5885 files changed
+2588 -195

No files matched your search

@@ -0,0 +1,244 @@
/**
* Phase 10 Task #68 — DSGVO personal-data export worker.
*
* Aggregates every row that references `userId` across the
* USER_FK_MAP (see packages/db/src/dsgvo/user-fk-map.ts) plus the
* canonical `User` row, serialises the result as a single
* UTF-8 JSON bundle, and streams it to S3 via the multipart sink
* shared with Phase 03 #46.
*
* The worker is host-agnostic: the bootstrap (apps/worker/src/index.ts)
* passes in the dependencies (table iterator, sink factory, audit
* writer, signed-URL builder, mailer) so this module can be unit-tested
* in isolation without S3 / Prisma / SES.
*
* Job lifecycle:
* 1. Resolve the user (loadUser).
* 2. For each entry in USER_FK_MAP with includeInExport=true, stream
* its rows into the JSON bundle under a section keyed by model.
* 3. Finalise the multipart upload, mint a signed URL with the
* configured validity, send the "export ready" notification email,
* and emit an AuditEvent with action=EXPORT_PERSONAL_DATA,
* regulatoryScope=[gdpr], dataSensitivity=pii.
*
* Errors short-circuit the pipeline; the caller (BullMQ worker) is
* responsible for retry semantics. The sink aborts its multipart upload
* automatically on stream destruction so no orphan parts linger in S3.
*/
import type { Writable } from "node:stream";
import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
export type DsgvoExportJobInput = {
userId: string;
/** Tenant scope for the AuditEvent + S3 metadata. */
tenantId: string | null;
/** Idempotency token (mirrors User.deletionRequestId for repeats). */
requestId: string;
/** Initiator: 'user' (self-service) or 'admin' (operator-on-behalf). */
actorType: "user" | "admin";
/** Actor identity for the AuditEvent. */
actorId: string;
/** Optional admin reason (admin path only). */
reason?: string | null;
};
export type DsgvoExportSection = {
model: string;
column: string;
rowCount: number;
};
export type DsgvoExportResult = {
bucket: string;
key: string;
rowCount: number;
sizeBytes: number;
signedUrl: string;
signedUrlExpiresAt: Date;
sections: DsgvoExportSection[];
};
export type DsgvoExportSink = {
writeStream: Writable;
completionPromise: Promise<{ bucket: string; key: string; sizeBytes: number }>;
};
export type DsgvoExportTableSource = {
/**
* Async-iterate every row in `entry.model` filtered by
* `${entry.column} = userId`. Implementations typically use a Prisma
* cursor; the worker buffers nothing.
*/
iterate(userId: string, entry: UserFkMapEntry): AsyncIterable<Record<string, unknown>>;
};
export type DsgvoExportUserRecord = {
id: string;
email: string;
firstName: string;
lastName: string;
createdAt: Date;
phone?: string | null;
avatar?: string | null;
locale?: string | null;
};
export type DsgvoExportDependencies = {
/** Load the canonical User row (header section of the bundle). */
loadUser: (userId: string) => Promise<DsgvoExportUserRecord | null>;
/** Iterate every user-fk-map entry that opts into export. */
tableSource: DsgvoExportTableSource;
/** Open the streaming sink (typically createS3MultipartUploadSink). */
openSink: (objectKey: string, contentType: string, metadata: Record<string, string>) => DsgvoExportSink;
/** Mint a presigned URL pointing at the finished artifact. */
buildSignedUrl: (input: { bucket: string; key: string }) => Promise<{ url: string; expiresAt: Date }>;
/** Persist the AuditEvent (createAuditEvent). */
writeAuditEvent: (input: {
tenantId: string | null;
actorId: string;
actorType: string;
action: string;
targetType: string;
targetId: string;
regulatoryScope: string[];
dataSensitivity: "pii";
outcome: "success" | "failure";
metadata: Record<string, unknown>;
}) => Promise<void>;
/** Notify the user that the export is ready. */
sendExportReadyEmail: (input: {
userId: string;
toEmail: string;
signedUrl: string;
expiresAt: Date;
sizeBytes: number;
rowCount: number;
}) => Promise<void>;
/** Object-key builder — caller controls bucket + path layout. */
buildObjectKey: (input: { userId: string; requestId: string }) => { bucket: string; key: string };
/** Logger seam — no console.* in production code. */
logger: { info: (fields: Record<string, unknown>, msg: string) => void; error: (fields: Record<string, unknown>, msg: string) => void };
};
const EXPORT_CONTENT_TYPE = "application/json; charset=utf-8";
const BUNDLE_VERSION = "1.0";
export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoExportJobInput): Promise<DsgvoExportResult> {
const user = await deps.loadUser(job.userId);
if (!user) {
await deps.writeAuditEvent({
tenantId: job.tenantId,
actorId: job.actorId,
actorType: job.actorType,
action: "EXPORT_PERSONAL_DATA",
targetType: "User",
targetId: job.userId,
regulatoryScope: ["gdpr"],
dataSensitivity: "pii",
outcome: "failure",
metadata: { requestId: job.requestId, error: "user_not_found", reason: job.reason ?? null },
});
throw new Error(`dsgvo-export: user ${job.userId} not found`);
}
const exportEntries = USER_FK_MAP.filter((e) => e.includeInExport);
const sections: DsgvoExportSection[] = [];
let totalRows = 0;
const { key } = deps.buildObjectKey({ userId: job.userId, requestId: job.requestId });
const sink = deps.openSink(key, EXPORT_CONTENT_TYPE, {
"x-tc-user-id": job.userId,
"x-tc-request-id": job.requestId,
});
try {
sink.writeStream.write(`{\n "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
sink.writeStream.write(` "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
sink.writeStream.write(` "requestId": ${JSON.stringify(job.requestId)},\n`);
sink.writeStream.write(` "user": ${JSON.stringify(user)},\n`);
sink.writeStream.write(` "sections": {\n`);
let firstSection = true;
for (const entry of exportEntries) {
if (!firstSection) sink.writeStream.write(",\n");
firstSection = false;
sink.writeStream.write(` ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
let rowCount = 0;
let firstRow = true;
for await (const row of deps.tableSource.iterate(job.userId, entry)) {
if (!firstRow) sink.writeStream.write(",");
firstRow = false;
sink.writeStream.write(JSON.stringify(row));
rowCount += 1;
}
sink.writeStream.write("]");
sections.push({ model: entry.model, column: entry.column, rowCount });
totalRows += rowCount;
}
sink.writeStream.write("\n }\n}\n");
sink.writeStream.end();
const completion = await sink.completionPromise;
const { url, expiresAt } = await deps.buildSignedUrl({ bucket: completion.bucket, key: completion.key });
await deps.sendExportReadyEmail({
userId: job.userId,
toEmail: user.email,
signedUrl: url,
expiresAt,
sizeBytes: completion.sizeBytes,
rowCount: totalRows,
});
await deps.writeAuditEvent({
tenantId: job.tenantId,
actorId: job.actorId,
actorType: job.actorType,
action: "EXPORT_PERSONAL_DATA",
targetType: "User",
targetId: job.userId,
regulatoryScope: ["gdpr"],
dataSensitivity: "pii",
outcome: "success",
metadata: {
requestId: job.requestId,
rowCount: totalRows,
sizeBytes: completion.sizeBytes,
sectionCount: sections.length,
bucket: completion.bucket,
key: completion.key,
reason: job.reason ?? null,
},
});
deps.logger.info({ userId: job.userId, requestId: job.requestId, rowCount: totalRows }, "dsgvo-export.complete");
return {
bucket: completion.bucket,
key: completion.key,
rowCount: totalRows,
sizeBytes: completion.sizeBytes,
signedUrl: url,
signedUrlExpiresAt: expiresAt,
sections,
};
} catch (error) {
sink.writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
deps.logger.error({ userId: job.userId, requestId: job.requestId, err: String(error) }, "dsgvo-export.failed");
await deps.writeAuditEvent({
tenantId: job.tenantId,
actorId: job.actorId,
actorType: job.actorType,
action: "EXPORT_PERSONAL_DATA",
targetType: "User",
targetId: job.userId,
regulatoryScope: ["gdpr"],
dataSensitivity: "pii",
outcome: "failure",
metadata: { requestId: job.requestId, error: String(error), reason: job.reason ?? null },
});
throw error;
}
}