Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* Phase 10 Task #68 — DSGVO personal-data export worker.
|
||||
*
|
||||
* Aggregates every row that references `userId` across the
|
||||
* USER_FK_MAP (see packages/db/src/dsgvo/user-fk-map.ts) plus the
|
||||
* canonical `User` row, serialises the result as a single
|
||||
* UTF-8 JSON bundle, and streams it to S3 via the multipart sink
|
||||
* shared with Phase 03 #46.
|
||||
*
|
||||
* The worker is host-agnostic: the bootstrap (apps/worker/src/index.ts)
|
||||
* passes in the dependencies (table iterator, sink factory, audit
|
||||
* writer, signed-URL builder, mailer) so this module can be unit-tested
|
||||
* in isolation without S3 / Prisma / SES.
|
||||
*
|
||||
* Job lifecycle:
|
||||
* 1. Resolve the user (loadUser).
|
||||
* 2. For each entry in USER_FK_MAP with includeInExport=true, stream
|
||||
* its rows into the JSON bundle under a section keyed by model.
|
||||
* 3. Finalise the multipart upload, mint a signed URL with the
|
||||
* configured validity, send the "export ready" notification email,
|
||||
* and emit an AuditEvent with action=EXPORT_PERSONAL_DATA,
|
||||
* regulatoryScope=[gdpr], dataSensitivity=pii.
|
||||
*
|
||||
* Errors short-circuit the pipeline; the caller (BullMQ worker) is
|
||||
* responsible for retry semantics. The sink aborts its multipart upload
|
||||
* automatically on stream destruction so no orphan parts linger in S3.
|
||||
*/
|
||||
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
|
||||
|
||||
export type DsgvoExportJobInput = {
|
||||
userId: string;
|
||||
/** Tenant scope for the AuditEvent + S3 metadata. */
|
||||
tenantId: string | null;
|
||||
/** Idempotency token (mirrors User.deletionRequestId for repeats). */
|
||||
requestId: string;
|
||||
/** Initiator: 'user' (self-service) or 'admin' (operator-on-behalf). */
|
||||
actorType: "user" | "admin";
|
||||
/** Actor identity for the AuditEvent. */
|
||||
actorId: string;
|
||||
/** Optional admin reason (admin path only). */
|
||||
reason?: string | null;
|
||||
};
|
||||
|
||||
export type DsgvoExportSection = {
|
||||
model: string;
|
||||
column: string;
|
||||
rowCount: number;
|
||||
};
|
||||
|
||||
export type DsgvoExportResult = {
|
||||
bucket: string;
|
||||
key: string;
|
||||
rowCount: number;
|
||||
sizeBytes: number;
|
||||
signedUrl: string;
|
||||
signedUrlExpiresAt: Date;
|
||||
sections: DsgvoExportSection[];
|
||||
};
|
||||
|
||||
export type DsgvoExportSink = {
|
||||
writeStream: Writable;
|
||||
completionPromise: Promise<{ bucket: string; key: string; sizeBytes: number }>;
|
||||
};
|
||||
|
||||
export type DsgvoExportTableSource = {
|
||||
/**
|
||||
* Async-iterate every row in `entry.model` filtered by
|
||||
* `${entry.column} = userId`. Implementations typically use a Prisma
|
||||
* cursor; the worker buffers nothing.
|
||||
*/
|
||||
iterate(userId: string, entry: UserFkMapEntry): AsyncIterable<Record<string, unknown>>;
|
||||
};
|
||||
|
||||
export type DsgvoExportUserRecord = {
|
||||
id: string;
|
||||
email: string;
|
||||
firstName: string;
|
||||
lastName: string;
|
||||
createdAt: Date;
|
||||
phone?: string | null;
|
||||
avatar?: string | null;
|
||||
locale?: string | null;
|
||||
};
|
||||
|
||||
export type DsgvoExportDependencies = {
|
||||
/** Load the canonical User row (header section of the bundle). */
|
||||
loadUser: (userId: string) => Promise<DsgvoExportUserRecord | null>;
|
||||
/** Iterate every user-fk-map entry that opts into export. */
|
||||
tableSource: DsgvoExportTableSource;
|
||||
/** Open the streaming sink (typically createS3MultipartUploadSink). */
|
||||
openSink: (objectKey: string, contentType: string, metadata: Record<string, string>) => DsgvoExportSink;
|
||||
/** Mint a presigned URL pointing at the finished artifact. */
|
||||
buildSignedUrl: (input: { bucket: string; key: string }) => Promise<{ url: string; expiresAt: Date }>;
|
||||
/** Persist the AuditEvent (createAuditEvent). */
|
||||
writeAuditEvent: (input: {
|
||||
tenantId: string | null;
|
||||
actorId: string;
|
||||
actorType: string;
|
||||
action: string;
|
||||
targetType: string;
|
||||
targetId: string;
|
||||
regulatoryScope: string[];
|
||||
dataSensitivity: "pii";
|
||||
outcome: "success" | "failure";
|
||||
metadata: Record<string, unknown>;
|
||||
}) => Promise<void>;
|
||||
/** Notify the user that the export is ready. */
|
||||
sendExportReadyEmail: (input: {
|
||||
userId: string;
|
||||
toEmail: string;
|
||||
signedUrl: string;
|
||||
expiresAt: Date;
|
||||
sizeBytes: number;
|
||||
rowCount: number;
|
||||
}) => Promise<void>;
|
||||
/** Object-key builder — caller controls bucket + path layout. */
|
||||
buildObjectKey: (input: { userId: string; requestId: string }) => { bucket: string; key: string };
|
||||
/** Logger seam — no console.* in production code. */
|
||||
logger: { info: (fields: Record<string, unknown>, msg: string) => void; error: (fields: Record<string, unknown>, msg: string) => void };
|
||||
};
|
||||
|
||||
const EXPORT_CONTENT_TYPE = "application/json; charset=utf-8";
|
||||
const BUNDLE_VERSION = "1.0";
|
||||
|
||||
export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoExportJobInput): Promise<DsgvoExportResult> {
|
||||
const user = await deps.loadUser(job.userId);
|
||||
if (!user) {
|
||||
await deps.writeAuditEvent({
|
||||
tenantId: job.tenantId,
|
||||
actorId: job.actorId,
|
||||
actorType: job.actorType,
|
||||
action: "EXPORT_PERSONAL_DATA",
|
||||
targetType: "User",
|
||||
targetId: job.userId,
|
||||
regulatoryScope: ["gdpr"],
|
||||
dataSensitivity: "pii",
|
||||
outcome: "failure",
|
||||
metadata: { requestId: job.requestId, error: "user_not_found", reason: job.reason ?? null },
|
||||
});
|
||||
throw new Error(`dsgvo-export: user ${job.userId} not found`);
|
||||
}
|
||||
|
||||
const exportEntries = USER_FK_MAP.filter((e) => e.includeInExport);
|
||||
const sections: DsgvoExportSection[] = [];
|
||||
let totalRows = 0;
|
||||
const { key } = deps.buildObjectKey({ userId: job.userId, requestId: job.requestId });
|
||||
const sink = deps.openSink(key, EXPORT_CONTENT_TYPE, {
|
||||
"x-tc-user-id": job.userId,
|
||||
"x-tc-request-id": job.requestId,
|
||||
});
|
||||
|
||||
try {
|
||||
sink.writeStream.write(`{\n "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
|
||||
sink.writeStream.write(` "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
|
||||
sink.writeStream.write(` "requestId": ${JSON.stringify(job.requestId)},\n`);
|
||||
sink.writeStream.write(` "user": ${JSON.stringify(user)},\n`);
|
||||
sink.writeStream.write(` "sections": {\n`);
|
||||
|
||||
let firstSection = true;
|
||||
for (const entry of exportEntries) {
|
||||
if (!firstSection) sink.writeStream.write(",\n");
|
||||
firstSection = false;
|
||||
sink.writeStream.write(` ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
|
||||
let rowCount = 0;
|
||||
let firstRow = true;
|
||||
for await (const row of deps.tableSource.iterate(job.userId, entry)) {
|
||||
if (!firstRow) sink.writeStream.write(",");
|
||||
firstRow = false;
|
||||
sink.writeStream.write(JSON.stringify(row));
|
||||
rowCount += 1;
|
||||
}
|
||||
sink.writeStream.write("]");
|
||||
sections.push({ model: entry.model, column: entry.column, rowCount });
|
||||
totalRows += rowCount;
|
||||
}
|
||||
|
||||
sink.writeStream.write("\n }\n}\n");
|
||||
sink.writeStream.end();
|
||||
const completion = await sink.completionPromise;
|
||||
|
||||
const { url, expiresAt } = await deps.buildSignedUrl({ bucket: completion.bucket, key: completion.key });
|
||||
|
||||
await deps.sendExportReadyEmail({
|
||||
userId: job.userId,
|
||||
toEmail: user.email,
|
||||
signedUrl: url,
|
||||
expiresAt,
|
||||
sizeBytes: completion.sizeBytes,
|
||||
rowCount: totalRows,
|
||||
});
|
||||
|
||||
await deps.writeAuditEvent({
|
||||
tenantId: job.tenantId,
|
||||
actorId: job.actorId,
|
||||
actorType: job.actorType,
|
||||
action: "EXPORT_PERSONAL_DATA",
|
||||
targetType: "User",
|
||||
targetId: job.userId,
|
||||
regulatoryScope: ["gdpr"],
|
||||
dataSensitivity: "pii",
|
||||
outcome: "success",
|
||||
metadata: {
|
||||
requestId: job.requestId,
|
||||
rowCount: totalRows,
|
||||
sizeBytes: completion.sizeBytes,
|
||||
sectionCount: sections.length,
|
||||
bucket: completion.bucket,
|
||||
key: completion.key,
|
||||
reason: job.reason ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
deps.logger.info({ userId: job.userId, requestId: job.requestId, rowCount: totalRows }, "dsgvo-export.complete");
|
||||
|
||||
return {
|
||||
bucket: completion.bucket,
|
||||
key: completion.key,
|
||||
rowCount: totalRows,
|
||||
sizeBytes: completion.sizeBytes,
|
||||
signedUrl: url,
|
||||
signedUrlExpiresAt: expiresAt,
|
||||
sections,
|
||||
};
|
||||
} catch (error) {
|
||||
sink.writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
||||
deps.logger.error({ userId: job.userId, requestId: job.requestId, err: String(error) }, "dsgvo-export.failed");
|
||||
await deps.writeAuditEvent({
|
||||
tenantId: job.tenantId,
|
||||
actorId: job.actorId,
|
||||
actorType: job.actorType,
|
||||
action: "EXPORT_PERSONAL_DATA",
|
||||
targetType: "User",
|
||||
targetId: job.userId,
|
||||
regulatoryScope: ["gdpr"],
|
||||
dataSensitivity: "pii",
|
||||
outcome: "failure",
|
||||
metadata: { requestId: job.requestId, error: String(error), reason: job.reason ?? null },
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user