245 lines
8.5 KiB
TypeScript
245 lines
8.5 KiB
TypeScript
/**
|
|
* Phase 10 Task #68 — DSGVO personal-data export worker.
|
|
*
|
|
* Aggregates every row that references `userId` across the
|
|
* USER_FK_MAP (see packages/db/src/dsgvo/user-fk-map.ts) plus the
|
|
* canonical `User` row, serialises the result as a single
|
|
* UTF-8 JSON bundle, and streams it to S3 via the multipart sink
|
|
* shared with Phase 03 #46.
|
|
*
|
|
* The worker is host-agnostic: the bootstrap (apps/worker/src/index.ts)
|
|
* passes in the dependencies (table iterator, sink factory, audit
|
|
* writer, signed-URL builder, mailer) so this module can be unit-tested
|
|
* in isolation without S3 / Prisma / SES.
|
|
*
|
|
* Job lifecycle:
|
|
* 1. Resolve the user (loadUser).
|
|
* 2. For each entry in USER_FK_MAP with includeInExport=true, stream
|
|
* its rows into the JSON bundle under a section keyed by model.
|
|
* 3. Finalise the multipart upload, mint a signed URL with the
|
|
* configured validity, send the "export ready" notification email,
|
|
* and emit an AuditEvent with action=EXPORT_PERSONAL_DATA,
|
|
* regulatoryScope=[gdpr], dataSensitivity=pii.
|
|
*
|
|
* Errors short-circuit the pipeline; the caller (BullMQ worker) is
|
|
* responsible for retry semantics. The sink aborts its multipart upload
|
|
* automatically on stream destruction so no orphan parts linger in S3.
|
|
*/
|
|
|
|
import type { Writable } from "node:stream";
|
|
|
|
import { USER_FK_MAP, type UserFkMapEntry } from "@hivegrid/db/dsgvo/user-fk-map";
|
|
|
|
export type DsgvoExportJobInput = {
|
|
userId: string;
|
|
/** Tenant scope for the AuditEvent + S3 metadata. */
|
|
tenantId: string | null;
|
|
/** Idempotency token (mirrors User.deletionRequestId for repeats). */
|
|
requestId: string;
|
|
/** Initiator: 'user' (self-service) or 'admin' (operator-on-behalf). */
|
|
actorType: "user" | "admin";
|
|
/** Actor identity for the AuditEvent. */
|
|
actorId: string;
|
|
/** Optional admin reason (admin path only). */
|
|
reason?: string | null;
|
|
};
|
|
|
|
export type DsgvoExportSection = {
|
|
model: string;
|
|
column: string;
|
|
rowCount: number;
|
|
};
|
|
|
|
export type DsgvoExportResult = {
|
|
bucket: string;
|
|
key: string;
|
|
rowCount: number;
|
|
sizeBytes: number;
|
|
signedUrl: string;
|
|
signedUrlExpiresAt: Date;
|
|
sections: DsgvoExportSection[];
|
|
};
|
|
|
|
export type DsgvoExportSink = {
|
|
writeStream: Writable;
|
|
completionPromise: Promise<{ bucket: string; key: string; sizeBytes: number }>;
|
|
};
|
|
|
|
export type DsgvoExportTableSource = {
|
|
/**
|
|
* Async-iterate every row in `entry.model` filtered by
|
|
* `${entry.column} = userId`. Implementations typically use a Prisma
|
|
* cursor; the worker buffers nothing.
|
|
*/
|
|
iterate(userId: string, entry: UserFkMapEntry): AsyncIterable<Record<string, unknown>>;
|
|
};
|
|
|
|
export type DsgvoExportUserRecord = {
|
|
id: string;
|
|
email: string;
|
|
firstName: string;
|
|
lastName: string;
|
|
createdAt: Date;
|
|
phone?: string | null;
|
|
avatar?: string | null;
|
|
locale?: string | null;
|
|
};
|
|
|
|
export type DsgvoExportDependencies = {
|
|
/** Load the canonical User row (header section of the bundle). */
|
|
loadUser: (userId: string) => Promise<DsgvoExportUserRecord | null>;
|
|
/** Iterate every user-fk-map entry that opts into export. */
|
|
tableSource: DsgvoExportTableSource;
|
|
/** Open the streaming sink (typically createS3MultipartUploadSink). */
|
|
openSink: (objectKey: string, contentType: string, metadata: Record<string, string>) => DsgvoExportSink;
|
|
/** Mint a presigned URL pointing at the finished artifact. */
|
|
buildSignedUrl: (input: { bucket: string; key: string }) => Promise<{ url: string; expiresAt: Date }>;
|
|
/** Persist the AuditEvent (createAuditEvent). */
|
|
writeAuditEvent: (input: {
|
|
tenantId: string | null;
|
|
actorId: string;
|
|
actorType: string;
|
|
action: string;
|
|
targetType: string;
|
|
targetId: string;
|
|
regulatoryScope: string[];
|
|
dataSensitivity: "pii";
|
|
outcome: "success" | "failure";
|
|
metadata: Record<string, unknown>;
|
|
}) => Promise<void>;
|
|
/** Notify the user that the export is ready. */
|
|
sendExportReadyEmail: (input: {
|
|
userId: string;
|
|
toEmail: string;
|
|
signedUrl: string;
|
|
expiresAt: Date;
|
|
sizeBytes: number;
|
|
rowCount: number;
|
|
}) => Promise<void>;
|
|
/** Object-key builder — caller controls bucket + path layout. */
|
|
buildObjectKey: (input: { userId: string; requestId: string }) => { bucket: string; key: string };
|
|
/** Logger seam — no console.* in production code. */
|
|
logger: { info: (fields: Record<string, unknown>, msg: string) => void; error: (fields: Record<string, unknown>, msg: string) => void };
|
|
};
|
|
|
|
const EXPORT_CONTENT_TYPE = "application/json; charset=utf-8";
|
|
const BUNDLE_VERSION = "1.0";
|
|
|
|
export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoExportJobInput): Promise<DsgvoExportResult> {
|
|
const user = await deps.loadUser(job.userId);
|
|
if (!user) {
|
|
await deps.writeAuditEvent({
|
|
tenantId: job.tenantId,
|
|
actorId: job.actorId,
|
|
actorType: job.actorType,
|
|
action: "EXPORT_PERSONAL_DATA",
|
|
targetType: "User",
|
|
targetId: job.userId,
|
|
regulatoryScope: ["gdpr"],
|
|
dataSensitivity: "pii",
|
|
outcome: "failure",
|
|
metadata: { requestId: job.requestId, error: "user_not_found", reason: job.reason ?? null },
|
|
});
|
|
throw new Error(`dsgvo-export: user ${job.userId} not found`);
|
|
}
|
|
|
|
const exportEntries = USER_FK_MAP.filter((e) => e.includeInExport);
|
|
const sections: DsgvoExportSection[] = [];
|
|
let totalRows = 0;
|
|
const { key } = deps.buildObjectKey({ userId: job.userId, requestId: job.requestId });
|
|
const sink = deps.openSink(key, EXPORT_CONTENT_TYPE, {
|
|
"x-tc-user-id": job.userId,
|
|
"x-tc-request-id": job.requestId,
|
|
});
|
|
|
|
try {
|
|
sink.writeStream.write(`{\n "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
|
|
sink.writeStream.write(` "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
|
|
sink.writeStream.write(` "requestId": ${JSON.stringify(job.requestId)},\n`);
|
|
sink.writeStream.write(` "user": ${JSON.stringify(user)},\n`);
|
|
sink.writeStream.write(` "sections": {\n`);
|
|
|
|
let firstSection = true;
|
|
for (const entry of exportEntries) {
|
|
if (!firstSection) sink.writeStream.write(",\n");
|
|
firstSection = false;
|
|
sink.writeStream.write(` ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
|
|
let rowCount = 0;
|
|
let firstRow = true;
|
|
for await (const row of deps.tableSource.iterate(job.userId, entry)) {
|
|
if (!firstRow) sink.writeStream.write(",");
|
|
firstRow = false;
|
|
sink.writeStream.write(JSON.stringify(row));
|
|
rowCount += 1;
|
|
}
|
|
sink.writeStream.write("]");
|
|
sections.push({ model: entry.model, column: entry.column, rowCount });
|
|
totalRows += rowCount;
|
|
}
|
|
|
|
sink.writeStream.write("\n }\n}\n");
|
|
sink.writeStream.end();
|
|
const completion = await sink.completionPromise;
|
|
|
|
const { url, expiresAt } = await deps.buildSignedUrl({ bucket: completion.bucket, key: completion.key });
|
|
|
|
await deps.sendExportReadyEmail({
|
|
userId: job.userId,
|
|
toEmail: user.email,
|
|
signedUrl: url,
|
|
expiresAt,
|
|
sizeBytes: completion.sizeBytes,
|
|
rowCount: totalRows,
|
|
});
|
|
|
|
await deps.writeAuditEvent({
|
|
tenantId: job.tenantId,
|
|
actorId: job.actorId,
|
|
actorType: job.actorType,
|
|
action: "EXPORT_PERSONAL_DATA",
|
|
targetType: "User",
|
|
targetId: job.userId,
|
|
regulatoryScope: ["gdpr"],
|
|
dataSensitivity: "pii",
|
|
outcome: "success",
|
|
metadata: {
|
|
requestId: job.requestId,
|
|
rowCount: totalRows,
|
|
sizeBytes: completion.sizeBytes,
|
|
sectionCount: sections.length,
|
|
bucket: completion.bucket,
|
|
key: completion.key,
|
|
reason: job.reason ?? null,
|
|
},
|
|
});
|
|
|
|
deps.logger.info({ userId: job.userId, requestId: job.requestId, rowCount: totalRows }, "dsgvo-export.complete");
|
|
|
|
return {
|
|
bucket: completion.bucket,
|
|
key: completion.key,
|
|
rowCount: totalRows,
|
|
sizeBytes: completion.sizeBytes,
|
|
signedUrl: url,
|
|
signedUrlExpiresAt: expiresAt,
|
|
sections,
|
|
};
|
|
} catch (error) {
|
|
sink.writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
|
deps.logger.error({ userId: job.userId, requestId: job.requestId, err: String(error) }, "dsgvo-export.failed");
|
|
await deps.writeAuditEvent({
|
|
tenantId: job.tenantId,
|
|
actorId: job.actorId,
|
|
actorType: job.actorType,
|
|
action: "EXPORT_PERSONAL_DATA",
|
|
targetType: "User",
|
|
targetId: job.userId,
|
|
regulatoryScope: ["gdpr"],
|
|
dataSensitivity: "pii",
|
|
outcome: "failure",
|
|
metadata: { requestId: job.requestId, error: String(error), reason: job.reason ?? null },
|
|
});
|
|
throw error;
|
|
}
|
|
}
|