/** * `tenant:snapshot` — extract a single tenant from Postgres + storage, apply * the chosen PII anonymization profile, tar + (optionally) encrypt + upload. * * The Prisma layer is dependency-injected via the `PrismaRunner` interface so * the CLI package stays free of `@prisma/client`. The production entrypoint * (`bin/hivegrid-cli.mjs` of the consumer) wires up the real `PrismaClient`; * tests pass a fake. */ import { defineCommand } from "citty"; import consola from "consola"; import { randomBytes, createHash } from "node:crypto"; import { createWriteStream, createReadStream } from "node:fs"; import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir, hostname } from "node:os"; import { join } from "node:path"; import { Readable } from "node:stream"; import { GetObjectCommand, ListObjectsV2Command, S3Client, type S3ClientConfig, } from "@aws-sdk/client-s3"; import { create as createTar } from "tar"; import { PROFILES, anonymizeRow, isProfileName, type AnonymizationContext, type ProfileName, } from "./anonymize.js"; import { TENANT_MANIFEST_VERSION, hashSalt, serializeTenantManifest, type TenantManifestModelEntry, type TenantSnapshotManifest, } from "./manifest.js"; import { TENANT_LIFECYCLE_MODELS, tenantWhereForLifecycleModel, type TenantLifecycleModel, } from "./derived-models.js"; import { assertSuperAdminForNoneProfile, writeSnapshotAudit } from "./audit.js"; import { snapshotReplacer } from "./json-codec.js"; import { collectAndWriteGlobals, type GlobalsPrismaRunner } from "./globals-runner.js"; import { encryptFile, type EncryptionConfig } from "../backup/encryption.js"; import { uploadFileMultipart } from "../backup/s3-upload.js"; /** * Minimal Prisma-like surface the snapshot command needs. * * `findManyByIds` is added for the globals extractor (B7 follow-up): * after collecting the tenant slice we re-issue id-set lookups against * the source DB to fetch the referenced `User` rows (and any other * globals enumerated in `models.ts::GLOBAL_REFERENCED_MODELS`). It is * optional so legacy callers that only snapshot tenant-scoped rows * (e.g. `--include-globals=false`) can still pass a slimmer runner. */ export interface PrismaRunner extends Partial { /** * Stream rows from `modelName` where `tenantId = tenantId` in stable id order. * Implementations should use cursor-based pagination so large tenants do * not OOM. The default page size is 1000 unless the caller overrides. */ findManyByTenant( modelName: string, tenantField: string, tenantId: string, pageSize: number, ): AsyncIterable>; /** * Stream FK-derived tenant rows using a relation-filter where clause. * This is separate from findManyByTenant because these models deliberately * have no tenantId scalar of their own. */ findManyByWhere( modelName: string, where: Readonly>, pageSize: number, ): AsyncIterable>; } export interface SnapshotS3Output { readonly kind: "s3"; readonly bucket: string; readonly key: string; } export interface SnapshotLocalOutput { readonly kind: "local"; readonly path: string; } export type SnapshotOutputTarget = SnapshotS3Output | SnapshotLocalOutput; export interface RunSnapshotOptions { readonly tenantId: string; readonly profile: ProfileName; readonly output: SnapshotOutputTarget; readonly prisma: PrismaRunner; readonly s3?: S3ClientConfig; readonly s3Client?: S3Client; readonly storageBucket?: string; readonly storagePrefix?: string; readonly encryption?: EncryptionConfig; readonly now?: () => Date; /** Marker the operator must pass when invoking `--profile=none`. */ readonly superAdminConfirmed?: boolean; /** Optional file path to append the snapshot-audit JSON record to. */ readonly auditFilePath?: string; /** Stream the human-readable audit line is written to (defaults to stdout). */ readonly auditStdout?: NodeJS.WritableStream; readonly logger?: { info: (m: string) => void; warn: (m: string) => void }; readonly actorUserId?: string; readonly pageSize?: number; /** * Emit `globals/.jsonl` files for every model in * `GLOBAL_REFERENCED_MODELS`. Defaults to `true`. Set to `false` for * backward-compatible "tenant-only" snapshots (the operator must then * pre-seed referenced globals on the target before restore). * * Requires `prisma.findManyByIds` to be implemented. If a runner * without that method is supplied with `includeGlobals=true`, the * snapshot logs a warning and skips the globals step rather than * failing — useful for unit tests that only exercise the tenant path. */ readonly includeGlobals?: boolean; } export interface RunSnapshotResult { readonly snapshotId: string; readonly manifest: TenantSnapshotManifest; readonly archivePath: string; readonly uploaded?: { bucket: string; key: string }; } /** * Parse an --output value into a structured target. Accepts `s3://bucket/key` * or a local filesystem path. Returns `null` if undefined. */ export function parseOutputTarget(raw: string | undefined, defaultLocalPath: string): SnapshotOutputTarget { if (!raw || raw.length === 0) { return { kind: "local", path: defaultLocalPath }; } if (raw.startsWith("s3://")) { const rest = raw.slice("s3://".length); const slash = rest.indexOf("/"); if (slash <= 0 || slash === rest.length - 1) { throw new Error(`Invalid --output: expected s3:///, got ${raw}`); } return { kind: "s3", bucket: rest.slice(0, slash), key: rest.slice(slash + 1) }; } return { kind: "local", path: raw }; } export async function runTenantSnapshot(opts: RunSnapshotOptions): Promise { assertSuperAdminForNoneProfile(opts.profile, Boolean(opts.superAdminConfirmed)); const logger = opts.logger ?? consola.create({ defaults: { tag: "tenant:snapshot" } }); const now = (opts.now ?? (() => new Date()))(); const createdAt = now.toISOString(); const shortUuid = randomBytes(4).toString("hex"); const snapshotId = `tenant-snapshot-${opts.tenantId}-${createdAt.replace(/[:.]/gu, "")}-${shortUuid}`; const salt = randomBytes(32).toString("hex"); const ctx: AnonymizationContext = { salt, profileName: opts.profile }; const profile = PROFILES[opts.profile]; const tempRoot = await mkdtemp(join(tmpdir(), "tenant-snapshot-")); try { const stageDir = join(tempRoot, snapshotId); await mkdir(stageDir, { recursive: true }); const postgresDir = join(stageDir, "postgres"); await mkdir(postgresDir, { recursive: true }); const pageSize = opts.pageSize ?? 1000; const modelEntries: TenantManifestModelEntry[] = []; for (const entry of TENANT_LIFECYCLE_MODELS) { const filePath = join(postgresDir, `${entry.model}.jsonl`); logger.info(`Extracting model ${entry.model}`); const { rowCount, sizeBytes, sha256 } = await writeModelJsonl({ prisma: opts.prisma, entry, tenantId: opts.tenantId, pageSize, profile, ctx, outPath: filePath, }); modelEntries.push({ model: entry.model, path: `postgres/${entry.model}.jsonl`, rowCount, sizeBytes, checksum: { algorithm: "sha256", value: sha256 }, }); } // After the tenant-scoped slice is staged, harvest referenced // global rows (User + anything else enumerated in // GLOBAL_REFERENCED_MODELS). Streams the JSONL files we just // wrote, collects the FK-id sets per global, fetches the rows from // Prisma in chunks, anonymizes with the same per-snapshot salt // and writes `globals/.jsonl`. let globalsEntries: TenantSnapshotManifest["globals"]; const wantGlobals = opts.includeGlobals !== false; if (wantGlobals) { const globalsRunner = opts.prisma as Partial; if ( typeof globalsRunner.findManyByIds === "function" && typeof globalsRunner.findManyByForeignKeys === "function" ) { const globalsDir = join(stageDir, "globals"); await mkdir(globalsDir, { recursive: true }); const collected = await collectAndWriteGlobals({ postgresDir, globalsDir, prisma: globalsRunner as GlobalsPrismaRunner, profile: opts.profile, ctx, logger, }); if (collected.entries.length > 0) { globalsEntries = collected.entries; } } else { logger.warn( "Snapshot runner does not implement the globals lookup methods — skipping globals extraction. Restore into an empty DB will fail on FKs to global rows.", ); } } let storageEntry: TenantSnapshotManifest["storage"]; if (opts.storageBucket) { const s3 = ensureS3Client(opts); const storageDir = join(stageDir, "storage"); await mkdir(storageDir, { recursive: true }); const storageResult = await extractTenantStorage({ client: s3, bucket: opts.storageBucket, tenantPrefix: opts.storagePrefix ?? `tenants/${opts.tenantId}/`, outDir: storageDir, }); storageEntry = { path: "storage/objects.tar", objectCount: storageResult.objectCount, sizeBytes: storageResult.totalBytes, checksum: { algorithm: "sha256", value: storageResult.checksum }, }; } else { logger.warn("No storage bucket configured; skipping storage component."); } const manifest: TenantSnapshotManifest = { schemaVersion: TENANT_MANIFEST_VERSION, snapshotId, sourceTenantId: opts.tenantId, createdAt, creator: { cliVersion: "0.0.0", host: hostname(), actorUserId: opts.actorUserId }, anonymization: { profile: opts.profile, saltHash: hashSalt(salt) }, models: modelEntries, ...(globalsEntries ? { globals: globalsEntries } : {}), ...(storageEntry ? { storage: storageEntry } : {}), encryption: opts.encryption ? { method: opts.encryption.method, recipient: opts.encryption.recipient } : { method: "none" }, }; const manifestPath = join(stageDir, "manifest.json"); await writeFile(manifestPath, serializeTenantManifest(manifest)); const archivePath = join(tempRoot, `${snapshotId}.tar.gz`); logger.info(`Packaging ${archivePath}`); await createTar({ gzip: true, file: archivePath, cwd: tempRoot, portable: true }, [snapshotId]); let outputPath = archivePath; if (opts.encryption && opts.encryption.method !== "none") { const encryptedPath = `${archivePath}.${opts.encryption.method}`; logger.info(`Encrypting via ${opts.encryption.method}`); await encryptFile({ config: opts.encryption, inputPath: archivePath, outputPath: encryptedPath }); outputPath = encryptedPath; } let uploaded: { bucket: string; key: string } | undefined; if (opts.output.kind === "s3") { const s3 = ensureS3Client(opts); logger.info(`Uploading to s3://${opts.output.bucket}/${opts.output.key}`); await uploadFileMultipart({ client: s3, bucket: opts.output.bucket, key: opts.output.key, filePath: outputPath, contentType: "application/gzip", metadata: { "snapshot-id": snapshotId, "source-tenant-id": opts.tenantId, profile: opts.profile, encryption: opts.encryption?.method ?? "none", "manifest-version": String(TENANT_MANIFEST_VERSION), }, }); uploaded = { bucket: opts.output.bucket, key: opts.output.key }; } else { const { copyFile } = await import("node:fs/promises"); await copyFile(outputPath, opts.output.path); } await writeSnapshotAudit({ entry: { event: "TENANT_SNAPSHOT_CREATED", snapshotId, sourceTenantId: opts.tenantId, profile: opts.profile, actorUserId: opts.actorUserId ?? null, host: hostname(), createdAt, outputTarget: opts.output.kind === "s3" ? `s3://${opts.output.bucket}/${opts.output.key}` : opts.output.path, superAdminConfirmed: Boolean(opts.superAdminConfirmed), }, filePath: opts.auditFilePath, stdout: opts.auditStdout, }); return { snapshotId, manifest, archivePath: outputPath, uploaded }; } finally { await rm(tempRoot, { recursive: true, force: true }); } } interface WriteModelJsonlOptions { prisma: PrismaRunner; entry: TenantLifecycleModel; tenantId: string; pageSize: number; profile: (typeof PROFILES)[ProfileName]; ctx: AnonymizationContext; outPath: string; } async function writeModelJsonl( opts: WriteModelJsonlOptions, ): Promise<{ rowCount: number; sizeBytes: number; sha256: string }> { const stream = createWriteStream(opts.outPath, { encoding: "utf8" }); const hash = createHash("sha256"); let rowCount = 0; const rows = opts.entry.scope === "direct" ? opts.prisma.findManyByTenant( opts.entry.model, opts.entry.tenantField, opts.tenantId, opts.pageSize, ) : opts.prisma.findManyByWhere( opts.entry.model, tenantWhereForLifecycleModel(opts.entry, opts.tenantId), opts.pageSize, ); for await (const row of rows) { const anonymized = anonymizeRow(opts.profile, opts.entry.model, row, opts.ctx); const line = `${JSON.stringify(anonymized, snapshotReplacer)}\n`; hash.update(line); rowCount += 1; if (!stream.write(line)) { await new Promise((resolve) => stream.once("drain", () => resolve())); } } await new Promise((resolve, reject) => { stream.end((err: Error | null | undefined) => (err ? reject(err) : resolve())); }); const s = await stat(opts.outPath); return { rowCount, sizeBytes: s.size, sha256: hash.digest("hex") }; } interface ExtractStorageOptions { client: S3Client; bucket: string; tenantPrefix: string; outDir: string; } interface ExtractStorageResult { objectCount: number; totalBytes: number; checksum: string; } export async function extractTenantStorage(opts: ExtractStorageOptions): Promise { const stagingDir = join(opts.outDir, "_staging"); await mkdir(stagingDir, { recursive: true }); const manifestPath = join(opts.outDir, "manifest.jsonl"); const archivePath = join(opts.outDir, "objects.tar"); const entries: Array<{ key: string; relPath: string; size: number; etag: string | null }> = []; let totalBytes = 0; let continuation: string | undefined; do { const page = await opts.client.send( new ListObjectsV2Command({ Bucket: opts.bucket, Prefix: opts.tenantPrefix, ContinuationToken: continuation, }), ); for (const obj of page.Contents ?? []) { if (!obj.Key) continue; const rel = obj.Key.replace(/^\/+/u, ""); const stagedAbs = join(stagingDir, rel); await mkdir(dirnameOf(stagedAbs), { recursive: true }); const getResp = await opts.client.send(new GetObjectCommand({ Bucket: opts.bucket, Key: obj.Key })); await streamToFile(asReadable(getResp.Body), stagedAbs); entries.push({ key: obj.Key, relPath: rel, size: obj.Size ?? 0, etag: obj.ETag ?? null }); totalBytes += obj.Size ?? 0; } continuation = page.IsTruncated ? page.NextContinuationToken : undefined; } while (continuation); const manifestLines = entries.map((e) => JSON.stringify(e)).join("\n"); await writeFile(manifestPath, manifestLines.length === 0 ? "" : `${manifestLines}\n`); if (entries.length === 0) { const { open } = await import("node:fs/promises"); const fh = await open(archivePath, "w"); await fh.close(); } else { await createTar( { file: archivePath, cwd: stagingDir, portable: true }, entries.map((e) => e.relPath), ); } const checksum = await sha256File(archivePath); try { await rm(stagingDir, { recursive: true, force: true }); } catch { /* ignore */ } return { objectCount: entries.length, totalBytes, checksum }; } function ensureS3Client(opts: RunSnapshotOptions): S3Client { if (opts.s3Client) return opts.s3Client; if (!opts.s3) { throw new Error("S3 client or config is required for this operation"); } return new S3Client(opts.s3); } function dirnameOf(p: string): string { const idx = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\")); return idx === -1 ? "." : p.slice(0, idx); } function asReadable(body: unknown): Readable { if (body && typeof (body as Readable).pipe === "function") { return body as Readable; } if (body && typeof (body as { transformToWebStream?: () => unknown }).transformToWebStream === "function") { const webStream = (body as { transformToWebStream: () => ReadableStream }).transformToWebStream(); return Readable.fromWeb(webStream as never); } if (body && Symbol.asyncIterator in (body as object)) { return Readable.from(body as AsyncIterable); } throw new Error("Unsupported S3 GetObject body type"); } async function streamToFile(stream: Readable, path: string): Promise { const sink = createWriteStream(path); await new Promise((resolve, reject) => { stream.pipe(sink); sink.on("finish", () => resolve()); sink.on("error", reject); stream.on("error", reject); }); } async function sha256File(path: string): Promise { const hash = createHash("sha256"); const stream = createReadStream(path); for await (const chunk of stream) { hash.update(chunk as Buffer); } return hash.digest("hex"); } export const tenantSnapshotCommand = defineCommand({ meta: { name: "snapshot", description: "Create a PII-anonymized snapshot of a single tenant and upload to S3.", }, args: { "tenant-id": { type: "string", required: true, description: "Source tenant id." }, profile: { type: "string", default: "full-anonymized", description: "Anonymization profile: full-anonymized | minimal-anonymized | none.", }, output: { type: "string", description: "Local file path or s3://bucket/key destination." }, "i-am-the-super-admin": { type: "boolean", default: false, description: "Required confirmation flag when --profile=none.", }, "audit-file": { type: "string", description: "Append the structured TENANT_SNAPSHOT_CREATED record to this file.", }, "page-size": { type: "string", description: "Prisma findMany page size (default 1000). Tune for large tenants.", }, "include-globals": { type: "boolean", default: true, description: "Extract referenced global rows (User, etc.) into globals/.jsonl so restore works against an empty target DB. Disable for legacy tenant-only snapshots.", }, }, async run({ args }) { const profile = String(args.profile); if (!isProfileName(profile)) { throw new Error(`Invalid --profile: ${profile}; expected full-anonymized | minimal-anonymized | none.`); } assertSuperAdminForNoneProfile(profile, Boolean(args["i-am-the-super-admin"])); const tenantId = String(args["tenant-id"]); const outputRaw = args.output === undefined ? undefined : String(args.output); const defaultLocal = `./tenant-snapshot-${tenantId}-${new Date().toISOString().replace(/[:.]/gu, "")}.tar.gz`; const output = parseOutputTarget(outputRaw, defaultLocal); const { resolveTenantSnapshotConfig } = await import("./env.js"); const cfg = resolveTenantSnapshotConfig(process.env as Record); const { getDbClient } = await import("@hivegrid/db/client.server"); const { createPrismaSnapshotRunner } = await import("./prisma-runner.js"); const prisma = getDbClient(); const runner = createPrismaSnapshotRunner(prisma as unknown as Record); const pageSize = args["page-size"] ? Number.parseInt(String(args["page-size"]), 10) : undefined; const result = await runTenantSnapshot({ tenantId, profile, output, prisma: runner, s3: cfg.s3, storageBucket: cfg.appStorageBucket ?? undefined, storagePrefix: cfg.storagePrefix, encryption: cfg.encryption.method === "none" ? undefined : cfg.encryption, superAdminConfirmed: Boolean(args["i-am-the-super-admin"]), auditFilePath: args["audit-file"] ? String(args["audit-file"]) : undefined, pageSize: pageSize && Number.isFinite(pageSize) && pageSize > 0 ? pageSize : undefined, includeGlobals: args["include-globals"] !== false, }); consola.success( `Snapshot ${result.snapshotId} written to ${result.uploaded ? `s3://${result.uploaded.bucket}/${result.uploaded.key}` : result.archivePath}`, ); consola.info( `Models: ${result.manifest.models.length} · rows: ${result.manifest.models.reduce((n, m) => n + m.rowCount, 0)} · profile: ${profile}`, ); }, });