Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/packages/cli/src/commands/tenant/snapshot.ts
T

565 lines
21 KiB
TypeScript

/**
* `tenant:snapshot` — extract a single tenant from Postgres + storage, apply
* the chosen PII anonymization profile, tar + (optionally) encrypt + upload.
*
* The Prisma layer is dependency-injected via the `PrismaRunner` interface so
* the CLI package stays free of `@prisma/client`. The production entrypoint
* (`bin/hivegrid-cli.mjs` of the consumer) wires up the real `PrismaClient`;
* tests pass a fake.
*/
import { defineCommand } from "citty";
import consola from "consola";
import { randomBytes, createHash } from "node:crypto";
import { createWriteStream, createReadStream } from "node:fs";
import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir, hostname } from "node:os";
import { join } from "node:path";
import { Readable } from "node:stream";
import {
GetObjectCommand,
ListObjectsV2Command,
S3Client,
type S3ClientConfig,
} from "@aws-sdk/client-s3";
import { create as createTar } from "tar";
import {
PROFILES,
anonymizeRow,
isProfileName,
type AnonymizationContext,
type ProfileName,
} from "./anonymize.js";
import {
TENANT_MANIFEST_VERSION,
hashSalt,
serializeTenantManifest,
type TenantManifestModelEntry,
type TenantSnapshotManifest,
} from "./manifest.js";
import {
TENANT_LIFECYCLE_MODELS,
tenantWhereForLifecycleModel,
type TenantLifecycleModel,
} from "./derived-models.js";
import { assertSuperAdminForNoneProfile, writeSnapshotAudit } from "./audit.js";
import { snapshotReplacer } from "./json-codec.js";
import { collectAndWriteGlobals, type GlobalsPrismaRunner } from "./globals-runner.js";
import { encryptFile, type EncryptionConfig } from "../backup/encryption.js";
import { uploadFileMultipart } from "../backup/s3-upload.js";
/**
* Minimal Prisma-like surface the snapshot command needs.
*
* `findManyByIds` is added for the globals extractor (B7 follow-up):
* after collecting the tenant slice we re-issue id-set lookups against
* the source DB to fetch the referenced `User` rows (and any other
* globals enumerated in `models.ts::GLOBAL_REFERENCED_MODELS`). It is
* optional so legacy callers that only snapshot tenant-scoped rows
* (e.g. `--include-globals=false`) can still pass a slimmer runner.
*/
export interface PrismaRunner extends Partial<GlobalsPrismaRunner> {
/**
* Stream rows from `modelName` where `tenantId = tenantId` in stable id order.
* Implementations should use cursor-based pagination so large tenants do
* not OOM. The default page size is 1000 unless the caller overrides.
*/
findManyByTenant(
modelName: string,
tenantField: string,
tenantId: string,
pageSize: number,
): AsyncIterable<Record<string, unknown>>;
/**
* Stream FK-derived tenant rows using a relation-filter where clause.
* This is separate from findManyByTenant because these models deliberately
* have no tenantId scalar of their own.
*/
findManyByWhere(
modelName: string,
where: Readonly<Record<string, unknown>>,
pageSize: number,
): AsyncIterable<Record<string, unknown>>;
}
export interface SnapshotS3Output {
readonly kind: "s3";
readonly bucket: string;
readonly key: string;
}
export interface SnapshotLocalOutput {
readonly kind: "local";
readonly path: string;
}
export type SnapshotOutputTarget = SnapshotS3Output | SnapshotLocalOutput;
export interface RunSnapshotOptions {
readonly tenantId: string;
readonly profile: ProfileName;
readonly output: SnapshotOutputTarget;
readonly prisma: PrismaRunner;
readonly s3?: S3ClientConfig;
readonly s3Client?: S3Client;
readonly storageBucket?: string;
readonly storagePrefix?: string;
readonly encryption?: EncryptionConfig;
readonly now?: () => Date;
/** Marker the operator must pass when invoking `--profile=none`. */
readonly superAdminConfirmed?: boolean;
/** Optional file path to append the snapshot-audit JSON record to. */
readonly auditFilePath?: string;
/** Stream the human-readable audit line is written to (defaults to stdout). */
readonly auditStdout?: NodeJS.WritableStream;
readonly logger?: { info: (m: string) => void; warn: (m: string) => void };
readonly actorUserId?: string;
readonly pageSize?: number;
/**
* Emit `globals/<Model>.jsonl` files for every model in
* `GLOBAL_REFERENCED_MODELS`. Defaults to `true`. Set to `false` for
* backward-compatible "tenant-only" snapshots (the operator must then
* pre-seed referenced globals on the target before restore).
*
* Requires `prisma.findManyByIds` to be implemented. If a runner
* without that method is supplied with `includeGlobals=true`, the
* snapshot logs a warning and skips the globals step rather than
* failing — useful for unit tests that only exercise the tenant path.
*/
readonly includeGlobals?: boolean;
}
export interface RunSnapshotResult {
readonly snapshotId: string;
readonly manifest: TenantSnapshotManifest;
readonly archivePath: string;
readonly uploaded?: { bucket: string; key: string };
}
/**
* Parse an --output value into a structured target. Accepts `s3://bucket/key`
* or a local filesystem path. Returns `null` if undefined.
*/
export function parseOutputTarget(raw: string | undefined, defaultLocalPath: string): SnapshotOutputTarget {
if (!raw || raw.length === 0) {
return { kind: "local", path: defaultLocalPath };
}
if (raw.startsWith("s3://")) {
const rest = raw.slice("s3://".length);
const slash = rest.indexOf("/");
if (slash <= 0 || slash === rest.length - 1) {
throw new Error(`Invalid --output: expected s3://<bucket>/<key>, got ${raw}`);
}
return { kind: "s3", bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
}
return { kind: "local", path: raw };
}
export async function runTenantSnapshot(opts: RunSnapshotOptions): Promise<RunSnapshotResult> {
assertSuperAdminForNoneProfile(opts.profile, Boolean(opts.superAdminConfirmed));
const logger = opts.logger ?? consola.create({ defaults: { tag: "tenant:snapshot" } });
const now = (opts.now ?? (() => new Date()))();
const createdAt = now.toISOString();
const shortUuid = randomBytes(4).toString("hex");
const snapshotId = `tenant-snapshot-${opts.tenantId}-${createdAt.replace(/[:.]/gu, "")}-${shortUuid}`;
const salt = randomBytes(32).toString("hex");
const ctx: AnonymizationContext = { salt, profileName: opts.profile };
const profile = PROFILES[opts.profile];
const tempRoot = await mkdtemp(join(tmpdir(), "tenant-snapshot-"));
try {
const stageDir = join(tempRoot, snapshotId);
await mkdir(stageDir, { recursive: true });
const postgresDir = join(stageDir, "postgres");
await mkdir(postgresDir, { recursive: true });
const pageSize = opts.pageSize ?? 1000;
const modelEntries: TenantManifestModelEntry[] = [];
for (const entry of TENANT_LIFECYCLE_MODELS) {
const filePath = join(postgresDir, `${entry.model}.jsonl`);
logger.info(`Extracting model ${entry.model}`);
const { rowCount, sizeBytes, sha256 } = await writeModelJsonl({
prisma: opts.prisma,
entry,
tenantId: opts.tenantId,
pageSize,
profile,
ctx,
outPath: filePath,
});
modelEntries.push({
model: entry.model,
path: `postgres/${entry.model}.jsonl`,
rowCount,
sizeBytes,
checksum: { algorithm: "sha256", value: sha256 },
});
}
// After the tenant-scoped slice is staged, harvest referenced
// global rows (User + anything else enumerated in
// GLOBAL_REFERENCED_MODELS). Streams the JSONL files we just
// wrote, collects the FK-id sets per global, fetches the rows from
// Prisma in chunks, anonymizes with the same per-snapshot salt
// and writes `globals/<Model>.jsonl`.
let globalsEntries: TenantSnapshotManifest["globals"];
const wantGlobals = opts.includeGlobals !== false;
if (wantGlobals) {
const globalsRunner = opts.prisma as Partial<GlobalsPrismaRunner>;
if (
typeof globalsRunner.findManyByIds === "function" &&
typeof globalsRunner.findManyByForeignKeys === "function"
) {
const globalsDir = join(stageDir, "globals");
await mkdir(globalsDir, { recursive: true });
const collected = await collectAndWriteGlobals({
postgresDir,
globalsDir,
prisma: globalsRunner as GlobalsPrismaRunner,
profile: opts.profile,
ctx,
logger,
});
if (collected.entries.length > 0) {
globalsEntries = collected.entries;
}
} else {
logger.warn(
"Snapshot runner does not implement the globals lookup methods — skipping globals extraction. Restore into an empty DB will fail on FKs to global rows.",
);
}
}
let storageEntry: TenantSnapshotManifest["storage"];
if (opts.storageBucket) {
const s3 = ensureS3Client(opts);
const storageDir = join(stageDir, "storage");
await mkdir(storageDir, { recursive: true });
const storageResult = await extractTenantStorage({
client: s3,
bucket: opts.storageBucket,
tenantPrefix: opts.storagePrefix ?? `tenants/${opts.tenantId}/`,
outDir: storageDir,
});
storageEntry = {
path: "storage/objects.tar",
objectCount: storageResult.objectCount,
sizeBytes: storageResult.totalBytes,
checksum: { algorithm: "sha256", value: storageResult.checksum },
};
} else {
logger.warn("No storage bucket configured; skipping storage component.");
}
const manifest: TenantSnapshotManifest = {
schemaVersion: TENANT_MANIFEST_VERSION,
snapshotId,
sourceTenantId: opts.tenantId,
createdAt,
creator: { cliVersion: "0.0.0", host: hostname(), actorUserId: opts.actorUserId },
anonymization: { profile: opts.profile, saltHash: hashSalt(salt) },
models: modelEntries,
...(globalsEntries ? { globals: globalsEntries } : {}),
...(storageEntry ? { storage: storageEntry } : {}),
encryption: opts.encryption
? { method: opts.encryption.method, recipient: opts.encryption.recipient }
: { method: "none" },
};
const manifestPath = join(stageDir, "manifest.json");
await writeFile(manifestPath, serializeTenantManifest(manifest));
const archivePath = join(tempRoot, `${snapshotId}.tar.gz`);
logger.info(`Packaging ${archivePath}`);
await createTar({ gzip: true, file: archivePath, cwd: tempRoot, portable: true }, [snapshotId]);
let outputPath = archivePath;
if (opts.encryption && opts.encryption.method !== "none") {
const encryptedPath = `${archivePath}.${opts.encryption.method}`;
logger.info(`Encrypting via ${opts.encryption.method}`);
await encryptFile({ config: opts.encryption, inputPath: archivePath, outputPath: encryptedPath });
outputPath = encryptedPath;
}
let uploaded: { bucket: string; key: string } | undefined;
if (opts.output.kind === "s3") {
const s3 = ensureS3Client(opts);
logger.info(`Uploading to s3://${opts.output.bucket}/${opts.output.key}`);
await uploadFileMultipart({
client: s3,
bucket: opts.output.bucket,
key: opts.output.key,
filePath: outputPath,
contentType: "application/gzip",
metadata: {
"snapshot-id": snapshotId,
"source-tenant-id": opts.tenantId,
profile: opts.profile,
encryption: opts.encryption?.method ?? "none",
"manifest-version": String(TENANT_MANIFEST_VERSION),
},
});
uploaded = { bucket: opts.output.bucket, key: opts.output.key };
} else {
const { copyFile } = await import("node:fs/promises");
await copyFile(outputPath, opts.output.path);
}
await writeSnapshotAudit({
entry: {
event: "TENANT_SNAPSHOT_CREATED",
snapshotId,
sourceTenantId: opts.tenantId,
profile: opts.profile,
actorUserId: opts.actorUserId ?? null,
host: hostname(),
createdAt,
outputTarget:
opts.output.kind === "s3"
? `s3://${opts.output.bucket}/${opts.output.key}`
: opts.output.path,
superAdminConfirmed: Boolean(opts.superAdminConfirmed),
},
filePath: opts.auditFilePath,
stdout: opts.auditStdout,
});
return { snapshotId, manifest, archivePath: outputPath, uploaded };
} finally {
await rm(tempRoot, { recursive: true, force: true });
}
}
interface WriteModelJsonlOptions {
prisma: PrismaRunner;
entry: TenantLifecycleModel;
tenantId: string;
pageSize: number;
profile: (typeof PROFILES)[ProfileName];
ctx: AnonymizationContext;
outPath: string;
}
async function writeModelJsonl(
opts: WriteModelJsonlOptions,
): Promise<{ rowCount: number; sizeBytes: number; sha256: string }> {
const stream = createWriteStream(opts.outPath, { encoding: "utf8" });
const hash = createHash("sha256");
let rowCount = 0;
const rows =
opts.entry.scope === "direct"
? opts.prisma.findManyByTenant(
opts.entry.model,
opts.entry.tenantField,
opts.tenantId,
opts.pageSize,
)
: opts.prisma.findManyByWhere(
opts.entry.model,
tenantWhereForLifecycleModel(opts.entry, opts.tenantId),
opts.pageSize,
);
for await (const row of rows) {
const anonymized = anonymizeRow(opts.profile, opts.entry.model, row, opts.ctx);
const line = `${JSON.stringify(anonymized, snapshotReplacer)}\n`;
hash.update(line);
rowCount += 1;
if (!stream.write(line)) {
await new Promise<void>((resolve) => stream.once("drain", () => resolve()));
}
}
await new Promise<void>((resolve, reject) => {
stream.end((err: Error | null | undefined) => (err ? reject(err) : resolve()));
});
const s = await stat(opts.outPath);
return { rowCount, sizeBytes: s.size, sha256: hash.digest("hex") };
}
interface ExtractStorageOptions {
client: S3Client;
bucket: string;
tenantPrefix: string;
outDir: string;
}
interface ExtractStorageResult {
objectCount: number;
totalBytes: number;
checksum: string;
}
export async function extractTenantStorage(opts: ExtractStorageOptions): Promise<ExtractStorageResult> {
const stagingDir = join(opts.outDir, "_staging");
await mkdir(stagingDir, { recursive: true });
const manifestPath = join(opts.outDir, "manifest.jsonl");
const archivePath = join(opts.outDir, "objects.tar");
const entries: Array<{ key: string; relPath: string; size: number; etag: string | null }> = [];
let totalBytes = 0;
let continuation: string | undefined;
do {
const page = await opts.client.send(
new ListObjectsV2Command({
Bucket: opts.bucket,
Prefix: opts.tenantPrefix,
ContinuationToken: continuation,
}),
);
for (const obj of page.Contents ?? []) {
if (!obj.Key) continue;
const rel = obj.Key.replace(/^\/+/u, "");
const stagedAbs = join(stagingDir, rel);
await mkdir(dirnameOf(stagedAbs), { recursive: true });
const getResp = await opts.client.send(new GetObjectCommand({ Bucket: opts.bucket, Key: obj.Key }));
await streamToFile(asReadable(getResp.Body), stagedAbs);
entries.push({ key: obj.Key, relPath: rel, size: obj.Size ?? 0, etag: obj.ETag ?? null });
totalBytes += obj.Size ?? 0;
}
continuation = page.IsTruncated ? page.NextContinuationToken : undefined;
} while (continuation);
const manifestLines = entries.map((e) => JSON.stringify(e)).join("\n");
await writeFile(manifestPath, manifestLines.length === 0 ? "" : `${manifestLines}\n`);
if (entries.length === 0) {
const { open } = await import("node:fs/promises");
const fh = await open(archivePath, "w");
await fh.close();
} else {
await createTar(
{ file: archivePath, cwd: stagingDir, portable: true },
entries.map((e) => e.relPath),
);
}
const checksum = await sha256File(archivePath);
try {
await rm(stagingDir, { recursive: true, force: true });
} catch {
/* ignore */
}
return { objectCount: entries.length, totalBytes, checksum };
}
function ensureS3Client(opts: RunSnapshotOptions): S3Client {
if (opts.s3Client) return opts.s3Client;
if (!opts.s3) {
throw new Error("S3 client or config is required for this operation");
}
return new S3Client(opts.s3);
}
function dirnameOf(p: string): string {
const idx = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\"));
return idx === -1 ? "." : p.slice(0, idx);
}
function asReadable(body: unknown): Readable {
if (body && typeof (body as Readable).pipe === "function") {
return body as Readable;
}
if (body && typeof (body as { transformToWebStream?: () => unknown }).transformToWebStream === "function") {
const webStream = (body as { transformToWebStream: () => ReadableStream<Uint8Array> }).transformToWebStream();
return Readable.fromWeb(webStream as never);
}
if (body && Symbol.asyncIterator in (body as object)) {
return Readable.from(body as AsyncIterable<Buffer>);
}
throw new Error("Unsupported S3 GetObject body type");
}
async function streamToFile(stream: Readable, path: string): Promise<void> {
const sink = createWriteStream(path);
await new Promise<void>((resolve, reject) => {
stream.pipe(sink);
sink.on("finish", () => resolve());
sink.on("error", reject);
stream.on("error", reject);
});
}
async function sha256File(path: string): Promise<string> {
const hash = createHash("sha256");
const stream = createReadStream(path);
for await (const chunk of stream) {
hash.update(chunk as Buffer);
}
return hash.digest("hex");
}
export const tenantSnapshotCommand = defineCommand({
meta: {
name: "snapshot",
description: "Create a PII-anonymized snapshot of a single tenant and upload to S3.",
},
args: {
"tenant-id": { type: "string", required: true, description: "Source tenant id." },
profile: {
type: "string",
default: "full-anonymized",
description: "Anonymization profile: full-anonymized | minimal-anonymized | none.",
},
output: { type: "string", description: "Local file path or s3://bucket/key destination." },
"i-am-the-super-admin": {
type: "boolean",
default: false,
description: "Required confirmation flag when --profile=none.",
},
"audit-file": {
type: "string",
description: "Append the structured TENANT_SNAPSHOT_CREATED record to this file.",
},
"page-size": {
type: "string",
description: "Prisma findMany page size (default 1000). Tune for large tenants.",
},
"include-globals": {
type: "boolean",
default: true,
description:
"Extract referenced global rows (User, etc.) into globals/<Model>.jsonl so restore works against an empty target DB. Disable for legacy tenant-only snapshots.",
},
},
async run({ args }) {
const profile = String(args.profile);
if (!isProfileName(profile)) {
throw new Error(`Invalid --profile: ${profile}; expected full-anonymized | minimal-anonymized | none.`);
}
assertSuperAdminForNoneProfile(profile, Boolean(args["i-am-the-super-admin"]));
const tenantId = String(args["tenant-id"]);
const outputRaw = args.output === undefined ? undefined : String(args.output);
const defaultLocal = `./tenant-snapshot-${tenantId}-${new Date().toISOString().replace(/[:.]/gu, "")}.tar.gz`;
const output = parseOutputTarget(outputRaw, defaultLocal);
const { resolveTenantSnapshotConfig } = await import("./env.js");
const cfg = resolveTenantSnapshotConfig(process.env as Record<string, string | undefined>);
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createPrismaSnapshotRunner } = await import("./prisma-runner.js");
const prisma = getDbClient();
const runner = createPrismaSnapshotRunner(prisma as unknown as Record<string, unknown>);
const pageSize = args["page-size"] ? Number.parseInt(String(args["page-size"]), 10) : undefined;
const result = await runTenantSnapshot({
tenantId,
profile,
output,
prisma: runner,
s3: cfg.s3,
storageBucket: cfg.appStorageBucket ?? undefined,
storagePrefix: cfg.storagePrefix,
encryption: cfg.encryption.method === "none" ? undefined : cfg.encryption,
superAdminConfirmed: Boolean(args["i-am-the-super-admin"]),
auditFilePath: args["audit-file"] ? String(args["audit-file"]) : undefined,
pageSize: pageSize && Number.isFinite(pageSize) && pageSize > 0 ? pageSize : undefined,
includeGlobals: args["include-globals"] !== false,
});
consola.success(
`Snapshot ${result.snapshotId} written to ${result.uploaded ? `s3://${result.uploaded.bucket}/${result.uploaded.key}` : result.archivePath}`,
);
consola.info(
`Models: ${result.manifest.models.length} · rows: ${result.manifest.models.reduce((n, m) => n + m.rowCount, 0)} · profile: ${profile}`,
);
},
});