Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/packages/cli/src/commands/tenant/restore.ts
T

491 lines
20 KiB
TypeScript

/**
* `tenant:restore` — read a snapshot tarball (local or S3), decrypt if needed,
* extract the manifest + per-model JSONL, replay rows into the target Prisma
* with the new tenant id, then upload tenant storage objects under the new
* `tenants/<new-id>/` prefix.
*
* The Prisma layer is dependency-injected via the `PrismaRestoreRunner`
* interface so the CLI package stays free of `@prisma/client`. Production
* callers wire up a real client in `bin/hivegrid-cli.mjs`; tests pass a fake.
*
* Foreign-key remapping: direct tenant rows receive the target `tenantId`;
* FK-derived children keep their original foreign keys because parent primary
* keys are preserved. The snapshot is intended to land in a separate
* environment, so id collisions with other tenants in that target are not
* handled here.
*/
import { defineCommand } from "citty";
import consola from "consola";
import { createReadStream, createWriteStream } from "node:fs";
import { mkdir, mkdtemp, readFile, rm, stat } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createInterface } from "node:readline";
import { Readable } from "node:stream";
import { extract as extractTar } from "tar";
import {
GetObjectCommand,
ListObjectsV2Command,
S3Client,
type S3ClientConfig,
} from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import {
deserializeTenantManifest,
type TenantSnapshotManifest,
} from "./manifest.js";
import { isTenantDerivedModel } from "./derived-models.js";
import { decryptFile, type EncryptionConfig } from "../backup/encryption.js";
export interface PrismaRestoreRunner {
/**
* Bulk-create rows into `modelName`, returning the number of rows actually
* persisted (after skipping duplicates the caller chose to ignore).
*/
createMany(modelName: string, rows: Record<string, unknown>[]): Promise<{ written: number }>;
/**
* Upsert rows into `modelName` keyed by `id`. Returns counts split by
* whether each row was created fresh or matched an existing row.
* Optional so legacy fake runners only used for the tenant-only path
* keep working — when missing, the restore falls back to createMany
* with skipDuplicates for the globals slice (same semantics as
* `--globals=skip`).
*/
upsertManyById?(
modelName: string,
rows: Record<string, unknown>[],
): Promise<{ created: number; updated: number }>;
}
/**
* How to handle global rows (User, etc.) during restore.
*
* - `upsert` (default): `prisma.<global>.upsert({ where:{id}, ... })` per
* row. Existing globals on the target are left in place; new globals
* get inserted. Best for typical "merge into existing dev/staging DB"
* scenarios.
* - `skip`: `createMany({ skipDuplicates: true })`. Same effect as
* upsert when only inserting; differs in that conflicting unique
* columns OTHER than `id` (e.g. `email`) silently drop the new row.
* - `error`: refuse to restore if any global id already exists on
* target. Useful for "sensitive restore into a known-empty DB"
* where overlap means the target wasn't actually empty.
*/
export type GlobalsRestoreMode = "upsert" | "skip" | "error";
export function isGlobalsRestoreMode(value: string): value is GlobalsRestoreMode {
return value === "upsert" || value === "skip" || value === "error";
}
export interface RunRestoreOptions {
/** Target tenant id the snapshot should land under. */
readonly tenantId: string;
/** Either an s3://bucket/key URL or a local filesystem path. */
readonly source: string;
readonly prisma: PrismaRestoreRunner;
readonly s3?: S3ClientConfig;
readonly s3Client?: S3Client;
readonly storageBucket?: string;
readonly encryption?: EncryptionConfig;
/** When true, skip the interactive confirmation prompt. */
readonly yes?: boolean;
/** Stream used to ask for confirmation; defaults to process.stdin. */
readonly stdin?: NodeJS.ReadableStream;
/** Stream used to print the confirmation question; defaults to process.stdout. */
readonly stdout?: NodeJS.WritableStream;
readonly logger?: { info: (m: string) => void; warn: (m: string) => void };
/** Refuse to run when NODE_ENV=production unless explicitly allowed. */
readonly allowProductionRestore?: boolean;
/**
* Strategy for the globals slice. Defaults to `upsert`. See the
* `GlobalsRestoreMode` docblock for the three modes.
*/
readonly globalsMode?: GlobalsRestoreMode;
}
export interface RunRestoreResult {
readonly manifest: TenantSnapshotManifest;
readonly perModel: ReadonlyArray<{ model: string; written: number; expected: number }>;
/**
* Per-global summary: how many rows the manifest carried, how many
* were inserted fresh, and how many already existed on target (under
* `upsert` they were left in place; under `skip` they were dropped;
* under `error` the restore aborts before this is populated).
*/
readonly globals: ReadonlyArray<{ model: string; expected: number; created: number; existed: number }>;
readonly storageObjectsUploaded: number;
}
export async function runTenantRestore(opts: RunRestoreOptions): Promise<RunRestoreResult> {
const logger = opts.logger ?? consola.create({ defaults: { tag: "tenant:restore" } });
if (process.env.NODE_ENV === "production" && !opts.allowProductionRestore) {
throw new Error(
"Refusing to run tenant:restore in NODE_ENV=production without allowProductionRestore=true (pass --allow-production-restore).",
);
}
const workRoot = await mkdtemp(join(tmpdir(), "tenant-restore-"));
try {
// 1. Download (if S3) into the work dir; decrypt if needed; untar.
const downloadedPath = await materializeSource(opts, workRoot, logger);
let archivePath = downloadedPath;
if (opts.encryption && opts.encryption.method !== "none") {
const decryptedPath = join(workRoot, "decrypted.tar.gz");
logger.info(`Decrypting via ${opts.encryption.method}`);
await decryptFile({ config: opts.encryption, inputPath: archivePath, outputPath: decryptedPath });
archivePath = decryptedPath;
}
const extractDir = join(workRoot, "extracted");
await mkdir(extractDir, { recursive: true });
await extractTar({ file: archivePath, cwd: extractDir });
// 2. Locate the (single) top-level snapshot directory and read the manifest.
const { readdir } = await import("node:fs/promises");
const tops = await readdir(extractDir);
const snapshotDir = tops.find((d) => d.startsWith("tenant-snapshot-"));
if (!snapshotDir) {
throw new Error(`No tenant-snapshot-* directory found inside ${archivePath}`);
}
const stageDir = join(extractDir, snapshotDir);
const manifestRaw = await readFile(join(stageDir, "manifest.json"), "utf8");
const manifest = deserializeTenantManifest(manifestRaw);
logger.info(
`Manifest: source=${manifest.sourceTenantId} profile=${manifest.anonymization.profile} models=${manifest.models.length}`,
);
// 3. Confirmation prompt unless --yes.
if (!opts.yes) {
const answered = await promptYesNo(
`About to restore snapshot of tenant ${manifest.sourceTenantId} into ${opts.tenantId}. Continue?`,
opts.stdin ?? process.stdin,
opts.stdout ?? process.stdout,
);
if (!answered) {
throw new Error("Aborted by operator.");
}
}
// 4a. Replay globals FIRST (in FK-correct order) so the tenant-scoped
// models that follow can resolve their FK columns (User, etc.).
// Manifests created before this extractor landed carry no `globals`
// array — we tolerate that and warn so the operator knows they still
// need pre-seeded globals on the target DB.
const mode: GlobalsRestoreMode = opts.globalsMode ?? "upsert";
const globalsSummary: { model: string; expected: number; created: number; existed: number }[] = [];
if (manifest.globals && manifest.globals.length > 0) {
logger.info(`Replaying ${manifest.globals.length} globals slice(s) with mode=${mode}.`);
for (const entry of manifest.globals) {
const filePath = join(stageDir, entry.path);
const rows = await readJsonl(filePath);
// Globals are NOT remapped onto a new tenantId — they live in
// global space. Pass rows through verbatim.
const summary = await replayGlobalsForModel({
modelName: entry.model,
rows,
mode,
prisma: opts.prisma,
logger,
});
globalsSummary.push({
model: entry.model,
expected: entry.rowCount,
created: summary.created,
existed: summary.existed,
});
}
} else {
logger.warn(
"Manifest carries no globals slice (legacy snapshot). Tenant-scoped models with FKs into global rows will fail unless those rows already exist on the target DB.",
);
}
// 4b. Replay each tenant-scoped model with the new tenantId.
const perModel: { model: string; written: number; expected: number }[] = [];
for (const entry of manifest.models) {
const filePath = join(stageDir, entry.path);
const rows = await readJsonl(filePath);
const remapped = isTenantDerivedModel(entry.model)
? rows
: rows.map((row) => ({ ...row, tenantId: opts.tenantId }));
const result = await opts.prisma.createMany(entry.model, remapped);
perModel.push({ model: entry.model, written: result.written, expected: entry.rowCount });
if (result.written !== entry.rowCount) {
logger.warn(`Model ${entry.model}: wrote ${result.written} rows but manifest expected ${entry.rowCount}.`);
}
}
// 5. Re-upload storage objects to the new tenant prefix.
let storageObjectsUploaded = 0;
if (manifest.storage && opts.storageBucket) {
storageObjectsUploaded = await restoreStorage({
stageDir,
storageEntry: manifest.storage,
targetBucket: opts.storageBucket,
targetPrefix: `tenants/${opts.tenantId}/`,
sourcePrefix: `tenants/${manifest.sourceTenantId}/`,
client: ensureS3Client(opts),
});
}
// 6. Sanity check: per-model row counts must match within tolerance.
const mismatched = perModel.filter((m) => m.written !== m.expected);
if (mismatched.length > 0) {
logger.warn(`Row-count mismatch in ${mismatched.length} models. See restore log for details.`);
}
return { manifest, perModel, globals: globalsSummary, storageObjectsUploaded };
} finally {
await rm(workRoot, { recursive: true, force: true });
}
}
interface ReplayGlobalsForModelOptions {
readonly modelName: string;
readonly rows: Record<string, unknown>[];
readonly mode: GlobalsRestoreMode;
readonly prisma: PrismaRestoreRunner;
readonly logger: { info: (m: string) => void; warn: (m: string) => void };
}
/**
* Insert/upsert globals according to the chosen mode.
*
* Strategy
* --------
* The PrismaRestoreRunner interface treats `upsertManyById` as
* optional. When the production wiring provides it (real PrismaClient,
* see `prisma-runner.ts`), `upsert` and `error` modes use it. Test
* fakes that only implement `createMany` fall through to a
* `createMany({skipDuplicates:true})` shim and emit a warning telling
* the operator the mode they asked for could not be honored.
*
* `error` mode pre-checks for any pre-existing id by performing an
* upsert that throws on update (the runner's upsertManyById returns
* `updated > 0` when a row already existed; we treat that as a fail).
*/
async function replayGlobalsForModel(
opts: ReplayGlobalsForModelOptions,
): Promise<{ created: number; existed: number }> {
if (opts.rows.length === 0) {
return { created: 0, existed: 0 };
}
const supportsUpsert = typeof opts.prisma.upsertManyById === "function";
if (opts.mode === "skip" || !supportsUpsert) {
if (opts.mode !== "skip" && !supportsUpsert) {
opts.logger.warn(
`Globals restore mode=${opts.mode} requested but runner does not implement upsertManyById; falling back to skip-duplicates semantics.`,
);
}
const result = await opts.prisma.createMany(opts.modelName, opts.rows);
return { created: result.written, existed: opts.rows.length - result.written };
}
const result = await opts.prisma.upsertManyById!(opts.modelName, opts.rows);
if (opts.mode === "error" && result.updated > 0) {
throw new Error(
`tenant:restore aborted (--globals=error): ${opts.modelName} had ${result.updated} pre-existing row(s) on the target. Pass --globals=upsert to overlay snapshot rows over existing ones, or --globals=skip to leave them in place.`,
);
}
return { created: result.created, existed: result.updated };
}
async function materializeSource(
opts: RunRestoreOptions,
workRoot: string,
logger: { info: (m: string) => void; warn: (m: string) => void },
): Promise<string> {
if (opts.source.startsWith("s3://")) {
const rest = opts.source.slice("s3://".length);
const slash = rest.indexOf("/");
if (slash <= 0) throw new Error(`Invalid --source: ${opts.source}`);
const bucket = rest.slice(0, slash);
const key = rest.slice(slash + 1);
const dest = join(workRoot, "snapshot.tar.gz");
logger.info(`Downloading s3://${bucket}/${key}`);
const client = ensureS3Client(opts);
const resp = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
await streamToFile(asReadable(resp.Body), dest);
return dest;
}
return opts.source;
}
async function readJsonl(path: string): Promise<Record<string, unknown>[]> {
try {
await stat(path);
} catch {
return [];
}
const rows: Record<string, unknown>[] = [];
const stream = createReadStream(path, { encoding: "utf8" });
const rl = createInterface({ input: stream, crlfDelay: Infinity });
for await (const line of rl) {
if (line.trim().length === 0) continue;
rows.push(JSON.parse(line) as Record<string, unknown>);
}
return rows;
}
interface RestoreStorageOptions {
stageDir: string;
storageEntry: NonNullable<TenantSnapshotManifest["storage"]>;
targetBucket: string;
targetPrefix: string;
sourcePrefix: string;
client: S3Client;
}
async function restoreStorage(opts: RestoreStorageOptions): Promise<number> {
const storageDir = join(opts.stageDir, "storage");
const manifestLines = (await readFile(join(storageDir, "manifest.jsonl"), "utf8")).split("\n").filter((l) => l.length > 0);
if (manifestLines.length === 0) return 0;
const archive = join(storageDir, "objects.tar");
const stagingDir = join(opts.stageDir, "_storage_staging");
await mkdir(stagingDir, { recursive: true });
await extractTar({ file: archive, cwd: stagingDir });
let uploaded = 0;
for (const line of manifestLines) {
const entry = JSON.parse(line) as { key: string; relPath: string };
const localPath = join(stagingDir, entry.relPath);
const newKey = entry.key.startsWith(opts.sourcePrefix)
? `${opts.targetPrefix}${entry.key.slice(opts.sourcePrefix.length)}`
: `${opts.targetPrefix}${entry.relPath}`;
const stream = createReadStream(localPath);
const upload = new Upload({
client: opts.client,
params: { Bucket: opts.targetBucket, Key: newKey, Body: stream },
});
await upload.done();
uploaded += 1;
}
return uploaded;
}
function promptYesNo(
question: string,
stdin: NodeJS.ReadableStream,
stdout: NodeJS.WritableStream,
): Promise<boolean> {
return new Promise((resolve) => {
const rl = createInterface({ input: stdin, output: stdout });
rl.question(`${question} [y/N] `, (answer) => {
rl.close();
resolve(/^y(es)?$/iu.test(answer.trim()));
});
});
}
function ensureS3Client(opts: RunRestoreOptions): S3Client {
if (opts.s3Client) return opts.s3Client;
if (!opts.s3) throw new Error("S3 client or config is required");
return new S3Client(opts.s3);
}
function asReadable(body: unknown): Readable {
if (body && typeof (body as Readable).pipe === "function") {
return body as Readable;
}
if (body && typeof (body as { transformToWebStream?: () => unknown }).transformToWebStream === "function") {
const webStream = (body as { transformToWebStream: () => ReadableStream<Uint8Array> }).transformToWebStream();
return Readable.fromWeb(webStream as never);
}
if (body && Symbol.asyncIterator in (body as object)) {
return Readable.from(body as AsyncIterable<Buffer>);
}
throw new Error("Unsupported S3 GetObject body type");
}
async function streamToFile(stream: Readable, path: string): Promise<void> {
const sink = createWriteStream(path);
await new Promise<void>((resolve, reject) => {
stream.pipe(sink);
sink.on("finish", () => resolve());
sink.on("error", reject);
stream.on("error", reject);
});
}
// Suppress the unused-import lint for ListObjectsV2Command; it remains in the
// import list so a future iteration that wants to discover unknown keys in
// the bucket can pick it up without re-importing.
void ListObjectsV2Command;
export const tenantRestoreCommand = defineCommand({
meta: {
name: "restore",
description: "Restore a tenant snapshot into a new tenant id (FK-remapped).",
},
args: {
"tenant-id": { type: "string", required: true, description: "Target tenant id to create." },
source: { type: "string", required: true, description: "s3://bucket/key or local path of the snapshot." },
yes: { type: "boolean", default: false, description: "Skip confirmation prompts." },
"allow-production-restore": {
type: "boolean",
default: false,
description: "Permit restore when NODE_ENV=production (DANGEROUS).",
},
globals: {
type: "string",
default: "upsert",
description:
"How to handle global rows (User, etc.) on the target: upsert (default, idempotent overlay) | skip (leave conflicting rows in place) | error (abort if any global already exists — for sensitive restores into a known-empty DB).",
},
},
async run({ args }) {
const { resolveTenantSnapshotConfig } = await import("./env.js");
const cfg = resolveTenantSnapshotConfig(process.env as Record<string, string | undefined>);
const { getDbClient } = await import("@hivegrid/db/client.server");
const { createPrismaRestoreRunner } = await import("./prisma-runner.js");
const { parseSchemaScalarTypes } = await import("./json-codec.js");
const { dirname, join } = await import("node:path");
const { fileURLToPath } = await import("node:url");
const here = dirname(fileURLToPath(import.meta.url));
// From packages/cli/src/commands/tenant/restore.ts → repo root is 5 levels up.
const schemaDir = join(here, "..", "..", "..", "..", "..", "prisma", "schema");
const scalarTypes = parseSchemaScalarTypes(schemaDir);
const prisma = getDbClient();
const runner = createPrismaRestoreRunner(prisma as unknown as Record<string, unknown>, scalarTypes);
const globalsRaw = args.globals === undefined ? "upsert" : String(args.globals);
if (!isGlobalsRestoreMode(globalsRaw)) {
throw new Error(`Invalid --globals: ${globalsRaw}; expected upsert | skip | error.`);
}
const result = await runTenantRestore({
tenantId: String(args["tenant-id"]),
source: String(args.source),
prisma: runner,
s3: cfg.s3,
storageBucket: cfg.appStorageBucket ?? undefined,
encryption: cfg.encryption.method === "none" ? undefined : cfg.encryption,
yes: Boolean(args.yes),
allowProductionRestore: Boolean(args["allow-production-restore"]),
globalsMode: globalsRaw,
});
const totalWritten = result.perModel.reduce((n, m) => n + m.written, 0);
const totalExpected = result.perModel.reduce((n, m) => n + m.expected, 0);
const mismatches = result.perModel.filter((m) => m.written !== m.expected);
consola.success(
`Restored snapshot of tenant ${result.manifest.sourceTenantId} → ${String(args["tenant-id"])}: ${totalWritten}/${totalExpected} rows across ${result.perModel.length} models.`,
);
if (result.globals.length > 0) {
const summary = result.globals
.map((g) => `${g.model}(created=${g.created} existed=${g.existed} expected=${g.expected})`)
.join(", ");
consola.info(`Globals (mode=${globalsRaw}): ${summary}.`);
}
if (result.storageObjectsUploaded > 0) {
consola.info(`Storage objects re-uploaded: ${result.storageObjectsUploaded}.`);
}
if (mismatches.length > 0) {
consola.warn(
`Row-count mismatch in ${mismatches.length} models: ${mismatches.map((m) => `${m.model}(${m.written}/${m.expected})`).join(", ")}`,
);
}
},
});