Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,177 @@
|
||||
import { defineCommand } from "citty";
|
||||
import consola from "consola";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { create as createTar } from "tar";
|
||||
import { hostname } from "node:os";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
BACKUP_MANIFEST_VERSION,
|
||||
type BackupManifest,
|
||||
classifyBackup,
|
||||
serializeManifest,
|
||||
} from "./manifest.js";
|
||||
import { dumpPostgresToFile } from "./postgres.js";
|
||||
import { backupStorageBucket } from "./storage.js";
|
||||
import { maybeSnapshotClickhouse } from "./clickhouse.js";
|
||||
import { encryptFile } from "./encryption.js";
|
||||
import { uploadFileMultipart } from "./s3-upload.js";
|
||||
import { resolveBackupConfig, type BackupEnv } from "./env.js";
|
||||
|
||||
export interface RunCreateBackupOptions {
|
||||
env?: BackupEnv;
|
||||
/** Inject a pre-built S3 client (tests). */
|
||||
client?: S3Client;
|
||||
/** Override now() for deterministic naming. */
|
||||
now?: () => Date;
|
||||
/** Skip storage component (CLI --skip-storage flag). */
|
||||
skipStorage?: boolean;
|
||||
/** Logger override (tests). */
|
||||
logger?: { info: (msg: string) => void; warn: (msg: string) => void };
|
||||
}
|
||||
|
||||
export interface RunCreateBackupResult {
|
||||
backupId: string;
|
||||
key: string;
|
||||
sizeBytes: number;
|
||||
manifest: BackupManifest;
|
||||
}
|
||||
|
||||
/** Run the backup-create flow. Exposed as a function so other commands and tests can re-use it. */
|
||||
export async function runCreateBackup(opts: RunCreateBackupOptions = {}): Promise<RunCreateBackupResult> {
|
||||
const env = opts.env ?? (process.env as unknown as BackupEnv);
|
||||
const logger = opts.logger ?? consola.create({ defaults: { tag: "backup:create" } });
|
||||
const config = resolveBackupConfig(env);
|
||||
const now = (opts.now ?? (() => new Date()))();
|
||||
const createdAt = now.toISOString();
|
||||
const shortUuid = randomBytes(4).toString("hex");
|
||||
const backupId = `backup-${createdAt.replace(/[:.]/gu, "")}-${shortUuid}`;
|
||||
const objectKey = `${backupId}.tar.gz${config.encryption.method === "none" ? "" : `.${config.encryption.method}`}`;
|
||||
|
||||
const tempRoot = await mkdtemp(join(tmpdir(), "backup-create-"));
|
||||
try {
|
||||
const stageDir = join(tempRoot, backupId);
|
||||
await mkdir(stageDir, { recursive: true });
|
||||
|
||||
logger.info(`Dumping Postgres into ${stageDir}/postgres/dump.sql.gz`);
|
||||
const pgDir = join(stageDir, "postgres");
|
||||
await mkdir(pgDir, { recursive: true });
|
||||
const pgPath = join(pgDir, "dump.sql.gz");
|
||||
const pgResult = await dumpPostgresToFile({ connectionString: config.databaseUrl, outPath: pgPath });
|
||||
|
||||
let storageEntry: BackupManifest["components"]["storage"] | undefined;
|
||||
if (!opts.skipStorage && config.appStorageBucket) {
|
||||
logger.info(`Backing up storage bucket ${config.appStorageBucket}`);
|
||||
const storageDir = join(stageDir, "storage");
|
||||
const storageResult = await backupStorageBucket({
|
||||
s3: config.s3,
|
||||
bucket: config.appStorageBucket,
|
||||
outDir: storageDir,
|
||||
client: opts.client,
|
||||
});
|
||||
storageEntry = {
|
||||
path: "storage/objects.tar",
|
||||
sizeBytes: storageResult.totalBytes,
|
||||
checksum: storageResult.checksum,
|
||||
objectCount: storageResult.objectCount,
|
||||
};
|
||||
} else if (opts.skipStorage) {
|
||||
logger.warn("--skip-storage flag set; skipping storage component");
|
||||
} else {
|
||||
logger.warn("STORAGE_BUCKET env not set; skipping storage component");
|
||||
}
|
||||
|
||||
let clickhouseEntry: BackupManifest["components"]["clickhouse"] | undefined;
|
||||
if (config.includeClickhouse) {
|
||||
logger.info("Snapshotting ClickHouse");
|
||||
const chDir = join(stageDir, "clickhouse");
|
||||
const chResult = await maybeSnapshotClickhouse({ outDir: chDir, env: env as never });
|
||||
if (chResult) {
|
||||
clickhouseEntry = {
|
||||
path: "clickhouse/snapshot.native.gz",
|
||||
sizeBytes: chResult.sizeBytes,
|
||||
checksum: chResult.checksum,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const manifest: BackupManifest = {
|
||||
schemaVersion: BACKUP_MANIFEST_VERSION,
|
||||
backupId,
|
||||
createdAt,
|
||||
creator: { cliVersion: "0.0.0", host: hostname() },
|
||||
classification: classifyBackup(createdAt),
|
||||
components: {
|
||||
postgres: {
|
||||
path: "postgres/dump.sql.gz",
|
||||
sizeBytes: pgResult.sizeBytes,
|
||||
checksum: pgResult.checksum,
|
||||
},
|
||||
...(storageEntry ? { storage: storageEntry } : {}),
|
||||
...(clickhouseEntry ? { clickhouse: clickhouseEntry } : {}),
|
||||
},
|
||||
encryption: {
|
||||
method: config.encryption.method,
|
||||
recipient: config.encryption.recipient,
|
||||
},
|
||||
};
|
||||
const manifestPath = join(stageDir, "manifest.json");
|
||||
await writeFile(manifestPath, serializeManifest(manifest));
|
||||
|
||||
const archivePath = join(tempRoot, `${backupId}.tar.gz`);
|
||||
logger.info(`Packaging ${archivePath}`);
|
||||
await createTar(
|
||||
{ gzip: true, file: archivePath, cwd: tempRoot, portable: true },
|
||||
[backupId],
|
||||
);
|
||||
|
||||
let uploadPath = archivePath;
|
||||
if (config.encryption.method !== "none") {
|
||||
const encryptedPath = `${archivePath}.${config.encryption.method}`;
|
||||
logger.info(`Encrypting via ${config.encryption.method}`);
|
||||
await encryptFile({ config: config.encryption, inputPath: archivePath, outputPath: encryptedPath });
|
||||
uploadPath = encryptedPath;
|
||||
}
|
||||
|
||||
const client = opts.client ?? new S3Client(config.s3);
|
||||
logger.info(`Uploading to s3://${config.backupBucket}/${objectKey}`);
|
||||
await uploadFileMultipart({
|
||||
client,
|
||||
bucket: config.backupBucket,
|
||||
key: objectKey,
|
||||
filePath: uploadPath,
|
||||
contentType: "application/gzip",
|
||||
metadata: {
|
||||
"backup-id": backupId,
|
||||
"manifest-version": String(BACKUP_MANIFEST_VERSION),
|
||||
encryption: config.encryption.method,
|
||||
classification: manifest.classification.join(","),
|
||||
},
|
||||
});
|
||||
|
||||
const uploadStat = await readFile(uploadPath).then((b) => b.length);
|
||||
return { backupId, key: objectKey, sizeBytes: uploadStat, manifest };
|
||||
} finally {
|
||||
await rm(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export const createBackupCommand = defineCommand({
|
||||
meta: {
|
||||
name: "create",
|
||||
description: "Create a new backup (Postgres + Storage) and upload to S3.",
|
||||
},
|
||||
args: {
|
||||
"skip-storage": {
|
||||
type: "boolean",
|
||||
description: "Do not include the application storage bucket.",
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
async run({ args }) {
|
||||
const result = await runCreateBackup({ skipStorage: Boolean(args["skip-storage"]) });
|
||||
consola.success(`Created ${result.backupId} (${result.sizeBytes} bytes) at key ${result.key}`);
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user