Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
+263
@@ -0,0 +1,263 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
anonymizeRow,
|
||||
applyStrategy,
|
||||
FULL_ANONYMIZED_PROFILE,
|
||||
MINIMAL_ANONYMIZED_PROFILE,
|
||||
NONE_PROFILE,
|
||||
PROFILES,
|
||||
isProfileName,
|
||||
type AnonymizationContext,
|
||||
} from "./anonymize.js";
|
||||
|
||||
const ctx: AnonymizationContext = { salt: "test-salt-deadbeef", profileName: "full-anonymized" };
|
||||
|
||||
describe("anonymize", () => {
|
||||
it("isProfileName recognises the three supported profiles", () => {
|
||||
expect(isProfileName("full-anonymized")).toBe(true);
|
||||
expect(isProfileName("minimal-anonymized")).toBe(true);
|
||||
expect(isProfileName("none")).toBe(true);
|
||||
expect(isProfileName("zalgo")).toBe(false);
|
||||
});
|
||||
|
||||
it("PROFILES exposes all three profiles", () => {
|
||||
expect(PROFILES["full-anonymized"]).toBe(FULL_ANONYMIZED_PROFILE);
|
||||
expect(PROFILES["minimal-anonymized"]).toBe(MINIMAL_ANONYMIZED_PROFILE);
|
||||
expect(PROFILES.none).toBe(NONE_PROFILE);
|
||||
});
|
||||
|
||||
describe("determinism", () => {
|
||||
it("same value + same salt → same output", () => {
|
||||
const a = applyStrategy("email", "alice@example.com", ctx);
|
||||
const b = applyStrategy("email", "alice@example.com", ctx);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
|
||||
it("same value + different salt → different output", () => {
|
||||
const a = applyStrategy("email", "alice@example.com", ctx);
|
||||
const b = applyStrategy("email", "alice@example.com", { ...ctx, salt: "other-salt" });
|
||||
expect(a).not.toEqual(b);
|
||||
});
|
||||
|
||||
it("two different originals map to two different anonymized values", () => {
|
||||
const a = applyStrategy("email", "alice@example.com", ctx);
|
||||
const b = applyStrategy("email", "bob@example.com", ctx);
|
||||
expect(a).not.toEqual(b);
|
||||
});
|
||||
|
||||
it("FK consistency: the same original yields a stable anonymized value across rows", () => {
|
||||
const row1 = anonymizeRow(FULL_ANONYMIZED_PROFILE, "PortalUser", { email: "x@y.io" }, ctx);
|
||||
const row2 = anonymizeRow(FULL_ANONYMIZED_PROFILE, "PortalUser", { email: "x@y.io" }, ctx);
|
||||
expect(row1.email).toEqual(row2.email);
|
||||
});
|
||||
});
|
||||
|
||||
describe("full-anonymized profile", () => {
|
||||
it("rewrites email to *@anonymized.test", () => {
|
||||
const row = anonymizeRow(
|
||||
FULL_ANONYMIZED_PROFILE,
|
||||
"TenantUserInvitation",
|
||||
{ email: "alice@example.com", firstName: "Alice", lastName: "Wonderland" },
|
||||
ctx,
|
||||
);
|
||||
expect(row.email).toMatch(/@anonymized\.test$/u);
|
||||
expect(row.firstName).not.toBe("Alice");
|
||||
expect(row.lastName).not.toBe("Wonderland");
|
||||
});
|
||||
|
||||
it("masks IP to 127.0.0.x", () => {
|
||||
const row = anonymizeRow(
|
||||
FULL_ANONYMIZED_PROFILE,
|
||||
"TenantIpAddress",
|
||||
{ ip: "203.0.113.42", fromUrl: "https://example.com" },
|
||||
ctx,
|
||||
);
|
||||
expect(row.ip).toMatch(/^127\.0\.0\.\d{1,3}$/u);
|
||||
expect(row.fromUrl).toMatch(/^https:\/\/anonymized\.test\//u);
|
||||
});
|
||||
|
||||
it("rewrites Stripe customer/subscription ids with anon prefix", () => {
|
||||
const row = anonymizeRow(
|
||||
FULL_ANONYMIZED_PROFILE,
|
||||
"TenantSubscription",
|
||||
{ stripeCustomerId: "cus_real", stripeSubscriptionId: "sub_real" },
|
||||
ctx,
|
||||
);
|
||||
expect(row.stripeCustomerId).toMatch(/^cus_anon[0-9a-f]{8}$/u);
|
||||
expect(row.stripeSubscriptionId).toMatch(/^sub_anon[0-9a-f]{8}$/u);
|
||||
});
|
||||
|
||||
it("redacts secrets to [REDACTED]", () => {
|
||||
const row = anonymizeRow(
|
||||
FULL_ANONYMIZED_PROFILE,
|
||||
"WorkflowCredential",
|
||||
{ secret: "very-secret-token", apiKey: "sk_live_real" },
|
||||
ctx,
|
||||
);
|
||||
expect(row.secret).toBe("[REDACTED]");
|
||||
expect(row.apiKey).toBe("[REDACTED]");
|
||||
});
|
||||
|
||||
it("leaves unrelated fields untouched", () => {
|
||||
const row = anonymizeRow(
|
||||
FULL_ANONYMIZED_PROFILE,
|
||||
"TenantSubscription",
|
||||
{ stripeCustomerId: "cus_real", planTier: "pro", active: true },
|
||||
ctx,
|
||||
);
|
||||
expect(row.planTier).toBe("pro");
|
||||
expect(row.active).toBe(true);
|
||||
});
|
||||
|
||||
it("passes null/undefined fields through without invocation", () => {
|
||||
const row = anonymizeRow(
|
||||
FULL_ANONYMIZED_PROFILE,
|
||||
"UserNotificationPreference",
|
||||
{ email: null, phone: undefined },
|
||||
ctx,
|
||||
);
|
||||
expect(row.email).toBeNull();
|
||||
expect(row.phone).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("minimal-anonymized profile", () => {
|
||||
it("keeps firstName/lastName", () => {
|
||||
const row = anonymizeRow(
|
||||
MINIMAL_ANONYMIZED_PROFILE,
|
||||
"TenantUserInvitation",
|
||||
{ email: "alice@example.com", firstName: "Alice", lastName: "Wonderland" },
|
||||
{ ...ctx, profileName: "minimal-anonymized" },
|
||||
);
|
||||
expect(row.firstName).toBe("Alice");
|
||||
expect(row.lastName).toBe("Wonderland");
|
||||
});
|
||||
|
||||
it("still scrubs email", () => {
|
||||
const row = anonymizeRow(
|
||||
MINIMAL_ANONYMIZED_PROFILE,
|
||||
"TenantUserInvitation",
|
||||
{ email: "alice@example.com" },
|
||||
{ ...ctx, profileName: "minimal-anonymized" },
|
||||
);
|
||||
expect(row.email).toMatch(/@anonymized\.test$/u);
|
||||
});
|
||||
|
||||
it("still scrubs IP address", () => {
|
||||
const row = anonymizeRow(
|
||||
MINIMAL_ANONYMIZED_PROFILE,
|
||||
"TenantIpAddress",
|
||||
{ ip: "203.0.113.42" },
|
||||
{ ...ctx, profileName: "minimal-anonymized" },
|
||||
);
|
||||
expect(row.ip).toMatch(/^127\.0\.0\.\d{1,3}$/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("none profile", () => {
|
||||
it("passes everything through unchanged", () => {
|
||||
const original = { email: "alice@example.com", stripeCustomerId: "cus_real" };
|
||||
const row = anonymizeRow(NONE_PROFILE, "TenantSubscription", original, {
|
||||
...ctx,
|
||||
profileName: "none",
|
||||
});
|
||||
expect(row).toEqual(original);
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyStrategy edge cases", () => {
|
||||
it("redact returns [REDACTED] for any value", () => {
|
||||
expect(applyStrategy("redact", { nested: "object" }, ctx)).toBe("[REDACTED]");
|
||||
expect(applyStrategy("redact", 42, ctx)).toBe("[REDACTED]");
|
||||
});
|
||||
|
||||
it("truncate-32 trims to 32 chars with ellipsis", () => {
|
||||
const result = applyStrategy(
|
||||
"truncate-32",
|
||||
"a very long string that exceeds thirty two characters",
|
||||
ctx,
|
||||
);
|
||||
expect(String(result).length).toBe(32);
|
||||
expect(String(result)).toMatch(/\.\.\.$/u);
|
||||
});
|
||||
|
||||
it("sha256-prefix returns anon- followed by 16 hex chars", () => {
|
||||
const result = applyStrategy("sha256-prefix", "some-original", ctx);
|
||||
expect(result).toMatch(/^anon-[0-9a-f]{16}$/u);
|
||||
});
|
||||
|
||||
it("avatar-url returns an anonymized.test url", () => {
|
||||
const result = applyStrategy("avatar-url", "https://example.com/me.png", ctx);
|
||||
expect(result).toMatch(/^https:\/\/anonymized\.test\/avatar\/[0-9a-f]{8}\.png$/u);
|
||||
});
|
||||
|
||||
it("zip-code preserves 2-digit prefix when input is numeric", () => {
|
||||
const result = applyStrategy("zip-code", "12345", ctx);
|
||||
expect(String(result)).toMatch(/^12\d{3}$/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("User (globals slice)", () => {
|
||||
const user = {
|
||||
id: "u_real",
|
||||
email: "alice@real.example",
|
||||
firstName: "Alice",
|
||||
lastName: "Anderson",
|
||||
phone: "+491701234567",
|
||||
avatar: "https://cdn.example/alice.png",
|
||||
passwordHash: "$argon2id$...",
|
||||
verifyToken: "verif-token",
|
||||
totpSecretEncrypted: "enc(secret)",
|
||||
totpPendingSecretEncrypted: "enc(pending)",
|
||||
mfaRecoveryCodeHashes: ["h1", "h2"],
|
||||
deletionRequestId: "del-abc",
|
||||
active: true,
|
||||
locale: "en",
|
||||
};
|
||||
|
||||
it("full-anonymized scrubs every PII column and nulls credentials", () => {
|
||||
const row = anonymizeRow(FULL_ANONYMIZED_PROFILE, "User", user, ctx);
|
||||
expect(row.id).toBe("u_real");
|
||||
expect(row.email).toMatch(/@anonymized\.test$/u);
|
||||
expect(row.firstName).not.toBe("Alice");
|
||||
expect(row.lastName).not.toBe("Anderson");
|
||||
expect(row.phone).toBeNull();
|
||||
expect(row.avatar).toMatch(/^https:\/\/anonymized\.test\/avatar\//u);
|
||||
expect(row.passwordHash).toBeNull();
|
||||
expect(row.verifyToken).toBeNull();
|
||||
expect(row.totpSecretEncrypted).toBeNull();
|
||||
expect(row.totpPendingSecretEncrypted).toBeNull();
|
||||
expect(row.mfaRecoveryCodeHashes).toBeNull();
|
||||
expect(row.deletionRequestId).toBeNull();
|
||||
// non-PII columns ride through unchanged.
|
||||
expect(row.active).toBe(true);
|
||||
expect(row.locale).toBe("en");
|
||||
});
|
||||
|
||||
it("minimal-anonymized keeps display names but still nulls credentials", () => {
|
||||
const row = anonymizeRow(MINIMAL_ANONYMIZED_PROFILE, "User", user, ctx);
|
||||
expect(row.firstName).toBe("Alice");
|
||||
expect(row.lastName).toBe("Anderson");
|
||||
expect(row.email).toMatch(/@anonymized\.test$/u);
|
||||
expect(row.passwordHash).toBeNull();
|
||||
expect(row.totpSecretEncrypted).toBeNull();
|
||||
});
|
||||
|
||||
it("two snapshots of the same source produce identical anonymized user (deterministic under shared salt)", () => {
|
||||
const a = anonymizeRow(FULL_ANONYMIZED_PROFILE, "User", user, ctx);
|
||||
const b = anonymizeRow(FULL_ANONYMIZED_PROFILE, "User", user, ctx);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
|
||||
it("output does not contain any of the original PII substrings", () => {
|
||||
const row = anonymizeRow(FULL_ANONYMIZED_PROFILE, "User", user, ctx);
|
||||
const text = JSON.stringify(row);
|
||||
expect(text).not.toContain("alice@real.example");
|
||||
expect(text).not.toContain("Alice");
|
||||
expect(text).not.toContain("Anderson");
|
||||
expect(text).not.toContain("+491701234567");
|
||||
expect(text).not.toContain("$argon2id$");
|
||||
expect(text).not.toContain("cdn.example");
|
||||
});
|
||||
});
|
||||
});
|
||||
+415
@@ -0,0 +1,415 @@
|
||||
/**
|
||||
* PII anonymization engine for tenant snapshots.
|
||||
*
|
||||
* Three profiles are supported:
|
||||
* - `full-anonymized` default; replaces every known PII field with a
|
||||
* synthetic value derived from a per-snapshot salt.
|
||||
* - `minimal-anonymized` keeps display names but scrubs strictly-regulated
|
||||
* PII (email, phone, address, IP, payment ids).
|
||||
* - `none` raw passthrough; gated behind a super-admin flag at
|
||||
* the snapshot-command layer (NEVER use without an
|
||||
* explicit audit trail — see docs/ops/tenant-snapshot.md).
|
||||
*
|
||||
* Determinism: every anonymizer is seeded by `sha256(salt + originalValue)`.
|
||||
* The same original value within the same snapshot always maps to the same
|
||||
* anonymized value, preserving foreign-key integrity (e.g. an email used as
|
||||
* a join key in another row stays consistent across rows).
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export type FieldStrategy =
|
||||
| "keep"
|
||||
| "null"
|
||||
| "email"
|
||||
| "first-name"
|
||||
| "last-name"
|
||||
| "full-name"
|
||||
| "phone"
|
||||
| "address-street"
|
||||
| "address-city"
|
||||
| "address-country"
|
||||
| "zip-code"
|
||||
| "ip-address"
|
||||
| "stripe-customer-id"
|
||||
| "stripe-subscription-id"
|
||||
| "stripe-payment-method-id"
|
||||
| "stripe-id-generic"
|
||||
| "url-rewrite"
|
||||
| "avatar-url"
|
||||
| "redact"
|
||||
| "sha256-prefix"
|
||||
| "truncate-32";
|
||||
|
||||
export type ProfileName = "full-anonymized" | "minimal-anonymized" | "none";
|
||||
|
||||
/** Map from model name → { field → strategy }. Fields not listed are passed through unchanged. */
|
||||
export type AnonymizationProfile = Readonly<Record<string, Readonly<Record<string, FieldStrategy>>>>;
|
||||
|
||||
/**
|
||||
* The full-anonymized profile. This is the source-of-truth PII map; the
|
||||
* minimal profile is derived from it by relaxing display-name strategies.
|
||||
*/
|
||||
export const FULL_ANONYMIZED_PROFILE: AnonymizationProfile = {
|
||||
TenantUserInvitation: {
|
||||
email: "email",
|
||||
firstName: "first-name",
|
||||
lastName: "last-name",
|
||||
},
|
||||
TenantIpAddress: {
|
||||
ip: "ip-address",
|
||||
fromUrl: "url-rewrite",
|
||||
},
|
||||
WorkspaceAuthProvider: {
|
||||
clientId: "redact",
|
||||
certificateFingerprint: "redact",
|
||||
issuer: "url-rewrite",
|
||||
authorizationUrl: "url-rewrite",
|
||||
tokenUrl: "url-rewrite",
|
||||
jwksUrl: "url-rewrite",
|
||||
ssoUrl: "url-rewrite",
|
||||
},
|
||||
ImpersonationAudit: {
|
||||
reason: "redact",
|
||||
metadata: "redact",
|
||||
},
|
||||
ApiKey: {
|
||||
name: "truncate-32",
|
||||
description: "redact",
|
||||
hash: "sha256-prefix",
|
||||
prefix: "sha256-prefix",
|
||||
},
|
||||
ApiKeyLog: {
|
||||
ip: "ip-address",
|
||||
params: "redact",
|
||||
},
|
||||
AuditEvent: {
|
||||
actorIp: "ip-address",
|
||||
metadata: "redact",
|
||||
},
|
||||
Event: {
|
||||
// Schema field is `data` (free-form string used by callers for any
|
||||
// event-shaped payload). Catches plaintext emails and identifiers
|
||||
// that callers stuff in for debugging.
|
||||
data: "redact",
|
||||
description: "truncate-32",
|
||||
},
|
||||
Log: {
|
||||
error: "redact",
|
||||
url: "url-rewrite",
|
||||
},
|
||||
Campaign: {
|
||||
fromEmail: "email",
|
||||
fromName: "full-name",
|
||||
replyToEmail: "email",
|
||||
},
|
||||
EmailSender: {
|
||||
fromEmail: "email",
|
||||
fromName: "full-name",
|
||||
apiKey: "redact",
|
||||
domain: "url-rewrite",
|
||||
},
|
||||
TenantInboundAddress: {
|
||||
address: "email",
|
||||
forwardTo: "email",
|
||||
},
|
||||
OutboundEmail: {
|
||||
fromEmail: "email",
|
||||
fromName: "full-name",
|
||||
toEmail: "email",
|
||||
replyToEmail: "email",
|
||||
subject: "truncate-32",
|
||||
htmlBody: "redact",
|
||||
textBody: "redact",
|
||||
},
|
||||
TenantSubscription: {
|
||||
stripeCustomerId: "stripe-customer-id",
|
||||
stripeSubscriptionId: "stripe-subscription-id",
|
||||
},
|
||||
TenantProductConfiguration: {
|
||||
stripeAccountId: "stripe-id-generic",
|
||||
},
|
||||
Credit: {
|
||||
stripeInvoiceId: "stripe-id-generic",
|
||||
},
|
||||
WorkflowCredential: {
|
||||
secret: "redact",
|
||||
apiKey: "redact",
|
||||
accessToken: "redact",
|
||||
refreshToken: "redact",
|
||||
},
|
||||
Workflow: {
|
||||
webhookSecret: "redact",
|
||||
},
|
||||
WorkflowExecution: {
|
||||
contextJson: "redact",
|
||||
errorMessage: "redact",
|
||||
},
|
||||
DataConnection: {
|
||||
accessToken: "redact",
|
||||
refreshToken: "redact",
|
||||
secret: "redact",
|
||||
config: "redact",
|
||||
},
|
||||
App: {
|
||||
secret: "redact",
|
||||
},
|
||||
ImportJob: {
|
||||
inputUrl: "url-rewrite",
|
||||
errorMessage: "redact",
|
||||
},
|
||||
ExportJob: {
|
||||
resultUrl: "url-rewrite",
|
||||
errorMessage: "redact",
|
||||
},
|
||||
Feedback: {
|
||||
message: "redact",
|
||||
},
|
||||
SupportTicket: {
|
||||
// Schema fields are `subject` and `message`, not `title`/`description`
|
||||
// (the previous map referenced nonexistent fields — see B7 verify).
|
||||
subject: "truncate-32",
|
||||
message: "redact",
|
||||
fromEmail: "email",
|
||||
fromUrl: "url-rewrite",
|
||||
},
|
||||
Portal: {
|
||||
customDomain: "url-rewrite",
|
||||
},
|
||||
PortalUser: {
|
||||
email: "email",
|
||||
firstName: "first-name",
|
||||
lastName: "last-name",
|
||||
},
|
||||
NotificationDelivery: {
|
||||
payload: "redact",
|
||||
email: "email",
|
||||
},
|
||||
UserNotificationPreference: {
|
||||
email: "email",
|
||||
phone: "phone",
|
||||
},
|
||||
Site: {
|
||||
customDomain: "url-rewrite",
|
||||
},
|
||||
TenantSettingsRow: {
|
||||
settings: "redact",
|
||||
},
|
||||
KnowledgeBase: {
|
||||
description: "redact",
|
||||
},
|
||||
/**
|
||||
* Global rows extracted alongside the tenant slice. These rows are
|
||||
* referenced by tenant-scoped FKs and otherwise break restore into an
|
||||
* empty target DB. See models.ts::GLOBAL_REFERENCED_MODELS.
|
||||
*
|
||||
* Every PII column on `User` must be scrubbed. `passwordHash`,
|
||||
* `verifyToken`, MFA secrets and OAuth subject ids are sensitive
|
||||
* credentials, not PII-with-utility — set them to null/redacted so
|
||||
* the restored user can never be logged into via the original creds.
|
||||
* `id` is intentionally NOT touched: tenant-scoped FK columns must
|
||||
* keep pointing at the same id after restore.
|
||||
*/
|
||||
User: {
|
||||
email: "email",
|
||||
firstName: "first-name",
|
||||
lastName: "last-name",
|
||||
phone: "phone",
|
||||
avatar: "avatar-url",
|
||||
passwordHash: "null",
|
||||
verifyToken: "null",
|
||||
totpSecretEncrypted: "null",
|
||||
totpPendingSecretEncrypted: "null",
|
||||
mfaRecoveryCodeHashes: "null",
|
||||
deletionRequestId: "null",
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* The minimal profile keeps display names (firstName/lastName) but still
|
||||
* scrubs DSGVO-relevant identifiers, payment ids, raw IPs and secrets.
|
||||
*/
|
||||
export const MINIMAL_ANONYMIZED_PROFILE: AnonymizationProfile = (() => {
|
||||
const out: Record<string, Record<string, FieldStrategy>> = {};
|
||||
for (const [model, fields] of Object.entries(FULL_ANONYMIZED_PROFILE)) {
|
||||
const next: Record<string, FieldStrategy> = {};
|
||||
for (const [field, strategy] of Object.entries(fields)) {
|
||||
if (strategy === "first-name" || strategy === "last-name" || strategy === "full-name") {
|
||||
next[field] = "keep";
|
||||
} else {
|
||||
next[field] = strategy;
|
||||
}
|
||||
}
|
||||
out[model] = next;
|
||||
}
|
||||
return out;
|
||||
})();
|
||||
|
||||
export const NONE_PROFILE: AnonymizationProfile = {};
|
||||
|
||||
export const PROFILES: Readonly<Record<ProfileName, AnonymizationProfile>> = {
|
||||
"full-anonymized": FULL_ANONYMIZED_PROFILE,
|
||||
"minimal-anonymized": MINIMAL_ANONYMIZED_PROFILE,
|
||||
none: NONE_PROFILE,
|
||||
};
|
||||
|
||||
export function isProfileName(value: string): value is ProfileName {
|
||||
return value === "full-anonymized" || value === "minimal-anonymized" || value === "none";
|
||||
}
|
||||
|
||||
export interface AnonymizationContext {
|
||||
/** Per-snapshot salt. Combined with `originalValue` to seed the transform. */
|
||||
readonly salt: string;
|
||||
/** Name of the profile being applied; included in errors for debuggability. */
|
||||
readonly profileName: ProfileName;
|
||||
}
|
||||
|
||||
const FIRST_NAMES = [
|
||||
"Alex", "Sam", "Jordan", "Taylor", "Morgan", "Casey", "Avery", "Riley",
|
||||
"Quinn", "Reese", "Skyler", "Drew", "Elliot", "Hayden", "Jamie", "Kai",
|
||||
"Logan", "Parker", "Robin", "Sage", "Tatum", "River", "Phoenix", "Rowan",
|
||||
];
|
||||
|
||||
const LAST_NAMES = [
|
||||
"Smith", "Johnson", "Lee", "Patel", "Garcia", "Mueller", "Schmidt", "Nguyen",
|
||||
"Kim", "Davis", "Brown", "Wilson", "Mendez", "Rossi", "Andersen", "Becker",
|
||||
];
|
||||
|
||||
const STREET_NAMES = [
|
||||
"Main", "Oak", "Pine", "Maple", "Elm", "Cedar", "Birch", "Willow", "Cherry",
|
||||
"Hauptstrasse", "Bahnhofstrasse",
|
||||
];
|
||||
|
||||
const CITIES = [
|
||||
"Springfield", "Riverton", "Oakville", "Lakeside", "Hillview", "Mapleton",
|
||||
"Pinehurst", "Bergstadt", "Seedorf",
|
||||
];
|
||||
|
||||
const COUNTRIES = ["AT", "DE", "CH", "NL", "PT", "IE", "FI", "SE"];
|
||||
|
||||
/**
|
||||
* Apply an anonymization profile to a row from the named model. Returns a
|
||||
* shallow copy with anonymized field values. Unknown fields and fields not in
|
||||
* the profile are passed through unchanged.
|
||||
*/
|
||||
export function anonymizeRow(
|
||||
profile: AnonymizationProfile,
|
||||
modelName: string,
|
||||
row: Record<string, unknown>,
|
||||
ctx: AnonymizationContext,
|
||||
): Record<string, unknown> {
|
||||
const mapping = profile[modelName];
|
||||
if (!mapping) return { ...row };
|
||||
const out: Record<string, unknown> = { ...row };
|
||||
for (const [field, strategy] of Object.entries(mapping)) {
|
||||
if (!(field in out)) continue;
|
||||
const original = out[field];
|
||||
if (original === null || original === undefined) continue;
|
||||
out[field] = applyStrategy(strategy, original, ctx);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Apply a single field strategy. Exported for unit tests. */
|
||||
export function applyStrategy(
|
||||
strategy: FieldStrategy,
|
||||
value: unknown,
|
||||
ctx: AnonymizationContext,
|
||||
): unknown {
|
||||
switch (strategy) {
|
||||
case "keep":
|
||||
return value;
|
||||
case "null":
|
||||
return null;
|
||||
case "redact":
|
||||
return "[REDACTED]";
|
||||
case "email":
|
||||
return anonEmail(value, ctx);
|
||||
case "first-name":
|
||||
return pick(FIRST_NAMES, value, ctx);
|
||||
case "last-name":
|
||||
return pick(LAST_NAMES, value, ctx);
|
||||
case "full-name":
|
||||
return `${pick(FIRST_NAMES, value, ctx)} ${pick(LAST_NAMES, value, ctx)}`;
|
||||
case "phone":
|
||||
return null;
|
||||
case "address-street": {
|
||||
const street = pick(STREET_NAMES, value, ctx);
|
||||
const num = seededInt(value, ctx, 1, 999);
|
||||
return `${num} ${street} St`;
|
||||
}
|
||||
case "address-city":
|
||||
return pick(CITIES, value, ctx);
|
||||
case "address-country":
|
||||
return pick(COUNTRIES, value, ctx);
|
||||
case "zip-code":
|
||||
return anonZip(value, ctx);
|
||||
case "ip-address":
|
||||
return anonIp(value, ctx);
|
||||
case "stripe-customer-id":
|
||||
return `cus_anon${seededHex(value, ctx, 8)}`;
|
||||
case "stripe-subscription-id":
|
||||
return `sub_anon${seededHex(value, ctx, 8)}`;
|
||||
case "stripe-payment-method-id":
|
||||
return `pm_anon${seededHex(value, ctx, 8)}`;
|
||||
case "stripe-id-generic":
|
||||
return `anon_${seededHex(value, ctx, 12)}`;
|
||||
case "url-rewrite":
|
||||
return `https://anonymized.test/${seededHex(value, ctx, 10)}`;
|
||||
case "avatar-url":
|
||||
return `https://anonymized.test/avatar/${seededHex(value, ctx, 8)}.png`;
|
||||
case "sha256-prefix":
|
||||
return `anon-${seedHex(stringify(value), ctx).slice(0, 16)}`;
|
||||
case "truncate-32": {
|
||||
const s = stringify(value);
|
||||
return s.length > 32 ? `${s.slice(0, 29)}...` : s;
|
||||
}
|
||||
default: {
|
||||
const exhaustive: never = strategy;
|
||||
throw new Error(`Unknown anonymization strategy: ${String(exhaustive)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function stringify(value: unknown): string {
|
||||
if (typeof value === "string") return value;
|
||||
if (value === null || value === undefined) return "";
|
||||
if (typeof value === "object") return JSON.stringify(value);
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function seedHex(originalValue: string, ctx: AnonymizationContext): string {
|
||||
return createHash("sha256").update(`${ctx.salt}::${originalValue}`).digest("hex");
|
||||
}
|
||||
|
||||
function seededInt(value: unknown, ctx: AnonymizationContext, min: number, max: number): number {
|
||||
const hex = seedHex(stringify(value), ctx).slice(0, 12);
|
||||
const n = Number.parseInt(hex, 16);
|
||||
const range = max - min + 1;
|
||||
return min + (n % range);
|
||||
}
|
||||
|
||||
function seededHex(value: unknown, ctx: AnonymizationContext, length: number): string {
|
||||
return seedHex(stringify(value), ctx).slice(0, length);
|
||||
}
|
||||
|
||||
function pick<T>(arr: readonly T[], value: unknown, ctx: AnonymizationContext): T {
|
||||
const idx = seededInt(value, ctx, 0, arr.length - 1);
|
||||
return arr[idx];
|
||||
}
|
||||
|
||||
function anonEmail(value: unknown, ctx: AnonymizationContext): string {
|
||||
const local = seededHex(value, ctx, 12);
|
||||
return `user-${local}@anonymized.test`;
|
||||
}
|
||||
|
||||
function anonIp(value: unknown, ctx: AnonymizationContext): string {
|
||||
const oct4 = seededInt(value, ctx, 0, 254);
|
||||
return `127.0.0.${oct4}`;
|
||||
}
|
||||
|
||||
function anonZip(value: unknown, ctx: AnonymizationContext): string {
|
||||
const s = stringify(value);
|
||||
const prefix = s.slice(0, 2).padEnd(2, "0");
|
||||
const rest = String(seededInt(value, ctx, 0, 999)).padStart(3, "0");
|
||||
return `${prefix}${rest}`;
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { Writable } from "node:stream";
|
||||
import {
|
||||
assertSuperAdminForNoneProfile,
|
||||
writeSnapshotAudit,
|
||||
type SnapshotAuditEntry,
|
||||
} from "./audit.js";
|
||||
|
||||
const baseEntry: SnapshotAuditEntry = {
|
||||
event: "TENANT_SNAPSHOT_CREATED",
|
||||
snapshotId: "tenant-snapshot-ten_a-2026-05-17T03:00:00Z-abcd1234",
|
||||
sourceTenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
actorUserId: "user_admin",
|
||||
host: "ops-jumphost",
|
||||
createdAt: "2026-05-17T03:00:00.000Z",
|
||||
outputTarget: "s3://snapshots/ten_a/snap.tar.gz",
|
||||
superAdminConfirmed: false,
|
||||
};
|
||||
|
||||
describe("assertSuperAdminForNoneProfile", () => {
|
||||
it("passes for full-anonymized regardless of flag", () => {
|
||||
expect(() => assertSuperAdminForNoneProfile("full-anonymized", false)).not.toThrow();
|
||||
expect(() => assertSuperAdminForNoneProfile("full-anonymized", true)).not.toThrow();
|
||||
});
|
||||
|
||||
it("passes for minimal-anonymized regardless of flag", () => {
|
||||
expect(() => assertSuperAdminForNoneProfile("minimal-anonymized", false)).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws for `none` without confirmation", () => {
|
||||
expect(() => assertSuperAdminForNoneProfile("none", false)).toThrow(
|
||||
/--i-am-the-super-admin/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("passes for `none` with confirmation", () => {
|
||||
expect(() => assertSuperAdminForNoneProfile("none", true)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("writeSnapshotAudit", () => {
|
||||
let workdir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), "tenant-audit-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("emits a one-line summary on stdout", async () => {
|
||||
const chunks: Buffer[] = [];
|
||||
const sink = new Writable({
|
||||
write(chunk, _enc, cb) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string));
|
||||
cb();
|
||||
},
|
||||
});
|
||||
await writeSnapshotAudit({ entry: baseEntry, stdout: sink });
|
||||
const text = Buffer.concat(chunks).toString("utf8");
|
||||
expect(text).toContain("[audit] TENANT_SNAPSHOT_CREATED");
|
||||
expect(text).toContain("tenant=ten_a");
|
||||
expect(text).toContain("profile=full-anonymized");
|
||||
});
|
||||
|
||||
it("appends a JSON line to the optional file", async () => {
|
||||
const filePath = join(workdir, "audit.log");
|
||||
const sink = new Writable({ write: (_c, _e, cb) => cb() });
|
||||
await writeSnapshotAudit({ entry: baseEntry, filePath, stdout: sink });
|
||||
await writeSnapshotAudit({
|
||||
entry: { ...baseEntry, snapshotId: "tenant-snapshot-2" },
|
||||
filePath,
|
||||
stdout: sink,
|
||||
});
|
||||
const content = await readFile(filePath, "utf8");
|
||||
const lines = content.trim().split("\n");
|
||||
expect(lines).toHaveLength(2);
|
||||
for (const line of lines) {
|
||||
const parsed = JSON.parse(line) as SnapshotAuditEntry;
|
||||
expect(parsed.event).toBe("TENANT_SNAPSHOT_CREATED");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Snapshot-event audit log. Spec doesn't mandate a DB-write (the source
|
||||
* system's AuditEvent table is reachable but requires Prisma which we keep
|
||||
* out of the CLI); instead this module produces a structured, line-oriented
|
||||
* audit record that operators redirect to a file or syslog.
|
||||
*
|
||||
* The `none` profile is the only one that requires the super-admin guardrail
|
||||
* at the command layer; the audit entry is always written regardless of
|
||||
* profile so that every snapshot leaves a trail.
|
||||
*/
|
||||
import { appendFile } from "node:fs/promises";
|
||||
|
||||
export type SnapshotProfile = "full-anonymized" | "minimal-anonymized" | "none";
|
||||
|
||||
export interface SnapshotAuditEntry {
|
||||
readonly event: "TENANT_SNAPSHOT_CREATED";
|
||||
readonly snapshotId: string;
|
||||
readonly sourceTenantId: string;
|
||||
readonly profile: SnapshotProfile;
|
||||
readonly actorUserId: string | null;
|
||||
readonly host: string | null;
|
||||
readonly createdAt: string;
|
||||
readonly outputTarget: string;
|
||||
/** Whether the super-admin escape hatch was invoked for `none`. */
|
||||
readonly superAdminConfirmed: boolean;
|
||||
}
|
||||
|
||||
export interface WriteSnapshotAuditOptions {
|
||||
readonly entry: SnapshotAuditEntry;
|
||||
/** Optional file path to append the JSON line to. */
|
||||
readonly filePath?: string;
|
||||
/** Sink for the human-readable line (defaults to process.stdout). */
|
||||
readonly stdout?: NodeJS.WritableStream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a structured audit entry to an optional file and emit a one-line
|
||||
* human summary on stdout. Returns the serialized JSON.
|
||||
*/
|
||||
export async function writeSnapshotAudit(opts: WriteSnapshotAuditOptions): Promise<string> {
|
||||
const json = JSON.stringify(opts.entry);
|
||||
const line = `[audit] ${opts.entry.event} snapshot=${opts.entry.snapshotId} tenant=${opts.entry.sourceTenantId} profile=${opts.entry.profile} actor=${opts.entry.actorUserId ?? "?"} target=${opts.entry.outputTarget}\n`;
|
||||
const out = opts.stdout ?? process.stdout;
|
||||
out.write(line);
|
||||
if (opts.filePath) {
|
||||
await appendFile(opts.filePath, `${json}\n`);
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that running the `none` profile is authorized. Throws if not.
|
||||
*/
|
||||
export function assertSuperAdminForNoneProfile(
|
||||
profile: SnapshotProfile,
|
||||
confirmed: boolean,
|
||||
): void {
|
||||
if (profile !== "none") return;
|
||||
if (!confirmed) {
|
||||
throw new Error(
|
||||
"Refusing to run --profile=none without --i-am-the-super-admin. " +
|
||||
"Raw exports of tenant data require an explicit super-admin confirmation and leave " +
|
||||
"a permanent audit trail. See docs/ops/tenant-snapshot.md.",
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveTenantSnapshotConfig, TenantSnapshotConfigError } from "./env.js";
|
||||
|
||||
describe("resolveTenantSnapshotConfig", () => {
|
||||
it("defaults to encryption=none and no app storage bucket", () => {
|
||||
const cfg = resolveTenantSnapshotConfig({});
|
||||
expect(cfg.encryption.method).toBe("none");
|
||||
expect(cfg.appStorageBucket).toBeNull();
|
||||
expect(cfg.s3.region).toBe("us-east-1");
|
||||
});
|
||||
|
||||
it("rejects an unknown BACKUP_ENCRYPTION value", () => {
|
||||
expect(() => resolveTenantSnapshotConfig({ BACKUP_ENCRYPTION: "rot13" })).toThrow(
|
||||
TenantSnapshotConfigError,
|
||||
);
|
||||
});
|
||||
|
||||
it("requires age key material when encryption=age", () => {
|
||||
expect(() => resolveTenantSnapshotConfig({ BACKUP_ENCRYPTION: "age" })).toThrow(
|
||||
/BACKUP_AGE_PUBLIC_KEY .* BACKUP_AGE_IDENTITY_FILE/u,
|
||||
);
|
||||
expect(() =>
|
||||
resolveTenantSnapshotConfig({
|
||||
BACKUP_ENCRYPTION: "age",
|
||||
BACKUP_AGE_PUBLIC_KEY: "age1xyz",
|
||||
}),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
resolveTenantSnapshotConfig({
|
||||
BACKUP_ENCRYPTION: "age",
|
||||
BACKUP_AGE_IDENTITY_FILE: "/tmp/key.txt",
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("threads STORAGE_BUCKET and optional prefix through", () => {
|
||||
const cfg = resolveTenantSnapshotConfig({
|
||||
STORAGE_BUCKET: "boilerplate-files",
|
||||
TENANT_SNAPSHOT_STORAGE_PREFIX: "tenants/abc/",
|
||||
});
|
||||
expect(cfg.appStorageBucket).toBe("boilerplate-files");
|
||||
expect(cfg.storagePrefix).toBe("tenants/abc/");
|
||||
});
|
||||
|
||||
it("threads S3 credentials when both id and secret are set", () => {
|
||||
const cfg = resolveTenantSnapshotConfig({
|
||||
BACKUP_S3_ENDPOINT: "https://s3.example.com",
|
||||
BACKUP_S3_REGION: "eu-central-1",
|
||||
BACKUP_S3_FORCE_PATH_STYLE: "true",
|
||||
BACKUP_S3_ACCESS_KEY_ID: "AKIA",
|
||||
BACKUP_S3_SECRET_ACCESS_KEY: "secret",
|
||||
});
|
||||
expect(cfg.s3.endpoint).toBe("https://s3.example.com");
|
||||
expect(cfg.s3.region).toBe("eu-central-1");
|
||||
expect(cfg.s3.forcePathStyle).toBe(true);
|
||||
expect(cfg.s3.credentials).toMatchObject({ accessKeyId: "AKIA", secretAccessKey: "secret" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* Environment-variable resolver for the tenant snapshot / restore commands.
|
||||
*
|
||||
* Tenant ops reuse the same `BACKUP_S3_*` and `BACKUP_AGE_*` envs as the
|
||||
* backup CLI so operators only have to learn one secret-set, but the
|
||||
* requirements are looser:
|
||||
*
|
||||
* - **S3 is optional.** Snapshots can be written to a local path
|
||||
* (`--output ./snap.tar.gz`), so S3 envs are only validated when the
|
||||
* chosen output is an `s3://...` URL or when the snapshot needs to pull
|
||||
* tenant files out of an S3 bucket (`STORAGE_BUCKET` set).
|
||||
* - **Encryption is optional.** Defaults to `none` so the operator can do
|
||||
* a quick local round-trip without provisioning `age` keys. The runbook
|
||||
* still recommends `age` for any snapshot that leaves the host.
|
||||
*/
|
||||
import type { S3ClientConfig } from "@aws-sdk/client-s3";
|
||||
import type { EncryptionConfig, EncryptionMethod } from "../backup/encryption.js";
|
||||
|
||||
export interface TenantSnapshotEnv {
|
||||
BACKUP_S3_ENDPOINT?: string;
|
||||
BACKUP_S3_REGION?: string;
|
||||
BACKUP_S3_ACCESS_KEY_ID?: string;
|
||||
BACKUP_S3_SECRET_ACCESS_KEY?: string;
|
||||
BACKUP_S3_SESSION_TOKEN?: string;
|
||||
BACKUP_S3_FORCE_PATH_STYLE?: string;
|
||||
BACKUP_ENCRYPTION?: string;
|
||||
BACKUP_AGE_PUBLIC_KEY?: string;
|
||||
BACKUP_AGE_IDENTITY_FILE?: string;
|
||||
BACKUP_GPG_RECIPIENT?: string;
|
||||
STORAGE_BUCKET?: string;
|
||||
TENANT_SNAPSHOT_STORAGE_PREFIX?: string;
|
||||
}
|
||||
|
||||
export interface ResolvedTenantSnapshotConfig {
|
||||
/** S3 client config, always populated. Operators using local-only flows can ignore it. */
|
||||
s3: S3ClientConfig;
|
||||
/** Source tenant files live under `tenants/<tenantId>/` in this bucket, if configured. */
|
||||
appStorageBucket: string | null;
|
||||
/** Optional override for the storage object prefix. Defaults to `tenants/<tenantId>/`. */
|
||||
storagePrefix: string | undefined;
|
||||
/** Encryption config. Method defaults to `none` so the local quick-path works without keys. */
|
||||
encryption: EncryptionConfig;
|
||||
}
|
||||
|
||||
export class TenantSnapshotConfigError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "TenantSnapshotConfigError";
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveTenantSnapshotConfig(env: TenantSnapshotEnv): ResolvedTenantSnapshotConfig {
|
||||
const encryptionMethod = (env.BACKUP_ENCRYPTION ?? "none") as EncryptionMethod;
|
||||
if (encryptionMethod !== "age" && encryptionMethod !== "gpg" && encryptionMethod !== "none") {
|
||||
throw new TenantSnapshotConfigError(
|
||||
`Invalid BACKUP_ENCRYPTION=${env.BACKUP_ENCRYPTION}; expected age|gpg|none.`,
|
||||
);
|
||||
}
|
||||
const encryption: EncryptionConfig = {
|
||||
method: encryptionMethod,
|
||||
recipient: env.BACKUP_AGE_PUBLIC_KEY,
|
||||
identityFile: env.BACKUP_AGE_IDENTITY_FILE,
|
||||
gpgRecipient: env.BACKUP_GPG_RECIPIENT,
|
||||
};
|
||||
if (encryptionMethod === "age" && !encryption.recipient && !encryption.identityFile) {
|
||||
throw new TenantSnapshotConfigError(
|
||||
"BACKUP_ENCRYPTION=age requires BACKUP_AGE_PUBLIC_KEY (for snapshot) or BACKUP_AGE_IDENTITY_FILE (for restore).",
|
||||
);
|
||||
}
|
||||
|
||||
const s3: S3ClientConfig = {
|
||||
region: env.BACKUP_S3_REGION ?? "us-east-1",
|
||||
endpoint: env.BACKUP_S3_ENDPOINT,
|
||||
forcePathStyle: env.BACKUP_S3_FORCE_PATH_STYLE === "true",
|
||||
};
|
||||
if (env.BACKUP_S3_ACCESS_KEY_ID && env.BACKUP_S3_SECRET_ACCESS_KEY) {
|
||||
s3.credentials = {
|
||||
accessKeyId: env.BACKUP_S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.BACKUP_S3_SECRET_ACCESS_KEY,
|
||||
sessionToken: env.BACKUP_S3_SESSION_TOKEN,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
s3,
|
||||
appStorageBucket: env.STORAGE_BUCKET && env.STORAGE_BUCKET.length > 0 ? env.STORAGE_BUCKET : null,
|
||||
storagePrefix: env.TENANT_SNAPSHOT_STORAGE_PREFIX,
|
||||
encryption,
|
||||
};
|
||||
}
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { collectAndWriteGlobals, type GlobalsPrismaRunner } from "./globals-runner.js";
|
||||
|
||||
function makeRunner(rows: Record<string, Record<string, unknown>[]>): {
|
||||
runner: GlobalsPrismaRunner;
|
||||
calls: Array<{ model: string; ids: readonly string[] }>;
|
||||
} {
|
||||
const calls: Array<{ model: string; ids: readonly string[] }> = [];
|
||||
const runner: GlobalsPrismaRunner = {
|
||||
async findManyByIds(modelName, ids) {
|
||||
calls.push({ model: modelName, ids });
|
||||
const table = rows[modelName] ?? [];
|
||||
return table.filter((r) => typeof r.id === "string" && ids.includes(r.id as string));
|
||||
},
|
||||
};
|
||||
return { runner, calls };
|
||||
}
|
||||
|
||||
describe("collectAndWriteGlobals", () => {
|
||||
let workdir: string;
|
||||
let postgresDir: string;
|
||||
let globalsDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), "globals-runner-test-"));
|
||||
postgresDir = join(workdir, "postgres");
|
||||
globalsDir = join(workdir, "globals");
|
||||
await mkdir(postgresDir, { recursive: true });
|
||||
await mkdir(globalsDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("emits an empty JSONL when no tenant rows reference the global", async () => {
|
||||
// No JSONL files in postgresDir — every collectFkIdsFromJsonl returns
|
||||
// an empty set, so findManyByIds is never invoked.
|
||||
const { runner, calls } = makeRunner({});
|
||||
const result = await collectAndWriteGlobals({
|
||||
postgresDir,
|
||||
globalsDir,
|
||||
prisma: runner,
|
||||
profile: "full-anonymized",
|
||||
ctx: { salt: "deadbeef", profileName: "full-anonymized" },
|
||||
});
|
||||
|
||||
expect(result.entries).toHaveLength(1); // just User today
|
||||
const userEntry = result.entries.find((e) => e.model === "User");
|
||||
expect(userEntry).toBeDefined();
|
||||
expect(userEntry?.rowCount).toBe(0);
|
||||
expect(userEntry?.path).toBe("globals/User.jsonl");
|
||||
expect(calls).toHaveLength(0);
|
||||
const userFile = await readFile(join(globalsDir, "User.jsonl"), "utf8");
|
||||
expect(userFile).toBe("");
|
||||
});
|
||||
|
||||
it("deduplicates user ids referenced by multiple tenant rows", async () => {
|
||||
// Two TenantUser rows reference the same userId; the user is fetched once.
|
||||
await writeFile(
|
||||
join(postgresDir, "TenantUser.jsonl"),
|
||||
`${JSON.stringify({ id: "tu1", tenantId: "t1", userId: "u1" })}\n${JSON.stringify({ id: "tu2", tenantId: "t1", userId: "u1" })}\n`,
|
||||
);
|
||||
const { runner, calls } = makeRunner({
|
||||
User: [
|
||||
{
|
||||
id: "u1",
|
||||
email: "real@example.org",
|
||||
firstName: "Real",
|
||||
lastName: "Person",
|
||||
phone: "+10000",
|
||||
avatar: "https://cdn/old.png",
|
||||
passwordHash: "$argon2id$realhash",
|
||||
createdAt: "2026-05-25T00:00:00.000Z",
|
||||
updatedAt: "2026-05-25T00:00:00.000Z",
|
||||
active: true,
|
||||
pendingDeletion: false,
|
||||
pseudonymized: false,
|
||||
},
|
||||
],
|
||||
});
|
||||
const result = await collectAndWriteGlobals({
|
||||
postgresDir,
|
||||
globalsDir,
|
||||
prisma: runner,
|
||||
profile: "full-anonymized",
|
||||
ctx: { salt: "saltsaltsalt", profileName: "full-anonymized" },
|
||||
});
|
||||
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]).toEqual({ model: "User", ids: ["u1"] });
|
||||
|
||||
const userEntry = result.entries.find((e) => e.model === "User");
|
||||
expect(userEntry?.rowCount).toBe(1);
|
||||
|
||||
const written = (await readFile(join(globalsDir, "User.jsonl"), "utf8")).trim();
|
||||
const row = JSON.parse(written);
|
||||
expect(row.id).toBe("u1");
|
||||
// Anonymized — no original email leaks through.
|
||||
expect(written).not.toContain("real@example.org");
|
||||
expect(row.email).toMatch(/@anonymized\.test$/);
|
||||
expect(row.passwordHash).toBeNull();
|
||||
});
|
||||
|
||||
it("collects ids across multiple source models and FK columns", async () => {
|
||||
// Three FK columns on TenantUserInvitation point at User; two on
|
||||
// ImpersonationAudit do too. Union the lot in one chunked call.
|
||||
await writeFile(
|
||||
join(postgresDir, "TenantUserInvitation.jsonl"),
|
||||
`${JSON.stringify({ id: "inv1", tenantId: "t1", fromUserId: "u_from", acceptedByUserId: "u_acc", revokedByUserId: null })}\n`,
|
||||
);
|
||||
await writeFile(
|
||||
join(postgresDir, "ImpersonationAudit.jsonl"),
|
||||
`${JSON.stringify({ id: "imp1", actorUserId: "u_actor", targetUserId: "u_target" })}\n`,
|
||||
);
|
||||
const userRows = ["u_from", "u_acc", "u_actor", "u_target"].map((id) => ({
|
||||
id,
|
||||
email: `${id}@x.org`,
|
||||
firstName: "Fn",
|
||||
lastName: "Ln",
|
||||
active: true,
|
||||
pendingDeletion: false,
|
||||
pseudonymized: false,
|
||||
}));
|
||||
const { runner, calls } = makeRunner({ User: userRows });
|
||||
const result = await collectAndWriteGlobals({
|
||||
postgresDir,
|
||||
globalsDir,
|
||||
prisma: runner,
|
||||
profile: "full-anonymized",
|
||||
ctx: { salt: "x", profileName: "full-anonymized" },
|
||||
});
|
||||
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(new Set(calls[0].ids)).toEqual(new Set(["u_from", "u_acc", "u_actor", "u_target"]));
|
||||
expect(result.entries.find((e) => e.model === "User")?.rowCount).toBe(4);
|
||||
});
|
||||
|
||||
it("warns and continues when a referenced id is missing from the source DB", async () => {
|
||||
await writeFile(
|
||||
join(postgresDir, "TenantUser.jsonl"),
|
||||
`${JSON.stringify({ id: "tu1", tenantId: "t1", userId: "u_present" })}\n${JSON.stringify({ id: "tu2", tenantId: "t1", userId: "u_missing" })}\n`,
|
||||
);
|
||||
const { runner } = makeRunner({
|
||||
User: [{ id: "u_present", email: "p@x.org", firstName: "P", lastName: "X" }],
|
||||
});
|
||||
const warn = vi.fn();
|
||||
const result = await collectAndWriteGlobals({
|
||||
postgresDir,
|
||||
globalsDir,
|
||||
prisma: runner,
|
||||
profile: "full-anonymized",
|
||||
ctx: { salt: "x", profileName: "full-anonymized" },
|
||||
logger: { info: () => undefined, warn },
|
||||
});
|
||||
|
||||
expect(result.entries.find((e) => e.model === "User")?.rowCount).toBe(1);
|
||||
expect(result.missingByModel["User"]).toBe(1);
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("dangling FK"));
|
||||
});
|
||||
|
||||
it("chunks findManyByIds calls according to chunkSize", async () => {
|
||||
// 7 ids with chunkSize=3 → ceil(7/3) = 3 calls.
|
||||
const ids = ["u1", "u2", "u3", "u4", "u5", "u6", "u7"];
|
||||
await writeFile(
|
||||
join(postgresDir, "TenantUser.jsonl"),
|
||||
ids.map((u, i) => JSON.stringify({ id: `tu${i}`, tenantId: "t1", userId: u })).join("\n") + "\n",
|
||||
);
|
||||
const userRows = ids.map((id) => ({ id, email: `${id}@x.org`, firstName: "F", lastName: "L" }));
|
||||
const { runner, calls } = makeRunner({ User: userRows });
|
||||
const result = await collectAndWriteGlobals({
|
||||
postgresDir,
|
||||
globalsDir,
|
||||
prisma: runner,
|
||||
profile: "full-anonymized",
|
||||
ctx: { salt: "x", profileName: "full-anonymized" },
|
||||
chunkSize: 3,
|
||||
});
|
||||
|
||||
expect(calls.length).toBe(3);
|
||||
expect(calls[0].ids.length).toBe(3);
|
||||
expect(calls[1].ids.length).toBe(3);
|
||||
expect(calls[2].ids.length).toBe(1);
|
||||
expect(result.entries.find((e) => e.model === "User")?.rowCount).toBe(7);
|
||||
});
|
||||
|
||||
it("produces a checksum that matches the file contents", async () => {
|
||||
await writeFile(
|
||||
join(postgresDir, "TenantUser.jsonl"),
|
||||
`${JSON.stringify({ id: "tu1", tenantId: "t1", userId: "u1" })}\n`,
|
||||
);
|
||||
const { runner } = makeRunner({
|
||||
User: [{ id: "u1", email: "e@x.org", firstName: "F", lastName: "L" }],
|
||||
});
|
||||
const result = await collectAndWriteGlobals({
|
||||
postgresDir,
|
||||
globalsDir,
|
||||
prisma: runner,
|
||||
profile: "full-anonymized",
|
||||
ctx: { salt: "x", profileName: "full-anonymized" },
|
||||
});
|
||||
const entry = result.entries.find((e) => e.model === "User");
|
||||
expect(entry?.checksum.algorithm).toBe("sha256");
|
||||
expect(entry?.checksum.value).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(entry?.sizeBytes).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* Globals extractor for tenant snapshots.
|
||||
*
|
||||
* Closes the design-limit documented in `docs/ops/tenant-snapshot.md`
|
||||
* (added by commit 377b96a4 / B7 verification): tenant-scoped models
|
||||
* carry foreign keys into globally-scoped tables (today only `User`),
|
||||
* which means restoring into a truly empty target database fails on
|
||||
* those models with `ForeignKeyConstraintViolation`. This runner walks
|
||||
* every tenant-scoped row already collected by `findManyByTenant`,
|
||||
* harvests the referenced global ids, and emits one JSONL file per
|
||||
* global model into `globals/<Model>.jsonl` inside the snapshot
|
||||
* archive.
|
||||
*
|
||||
* Design notes
|
||||
* ------------
|
||||
* - Two-pass against the source: pass one (`findManyByTenant`) is the
|
||||
* existing tenant-scoped extractor; pass two streams those rows
|
||||
* through `collectGlobalIds()` to build the FK-id sets, then issues
|
||||
* chunked `findManyByIds()` calls against the Prisma client. We do
|
||||
* NOT re-issue the tenant queries — we read the JSONL files we just
|
||||
* wrote so the pass-two memory footprint is bounded by chunk size
|
||||
* regardless of tenant size.
|
||||
* - Anonymization is applied to the global rows using the same
|
||||
* per-snapshot salt as the tenant slice. Determinism: if two tenant
|
||||
* rows reference the same user, both restored rows still see the
|
||||
* same anonymized email/firstName/lastName because the seed is
|
||||
* `sha256(salt + originalValue)` (same as the tenant-side path).
|
||||
* - "Missing global" tolerance: if a referenced id is not present in
|
||||
* the source DB (e.g. the user was hard-deleted but a tenant row
|
||||
* still references it via a nullable FK that was never cleared), we
|
||||
* warn via the supplied logger and drop the id. Failing the snapshot
|
||||
* for this case would be wrong: the source is already in an
|
||||
* inconsistent state, and restoring the tenant slice without the
|
||||
* dangling reference is the conservative outcome.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import {
|
||||
PROFILES,
|
||||
anonymizeRow,
|
||||
type AnonymizationContext,
|
||||
type ProfileName,
|
||||
} from "./anonymize.js";
|
||||
import { snapshotReplacer } from "./json-codec.js";
|
||||
import { GLOBAL_REFERENCED_MODELS } from "./models.js";
|
||||
import type { TenantManifestGlobalsEntry } from "./manifest.js";
|
||||
|
||||
/**
|
||||
* Minimal Prisma-like surface the globals extractor needs in addition
|
||||
* to the existing `PrismaRunner.findManyByTenant`. Decoupled so callers
|
||||
* can fake it in unit tests.
|
||||
*/
|
||||
export interface GlobalsPrismaRunner {
|
||||
/**
|
||||
* Return every row of `modelName` whose `id` is in `ids`. The caller
|
||||
* pages the ids in chunks; this method must not perform its own
|
||||
* pagination on top.
|
||||
*/
|
||||
findManyByIds(modelName: string, ids: readonly string[]): Promise<Record<string, unknown>[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk a JSONL file row-by-row, returning ids referenced via the named
|
||||
* FK columns. Streams the file — never materializes the whole thing in
|
||||
* memory.
|
||||
*/
|
||||
async function collectFkIdsFromJsonl(
|
||||
filePath: string,
|
||||
fkColumns: readonly string[],
|
||||
): Promise<Set<string>> {
|
||||
const out = new Set<string>();
|
||||
try {
|
||||
await stat(filePath);
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
const stream = createReadStream(filePath, { encoding: "utf8" });
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||
for await (const line of rl) {
|
||||
if (line.trim().length === 0) continue;
|
||||
const row = JSON.parse(line) as Record<string, unknown>;
|
||||
for (const col of fkColumns) {
|
||||
const v = row[col];
|
||||
if (typeof v === "string" && v.length > 0) out.add(v);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface CollectGlobalsOptions {
|
||||
/** Directory containing the already-written tenant JSONL files. */
|
||||
readonly postgresDir: string;
|
||||
/** Directory the globals JSONL files should be written into. */
|
||||
readonly globalsDir: string;
|
||||
/** Prisma client capable of bulk id lookups. */
|
||||
readonly prisma: GlobalsPrismaRunner;
|
||||
/** Anonymization profile (full/minimal/none). */
|
||||
readonly profile: ProfileName;
|
||||
/** Salt used for the tenant slice — reused so anonymization is consistent. */
|
||||
readonly ctx: AnonymizationContext;
|
||||
/** Chunk size for `findManyByIds`. Defaults to 500 (Prisma `IN ()` ceiling is much higher; 500 is conservative for Postgres-side memory). */
|
||||
readonly chunkSize?: number;
|
||||
readonly logger?: { info: (m: string) => void; warn: (m: string) => void };
|
||||
}
|
||||
|
||||
export interface CollectGlobalsResult {
|
||||
readonly entries: TenantManifestGlobalsEntry[];
|
||||
/** Per-global, the count of ids that were referenced but not found in source. */
|
||||
readonly missingByModel: Readonly<Record<string, number>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk every tenant-scoped JSONL file in `postgresDir`, harvest FK ids
|
||||
* for each global model, fetch the rows from Prisma in chunks, run the
|
||||
* anonymizer, and write `globals/<Model>.jsonl`. Returns the manifest
|
||||
* entries describing what was written.
|
||||
*/
|
||||
export async function collectAndWriteGlobals(
|
||||
opts: CollectGlobalsOptions,
|
||||
): Promise<CollectGlobalsResult> {
|
||||
const chunkSize = opts.chunkSize ?? 500;
|
||||
const logger = opts.logger;
|
||||
const profile = PROFILES[opts.profile];
|
||||
const entries: TenantManifestGlobalsEntry[] = [];
|
||||
const missingByModel: Record<string, number> = {};
|
||||
|
||||
for (const global of GLOBAL_REFERENCED_MODELS) {
|
||||
// 1. Build the union of referenced ids across every source model.
|
||||
const referencedIds = new Set<string>();
|
||||
// Group columns by source model so we read each JSONL file at most once.
|
||||
const columnsBySource = new Map<string, string[]>();
|
||||
for (const ref of global.referencedBy) {
|
||||
const arr = columnsBySource.get(ref.model) ?? [];
|
||||
arr.push(ref.fkColumn);
|
||||
columnsBySource.set(ref.model, arr);
|
||||
}
|
||||
for (const [sourceModel, cols] of columnsBySource) {
|
||||
const filePath = join(opts.postgresDir, `${sourceModel}.jsonl`);
|
||||
const ids = await collectFkIdsFromJsonl(filePath, cols);
|
||||
for (const id of ids) referencedIds.add(id);
|
||||
}
|
||||
|
||||
// 2. Fetch the rows in chunks and stream-write the JSONL.
|
||||
const outPath = join(opts.globalsDir, `${global.model}.jsonl`);
|
||||
const hash = createHash("sha256");
|
||||
const writeStream = createWriteStream(outPath, { encoding: "utf8" });
|
||||
let rowCount = 0;
|
||||
let missing = 0;
|
||||
const idArr = Array.from(referencedIds);
|
||||
if (idArr.length > 0) {
|
||||
logger?.info(`Globals: ${global.model} → ${idArr.length} referenced ids`);
|
||||
}
|
||||
for (let i = 0; i < idArr.length; i += chunkSize) {
|
||||
const chunk = idArr.slice(i, i + chunkSize);
|
||||
const rows = await opts.prisma.findManyByIds(global.model, chunk);
|
||||
const foundIds = new Set<string>();
|
||||
for (const row of rows) {
|
||||
const id = row["id"];
|
||||
if (typeof id === "string") foundIds.add(id);
|
||||
const anonymized = anonymizeRow(profile, global.model, row, opts.ctx);
|
||||
const line = `${JSON.stringify(anonymized, snapshotReplacer)}\n`;
|
||||
hash.update(line);
|
||||
rowCount += 1;
|
||||
if (!writeStream.write(line)) {
|
||||
await new Promise<void>((resolve) => writeStream.once("drain", () => resolve()));
|
||||
}
|
||||
}
|
||||
for (const requested of chunk) {
|
||||
if (!foundIds.has(requested)) missing += 1;
|
||||
}
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writeStream.end((err: Error | null | undefined) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
|
||||
if (missing > 0) {
|
||||
logger?.warn(
|
||||
`Globals: ${global.model} had ${missing} referenced id(s) with no source row (dangling FK). Continuing.`,
|
||||
);
|
||||
missingByModel[global.model] = missing;
|
||||
}
|
||||
|
||||
const s = await stat(outPath);
|
||||
entries.push({
|
||||
model: global.model,
|
||||
path: `globals/${global.model}.jsonl`,
|
||||
rowCount,
|
||||
sizeBytes: s.size,
|
||||
checksum: { algorithm: "sha256", value: hash.digest("hex") },
|
||||
});
|
||||
}
|
||||
|
||||
return { entries, missingByModel };
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { defineCommand } from "citty";
|
||||
import { tenantSnapshotCommand } from "./snapshot.js";
|
||||
import { tenantRestoreCommand } from "./restore.js";
|
||||
import { tenantListSnapshotsCommand } from "./list-snapshots.js";
|
||||
|
||||
export const tenantCommand = defineCommand({
|
||||
meta: {
|
||||
name: "tenant",
|
||||
description: "Tenant snapshot, restore, and listing (PII-anonymized).",
|
||||
},
|
||||
subCommands: {
|
||||
snapshot: tenantSnapshotCommand,
|
||||
restore: tenantRestoreCommand,
|
||||
"list-snapshots": tenantListSnapshotsCommand,
|
||||
},
|
||||
});
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
// @vitest-environment node
|
||||
// Buffer is a Uint8Array subclass at the Node realm; under jsdom the global
|
||||
// `Uint8Array` resolves to a different realm and `instanceof` returns false,
|
||||
// breaking snapshotReplacer's Bytes branch. This file exercises a Node-only
|
||||
// CLI codec, so pin the runtime explicitly.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
decodeRowForPrisma,
|
||||
parseSchemaScalarTypes,
|
||||
snapshotReplacer,
|
||||
} from "./json-codec.js";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = join(here, "..", "..", "..", "..", "..");
|
||||
const schemaDir = join(repoRoot, "prisma", "schema");
|
||||
|
||||
describe("snapshotReplacer", () => {
|
||||
it("serializes BigInt as a decimal string", () => {
|
||||
const out = JSON.stringify({ n: 9007199254740993n }, snapshotReplacer);
|
||||
expect(out).toBe('{"n":"9007199254740993"}');
|
||||
});
|
||||
|
||||
it("serializes Buffer / Uint8Array as base64", () => {
|
||||
const out = JSON.parse(JSON.stringify({ bytes: Buffer.from("hello") }, snapshotReplacer));
|
||||
expect(out.bytes).toBe(Buffer.from("hello").toString("base64"));
|
||||
});
|
||||
|
||||
it("serializes Date via its native ISO toJSON", () => {
|
||||
const d = new Date("2026-05-24T10:00:00.000Z");
|
||||
const out = JSON.parse(JSON.stringify({ at: d }, snapshotReplacer));
|
||||
expect(out.at).toBe("2026-05-24T10:00:00.000Z");
|
||||
});
|
||||
|
||||
it("serializes a Decimal-like object via its toString", () => {
|
||||
class Decimal {
|
||||
constructor(private readonly v: string) {}
|
||||
toString() {
|
||||
return this.v;
|
||||
}
|
||||
toFixed() {
|
||||
return this.v;
|
||||
}
|
||||
toExponential() {
|
||||
return this.v;
|
||||
}
|
||||
}
|
||||
const out = JSON.parse(JSON.stringify({ price: new Decimal("12.34") }, snapshotReplacer));
|
||||
expect(out.price).toBe("12.34");
|
||||
});
|
||||
|
||||
it("passes through plain JSON-friendly values", () => {
|
||||
const obj = { a: "x", b: 1, c: true, d: null, e: { nested: ["v"] } };
|
||||
const out = JSON.parse(JSON.stringify(obj, snapshotReplacer));
|
||||
expect(out).toEqual(obj);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseSchemaScalarTypes", () => {
|
||||
const map = parseSchemaScalarTypes(schemaDir);
|
||||
|
||||
it("captures CollabDoc.ydocState as Bytes", () => {
|
||||
const collab = map.get("CollabDoc");
|
||||
expect(collab?.fields.get("ydocState")).toBe("Bytes");
|
||||
});
|
||||
|
||||
it("captures RowValue.numberValue as Decimal (nullable + non-tenant-scoped)", () => {
|
||||
// Use a Decimal field that actually exists in the schema. Decimal
|
||||
// columns are rare in tenant-scoped models today — the codec must
|
||||
// still parse them so non-tenant-scoped models referenced indirectly
|
||||
// (or future schema growth) decode correctly.
|
||||
const rowValue = map.get("RowValue");
|
||||
expect(rowValue?.fields.get("numberValue")).toBe("Decimal");
|
||||
});
|
||||
|
||||
it("captures TenantUser.createdAt as DateTime", () => {
|
||||
const tu = map.get("TenantUser");
|
||||
expect(tu?.fields.get("createdAt")).toBe("DateTime");
|
||||
});
|
||||
|
||||
it("does not classify scalar String fields as coded", () => {
|
||||
const tu = map.get("TenantUser");
|
||||
expect(tu?.fields.get("userId")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("decodeRowForPrisma", () => {
|
||||
it("turns DateTime strings into Date objects", () => {
|
||||
const map = new Map<string, "DateTime">([["createdAt", "DateTime"]]);
|
||||
const row = decodeRowForPrisma(
|
||||
{ id: "x", createdAt: "2026-05-24T10:00:00.000Z" },
|
||||
map,
|
||||
);
|
||||
expect(row.createdAt).toBeInstanceOf(Date);
|
||||
expect((row.createdAt as Date).toISOString()).toBe("2026-05-24T10:00:00.000Z");
|
||||
});
|
||||
|
||||
it("turns Bytes base64 strings into Buffer instances", () => {
|
||||
const map = new Map<string, "Bytes">([["data", "Bytes"]]);
|
||||
const row = decodeRowForPrisma(
|
||||
{ id: "x", data: Buffer.from("hello").toString("base64") },
|
||||
map,
|
||||
);
|
||||
expect(Buffer.isBuffer(row.data)).toBe(true);
|
||||
expect((row.data as Buffer).toString("utf8")).toBe("hello");
|
||||
});
|
||||
|
||||
it("turns BigInt strings into BigInts", () => {
|
||||
const map = new Map<string, "BigInt">([["counter", "BigInt"]]);
|
||||
const row = decodeRowForPrisma({ counter: "9007199254740993" }, map);
|
||||
expect(typeof row.counter).toBe("bigint");
|
||||
expect(row.counter).toBe(9007199254740993n);
|
||||
});
|
||||
|
||||
it("keeps Decimal as string for Prisma createMany", () => {
|
||||
const map = new Map<string, "Decimal">([["price", "Decimal"]]);
|
||||
const row = decodeRowForPrisma({ price: "12.34" }, map);
|
||||
expect(row.price).toBe("12.34");
|
||||
});
|
||||
|
||||
it("passes null through for nullable coded fields", () => {
|
||||
const map = new Map<string, "DateTime">([["deletedAt", "DateTime"]]);
|
||||
const row = decodeRowForPrisma({ deletedAt: null }, map);
|
||||
expect(row.deletedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("returns the row unchanged when no coded fields exist", () => {
|
||||
const row = decodeRowForPrisma({ id: "x", name: "y" }, undefined);
|
||||
expect(row).toEqual({ id: "x", name: "y" });
|
||||
});
|
||||
});
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* JSON serialization helpers for the tenant snapshot ↔ restore round-trip.
|
||||
*
|
||||
* Prisma rows carry types that vanilla `JSON.stringify` either rejects
|
||||
* outright (BigInt) or mangles into shapes the round-trip cannot reverse
|
||||
* (Buffer → `{type:"Buffer",data:[...]}`). The snapshot side uses
|
||||
* {@link snapshotReplacer} so the JSONL line is a portable plain-JSON
|
||||
* value; the restore side uses {@link decodeRowForPrisma} with a
|
||||
* per-model scalar-type map (parsed from the prisma schema files) to
|
||||
* coerce strings back into the native types Prisma's `createMany`
|
||||
* expects.
|
||||
*
|
||||
* Scalar handling
|
||||
* - `DateTime` → ISO string on the wire, `Date` on the way back in.
|
||||
* - `BigInt` → decimal string on the wire, `bigint` on the way back in.
|
||||
* - `Bytes` → base64 string on the wire, `Buffer` on the way back in.
|
||||
* - `Decimal` → decimal string on the wire, kept as `string` on restore
|
||||
* because Prisma's `Decimal` columns accept string inputs
|
||||
* to `createMany` and parse them safely.
|
||||
* - everything else passes through untouched (`Json` columns ride as the
|
||||
* plain object/array the row contained, scalar strings/numbers/booleans
|
||||
* stay as-is, nulls stay null).
|
||||
*/
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
/**
|
||||
* JSON.stringify replacer that turns Prisma's non-JSON-native scalars into
|
||||
* lossless plain-JSON representations. Pass it as the second argument to
|
||||
* `JSON.stringify(value, snapshotReplacer)`.
|
||||
*
|
||||
* V8 invokes `toJSON` BEFORE the replacer is called, so by the time we
|
||||
* see `value` for a `Date` or `Buffer`, it has already been mangled
|
||||
* (Date → ISO string, Buffer → `{type:"Buffer",data:[...]}`). We reach
|
||||
* past that by reading `this[key]` — `this` is the holder object that
|
||||
* still references the original native instance. The root call binds
|
||||
* `this` to `{"": <rootValue>}` per the spec, so the same lookup works
|
||||
* uniformly. `value` is only inspected as a fallback for the BigInt
|
||||
* case (BigInts have no `toJSON` and so survive intact in both slots).
|
||||
*/
|
||||
export function snapshotReplacer(this: unknown, key: string, value: unknown): unknown {
|
||||
const holder = this as Record<string, unknown> | undefined;
|
||||
const raw = holder !== undefined && key in holder ? holder[key] : value;
|
||||
if (raw === null || raw === undefined) return value;
|
||||
if (typeof raw === "bigint") return raw.toString();
|
||||
if (raw instanceof Date) return raw.toISOString();
|
||||
if (raw instanceof Uint8Array) {
|
||||
// Buffer is a Uint8Array subclass; this branch catches both.
|
||||
return Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength).toString("base64");
|
||||
}
|
||||
// Prisma Decimal duck-typing. Both decimal.js and decimal.js-light
|
||||
// expose `toFixed`, `toExponential` and a constructor named "Decimal".
|
||||
// We guard against plain numbers (which also satisfy `toFixed`) by
|
||||
// requiring an object that is *not* a Number wrapper.
|
||||
if (
|
||||
typeof raw === "object" &&
|
||||
typeof (raw as { toFixed?: unknown }).toFixed === "function" &&
|
||||
typeof (raw as { toExponential?: unknown }).toExponential === "function" &&
|
||||
(raw as { constructor?: { name?: string } }).constructor?.name === "Decimal"
|
||||
) {
|
||||
return (raw as { toString: () => string }).toString();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* The subset of Prisma scalar types that need special restore-side handling.
|
||||
* Anything not in this list rides through as-is.
|
||||
*/
|
||||
export type CodedScalar = "DateTime" | "BigInt" | "Bytes" | "Decimal";
|
||||
|
||||
export interface ModelScalarTypeMap {
|
||||
readonly modelName: string;
|
||||
/** Map field name → Prisma scalar type (only includes interesting ones). */
|
||||
readonly fields: ReadonlyMap<string, CodedScalar>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse every `prisma/schema/*.prisma` file under `schemaDir` and produce
|
||||
* a per-model map of fields whose scalar type needs special decoding on
|
||||
* restore.
|
||||
*
|
||||
* The parser is intentionally regex-based to avoid pulling the Prisma
|
||||
* schema parser as a runtime dependency. It handles the subset the
|
||||
* codebase actually uses today (single-line field declarations with
|
||||
* `Name Type modifiers...`). The companion test in `json-codec.test.ts`
|
||||
* cross-checks the output against a sample of known fields so a schema
|
||||
* format change fails CI loudly.
|
||||
*/
|
||||
export function parseSchemaScalarTypes(schemaDir: string): Map<string, ModelScalarTypeMap> {
|
||||
const result = new Map<string, ModelScalarTypeMap>();
|
||||
const files = readdirSync(schemaDir).filter((f) => f.endsWith(".prisma"));
|
||||
const modelBlock = /model\s+(\w+)\s*\{([\s\S]*?)\n\}/g;
|
||||
// `name Type` — Type captured without trailing `?`/`[]`.
|
||||
const fieldLine = /^\s*(\w+)\s+(\w+)(\??)(\[\])?/u;
|
||||
|
||||
for (const file of files) {
|
||||
const text = readFileSync(join(schemaDir, file), "utf8");
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = modelBlock.exec(text)) !== null) {
|
||||
const modelName = m[1]!;
|
||||
const body = m[2]!;
|
||||
const fields = new Map<string, CodedScalar>();
|
||||
for (const line of body.split("\n")) {
|
||||
const fm = fieldLine.exec(line);
|
||||
if (!fm) continue;
|
||||
const fieldName = fm[1]!;
|
||||
const fieldType = fm[2]!;
|
||||
// Skip relation/block-attribute lines that look field-shaped.
|
||||
if (fieldName.startsWith("@")) continue;
|
||||
if (fieldType === "DateTime" || fieldType === "BigInt" || fieldType === "Bytes" || fieldType === "Decimal") {
|
||||
fields.set(fieldName, fieldType);
|
||||
}
|
||||
}
|
||||
if (fields.size > 0) {
|
||||
result.set(modelName, { modelName, fields });
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce a JSON-parsed row back into the native types Prisma expects.
|
||||
*
|
||||
* - DateTime strings → `Date` objects (invalid strings stay as-is and
|
||||
* surface as a Prisma error later — visible failures beat silent
|
||||
* corruption).
|
||||
* - BigInt strings → `bigint`.
|
||||
* - Bytes strings → `Buffer` (decoded from base64).
|
||||
* - Decimal strings → passed through; Prisma accepts strings for Decimal
|
||||
* columns and parses them with its own canonical Decimal class.
|
||||
* - Fields not in `fieldTypes` (or whose value is null/already-typed)
|
||||
* pass through unchanged.
|
||||
*/
|
||||
export function decodeRowForPrisma(
|
||||
row: Record<string, unknown>,
|
||||
fieldTypes: ReadonlyMap<string, CodedScalar> | undefined,
|
||||
): Record<string, unknown> {
|
||||
if (!fieldTypes || fieldTypes.size === 0) return row;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
const scalar = fieldTypes.get(key);
|
||||
if (value === null || value === undefined || scalar === undefined) {
|
||||
out[key] = value;
|
||||
continue;
|
||||
}
|
||||
if (scalar === "DateTime") {
|
||||
out[key] = typeof value === "string" ? new Date(value) : value;
|
||||
continue;
|
||||
}
|
||||
if (scalar === "BigInt") {
|
||||
out[key] = typeof value === "string" || typeof value === "number" ? BigInt(value) : value;
|
||||
continue;
|
||||
}
|
||||
if (scalar === "Bytes") {
|
||||
out[key] = typeof value === "string" ? Buffer.from(value, "base64") : value;
|
||||
continue;
|
||||
}
|
||||
// Decimal — keep as string.
|
||||
out[key] = value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { mockClient } from "aws-sdk-client-mock";
|
||||
import { ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
detectEncryption,
|
||||
listTenantSnapshots,
|
||||
parseTenantIdFromKey,
|
||||
} from "./list-snapshots.js";
|
||||
|
||||
describe("parseTenantIdFromKey", () => {
|
||||
it("returns the tenant id from a canonical key", () => {
|
||||
expect(parseTenantIdFromKey("tenant-snapshot-tenant_abc-2026-05-17T03:00:00Z-deadbeef.tar.gz")).toBe(
|
||||
"tenant_abc",
|
||||
);
|
||||
});
|
||||
|
||||
it("strips an S3 directory prefix before parsing", () => {
|
||||
expect(parseTenantIdFromKey("snapshots/tenant-snapshot-tenant_xyz-2026-05-17.tar.gz.age")).toBe(
|
||||
"tenant_xyz",
|
||||
);
|
||||
});
|
||||
|
||||
it("returns null for non-matching keys", () => {
|
||||
expect(parseTenantIdFromKey("random-blob.tar.gz")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectEncryption", () => {
|
||||
it.each([
|
||||
["snap.tar.gz", "none"],
|
||||
["snap.tar.gz.age", "age"],
|
||||
["snap.tar.gz.AGE", "age"],
|
||||
["snap.tar.gz.gpg", "gpg"],
|
||||
])("detects %s as %s", (key, expected) => {
|
||||
expect(detectEncryption(key)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("listTenantSnapshots", () => {
|
||||
it("paginates ListObjectsV2 and returns parsed entries", async () => {
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
let calls = 0;
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake(() => {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
return {
|
||||
Contents: [
|
||||
{ Key: "tenant-snapshot-tenant_a-2026-05-17.tar.gz.age", Size: 1024, ETag: '"a1"' },
|
||||
{ Key: "tenant-snapshot-tenant_b-2026-05-16.tar.gz", Size: 512, ETag: '"b1"' },
|
||||
],
|
||||
IsTruncated: true,
|
||||
NextContinuationToken: "next",
|
||||
};
|
||||
}
|
||||
return {
|
||||
Contents: [{ Key: "tenant-snapshot-tenant_c-2026-05-15.tar.gz.gpg", Size: 256, ETag: '"c1"' }],
|
||||
IsTruncated: false,
|
||||
};
|
||||
});
|
||||
|
||||
const entries = await listTenantSnapshots({
|
||||
bucket: "snapshots",
|
||||
client: new S3Client({ region: "us-east-1" }),
|
||||
});
|
||||
expect(entries).toHaveLength(3);
|
||||
expect(entries[0].tenantId).toBe("tenant_a");
|
||||
expect(entries[0].encryption).toBe("age");
|
||||
expect(entries[1].encryption).toBe("none");
|
||||
expect(entries[2].encryption).toBe("gpg");
|
||||
});
|
||||
|
||||
it("returns an empty list when bucket is empty", async () => {
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake(() => ({ Contents: [], IsTruncated: false }));
|
||||
const entries = await listTenantSnapshots({
|
||||
bucket: "snapshots",
|
||||
client: new S3Client({ region: "us-east-1" }),
|
||||
});
|
||||
expect(entries).toEqual([]);
|
||||
});
|
||||
});
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* `tenant:list-snapshots` — enumerate tenant-snapshot objects in an S3 bucket
|
||||
* and emit a tabular summary. Optionally fetches and decodes each manifest
|
||||
* (lightweight head + GetObject) to surface anonymization-profile and row
|
||||
* counts; off by default to keep the listing cheap.
|
||||
*/
|
||||
import { defineCommand } from "citty";
|
||||
import consola from "consola";
|
||||
import {
|
||||
ListObjectsV2Command,
|
||||
S3Client,
|
||||
type S3ClientConfig,
|
||||
} from "@aws-sdk/client-s3";
|
||||
|
||||
export interface TenantSnapshotListEntry {
|
||||
readonly key: string;
|
||||
readonly sizeBytes: number;
|
||||
readonly lastModified: Date | null;
|
||||
readonly etag: string | null;
|
||||
/** Parsed from the key — `null` if the convention does not match. */
|
||||
readonly tenantId: string | null;
|
||||
/** Parsed from the key — encrypted snapshots end in `.age` or `.gpg`. */
|
||||
readonly encryption: "age" | "gpg" | "none";
|
||||
}
|
||||
|
||||
export interface ListTenantSnapshotsOptions {
|
||||
readonly bucket: string;
|
||||
readonly client?: S3Client;
|
||||
readonly s3?: S3ClientConfig;
|
||||
/** Override the default `tenant-snapshot-` listing prefix. */
|
||||
readonly prefix?: string;
|
||||
}
|
||||
|
||||
const SNAPSHOT_KEY_RE = /^tenant-snapshot-([^/-][^/-]*)-/u;
|
||||
const ENCRYPTION_SUFFIX_RE = /\.(age|gpg)$/iu;
|
||||
|
||||
export async function listTenantSnapshots(
|
||||
opts: ListTenantSnapshotsOptions,
|
||||
): Promise<TenantSnapshotListEntry[]> {
|
||||
const client = opts.client ?? new S3Client(opts.s3 ?? {});
|
||||
const out: TenantSnapshotListEntry[] = [];
|
||||
let continuation: string | undefined;
|
||||
do {
|
||||
const page = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: opts.bucket,
|
||||
Prefix: opts.prefix ?? "tenant-snapshot-",
|
||||
ContinuationToken: continuation,
|
||||
}),
|
||||
);
|
||||
for (const obj of page.Contents ?? []) {
|
||||
if (!obj.Key) continue;
|
||||
out.push({
|
||||
key: obj.Key,
|
||||
sizeBytes: obj.Size ?? 0,
|
||||
lastModified: obj.LastModified ?? null,
|
||||
etag: obj.ETag ?? null,
|
||||
tenantId: parseTenantIdFromKey(obj.Key),
|
||||
encryption: detectEncryption(obj.Key),
|
||||
});
|
||||
}
|
||||
continuation = page.IsTruncated ? page.NextContinuationToken : undefined;
|
||||
} while (continuation);
|
||||
return out;
|
||||
}
|
||||
|
||||
export function parseTenantIdFromKey(key: string): string | null {
|
||||
// Strip directory prefix if present.
|
||||
const last = key.lastIndexOf("/");
|
||||
const file = last === -1 ? key : key.slice(last + 1);
|
||||
const m = SNAPSHOT_KEY_RE.exec(file);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
export function detectEncryption(key: string): "age" | "gpg" | "none" {
|
||||
const m = ENCRYPTION_SUFFIX_RE.exec(key);
|
||||
if (!m) return "none";
|
||||
const suffix = m[1].toLowerCase();
|
||||
return suffix === "age" || suffix === "gpg" ? suffix : "none";
|
||||
}
|
||||
|
||||
export const tenantListSnapshotsCommand = defineCommand({
|
||||
meta: {
|
||||
name: "list-snapshots",
|
||||
description: "List tenant snapshots stored in the snapshot S3 bucket.",
|
||||
},
|
||||
args: {
|
||||
bucket: { type: "string", description: "S3 bucket to list (defaults to BACKUP_S3_BUCKET)." },
|
||||
prefix: { type: "string", description: "Override the default key prefix (`tenant-snapshot-`)." },
|
||||
},
|
||||
async run({ args }) {
|
||||
const bucket = String(args.bucket ?? process.env.BACKUP_S3_BUCKET ?? "");
|
||||
if (bucket.length === 0) {
|
||||
throw new Error("Provide --bucket or set BACKUP_S3_BUCKET");
|
||||
}
|
||||
const entries = await listTenantSnapshots({ bucket, prefix: args.prefix ? String(args.prefix) : undefined });
|
||||
if (entries.length === 0) {
|
||||
consola.info("No tenant snapshots found.");
|
||||
return;
|
||||
}
|
||||
consola.box(
|
||||
[
|
||||
"KEY".padEnd(60) + " | " + "SIZE".padStart(10) + " | TENANT".padEnd(20) + " | ENC",
|
||||
...entries.map(
|
||||
(e) =>
|
||||
e.key.padEnd(60) +
|
||||
" | " +
|
||||
String(e.sizeBytes).padStart(10) +
|
||||
" | " +
|
||||
(e.tenantId ?? "?").padEnd(18) +
|
||||
" | " +
|
||||
e.encryption,
|
||||
),
|
||||
].join("\n"),
|
||||
);
|
||||
},
|
||||
});
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
TENANT_MANIFEST_VERSION,
|
||||
TenantSnapshotManifestSchema,
|
||||
deserializeTenantManifest,
|
||||
hashSalt,
|
||||
serializeTenantManifest,
|
||||
sha256Hex,
|
||||
type TenantSnapshotManifest,
|
||||
} from "./manifest.js";
|
||||
|
||||
const baseManifest: TenantSnapshotManifest = {
|
||||
schemaVersion: TENANT_MANIFEST_VERSION,
|
||||
snapshotId: "tenant-snapshot-acme-2026-05-17T03:00:00Z-abcd1234",
|
||||
sourceTenantId: "tenant_abc123",
|
||||
sourceTenantSlug: "acme",
|
||||
createdAt: "2026-05-17T03:00:00.000Z",
|
||||
creator: { cliVersion: "0.0.0", host: "ops-jumphost", actorUserId: "user_admin" },
|
||||
anonymization: {
|
||||
profile: "full-anonymized",
|
||||
saltHash: "a".repeat(64),
|
||||
},
|
||||
models: [
|
||||
{
|
||||
model: "TenantUser",
|
||||
path: "postgres/TenantUser.jsonl",
|
||||
rowCount: 12,
|
||||
sizeBytes: 1234,
|
||||
checksum: { algorithm: "sha256", value: "b".repeat(64) },
|
||||
},
|
||||
],
|
||||
storage: {
|
||||
path: "storage/objects.tar",
|
||||
objectCount: 7,
|
||||
sizeBytes: 2048,
|
||||
checksum: { algorithm: "sha256", value: "c".repeat(64) },
|
||||
},
|
||||
encryption: { method: "age", recipient: "age1example..." },
|
||||
};
|
||||
|
||||
describe("tenant snapshot manifest schema", () => {
|
||||
it("round-trips a valid manifest", () => {
|
||||
const serialized = serializeTenantManifest(baseManifest);
|
||||
expect(serialized.endsWith("\n")).toBe(true);
|
||||
const parsed = deserializeTenantManifest(serialized);
|
||||
expect(parsed).toEqual(baseManifest);
|
||||
});
|
||||
|
||||
it("rejects an unsupported profile name", () => {
|
||||
const invalid = {
|
||||
...baseManifest,
|
||||
anonymization: { ...baseManifest.anonymization, profile: "zalgo" },
|
||||
} as unknown;
|
||||
expect(() => TenantSnapshotManifestSchema.parse(invalid)).toThrow();
|
||||
});
|
||||
|
||||
it("rejects an invalid salt hash format", () => {
|
||||
const invalid: TenantSnapshotManifest = {
|
||||
...baseManifest,
|
||||
anonymization: { ...baseManifest.anonymization, saltHash: "not-hex" },
|
||||
};
|
||||
expect(() => TenantSnapshotManifestSchema.parse(invalid)).toThrow();
|
||||
});
|
||||
|
||||
it("rejects unknown top-level keys (strict)", () => {
|
||||
const invalid = { ...baseManifest, oops: true } as unknown;
|
||||
expect(() => TenantSnapshotManifestSchema.parse(invalid)).toThrow();
|
||||
});
|
||||
|
||||
it("rejects negative row counts", () => {
|
||||
const invalid: TenantSnapshotManifest = {
|
||||
...baseManifest,
|
||||
models: [{ ...baseManifest.models[0], rowCount: -1 }],
|
||||
};
|
||||
expect(() => TenantSnapshotManifestSchema.parse(invalid)).toThrow();
|
||||
});
|
||||
|
||||
it("accepts manifests with zero models (edge case: empty tenant)", () => {
|
||||
const empty: TenantSnapshotManifest = { ...baseManifest, models: [] };
|
||||
expect(() => TenantSnapshotManifestSchema.parse(empty)).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts manifests with no storage component", () => {
|
||||
const noStorage = { ...baseManifest };
|
||||
delete (noStorage as Partial<TenantSnapshotManifest>).storage;
|
||||
expect(() => TenantSnapshotManifestSchema.parse(noStorage)).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts manifests with a globals slice", () => {
|
||||
const withGlobals: TenantSnapshotManifest = {
|
||||
...baseManifest,
|
||||
globals: [
|
||||
{
|
||||
model: "User",
|
||||
path: "globals/User.jsonl",
|
||||
rowCount: 3,
|
||||
sizeBytes: 512,
|
||||
checksum: { algorithm: "sha256", value: "d".repeat(64) },
|
||||
},
|
||||
],
|
||||
};
|
||||
const parsed = deserializeTenantManifest(serializeTenantManifest(withGlobals));
|
||||
expect(parsed.globals?.[0]?.model).toBe("User");
|
||||
expect(parsed.globals?.[0]?.rowCount).toBe(3);
|
||||
});
|
||||
|
||||
it("treats absence of globals as backward-compatible (legacy snapshots)", () => {
|
||||
const legacy = { ...baseManifest };
|
||||
delete (legacy as Partial<TenantSnapshotManifest>).globals;
|
||||
const parsed = deserializeTenantManifest(serializeTenantManifest(legacy));
|
||||
expect(parsed.globals).toBeUndefined();
|
||||
});
|
||||
|
||||
it("hashSalt returns deterministic hex", () => {
|
||||
const a = hashSalt("salt-1");
|
||||
const b = hashSalt("salt-1");
|
||||
expect(a).toBe(b);
|
||||
expect(a).toMatch(/^[a-f0-9]{64}$/u);
|
||||
});
|
||||
|
||||
it("hashSalt yields different output for different inputs", () => {
|
||||
expect(hashSalt("salt-1")).not.toBe(hashSalt("salt-2"));
|
||||
});
|
||||
|
||||
it("sha256Hex returns lowercase 64-char hex", () => {
|
||||
const hex = sha256Hex("hello world");
|
||||
expect(hex).toHaveLength(64);
|
||||
expect(hex).toMatch(/^[a-f0-9]{64}$/u);
|
||||
});
|
||||
});
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* Tenant-snapshot manifest. Mirrors the shape of `backup/manifest.ts` so the
|
||||
* operator UX stays consistent, but adds tenant-specific metadata: source
|
||||
* tenant id, anonymization profile, salt hash (NOT salt itself), per-model
|
||||
* row counts, storage object count.
|
||||
*
|
||||
* The salt itself is intentionally not stored: only `sha256(salt)` lives in
|
||||
* the manifest as a reproducibility marker. The original salt is discarded
|
||||
* after snapshot creation so that no reverse-engineering path exists.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { z } from "zod";
|
||||
|
||||
export const TENANT_MANIFEST_VERSION = 1;
|
||||
|
||||
export const TenantManifestChecksumSchema = z
|
||||
.object({
|
||||
algorithm: z.literal("sha256"),
|
||||
value: z.string().regex(/^[a-f0-9]{64}$/u, "expected lowercase hex sha256"),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const TenantManifestModelEntrySchema = z
|
||||
.object({
|
||||
/** Prisma model name (PascalCase as written in `schema.prisma`). */
|
||||
model: z.string().min(1),
|
||||
/** Path within the tar archive (relative to manifest.json). */
|
||||
path: z.string().min(1),
|
||||
/** Row count after anonymization. */
|
||||
rowCount: z.number().int().nonnegative(),
|
||||
/** sha256 of the JSONL file. */
|
||||
checksum: TenantManifestChecksumSchema,
|
||||
sizeBytes: z.number().int().nonnegative(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
/**
|
||||
* Global (non-tenant-scoped) row slice shipped alongside the tenant
|
||||
* data. Same shape as `TenantManifestModelEntry`; lives under
|
||||
* `globals/<Model>.jsonl` inside the archive. See
|
||||
* `packages/cli/src/commands/tenant/models.ts::GLOBAL_REFERENCED_MODELS`
|
||||
* for what gets extracted.
|
||||
*
|
||||
* Backward-compat: snapshots taken before the globals extractor landed
|
||||
* carry no `globals` entry in the manifest. The restore path treats
|
||||
* that as "no globals to replay" and falls back to the pre-globals
|
||||
* behaviour where the operator must pre-seed referenced rows.
|
||||
*/
|
||||
export const TenantManifestGlobalsEntrySchema = z
|
||||
.object({
|
||||
model: z.string().min(1),
|
||||
path: z.string().min(1),
|
||||
rowCount: z.number().int().nonnegative(),
|
||||
checksum: TenantManifestChecksumSchema,
|
||||
sizeBytes: z.number().int().nonnegative(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const TenantManifestStorageEntrySchema = z
|
||||
.object({
|
||||
path: z.string().min(1),
|
||||
objectCount: z.number().int().nonnegative(),
|
||||
sizeBytes: z.number().int().nonnegative(),
|
||||
checksum: TenantManifestChecksumSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const TenantManifestEncryptionSchema = z
|
||||
.object({
|
||||
method: z.enum(["age", "gpg", "none"]),
|
||||
recipient: z.string().optional(),
|
||||
keyFingerprint: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const TenantSnapshotManifestSchema = z
|
||||
.object({
|
||||
schemaVersion: z.literal(TENANT_MANIFEST_VERSION),
|
||||
snapshotId: z.string().min(1),
|
||||
sourceTenantId: z.string().min(1),
|
||||
sourceTenantSlug: z.string().optional(),
|
||||
createdAt: z.string().datetime({ offset: false }),
|
||||
creator: z
|
||||
.object({
|
||||
cliVersion: z.string().min(1),
|
||||
host: z.string().optional(),
|
||||
actorUserId: z.string().optional(),
|
||||
})
|
||||
.strict(),
|
||||
anonymization: z
|
||||
.object({
|
||||
profile: z.enum(["full-anonymized", "minimal-anonymized", "none"]),
|
||||
saltHash: z.string().regex(/^[a-f0-9]{64}$/u, "expected lowercase hex sha256"),
|
||||
})
|
||||
.strict(),
|
||||
models: TenantManifestModelEntrySchema.array(),
|
||||
/**
|
||||
* Globals slice (referenced non-tenant-scoped rows like `User`).
|
||||
* Absent on snapshots taken before this extractor landed; restore
|
||||
* tolerates that and warns.
|
||||
*/
|
||||
globals: TenantManifestGlobalsEntrySchema.array().optional(),
|
||||
storage: TenantManifestStorageEntrySchema.optional(),
|
||||
encryption: TenantManifestEncryptionSchema,
|
||||
notes: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type TenantManifestChecksum = z.infer<typeof TenantManifestChecksumSchema>;
|
||||
export type TenantManifestModelEntry = z.infer<typeof TenantManifestModelEntrySchema>;
|
||||
export type TenantManifestGlobalsEntry = z.infer<typeof TenantManifestGlobalsEntrySchema>;
|
||||
export type TenantManifestStorageEntry = z.infer<typeof TenantManifestStorageEntrySchema>;
|
||||
export type TenantSnapshotManifest = z.infer<typeof TenantSnapshotManifestSchema>;
|
||||
|
||||
export function serializeTenantManifest(manifest: TenantSnapshotManifest): string {
|
||||
const parsed = TenantSnapshotManifestSchema.parse(manifest);
|
||||
return `${JSON.stringify(parsed, null, 2)}\n`;
|
||||
}
|
||||
|
||||
export function deserializeTenantManifest(raw: string | Buffer): TenantSnapshotManifest {
|
||||
const text = typeof raw === "string" ? raw : raw.toString("utf8");
|
||||
const data = JSON.parse(text) as unknown;
|
||||
return TenantSnapshotManifestSchema.parse(data);
|
||||
}
|
||||
|
||||
export function hashSalt(salt: string): string {
|
||||
return createHash("sha256").update(salt).digest("hex");
|
||||
}
|
||||
|
||||
export function sha256Hex(input: Buffer | string): string {
|
||||
return createHash("sha256").update(input).digest("hex");
|
||||
}
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
GLOBAL_REFERENCED_MODELS,
|
||||
PLATFORM_CATALOG_GLOBALS,
|
||||
TENANT_SCOPED_MODELS,
|
||||
globalSnapshotModelNames,
|
||||
isGlobalSnapshotModel,
|
||||
isTenantScopedModel,
|
||||
tenantScopedModelNames,
|
||||
} from "./models.js";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = join(here, "..", "..", "..", "..", "..");
|
||||
const schemaDir = join(repoRoot, "prisma", "schema");
|
||||
|
||||
interface SchemaModel {
|
||||
readonly name: string;
|
||||
readonly schema: string;
|
||||
readonly hasTenantId: boolean;
|
||||
}
|
||||
|
||||
function listSchemaModels(): SchemaModel[] {
|
||||
const files = readdirSync(schemaDir).filter((f) => f.endsWith(".prisma"));
|
||||
const models: SchemaModel[] = [];
|
||||
for (const file of files) {
|
||||
const text = readFileSync(join(schemaDir, file), "utf8");
|
||||
const re = /model\s+(\w+)\s*\{([\s\S]*?)\n\}/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
const body = m[2];
|
||||
const hasTenantId = /^\s*tenantId\s+String/m.test(body);
|
||||
models.push({ name: m[1], schema: file, hasTenantId });
|
||||
}
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
describe("TENANT_SCOPED_MODELS", () => {
|
||||
const schemaModels = listSchemaModels();
|
||||
const schemaTenantModels = schemaModels.filter((m) => m.hasTenantId);
|
||||
|
||||
it("covers every prisma model that has a tenantId String column", () => {
|
||||
const listed = new Set(TENANT_SCOPED_MODELS.map((m) => m.model));
|
||||
const missing = schemaTenantModels.filter((m) => !listed.has(m.name));
|
||||
expect(missing, `missing models in TENANT_SCOPED_MODELS: ${missing.map((x) => x.name).join(", ")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not list a model that is not actually tenant-scoped", () => {
|
||||
const schemaByName = new Map(schemaModels.map((m) => [m.name, m]));
|
||||
const bogus = TENANT_SCOPED_MODELS.filter((m) => {
|
||||
const s = schemaByName.get(m.model);
|
||||
return !s || !s.hasTenantId;
|
||||
});
|
||||
expect(bogus, `tenant-scoped list contains non-tenant models: ${bogus.map((x) => x.model).join(", ")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("declares unique model names", () => {
|
||||
const names = TENANT_SCOPED_MODELS.map((m) => m.model);
|
||||
expect(new Set(names).size).toBe(names.length);
|
||||
});
|
||||
|
||||
it("isTenantScopedModel returns true for every known model and false otherwise", () => {
|
||||
for (const m of TENANT_SCOPED_MODELS) {
|
||||
expect(isTenantScopedModel(m.model)).toBe(true);
|
||||
}
|
||||
expect(isTenantScopedModel("DefinitelyNotARealModel")).toBe(false);
|
||||
});
|
||||
|
||||
it("tenantScopedModelNames returns the list in declared order", () => {
|
||||
expect(tenantScopedModelNames()).toEqual(TENANT_SCOPED_MODELS.map((m) => m.model));
|
||||
});
|
||||
});
|
||||
|
||||
interface RelationRef {
|
||||
readonly sourceModel: string;
|
||||
readonly targetModel: string;
|
||||
readonly fkColumn: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse every prisma schema file and return one entry per FK relation
|
||||
* declared on a tenant-scoped model (i.e. lines with both `@relation` and
|
||||
* `fields: [...]`). Used by the globals-coverage tests below to prove
|
||||
* that every global referenced from the tenant-scoped slice is either
|
||||
* listed in GLOBAL_REFERENCED_MODELS, on PLATFORM_CATALOG_GLOBALS, or is
|
||||
* itself a tenant-scoped model (i.e. already inside the snapshot).
|
||||
*/
|
||||
function collectTenantScopedFkRelations(): RelationRef[] {
|
||||
const files = readdirSync(schemaDir).filter((f) => f.endsWith(".prisma"));
|
||||
const all: RelationRef[] = [];
|
||||
const tenantScopedNames = new Set(TENANT_SCOPED_MODELS.map((m) => m.model));
|
||||
for (const file of files) {
|
||||
const text = readFileSync(join(schemaDir, file), "utf8");
|
||||
const modelBlock = /model\s+(\w+)\s*\{([\s\S]*?)\n\}/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = modelBlock.exec(text)) !== null) {
|
||||
const sourceModel = m[1];
|
||||
const body = m[2];
|
||||
if (!tenantScopedNames.has(sourceModel)) continue;
|
||||
for (const line of body.split("\n")) {
|
||||
const s = line.trim();
|
||||
if (!s.includes("@relation")) continue;
|
||||
if (!s.includes("fields:")) continue;
|
||||
const fieldDecl = /^(\w+)\s+(\w+)(\??)/u.exec(s);
|
||||
if (!fieldDecl) continue;
|
||||
const targetModel = fieldDecl[2];
|
||||
const fieldsMatch = /fields:\s*\[([^\]]+)\]/u.exec(s);
|
||||
if (!fieldsMatch) continue;
|
||||
const cols = fieldsMatch[1].split(",").map((x) => x.trim()).filter(Boolean);
|
||||
for (const fk of cols) {
|
||||
all.push({ sourceModel, targetModel, fkColumn: fk });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
describe("GLOBAL_REFERENCED_MODELS + PLATFORM_CATALOG_GLOBALS coverage", () => {
|
||||
const schemaModels = listSchemaModels();
|
||||
const tenantScopedNames = new Set(TENANT_SCOPED_MODELS.map((m) => m.model));
|
||||
const globalNames = new Set(GLOBAL_REFERENCED_MODELS.map((m) => m.model));
|
||||
const platformNames = new Set(PLATFORM_CATALOG_GLOBALS);
|
||||
|
||||
it("every tenant-scoped FK relation targets a tenant-scoped model, a snapshot-global, or a platform-catalog global", () => {
|
||||
const relations = collectTenantScopedFkRelations();
|
||||
const uncategorized: RelationRef[] = [];
|
||||
for (const rel of relations) {
|
||||
if (tenantScopedNames.has(rel.targetModel)) continue;
|
||||
if (globalNames.has(rel.targetModel)) continue;
|
||||
if (platformNames.has(rel.targetModel)) continue;
|
||||
// `Tenant` itself is the snapshot root — restore creates the target
|
||||
// tenant row before replay, so it is never copied via the globals
|
||||
// pathway.
|
||||
if (rel.targetModel === "Tenant") continue;
|
||||
uncategorized.push(rel);
|
||||
}
|
||||
expect(
|
||||
uncategorized,
|
||||
`Uncategorized globals: ${uncategorized.map((r) => `${r.sourceModel}.${r.fkColumn} → ${r.targetModel}`).join(", ")}. Add the target model to GLOBAL_REFERENCED_MODELS (snapshot it) or PLATFORM_CATALOG_GLOBALS (treat as deployment baseline).`,
|
||||
).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every entry in GLOBAL_REFERENCED_MODELS corresponds to an actual prisma model with no tenantId column", () => {
|
||||
const byName = new Map(schemaModels.map((m) => [m.name, m]));
|
||||
const bogus = GLOBAL_REFERENCED_MODELS.filter((g) => {
|
||||
const s = byName.get(g.model);
|
||||
return !s || s.hasTenantId;
|
||||
});
|
||||
expect(bogus, `bogus global entries: ${bogus.map((x) => x.model).join(", ")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every entry in PLATFORM_CATALOG_GLOBALS corresponds to an actual prisma model with no tenantId column", () => {
|
||||
const byName = new Map(schemaModels.map((m) => [m.name, m]));
|
||||
const bogus = PLATFORM_CATALOG_GLOBALS.filter((name) => {
|
||||
const s = byName.get(name);
|
||||
return !s || s.hasTenantId;
|
||||
});
|
||||
expect(bogus, `bogus platform-catalog entries: ${bogus.join(", ")}`).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("every declared referencedBy entry resolves to a tenant-scoped model with the named FK column", () => {
|
||||
const tenantScopedSet = new Set(TENANT_SCOPED_MODELS.map((m) => m.model));
|
||||
const violations: string[] = [];
|
||||
for (const g of GLOBAL_REFERENCED_MODELS) {
|
||||
for (const ref of g.referencedBy) {
|
||||
if (!tenantScopedSet.has(ref.model)) {
|
||||
violations.push(`${g.model}.referencedBy: ${ref.model} is not tenant-scoped`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(violations).toEqual([]);
|
||||
});
|
||||
|
||||
it("isGlobalSnapshotModel + globalSnapshotModelNames agree with GLOBAL_REFERENCED_MODELS", () => {
|
||||
for (const g of GLOBAL_REFERENCED_MODELS) {
|
||||
expect(isGlobalSnapshotModel(g.model)).toBe(true);
|
||||
}
|
||||
expect(isGlobalSnapshotModel("DefinitelyNotARealGlobal")).toBe(false);
|
||||
expect(globalSnapshotModelNames()).toEqual(GLOBAL_REFERENCED_MODELS.map((g) => g.model));
|
||||
});
|
||||
|
||||
it("snapshot-globals and platform-catalog-globals are disjoint", () => {
|
||||
const overlap = GLOBAL_REFERENCED_MODELS.map((g) => g.model).filter((n) => platformNames.has(n));
|
||||
expect(overlap).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
/**
|
||||
* Static enumeration of every Prisma model that is scoped by a `tenantId`
|
||||
* String column. Used by tenant:snapshot/restore to know what to extract.
|
||||
*
|
||||
* Keep this list in sync with `prisma/schema/*.prisma` — a Vitest test
|
||||
* (`models.test.ts`) parses the schema files and asserts coverage so that a
|
||||
* forgotten model fails CI instead of silently leaking out of snapshots.
|
||||
*
|
||||
* Ordering matters for the restore path: parents before children so that the
|
||||
* naive insert-in-list-order strategy produces no foreign-key violations.
|
||||
* The dependency graph below is conservative — re-orderings are safe as long
|
||||
* as a model never references another model that comes later in the list via
|
||||
* a NOT NULL foreign key with onDelete:Restrict.
|
||||
*/
|
||||
export interface TenantScopedModel {
|
||||
/** Prisma model name (PascalCase as written in `schema.prisma`). */
|
||||
readonly model: string;
|
||||
/** The tenant foreign-key field on the model. Always `tenantId` today. */
|
||||
readonly tenantField: "tenantId";
|
||||
/** Schema file the model lives in, for diagnostics. */
|
||||
readonly schema: string;
|
||||
}
|
||||
|
||||
export const TENANT_SCOPED_MODELS: readonly TenantScopedModel[] = [
|
||||
// accounts.prisma
|
||||
{ model: "TenantUser", tenantField: "tenantId", schema: "accounts.prisma" },
|
||||
{ model: "TenantUserInvitation", tenantField: "tenantId", schema: "accounts.prisma" },
|
||||
{ model: "TenantIpAddress", tenantField: "tenantId", schema: "accounts.prisma" },
|
||||
{ model: "WorkspaceAuthProvider", tenantField: "tenantId", schema: "accounts.prisma" },
|
||||
{ model: "ImpersonationAudit", tenantField: "tenantId", schema: "accounts.prisma" },
|
||||
|
||||
// permissions.prisma
|
||||
{ model: "Role", tenantField: "tenantId", schema: "permissions.prisma" },
|
||||
{ model: "Group", tenantField: "tenantId", schema: "permissions.prisma" },
|
||||
{ model: "UserRole", tenantField: "tenantId", schema: "permissions.prisma" },
|
||||
|
||||
// core.prisma
|
||||
{ model: "ApiKey", tenantField: "tenantId", schema: "core.prisma" },
|
||||
{ model: "ApiKeyLog", tenantField: "tenantId", schema: "core.prisma" },
|
||||
{ model: "AuditEvent", tenantField: "tenantId", schema: "core.prisma" },
|
||||
{ model: "Event", tenantField: "tenantId", schema: "core.prisma" },
|
||||
{ model: "Log", tenantField: "tenantId", schema: "core.prisma" },
|
||||
{ model: "MetricLog", tenantField: "tenantId", schema: "core.prisma" },
|
||||
{ model: "Widget", tenantField: "tenantId", schema: "core.prisma" },
|
||||
|
||||
// entityBuilder.prisma
|
||||
{ model: "Property", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
{ model: "EntityTemplate", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
{ model: "EntityView", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
{ model: "EntityGroupConfiguration", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
{ model: "Row", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
{ model: "RowPermission", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
{ model: "TenantSettingsRow", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
{ model: "FormulaLog", tenantField: "tenantId", schema: "entityBuilder.prisma" },
|
||||
|
||||
// subscriptions.prisma
|
||||
{ model: "TenantSubscription", tenantField: "tenantId", schema: "subscriptions.prisma" },
|
||||
{ model: "TenantFeatureOverride", tenantField: "tenantId", schema: "subscriptions.prisma" },
|
||||
{ model: "TenantProductConfiguration", tenantField: "tenantId", schema: "subscriptions.prisma" },
|
||||
{ model: "Credit", tenantField: "tenantId", schema: "subscriptions.prisma" },
|
||||
|
||||
// email.prisma
|
||||
{ model: "EmailSender", tenantField: "tenantId", schema: "email.prisma" },
|
||||
{ model: "TenantInboundAddress", tenantField: "tenantId", schema: "email.prisma" },
|
||||
{ model: "Campaign", tenantField: "tenantId", schema: "email.prisma" },
|
||||
{ model: "OutboundEmail", tenantField: "tenantId", schema: "email.prisma" },
|
||||
|
||||
// workflows.prisma
|
||||
{ model: "Workflow", tenantField: "tenantId", schema: "workflows.prisma" },
|
||||
{ model: "WorkflowExecution", tenantField: "tenantId", schema: "workflows.prisma" },
|
||||
{ model: "WorkflowVariable", tenantField: "tenantId", schema: "workflows.prisma" },
|
||||
{ model: "WorkflowCredential", tenantField: "tenantId", schema: "workflows.prisma" },
|
||||
|
||||
// appsEngine.prisma
|
||||
{ model: "App", tenantField: "tenantId", schema: "appsEngine.prisma" },
|
||||
{ model: "AppSyncRun", tenantField: "tenantId", schema: "appsEngine.prisma" },
|
||||
{ model: "DataConnection", tenantField: "tenantId", schema: "appsEngine.prisma" },
|
||||
|
||||
// featureFlags.prisma
|
||||
{ model: "FeatureFlag", tenantField: "tenantId", schema: "featureFlags.prisma" },
|
||||
|
||||
// helpDesk.prisma
|
||||
{ model: "Feedback", tenantField: "tenantId", schema: "helpDesk.prisma" },
|
||||
{ model: "SupportTicket", tenantField: "tenantId", schema: "helpDesk.prisma" },
|
||||
|
||||
// imports.prisma
|
||||
{ model: "ImportJob", tenantField: "tenantId", schema: "imports.prisma" },
|
||||
|
||||
// queue.prisma
|
||||
{ model: "ExportJob", tenantField: "tenantId", schema: "queue.prisma" },
|
||||
|
||||
// knowledgeBase.prisma
|
||||
{ model: "KnowledgeBase", tenantField: "tenantId", schema: "knowledgeBase.prisma" },
|
||||
|
||||
// notifications.prisma
|
||||
{ model: "NotificationDelivery", tenantField: "tenantId", schema: "notifications.prisma" },
|
||||
{ model: "UserNotificationPreference", tenantField: "tenantId", schema: "notifications.prisma" },
|
||||
|
||||
// onboarding.prisma
|
||||
{ model: "OnboardingSession", tenantField: "tenantId", schema: "onboarding.prisma" },
|
||||
|
||||
// portals.prisma
|
||||
{ model: "Portal", tenantField: "tenantId", schema: "portals.prisma" },
|
||||
{ model: "PortalUser", tenantField: "tenantId", schema: "portals.prisma" },
|
||||
|
||||
// sites.prisma
|
||||
{ model: "Site", tenantField: "tenantId", schema: "sites.prisma" },
|
||||
|
||||
// formWidget.prisma (Phase 05 #58 — embeddable form widget)
|
||||
{ model: "Form", tenantField: "tenantId", schema: "formWidget.prisma" },
|
||||
{ model: "FormSubmission", tenantField: "tenantId", schema: "formWidget.prisma" },
|
||||
|
||||
// outbox.prisma (tenantId is nullable — system-wide events use NULL)
|
||||
{ model: "OutboxEvent", tenantField: "tenantId", schema: "outbox.prisma" },
|
||||
|
||||
// activity.prisma (M00 — Activity-Stream)
|
||||
{ model: "Activity", tenantField: "tenantId", schema: "activity.prisma" },
|
||||
|
||||
// plugins.prisma (#21 + #22)
|
||||
{ model: "Plugin", tenantField: "tenantId", schema: "plugins.prisma" },
|
||||
{ model: "PluginInvocation", tenantField: "tenantId", schema: "plugins.prisma" },
|
||||
{ model: "PluginCapabilityUse", tenantField: "tenantId", schema: "plugins.prisma" },
|
||||
{ model: "TenantPluginCapabilityGrant", tenantField: "tenantId", schema: "plugins.prisma" },
|
||||
|
||||
// collabDocs.prisma
|
||||
{ model: "CollabDoc", tenantField: "tenantId", schema: "collabDocs.prisma" },
|
||||
|
||||
// emailSequences.prisma
|
||||
{ model: "EmailSequence", tenantField: "tenantId", schema: "emailSequences.prisma" },
|
||||
{ model: "EmailSequenceRun", tenantField: "tenantId", schema: "emailSequences.prisma" },
|
||||
|
||||
// marketingLeads.prisma
|
||||
{ model: "TenantAcquisition", tenantField: "tenantId", schema: "marketingLeads.prisma" },
|
||||
|
||||
// notifications.prisma — push tokens
|
||||
{ model: "DeviceToken", tenantField: "tenantId", schema: "notifications.prisma" },
|
||||
];
|
||||
|
||||
/** Constant-time check that a model name is known to be tenant-scoped. */
|
||||
export function isTenantScopedModel(model: string): boolean {
|
||||
return TENANT_SCOPED_MODELS.some((m) => m.model === model);
|
||||
}
|
||||
|
||||
/** Return the list of model names, in restore-safe iteration order. */
|
||||
export function tenantScopedModelNames(): readonly string[] {
|
||||
return TENANT_SCOPED_MODELS.map((m) => m.model);
|
||||
}
|
||||
|
||||
/**
|
||||
* Global (NON-tenant-scoped) Prisma models that are referenced by foreign
|
||||
* keys from at least one tenant-scoped model. Without these in the
|
||||
* snapshot, restoring a tenant into an empty target database fails on
|
||||
* `ForeignKeyConstraintViolation` for every tenant-scoped row that
|
||||
* carries a FK column pointing at a global row.
|
||||
*
|
||||
* Ordering matters for the restore path: parents before children. `User`
|
||||
* has no incoming FK from another global on this list, so it goes first.
|
||||
*
|
||||
* Coverage notes
|
||||
* --------------
|
||||
* `User` is the only global referenced by the six models that the B7
|
||||
* staging verification flagged as broken (`TenantUser`, `TenantIpAddress`,
|
||||
* `UserRole`, `Event`, `Log`, `SupportTicket`). All other globals reached
|
||||
* from tenant-scoped models in the current schema are platform-level
|
||||
* catalogs (`Entity`, `Formula`, `EntityGroup`, `Onboarding`,
|
||||
* `RowComment`, `WorkflowBlock`) — these are expected to
|
||||
* already exist on the target deployment because they are seeded by
|
||||
* migrations/setup scripts, NOT by tenant onboarding. Their absence on
|
||||
* an "empty" target is therefore an operator-environment issue, not a
|
||||
* tenant-snapshot issue, and is intentionally out of scope here.
|
||||
*
|
||||
* The `models.test.ts` companion asserts that every global referenced by
|
||||
* a tenant-scoped FK is either listed here OR explicitly tagged below as
|
||||
* a "platform catalog" so adding a new schema relation forces a
|
||||
* conscious decision instead of silently regressing the round-trip.
|
||||
*/
|
||||
export interface GlobalReferencedModel {
|
||||
/** Prisma model name (PascalCase). */
|
||||
readonly model: string;
|
||||
/** Schema file the model lives in, for diagnostics. */
|
||||
readonly schema: string;
|
||||
/**
|
||||
* Tenant-scoped models known to reference this global. Stored as
|
||||
* documentation + drives the FK-id-collection pass in the snapshot
|
||||
* runner (each entry: source-model, local-FK-column).
|
||||
*/
|
||||
readonly referencedBy: ReadonlyArray<{ readonly model: string; readonly fkColumn: string }>;
|
||||
}
|
||||
|
||||
export const GLOBAL_REFERENCED_MODELS: readonly GlobalReferencedModel[] = [
|
||||
{
|
||||
model: "User",
|
||||
schema: "accounts.prisma",
|
||||
referencedBy: [
|
||||
// (kept in alphabetical order by source model for review-friendliness)
|
||||
{ model: "ApiKey", fkColumn: "createdByUserId" },
|
||||
{ model: "App", fkColumn: "installedByUserId" },
|
||||
{ model: "AppSyncRun", fkColumn: "createdByUserId" },
|
||||
{ model: "Credit", fkColumn: "userId" },
|
||||
{ model: "DeviceToken", fkColumn: "userId" },
|
||||
{ model: "EntityView", fkColumn: "createdByUserId" },
|
||||
{ model: "EntityView", fkColumn: "userId" },
|
||||
{ model: "Event", fkColumn: "userId" },
|
||||
{ model: "Feedback", fkColumn: "userId" },
|
||||
{ model: "FormulaLog", fkColumn: "userId" },
|
||||
{ model: "Group", fkColumn: "createdByUserId" },
|
||||
{ model: "ImpersonationAudit", fkColumn: "actorUserId" },
|
||||
{ model: "ImpersonationAudit", fkColumn: "targetUserId" },
|
||||
{ model: "Log", fkColumn: "userId" },
|
||||
{ model: "MetricLog", fkColumn: "userId" },
|
||||
{ model: "NotificationDelivery", fkColumn: "userId" },
|
||||
{ model: "OnboardingSession", fkColumn: "userId" },
|
||||
{ model: "Portal", fkColumn: "createdByUserId" },
|
||||
{ model: "Row", fkColumn: "createdByUserId" },
|
||||
{ model: "RowPermission", fkColumn: "userId" },
|
||||
{ model: "Site", fkColumn: "createdByUserId" },
|
||||
{ model: "SupportTicket", fkColumn: "userId" },
|
||||
{ model: "TenantIpAddress", fkColumn: "userId" },
|
||||
{ model: "TenantUser", fkColumn: "userId" },
|
||||
{ model: "TenantUserInvitation", fkColumn: "acceptedByUserId" },
|
||||
{ model: "TenantUserInvitation", fkColumn: "createdUserId" },
|
||||
{ model: "TenantUserInvitation", fkColumn: "fromUserId" },
|
||||
{ model: "TenantUserInvitation", fkColumn: "revokedByUserId" },
|
||||
{ model: "UserNotificationPreference", fkColumn: "userId" },
|
||||
{ model: "UserRole", fkColumn: "userId" },
|
||||
{ model: "Workflow", fkColumn: "createdByUserId" },
|
||||
{ model: "WorkflowCredential", fkColumn: "userId" },
|
||||
{ model: "WorkflowExecution", fkColumn: "createdByUserId" },
|
||||
{ model: "WorkflowVariable", fkColumn: "userId" },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Platform-catalog globals: referenced by tenant-scoped models but
|
||||
* explicitly NOT shipped in tenant snapshots because they are part of
|
||||
* the target deployment's seeded baseline. Listed here so the models
|
||||
* test can prove every global reference is consciously categorized.
|
||||
*/
|
||||
export const PLATFORM_CATALOG_GLOBALS: readonly string[] = [
|
||||
"Entity",
|
||||
"EntityGroup",
|
||||
"Formula",
|
||||
"Onboarding",
|
||||
"RowComment",
|
||||
"WorkflowBlock",
|
||||
];
|
||||
|
||||
/** Constant-time check that a model is on the snapshot-globals list. */
|
||||
export function isGlobalSnapshotModel(model: string): boolean {
|
||||
return GLOBAL_REFERENCED_MODELS.some((m) => m.model === model);
|
||||
}
|
||||
|
||||
/** Return the list of global model names, in restore-safe iteration order. */
|
||||
export function globalSnapshotModelNames(): readonly string[] {
|
||||
return GLOBAL_REFERENCED_MODELS.map((m) => m.model);
|
||||
}
|
||||
+238
@@ -0,0 +1,238 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertClientCoversTenantScopedModels,
|
||||
createPrismaRestoreRunner,
|
||||
createPrismaSnapshotRunner,
|
||||
prismaModelAccessor,
|
||||
type PrismaLikeClient,
|
||||
} from "./prisma-runner.js";
|
||||
import { TENANT_SCOPED_MODELS } from "./models.js";
|
||||
import type { ModelScalarTypeMap } from "./json-codec.js";
|
||||
|
||||
describe("prismaModelAccessor", () => {
|
||||
it("lowercases the first character", () => {
|
||||
expect(prismaModelAccessor("TenantUser")).toBe("tenantUser");
|
||||
expect(prismaModelAccessor("Row")).toBe("row");
|
||||
expect(prismaModelAccessor("TenantSettingsRow")).toBe("tenantSettingsRow");
|
||||
});
|
||||
});
|
||||
|
||||
describe("createPrismaSnapshotRunner", () => {
|
||||
it("paginates by id cursor and yields every row exactly once", async () => {
|
||||
const rows = Array.from({ length: 5 }, (_, i) => ({ id: `r${i + 1}`, tenantId: "t1", v: i }));
|
||||
const calls: Array<Record<string, unknown>> = [];
|
||||
const client: PrismaLikeClient = {
|
||||
tenantUser: {
|
||||
findMany: async (args: Record<string, unknown>) => {
|
||||
calls.push(args);
|
||||
const take = args.take as number;
|
||||
const skip = (args.skip as number | undefined) ?? 0;
|
||||
const cursor = args.cursor as { id?: string } | undefined;
|
||||
let startIdx = 0;
|
||||
if (cursor?.id) {
|
||||
startIdx = rows.findIndex((r) => r.id === cursor.id) + skip;
|
||||
}
|
||||
return rows.slice(startIdx, startIdx + take);
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createPrismaSnapshotRunner(client);
|
||||
const out: Array<Record<string, unknown>> = [];
|
||||
for await (const r of runner.findManyByTenant("TenantUser", "tenantId", "t1", 2)) {
|
||||
out.push(r);
|
||||
}
|
||||
expect(out.map((r) => r.id)).toEqual(["r1", "r2", "r3", "r4", "r5"]);
|
||||
// First page: no cursor; subsequent pages must include cursor + skip=1.
|
||||
expect(calls[0]?.cursor).toBeUndefined();
|
||||
expect(calls[1]?.cursor).toEqual({ id: "r2" });
|
||||
expect(calls[1]?.skip).toBe(1);
|
||||
});
|
||||
|
||||
it("uses tenantId as the cursor for TenantSettingsRow", async () => {
|
||||
const calls: Array<Record<string, unknown>> = [];
|
||||
const client: PrismaLikeClient = {
|
||||
tenantSettingsRow: {
|
||||
findMany: async (args: Record<string, unknown>) => {
|
||||
calls.push(args);
|
||||
if ((args.cursor as { tenantId?: string } | undefined)?.tenantId === "t1") return [];
|
||||
return [{ tenantId: "t1", foo: "bar" }];
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createPrismaSnapshotRunner(client);
|
||||
const out: Array<Record<string, unknown>> = [];
|
||||
for await (const r of runner.findManyByTenant("TenantSettingsRow", "tenantId", "t1", 100)) {
|
||||
out.push(r);
|
||||
}
|
||||
expect(out).toEqual([{ tenantId: "t1", foo: "bar" }]);
|
||||
expect(calls[0]?.orderBy).toEqual({ tenantId: "asc" });
|
||||
});
|
||||
|
||||
it("throws when the client has no delegate for the model", async () => {
|
||||
const client: PrismaLikeClient = {};
|
||||
const runner = createPrismaSnapshotRunner(client);
|
||||
const iter = runner.findManyByTenant("TenantUser", "tenantId", "t1", 100);
|
||||
await expect((async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
for await (const _ of iter) {
|
||||
/* unreachable */
|
||||
}
|
||||
})()).rejects.toThrow(/no delegate for model "TenantUser"/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createPrismaRestoreRunner", () => {
|
||||
it("calls createMany with skipDuplicates and returns the written count", async () => {
|
||||
let received: Record<string, unknown> | undefined;
|
||||
const client: PrismaLikeClient = {
|
||||
tenantUser: {
|
||||
createMany: async (args: Record<string, unknown>) => {
|
||||
received = args;
|
||||
return { count: (args.data as unknown[]).length };
|
||||
},
|
||||
},
|
||||
};
|
||||
const types = new Map<string, ModelScalarTypeMap>();
|
||||
const runner = createPrismaRestoreRunner(client, types);
|
||||
const result = await runner.createMany("TenantUser", [
|
||||
{ id: "tu1", tenantId: "t2", userId: "u1" },
|
||||
{ id: "tu2", tenantId: "t2", userId: "u2" },
|
||||
]);
|
||||
expect(result.written).toBe(2);
|
||||
expect(received?.skipDuplicates).toBe(true);
|
||||
expect((received?.data as unknown[]).length).toBe(2);
|
||||
});
|
||||
|
||||
it("decodes coded scalar fields before passing rows to Prisma", async () => {
|
||||
let received: Array<Record<string, unknown>> | undefined;
|
||||
const client: PrismaLikeClient = {
|
||||
collabDoc: {
|
||||
createMany: async (args: Record<string, unknown>) => {
|
||||
received = args.data as Array<Record<string, unknown>>;
|
||||
return { count: received.length };
|
||||
},
|
||||
},
|
||||
};
|
||||
const types = new Map<string, ModelScalarTypeMap>([
|
||||
[
|
||||
"CollabDoc",
|
||||
{
|
||||
modelName: "CollabDoc",
|
||||
fields: new Map([
|
||||
["createdAt", "DateTime"],
|
||||
["ydocState", "Bytes"],
|
||||
]),
|
||||
},
|
||||
],
|
||||
]);
|
||||
const runner = createPrismaRestoreRunner(client, types);
|
||||
const base64 = Buffer.from("hi").toString("base64");
|
||||
await runner.createMany("CollabDoc", [
|
||||
{ id: "c1", tenantId: "t1", createdAt: "2026-05-24T12:00:00.000Z", ydocState: base64 },
|
||||
]);
|
||||
const row = received![0]!;
|
||||
expect(row.createdAt).toBeInstanceOf(Date);
|
||||
expect(Buffer.isBuffer(row.ydocState)).toBe(true);
|
||||
expect((row.ydocState as Buffer).toString()).toBe("hi");
|
||||
});
|
||||
|
||||
it("findManyByIds delegates to findMany with where.id.in", async () => {
|
||||
let received: Record<string, unknown> | undefined;
|
||||
const client: PrismaLikeClient = {
|
||||
user: {
|
||||
findMany: async (args: Record<string, unknown>) => {
|
||||
received = args;
|
||||
return [{ id: "u1" }, { id: "u2" }];
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createPrismaSnapshotRunner(client);
|
||||
const rows = await runner.findManyByIds!("User", ["u1", "u2"]);
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(received).toEqual({ where: { id: { in: ["u1", "u2"] } } });
|
||||
});
|
||||
|
||||
it("findManyByIds returns [] without calling Prisma for an empty id list", async () => {
|
||||
let invoked = false;
|
||||
const client: PrismaLikeClient = {
|
||||
user: {
|
||||
findMany: async () => {
|
||||
invoked = true;
|
||||
return [];
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createPrismaSnapshotRunner(client);
|
||||
const rows = await runner.findManyByIds!("User", []);
|
||||
expect(rows).toEqual([]);
|
||||
expect(invoked).toBe(false);
|
||||
});
|
||||
|
||||
it("upsertManyById creates fresh rows and reports updated=N for pre-existing ids", async () => {
|
||||
const existing = new Set(["u_exists"]);
|
||||
const createdRows: Array<Record<string, unknown>> = [];
|
||||
const client: PrismaLikeClient = {
|
||||
user: {
|
||||
findUnique: async (args: { where: { id: string } }) => {
|
||||
return existing.has(args.where.id) ? { id: args.where.id } : null;
|
||||
},
|
||||
create: async (args: { data: Record<string, unknown> }) => {
|
||||
createdRows.push(args.data);
|
||||
return args.data;
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createPrismaRestoreRunner(client, new Map());
|
||||
const result = await runner.upsertManyById!("User", [
|
||||
{ id: "u_exists", email: "old@x" },
|
||||
{ id: "u_new", email: "new@x" },
|
||||
]);
|
||||
expect(result).toEqual({ created: 1, updated: 1 });
|
||||
expect(createdRows).toHaveLength(1);
|
||||
expect(createdRows[0]?.id).toBe("u_new");
|
||||
});
|
||||
|
||||
it("upsertManyById throws when a row has no string id", async () => {
|
||||
const client: PrismaLikeClient = {
|
||||
user: {
|
||||
findUnique: async () => null,
|
||||
create: async (args: { data: Record<string, unknown> }) => args.data,
|
||||
},
|
||||
};
|
||||
const runner = createPrismaRestoreRunner(client, new Map());
|
||||
await expect(runner.upsertManyById!("User", [{ email: "x@y" }])).rejects.toThrow(
|
||||
/has no string id/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("returns { written: 0 } for an empty row list without calling Prisma", async () => {
|
||||
let invoked = false;
|
||||
const client: PrismaLikeClient = {
|
||||
tenantUser: {
|
||||
createMany: async () => {
|
||||
invoked = true;
|
||||
return { count: 99 };
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createPrismaRestoreRunner(client, new Map());
|
||||
const result = await runner.createMany("TenantUser", []);
|
||||
expect(result.written).toBe(0);
|
||||
expect(invoked).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("assertClientCoversTenantScopedModels", () => {
|
||||
it("passes for a client exposing every accessor", () => {
|
||||
const client: PrismaLikeClient = {};
|
||||
for (const { model } of TENANT_SCOPED_MODELS) {
|
||||
client[prismaModelAccessor(model)] = { findMany: () => Promise.resolve([]) };
|
||||
}
|
||||
expect(() => assertClientCoversTenantScopedModels(client)).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws listing the missing models when accessors are absent", () => {
|
||||
const client: PrismaLikeClient = {};
|
||||
expect(() => assertClientCoversTenantScopedModels(client)).toThrow(/missing accessors/u);
|
||||
});
|
||||
});
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* Bridge between the generic, dependency-injected `PrismaRunner` /
|
||||
* `PrismaRestoreRunner` interfaces and a real `@toolctrl/db` PrismaClient.
|
||||
*
|
||||
* Kept separate from `snapshot.ts` / `restore.ts` so those modules stay
|
||||
* Prisma-free (and trivially unit-testable with plain fakes). The CLI bin
|
||||
* wiring imports from here lazily inside the citty `run()` body so the
|
||||
* command tree loads without pulling Prisma into memory.
|
||||
*/
|
||||
import type { PrismaRunner } from "./snapshot.js";
|
||||
import type { PrismaRestoreRunner } from "./restore.js";
|
||||
import { TENANT_SCOPED_MODELS } from "./models.js";
|
||||
import { decodeRowForPrisma, type ModelScalarTypeMap } from "./json-codec.js";
|
||||
|
||||
/** Minimal shape of `PrismaClient` we lean on — anything `[modelAccessor]` shaped. */
|
||||
export interface PrismaLikeClient {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prisma model names are PascalCase; the runtime client exposes them as
|
||||
* camelCase accessors (`Prisma.tenantUser`). The one exception in our
|
||||
* schema today is `Row` → `prisma.row`. There is no other casing quirk
|
||||
* to handle.
|
||||
*/
|
||||
export function prismaModelAccessor(modelName: string): string {
|
||||
return modelName.charAt(0).toLowerCase() + modelName.slice(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* A small fixup map for models whose primary-key column is not literally
|
||||
* `id`. Tenant-scoped today: only `TenantSettingsRow` (PK == `tenantId`).
|
||||
* Used to drive cursor pagination correctly.
|
||||
*/
|
||||
const CURSOR_FIELD_OVERRIDES: Record<string, string> = {
|
||||
TenantSettingsRow: "tenantId",
|
||||
};
|
||||
|
||||
function cursorFieldFor(modelName: string): string {
|
||||
return CURSOR_FIELD_OVERRIDES[modelName] ?? "id";
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a snapshot-side `PrismaRunner` that streams rows out of the
|
||||
* supplied client using cursor pagination keyed by primary key.
|
||||
*
|
||||
* Single-row models (`TenantSettingsRow`) still walk the same cursor
|
||||
* code path because their PK column is included in the rows themselves.
|
||||
*/
|
||||
export function createPrismaSnapshotRunner(prisma: PrismaLikeClient): PrismaRunner {
|
||||
return {
|
||||
async findManyByIds(modelName, ids) {
|
||||
if (ids.length === 0) return [];
|
||||
const accessor = prismaModelAccessor(modelName);
|
||||
const delegate = prisma[accessor] as
|
||||
| undefined
|
||||
| {
|
||||
findMany: (args: Record<string, unknown>) => Promise<Array<Record<string, unknown>>>;
|
||||
};
|
||||
if (!delegate || typeof delegate.findMany !== "function") {
|
||||
throw new Error(
|
||||
`PrismaRunner.findManyByIds: client has no delegate for model "${modelName}" (accessor "${accessor}").`,
|
||||
);
|
||||
}
|
||||
return delegate.findMany({ where: { id: { in: Array.from(ids) } } });
|
||||
},
|
||||
async *findManyByTenant(modelName, tenantField, tenantId, pageSize) {
|
||||
const accessor = prismaModelAccessor(modelName);
|
||||
const delegate = prisma[accessor] as
|
||||
| undefined
|
||||
| {
|
||||
findMany: (args: Record<string, unknown>) => Promise<Array<Record<string, unknown>>>;
|
||||
};
|
||||
if (!delegate || typeof delegate.findMany !== "function") {
|
||||
throw new Error(
|
||||
`PrismaRunner: client has no delegate for model "${modelName}" (accessor "${accessor}").`,
|
||||
);
|
||||
}
|
||||
const cursorField = cursorFieldFor(modelName);
|
||||
let cursor: Record<string, unknown> | undefined;
|
||||
// Belt-and-suspenders: cap the outer loop so a misbehaving page
|
||||
// (always returning the same `pageSize` rows with the same last id)
|
||||
// cannot wedge the snapshot. 10 M rows / pageSize is a sane ceiling.
|
||||
const maxIterations = Math.max(1, Math.ceil(10_000_000 / pageSize));
|
||||
let iterations = 0;
|
||||
while (iterations++ < maxIterations) {
|
||||
const args: Record<string, unknown> = {
|
||||
where: { [tenantField]: tenantId },
|
||||
take: pageSize,
|
||||
orderBy: { [cursorField]: "asc" },
|
||||
};
|
||||
if (cursor !== undefined) {
|
||||
args.cursor = cursor;
|
||||
args.skip = 1;
|
||||
}
|
||||
const page = await delegate.findMany(args);
|
||||
if (page.length === 0) break;
|
||||
for (const row of page) yield row;
|
||||
if (page.length < pageSize) break;
|
||||
const last = page[page.length - 1] as Record<string, unknown>;
|
||||
const lastCursor = last[cursorField];
|
||||
if (lastCursor === undefined || lastCursor === null) {
|
||||
throw new Error(
|
||||
`PrismaRunner: cursor field "${cursorField}" missing on a row of ${modelName}; cannot paginate.`,
|
||||
);
|
||||
}
|
||||
cursor = { [cursorField]: lastCursor };
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a restore-side `PrismaRestoreRunner` that decodes rows back into
|
||||
* native Prisma types and writes them with `createMany({ skipDuplicates }).
|
||||
*
|
||||
* `schemaScalarTypes` comes from `parseSchemaScalarTypes(schemaDir)` and
|
||||
* tells us per-model which fields need DateTime/Bytes/BigInt/Decimal
|
||||
* coercion. Models absent from the map are written as-is.
|
||||
*/
|
||||
export function createPrismaRestoreRunner(
|
||||
prisma: PrismaLikeClient,
|
||||
schemaScalarTypes: ReadonlyMap<string, ModelScalarTypeMap>,
|
||||
): PrismaRestoreRunner {
|
||||
return {
|
||||
async createMany(modelName, rows) {
|
||||
if (rows.length === 0) return { written: 0 };
|
||||
const accessor = prismaModelAccessor(modelName);
|
||||
const delegate = prisma[accessor] as
|
||||
| undefined
|
||||
| {
|
||||
createMany: (args: { data: Record<string, unknown>[]; skipDuplicates?: boolean }) => Promise<{ count: number }>;
|
||||
};
|
||||
if (!delegate || typeof delegate.createMany !== "function") {
|
||||
throw new Error(
|
||||
`PrismaRestoreRunner: client has no delegate for model "${modelName}" (accessor "${accessor}").`,
|
||||
);
|
||||
}
|
||||
const fieldTypes = schemaScalarTypes.get(modelName)?.fields;
|
||||
const decoded = rows.map((row) => decodeRowForPrisma(row, fieldTypes));
|
||||
const result = await delegate.createMany({ data: decoded, skipDuplicates: true });
|
||||
return { written: result?.count ?? decoded.length };
|
||||
},
|
||||
async upsertManyById(modelName, rows) {
|
||||
// Per-row upsert keyed by `id`. We deliberately don't issue this
|
||||
// as a batch (Prisma has no batch upsert) — globals are small
|
||||
// (referenced-User rows for the tenant slice; rarely > 1000).
|
||||
// `update: {}` means "if already present, do not mutate" so we
|
||||
// never silently overwrite existing globals; the existence is
|
||||
// surfaced to the caller via `updated`.
|
||||
if (rows.length === 0) return { created: 0, updated: 0 };
|
||||
const accessor = prismaModelAccessor(modelName);
|
||||
const delegate = prisma[accessor] as
|
||||
| undefined
|
||||
| {
|
||||
findUnique: (args: { where: { id: string } }) => Promise<unknown>;
|
||||
create: (args: { data: Record<string, unknown> }) => Promise<unknown>;
|
||||
};
|
||||
if (
|
||||
!delegate ||
|
||||
typeof delegate.findUnique !== "function" ||
|
||||
typeof delegate.create !== "function"
|
||||
) {
|
||||
throw new Error(
|
||||
`PrismaRestoreRunner.upsertManyById: client has no findUnique/create for model "${modelName}" (accessor "${accessor}").`,
|
||||
);
|
||||
}
|
||||
const fieldTypes = schemaScalarTypes.get(modelName)?.fields;
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
for (const row of rows) {
|
||||
const id = row["id"];
|
||||
if (typeof id !== "string" || id.length === 0) {
|
||||
throw new Error(
|
||||
`PrismaRestoreRunner.upsertManyById: row in ${modelName} has no string id; cannot upsert by id.`,
|
||||
);
|
||||
}
|
||||
const existing = await delegate.findUnique({ where: { id } });
|
||||
if (existing) {
|
||||
updated += 1;
|
||||
continue;
|
||||
}
|
||||
const decoded = decodeRowForPrisma(row, fieldTypes);
|
||||
await delegate.create({ data: decoded });
|
||||
created += 1;
|
||||
}
|
||||
return { created, updated };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Defensive cross-check: every tenant-scoped model declared in `models.ts`
|
||||
* must resolve to an accessor on the supplied client. Surfaces a clear
|
||||
* error before a snapshot or restore tries to stream from a missing
|
||||
* delegate. Pure helper — not invoked by the runners themselves.
|
||||
*/
|
||||
export function assertClientCoversTenantScopedModels(prisma: PrismaLikeClient): void {
|
||||
const missing: string[] = [];
|
||||
for (const { model } of TENANT_SCOPED_MODELS) {
|
||||
if (prisma[prismaModelAccessor(model)] === undefined) {
|
||||
missing.push(model);
|
||||
}
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
throw new Error(
|
||||
`PrismaClient is missing accessors for tenant-scoped models: ${missing.join(", ")}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
+467
@@ -0,0 +1,467 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { Readable } from "node:stream";
|
||||
import { mockClient } from "aws-sdk-client-mock";
|
||||
import { GetObjectCommand, ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import { runTenantSnapshot, type PrismaRunner } from "./snapshot.js";
|
||||
import {
|
||||
isGlobalsRestoreMode,
|
||||
runTenantRestore,
|
||||
type PrismaRestoreRunner,
|
||||
} from "./restore.js";
|
||||
|
||||
class FakeSourcePrisma implements PrismaRunner {
|
||||
constructor(
|
||||
private readonly rows: Record<string, Record<string, unknown>[]>,
|
||||
private readonly globals: Record<string, Record<string, unknown>[]> = {},
|
||||
) {}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async *findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>> {
|
||||
void tenantField;
|
||||
void tenantId;
|
||||
void pageSize;
|
||||
const list = this.rows[modelName] ?? [];
|
||||
for (const r of list) yield r;
|
||||
}
|
||||
async findManyByIds(modelName: string, ids: readonly string[]): Promise<Record<string, unknown>[]> {
|
||||
const list = this.globals[modelName] ?? [];
|
||||
return list.filter((r) => typeof r.id === "string" && ids.includes(r.id as string));
|
||||
}
|
||||
}
|
||||
|
||||
class FakeTargetPrisma implements PrismaRestoreRunner {
|
||||
readonly writes: Record<string, Record<string, unknown>[]> = {};
|
||||
/** Rows that already exist on the target — used by upsertManyById to simulate prior state. */
|
||||
readonly preExisting: Record<string, Set<string>> = {};
|
||||
async createMany(modelName: string, rows: Record<string, unknown>[]): Promise<{ written: number }> {
|
||||
this.writes[modelName] = (this.writes[modelName] ?? []).concat(rows);
|
||||
return { written: rows.length };
|
||||
}
|
||||
async upsertManyById(
|
||||
modelName: string,
|
||||
rows: Record<string, unknown>[],
|
||||
): Promise<{ created: number; updated: number }> {
|
||||
const existing = this.preExisting[modelName] ?? new Set<string>();
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
for (const row of rows) {
|
||||
const id = row.id as string;
|
||||
if (existing.has(id)) {
|
||||
updated += 1;
|
||||
continue;
|
||||
}
|
||||
this.writes[modelName] = (this.writes[modelName] ?? []).concat(row);
|
||||
created += 1;
|
||||
}
|
||||
return { created, updated };
|
||||
}
|
||||
}
|
||||
|
||||
/** Variant without upsertManyById — exercises the legacy-runner fallback. */
|
||||
class LegacyTargetPrisma implements PrismaRestoreRunner {
|
||||
readonly writes: Record<string, Record<string, unknown>[]> = {};
|
||||
async createMany(modelName: string, rows: Record<string, unknown>[]): Promise<{ written: number }> {
|
||||
this.writes[modelName] = (this.writes[modelName] ?? []).concat(rows);
|
||||
return { written: rows.length };
|
||||
}
|
||||
}
|
||||
|
||||
class PartialWriteTargetPrisma extends FakeTargetPrisma {
|
||||
constructor(private readonly partialModel: string) {
|
||||
super();
|
||||
}
|
||||
|
||||
override async createMany(modelName: string, rows: Record<string, unknown>[]): Promise<{ written: number }> {
|
||||
this.writes[modelName] = (this.writes[modelName] ?? []).concat(rows);
|
||||
return { written: modelName === this.partialModel ? Math.max(0, rows.length - 1) : rows.length };
|
||||
}
|
||||
}
|
||||
|
||||
describe("runTenantRestore", () => {
|
||||
let workdir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), "tenant-restore-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("recognizes supported globals restore modes", () => {
|
||||
expect(isGlobalsRestoreMode("upsert")).toBe(true);
|
||||
expect(isGlobalsRestoreMode("skip")).toBe(true);
|
||||
expect(isGlobalsRestoreMode("error")).toBe(true);
|
||||
expect(isGlobalsRestoreMode("merge")).toBe(false);
|
||||
expect(isGlobalsRestoreMode("")).toBe(false);
|
||||
});
|
||||
|
||||
it("round-trips a snapshot through restore with FK-remapped tenantId", async () => {
|
||||
// 1. Snapshot from FakeSourcePrisma.
|
||||
const sourcePrisma = new FakeSourcePrisma({
|
||||
TenantUser: [
|
||||
{ id: "tu1", tenantId: "tenant_a", userId: "u1", type: 1, joined: 1, status: 1 },
|
||||
{ id: "tu2", tenantId: "tenant_a", userId: "u2", type: 1, joined: 1, status: 1 },
|
||||
],
|
||||
TenantUserInvitation: [
|
||||
{ id: "inv1", tenantId: "tenant_a", email: "x@y.io", firstName: "X", lastName: "Y" },
|
||||
],
|
||||
});
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await runTenantSnapshot({
|
||||
tenantId: "tenant_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: sourcePrisma,
|
||||
output: { kind: "local", path: snapshotPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
// 2. Restore into FakeTargetPrisma.
|
||||
const target = new FakeTargetPrisma();
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: target,
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.manifest.sourceTenantId).toBe("tenant_a");
|
||||
const tuEntry = result.perModel.find((m) => m.model === "TenantUser");
|
||||
expect(tuEntry).toEqual({ model: "TenantUser", written: 2, expected: 2 });
|
||||
// All rows should have the new tenantId.
|
||||
expect(target.writes.TenantUser).toHaveLength(2);
|
||||
for (const row of target.writes.TenantUser) {
|
||||
expect(row.tenantId).toBe("tenant_a_clone");
|
||||
}
|
||||
// Anonymization preserved on the way in.
|
||||
expect(target.writes.TenantUserInvitation?.[0]?.email).toMatch(/@anonymized\.test$/u);
|
||||
});
|
||||
|
||||
it("aborts when the operator answers 'no' to the confirmation prompt", async () => {
|
||||
const sourcePrisma = new FakeSourcePrisma({
|
||||
TenantUser: [{ id: "tu1", tenantId: "tenant_a", userId: "u1", type: 1, joined: 1, status: 1 }],
|
||||
});
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await runTenantSnapshot({
|
||||
tenantId: "tenant_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: sourcePrisma,
|
||||
output: { kind: "local", path: snapshotPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
const target = new FakeTargetPrisma();
|
||||
const { Readable } = await import("node:stream");
|
||||
const { Writable } = await import("node:stream");
|
||||
const stdin = Readable.from(["n\n"]);
|
||||
const stdout = new Writable({ write: (_c, _e, cb) => cb() });
|
||||
await expect(
|
||||
runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: target,
|
||||
yes: false,
|
||||
stdin: stdin as NodeJS.ReadableStream,
|
||||
stdout,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/Aborted/u);
|
||||
});
|
||||
|
||||
it("refuses to run in production without --allow-production-restore", async () => {
|
||||
const prev = process.env.NODE_ENV;
|
||||
process.env.NODE_ENV = "production";
|
||||
try {
|
||||
await expect(
|
||||
runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: "/nonexistent.tar.gz",
|
||||
prisma: new FakeTargetPrisma(),
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/production/u);
|
||||
} finally {
|
||||
process.env.NODE_ENV = prev;
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects malformed s3 sources before attempting a restore", async () => {
|
||||
await expect(
|
||||
runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: "s3://bucket-only",
|
||||
prisma: new FakeTargetPrisma(),
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/Invalid --source/u);
|
||||
});
|
||||
|
||||
it("restores storage objects under the target tenant prefix", async () => {
|
||||
const sourcePrisma = new FakeSourcePrisma({
|
||||
TenantUser: [{ id: "tu1", tenantId: "tenant_a", userId: "u1", type: 1, joined: 1, status: 1 }],
|
||||
});
|
||||
const sourceS3 = mockClient(S3Client as never);
|
||||
sourceS3.on(ListObjectsV2Command as never).callsFake(() => ({
|
||||
Contents: [{ Key: "tenants/tenant_a/uploads/file.txt", Size: 11, ETag: '"etag"' }],
|
||||
IsTruncated: false,
|
||||
}));
|
||||
sourceS3.on(GetObjectCommand as never).callsFake(() => ({
|
||||
Body: Readable.from([Buffer.from("hello world")]),
|
||||
}));
|
||||
|
||||
const snapshotPath = join(workdir, "snap-with-storage.tar.gz");
|
||||
await runTenantSnapshot({
|
||||
tenantId: "tenant_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: sourcePrisma,
|
||||
output: { kind: "local", path: snapshotPath },
|
||||
s3Client: sourceS3 as unknown as S3Client,
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
const uploadedKeys: string[] = [];
|
||||
const origDone = Upload.prototype.done;
|
||||
Upload.prototype.done = vi.fn(function (this: Upload) {
|
||||
const params = this["params"] as { Key?: string };
|
||||
if (params.Key) uploadedKeys.push(params.Key);
|
||||
return Promise.resolve({ ETag: '"uploaded"' });
|
||||
}) as never;
|
||||
try {
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_b",
|
||||
source: snapshotPath,
|
||||
prisma: new FakeTargetPrisma(),
|
||||
storageBucket: "target-storage",
|
||||
s3Client: new S3Client({ region: "us-east-1" }),
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.storageObjectsUploaded).toBe(1);
|
||||
expect(uploadedKeys).toEqual(["tenants/tenant_b/uploads/file.txt"]);
|
||||
} finally {
|
||||
Upload.prototype.done = origDone;
|
||||
}
|
||||
});
|
||||
|
||||
it("warns when tenant-scoped model writes do not match manifest counts", async () => {
|
||||
const sourcePrisma = new FakeSourcePrisma({
|
||||
TenantUser: [
|
||||
{ id: "tu1", tenantId: "tenant_a", userId: "u1", type: 1, joined: 1, status: 1 },
|
||||
{ id: "tu2", tenantId: "tenant_a", userId: "u2", type: 1, joined: 1, status: 1 },
|
||||
],
|
||||
});
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await runTenantSnapshot({
|
||||
tenantId: "tenant_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: sourcePrisma,
|
||||
output: { kind: "local", path: snapshotPath },
|
||||
includeGlobals: false,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
const warn = vi.fn();
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: new PartialWriteTargetPrisma("TenantUser"),
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn },
|
||||
});
|
||||
|
||||
expect(result.perModel.find((m) => m.model === "TenantUser")).toEqual({
|
||||
model: "TenantUser",
|
||||
written: 1,
|
||||
expected: 2,
|
||||
});
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("Model TenantUser: wrote 1 rows"));
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("Row-count mismatch"));
|
||||
});
|
||||
|
||||
describe("globals replay", () => {
|
||||
async function snapshotWithUsers(snapshotPath: string): Promise<void> {
|
||||
const sourcePrisma = new FakeSourcePrisma(
|
||||
{
|
||||
TenantUser: [
|
||||
{ id: "tu1", tenantId: "tenant_a", userId: "u1", type: 1, joined: 1, status: 1 },
|
||||
{ id: "tu2", tenantId: "tenant_a", userId: "u2", type: 1, joined: 1, status: 1 },
|
||||
],
|
||||
},
|
||||
{
|
||||
User: [
|
||||
{ id: "u1", email: "first@real.com", firstName: "First", lastName: "Real" },
|
||||
{ id: "u2", email: "second@real.com", firstName: "Second", lastName: "Real" },
|
||||
],
|
||||
},
|
||||
);
|
||||
await runTenantSnapshot({
|
||||
tenantId: "tenant_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: sourcePrisma,
|
||||
output: { kind: "local", path: snapshotPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
}
|
||||
|
||||
it("replays globals BEFORE tenant-scoped models (default upsert mode)", async () => {
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await snapshotWithUsers(snapshotPath);
|
||||
const target = new FakeTargetPrisma();
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: target,
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
const userGlobal = result.globals.find((g) => g.model === "User");
|
||||
expect(userGlobal).toEqual({ model: "User", expected: 2, created: 2, existed: 0 });
|
||||
// Users landed in target.
|
||||
expect(target.writes.User).toHaveLength(2);
|
||||
// And TenantUser rows landed under the new tenantId.
|
||||
expect(target.writes.TenantUser).toHaveLength(2);
|
||||
for (const row of target.writes.TenantUser) {
|
||||
expect(row.tenantId).toBe("tenant_a_clone");
|
||||
}
|
||||
});
|
||||
|
||||
it("upsert mode reports existed=N when globals already exist on target", async () => {
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await snapshotWithUsers(snapshotPath);
|
||||
const target = new FakeTargetPrisma();
|
||||
target.preExisting.User = new Set(["u1"]); // one of two users already present
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: target,
|
||||
yes: true,
|
||||
globalsMode: "upsert",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.globals.find((g) => g.model === "User")).toEqual({
|
||||
model: "User",
|
||||
expected: 2,
|
||||
created: 1,
|
||||
existed: 1,
|
||||
});
|
||||
// Only the new user landed; the existing u1 was left in place.
|
||||
expect(target.writes.User).toHaveLength(1);
|
||||
expect((target.writes.User[0] as { id: string }).id).toBe("u2");
|
||||
});
|
||||
|
||||
it("error mode aborts when a global already exists on target", async () => {
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await snapshotWithUsers(snapshotPath);
|
||||
const target = new FakeTargetPrisma();
|
||||
target.preExisting.User = new Set(["u1"]);
|
||||
await expect(
|
||||
runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: target,
|
||||
yes: true,
|
||||
globalsMode: "error",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/--globals=error/u);
|
||||
});
|
||||
|
||||
it("skip mode falls back to createMany semantics for legacy runners", async () => {
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await snapshotWithUsers(snapshotPath);
|
||||
const target = new LegacyTargetPrisma();
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: target,
|
||||
yes: true,
|
||||
globalsMode: "skip",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.globals.find((g) => g.model === "User")?.created).toBe(2);
|
||||
expect(target.writes.User).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("default upsert mode warns and falls back for legacy runners", async () => {
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await snapshotWithUsers(snapshotPath);
|
||||
const target = new LegacyTargetPrisma();
|
||||
const warn = vi.fn();
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: target,
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn },
|
||||
});
|
||||
|
||||
expect(result.globals.find((g) => g.model === "User")).toEqual({
|
||||
model: "User",
|
||||
expected: 2,
|
||||
created: 2,
|
||||
existed: 0,
|
||||
});
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("does not implement upsertManyById"));
|
||||
});
|
||||
|
||||
it("skip mode reports existing globals from partial createMany writes", async () => {
|
||||
const snapshotPath = join(workdir, "snap.tar.gz");
|
||||
await snapshotWithUsers(snapshotPath);
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: new PartialWriteTargetPrisma("User"),
|
||||
yes: true,
|
||||
globalsMode: "skip",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.globals.find((g) => g.model === "User")).toEqual({
|
||||
model: "User",
|
||||
expected: 2,
|
||||
created: 1,
|
||||
existed: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("legacy snapshot (no globals slice) is tolerated with a warn line", async () => {
|
||||
// Build a snapshot using a runner without findManyByIds — manifest will
|
||||
// have no `globals` array (matches pre-feature snapshots).
|
||||
const legacySource: PrismaRunner = {
|
||||
async *findManyByTenant() {
|
||||
// empty
|
||||
},
|
||||
};
|
||||
const snapshotPath = join(workdir, "legacy.tar.gz");
|
||||
await runTenantSnapshot({
|
||||
tenantId: "tenant_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: legacySource,
|
||||
output: { kind: "local", path: snapshotPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
const warn = vi.fn();
|
||||
const result = await runTenantRestore({
|
||||
tenantId: "tenant_a_clone",
|
||||
source: snapshotPath,
|
||||
prisma: new FakeTargetPrisma(),
|
||||
yes: true,
|
||||
logger: { info: vi.fn(), warn },
|
||||
});
|
||||
expect(result.globals).toEqual([]);
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("no globals slice"));
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,490 @@
|
||||
/**
|
||||
* `tenant:restore` — read a snapshot tarball (local or S3), decrypt if needed,
|
||||
* extract the manifest + per-model JSONL, replay rows into the target Prisma
|
||||
* with the new tenant id, then upload tenant storage objects under the new
|
||||
* `tenants/<new-id>/` prefix.
|
||||
*
|
||||
* The Prisma layer is dependency-injected via the `PrismaRestoreRunner`
|
||||
* interface so the CLI package stays free of `@prisma/client`. Production
|
||||
* callers wire up a real client in `bin/toolctrl-cli.mjs`; tests pass a fake.
|
||||
*
|
||||
* Foreign-key remapping: only the `tenantId` column is rewritten. Row primary
|
||||
* keys are preserved (the snapshot is intended to land in a separate
|
||||
* environment, so id collisions with other tenants in that target are not
|
||||
* the concern this commit addresses; the schema's tenant-scoped uniqueness
|
||||
* indexes do the rest). The roadmap notes (`tenant-snapshot.md` step 6) call
|
||||
* out a deeper FK-aware remapper as a follow-up if cross-tenant FK chains
|
||||
* ever appear (today none of the tenant-scoped models cross-reference each
|
||||
* other in a way that survives a snapshot — they all root at `tenantId`).
|
||||
*/
|
||||
import { defineCommand } from "citty";
|
||||
import consola from "consola";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { mkdir, mkdtemp, readFile, rm, stat } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { Readable } from "node:stream";
|
||||
import { extract as extractTar } from "tar";
|
||||
import {
|
||||
GetObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
S3Client,
|
||||
type S3ClientConfig,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import {
|
||||
deserializeTenantManifest,
|
||||
type TenantSnapshotManifest,
|
||||
} from "./manifest.js";
|
||||
import { decryptFile, type EncryptionConfig } from "../backup/encryption.js";
|
||||
|
||||
export interface PrismaRestoreRunner {
|
||||
/**
|
||||
* Bulk-create rows into `modelName`, returning the number of rows actually
|
||||
* persisted (after skipping duplicates the caller chose to ignore).
|
||||
*/
|
||||
createMany(modelName: string, rows: Record<string, unknown>[]): Promise<{ written: number }>;
|
||||
/**
|
||||
* Upsert rows into `modelName` keyed by `id`. Returns counts split by
|
||||
* whether each row was created fresh or matched an existing row.
|
||||
* Optional so legacy fake runners only used for the tenant-only path
|
||||
* keep working — when missing, the restore falls back to createMany
|
||||
* with skipDuplicates for the globals slice (same semantics as
|
||||
* `--globals=skip`).
|
||||
*/
|
||||
upsertManyById?(
|
||||
modelName: string,
|
||||
rows: Record<string, unknown>[],
|
||||
): Promise<{ created: number; updated: number }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* How to handle global rows (User, etc.) during restore.
|
||||
*
|
||||
* - `upsert` (default): `prisma.<global>.upsert({ where:{id}, ... })` per
|
||||
* row. Existing globals on the target are left in place; new globals
|
||||
* get inserted. Best for typical "merge into existing dev/staging DB"
|
||||
* scenarios.
|
||||
* - `skip`: `createMany({ skipDuplicates: true })`. Same effect as
|
||||
* upsert when only inserting; differs in that conflicting unique
|
||||
* columns OTHER than `id` (e.g. `email`) silently drop the new row.
|
||||
* - `error`: refuse to restore if any global id already exists on
|
||||
* target. Useful for "sensitive restore into a known-empty DB"
|
||||
* where overlap means the target wasn't actually empty.
|
||||
*/
|
||||
export type GlobalsRestoreMode = "upsert" | "skip" | "error";
|
||||
|
||||
export function isGlobalsRestoreMode(value: string): value is GlobalsRestoreMode {
|
||||
return value === "upsert" || value === "skip" || value === "error";
|
||||
}
|
||||
|
||||
export interface RunRestoreOptions {
|
||||
/** Target tenant id the snapshot should land under. */
|
||||
readonly tenantId: string;
|
||||
/** Either an s3://bucket/key URL or a local filesystem path. */
|
||||
readonly source: string;
|
||||
readonly prisma: PrismaRestoreRunner;
|
||||
readonly s3?: S3ClientConfig;
|
||||
readonly s3Client?: S3Client;
|
||||
readonly storageBucket?: string;
|
||||
readonly encryption?: EncryptionConfig;
|
||||
/** When true, skip the interactive confirmation prompt. */
|
||||
readonly yes?: boolean;
|
||||
/** Stream used to ask for confirmation; defaults to process.stdin. */
|
||||
readonly stdin?: NodeJS.ReadableStream;
|
||||
/** Stream used to print the confirmation question; defaults to process.stdout. */
|
||||
readonly stdout?: NodeJS.WritableStream;
|
||||
readonly logger?: { info: (m: string) => void; warn: (m: string) => void };
|
||||
/** Refuse to run when NODE_ENV=production unless explicitly allowed. */
|
||||
readonly allowProductionRestore?: boolean;
|
||||
/**
|
||||
* Strategy for the globals slice. Defaults to `upsert`. See the
|
||||
* `GlobalsRestoreMode` docblock for the three modes.
|
||||
*/
|
||||
readonly globalsMode?: GlobalsRestoreMode;
|
||||
}
|
||||
|
||||
export interface RunRestoreResult {
|
||||
readonly manifest: TenantSnapshotManifest;
|
||||
readonly perModel: ReadonlyArray<{ model: string; written: number; expected: number }>;
|
||||
/**
|
||||
* Per-global summary: how many rows the manifest carried, how many
|
||||
* were inserted fresh, and how many already existed on target (under
|
||||
* `upsert` they were left in place; under `skip` they were dropped;
|
||||
* under `error` the restore aborts before this is populated).
|
||||
*/
|
||||
readonly globals: ReadonlyArray<{ model: string; expected: number; created: number; existed: number }>;
|
||||
readonly storageObjectsUploaded: number;
|
||||
}
|
||||
|
||||
export async function runTenantRestore(opts: RunRestoreOptions): Promise<RunRestoreResult> {
|
||||
const logger = opts.logger ?? consola.create({ defaults: { tag: "tenant:restore" } });
|
||||
if (process.env.NODE_ENV === "production" && !opts.allowProductionRestore) {
|
||||
throw new Error(
|
||||
"Refusing to run tenant:restore in NODE_ENV=production without allowProductionRestore=true (pass --allow-production-restore).",
|
||||
);
|
||||
}
|
||||
|
||||
const workRoot = await mkdtemp(join(tmpdir(), "tenant-restore-"));
|
||||
try {
|
||||
// 1. Download (if S3) into the work dir; decrypt if needed; untar.
|
||||
const downloadedPath = await materializeSource(opts, workRoot, logger);
|
||||
let archivePath = downloadedPath;
|
||||
if (opts.encryption && opts.encryption.method !== "none") {
|
||||
const decryptedPath = join(workRoot, "decrypted.tar.gz");
|
||||
logger.info(`Decrypting via ${opts.encryption.method}`);
|
||||
await decryptFile({ config: opts.encryption, inputPath: archivePath, outputPath: decryptedPath });
|
||||
archivePath = decryptedPath;
|
||||
}
|
||||
const extractDir = join(workRoot, "extracted");
|
||||
await mkdir(extractDir, { recursive: true });
|
||||
await extractTar({ file: archivePath, cwd: extractDir });
|
||||
|
||||
// 2. Locate the (single) top-level snapshot directory and read the manifest.
|
||||
const { readdir } = await import("node:fs/promises");
|
||||
const tops = await readdir(extractDir);
|
||||
const snapshotDir = tops.find((d) => d.startsWith("tenant-snapshot-"));
|
||||
if (!snapshotDir) {
|
||||
throw new Error(`No tenant-snapshot-* directory found inside ${archivePath}`);
|
||||
}
|
||||
const stageDir = join(extractDir, snapshotDir);
|
||||
const manifestRaw = await readFile(join(stageDir, "manifest.json"), "utf8");
|
||||
const manifest = deserializeTenantManifest(manifestRaw);
|
||||
logger.info(
|
||||
`Manifest: source=${manifest.sourceTenantId} profile=${manifest.anonymization.profile} models=${manifest.models.length}`,
|
||||
);
|
||||
|
||||
// 3. Confirmation prompt unless --yes.
|
||||
if (!opts.yes) {
|
||||
const answered = await promptYesNo(
|
||||
`About to restore snapshot of tenant ${manifest.sourceTenantId} into ${opts.tenantId}. Continue?`,
|
||||
opts.stdin ?? process.stdin,
|
||||
opts.stdout ?? process.stdout,
|
||||
);
|
||||
if (!answered) {
|
||||
throw new Error("Aborted by operator.");
|
||||
}
|
||||
}
|
||||
|
||||
// 4a. Replay globals FIRST (in FK-correct order) so the tenant-scoped
|
||||
// models that follow can resolve their FK columns (User, etc.).
|
||||
// Manifests created before this extractor landed carry no `globals`
|
||||
// array — we tolerate that and warn so the operator knows they still
|
||||
// need pre-seeded globals on the target DB.
|
||||
const mode: GlobalsRestoreMode = opts.globalsMode ?? "upsert";
|
||||
const globalsSummary: { model: string; expected: number; created: number; existed: number }[] = [];
|
||||
if (manifest.globals && manifest.globals.length > 0) {
|
||||
logger.info(`Replaying ${manifest.globals.length} globals slice(s) with mode=${mode}.`);
|
||||
for (const entry of manifest.globals) {
|
||||
const filePath = join(stageDir, entry.path);
|
||||
const rows = await readJsonl(filePath);
|
||||
// Globals are NOT remapped onto a new tenantId — they live in
|
||||
// global space. Pass rows through verbatim.
|
||||
const summary = await replayGlobalsForModel({
|
||||
modelName: entry.model,
|
||||
rows,
|
||||
mode,
|
||||
prisma: opts.prisma,
|
||||
logger,
|
||||
});
|
||||
globalsSummary.push({
|
||||
model: entry.model,
|
||||
expected: entry.rowCount,
|
||||
created: summary.created,
|
||||
existed: summary.existed,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
logger.warn(
|
||||
"Manifest carries no globals slice (legacy snapshot). Tenant-scoped models with FKs into global rows will fail unless those rows already exist on the target DB.",
|
||||
);
|
||||
}
|
||||
|
||||
// 4b. Replay each tenant-scoped model with the new tenantId.
|
||||
const perModel: { model: string; written: number; expected: number }[] = [];
|
||||
for (const entry of manifest.models) {
|
||||
const filePath = join(stageDir, entry.path);
|
||||
const rows = await readJsonl(filePath);
|
||||
const remapped = rows.map((row) => ({ ...row, tenantId: opts.tenantId }));
|
||||
const result = await opts.prisma.createMany(entry.model, remapped);
|
||||
perModel.push({ model: entry.model, written: result.written, expected: entry.rowCount });
|
||||
if (result.written !== entry.rowCount) {
|
||||
logger.warn(`Model ${entry.model}: wrote ${result.written} rows but manifest expected ${entry.rowCount}.`);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Re-upload storage objects to the new tenant prefix.
|
||||
let storageObjectsUploaded = 0;
|
||||
if (manifest.storage && opts.storageBucket) {
|
||||
storageObjectsUploaded = await restoreStorage({
|
||||
stageDir,
|
||||
storageEntry: manifest.storage,
|
||||
targetBucket: opts.storageBucket,
|
||||
targetPrefix: `tenants/${opts.tenantId}/`,
|
||||
sourcePrefix: `tenants/${manifest.sourceTenantId}/`,
|
||||
client: ensureS3Client(opts),
|
||||
});
|
||||
}
|
||||
|
||||
// 6. Sanity check: per-model row counts must match within tolerance.
|
||||
const mismatched = perModel.filter((m) => m.written !== m.expected);
|
||||
if (mismatched.length > 0) {
|
||||
logger.warn(`Row-count mismatch in ${mismatched.length} models. See restore log for details.`);
|
||||
}
|
||||
|
||||
return { manifest, perModel, globals: globalsSummary, storageObjectsUploaded };
|
||||
} finally {
|
||||
await rm(workRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
interface ReplayGlobalsForModelOptions {
|
||||
readonly modelName: string;
|
||||
readonly rows: Record<string, unknown>[];
|
||||
readonly mode: GlobalsRestoreMode;
|
||||
readonly prisma: PrismaRestoreRunner;
|
||||
readonly logger: { info: (m: string) => void; warn: (m: string) => void };
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert/upsert globals according to the chosen mode.
|
||||
*
|
||||
* Strategy
|
||||
* --------
|
||||
* The PrismaRestoreRunner interface treats `upsertManyById` as
|
||||
* optional. When the production wiring provides it (real PrismaClient,
|
||||
* see `prisma-runner.ts`), `upsert` and `error` modes use it. Test
|
||||
* fakes that only implement `createMany` fall through to a
|
||||
* `createMany({skipDuplicates:true})` shim and emit a warning telling
|
||||
* the operator the mode they asked for could not be honored.
|
||||
*
|
||||
* `error` mode pre-checks for any pre-existing id by performing an
|
||||
* upsert that throws on update (the runner's upsertManyById returns
|
||||
* `updated > 0` when a row already existed; we treat that as a fail).
|
||||
*/
|
||||
async function replayGlobalsForModel(
|
||||
opts: ReplayGlobalsForModelOptions,
|
||||
): Promise<{ created: number; existed: number }> {
|
||||
if (opts.rows.length === 0) {
|
||||
return { created: 0, existed: 0 };
|
||||
}
|
||||
const supportsUpsert = typeof opts.prisma.upsertManyById === "function";
|
||||
if (opts.mode === "skip" || !supportsUpsert) {
|
||||
if (opts.mode !== "skip" && !supportsUpsert) {
|
||||
opts.logger.warn(
|
||||
`Globals restore mode=${opts.mode} requested but runner does not implement upsertManyById; falling back to skip-duplicates semantics.`,
|
||||
);
|
||||
}
|
||||
const result = await opts.prisma.createMany(opts.modelName, opts.rows);
|
||||
return { created: result.written, existed: opts.rows.length - result.written };
|
||||
}
|
||||
const result = await opts.prisma.upsertManyById!(opts.modelName, opts.rows);
|
||||
if (opts.mode === "error" && result.updated > 0) {
|
||||
throw new Error(
|
||||
`tenant:restore aborted (--globals=error): ${opts.modelName} had ${result.updated} pre-existing row(s) on the target. Pass --globals=upsert to overlay snapshot rows over existing ones, or --globals=skip to leave them in place.`,
|
||||
);
|
||||
}
|
||||
return { created: result.created, existed: result.updated };
|
||||
}
|
||||
|
||||
async function materializeSource(
|
||||
opts: RunRestoreOptions,
|
||||
workRoot: string,
|
||||
logger: { info: (m: string) => void; warn: (m: string) => void },
|
||||
): Promise<string> {
|
||||
if (opts.source.startsWith("s3://")) {
|
||||
const rest = opts.source.slice("s3://".length);
|
||||
const slash = rest.indexOf("/");
|
||||
if (slash <= 0) throw new Error(`Invalid --source: ${opts.source}`);
|
||||
const bucket = rest.slice(0, slash);
|
||||
const key = rest.slice(slash + 1);
|
||||
const dest = join(workRoot, "snapshot.tar.gz");
|
||||
logger.info(`Downloading s3://${bucket}/${key}`);
|
||||
const client = ensureS3Client(opts);
|
||||
const resp = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
|
||||
await streamToFile(asReadable(resp.Body), dest);
|
||||
return dest;
|
||||
}
|
||||
return opts.source;
|
||||
}
|
||||
|
||||
async function readJsonl(path: string): Promise<Record<string, unknown>[]> {
|
||||
try {
|
||||
await stat(path);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
const stream = createReadStream(path, { encoding: "utf8" });
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||
for await (const line of rl) {
|
||||
if (line.trim().length === 0) continue;
|
||||
rows.push(JSON.parse(line) as Record<string, unknown>);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
interface RestoreStorageOptions {
|
||||
stageDir: string;
|
||||
storageEntry: NonNullable<TenantSnapshotManifest["storage"]>;
|
||||
targetBucket: string;
|
||||
targetPrefix: string;
|
||||
sourcePrefix: string;
|
||||
client: S3Client;
|
||||
}
|
||||
|
||||
async function restoreStorage(opts: RestoreStorageOptions): Promise<number> {
|
||||
const storageDir = join(opts.stageDir, "storage");
|
||||
const manifestLines = (await readFile(join(storageDir, "manifest.jsonl"), "utf8")).split("\n").filter((l) => l.length > 0);
|
||||
if (manifestLines.length === 0) return 0;
|
||||
const archive = join(storageDir, "objects.tar");
|
||||
const stagingDir = join(opts.stageDir, "_storage_staging");
|
||||
await mkdir(stagingDir, { recursive: true });
|
||||
await extractTar({ file: archive, cwd: stagingDir });
|
||||
|
||||
let uploaded = 0;
|
||||
for (const line of manifestLines) {
|
||||
const entry = JSON.parse(line) as { key: string; relPath: string };
|
||||
const localPath = join(stagingDir, entry.relPath);
|
||||
const newKey = entry.key.startsWith(opts.sourcePrefix)
|
||||
? `${opts.targetPrefix}${entry.key.slice(opts.sourcePrefix.length)}`
|
||||
: `${opts.targetPrefix}${entry.relPath}`;
|
||||
const stream = createReadStream(localPath);
|
||||
const upload = new Upload({
|
||||
client: opts.client,
|
||||
params: { Bucket: opts.targetBucket, Key: newKey, Body: stream },
|
||||
});
|
||||
await upload.done();
|
||||
uploaded += 1;
|
||||
}
|
||||
return uploaded;
|
||||
}
|
||||
|
||||
function promptYesNo(
|
||||
question: string,
|
||||
stdin: NodeJS.ReadableStream,
|
||||
stdout: NodeJS.WritableStream,
|
||||
): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
const rl = createInterface({ input: stdin, output: stdout });
|
||||
rl.question(`${question} [y/N] `, (answer) => {
|
||||
rl.close();
|
||||
resolve(/^y(es)?$/iu.test(answer.trim()));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function ensureS3Client(opts: RunRestoreOptions): S3Client {
|
||||
if (opts.s3Client) return opts.s3Client;
|
||||
if (!opts.s3) throw new Error("S3 client or config is required");
|
||||
return new S3Client(opts.s3);
|
||||
}
|
||||
|
||||
function asReadable(body: unknown): Readable {
|
||||
if (body && typeof (body as Readable).pipe === "function") {
|
||||
return body as Readable;
|
||||
}
|
||||
if (body && typeof (body as { transformToWebStream?: () => unknown }).transformToWebStream === "function") {
|
||||
const webStream = (body as { transformToWebStream: () => ReadableStream<Uint8Array> }).transformToWebStream();
|
||||
return Readable.fromWeb(webStream as never);
|
||||
}
|
||||
if (body && Symbol.asyncIterator in (body as object)) {
|
||||
return Readable.from(body as AsyncIterable<Buffer>);
|
||||
}
|
||||
throw new Error("Unsupported S3 GetObject body type");
|
||||
}
|
||||
|
||||
async function streamToFile(stream: Readable, path: string): Promise<void> {
|
||||
const sink = createWriteStream(path);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
stream.pipe(sink);
|
||||
sink.on("finish", () => resolve());
|
||||
sink.on("error", reject);
|
||||
stream.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
// Suppress the unused-import lint for ListObjectsV2Command; it remains in the
|
||||
// import list so a future iteration that wants to discover unknown keys in
|
||||
// the bucket can pick it up without re-importing.
|
||||
void ListObjectsV2Command;
|
||||
|
||||
export const tenantRestoreCommand = defineCommand({
|
||||
meta: {
|
||||
name: "restore",
|
||||
description: "Restore a tenant snapshot into a new tenant id (FK-remapped).",
|
||||
},
|
||||
args: {
|
||||
"tenant-id": { type: "string", required: true, description: "Target tenant id to create." },
|
||||
source: { type: "string", required: true, description: "s3://bucket/key or local path of the snapshot." },
|
||||
yes: { type: "boolean", default: false, description: "Skip confirmation prompts." },
|
||||
"allow-production-restore": {
|
||||
type: "boolean",
|
||||
default: false,
|
||||
description: "Permit restore when NODE_ENV=production (DANGEROUS).",
|
||||
},
|
||||
globals: {
|
||||
type: "string",
|
||||
default: "upsert",
|
||||
description:
|
||||
"How to handle global rows (User, etc.) on the target: upsert (default, idempotent overlay) | skip (leave conflicting rows in place) | error (abort if any global already exists — for sensitive restores into a known-empty DB).",
|
||||
},
|
||||
},
|
||||
async run({ args }) {
|
||||
const { resolveTenantSnapshotConfig } = await import("./env.js");
|
||||
const cfg = resolveTenantSnapshotConfig(process.env as Record<string, string | undefined>);
|
||||
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { createPrismaRestoreRunner } = await import("./prisma-runner.js");
|
||||
const { parseSchemaScalarTypes } = await import("./json-codec.js");
|
||||
|
||||
const { dirname, join } = await import("node:path");
|
||||
const { fileURLToPath } = await import("node:url");
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
// From packages/cli/src/commands/tenant/restore.ts → repo root is 5 levels up.
|
||||
const schemaDir = join(here, "..", "..", "..", "..", "..", "prisma", "schema");
|
||||
const scalarTypes = parseSchemaScalarTypes(schemaDir);
|
||||
|
||||
const prisma = getDbClient();
|
||||
const runner = createPrismaRestoreRunner(prisma as unknown as Record<string, unknown>, scalarTypes);
|
||||
|
||||
const globalsRaw = args.globals === undefined ? "upsert" : String(args.globals);
|
||||
if (!isGlobalsRestoreMode(globalsRaw)) {
|
||||
throw new Error(`Invalid --globals: ${globalsRaw}; expected upsert | skip | error.`);
|
||||
}
|
||||
|
||||
const result = await runTenantRestore({
|
||||
tenantId: String(args["tenant-id"]),
|
||||
source: String(args.source),
|
||||
prisma: runner,
|
||||
s3: cfg.s3,
|
||||
storageBucket: cfg.appStorageBucket ?? undefined,
|
||||
encryption: cfg.encryption.method === "none" ? undefined : cfg.encryption,
|
||||
yes: Boolean(args.yes),
|
||||
allowProductionRestore: Boolean(args["allow-production-restore"]),
|
||||
globalsMode: globalsRaw,
|
||||
});
|
||||
|
||||
const totalWritten = result.perModel.reduce((n, m) => n + m.written, 0);
|
||||
const totalExpected = result.perModel.reduce((n, m) => n + m.expected, 0);
|
||||
const mismatches = result.perModel.filter((m) => m.written !== m.expected);
|
||||
consola.success(
|
||||
`Restored snapshot of tenant ${result.manifest.sourceTenantId} → ${String(args["tenant-id"])}: ${totalWritten}/${totalExpected} rows across ${result.perModel.length} models.`,
|
||||
);
|
||||
if (result.globals.length > 0) {
|
||||
const summary = result.globals
|
||||
.map((g) => `${g.model}(created=${g.created} existed=${g.existed} expected=${g.expected})`)
|
||||
.join(", ");
|
||||
consola.info(`Globals (mode=${globalsRaw}): ${summary}.`);
|
||||
}
|
||||
if (result.storageObjectsUploaded > 0) {
|
||||
consola.info(`Storage objects re-uploaded: ${result.storageObjectsUploaded}.`);
|
||||
}
|
||||
if (mismatches.length > 0) {
|
||||
consola.warn(
|
||||
`Row-count mismatch in ${mismatches.length} models: ${mismatches.map((m) => `${m.model}(${m.written}/${m.expected})`).join(", ")}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
/**
|
||||
* End-to-end round-trip: seed a synthetic tenant in a real source Postgres,
|
||||
* take a snapshot with the real PrismaRunner, restore into a SEPARATE
|
||||
* target Postgres under a fresh tenant id, and verify row counts +
|
||||
* anonymization.
|
||||
*
|
||||
* Why two databases: `runTenantRestore` preserves source row primary keys
|
||||
* by design (see the docblock at the top of `restore.ts`). Restoring into
|
||||
* the same database the snapshot was taken from would silently drop every
|
||||
* row to `skipDuplicates: true`, which is correct production behaviour but
|
||||
* useless for verifying the round-trip. Operators in CI / on staging
|
||||
* provision two databases.
|
||||
*
|
||||
* Opt-in via:
|
||||
* TENANT_TEST_DATABASE_URL — source DB (or DATABASE_URL if real)
|
||||
* TENANT_TEST_TARGET_DATABASE_URL — target DB (must be a different
|
||||
* database, same schema version)
|
||||
*
|
||||
* The suite skips when either is missing so a developer without two
|
||||
* provisioned databases does not see a CI failure.
|
||||
*
|
||||
* Storage extractor / S3 paths are deliberately out of scope here — they
|
||||
* are covered by `snapshot.test.ts` with aws-sdk-client-mock. This test
|
||||
* only validates the Postgres-side round-trip exercises the real Prisma
|
||||
* client (DateTime + Decimal + Bytes coercion) end to end.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { mkdtemp } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createDbClient } from "@toolctrl/db/client.server";
|
||||
import { runTenantSnapshot } from "./snapshot.js";
|
||||
import { runTenantRestore } from "./restore.js";
|
||||
import { createPrismaSnapshotRunner, createPrismaRestoreRunner } from "./prisma-runner.js";
|
||||
import { parseSchemaScalarTypes } from "./json-codec.js";
|
||||
import { TENANT_SCOPED_MODELS } from "./models.js";
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL?.trim();
|
||||
const sourceUrl =
|
||||
process.env.TENANT_TEST_DATABASE_URL?.trim() ||
|
||||
(DATABASE_URL && !DATABASE_URL.includes("example.invalid") ? DATABASE_URL : undefined);
|
||||
const targetUrl = process.env.TENANT_TEST_TARGET_DATABASE_URL?.trim();
|
||||
const skipIntegration = !sourceUrl || !targetUrl || sourceUrl === targetUrl;
|
||||
const describeIntegration = skipIntegration ? describe.skip : describe;
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = join(here, "..", "..", "..", "..", "..");
|
||||
const schemaDir = join(repoRoot, "prisma", "schema");
|
||||
|
||||
describeIntegration("tenant snapshot ↔ restore (postgres round-trip, source → target DB)", () => {
|
||||
const source = createDbClient({ connectionString: sourceUrl });
|
||||
const target = createDbClient({ connectionString: targetUrl });
|
||||
const sourceTenantId = `ten_snap_src_${Math.random().toString(36).slice(2, 10)}`;
|
||||
const targetTenantId = `ten_snap_dst_${Math.random().toString(36).slice(2, 10)}`;
|
||||
let workdir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), "tenant-roundtrip-"));
|
||||
await source.$connect();
|
||||
await target.$connect();
|
||||
// Seed parent rows so the tenant-scoped fixtures have an FK target.
|
||||
await source.tenant.upsert({
|
||||
where: { id: sourceTenantId },
|
||||
create: { id: sourceTenantId, slug: sourceTenantId, name: "snapshot-src" },
|
||||
update: {},
|
||||
});
|
||||
await target.tenant.upsert({
|
||||
where: { id: targetTenantId },
|
||||
create: { id: targetTenantId, slug: targetTenantId, name: "snapshot-dst" },
|
||||
update: {},
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Best-effort cleanup on both sides. Most failures here are noise
|
||||
// (tables that exist in one schema version but not the other) — we
|
||||
// log nothing because the surrounding test runner already shows the
|
||||
// database error if cleanup actually matters.
|
||||
for (const { model } of TENANT_SCOPED_MODELS) {
|
||||
const accessor = model.charAt(0).toLowerCase() + model.slice(1);
|
||||
for (const [client, tenantId] of [
|
||||
[source, sourceTenantId],
|
||||
[target, targetTenantId],
|
||||
] as const) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const delegate = (client as any)[accessor];
|
||||
if (!delegate?.deleteMany) continue;
|
||||
try {
|
||||
await delegate.deleteMany({ where: { tenantId } });
|
||||
} catch {
|
||||
/* swallow */
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
await source.tenant.delete({ where: { id: sourceTenantId } });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
try {
|
||||
await target.tenant.delete({ where: { id: targetTenantId } });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
await source.$disconnect();
|
||||
await target.$disconnect();
|
||||
});
|
||||
|
||||
it("snapshots a seeded tenant from source and restores its rows into target", async () => {
|
||||
// 1. Seed two AuditEvent rows on source — AuditEvent is tenant-scoped
|
||||
// and has a permissive shape (no required FK chain beyond Tenant).
|
||||
await source.auditEvent.createMany({
|
||||
data: [
|
||||
{
|
||||
tenantId: sourceTenantId,
|
||||
actorType: "SYSTEM",
|
||||
action: "ROUNDTRIP_TEST_A",
|
||||
targetType: "test",
|
||||
},
|
||||
{
|
||||
tenantId: sourceTenantId,
|
||||
actorType: "SYSTEM",
|
||||
action: "ROUNDTRIP_TEST_B",
|
||||
targetType: "test",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// 2. Snapshot from source using the real PrismaRunner.
|
||||
const snapshotRunner = createPrismaSnapshotRunner(source as unknown as Record<string, unknown>);
|
||||
const snapPath = join(workdir, "snap.tar.gz");
|
||||
const snap = await runTenantSnapshot({
|
||||
tenantId: sourceTenantId,
|
||||
profile: "full-anonymized",
|
||||
output: { kind: "local", path: snapPath },
|
||||
prisma: snapshotRunner,
|
||||
logger: { info: () => undefined, warn: () => undefined },
|
||||
});
|
||||
const auditEntry = snap.manifest.models.find((m) => m.model === "AuditEvent");
|
||||
expect(auditEntry?.rowCount).toBe(2);
|
||||
|
||||
// 3. Restore into target.
|
||||
const scalarTypes = parseSchemaScalarTypes(schemaDir);
|
||||
const restoreRunner = createPrismaRestoreRunner(
|
||||
target as unknown as Record<string, unknown>,
|
||||
scalarTypes,
|
||||
);
|
||||
const result = await runTenantRestore({
|
||||
tenantId: targetTenantId,
|
||||
source: snapPath,
|
||||
prisma: restoreRunner,
|
||||
yes: true,
|
||||
logger: { info: () => undefined, warn: () => undefined },
|
||||
});
|
||||
|
||||
// 4. Snapshot extracted 2 AuditEvent rows, restore must write 2.
|
||||
const writtenAudit = result.perModel.find((m) => m.model === "AuditEvent");
|
||||
expect(writtenAudit?.written).toBe(2);
|
||||
expect(writtenAudit?.expected).toBe(2);
|
||||
|
||||
// 5. The two rows must now live under the target tenant id in the
|
||||
// target database, and their createdAt must be a Date (the codec
|
||||
// hydrated the DateTime string back into a real Date).
|
||||
const restored = await target.auditEvent.findMany({
|
||||
where: { tenantId: targetTenantId, action: { in: ["ROUNDTRIP_TEST_A", "ROUNDTRIP_TEST_B"] } },
|
||||
orderBy: { action: "asc" },
|
||||
});
|
||||
expect(restored.map((r) => r.action)).toEqual(["ROUNDTRIP_TEST_A", "ROUNDTRIP_TEST_B"]);
|
||||
expect(restored[0]?.createdAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("snapshots tenant + referenced global Users and restores into a truly empty target", async () => {
|
||||
// This is the B7 follow-up test: the six models that previously
|
||||
// failed on FK-into-User get a row each, plus an AuditEvent for
|
||||
// good measure. The target tenant id exists (created in beforeAll)
|
||||
// but the referenced User rows DO NOT exist on the target. The
|
||||
// round-trip must replay globals first and then all six
|
||||
// tenant-scoped models so the final row counts match.
|
||||
const sourceUserIdA = `usr_src_${Math.random().toString(36).slice(2, 8)}_a`;
|
||||
const sourceUserIdB = `usr_src_${Math.random().toString(36).slice(2, 8)}_b`;
|
||||
try {
|
||||
await source.user.createMany({
|
||||
data: [
|
||||
{
|
||||
id: sourceUserIdA,
|
||||
email: `${sourceUserIdA}@roundtrip.test`,
|
||||
firstName: "Alice",
|
||||
lastName: "RoundtripA",
|
||||
active: true,
|
||||
},
|
||||
{
|
||||
id: sourceUserIdB,
|
||||
email: `${sourceUserIdB}@roundtrip.test`,
|
||||
firstName: "Bob",
|
||||
lastName: "RoundtripB",
|
||||
active: true,
|
||||
},
|
||||
],
|
||||
skipDuplicates: true,
|
||||
});
|
||||
|
||||
// Seed one row on every previously-broken model. Each row must FK
|
||||
// to a User and to the source Tenant.
|
||||
await source.tenantUser.create({
|
||||
data: {
|
||||
tenantId: sourceTenantId,
|
||||
userId: sourceUserIdA,
|
||||
type: 1,
|
||||
joined: 1,
|
||||
status: 1,
|
||||
},
|
||||
});
|
||||
await source.tenantIpAddress.create({
|
||||
data: {
|
||||
tenantId: sourceTenantId,
|
||||
userId: sourceUserIdA,
|
||||
ip: "10.0.0.42",
|
||||
fromUrl: "https://src.example",
|
||||
},
|
||||
});
|
||||
// UserRole needs a Role in the same tenant.
|
||||
const role = await source.role.create({
|
||||
data: {
|
||||
tenantId: sourceTenantId,
|
||||
name: "roundtrip-tester",
|
||||
description: "ephemeral",
|
||||
assignToNewUsers: false,
|
||||
isDefault: false,
|
||||
order: 0,
|
||||
type: "app",
|
||||
},
|
||||
});
|
||||
await source.userRole.create({
|
||||
data: { tenantId: sourceTenantId, userId: sourceUserIdA, roleId: role.id },
|
||||
});
|
||||
await source.event.create({
|
||||
data: {
|
||||
tenantId: sourceTenantId,
|
||||
userId: sourceUserIdB,
|
||||
name: "roundtrip.fired",
|
||||
data: "{}",
|
||||
},
|
||||
});
|
||||
await source.log.create({
|
||||
data: {
|
||||
tenantId: sourceTenantId,
|
||||
userId: sourceUserIdA,
|
||||
action: "ROUNDTRIP.LOG",
|
||||
url: "https://src.example/page",
|
||||
},
|
||||
});
|
||||
await source.supportTicket.create({
|
||||
data: {
|
||||
tenantId: sourceTenantId,
|
||||
userId: sourceUserIdB,
|
||||
source: "in-app",
|
||||
subject: "roundtrip ticket",
|
||||
message: "test body",
|
||||
status: "open",
|
||||
fromEmail: "ticket@roundtrip.test",
|
||||
fromUrl: "https://src.example/help",
|
||||
},
|
||||
});
|
||||
|
||||
// Snapshot from source — globals extractor harvests both users.
|
||||
const snapshotRunner = createPrismaSnapshotRunner(source as unknown as Record<string, unknown>);
|
||||
const snapPath = join(workdir, "with-globals.tar.gz");
|
||||
const snap = await runTenantSnapshot({
|
||||
tenantId: sourceTenantId,
|
||||
profile: "full-anonymized",
|
||||
output: { kind: "local", path: snapPath },
|
||||
prisma: snapshotRunner,
|
||||
logger: { info: () => undefined, warn: () => undefined },
|
||||
});
|
||||
// Manifest must carry the globals slice with both users.
|
||||
expect(snap.manifest.globals).toBeDefined();
|
||||
const userGlobals = snap.manifest.globals?.find((g) => g.model === "User");
|
||||
expect(userGlobals?.rowCount).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// Restore into target — referenced Users DO NOT exist on target
|
||||
// yet (target.tenant exists, but no users).
|
||||
const targetUserCountBefore = await target.user.count({
|
||||
where: { id: { in: [sourceUserIdA, sourceUserIdB] } },
|
||||
});
|
||||
expect(targetUserCountBefore).toBe(0);
|
||||
|
||||
const scalarTypes = parseSchemaScalarTypes(schemaDir);
|
||||
const restoreRunner = createPrismaRestoreRunner(
|
||||
target as unknown as Record<string, unknown>,
|
||||
scalarTypes,
|
||||
);
|
||||
const result = await runTenantRestore({
|
||||
tenantId: targetTenantId,
|
||||
source: snapPath,
|
||||
prisma: restoreRunner,
|
||||
yes: true,
|
||||
globalsMode: "upsert",
|
||||
logger: { info: () => undefined, warn: () => undefined },
|
||||
});
|
||||
|
||||
// Users now exist on target (created by the globals replay).
|
||||
const targetUserCountAfter = await target.user.count({
|
||||
where: { id: { in: [sourceUserIdA, sourceUserIdB] } },
|
||||
});
|
||||
expect(targetUserCountAfter).toBe(2);
|
||||
|
||||
// Each previously-broken model must report written === expected.
|
||||
const previouslyBroken = ["TenantUser", "TenantIpAddress", "UserRole", "Event", "Log", "SupportTicket"];
|
||||
for (const model of previouslyBroken) {
|
||||
const entry = result.perModel.find((m) => m.model === model);
|
||||
expect(entry, `model ${model} missing from perModel`).toBeDefined();
|
||||
expect(entry!.written, `${model}: written ${entry!.written} ≠ expected ${entry!.expected}`).toBe(entry!.expected);
|
||||
expect(entry!.expected, `${model}: expected count should be ≥1`).toBeGreaterThan(0);
|
||||
}
|
||||
|
||||
// Spot-check: TenantUser landed under the new tenantId AND its
|
||||
// userId resolves to a now-existing target user.
|
||||
const restoredTenantUsers = await target.tenantUser.findMany({
|
||||
where: { tenantId: targetTenantId },
|
||||
});
|
||||
expect(restoredTenantUsers.length).toBeGreaterThan(0);
|
||||
for (const tu of restoredTenantUsers) {
|
||||
const userExists = await target.user.findUnique({ where: { id: tu.userId } });
|
||||
expect(userExists, `restored TenantUser.userId=${tu.userId} dangles`).not.toBeNull();
|
||||
}
|
||||
|
||||
// Globals summary in the result must list User with the expected counts.
|
||||
const userGlobalsResult = result.globals.find((g) => g.model === "User");
|
||||
expect(userGlobalsResult?.created).toBeGreaterThanOrEqual(2);
|
||||
} finally {
|
||||
// Best-effort cleanup of the source-side seed rows (target rows
|
||||
// are handled by the suite's afterAll because they live under
|
||||
// targetTenantId).
|
||||
try {
|
||||
await source.userRole.deleteMany({ where: { tenantId: sourceTenantId } });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
try {
|
||||
await source.role.deleteMany({ where: { tenantId: sourceTenantId } });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
for (const userId of [sourceUserIdA, sourceUserIdB]) {
|
||||
try {
|
||||
await source.user.delete({ where: { id: userId } });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
try {
|
||||
await target.user.delete({ where: { id: userId } });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("survives a tenant with zero tenant-scoped rows in every model", async () => {
|
||||
// No seed — pure 0-row round-trip across every model. The
|
||||
// interesting failure mode would be a model whose accessor name
|
||||
// doesn't match the camelCase rule (this test surfaces it loudly).
|
||||
const snapshotRunner = createPrismaSnapshotRunner(source as unknown as Record<string, unknown>);
|
||||
const snapPath = join(workdir, "empty.tar.gz");
|
||||
// Use a fresh source tenant id with no rows so a possible leftover
|
||||
// from the first test does not affect this assertion.
|
||||
const emptyTenantId = `ten_snap_empty_${Math.random().toString(36).slice(2, 10)}`;
|
||||
await source.tenant.upsert({
|
||||
where: { id: emptyTenantId },
|
||||
create: { id: emptyTenantId, slug: emptyTenantId, name: "snapshot-empty" },
|
||||
update: {},
|
||||
});
|
||||
try {
|
||||
const snap = await runTenantSnapshot({
|
||||
tenantId: emptyTenantId,
|
||||
profile: "full-anonymized",
|
||||
output: { kind: "local", path: snapPath },
|
||||
prisma: snapshotRunner,
|
||||
logger: { info: () => undefined, warn: () => undefined },
|
||||
});
|
||||
const totalRows = snap.manifest.models.reduce((n, m) => n + m.rowCount, 0);
|
||||
expect(totalRows).toBe(0);
|
||||
|
||||
const scalarTypes = parseSchemaScalarTypes(schemaDir);
|
||||
const restoreRunner = createPrismaRestoreRunner(
|
||||
target as unknown as Record<string, unknown>,
|
||||
scalarTypes,
|
||||
);
|
||||
const emptyTargetTenantId = `ten_snap_empty_dst_${Math.random().toString(36).slice(2, 10)}`;
|
||||
await target.tenant.upsert({
|
||||
where: { id: emptyTargetTenantId },
|
||||
create: { id: emptyTargetTenantId, slug: emptyTargetTenantId, name: "snapshot-empty-dst" },
|
||||
update: {},
|
||||
});
|
||||
try {
|
||||
const result = await runTenantRestore({
|
||||
tenantId: emptyTargetTenantId,
|
||||
source: snapPath,
|
||||
prisma: restoreRunner,
|
||||
yes: true,
|
||||
logger: { info: () => undefined, warn: () => undefined },
|
||||
});
|
||||
const totalWritten = result.perModel.reduce((n, m) => n + m.written, 0);
|
||||
expect(totalWritten).toBe(0);
|
||||
expect(result.manifest.sourceTenantId).toBe(emptyTenantId);
|
||||
} finally {
|
||||
await target.tenant.delete({ where: { id: emptyTargetTenantId } }).catch(() => undefined);
|
||||
}
|
||||
} finally {
|
||||
await source.tenant.delete({ where: { id: emptyTenantId } }).catch(() => undefined);
|
||||
}
|
||||
});
|
||||
});
|
||||
+466
@@ -0,0 +1,466 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { mockClient } from "aws-sdk-client-mock";
|
||||
import { GetObjectCommand, ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
|
||||
import { Readable } from "node:stream";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import {
|
||||
parseOutputTarget,
|
||||
runTenantSnapshot,
|
||||
type PrismaRunner,
|
||||
} from "./snapshot.js";
|
||||
import { TENANT_SCOPED_MODELS } from "./models.js";
|
||||
|
||||
class FakePrisma implements PrismaRunner {
|
||||
constructor(
|
||||
private readonly rows: Record<string, Record<string, unknown>[]>,
|
||||
private readonly globals: Record<string, Record<string, unknown>[]> = {},
|
||||
) {}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async *findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>> {
|
||||
void tenantField;
|
||||
void tenantId;
|
||||
void pageSize;
|
||||
const list = this.rows[modelName] ?? [];
|
||||
for (const r of list) {
|
||||
yield r;
|
||||
}
|
||||
}
|
||||
async findManyByIds(modelName: string, ids: readonly string[]): Promise<Record<string, unknown>[]> {
|
||||
const list = this.globals[modelName] ?? [];
|
||||
return list.filter((r) => typeof r.id === "string" && ids.includes(r.id as string));
|
||||
}
|
||||
}
|
||||
|
||||
class RecordingPrisma extends FakePrisma {
|
||||
readonly pageSizes: number[] = [];
|
||||
|
||||
override async *findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>> {
|
||||
this.pageSizes.push(pageSize);
|
||||
yield* super.findManyByTenant(modelName, tenantField, tenantId, pageSize);
|
||||
}
|
||||
}
|
||||
|
||||
describe("parseOutputTarget", () => {
|
||||
it("falls back to default local path when undefined", () => {
|
||||
expect(parseOutputTarget(undefined, "/tmp/x.tar.gz")).toEqual({
|
||||
kind: "local",
|
||||
path: "/tmp/x.tar.gz",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats unprefixed paths as local files", () => {
|
||||
expect(parseOutputTarget("/snapshots/a.tar.gz", "/tmp/x")).toEqual({
|
||||
kind: "local",
|
||||
path: "/snapshots/a.tar.gz",
|
||||
});
|
||||
});
|
||||
|
||||
it("parses s3://bucket/key targets", () => {
|
||||
expect(parseOutputTarget("s3://my-bucket/path/to/snap.tar.gz", "")).toEqual({
|
||||
kind: "s3",
|
||||
bucket: "my-bucket",
|
||||
key: "path/to/snap.tar.gz",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed s3:// targets", () => {
|
||||
expect(() => parseOutputTarget("s3://bucket-only", "")).toThrow(/Invalid --output/u);
|
||||
expect(() => parseOutputTarget("s3://bucket/", "")).toThrow(/Invalid --output/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runTenantSnapshot", () => {
|
||||
let workdir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), "tenant-snapshot-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("writes a manifest with row counts per model and the chosen profile", async () => {
|
||||
const prisma = new FakePrisma({
|
||||
TenantUser: [
|
||||
{ id: "tu1", tenantId: "ten_a", userId: "u1" },
|
||||
{ id: "tu2", tenantId: "ten_a", userId: "u2" },
|
||||
],
|
||||
TenantUserInvitation: [
|
||||
{ id: "inv1", tenantId: "ten_a", email: "alice@example.com", firstName: "Alice", lastName: "X" },
|
||||
],
|
||||
});
|
||||
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.snapshotId).toMatch(/^tenant-snapshot-ten_a-/u);
|
||||
expect(result.manifest.anonymization.profile).toBe("full-anonymized");
|
||||
expect(result.manifest.anonymization.saltHash).toMatch(/^[a-f0-9]{64}$/u);
|
||||
expect(result.manifest.sourceTenantId).toBe("ten_a");
|
||||
const tuEntry = result.manifest.models.find((m) => m.model === "TenantUser");
|
||||
expect(tuEntry?.rowCount).toBe(2);
|
||||
const invEntry = result.manifest.models.find((m) => m.model === "TenantUserInvitation");
|
||||
expect(invEntry?.rowCount).toBe(1);
|
||||
// every other model should be present with rowCount=0.
|
||||
expect(result.manifest.models).toHaveLength(TENANT_SCOPED_MODELS.length);
|
||||
for (const m of result.manifest.models) {
|
||||
if (m.model !== "TenantUser" && m.model !== "TenantUserInvitation") {
|
||||
expect(m.rowCount).toBe(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("requires explicit super-admin confirmation for the none anonymization profile", async () => {
|
||||
await expect(
|
||||
runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "none",
|
||||
prisma: new FakePrisma({}),
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/super-admin/iu);
|
||||
});
|
||||
|
||||
it("records super-admin confirmation in the audit entry for profile none", async () => {
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const auditPath = join(workdir, "audit.jsonl");
|
||||
const fixedNow = new Date("2026-05-25T12:00:00.000Z");
|
||||
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "none",
|
||||
prisma: new FakePrisma({ TenantUser: [{ id: "tu1", tenantId: "ten_a", userId: "u1" }] }),
|
||||
output: { kind: "local", path: outPath },
|
||||
superAdminConfirmed: true,
|
||||
auditFilePath: auditPath,
|
||||
actorUserId: "user_admin",
|
||||
now: () => fixedNow,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.manifest.anonymization.profile).toBe("none");
|
||||
const audit = JSON.parse((await readFile(auditPath, "utf8")).trim()) as {
|
||||
event: string;
|
||||
sourceTenantId: string;
|
||||
actorUserId: string;
|
||||
createdAt: string;
|
||||
outputTarget: string;
|
||||
superAdminConfirmed: boolean;
|
||||
};
|
||||
expect(audit).toMatchObject({
|
||||
event: "TENANT_SNAPSHOT_CREATED",
|
||||
sourceTenantId: "ten_a",
|
||||
actorUserId: "user_admin",
|
||||
createdAt: "2026-05-25T12:00:00.000Z",
|
||||
outputTarget: outPath,
|
||||
superAdminConfirmed: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("passes the configured page size to every tenant-scoped model query", async () => {
|
||||
const prisma = new RecordingPrisma({});
|
||||
await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
pageSize: 17,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(prisma.pageSizes).toHaveLength(TENANT_SCOPED_MODELS.length);
|
||||
expect(new Set(prisma.pageSizes)).toEqual(new Set([17]));
|
||||
});
|
||||
|
||||
it("anonymizes PII inline so the snapshot file contains no raw emails", async () => {
|
||||
const prisma = new FakePrisma({
|
||||
TenantUserInvitation: [
|
||||
{ id: "inv1", tenantId: "ten_a", email: "real-person@example.org", firstName: "Real", lastName: "Person" },
|
||||
],
|
||||
});
|
||||
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
// Untar locally and read the model file.
|
||||
const { mkdtemp: mk } = await import("node:fs/promises");
|
||||
const { extract } = await import("tar");
|
||||
const untarDir = await mk(join(tmpdir(), "tenant-snap-untar-"));
|
||||
try {
|
||||
await extract({ file: outPath, cwd: untarDir });
|
||||
const modelFile = join(untarDir, result.snapshotId, "postgres", "TenantUserInvitation.jsonl");
|
||||
const content = await readFile(modelFile, "utf8");
|
||||
expect(content).not.toContain("real-person@example.org");
|
||||
expect(content).toMatch(/@anonymized\.test/u);
|
||||
} finally {
|
||||
await rm(untarDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("uploads to S3 when --output is s3://bucket/key", async () => {
|
||||
const prisma = new FakePrisma({});
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake(() => ({ Contents: [], IsTruncated: false }));
|
||||
const fakeUpload = vi.fn().mockImplementation(() => ({
|
||||
done: vi.fn().mockResolvedValue({ ETag: '"deadbeef"' }),
|
||||
}));
|
||||
// Monkey-patch the Upload constructor via the global hook in the source.
|
||||
// We use the lib-storage Upload import inside s3-upload.ts; the helper
|
||||
// accepts an `uploadCtor` override but runTenantSnapshot does not expose it.
|
||||
// Instead we stub Upload.prototype.done.
|
||||
const origDone = Upload.prototype.done;
|
||||
Upload.prototype.done = vi.fn().mockResolvedValue({ ETag: '"deadbeef"' }) as never;
|
||||
try {
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "minimal-anonymized",
|
||||
prisma,
|
||||
output: { kind: "s3", bucket: "snapshots", key: "ten_a/snap.tar.gz" },
|
||||
s3Client: s3Mock as unknown as S3Client,
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.uploaded).toEqual({ bucket: "snapshots", key: "ten_a/snap.tar.gz" });
|
||||
expect(result.manifest.storage).toBeDefined();
|
||||
expect(result.manifest.storage?.objectCount).toBe(0);
|
||||
} finally {
|
||||
Upload.prototype.done = origDone;
|
||||
void fakeUpload;
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects storage snapshots without an S3 client or config", async () => {
|
||||
await expect(
|
||||
runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: new FakePrisma({}),
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/S3 client or config is required/u);
|
||||
});
|
||||
|
||||
it("includes tenant storage objects when bucket is configured", async () => {
|
||||
const prisma = new FakePrisma({});
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake(() => ({
|
||||
Contents: [
|
||||
{ Key: "tenants/ten_a/avatars/me.png", Size: 17, ETag: '"abc"' },
|
||||
{ Key: "tenants/ten_a/uploads/file.txt", Size: 5, ETag: '"def"' },
|
||||
],
|
||||
IsTruncated: false,
|
||||
}));
|
||||
s3Mock.on(GetObjectCommand as never).callsFake(() => ({
|
||||
Body: Readable.from([Buffer.from("xx")]),
|
||||
}));
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
s3Client: s3Mock as unknown as S3Client,
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.manifest.storage?.objectCount).toBe(2);
|
||||
});
|
||||
|
||||
it("paginates tenant storage extraction and ignores objects without keys", async () => {
|
||||
const prisma = new FakePrisma({});
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake((input: { ContinuationToken?: string }) => {
|
||||
if (!input.ContinuationToken) {
|
||||
return {
|
||||
Contents: [
|
||||
{ Size: 999, ETag: '"missing-key"' },
|
||||
{ Key: "tenants/ten_a/avatars/me.png", Size: 17, ETag: '"abc"' },
|
||||
],
|
||||
IsTruncated: true,
|
||||
NextContinuationToken: "page-2",
|
||||
};
|
||||
}
|
||||
return {
|
||||
Contents: [{ Key: "tenants/ten_a/uploads/file.txt", Size: 5, ETag: '"def"' }],
|
||||
IsTruncated: false,
|
||||
};
|
||||
});
|
||||
s3Mock.on(GetObjectCommand as never).callsFake(() => ({
|
||||
Body: Readable.from([Buffer.from("xx")]),
|
||||
}));
|
||||
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
s3Client: s3Mock as unknown as S3Client,
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.manifest.storage?.objectCount).toBe(2);
|
||||
expect(result.manifest.storage?.sizeBytes).toBe(22);
|
||||
const listCalls = s3Mock.commandCalls(ListObjectsV2Command as never);
|
||||
expect(listCalls).toHaveLength(2);
|
||||
expect(listCalls[1].args[0].input).toMatchObject({ ContinuationToken: "page-2" });
|
||||
expect(s3Mock.commandCalls(GetObjectCommand as never)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("emits globals/User.jsonl with referenced + anonymized user rows", async () => {
|
||||
const prisma = new FakePrisma(
|
||||
{
|
||||
TenantUser: [
|
||||
{ id: "tu1", tenantId: "ten_a", userId: "u1", type: 1, joined: 1, status: 1 },
|
||||
{ id: "tu2", tenantId: "ten_a", userId: "u2", type: 1, joined: 1, status: 1 },
|
||||
],
|
||||
},
|
||||
{
|
||||
User: [
|
||||
{ id: "u1", email: "first@real.com", firstName: "First", lastName: "Real", phone: "+10001" },
|
||||
{ id: "u2", email: "second@real.com", firstName: "Second", lastName: "Real", phone: "+10002" },
|
||||
{ id: "u_unused", email: "unused@real.com", firstName: "Unused", lastName: "U" },
|
||||
],
|
||||
},
|
||||
);
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.manifest.globals).toBeDefined();
|
||||
const userEntry = result.manifest.globals?.find((g) => g.model === "User");
|
||||
expect(userEntry?.rowCount).toBe(2);
|
||||
expect(userEntry?.path).toBe("globals/User.jsonl");
|
||||
|
||||
const { extract } = await import("tar");
|
||||
const untar = await mkdtemp(join(tmpdir(), "snap-globals-untar-"));
|
||||
try {
|
||||
await extract({ file: outPath, cwd: untar });
|
||||
const text = await readFile(join(untar, result.snapshotId, "globals", "User.jsonl"), "utf8");
|
||||
expect(text).not.toContain("first@real.com");
|
||||
expect(text).not.toContain("second@real.com");
|
||||
expect(text).not.toContain("unused@real.com");
|
||||
const lines = text.trim().split("\n").map((l) => JSON.parse(l));
|
||||
expect(lines.length).toBe(2);
|
||||
const ids = new Set(lines.map((l) => l.id));
|
||||
expect(ids).toEqual(new Set(["u1", "u2"]));
|
||||
for (const u of lines) {
|
||||
expect(u.email).toMatch(/@anonymized\.test$/);
|
||||
// PII columns scrubbed.
|
||||
expect(u.firstName).not.toMatch(/^(First|Second)$/);
|
||||
expect(u.phone).toBeNull();
|
||||
}
|
||||
} finally {
|
||||
await rm(untar, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("skips globals extraction when include-globals=false", async () => {
|
||||
const prisma = new FakePrisma(
|
||||
{ TenantUser: [{ id: "tu1", tenantId: "ten_a", userId: "u1", type: 1, joined: 1, status: 1 }] },
|
||||
{ User: [{ id: "u1", email: "a@b.io", firstName: "A", lastName: "B" }] },
|
||||
);
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
includeGlobals: false,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.manifest.globals).toBeUndefined();
|
||||
});
|
||||
|
||||
it("warns and skips globals when the runner has no findManyByIds method", async () => {
|
||||
// A minimal runner without findManyByIds — simulates a legacy adapter.
|
||||
class LegacyRunner implements PrismaRunner {
|
||||
async *findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>> {
|
||||
void modelName;
|
||||
void tenantField;
|
||||
void tenantId;
|
||||
void pageSize;
|
||||
yield* [];
|
||||
}
|
||||
}
|
||||
const warn = vi.fn();
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: new LegacyRunner(),
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn },
|
||||
});
|
||||
expect(result.manifest.globals).toBeUndefined();
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("findManyByIds"));
|
||||
});
|
||||
|
||||
it("preserves anonymization-FK consistency across calls (same salt seed inside one run)", async () => {
|
||||
const prisma = new FakePrisma({
|
||||
TenantUserInvitation: [
|
||||
{ id: "1", tenantId: "ten_a", email: "alice@example.com", firstName: "Alice", lastName: "Z" },
|
||||
{ id: "2", tenantId: "ten_a", email: "alice@example.com", firstName: "Alice", lastName: "Z" },
|
||||
],
|
||||
});
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
const { extract } = await import("tar");
|
||||
const untarDir = await mkdtemp(join(tmpdir(), "tenant-snap-untar-"));
|
||||
try {
|
||||
await extract({ file: outPath, cwd: untarDir });
|
||||
const file = join(untarDir, result.snapshotId, "postgres", "TenantUserInvitation.jsonl");
|
||||
const lines = (await readFile(file, "utf8")).trim().split("\n").map((l) => JSON.parse(l));
|
||||
// Both rows have the same original email → must have the same anonymized email.
|
||||
expect(lines[0].email).toBe(lines[1].email);
|
||||
} finally {
|
||||
await rm(untarDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
+541
@@ -0,0 +1,541 @@
|
||||
/**
|
||||
* `tenant:snapshot` — extract a single tenant from Postgres + storage, apply
|
||||
* the chosen PII anonymization profile, tar + (optionally) encrypt + upload.
|
||||
*
|
||||
* The Prisma layer is dependency-injected via the `PrismaRunner` interface so
|
||||
* the CLI package stays free of `@prisma/client`. The production entrypoint
|
||||
* (`bin/toolctrl-cli.mjs` of the consumer) wires up the real `PrismaClient`;
|
||||
* tests pass a fake.
|
||||
*/
|
||||
import { defineCommand } from "citty";
|
||||
import consola from "consola";
|
||||
import { randomBytes, createHash } from "node:crypto";
|
||||
import { createWriteStream, createReadStream } from "node:fs";
|
||||
import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir, hostname } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
GetObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
S3Client,
|
||||
type S3ClientConfig,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { create as createTar } from "tar";
|
||||
import {
|
||||
PROFILES,
|
||||
anonymizeRow,
|
||||
isProfileName,
|
||||
type AnonymizationContext,
|
||||
type ProfileName,
|
||||
} from "./anonymize.js";
|
||||
import {
|
||||
TENANT_MANIFEST_VERSION,
|
||||
hashSalt,
|
||||
serializeTenantManifest,
|
||||
type TenantManifestModelEntry,
|
||||
type TenantSnapshotManifest,
|
||||
} from "./manifest.js";
|
||||
import { TENANT_SCOPED_MODELS } from "./models.js";
|
||||
import { assertSuperAdminForNoneProfile, writeSnapshotAudit } from "./audit.js";
|
||||
import { snapshotReplacer } from "./json-codec.js";
|
||||
import { collectAndWriteGlobals, type GlobalsPrismaRunner } from "./globals-runner.js";
|
||||
import { encryptFile, type EncryptionConfig } from "../backup/encryption.js";
|
||||
import { uploadFileMultipart } from "../backup/s3-upload.js";
|
||||
|
||||
/**
|
||||
* Minimal Prisma-like surface the snapshot command needs.
|
||||
*
|
||||
* `findManyByIds` is added for the globals extractor (B7 follow-up):
|
||||
* after collecting the tenant slice we re-issue id-set lookups against
|
||||
* the source DB to fetch the referenced `User` rows (and any other
|
||||
* globals enumerated in `models.ts::GLOBAL_REFERENCED_MODELS`). It is
|
||||
* optional so legacy callers that only snapshot tenant-scoped rows
|
||||
* (e.g. `--include-globals=false`) can still pass a slimmer runner.
|
||||
*/
|
||||
export interface PrismaRunner extends Partial<GlobalsPrismaRunner> {
|
||||
/**
|
||||
* Stream rows from `modelName` where `tenantId = tenantId` in stable id order.
|
||||
* Implementations should use cursor-based pagination so large tenants do
|
||||
* not OOM. The default page size is 1000 unless the caller overrides.
|
||||
*/
|
||||
findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface SnapshotS3Output {
|
||||
readonly kind: "s3";
|
||||
readonly bucket: string;
|
||||
readonly key: string;
|
||||
}
|
||||
|
||||
export interface SnapshotLocalOutput {
|
||||
readonly kind: "local";
|
||||
readonly path: string;
|
||||
}
|
||||
|
||||
export type SnapshotOutputTarget = SnapshotS3Output | SnapshotLocalOutput;
|
||||
|
||||
export interface RunSnapshotOptions {
|
||||
readonly tenantId: string;
|
||||
readonly profile: ProfileName;
|
||||
readonly output: SnapshotOutputTarget;
|
||||
readonly prisma: PrismaRunner;
|
||||
readonly s3?: S3ClientConfig;
|
||||
readonly s3Client?: S3Client;
|
||||
readonly storageBucket?: string;
|
||||
readonly storagePrefix?: string;
|
||||
readonly encryption?: EncryptionConfig;
|
||||
readonly now?: () => Date;
|
||||
/** Marker the operator must pass when invoking `--profile=none`. */
|
||||
readonly superAdminConfirmed?: boolean;
|
||||
/** Optional file path to append the snapshot-audit JSON record to. */
|
||||
readonly auditFilePath?: string;
|
||||
/** Stream the human-readable audit line is written to (defaults to stdout). */
|
||||
readonly auditStdout?: NodeJS.WritableStream;
|
||||
readonly logger?: { info: (m: string) => void; warn: (m: string) => void };
|
||||
readonly actorUserId?: string;
|
||||
readonly pageSize?: number;
|
||||
/**
|
||||
* Emit `globals/<Model>.jsonl` files for every model in
|
||||
* `GLOBAL_REFERENCED_MODELS`. Defaults to `true`. Set to `false` for
|
||||
* backward-compatible "tenant-only" snapshots (the operator must then
|
||||
* pre-seed referenced globals on the target before restore).
|
||||
*
|
||||
* Requires `prisma.findManyByIds` to be implemented. If a runner
|
||||
* without that method is supplied with `includeGlobals=true`, the
|
||||
* snapshot logs a warning and skips the globals step rather than
|
||||
* failing — useful for unit tests that only exercise the tenant path.
|
||||
*/
|
||||
readonly includeGlobals?: boolean;
|
||||
}
|
||||
|
||||
export interface RunSnapshotResult {
|
||||
readonly snapshotId: string;
|
||||
readonly manifest: TenantSnapshotManifest;
|
||||
readonly archivePath: string;
|
||||
readonly uploaded?: { bucket: string; key: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an --output value into a structured target. Accepts `s3://bucket/key`
|
||||
* or a local filesystem path. Returns `null` if undefined.
|
||||
*/
|
||||
export function parseOutputTarget(raw: string | undefined, defaultLocalPath: string): SnapshotOutputTarget {
|
||||
if (!raw || raw.length === 0) {
|
||||
return { kind: "local", path: defaultLocalPath };
|
||||
}
|
||||
if (raw.startsWith("s3://")) {
|
||||
const rest = raw.slice("s3://".length);
|
||||
const slash = rest.indexOf("/");
|
||||
if (slash <= 0 || slash === rest.length - 1) {
|
||||
throw new Error(`Invalid --output: expected s3://<bucket>/<key>, got ${raw}`);
|
||||
}
|
||||
return { kind: "s3", bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
|
||||
}
|
||||
return { kind: "local", path: raw };
|
||||
}
|
||||
|
||||
export async function runTenantSnapshot(opts: RunSnapshotOptions): Promise<RunSnapshotResult> {
|
||||
assertSuperAdminForNoneProfile(opts.profile, Boolean(opts.superAdminConfirmed));
|
||||
const logger = opts.logger ?? consola.create({ defaults: { tag: "tenant:snapshot" } });
|
||||
const now = (opts.now ?? (() => new Date()))();
|
||||
const createdAt = now.toISOString();
|
||||
const shortUuid = randomBytes(4).toString("hex");
|
||||
const snapshotId = `tenant-snapshot-${opts.tenantId}-${createdAt.replace(/[:.]/gu, "")}-${shortUuid}`;
|
||||
const salt = randomBytes(32).toString("hex");
|
||||
const ctx: AnonymizationContext = { salt, profileName: opts.profile };
|
||||
const profile = PROFILES[opts.profile];
|
||||
|
||||
const tempRoot = await mkdtemp(join(tmpdir(), "tenant-snapshot-"));
|
||||
try {
|
||||
const stageDir = join(tempRoot, snapshotId);
|
||||
await mkdir(stageDir, { recursive: true });
|
||||
const postgresDir = join(stageDir, "postgres");
|
||||
await mkdir(postgresDir, { recursive: true });
|
||||
|
||||
const pageSize = opts.pageSize ?? 1000;
|
||||
const modelEntries: TenantManifestModelEntry[] = [];
|
||||
for (const { model, tenantField } of TENANT_SCOPED_MODELS) {
|
||||
const filePath = join(postgresDir, `${model}.jsonl`);
|
||||
logger.info(`Extracting model ${model}`);
|
||||
const { rowCount, sizeBytes, sha256 } = await writeModelJsonl({
|
||||
prisma: opts.prisma,
|
||||
modelName: model,
|
||||
tenantField,
|
||||
tenantId: opts.tenantId,
|
||||
pageSize,
|
||||
profile,
|
||||
ctx,
|
||||
outPath: filePath,
|
||||
});
|
||||
modelEntries.push({
|
||||
model,
|
||||
path: `postgres/${model}.jsonl`,
|
||||
rowCount,
|
||||
sizeBytes,
|
||||
checksum: { algorithm: "sha256", value: sha256 },
|
||||
});
|
||||
}
|
||||
|
||||
// After the tenant-scoped slice is staged, harvest referenced
|
||||
// global rows (User + anything else enumerated in
|
||||
// GLOBAL_REFERENCED_MODELS). Streams the JSONL files we just
|
||||
// wrote, collects the FK-id sets per global, fetches the rows from
|
||||
// Prisma in chunks, anonymizes with the same per-snapshot salt
|
||||
// and writes `globals/<Model>.jsonl`.
|
||||
let globalsEntries: TenantSnapshotManifest["globals"];
|
||||
const wantGlobals = opts.includeGlobals !== false;
|
||||
if (wantGlobals) {
|
||||
const globalsRunner = opts.prisma as Partial<GlobalsPrismaRunner>;
|
||||
if (typeof globalsRunner.findManyByIds === "function") {
|
||||
const globalsDir = join(stageDir, "globals");
|
||||
await mkdir(globalsDir, { recursive: true });
|
||||
const collected = await collectAndWriteGlobals({
|
||||
postgresDir,
|
||||
globalsDir,
|
||||
prisma: globalsRunner as GlobalsPrismaRunner,
|
||||
profile: opts.profile,
|
||||
ctx,
|
||||
logger,
|
||||
});
|
||||
if (collected.entries.length > 0) {
|
||||
globalsEntries = collected.entries;
|
||||
}
|
||||
} else {
|
||||
logger.warn(
|
||||
"Snapshot runner does not implement findManyByIds — skipping globals extraction. Restore into an empty DB will fail on FKs to global rows.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let storageEntry: TenantSnapshotManifest["storage"];
|
||||
if (opts.storageBucket) {
|
||||
const s3 = ensureS3Client(opts);
|
||||
const storageDir = join(stageDir, "storage");
|
||||
await mkdir(storageDir, { recursive: true });
|
||||
const storageResult = await extractTenantStorage({
|
||||
client: s3,
|
||||
bucket: opts.storageBucket,
|
||||
tenantPrefix: opts.storagePrefix ?? `tenants/${opts.tenantId}/`,
|
||||
outDir: storageDir,
|
||||
});
|
||||
storageEntry = {
|
||||
path: "storage/objects.tar",
|
||||
objectCount: storageResult.objectCount,
|
||||
sizeBytes: storageResult.totalBytes,
|
||||
checksum: { algorithm: "sha256", value: storageResult.checksum },
|
||||
};
|
||||
} else {
|
||||
logger.warn("No storage bucket configured; skipping storage component.");
|
||||
}
|
||||
|
||||
const manifest: TenantSnapshotManifest = {
|
||||
schemaVersion: TENANT_MANIFEST_VERSION,
|
||||
snapshotId,
|
||||
sourceTenantId: opts.tenantId,
|
||||
createdAt,
|
||||
creator: { cliVersion: "0.0.0", host: hostname(), actorUserId: opts.actorUserId },
|
||||
anonymization: { profile: opts.profile, saltHash: hashSalt(salt) },
|
||||
models: modelEntries,
|
||||
...(globalsEntries ? { globals: globalsEntries } : {}),
|
||||
...(storageEntry ? { storage: storageEntry } : {}),
|
||||
encryption: opts.encryption
|
||||
? { method: opts.encryption.method, recipient: opts.encryption.recipient }
|
||||
: { method: "none" },
|
||||
};
|
||||
const manifestPath = join(stageDir, "manifest.json");
|
||||
await writeFile(manifestPath, serializeTenantManifest(manifest));
|
||||
|
||||
const archivePath = join(tempRoot, `${snapshotId}.tar.gz`);
|
||||
logger.info(`Packaging ${archivePath}`);
|
||||
await createTar({ gzip: true, file: archivePath, cwd: tempRoot, portable: true }, [snapshotId]);
|
||||
|
||||
let outputPath = archivePath;
|
||||
if (opts.encryption && opts.encryption.method !== "none") {
|
||||
const encryptedPath = `${archivePath}.${opts.encryption.method}`;
|
||||
logger.info(`Encrypting via ${opts.encryption.method}`);
|
||||
await encryptFile({ config: opts.encryption, inputPath: archivePath, outputPath: encryptedPath });
|
||||
outputPath = encryptedPath;
|
||||
}
|
||||
|
||||
let uploaded: { bucket: string; key: string } | undefined;
|
||||
if (opts.output.kind === "s3") {
|
||||
const s3 = ensureS3Client(opts);
|
||||
logger.info(`Uploading to s3://${opts.output.bucket}/${opts.output.key}`);
|
||||
await uploadFileMultipart({
|
||||
client: s3,
|
||||
bucket: opts.output.bucket,
|
||||
key: opts.output.key,
|
||||
filePath: outputPath,
|
||||
contentType: "application/gzip",
|
||||
metadata: {
|
||||
"snapshot-id": snapshotId,
|
||||
"source-tenant-id": opts.tenantId,
|
||||
profile: opts.profile,
|
||||
encryption: opts.encryption?.method ?? "none",
|
||||
"manifest-version": String(TENANT_MANIFEST_VERSION),
|
||||
},
|
||||
});
|
||||
uploaded = { bucket: opts.output.bucket, key: opts.output.key };
|
||||
} else {
|
||||
const { copyFile } = await import("node:fs/promises");
|
||||
await copyFile(outputPath, opts.output.path);
|
||||
}
|
||||
|
||||
await writeSnapshotAudit({
|
||||
entry: {
|
||||
event: "TENANT_SNAPSHOT_CREATED",
|
||||
snapshotId,
|
||||
sourceTenantId: opts.tenantId,
|
||||
profile: opts.profile,
|
||||
actorUserId: opts.actorUserId ?? null,
|
||||
host: hostname(),
|
||||
createdAt,
|
||||
outputTarget:
|
||||
opts.output.kind === "s3"
|
||||
? `s3://${opts.output.bucket}/${opts.output.key}`
|
||||
: opts.output.path,
|
||||
superAdminConfirmed: Boolean(opts.superAdminConfirmed),
|
||||
},
|
||||
filePath: opts.auditFilePath,
|
||||
stdout: opts.auditStdout,
|
||||
});
|
||||
|
||||
return { snapshotId, manifest, archivePath: outputPath, uploaded };
|
||||
} finally {
|
||||
await rm(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
interface WriteModelJsonlOptions {
|
||||
prisma: PrismaRunner;
|
||||
modelName: string;
|
||||
tenantField: string;
|
||||
tenantId: string;
|
||||
pageSize: number;
|
||||
profile: (typeof PROFILES)[ProfileName];
|
||||
ctx: AnonymizationContext;
|
||||
outPath: string;
|
||||
}
|
||||
|
||||
async function writeModelJsonl(
|
||||
opts: WriteModelJsonlOptions,
|
||||
): Promise<{ rowCount: number; sizeBytes: number; sha256: string }> {
|
||||
const stream = createWriteStream(opts.outPath, { encoding: "utf8" });
|
||||
const hash = createHash("sha256");
|
||||
let rowCount = 0;
|
||||
|
||||
for await (const row of opts.prisma.findManyByTenant(
|
||||
opts.modelName,
|
||||
opts.tenantField,
|
||||
opts.tenantId,
|
||||
opts.pageSize,
|
||||
)) {
|
||||
const anonymized = anonymizeRow(opts.profile, opts.modelName, row, opts.ctx);
|
||||
const line = `${JSON.stringify(anonymized, snapshotReplacer)}\n`;
|
||||
hash.update(line);
|
||||
rowCount += 1;
|
||||
if (!stream.write(line)) {
|
||||
await new Promise<void>((resolve) => stream.once("drain", () => resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
stream.end((err: Error | null | undefined) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
const s = await stat(opts.outPath);
|
||||
return { rowCount, sizeBytes: s.size, sha256: hash.digest("hex") };
|
||||
}
|
||||
|
||||
interface ExtractStorageOptions {
|
||||
client: S3Client;
|
||||
bucket: string;
|
||||
tenantPrefix: string;
|
||||
outDir: string;
|
||||
}
|
||||
|
||||
interface ExtractStorageResult {
|
||||
objectCount: number;
|
||||
totalBytes: number;
|
||||
checksum: string;
|
||||
}
|
||||
|
||||
export async function extractTenantStorage(opts: ExtractStorageOptions): Promise<ExtractStorageResult> {
|
||||
const stagingDir = join(opts.outDir, "_staging");
|
||||
await mkdir(stagingDir, { recursive: true });
|
||||
const manifestPath = join(opts.outDir, "manifest.jsonl");
|
||||
const archivePath = join(opts.outDir, "objects.tar");
|
||||
|
||||
const entries: Array<{ key: string; relPath: string; size: number; etag: string | null }> = [];
|
||||
let totalBytes = 0;
|
||||
let continuation: string | undefined;
|
||||
do {
|
||||
const page = await opts.client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: opts.bucket,
|
||||
Prefix: opts.tenantPrefix,
|
||||
ContinuationToken: continuation,
|
||||
}),
|
||||
);
|
||||
for (const obj of page.Contents ?? []) {
|
||||
if (!obj.Key) continue;
|
||||
const rel = obj.Key.replace(/^\/+/u, "");
|
||||
const stagedAbs = join(stagingDir, rel);
|
||||
await mkdir(dirnameOf(stagedAbs), { recursive: true });
|
||||
const getResp = await opts.client.send(new GetObjectCommand({ Bucket: opts.bucket, Key: obj.Key }));
|
||||
await streamToFile(asReadable(getResp.Body), stagedAbs);
|
||||
entries.push({ key: obj.Key, relPath: rel, size: obj.Size ?? 0, etag: obj.ETag ?? null });
|
||||
totalBytes += obj.Size ?? 0;
|
||||
}
|
||||
continuation = page.IsTruncated ? page.NextContinuationToken : undefined;
|
||||
} while (continuation);
|
||||
|
||||
const manifestLines = entries.map((e) => JSON.stringify(e)).join("\n");
|
||||
await writeFile(manifestPath, manifestLines.length === 0 ? "" : `${manifestLines}\n`);
|
||||
|
||||
if (entries.length === 0) {
|
||||
const { open } = await import("node:fs/promises");
|
||||
const fh = await open(archivePath, "w");
|
||||
await fh.close();
|
||||
} else {
|
||||
await createTar(
|
||||
{ file: archivePath, cwd: stagingDir, portable: true },
|
||||
entries.map((e) => e.relPath),
|
||||
);
|
||||
}
|
||||
const checksum = await sha256File(archivePath);
|
||||
try {
|
||||
await rm(stagingDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return { objectCount: entries.length, totalBytes, checksum };
|
||||
}
|
||||
|
||||
function ensureS3Client(opts: RunSnapshotOptions): S3Client {
|
||||
if (opts.s3Client) return opts.s3Client;
|
||||
if (!opts.s3) {
|
||||
throw new Error("S3 client or config is required for this operation");
|
||||
}
|
||||
return new S3Client(opts.s3);
|
||||
}
|
||||
|
||||
function dirnameOf(p: string): string {
|
||||
const idx = Math.max(p.lastIndexOf("/"), p.lastIndexOf("\\"));
|
||||
return idx === -1 ? "." : p.slice(0, idx);
|
||||
}
|
||||
|
||||
function asReadable(body: unknown): Readable {
|
||||
if (body && typeof (body as Readable).pipe === "function") {
|
||||
return body as Readable;
|
||||
}
|
||||
if (body && typeof (body as { transformToWebStream?: () => unknown }).transformToWebStream === "function") {
|
||||
const webStream = (body as { transformToWebStream: () => ReadableStream<Uint8Array> }).transformToWebStream();
|
||||
return Readable.fromWeb(webStream as never);
|
||||
}
|
||||
if (body && Symbol.asyncIterator in (body as object)) {
|
||||
return Readable.from(body as AsyncIterable<Buffer>);
|
||||
}
|
||||
throw new Error("Unsupported S3 GetObject body type");
|
||||
}
|
||||
|
||||
async function streamToFile(stream: Readable, path: string): Promise<void> {
|
||||
const sink = createWriteStream(path);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
stream.pipe(sink);
|
||||
sink.on("finish", () => resolve());
|
||||
sink.on("error", reject);
|
||||
stream.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function sha256File(path: string): Promise<string> {
|
||||
const hash = createHash("sha256");
|
||||
const stream = createReadStream(path);
|
||||
for await (const chunk of stream) {
|
||||
hash.update(chunk as Buffer);
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
export const tenantSnapshotCommand = defineCommand({
|
||||
meta: {
|
||||
name: "snapshot",
|
||||
description: "Create a PII-anonymized snapshot of a single tenant and upload to S3.",
|
||||
},
|
||||
args: {
|
||||
"tenant-id": { type: "string", required: true, description: "Source tenant id." },
|
||||
profile: {
|
||||
type: "string",
|
||||
default: "full-anonymized",
|
||||
description: "Anonymization profile: full-anonymized | minimal-anonymized | none.",
|
||||
},
|
||||
output: { type: "string", description: "Local file path or s3://bucket/key destination." },
|
||||
"i-am-the-super-admin": {
|
||||
type: "boolean",
|
||||
default: false,
|
||||
description: "Required confirmation flag when --profile=none.",
|
||||
},
|
||||
"audit-file": {
|
||||
type: "string",
|
||||
description: "Append the structured TENANT_SNAPSHOT_CREATED record to this file.",
|
||||
},
|
||||
"page-size": {
|
||||
type: "string",
|
||||
description: "Prisma findMany page size (default 1000). Tune for large tenants.",
|
||||
},
|
||||
"include-globals": {
|
||||
type: "boolean",
|
||||
default: true,
|
||||
description:
|
||||
"Extract referenced global rows (User, etc.) into globals/<Model>.jsonl so restore works against an empty target DB. Disable for legacy tenant-only snapshots.",
|
||||
},
|
||||
},
|
||||
async run({ args }) {
|
||||
const profile = String(args.profile);
|
||||
if (!isProfileName(profile)) {
|
||||
throw new Error(`Invalid --profile: ${profile}; expected full-anonymized | minimal-anonymized | none.`);
|
||||
}
|
||||
assertSuperAdminForNoneProfile(profile, Boolean(args["i-am-the-super-admin"]));
|
||||
|
||||
const tenantId = String(args["tenant-id"]);
|
||||
const outputRaw = args.output === undefined ? undefined : String(args.output);
|
||||
const defaultLocal = `./tenant-snapshot-${tenantId}-${new Date().toISOString().replace(/[:.]/gu, "")}.tar.gz`;
|
||||
const output = parseOutputTarget(outputRaw, defaultLocal);
|
||||
|
||||
const { resolveTenantSnapshotConfig } = await import("./env.js");
|
||||
const cfg = resolveTenantSnapshotConfig(process.env as Record<string, string | undefined>);
|
||||
|
||||
const { getDbClient } = await import("@toolctrl/db/client.server");
|
||||
const { createPrismaSnapshotRunner } = await import("./prisma-runner.js");
|
||||
const prisma = getDbClient();
|
||||
const runner = createPrismaSnapshotRunner(prisma as unknown as Record<string, unknown>);
|
||||
|
||||
const pageSize = args["page-size"] ? Number.parseInt(String(args["page-size"]), 10) : undefined;
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId,
|
||||
profile,
|
||||
output,
|
||||
prisma: runner,
|
||||
s3: cfg.s3,
|
||||
storageBucket: cfg.appStorageBucket ?? undefined,
|
||||
storagePrefix: cfg.storagePrefix,
|
||||
encryption: cfg.encryption.method === "none" ? undefined : cfg.encryption,
|
||||
superAdminConfirmed: Boolean(args["i-am-the-super-admin"]),
|
||||
auditFilePath: args["audit-file"] ? String(args["audit-file"]) : undefined,
|
||||
pageSize: pageSize && Number.isFinite(pageSize) && pageSize > 0 ? pageSize : undefined,
|
||||
includeGlobals: args["include-globals"] !== false,
|
||||
});
|
||||
consola.success(
|
||||
`Snapshot ${result.snapshotId} written to ${result.uploaded ? `s3://${result.uploaded.bucket}/${result.uploaded.key}` : result.archivePath}`,
|
||||
);
|
||||
consola.info(
|
||||
`Models: ${result.manifest.models.length} · rows: ${result.manifest.models.reduce((n, m) => n + m.rowCount, 0)} · profile: ${profile}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user