Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
+466
@@ -0,0 +1,466 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { mockClient } from "aws-sdk-client-mock";
|
||||
import { GetObjectCommand, ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
|
||||
import { Readable } from "node:stream";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import {
|
||||
parseOutputTarget,
|
||||
runTenantSnapshot,
|
||||
type PrismaRunner,
|
||||
} from "./snapshot.js";
|
||||
import { TENANT_SCOPED_MODELS } from "./models.js";
|
||||
|
||||
class FakePrisma implements PrismaRunner {
|
||||
constructor(
|
||||
private readonly rows: Record<string, Record<string, unknown>[]>,
|
||||
private readonly globals: Record<string, Record<string, unknown>[]> = {},
|
||||
) {}
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
async *findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>> {
|
||||
void tenantField;
|
||||
void tenantId;
|
||||
void pageSize;
|
||||
const list = this.rows[modelName] ?? [];
|
||||
for (const r of list) {
|
||||
yield r;
|
||||
}
|
||||
}
|
||||
async findManyByIds(modelName: string, ids: readonly string[]): Promise<Record<string, unknown>[]> {
|
||||
const list = this.globals[modelName] ?? [];
|
||||
return list.filter((r) => typeof r.id === "string" && ids.includes(r.id as string));
|
||||
}
|
||||
}
|
||||
|
||||
class RecordingPrisma extends FakePrisma {
|
||||
readonly pageSizes: number[] = [];
|
||||
|
||||
override async *findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>> {
|
||||
this.pageSizes.push(pageSize);
|
||||
yield* super.findManyByTenant(modelName, tenantField, tenantId, pageSize);
|
||||
}
|
||||
}
|
||||
|
||||
describe("parseOutputTarget", () => {
|
||||
it("falls back to default local path when undefined", () => {
|
||||
expect(parseOutputTarget(undefined, "/tmp/x.tar.gz")).toEqual({
|
||||
kind: "local",
|
||||
path: "/tmp/x.tar.gz",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats unprefixed paths as local files", () => {
|
||||
expect(parseOutputTarget("/snapshots/a.tar.gz", "/tmp/x")).toEqual({
|
||||
kind: "local",
|
||||
path: "/snapshots/a.tar.gz",
|
||||
});
|
||||
});
|
||||
|
||||
it("parses s3://bucket/key targets", () => {
|
||||
expect(parseOutputTarget("s3://my-bucket/path/to/snap.tar.gz", "")).toEqual({
|
||||
kind: "s3",
|
||||
bucket: "my-bucket",
|
||||
key: "path/to/snap.tar.gz",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed s3:// targets", () => {
|
||||
expect(() => parseOutputTarget("s3://bucket-only", "")).toThrow(/Invalid --output/u);
|
||||
expect(() => parseOutputTarget("s3://bucket/", "")).toThrow(/Invalid --output/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runTenantSnapshot", () => {
|
||||
let workdir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), "tenant-snapshot-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("writes a manifest with row counts per model and the chosen profile", async () => {
|
||||
const prisma = new FakePrisma({
|
||||
TenantUser: [
|
||||
{ id: "tu1", tenantId: "ten_a", userId: "u1" },
|
||||
{ id: "tu2", tenantId: "ten_a", userId: "u2" },
|
||||
],
|
||||
TenantUserInvitation: [
|
||||
{ id: "inv1", tenantId: "ten_a", email: "alice@example.com", firstName: "Alice", lastName: "X" },
|
||||
],
|
||||
});
|
||||
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.snapshotId).toMatch(/^tenant-snapshot-ten_a-/u);
|
||||
expect(result.manifest.anonymization.profile).toBe("full-anonymized");
|
||||
expect(result.manifest.anonymization.saltHash).toMatch(/^[a-f0-9]{64}$/u);
|
||||
expect(result.manifest.sourceTenantId).toBe("ten_a");
|
||||
const tuEntry = result.manifest.models.find((m) => m.model === "TenantUser");
|
||||
expect(tuEntry?.rowCount).toBe(2);
|
||||
const invEntry = result.manifest.models.find((m) => m.model === "TenantUserInvitation");
|
||||
expect(invEntry?.rowCount).toBe(1);
|
||||
// every other model should be present with rowCount=0.
|
||||
expect(result.manifest.models).toHaveLength(TENANT_SCOPED_MODELS.length);
|
||||
for (const m of result.manifest.models) {
|
||||
if (m.model !== "TenantUser" && m.model !== "TenantUserInvitation") {
|
||||
expect(m.rowCount).toBe(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("requires explicit super-admin confirmation for the none anonymization profile", async () => {
|
||||
await expect(
|
||||
runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "none",
|
||||
prisma: new FakePrisma({}),
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/super-admin/iu);
|
||||
});
|
||||
|
||||
it("records super-admin confirmation in the audit entry for profile none", async () => {
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const auditPath = join(workdir, "audit.jsonl");
|
||||
const fixedNow = new Date("2026-05-25T12:00:00.000Z");
|
||||
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "none",
|
||||
prisma: new FakePrisma({ TenantUser: [{ id: "tu1", tenantId: "ten_a", userId: "u1" }] }),
|
||||
output: { kind: "local", path: outPath },
|
||||
superAdminConfirmed: true,
|
||||
auditFilePath: auditPath,
|
||||
actorUserId: "user_admin",
|
||||
now: () => fixedNow,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.manifest.anonymization.profile).toBe("none");
|
||||
const audit = JSON.parse((await readFile(auditPath, "utf8")).trim()) as {
|
||||
event: string;
|
||||
sourceTenantId: string;
|
||||
actorUserId: string;
|
||||
createdAt: string;
|
||||
outputTarget: string;
|
||||
superAdminConfirmed: boolean;
|
||||
};
|
||||
expect(audit).toMatchObject({
|
||||
event: "TENANT_SNAPSHOT_CREATED",
|
||||
sourceTenantId: "ten_a",
|
||||
actorUserId: "user_admin",
|
||||
createdAt: "2026-05-25T12:00:00.000Z",
|
||||
outputTarget: outPath,
|
||||
superAdminConfirmed: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("passes the configured page size to every tenant-scoped model query", async () => {
|
||||
const prisma = new RecordingPrisma({});
|
||||
await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
pageSize: 17,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(prisma.pageSizes).toHaveLength(TENANT_SCOPED_MODELS.length);
|
||||
expect(new Set(prisma.pageSizes)).toEqual(new Set([17]));
|
||||
});
|
||||
|
||||
it("anonymizes PII inline so the snapshot file contains no raw emails", async () => {
|
||||
const prisma = new FakePrisma({
|
||||
TenantUserInvitation: [
|
||||
{ id: "inv1", tenantId: "ten_a", email: "real-person@example.org", firstName: "Real", lastName: "Person" },
|
||||
],
|
||||
});
|
||||
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
// Untar locally and read the model file.
|
||||
const { mkdtemp: mk } = await import("node:fs/promises");
|
||||
const { extract } = await import("tar");
|
||||
const untarDir = await mk(join(tmpdir(), "tenant-snap-untar-"));
|
||||
try {
|
||||
await extract({ file: outPath, cwd: untarDir });
|
||||
const modelFile = join(untarDir, result.snapshotId, "postgres", "TenantUserInvitation.jsonl");
|
||||
const content = await readFile(modelFile, "utf8");
|
||||
expect(content).not.toContain("real-person@example.org");
|
||||
expect(content).toMatch(/@anonymized\.test/u);
|
||||
} finally {
|
||||
await rm(untarDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("uploads to S3 when --output is s3://bucket/key", async () => {
|
||||
const prisma = new FakePrisma({});
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake(() => ({ Contents: [], IsTruncated: false }));
|
||||
const fakeUpload = vi.fn().mockImplementation(() => ({
|
||||
done: vi.fn().mockResolvedValue({ ETag: '"deadbeef"' }),
|
||||
}));
|
||||
// Monkey-patch the Upload constructor via the global hook in the source.
|
||||
// We use the lib-storage Upload import inside s3-upload.ts; the helper
|
||||
// accepts an `uploadCtor` override but runTenantSnapshot does not expose it.
|
||||
// Instead we stub Upload.prototype.done.
|
||||
const origDone = Upload.prototype.done;
|
||||
Upload.prototype.done = vi.fn().mockResolvedValue({ ETag: '"deadbeef"' }) as never;
|
||||
try {
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "minimal-anonymized",
|
||||
prisma,
|
||||
output: { kind: "s3", bucket: "snapshots", key: "ten_a/snap.tar.gz" },
|
||||
s3Client: s3Mock as unknown as S3Client,
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.uploaded).toEqual({ bucket: "snapshots", key: "ten_a/snap.tar.gz" });
|
||||
expect(result.manifest.storage).toBeDefined();
|
||||
expect(result.manifest.storage?.objectCount).toBe(0);
|
||||
} finally {
|
||||
Upload.prototype.done = origDone;
|
||||
void fakeUpload;
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects storage snapshots without an S3 client or config", async () => {
|
||||
await expect(
|
||||
runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: new FakePrisma({}),
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
}),
|
||||
).rejects.toThrow(/S3 client or config is required/u);
|
||||
});
|
||||
|
||||
it("includes tenant storage objects when bucket is configured", async () => {
|
||||
const prisma = new FakePrisma({});
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake(() => ({
|
||||
Contents: [
|
||||
{ Key: "tenants/ten_a/avatars/me.png", Size: 17, ETag: '"abc"' },
|
||||
{ Key: "tenants/ten_a/uploads/file.txt", Size: 5, ETag: '"def"' },
|
||||
],
|
||||
IsTruncated: false,
|
||||
}));
|
||||
s3Mock.on(GetObjectCommand as never).callsFake(() => ({
|
||||
Body: Readable.from([Buffer.from("xx")]),
|
||||
}));
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
s3Client: s3Mock as unknown as S3Client,
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.manifest.storage?.objectCount).toBe(2);
|
||||
});
|
||||
|
||||
it("paginates tenant storage extraction and ignores objects without keys", async () => {
|
||||
const prisma = new FakePrisma({});
|
||||
const s3Mock = mockClient(S3Client as never);
|
||||
s3Mock.on(ListObjectsV2Command as never).callsFake((input: { ContinuationToken?: string }) => {
|
||||
if (!input.ContinuationToken) {
|
||||
return {
|
||||
Contents: [
|
||||
{ Size: 999, ETag: '"missing-key"' },
|
||||
{ Key: "tenants/ten_a/avatars/me.png", Size: 17, ETag: '"abc"' },
|
||||
],
|
||||
IsTruncated: true,
|
||||
NextContinuationToken: "page-2",
|
||||
};
|
||||
}
|
||||
return {
|
||||
Contents: [{ Key: "tenants/ten_a/uploads/file.txt", Size: 5, ETag: '"def"' }],
|
||||
IsTruncated: false,
|
||||
};
|
||||
});
|
||||
s3Mock.on(GetObjectCommand as never).callsFake(() => ({
|
||||
Body: Readable.from([Buffer.from("xx")]),
|
||||
}));
|
||||
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: join(workdir, "snap.tar.gz") },
|
||||
s3Client: s3Mock as unknown as S3Client,
|
||||
storageBucket: "app-storage",
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.manifest.storage?.objectCount).toBe(2);
|
||||
expect(result.manifest.storage?.sizeBytes).toBe(22);
|
||||
const listCalls = s3Mock.commandCalls(ListObjectsV2Command as never);
|
||||
expect(listCalls).toHaveLength(2);
|
||||
expect(listCalls[1].args[0].input).toMatchObject({ ContinuationToken: "page-2" });
|
||||
expect(s3Mock.commandCalls(GetObjectCommand as never)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("emits globals/User.jsonl with referenced + anonymized user rows", async () => {
|
||||
const prisma = new FakePrisma(
|
||||
{
|
||||
TenantUser: [
|
||||
{ id: "tu1", tenantId: "ten_a", userId: "u1", type: 1, joined: 1, status: 1 },
|
||||
{ id: "tu2", tenantId: "ten_a", userId: "u2", type: 1, joined: 1, status: 1 },
|
||||
],
|
||||
},
|
||||
{
|
||||
User: [
|
||||
{ id: "u1", email: "first@real.com", firstName: "First", lastName: "Real", phone: "+10001" },
|
||||
{ id: "u2", email: "second@real.com", firstName: "Second", lastName: "Real", phone: "+10002" },
|
||||
{ id: "u_unused", email: "unused@real.com", firstName: "Unused", lastName: "U" },
|
||||
],
|
||||
},
|
||||
);
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(result.manifest.globals).toBeDefined();
|
||||
const userEntry = result.manifest.globals?.find((g) => g.model === "User");
|
||||
expect(userEntry?.rowCount).toBe(2);
|
||||
expect(userEntry?.path).toBe("globals/User.jsonl");
|
||||
|
||||
const { extract } = await import("tar");
|
||||
const untar = await mkdtemp(join(tmpdir(), "snap-globals-untar-"));
|
||||
try {
|
||||
await extract({ file: outPath, cwd: untar });
|
||||
const text = await readFile(join(untar, result.snapshotId, "globals", "User.jsonl"), "utf8");
|
||||
expect(text).not.toContain("first@real.com");
|
||||
expect(text).not.toContain("second@real.com");
|
||||
expect(text).not.toContain("unused@real.com");
|
||||
const lines = text.trim().split("\n").map((l) => JSON.parse(l));
|
||||
expect(lines.length).toBe(2);
|
||||
const ids = new Set(lines.map((l) => l.id));
|
||||
expect(ids).toEqual(new Set(["u1", "u2"]));
|
||||
for (const u of lines) {
|
||||
expect(u.email).toMatch(/@anonymized\.test$/);
|
||||
// PII columns scrubbed.
|
||||
expect(u.firstName).not.toMatch(/^(First|Second)$/);
|
||||
expect(u.phone).toBeNull();
|
||||
}
|
||||
} finally {
|
||||
await rm(untar, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("skips globals extraction when include-globals=false", async () => {
|
||||
const prisma = new FakePrisma(
|
||||
{ TenantUser: [{ id: "tu1", tenantId: "ten_a", userId: "u1", type: 1, joined: 1, status: 1 }] },
|
||||
{ User: [{ id: "u1", email: "a@b.io", firstName: "A", lastName: "B" }] },
|
||||
);
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
includeGlobals: false,
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
expect(result.manifest.globals).toBeUndefined();
|
||||
});
|
||||
|
||||
it("warns and skips globals when the runner has no findManyByIds method", async () => {
|
||||
// A minimal runner without findManyByIds — simulates a legacy adapter.
|
||||
class LegacyRunner implements PrismaRunner {
|
||||
async *findManyByTenant(
|
||||
modelName: string,
|
||||
tenantField: string,
|
||||
tenantId: string,
|
||||
pageSize: number,
|
||||
): AsyncIterable<Record<string, unknown>> {
|
||||
void modelName;
|
||||
void tenantField;
|
||||
void tenantId;
|
||||
void pageSize;
|
||||
yield* [];
|
||||
}
|
||||
}
|
||||
const warn = vi.fn();
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma: new LegacyRunner(),
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn },
|
||||
});
|
||||
expect(result.manifest.globals).toBeUndefined();
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("findManyByIds"));
|
||||
});
|
||||
|
||||
it("preserves anonymization-FK consistency across calls (same salt seed inside one run)", async () => {
|
||||
const prisma = new FakePrisma({
|
||||
TenantUserInvitation: [
|
||||
{ id: "1", tenantId: "ten_a", email: "alice@example.com", firstName: "Alice", lastName: "Z" },
|
||||
{ id: "2", tenantId: "ten_a", email: "alice@example.com", firstName: "Alice", lastName: "Z" },
|
||||
],
|
||||
});
|
||||
const outPath = join(workdir, "snap.tar.gz");
|
||||
const result = await runTenantSnapshot({
|
||||
tenantId: "ten_a",
|
||||
profile: "full-anonymized",
|
||||
prisma,
|
||||
output: { kind: "local", path: outPath },
|
||||
logger: { info: vi.fn(), warn: vi.fn() },
|
||||
});
|
||||
|
||||
const { extract } = await import("tar");
|
||||
const untarDir = await mkdtemp(join(tmpdir(), "tenant-snap-untar-"));
|
||||
try {
|
||||
await extract({ file: outPath, cwd: untarDir });
|
||||
const file = join(untarDir, result.snapshotId, "postgres", "TenantUserInvitation.jsonl");
|
||||
const lines = (await readFile(file, "utf8")).trim().split("\n").map((l) => JSON.parse(l));
|
||||
// Both rows have the same original email → must have the same anonymized email.
|
||||
expect(lines[0].email).toBe(lines[1].email);
|
||||
} finally {
|
||||
await rm(untarDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user