import { createHash } from "node:crypto"; import type { PlatformLeadImportInput } from "@hivegrid/db/platform-leads.server"; import { describe, expect, it, vi } from "vitest"; import { isSuccessfulPlatformLeadBackfillManifest, parsePlatformLeadBackfillInput, PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE, runPlatformLeadBackfill, type PlatformLeadLegacyStore, } from "./backfill.js"; function jsonl(...records: unknown[]): Buffer { return Buffer.from(records.map((record) => JSON.stringify(record)).join("\n"), "utf8"); } function parse(store: PlatformLeadLegacyStore, ...records: unknown[]) { return parsePlatformLeadBackfillInput({ legacyStore: store, format: "jsonl", bytes: jsonl(...records) }); } function clock(...values: string[]): () => Date { const dates = values.map((value) => new Date(value)); return () => dates.shift() ?? new Date("2026-07-16T12:00:00.000Z"); } describe("platform lead legacy backfill mapping", () => { it("maps lead-inbox rows with deterministic PII-safe identity, attribution and original capture time", () => { const result = parse("lead-inbox-postgres", { id: "legacy-row-1", site: "hivegrid.de", name: "Ada Lovelace", email: "ada@example.test", company: "Analytical Engines", phone: "+49 30 123456", message: "Bitte melden", source: "demo-form", meta: { utm_source: "messe", landing_page: "https://hivegrid.de/demo", form_id: "demo-request", consent: true }, created_at: "2026-03-10T09:30:00.000Z", }); expect(result.sourceKey).toBe("lead-inbox-legacy"); expect(result.scanned).toBe(1); expect(result.rejectionCounts).toEqual({}); expect(result.accepted[0]).toMatchObject({ legacyId: "legacy-row-1", capturedAt: new Date("2026-03-10T09:30:00.000Z"), payload: { firstName: "Ada", lastName: "Lovelace", email: "ada@example.test", sourceDetail: "demo-form", utmSource: "messe", landingPage: "https://hivegrid.de/demo", metadata: { legacyStore: "lead-inbox-postgres", formId: "demo-request", consent: true }, }, }); expect(result.accepted[0]?.payload.externalId).toMatch(/^legacy:lead-inbox-postgres:[a-f0-9]{64}$/u); expect(JSON.stringify(result.accepted[0]?.payload.metadata)).not.toContain("legacy-row-1"); }); it.each(["hivegrid-website-production-sqlite", "hivegrid-website-staging-sqlite"] as const)( "maps %s rows to the configured website source", (legacyStore) => { const result = parse(legacyStore, { id: 17, email: "website@example.test", name: "Grace Hopper", company: "Compiler GmbH", message: "Demo", created_at: "2026-03-11T08:00:00.000Z", }); expect(result.sourceKey).toBe("hivegrid-website"); expect(result.accepted[0]).toMatchObject({ legacyId: "17", capturedAt: new Date("2026-03-11T08:00:00.000Z"), payload: { firstName: "Grace", lastName: "Hopper", sourceDetail: "legacy-website-sqlite" }, }); } ); it("preserves native legacy status and maps supported Cloud activities with original timestamps", () => { const result = parse("hivegrid-cloud-legacy", { id: "cloud-lead-1", email: "cloud@example.test", firstName: "Katherine", lastName: "Johnson", source: "landing-pro", status: "qualified", firstSeenAt: "2026-03-01T10:00:00.000Z", utmSource: "partner", events: [ { id: "event-1", type: "re-captured", actorId: null, payload: { source: "pricing-form" }, createdAt: "2026-03-02T10:00:00.000Z", }, { id: "event-2", type: "status-changed", actorId: "legacy-operator", payload: { from: "new", to: "qualified" }, createdAt: "2026-03-03T10:00:00.000Z", }, ], }); expect(result.accepted[0]).toMatchObject({ initialStatus: "qualified", capturedAt: new Date("2026-03-01T10:00:00.000Z"), legacyActivities: [ { externalActivityId: "event-1", occurredAt: new Date("2026-03-02T10:00:00.000Z"), type: "captured", payload: { source: "pricing-form", legacyType: "re-captured" }, }, { externalActivityId: "event-2", occurredAt: new Date("2026-03-03T10:00:00.000Z"), type: "status_changed", payload: { from: "new", to: "qualified", legacyActorId: "legacy-operator" }, }, ], }); }); it.each([ ["trial", "qualified"], ["customer", "qualified"], ["churned", "disqualified"], ] as const)("maps legacy sales status %s without losing its original value", (legacyStatus, expectedStatus) => { const result = parse("hivegrid-cloud-legacy", { id: `legacy-${legacyStatus}`, email: `${legacyStatus}@example.test`, status: legacyStatus, firstSeenAt: "2026-03-01T10:00:00.000Z", events: [], }); expect(result.rejectionCounts).toEqual({}); expect(result.accepted[0]).toMatchObject({ initialStatus: expectedStatus, payload: { metadata: { legacyStatus } }, }); }); it("preserves score, conversion and legacy note history as typed import activities", () => { const result = parse( "hivegrid-cloud-legacy", { id: "sales-history", email: "one@example.test", status: "customer", firstSeenAt: "2026-03-01T10:00:00.000Z", notes: { body: "Aktuell\nmit Umbruch", updatedAt: "2026-03-05T10:00:00.000Z", history: [{ body: "Erste Notiz", at: "2026-03-02T08:00:00.000Z" }], }, events: [ { id: "score", type: "score-changed", payload: { from: 1, to: 2 }, createdAt: "2026-03-02T10:00:00.000Z" }, { id: "converted", type: "converted", payload: { tenantId: "legacy-tenant" }, createdAt: "2026-03-03T10:00:00.000Z" }, ], } ); expect(result.rejectionCounts).toEqual({}); expect(result.accepted[0]).toMatchObject({ initialStatus: "qualified", payload: { metadata: { legacyStatus: "customer" } }, legacyActivities: [ { externalActivityId: "score", type: "legacy_event", payload: { from: 1, to: 2, legacyType: "score-changed" } }, { externalActivityId: "converted", type: "legacy_event", payload: { tenantId: "legacy-tenant", legacyType: "converted" } }, { externalActivityId: "sales-history:notes:history:0", type: "note_added", payload: { note: "Erste Notiz", legacyType: "lead.notes.history" } }, { externalActivityId: "sales-history:notes:current", type: "note_added", payload: { note: "Aktuell\nmit Umbruch", legacyType: "lead.notes.current" } }, ], }); }); it("counts malformed JSON and duplicate legacy identities without returning row data", () => { const valid = JSON.stringify({ id: "duplicate", email: "duplicate@example.test", created_at: "2026-03-01T10:00:00.000Z", }); const bytes = Buffer.from(`${valid}\nnot-json\n${valid}`, "utf8"); const result = parsePlatformLeadBackfillInput({ legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes }); expect(result.scanned).toBe(3); expect(result.accepted).toHaveLength(1); expect(result.rejectionCounts).toEqual({ INVALID_JSON: 1, DUPLICATE_LEGACY_ID: 1 }); }); it("rejects unsupported backfill formats before parsing input", () => { expect(() => parsePlatformLeadBackfillInput({ legacyStore: "hivegrid-cloud-legacy", format: "csv" as never, bytes: Buffer.from("id,email", "utf8"), }) ).toThrow(); }); }); describe("platform lead backfill execution and manifests", () => { const sourceRecord = { id: "website-row-1", email: "secret.person@example.test", name: "Secret Person", company: "Secret Company", message: "Secret message", created_at: "2026-03-04T11:00:00.000Z", }; it("produces a PII-free dry-run manifest with hash, counts and time range and performs no imports", async () => { const bytes = jsonl(sourceRecord); const manifest = await runPlatformLeadBackfill({ mode: "dry-run", legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes, runId: "dry-run-1", clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"), }); expect(manifest).toMatchObject({ mode: "dry-run", sourceKey: "hivegrid-website", outcome: "validated", counts: { scanned: 1, accepted: 1, rejected: 0, created: 0, replayed: 0, failed: 0 }, timeRange: { firstCapturedAt: "2026-03-04T11:00:00.000Z", lastCapturedAt: "2026-03-04T11:00:00.000Z" }, notificationsTriggered: 0, automationsTriggered: 0, }); expect(manifest.input.sha256).toBe(createHash("sha256").update(bytes).digest("hex")); const serialized = JSON.stringify(manifest); for (const pii of ["secret.person@example.test", "Secret Person", "Secret Company", "Secret message", "website-row-1"]) { expect(serialized).not.toContain(pii); } }); it("applies through the single import port while preserving timestamp and disabling notification/automation fan-out", async () => { const bytes = jsonl(sourceRecord); const dryRun = parsePlatformLeadBackfillInput({ legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes }); const calls: PlatformLeadImportInput[] = []; const importer = vi.fn(async (input: PlatformLeadImportInput) => { calls.push(input); return { outcome: "created" as const, lead: { id: "native-1", status: "new" as const, createdAt: input.capturedAt.toISOString() } }; }); const manifest = await runPlatformLeadBackfill({ mode: "apply", legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes, runId: "apply-1", sourceId: "source_website", actorAdminUserId: "operator_1", expectedInputSha256: dryRun.inputSha256, expectedAccepted: 1, importer, clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"), }); expect(importer).toHaveBeenCalledTimes(1); expect(calls[0]).toMatchObject({ channel: "import", sourceId: "source_website", actorAdminUserId: "operator_1", capturedAt: new Date("2026-03-04T11:00:00.000Z"), payload: { email: "secret.person@example.test" }, legacyActivities: [], }); expect(manifest).toMatchObject({ outcome: "applied", counts: { created: 1, replayed: 0, failed: 0 } }); expect(isSuccessfulPlatformLeadBackfillManifest(manifest)).toBe(true); }); it("blocks apply before writes when the expected input hash or count changed or any row is rejected", async () => { const bytes = Buffer.from(`${JSON.stringify(sourceRecord)}\ninvalid-json`, "utf8"); const importer = vi.fn(); const manifest = await runPlatformLeadBackfill({ mode: "apply", legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes, runId: "blocked-1", sourceId: "source_website", actorAdminUserId: "operator_1", expectedInputSha256: "0".repeat(64), expectedAccepted: 2, importer, clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"), }); expect(importer).not.toHaveBeenCalled(); expect(manifest.outcome).toBe("blocked"); expect(manifest.gateFailures).toEqual(["INPUT_HASH_MISMATCH", "ACCEPTED_COUNT_MISMATCH", "REJECTED_ROWS_PRESENT"]); }); it("proves replay only when no new row is created", async () => { const bytes = jsonl(sourceRecord); const hash = createHash("sha256").update(bytes).digest("hex"); const replayed = await runPlatformLeadBackfill({ mode: "replay", legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes, runId: "replay-1", sourceId: "source_website", actorAdminUserId: "operator_1", expectedInputSha256: hash, expectedAccepted: 1, importer: async () => ({ outcome: "replayed", lead: { id: "native-1", status: "new", createdAt: "2026-03-04T11:00:00.000Z" } }), clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"), }); expect(replayed).toMatchObject({ outcome: "replayed", counts: { created: 0, replayed: 1, failed: 0 }, gateFailures: [] }); const unexpectedCreate = await runPlatformLeadBackfill({ mode: "replay", legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes, runId: "replay-2", sourceId: "source_website", actorAdminUserId: "operator_1", expectedInputSha256: hash, expectedAccepted: 1, importer: async () => ({ outcome: "created", lead: { id: "native-2", status: "new", createdAt: "2026-03-04T11:00:00.000Z" } }), clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"), }); expect(unexpectedCreate).toMatchObject({ outcome: "failed", counts: { created: 1 }, gateFailures: ["REPLAY_CREATED_ROWS"] }); }); it("declares the exact source mapping for all four stores", () => { expect(PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE).toEqual({ "lead-inbox-postgres": "lead-inbox-legacy", "hivegrid-website-production-sqlite": "hivegrid-website", "hivegrid-website-staging-sqlite": "hivegrid-website", "hivegrid-cloud-legacy": "hivegrid-legacy-cloud", }); }); });