HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/scripts/platform-leads/backfill.test.ts

import { createHash } from "node:crypto";

import type { PlatformLeadImportInput } from "@hivegrid/db/platform-leads.server";
import { describe, expect, it, vi } from "vitest";

import {
  isSuccessfulPlatformLeadBackfillManifest,
  parsePlatformLeadBackfillInput,
  PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE,
  runPlatformLeadBackfill,
  type PlatformLeadLegacyStore,
} from "./backfill.js";

function jsonl(...records: unknown[]): Buffer {
  return Buffer.from(records.map((record) => JSON.stringify(record)).join("\n"), "utf8");
}

function parse(store: PlatformLeadLegacyStore, ...records: unknown[]) {
  return parsePlatformLeadBackfillInput({ legacyStore: store, format: "jsonl", bytes: jsonl(...records) });
}

function clock(...values: string[]): () => Date {
  const dates = values.map((value) => new Date(value));
  return () => dates.shift() ?? new Date("2026-07-16T12:00:00.000Z");
}

describe("platform lead legacy backfill mapping", () => {
  it("maps lead-inbox rows with deterministic PII-safe identity, attribution and original capture time", () => {
    const result = parse("lead-inbox-postgres", {
      id: "legacy-row-1",
      site: "hivegrid.de",
      name: "Ada Lovelace",
      email: "ada@example.test",
      company: "Analytical Engines",
      phone: "+49 30 123456",
      message: "Bitte melden",
      source: "demo-form",
      meta: { utm_source: "messe", landing_page: "https://hivegrid.de/demo", form_id: "demo-request", consent: true },
      created_at: "2026-03-10T09:30:00.000Z",
    });

    expect(result.sourceKey).toBe("lead-inbox-legacy");
    expect(result.scanned).toBe(1);
    expect(result.rejectionCounts).toEqual({});
    expect(result.accepted[0]).toMatchObject({
      legacyId: "legacy-row-1",
      capturedAt: new Date("2026-03-10T09:30:00.000Z"),
      payload: {
        firstName: "Ada",
        lastName: "Lovelace",
        email: "ada@example.test",
        sourceDetail: "demo-form",
        utmSource: "messe",
        landingPage: "https://hivegrid.de/demo",
        metadata: { legacyStore: "lead-inbox-postgres", formId: "demo-request", consent: true },
      },
    });
    expect(result.accepted[0]?.payload.externalId).toMatch(/^legacy:lead-inbox-postgres:[a-f0-9]{64}$/u);
    expect(JSON.stringify(result.accepted[0]?.payload.metadata)).not.toContain("legacy-row-1");
  });

  it.each(["hivegrid-website-production-sqlite", "hivegrid-website-staging-sqlite"] as const)(
    "maps %s rows to the configured website source",
    (legacyStore) => {
      const result = parse(legacyStore, {
        id: 17,
        email: "website@example.test",
        name: "Grace Hopper",
        company: "Compiler GmbH",
        message: "Demo",
        created_at: "2026-03-11T08:00:00.000Z",
      });
      expect(result.sourceKey).toBe("hivegrid-website");
      expect(result.accepted[0]).toMatchObject({
        legacyId: "17",
        capturedAt: new Date("2026-03-11T08:00:00.000Z"),
        payload: { firstName: "Grace", lastName: "Hopper", sourceDetail: "legacy-website-sqlite" },
      });
    }
  );

  it("preserves native legacy status and maps supported Cloud activities with original timestamps", () => {
    const result = parse("hivegrid-cloud-legacy", {
      id: "cloud-lead-1",
      email: "cloud@example.test",
      firstName: "Katherine",
      lastName: "Johnson",
      source: "landing-pro",
      status: "qualified",
      firstSeenAt: "2026-03-01T10:00:00.000Z",
      utmSource: "partner",
      events: [
        {
          id: "event-1",
          type: "re-captured",
          actorId: null,
          payload: { source: "pricing-form" },
          createdAt: "2026-03-02T10:00:00.000Z",
        },
        {
          id: "event-2",
          type: "status-changed",
          actorId: "legacy-operator",
          payload: { from: "new", to: "qualified" },
          createdAt: "2026-03-03T10:00:00.000Z",
        },
      ],
    });

    expect(result.accepted[0]).toMatchObject({
      initialStatus: "qualified",
      capturedAt: new Date("2026-03-01T10:00:00.000Z"),
      legacyActivities: [
        {
          externalActivityId: "event-1",
          occurredAt: new Date("2026-03-02T10:00:00.000Z"),
          type: "captured",
          payload: { source: "pricing-form", legacyType: "re-captured" },
        },
        {
          externalActivityId: "event-2",
          occurredAt: new Date("2026-03-03T10:00:00.000Z"),
          type: "status_changed",
          payload: { from: "new", to: "qualified", legacyActorId: "legacy-operator" },
        },
      ],
    });
  });

  it.each([
    ["trial", "qualified"],
    ["customer", "qualified"],
    ["churned", "disqualified"],
  ] as const)("maps legacy sales status %s without losing its original value", (legacyStatus, expectedStatus) => {
    const result = parse("hivegrid-cloud-legacy", {
      id: `legacy-${legacyStatus}`,
      email: `${legacyStatus}@example.test`,
      status: legacyStatus,
      firstSeenAt: "2026-03-01T10:00:00.000Z",
      events: [],
    });

    expect(result.rejectionCounts).toEqual({});
    expect(result.accepted[0]).toMatchObject({
      initialStatus: expectedStatus,
      payload: { metadata: { legacyStatus } },
    });
  });

  it("preserves score, conversion and legacy note history as typed import activities", () => {
    const result = parse(
      "hivegrid-cloud-legacy",
      {
        id: "sales-history",
        email: "one@example.test",
        status: "customer",
        firstSeenAt: "2026-03-01T10:00:00.000Z",
        notes: {
          body: "Aktuell\nmit Umbruch",
          updatedAt: "2026-03-05T10:00:00.000Z",
          history: [{ body: "Erste Notiz", at: "2026-03-02T08:00:00.000Z" }],
        },
        events: [
          { id: "score", type: "score-changed", payload: { from: 1, to: 2 }, createdAt: "2026-03-02T10:00:00.000Z" },
          { id: "converted", type: "converted", payload: { tenantId: "legacy-tenant" }, createdAt: "2026-03-03T10:00:00.000Z" },
        ],
      }
    );

    expect(result.rejectionCounts).toEqual({});
    expect(result.accepted[0]).toMatchObject({
      initialStatus: "qualified",
      payload: { metadata: { legacyStatus: "customer" } },
      legacyActivities: [
        { externalActivityId: "score", type: "legacy_event", payload: { from: 1, to: 2, legacyType: "score-changed" } },
        { externalActivityId: "converted", type: "legacy_event", payload: { tenantId: "legacy-tenant", legacyType: "converted" } },
        { externalActivityId: "sales-history:notes:history:0", type: "note_added", payload: { note: "Erste Notiz", legacyType: "lead.notes.history" } },
        { externalActivityId: "sales-history:notes:current", type: "note_added", payload: { note: "Aktuell\nmit Umbruch", legacyType: "lead.notes.current" } },
      ],
    });
  });

  it("counts malformed JSON and duplicate legacy identities without returning row data", () => {
    const valid = JSON.stringify({
      id: "duplicate",
      email: "duplicate@example.test",
      created_at: "2026-03-01T10:00:00.000Z",
    });
    const bytes = Buffer.from(`${valid}\nnot-json\n${valid}`, "utf8");
    const result = parsePlatformLeadBackfillInput({ legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes });

    expect(result.scanned).toBe(3);
    expect(result.accepted).toHaveLength(1);
    expect(result.rejectionCounts).toEqual({ INVALID_JSON: 1, DUPLICATE_LEGACY_ID: 1 });
  });

  it("rejects unsupported backfill formats before parsing input", () => {
    expect(() =>
      parsePlatformLeadBackfillInput({
        legacyStore: "hivegrid-cloud-legacy",
        format: "csv" as never,
        bytes: Buffer.from("id,email", "utf8"),
      })
    ).toThrow();
  });
});

describe("platform lead backfill execution and manifests", () => {
  const sourceRecord = {
    id: "website-row-1",
    email: "secret.person@example.test",
    name: "Secret Person",
    company: "Secret Company",
    message: "Secret message",
    created_at: "2026-03-04T11:00:00.000Z",
  };

  it("produces a PII-free dry-run manifest with hash, counts and time range and performs no imports", async () => {
    const bytes = jsonl(sourceRecord);
    const manifest = await runPlatformLeadBackfill({
      mode: "dry-run",
      legacyStore: "hivegrid-website-staging-sqlite",
      format: "jsonl",
      bytes,
      runId: "dry-run-1",
      clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
    });

    expect(manifest).toMatchObject({
      mode: "dry-run",
      sourceKey: "hivegrid-website",
      outcome: "validated",
      counts: { scanned: 1, accepted: 1, rejected: 0, created: 0, replayed: 0, failed: 0 },
      timeRange: { firstCapturedAt: "2026-03-04T11:00:00.000Z", lastCapturedAt: "2026-03-04T11:00:00.000Z" },
      notificationsTriggered: 0,
      automationsTriggered: 0,
    });
    expect(manifest.input.sha256).toBe(createHash("sha256").update(bytes).digest("hex"));
    const serialized = JSON.stringify(manifest);
    for (const pii of ["secret.person@example.test", "Secret Person", "Secret Company", "Secret message", "website-row-1"]) {
      expect(serialized).not.toContain(pii);
    }
  });

  it("applies through the single import port while preserving timestamp and disabling notification/automation fan-out", async () => {
    const bytes = jsonl(sourceRecord);
    const dryRun = parsePlatformLeadBackfillInput({ legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes });
    const calls: PlatformLeadImportInput[] = [];
    const importer = vi.fn(async (input: PlatformLeadImportInput) => {
      calls.push(input);
      return { outcome: "created" as const, lead: { id: "native-1", status: "new" as const, createdAt: input.capturedAt.toISOString() } };
    });
    const manifest = await runPlatformLeadBackfill({
      mode: "apply",
      legacyStore: "hivegrid-website-staging-sqlite",
      format: "jsonl",
      bytes,
      runId: "apply-1",
      sourceId: "source_website",
      actorAdminUserId: "operator_1",
      expectedInputSha256: dryRun.inputSha256,
      expectedAccepted: 1,
      importer,
      clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
    });

    expect(importer).toHaveBeenCalledTimes(1);
    expect(calls[0]).toMatchObject({
      channel: "import",
      sourceId: "source_website",
      actorAdminUserId: "operator_1",
      capturedAt: new Date("2026-03-04T11:00:00.000Z"),
      payload: { email: "secret.person@example.test" },
      legacyActivities: [],
    });
    expect(manifest).toMatchObject({ outcome: "applied", counts: { created: 1, replayed: 0, failed: 0 } });
    expect(isSuccessfulPlatformLeadBackfillManifest(manifest)).toBe(true);
  });

  it("blocks apply before writes when the expected input hash or count changed or any row is rejected", async () => {
    const bytes = Buffer.from(`${JSON.stringify(sourceRecord)}\ninvalid-json`, "utf8");
    const importer = vi.fn();
    const manifest = await runPlatformLeadBackfill({
      mode: "apply",
      legacyStore: "hivegrid-website-staging-sqlite",
      format: "jsonl",
      bytes,
      runId: "blocked-1",
      sourceId: "source_website",
      actorAdminUserId: "operator_1",
      expectedInputSha256: "0".repeat(64),
      expectedAccepted: 2,
      importer,
      clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
    });

    expect(importer).not.toHaveBeenCalled();
    expect(manifest.outcome).toBe("blocked");
    expect(manifest.gateFailures).toEqual(["INPUT_HASH_MISMATCH", "ACCEPTED_COUNT_MISMATCH", "REJECTED_ROWS_PRESENT"]);
  });

  it("proves replay only when no new row is created", async () => {
    const bytes = jsonl(sourceRecord);
    const hash = createHash("sha256").update(bytes).digest("hex");
    const replayed = await runPlatformLeadBackfill({
      mode: "replay",
      legacyStore: "hivegrid-website-staging-sqlite",
      format: "jsonl",
      bytes,
      runId: "replay-1",
      sourceId: "source_website",
      actorAdminUserId: "operator_1",
      expectedInputSha256: hash,
      expectedAccepted: 1,
      importer: async () => ({ outcome: "replayed", lead: { id: "native-1", status: "new", createdAt: "2026-03-04T11:00:00.000Z" } }),
      clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
    });
    expect(replayed).toMatchObject({ outcome: "replayed", counts: { created: 0, replayed: 1, failed: 0 }, gateFailures: [] });

    const unexpectedCreate = await runPlatformLeadBackfill({
      mode: "replay",
      legacyStore: "hivegrid-website-staging-sqlite",
      format: "jsonl",
      bytes,
      runId: "replay-2",
      sourceId: "source_website",
      actorAdminUserId: "operator_1",
      expectedInputSha256: hash,
      expectedAccepted: 1,
      importer: async () => ({ outcome: "created", lead: { id: "native-2", status: "new", createdAt: "2026-03-04T11:00:00.000Z" } }),
      clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
    });
    expect(unexpectedCreate).toMatchObject({ outcome: "failed", counts: { created: 1 }, gateFailures: ["REPLAY_CREATED_ROWS"] });
  });

  it("declares the exact source mapping for all four stores", () => {
    expect(PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE).toEqual({
      "lead-inbox-postgres": "lead-inbox-legacy",
      "hivegrid-website-production-sqlite": "hivegrid-website",
      "hivegrid-website-staging-sqlite": "hivegrid-website",
      "hivegrid-cloud-legacy": "hivegrid-legacy-cloud",
    });
  });
});