Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { parsePlatformLeadBackfillCliArgs } from "./backfill-cli.js";
|
||||
|
||||
describe("platform lead backfill CLI contract", () => {
|
||||
it("parses a dry-run without database or mutation arguments", () => {
|
||||
expect(
|
||||
parsePlatformLeadBackfillCliArgs([
|
||||
"--",
|
||||
"--mode",
|
||||
"dry-run",
|
||||
"--store=hivegrid-website-staging-sqlite",
|
||||
"--input",
|
||||
"/tmp/staging.jsonl",
|
||||
"--manifest",
|
||||
"/tmp/staging.dry-run.json",
|
||||
"--run-id",
|
||||
"run-dry-1",
|
||||
])
|
||||
).toEqual({
|
||||
mode: "dry-run",
|
||||
legacyStore: "hivegrid-website-staging-sqlite",
|
||||
inputPath: "/tmp/staging.jsonl",
|
||||
manifestPath: "/tmp/staging.dry-run.json",
|
||||
format: "jsonl",
|
||||
runId: "run-dry-1",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires hash, accepted count and actor for apply and replay", () => {
|
||||
expect(() =>
|
||||
parsePlatformLeadBackfillCliArgs(["--mode", "apply", "--store", "lead-inbox-postgres", "--input", "/tmp/input.jsonl", "--manifest", "/tmp/apply.json"])
|
||||
).toThrow();
|
||||
|
||||
expect(
|
||||
parsePlatformLeadBackfillCliArgs([
|
||||
"--mode=replay",
|
||||
"--store=hivegrid-cloud-legacy",
|
||||
"--input=/tmp/leads.jsonl",
|
||||
"--manifest=/tmp/replay.json",
|
||||
"--actor-admin-user-id=operator_1",
|
||||
`--expected-input-sha256=${"a".repeat(64)}`,
|
||||
"--expected-accepted=2",
|
||||
"--run-id=replay-1",
|
||||
])
|
||||
).toMatchObject({
|
||||
mode: "replay",
|
||||
legacyStore: "hivegrid-cloud-legacy",
|
||||
format: "jsonl",
|
||||
actorAdminUserId: "operator_1",
|
||||
expectedInputSha256: "a".repeat(64),
|
||||
expectedAccepted: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects mutation-only proof arguments during dry-run and unknown arguments", () => {
|
||||
const base = ["--mode=dry-run", "--store=hivegrid-cloud-legacy", "--input=/tmp/leads.jsonl", "--manifest=/tmp/dry.json"];
|
||||
expect(() => parsePlatformLeadBackfillCliArgs([...base, "--expected-accepted=2"])).toThrow();
|
||||
expect(() => parsePlatformLeadBackfillCliArgs([...base, "--secret=must-not-exist"])).toThrow();
|
||||
expect(() => parsePlatformLeadBackfillCliArgs([...base, "--format=csv"])).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env tsx
|
||||
import "dotenv/config";
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { readFile, stat, writeFile } from "node:fs/promises";
|
||||
|
||||
import { createPrismaPlatformLeadService } from "@hivegrid/db/platform-leads-admin.server";
|
||||
import { disconnectDbClients } from "@hivegrid/db/client.server";
|
||||
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
|
||||
|
||||
import { closeSharedStores } from "../../apps/web/app/shared-store.server.js";
|
||||
import { userHasPermission } from "../../apps/web/app/utils/db/permissions/userRoles.db.server.js";
|
||||
|
||||
import {
|
||||
isPlatformLeadLegacyStore,
|
||||
isSuccessfulPlatformLeadBackfillManifest,
|
||||
PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE,
|
||||
PlatformLeadBackfillInputError,
|
||||
runPlatformLeadBackfill,
|
||||
type PlatformLeadBackfillInputFormat,
|
||||
type PlatformLeadBackfillMode,
|
||||
type PlatformLeadLegacyStore,
|
||||
} from "./backfill.js";
|
||||
|
||||
const MAX_INPUT_BYTES = 100 * 1024 * 1024;
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/u;
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9_-]+$/u;
|
||||
|
||||
export type PlatformLeadBackfillCliOptions = {
|
||||
mode: PlatformLeadBackfillMode;
|
||||
legacyStore: PlatformLeadLegacyStore;
|
||||
inputPath: string;
|
||||
manifestPath: string;
|
||||
format: PlatformLeadBackfillInputFormat;
|
||||
runId: string;
|
||||
actorAdminUserId?: string;
|
||||
expectedInputSha256?: string;
|
||||
expectedAccepted?: number;
|
||||
};
|
||||
|
||||
export function parsePlatformLeadBackfillCliArgs(argv: readonly string[]): PlatformLeadBackfillCliOptions {
|
||||
const values = parseNamedArguments(argv);
|
||||
rejectUnknownKeys(values, ["mode", "store", "input", "manifest", "format", "run-id", "actor-admin-user-id", "expected-input-sha256", "expected-accepted"]);
|
||||
|
||||
const mode = requireOneOf(values.get("mode"), ["dry-run", "apply", "replay"] as const);
|
||||
const storeValue = requireArgument(values.get("store"));
|
||||
if (!isPlatformLeadLegacyStore(storeValue)) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
const inputPath = requireArgument(values.get("input"));
|
||||
const manifestPath = requireArgument(values.get("manifest"));
|
||||
const format = requireOneOf(values.get("format") ?? "jsonl", ["jsonl"] as const);
|
||||
const runId = values.get("run-id")?.trim() || randomUUID();
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,190}$/u.test(runId)) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
|
||||
const options: PlatformLeadBackfillCliOptions = {
|
||||
mode,
|
||||
legacyStore: storeValue,
|
||||
inputPath,
|
||||
manifestPath,
|
||||
format,
|
||||
runId,
|
||||
};
|
||||
if (mode !== "dry-run") {
|
||||
options.actorAdminUserId = requireIdentifier(values.get("actor-admin-user-id"));
|
||||
options.expectedInputSha256 = requireSha256(values.get("expected-input-sha256"));
|
||||
options.expectedAccepted = requireNonNegativeInteger(values.get("expected-accepted"));
|
||||
} else if (values.has("actor-admin-user-id") || values.has("expected-input-sha256") || values.has("expected-accepted")) {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
export async function runPlatformLeadBackfillCli(argv: readonly string[] = process.argv.slice(2)): Promise<number> {
|
||||
let options: PlatformLeadBackfillCliOptions;
|
||||
try {
|
||||
options = parsePlatformLeadBackfillCliArgs(argv);
|
||||
} catch (error) {
|
||||
writeSafeError(error);
|
||||
return 1;
|
||||
}
|
||||
|
||||
try {
|
||||
const inputStat = await stat(options.inputPath);
|
||||
if (!inputStat.isFile() || inputStat.size > MAX_INPUT_BYTES) throw new PlatformLeadBackfillInputError("INVALID_INPUT");
|
||||
const bytes = await readFile(options.inputPath);
|
||||
const common = {
|
||||
mode: options.mode,
|
||||
legacyStore: options.legacyStore,
|
||||
format: options.format,
|
||||
bytes,
|
||||
runId: options.runId,
|
||||
expectedInputSha256: options.expectedInputSha256,
|
||||
expectedAccepted: options.expectedAccepted,
|
||||
} as const;
|
||||
|
||||
const manifest =
|
||||
options.mode === "dry-run"
|
||||
? await runPlatformLeadBackfill(common)
|
||||
: await runWithRlsContext({ userId: options.actorAdminUserId! }, async () => {
|
||||
if (!(await userHasPermission(options.actorAdminUserId!, null, "admin.leads.admin"))) {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
const service = createPrismaPlatformLeadService();
|
||||
const expectedSourceKey = PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE[options.legacyStore];
|
||||
const source = (await service.listSources()).find(({ key }) => key === expectedSourceKey);
|
||||
if (!source) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
return runPlatformLeadBackfill({
|
||||
...common,
|
||||
sourceId: source.id,
|
||||
actorAdminUserId: options.actorAdminUserId,
|
||||
importer: (input) => service.importLead(input),
|
||||
});
|
||||
});
|
||||
|
||||
const serialized = `${JSON.stringify(manifest, null, 2)}\n`;
|
||||
await writeFile(options.manifestPath, serialized, { encoding: "utf8", flag: "wx", mode: 0o600 });
|
||||
process.stdout.write(serialized);
|
||||
return isSuccessfulPlatformLeadBackfillManifest(manifest) ? 0 : 2;
|
||||
} catch (error) {
|
||||
writeSafeError(error);
|
||||
return 1;
|
||||
} finally {
|
||||
await Promise.allSettled([closeSharedStores(), disconnectDbClients()]);
|
||||
}
|
||||
}
|
||||
|
||||
function parseNamedArguments(argv: readonly string[]): Map<string, string> {
|
||||
const values = new Map<string, string>();
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index]!;
|
||||
if (argument === "--" && index === 0) continue;
|
||||
if (!argument.startsWith("--")) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
const equalsIndex = argument.indexOf("=");
|
||||
const key = argument.slice(2, equalsIndex === -1 ? undefined : equalsIndex);
|
||||
const value = equalsIndex === -1 ? argv[++index] : argument.slice(equalsIndex + 1);
|
||||
if (!key || !value || value.startsWith("--") || values.has(key)) {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
values.set(key, value);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function rejectUnknownKeys(values: ReadonlyMap<string, string>, allowedKeys: readonly string[]): void {
|
||||
const allowed = new Set(allowedKeys);
|
||||
if ([...values.keys()].some((key) => !allowed.has(key))) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
|
||||
function requireArgument(value: string | undefined): string {
|
||||
const normalized = value?.trim();
|
||||
if (!normalized || /[\u0000-\u001f\u007f]/u.test(normalized)) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireIdentifier(value: string | undefined): string {
|
||||
const normalized = requireArgument(value);
|
||||
if (normalized.length > 191 || !IDENTIFIER_PATTERN.test(normalized)) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireSha256(value: string | undefined): string {
|
||||
const normalized = requireArgument(value).toLowerCase();
|
||||
if (!SHA256_PATTERN.test(normalized)) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireNonNegativeInteger(value: string | undefined): number {
|
||||
const normalized = requireArgument(value);
|
||||
if (!/^\d+$/u.test(normalized)) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
const parsed = Number(normalized);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function requireOneOf<const T extends readonly string[]>(value: string | undefined, allowed: T): T[number] {
|
||||
const normalized = requireArgument(value);
|
||||
if (!(allowed as readonly string[]).includes(normalized)) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
return normalized as T[number];
|
||||
}
|
||||
|
||||
function writeSafeError(error: unknown): void {
|
||||
const code = error instanceof PlatformLeadBackfillInputError ? error.code : "OPERATION_FAILED";
|
||||
process.stderr.write(`${JSON.stringify({ ok: false, code })}\n`);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runPlatformLeadBackfillCli()
|
||||
.then((exitCode) => {
|
||||
process.exitCode = exitCode;
|
||||
})
|
||||
.catch(() => {
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import type { PlatformLeadImportInput } from "@hivegrid/db/platform-leads.server";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
isSuccessfulPlatformLeadBackfillManifest,
|
||||
parsePlatformLeadBackfillInput,
|
||||
PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE,
|
||||
runPlatformLeadBackfill,
|
||||
type PlatformLeadLegacyStore,
|
||||
} from "./backfill.js";
|
||||
|
||||
function jsonl(...records: unknown[]): Buffer {
|
||||
return Buffer.from(records.map((record) => JSON.stringify(record)).join("\n"), "utf8");
|
||||
}
|
||||
|
||||
function parse(store: PlatformLeadLegacyStore, ...records: unknown[]) {
|
||||
return parsePlatformLeadBackfillInput({ legacyStore: store, format: "jsonl", bytes: jsonl(...records) });
|
||||
}
|
||||
|
||||
function clock(...values: string[]): () => Date {
|
||||
const dates = values.map((value) => new Date(value));
|
||||
return () => dates.shift() ?? new Date("2026-07-16T12:00:00.000Z");
|
||||
}
|
||||
|
||||
describe("platform lead legacy backfill mapping", () => {
|
||||
it("maps lead-inbox rows with deterministic PII-safe identity, attribution and original capture time", () => {
|
||||
const result = parse("lead-inbox-postgres", {
|
||||
id: "legacy-row-1",
|
||||
site: "hivegrid.de",
|
||||
name: "Ada Lovelace",
|
||||
email: "ada@example.test",
|
||||
company: "Analytical Engines",
|
||||
phone: "+49 30 123456",
|
||||
message: "Bitte melden",
|
||||
source: "demo-form",
|
||||
meta: { utm_source: "messe", landing_page: "https://hivegrid.de/demo", form_id: "demo-request", consent: true },
|
||||
created_at: "2026-03-10T09:30:00.000Z",
|
||||
});
|
||||
|
||||
expect(result.sourceKey).toBe("lead-inbox-legacy");
|
||||
expect(result.scanned).toBe(1);
|
||||
expect(result.rejectionCounts).toEqual({});
|
||||
expect(result.accepted[0]).toMatchObject({
|
||||
legacyId: "legacy-row-1",
|
||||
capturedAt: new Date("2026-03-10T09:30:00.000Z"),
|
||||
payload: {
|
||||
firstName: "Ada",
|
||||
lastName: "Lovelace",
|
||||
email: "ada@example.test",
|
||||
sourceDetail: "demo-form",
|
||||
utmSource: "messe",
|
||||
landingPage: "https://hivegrid.de/demo",
|
||||
metadata: { legacyStore: "lead-inbox-postgres", formId: "demo-request", consent: true },
|
||||
},
|
||||
});
|
||||
expect(result.accepted[0]?.payload.externalId).toMatch(/^legacy:lead-inbox-postgres:[a-f0-9]{64}$/u);
|
||||
expect(JSON.stringify(result.accepted[0]?.payload.metadata)).not.toContain("legacy-row-1");
|
||||
});
|
||||
|
||||
it.each(["hivegrid-website-production-sqlite", "hivegrid-website-staging-sqlite"] as const)(
|
||||
"maps %s rows to the configured website source",
|
||||
(legacyStore) => {
|
||||
const result = parse(legacyStore, {
|
||||
id: 17,
|
||||
email: "website@example.test",
|
||||
name: "Grace Hopper",
|
||||
company: "Compiler GmbH",
|
||||
message: "Demo",
|
||||
created_at: "2026-03-11T08:00:00.000Z",
|
||||
});
|
||||
expect(result.sourceKey).toBe("hivegrid-website");
|
||||
expect(result.accepted[0]).toMatchObject({
|
||||
legacyId: "17",
|
||||
capturedAt: new Date("2026-03-11T08:00:00.000Z"),
|
||||
payload: { firstName: "Grace", lastName: "Hopper", sourceDetail: "legacy-website-sqlite" },
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
it("preserves native legacy status and maps supported Cloud activities with original timestamps", () => {
|
||||
const result = parse("hivegrid-cloud-legacy", {
|
||||
id: "cloud-lead-1",
|
||||
email: "cloud@example.test",
|
||||
firstName: "Katherine",
|
||||
lastName: "Johnson",
|
||||
source: "landing-pro",
|
||||
status: "qualified",
|
||||
firstSeenAt: "2026-03-01T10:00:00.000Z",
|
||||
utmSource: "partner",
|
||||
events: [
|
||||
{
|
||||
id: "event-1",
|
||||
type: "re-captured",
|
||||
actorId: null,
|
||||
payload: { source: "pricing-form" },
|
||||
createdAt: "2026-03-02T10:00:00.000Z",
|
||||
},
|
||||
{
|
||||
id: "event-2",
|
||||
type: "status-changed",
|
||||
actorId: "legacy-operator",
|
||||
payload: { from: "new", to: "qualified" },
|
||||
createdAt: "2026-03-03T10:00:00.000Z",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.accepted[0]).toMatchObject({
|
||||
initialStatus: "qualified",
|
||||
capturedAt: new Date("2026-03-01T10:00:00.000Z"),
|
||||
legacyActivities: [
|
||||
{
|
||||
externalActivityId: "event-1",
|
||||
occurredAt: new Date("2026-03-02T10:00:00.000Z"),
|
||||
type: "captured",
|
||||
payload: { source: "pricing-form", legacyType: "re-captured" },
|
||||
},
|
||||
{
|
||||
externalActivityId: "event-2",
|
||||
occurredAt: new Date("2026-03-03T10:00:00.000Z"),
|
||||
type: "status_changed",
|
||||
payload: { from: "new", to: "qualified", legacyActorId: "legacy-operator" },
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["trial", "qualified"],
|
||||
["customer", "qualified"],
|
||||
["churned", "disqualified"],
|
||||
] as const)("maps legacy sales status %s without losing its original value", (legacyStatus, expectedStatus) => {
|
||||
const result = parse("hivegrid-cloud-legacy", {
|
||||
id: `legacy-${legacyStatus}`,
|
||||
email: `${legacyStatus}@example.test`,
|
||||
status: legacyStatus,
|
||||
firstSeenAt: "2026-03-01T10:00:00.000Z",
|
||||
events: [],
|
||||
});
|
||||
|
||||
expect(result.rejectionCounts).toEqual({});
|
||||
expect(result.accepted[0]).toMatchObject({
|
||||
initialStatus: expectedStatus,
|
||||
payload: { metadata: { legacyStatus } },
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves score, conversion and legacy note history as typed import activities", () => {
|
||||
const result = parse(
|
||||
"hivegrid-cloud-legacy",
|
||||
{
|
||||
id: "sales-history",
|
||||
email: "one@example.test",
|
||||
status: "customer",
|
||||
firstSeenAt: "2026-03-01T10:00:00.000Z",
|
||||
notes: {
|
||||
body: "Aktuell\nmit Umbruch",
|
||||
updatedAt: "2026-03-05T10:00:00.000Z",
|
||||
history: [{ body: "Erste Notiz", at: "2026-03-02T08:00:00.000Z" }],
|
||||
},
|
||||
events: [
|
||||
{ id: "score", type: "score-changed", payload: { from: 1, to: 2 }, createdAt: "2026-03-02T10:00:00.000Z" },
|
||||
{ id: "converted", type: "converted", payload: { tenantId: "legacy-tenant" }, createdAt: "2026-03-03T10:00:00.000Z" },
|
||||
],
|
||||
}
|
||||
);
|
||||
|
||||
expect(result.rejectionCounts).toEqual({});
|
||||
expect(result.accepted[0]).toMatchObject({
|
||||
initialStatus: "qualified",
|
||||
payload: { metadata: { legacyStatus: "customer" } },
|
||||
legacyActivities: [
|
||||
{ externalActivityId: "score", type: "legacy_event", payload: { from: 1, to: 2, legacyType: "score-changed" } },
|
||||
{ externalActivityId: "converted", type: "legacy_event", payload: { tenantId: "legacy-tenant", legacyType: "converted" } },
|
||||
{ externalActivityId: "sales-history:notes:history:0", type: "note_added", payload: { note: "Erste Notiz", legacyType: "lead.notes.history" } },
|
||||
{ externalActivityId: "sales-history:notes:current", type: "note_added", payload: { note: "Aktuell\nmit Umbruch", legacyType: "lead.notes.current" } },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("counts malformed JSON and duplicate legacy identities without returning row data", () => {
|
||||
const valid = JSON.stringify({
|
||||
id: "duplicate",
|
||||
email: "duplicate@example.test",
|
||||
created_at: "2026-03-01T10:00:00.000Z",
|
||||
});
|
||||
const bytes = Buffer.from(`${valid}\nnot-json\n${valid}`, "utf8");
|
||||
const result = parsePlatformLeadBackfillInput({ legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes });
|
||||
|
||||
expect(result.scanned).toBe(3);
|
||||
expect(result.accepted).toHaveLength(1);
|
||||
expect(result.rejectionCounts).toEqual({ INVALID_JSON: 1, DUPLICATE_LEGACY_ID: 1 });
|
||||
});
|
||||
|
||||
it("rejects unsupported backfill formats before parsing input", () => {
|
||||
expect(() =>
|
||||
parsePlatformLeadBackfillInput({
|
||||
legacyStore: "hivegrid-cloud-legacy",
|
||||
format: "csv" as never,
|
||||
bytes: Buffer.from("id,email", "utf8"),
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("platform lead backfill execution and manifests", () => {
|
||||
const sourceRecord = {
|
||||
id: "website-row-1",
|
||||
email: "secret.person@example.test",
|
||||
name: "Secret Person",
|
||||
company: "Secret Company",
|
||||
message: "Secret message",
|
||||
created_at: "2026-03-04T11:00:00.000Z",
|
||||
};
|
||||
|
||||
it("produces a PII-free dry-run manifest with hash, counts and time range and performs no imports", async () => {
|
||||
const bytes = jsonl(sourceRecord);
|
||||
const manifest = await runPlatformLeadBackfill({
|
||||
mode: "dry-run",
|
||||
legacyStore: "hivegrid-website-staging-sqlite",
|
||||
format: "jsonl",
|
||||
bytes,
|
||||
runId: "dry-run-1",
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
|
||||
expect(manifest).toMatchObject({
|
||||
mode: "dry-run",
|
||||
sourceKey: "hivegrid-website",
|
||||
outcome: "validated",
|
||||
counts: { scanned: 1, accepted: 1, rejected: 0, created: 0, replayed: 0, failed: 0 },
|
||||
timeRange: { firstCapturedAt: "2026-03-04T11:00:00.000Z", lastCapturedAt: "2026-03-04T11:00:00.000Z" },
|
||||
notificationsTriggered: 0,
|
||||
automationsTriggered: 0,
|
||||
});
|
||||
expect(manifest.input.sha256).toBe(createHash("sha256").update(bytes).digest("hex"));
|
||||
const serialized = JSON.stringify(manifest);
|
||||
for (const pii of ["secret.person@example.test", "Secret Person", "Secret Company", "Secret message", "website-row-1"]) {
|
||||
expect(serialized).not.toContain(pii);
|
||||
}
|
||||
});
|
||||
|
||||
it("applies through the single import port while preserving timestamp and disabling notification/automation fan-out", async () => {
|
||||
const bytes = jsonl(sourceRecord);
|
||||
const dryRun = parsePlatformLeadBackfillInput({ legacyStore: "hivegrid-website-staging-sqlite", format: "jsonl", bytes });
|
||||
const calls: PlatformLeadImportInput[] = [];
|
||||
const importer = vi.fn(async (input: PlatformLeadImportInput) => {
|
||||
calls.push(input);
|
||||
return { outcome: "created" as const, lead: { id: "native-1", status: "new" as const, createdAt: input.capturedAt.toISOString() } };
|
||||
});
|
||||
const manifest = await runPlatformLeadBackfill({
|
||||
mode: "apply",
|
||||
legacyStore: "hivegrid-website-staging-sqlite",
|
||||
format: "jsonl",
|
||||
bytes,
|
||||
runId: "apply-1",
|
||||
sourceId: "source_website",
|
||||
actorAdminUserId: "operator_1",
|
||||
expectedInputSha256: dryRun.inputSha256,
|
||||
expectedAccepted: 1,
|
||||
importer,
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
|
||||
expect(importer).toHaveBeenCalledTimes(1);
|
||||
expect(calls[0]).toMatchObject({
|
||||
channel: "import",
|
||||
sourceId: "source_website",
|
||||
actorAdminUserId: "operator_1",
|
||||
capturedAt: new Date("2026-03-04T11:00:00.000Z"),
|
||||
payload: { email: "secret.person@example.test" },
|
||||
legacyActivities: [],
|
||||
});
|
||||
expect(manifest).toMatchObject({ outcome: "applied", counts: { created: 1, replayed: 0, failed: 0 } });
|
||||
expect(isSuccessfulPlatformLeadBackfillManifest(manifest)).toBe(true);
|
||||
});
|
||||
|
||||
it("blocks apply before writes when the expected input hash or count changed or any row is rejected", async () => {
|
||||
const bytes = Buffer.from(`${JSON.stringify(sourceRecord)}\ninvalid-json`, "utf8");
|
||||
const importer = vi.fn();
|
||||
const manifest = await runPlatformLeadBackfill({
|
||||
mode: "apply",
|
||||
legacyStore: "hivegrid-website-staging-sqlite",
|
||||
format: "jsonl",
|
||||
bytes,
|
||||
runId: "blocked-1",
|
||||
sourceId: "source_website",
|
||||
actorAdminUserId: "operator_1",
|
||||
expectedInputSha256: "0".repeat(64),
|
||||
expectedAccepted: 2,
|
||||
importer,
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
|
||||
expect(importer).not.toHaveBeenCalled();
|
||||
expect(manifest.outcome).toBe("blocked");
|
||||
expect(manifest.gateFailures).toEqual(["INPUT_HASH_MISMATCH", "ACCEPTED_COUNT_MISMATCH", "REJECTED_ROWS_PRESENT"]);
|
||||
});
|
||||
|
||||
it("proves replay only when no new row is created", async () => {
|
||||
const bytes = jsonl(sourceRecord);
|
||||
const hash = createHash("sha256").update(bytes).digest("hex");
|
||||
const replayed = await runPlatformLeadBackfill({
|
||||
mode: "replay",
|
||||
legacyStore: "hivegrid-website-staging-sqlite",
|
||||
format: "jsonl",
|
||||
bytes,
|
||||
runId: "replay-1",
|
||||
sourceId: "source_website",
|
||||
actorAdminUserId: "operator_1",
|
||||
expectedInputSha256: hash,
|
||||
expectedAccepted: 1,
|
||||
importer: async () => ({ outcome: "replayed", lead: { id: "native-1", status: "new", createdAt: "2026-03-04T11:00:00.000Z" } }),
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
expect(replayed).toMatchObject({ outcome: "replayed", counts: { created: 0, replayed: 1, failed: 0 }, gateFailures: [] });
|
||||
|
||||
const unexpectedCreate = await runPlatformLeadBackfill({
|
||||
mode: "replay",
|
||||
legacyStore: "hivegrid-website-staging-sqlite",
|
||||
format: "jsonl",
|
||||
bytes,
|
||||
runId: "replay-2",
|
||||
sourceId: "source_website",
|
||||
actorAdminUserId: "operator_1",
|
||||
expectedInputSha256: hash,
|
||||
expectedAccepted: 1,
|
||||
importer: async () => ({ outcome: "created", lead: { id: "native-2", status: "new", createdAt: "2026-03-04T11:00:00.000Z" } }),
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
expect(unexpectedCreate).toMatchObject({ outcome: "failed", counts: { created: 1 }, gateFailures: ["REPLAY_CREATED_ROWS"] });
|
||||
});
|
||||
|
||||
it("declares the exact source mapping for all four stores", () => {
|
||||
expect(PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE).toEqual({
|
||||
"lead-inbox-postgres": "lead-inbox-legacy",
|
||||
"hivegrid-website-production-sqlite": "hivegrid-website",
|
||||
"hivegrid-website-staging-sqlite": "hivegrid-website",
|
||||
"hivegrid-cloud-legacy": "hivegrid-legacy-cloud",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,698 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import {
|
||||
buildLegacyPlatformLeadExternalId,
|
||||
normalizePlatformLeadPayload,
|
||||
PLATFORM_LEAD_LEGACY_STORES,
|
||||
type PlatformLeadImportInput,
|
||||
type PlatformLeadIngestionResult,
|
||||
type PlatformLeadLegacyActivityInput,
|
||||
type PlatformLeadPayload,
|
||||
} from "@hivegrid/db/platform-leads.server";
|
||||
|
||||
export type PlatformLeadLegacyStore = (typeof PLATFORM_LEAD_LEGACY_STORES)[number];
|
||||
export type PlatformLeadBackfillMode = "dry-run" | "apply" | "replay";
|
||||
export type PlatformLeadBackfillInputFormat = "jsonl";
|
||||
export type PlatformLeadBackfillInitialStatus = NonNullable<PlatformLeadImportInput["initialStatus"]>;
|
||||
|
||||
export const PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE = Object.freeze({
|
||||
"lead-inbox-postgres": "lead-inbox-legacy",
|
||||
"hivegrid-website-production-sqlite": "hivegrid-website",
|
||||
"hivegrid-website-staging-sqlite": "hivegrid-website",
|
||||
"hivegrid-cloud-legacy": "hivegrid-legacy-cloud",
|
||||
} as const satisfies Record<PlatformLeadLegacyStore, string>);
|
||||
|
||||
export const PLATFORM_LEAD_BACKFILL_REJECTION_CODES = [
|
||||
"INVALID_JSON",
|
||||
"INVALID_RECORD",
|
||||
"MISSING_LEGACY_ID",
|
||||
"INVALID_TIMESTAMP",
|
||||
"INVALID_LEAD_PAYLOAD",
|
||||
"DUPLICATE_LEGACY_ID",
|
||||
"INVALID_ACTIVITY",
|
||||
"UNSUPPORTED_ACTIVITY_TYPE",
|
||||
"UNSUPPORTED_LEGACY_STATUS",
|
||||
] as const;
|
||||
|
||||
export type PlatformLeadBackfillRejectionCode = (typeof PLATFORM_LEAD_BACKFILL_REJECTION_CODES)[number];
|
||||
export type PlatformLeadBackfillGateFailure =
|
||||
| "INPUT_HASH_MISMATCH"
|
||||
| "ACCEPTED_COUNT_MISMATCH"
|
||||
| "REJECTED_ROWS_PRESENT"
|
||||
| "IMPORT_FAILURES_PRESENT"
|
||||
| "REPLAY_CREATED_ROWS";
|
||||
|
||||
export type PlatformLeadBackfillImportRecord = {
|
||||
legacyId: string;
|
||||
capturedAt: Date;
|
||||
initialStatus?: PlatformLeadBackfillInitialStatus;
|
||||
payload: PlatformLeadPayload & { externalId: string };
|
||||
legacyActivities: PlatformLeadLegacyActivityInput[];
|
||||
};
|
||||
|
||||
export type PlatformLeadBackfillParseResult = {
|
||||
legacyStore: PlatformLeadLegacyStore;
|
||||
sourceKey: string;
|
||||
inputSha256: string;
|
||||
inputBytes: number;
|
||||
scanned: number;
|
||||
accepted: PlatformLeadBackfillImportRecord[];
|
||||
rejectionCounts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>;
|
||||
};
|
||||
|
||||
export type PlatformLeadBackfillManifest = {
|
||||
schemaVersion: 1;
|
||||
runId: string;
|
||||
mode: PlatformLeadBackfillMode;
|
||||
legacyStore: PlatformLeadLegacyStore;
|
||||
sourceKey: string;
|
||||
sourceId: string | null;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
input: {
|
||||
sha256: string;
|
||||
bytes: number;
|
||||
expectedSha256: string | null;
|
||||
hashVerified: boolean | null;
|
||||
};
|
||||
counts: {
|
||||
scanned: number;
|
||||
accepted: number;
|
||||
expectedAccepted: number | null;
|
||||
acceptedCountVerified: boolean | null;
|
||||
rejected: number;
|
||||
created: number;
|
||||
replayed: number;
|
||||
failed: number;
|
||||
};
|
||||
rejectionCounts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>;
|
||||
timeRange: {
|
||||
firstCapturedAt: string | null;
|
||||
lastCapturedAt: string | null;
|
||||
};
|
||||
gateFailures: PlatformLeadBackfillGateFailure[];
|
||||
outcome: "validated" | "applied" | "replayed" | "blocked" | "failed";
|
||||
notificationsTriggered: 0;
|
||||
automationsTriggered: 0;
|
||||
};
|
||||
|
||||
export type PlatformLeadBackfillImporter = (input: PlatformLeadImportInput) => Promise<PlatformLeadIngestionResult>;
|
||||
|
||||
export type RunPlatformLeadBackfillInput = {
|
||||
mode: PlatformLeadBackfillMode;
|
||||
legacyStore: PlatformLeadLegacyStore;
|
||||
format: PlatformLeadBackfillInputFormat;
|
||||
bytes: Uint8Array;
|
||||
runId: string;
|
||||
sourceId?: string;
|
||||
actorAdminUserId?: string;
|
||||
expectedInputSha256?: string;
|
||||
expectedAccepted?: number;
|
||||
importer?: PlatformLeadBackfillImporter;
|
||||
clock?: () => Date;
|
||||
};
|
||||
|
||||
type CandidateRow = { rowNumber: number; value?: unknown; rejectionCode?: PlatformLeadBackfillRejectionCode };
|
||||
|
||||
class PlatformLeadBackfillRowError extends Error {
|
||||
public override readonly name = "PlatformLeadBackfillRowError";
|
||||
|
||||
public constructor(public readonly code: PlatformLeadBackfillRejectionCode) {
|
||||
super(code);
|
||||
}
|
||||
}
|
||||
|
||||
export class PlatformLeadBackfillInputError extends Error {
|
||||
public override readonly name = "PlatformLeadBackfillInputError";
|
||||
|
||||
public constructor(public readonly code: "INVALID_INPUT" | "INVALID_CONFIGURATION") {
|
||||
super(code);
|
||||
}
|
||||
}
|
||||
|
||||
export function isPlatformLeadLegacyStore(value: string): value is PlatformLeadLegacyStore {
|
||||
return (PLATFORM_LEAD_LEGACY_STORES as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
export function parsePlatformLeadBackfillInput(input: {
|
||||
legacyStore: PlatformLeadLegacyStore;
|
||||
format: PlatformLeadBackfillInputFormat;
|
||||
bytes: Uint8Array;
|
||||
}): PlatformLeadBackfillParseResult {
|
||||
if (input.format !== "jsonl") throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
|
||||
const inputBytes = Buffer.from(input.bytes);
|
||||
const inputSha256 = sha256(inputBytes);
|
||||
let text: string;
|
||||
try {
|
||||
text = new TextDecoder("utf-8", { fatal: true }).decode(inputBytes).replace(/^\uFEFF/u, "");
|
||||
} catch {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_INPUT");
|
||||
}
|
||||
|
||||
const candidates = parseJsonCandidates(text);
|
||||
const accepted: PlatformLeadBackfillImportRecord[] = [];
|
||||
const rejectionCounts: Partial<Record<PlatformLeadBackfillRejectionCode, number>> = {};
|
||||
const seenExternalIds = new Set<string>();
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.rejectionCode) {
|
||||
incrementCount(rejectionCounts, candidate.rejectionCode);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const mapped = mapCandidate(input.legacyStore, candidate.value);
|
||||
if (seenExternalIds.has(mapped.payload.externalId)) {
|
||||
throw new PlatformLeadBackfillRowError("DUPLICATE_LEGACY_ID");
|
||||
}
|
||||
seenExternalIds.add(mapped.payload.externalId);
|
||||
accepted.push(mapped);
|
||||
} catch (error) {
|
||||
incrementCount(rejectionCounts, toRejectionCode(error));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
legacyStore: input.legacyStore,
|
||||
sourceKey: PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE[input.legacyStore],
|
||||
inputSha256,
|
||||
inputBytes: inputBytes.byteLength,
|
||||
scanned: candidates.length,
|
||||
accepted,
|
||||
rejectionCounts,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runPlatformLeadBackfill(input: RunPlatformLeadBackfillInput): Promise<PlatformLeadBackfillManifest> {
|
||||
const clock = input.clock ?? (() => new Date());
|
||||
const startedAt = requireValidDate(clock());
|
||||
const runId = normalizeRunId(input.runId);
|
||||
const parsed = parsePlatformLeadBackfillInput(input);
|
||||
const expectedInputSha256 = input.expectedInputSha256?.trim().toLowerCase() ?? null;
|
||||
const hashVerified = expectedInputSha256 === null ? null : parsed.inputSha256 === expectedInputSha256;
|
||||
const acceptedCountVerified = input.expectedAccepted === undefined ? null : parsed.accepted.length === input.expectedAccepted;
|
||||
const rejected = sumCounts(parsed.rejectionCounts);
|
||||
const gateFailures: PlatformLeadBackfillGateFailure[] = [];
|
||||
if (hashVerified === false) gateFailures.push("INPUT_HASH_MISMATCH");
|
||||
if (acceptedCountVerified === false) gateFailures.push("ACCEPTED_COUNT_MISMATCH");
|
||||
if (rejected > 0) gateFailures.push("REJECTED_ROWS_PRESENT");
|
||||
|
||||
if (input.mode !== "dry-run") {
|
||||
if (!expectedInputSha256 || !/^[a-f0-9]{64}$/u.test(expectedInputSha256) || input.expectedAccepted === undefined) {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
if (!Number.isSafeInteger(input.expectedAccepted) || input.expectedAccepted < 0) {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
if (!input.sourceId || !input.actorAdminUserId || !input.importer) {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
}
|
||||
|
||||
let created = 0;
|
||||
let replayed = 0;
|
||||
let failed = 0;
|
||||
if (input.mode !== "dry-run" && gateFailures.length === 0) {
|
||||
for (const record of parsed.accepted) {
|
||||
try {
|
||||
const result = await input.importer!({
|
||||
channel: "import",
|
||||
sourceId: input.sourceId!,
|
||||
actorAdminUserId: input.actorAdminUserId!,
|
||||
reason: `R-LEADS deterministic ${input.legacyStore} backfill`,
|
||||
requestId: runId,
|
||||
capturedAt: record.capturedAt,
|
||||
initialStatus: record.initialStatus,
|
||||
payload: record.payload,
|
||||
legacyActivities: record.legacyActivities,
|
||||
});
|
||||
if (result.outcome === "created") created += 1;
|
||||
else replayed += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
if (failed > 0) gateFailures.push("IMPORT_FAILURES_PRESENT");
|
||||
if (input.mode === "replay" && created > 0) gateFailures.push("REPLAY_CREATED_ROWS");
|
||||
}
|
||||
|
||||
const finishedAt = requireValidDate(clock());
|
||||
const range = capturedAtRange(parsed.accepted);
|
||||
const outcome = resolveOutcome(input.mode, gateFailures);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
mode: input.mode,
|
||||
legacyStore: parsed.legacyStore,
|
||||
sourceKey: parsed.sourceKey,
|
||||
sourceId: input.sourceId ?? null,
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: finishedAt.toISOString(),
|
||||
input: {
|
||||
sha256: parsed.inputSha256,
|
||||
bytes: parsed.inputBytes,
|
||||
expectedSha256: expectedInputSha256,
|
||||
hashVerified,
|
||||
},
|
||||
counts: {
|
||||
scanned: parsed.scanned,
|
||||
accepted: parsed.accepted.length,
|
||||
expectedAccepted: input.expectedAccepted ?? null,
|
||||
acceptedCountVerified,
|
||||
rejected,
|
||||
created,
|
||||
replayed,
|
||||
failed,
|
||||
},
|
||||
rejectionCounts: parsed.rejectionCounts,
|
||||
timeRange: range,
|
||||
gateFailures,
|
||||
outcome,
|
||||
notificationsTriggered: 0,
|
||||
automationsTriggered: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function isSuccessfulPlatformLeadBackfillManifest(manifest: PlatformLeadBackfillManifest): boolean {
|
||||
return manifest.gateFailures.length === 0 && manifest.outcome !== "blocked" && manifest.outcome !== "failed";
|
||||
}
|
||||
|
||||
function parseJsonCandidates(text: string): CandidateRow[] {
|
||||
if (!text.trim()) return [];
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
if (!Array.isArray(parsed)) throw new PlatformLeadBackfillInputError("INVALID_INPUT");
|
||||
return parsed.map((value, index) => ({ rowNumber: index + 1, value }));
|
||||
} catch (error) {
|
||||
if (error instanceof PlatformLeadBackfillInputError) throw error;
|
||||
throw new PlatformLeadBackfillInputError("INVALID_INPUT");
|
||||
}
|
||||
}
|
||||
return text
|
||||
.split(/\r?\n/u)
|
||||
.map((line, index) => ({ line: line.trim(), rowNumber: index + 1 }))
|
||||
.filter(({ line }) => line.length > 0)
|
||||
.map(({ line, rowNumber }) => {
|
||||
try {
|
||||
return { rowNumber, value: JSON.parse(line) as unknown };
|
||||
} catch {
|
||||
return { rowNumber, rejectionCode: "INVALID_JSON" as const };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function mapCandidate(legacyStore: PlatformLeadLegacyStore, value: unknown): PlatformLeadBackfillImportRecord {
|
||||
const record = requireRecord(value);
|
||||
switch (legacyStore) {
|
||||
case "lead-inbox-postgres":
|
||||
return mapLeadInboxRecord(legacyStore, record);
|
||||
case "hivegrid-website-production-sqlite":
|
||||
case "hivegrid-website-staging-sqlite":
|
||||
return mapHiveGridWebsiteRecord(legacyStore, record);
|
||||
case "hivegrid-cloud-legacy":
|
||||
return mapHiveGridCloudRecord(legacyStore, record);
|
||||
}
|
||||
}
|
||||
|
||||
function mapLeadInboxRecord(legacyStore: PlatformLeadLegacyStore, record: Record<string, unknown>): PlatformLeadBackfillImportRecord {
|
||||
const legacyId = requireLegacyId(field(record, "id"));
|
||||
const meta = optionalObjectOrJson(field(record, "meta"));
|
||||
return buildRecord({
|
||||
legacyStore,
|
||||
legacyId,
|
||||
capturedAt: requireTimestamp(field(record, "created_at", "createdAt")),
|
||||
metadata: legacyMetadata(record, meta),
|
||||
payload: {
|
||||
...nameFields(record),
|
||||
email: optionalString(field(record, "email")),
|
||||
phone: optionalString(field(record, "phone")),
|
||||
company: optionalString(field(record, "company")),
|
||||
message: optionalString(field(record, "message")),
|
||||
sourceDetail: optionalString(field(record, "source", "site")),
|
||||
...attributionFields(record, meta),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function mapHiveGridWebsiteRecord(legacyStore: PlatformLeadLegacyStore, record: Record<string, unknown>): PlatformLeadBackfillImportRecord {
|
||||
const legacyId = requireLegacyId(field(record, "id"));
|
||||
return buildRecord({
|
||||
legacyStore,
|
||||
legacyId,
|
||||
capturedAt: requireTimestamp(field(record, "created_at", "createdAt")),
|
||||
payload: {
|
||||
...nameFields(record),
|
||||
email: optionalString(field(record, "email")),
|
||||
company: optionalString(field(record, "company")),
|
||||
message: optionalString(field(record, "message")),
|
||||
sourceDetail: "legacy-website-sqlite",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function mapHiveGridCloudRecord(legacyStore: PlatformLeadLegacyStore, record: Record<string, unknown>): PlatformLeadBackfillImportRecord {
|
||||
const legacyId = requireLegacyId(field(record, "id"));
|
||||
const legacyStatus = optionalLegacyStatus(field(record, "status"));
|
||||
const status = mapLegacyStatus(legacyStatus);
|
||||
const eventsValue = field(record, "events");
|
||||
if (eventsValue !== undefined && !Array.isArray(eventsValue)) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
||||
const capturedAt = requireTimestamp(field(record, "firstSeenAt", "first_seen_at", "createdAt", "created_at"));
|
||||
const legacyActivities = [
|
||||
...(eventsValue ?? []).map((event) => mapLegacyCloudActivity(event)),
|
||||
...mapLegacyCloudNotes({ legacyId, notes: field(record, "notes"), capturedAt }),
|
||||
];
|
||||
return buildRecord({
|
||||
legacyStore,
|
||||
legacyId,
|
||||
capturedAt,
|
||||
initialStatus: status,
|
||||
metadata: legacyStatus && legacyStatus !== status ? { legacyStatus } : undefined,
|
||||
legacyActivities,
|
||||
payload: {
|
||||
firstName: optionalString(field(record, "firstName", "first_name")),
|
||||
lastName: optionalString(field(record, "lastName", "last_name")),
|
||||
email: optionalString(field(record, "email")),
|
||||
phone: optionalString(field(record, "phone")),
|
||||
company: optionalString(field(record, "company")),
|
||||
sourceDetail: optionalString(field(record, "source")),
|
||||
...attributionFields(record),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function mapLegacyCloudActivity(value: unknown): PlatformLeadLegacyActivityInput {
|
||||
const event = requireRecord(value, "INVALID_ACTIVITY");
|
||||
const rawType = requireString(field(event, "type"), "INVALID_ACTIVITY");
|
||||
const type = mapLegacyActivityType(rawType);
|
||||
const payload = normalizeActivityPayload(field(event, "payload"));
|
||||
const actorId = optionalString(field(event, "actorId", "actor_id"), "INVALID_ACTIVITY");
|
||||
if (actorId) payload.legacyActorId = actorId;
|
||||
if (rawType === "re-captured" || type === "legacy_event") payload.legacyType = rawType;
|
||||
return {
|
||||
externalActivityId: requireLegacyId(field(event, "id"), "INVALID_ACTIVITY"),
|
||||
occurredAt: requireTimestamp(field(event, "createdAt", "created_at"), "INVALID_ACTIVITY"),
|
||||
type,
|
||||
payload,
|
||||
};
|
||||
}
|
||||
|
||||
function mapLegacyActivityType(value: string): PlatformLeadLegacyActivityInput["type"] {
|
||||
const mapping: Record<string, PlatformLeadLegacyActivityInput["type"] | undefined> = {
|
||||
captured: "captured",
|
||||
"re-captured": "captured",
|
||||
status_changed: "status_changed",
|
||||
"status-changed": "status_changed",
|
||||
assigned: "assigned",
|
||||
note_added: "note_added",
|
||||
"note-added": "note_added",
|
||||
archived: "archived",
|
||||
restored: "restored",
|
||||
duplicate_hint_recorded: "duplicate_hint_recorded",
|
||||
"score-changed": "legacy_event",
|
||||
score_changed: "legacy_event",
|
||||
converted: "legacy_event",
|
||||
};
|
||||
const mapped = mapping[value];
|
||||
if (!mapped) throw new PlatformLeadBackfillRowError("UNSUPPORTED_ACTIVITY_TYPE");
|
||||
return mapped;
|
||||
}
|
||||
|
||||
function mapLegacyCloudNotes(input: {
|
||||
legacyId: string;
|
||||
notes: unknown;
|
||||
capturedAt: Date;
|
||||
}): PlatformLeadLegacyActivityInput[] {
|
||||
if (input.notes === undefined || input.notes === null) return [];
|
||||
const notes = requireRecord(input.notes, "INVALID_ACTIVITY");
|
||||
const mapped: PlatformLeadLegacyActivityInput[] = [];
|
||||
const history = field(notes, "history");
|
||||
if (history !== undefined && !Array.isArray(history)) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
||||
for (const [index, value] of (history ?? []).entries()) {
|
||||
const entry = requireRecord(value, "INVALID_ACTIVITY");
|
||||
const note = optionalNote(field(entry, "body"));
|
||||
if (!note) continue;
|
||||
mapped.push({
|
||||
externalActivityId: `${input.legacyId}:notes:history:${index}`,
|
||||
occurredAt: optionalTimestamp(field(entry, "at")) ?? input.capturedAt,
|
||||
type: "note_added",
|
||||
payload: { note, legacyType: "lead.notes.history" },
|
||||
});
|
||||
}
|
||||
const current = optionalNote(field(notes, "body"));
|
||||
if (current) {
|
||||
mapped.push({
|
||||
externalActivityId: `${input.legacyId}:notes:current`,
|
||||
occurredAt: optionalTimestamp(field(notes, "updatedAt", "updated_at")) ?? input.capturedAt,
|
||||
type: "note_added",
|
||||
payload: { note: current, legacyType: "lead.notes.current" },
|
||||
});
|
||||
}
|
||||
return mapped;
|
||||
}
|
||||
|
||||
function normalizeActivityPayload(value: unknown): Record<string, string | number | boolean | null> {
|
||||
if (value === null || value === undefined) return {};
|
||||
const payload = requireRecord(value, "INVALID_ACTIVITY");
|
||||
const entries = Object.entries(payload);
|
||||
if (entries.length > 32) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
||||
const normalized: Record<string, string | number | boolean | null> = {};
|
||||
for (const [key, entry] of entries) {
|
||||
if (!/^[A-Za-z][A-Za-z0-9_-]{0,63}$/u.test(key)) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
||||
if (entry === null || typeof entry === "boolean" || (typeof entry === "number" && Number.isFinite(entry))) {
|
||||
normalized[key] = entry;
|
||||
} else if (typeof entry === "string" && entry.length <= 1_000 && !hasControlCharacters(entry)) {
|
||||
normalized[key] = entry;
|
||||
} else throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
||||
}
|
||||
if (Buffer.byteLength(JSON.stringify(normalized), "utf8") > 4_096) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function buildRecord(input: {
|
||||
legacyStore: PlatformLeadLegacyStore;
|
||||
legacyId: string;
|
||||
capturedAt: Date;
|
||||
initialStatus?: PlatformLeadBackfillInitialStatus;
|
||||
payload: PlatformLeadPayload;
|
||||
metadata?: Record<string, string | number | boolean | null>;
|
||||
legacyActivities?: PlatformLeadLegacyActivityInput[];
|
||||
}): PlatformLeadBackfillImportRecord {
|
||||
const externalId = buildLegacyPlatformLeadExternalId({ legacyStore: input.legacyStore, legacyId: input.legacyId });
|
||||
const payload: PlatformLeadPayload & { externalId: string } = {
|
||||
...input.payload,
|
||||
externalId,
|
||||
metadata: {
|
||||
...input.metadata,
|
||||
legacyId: sha256(Buffer.from(input.legacyId, "utf8")),
|
||||
legacyStore: input.legacyStore,
|
||||
},
|
||||
};
|
||||
try {
|
||||
normalizePlatformLeadPayload(payload);
|
||||
} catch {
|
||||
throw new PlatformLeadBackfillRowError("INVALID_LEAD_PAYLOAD");
|
||||
}
|
||||
return {
|
||||
legacyId: input.legacyId,
|
||||
capturedAt: input.capturedAt,
|
||||
initialStatus: input.initialStatus,
|
||||
payload,
|
||||
legacyActivities: input.legacyActivities ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function nameFields(record: Record<string, unknown>): Pick<PlatformLeadPayload, "firstName" | "lastName"> {
|
||||
const explicitFirstName = optionalString(field(record, "firstName", "first_name"));
|
||||
const explicitLastName = optionalString(field(record, "lastName", "last_name"));
|
||||
if (explicitFirstName || explicitLastName) return { firstName: explicitFirstName, lastName: explicitLastName };
|
||||
const fullName = optionalString(field(record, "name"));
|
||||
if (!fullName) return { firstName: null, lastName: null };
|
||||
const [firstName, ...rest] = fullName.split(/\s+/u);
|
||||
return { firstName: firstName ?? null, lastName: rest.join(" ") || null };
|
||||
}
|
||||
|
||||
function attributionFields(
|
||||
...records: Array<Record<string, unknown> | null>
|
||||
): Pick<PlatformLeadPayload, "utmSource" | "utmMedium" | "utmCampaign" | "utmTerm" | "utmContent" | "referrer" | "landingPage"> {
|
||||
const find = (...keys: string[]): unknown => {
|
||||
for (const record of records) {
|
||||
if (!record) continue;
|
||||
const value = field(record, ...keys);
|
||||
if (value !== undefined && value !== null && value !== "") return value;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
return {
|
||||
utmSource: optionalString(find("utmSource", "utm_source")),
|
||||
utmMedium: optionalString(find("utmMedium", "utm_medium")),
|
||||
utmCampaign: optionalString(find("utmCampaign", "utm_campaign")),
|
||||
utmTerm: optionalString(find("utmTerm", "utm_term")),
|
||||
utmContent: optionalString(find("utmContent", "utm_content")),
|
||||
referrer: optionalString(find("referrer")),
|
||||
landingPage: optionalString(find("landingPage", "landing_page")),
|
||||
};
|
||||
}
|
||||
|
||||
function legacyMetadata(...records: Array<Record<string, unknown> | null>): Record<string, string | number | boolean | null> {
|
||||
const aliases = {
|
||||
consent: ["consent"],
|
||||
formId: ["formId", "form_id"],
|
||||
formName: ["formName", "form_name"],
|
||||
locale: ["locale"],
|
||||
productInterest: ["productInterest", "product_interest"],
|
||||
submissionId: ["submissionId", "submission_id"],
|
||||
} as const;
|
||||
const result: Record<string, string | number | boolean | null> = {};
|
||||
for (const [target, keys] of Object.entries(aliases)) {
|
||||
let value: unknown;
|
||||
for (const record of records) {
|
||||
if (!record) continue;
|
||||
value = field(record, ...keys);
|
||||
if (value !== undefined) break;
|
||||
}
|
||||
if (value === undefined) continue;
|
||||
if (value === null || typeof value === "boolean" || (typeof value === "number" && Number.isFinite(value))) result[target] = value;
|
||||
else if (typeof value === "string" && value.length <= 500 && !hasControlCharacters(value)) result[target] = value;
|
||||
else throw new PlatformLeadBackfillRowError("INVALID_RECORD");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function optionalLegacyStatus(value: unknown): string | null {
|
||||
if (value === undefined || value === null || value === "") return null;
|
||||
if (typeof value !== "string") throw new PlatformLeadBackfillRowError("UNSUPPORTED_LEGACY_STATUS");
|
||||
if (["new", "contacted", "qualified", "trial", "customer", "churned", "disqualified"].includes(value)) return value;
|
||||
throw new PlatformLeadBackfillRowError("UNSUPPORTED_LEGACY_STATUS");
|
||||
}
|
||||
|
||||
function mapLegacyStatus(value: string | null): PlatformLeadBackfillInitialStatus {
|
||||
if (!value) return "new";
|
||||
if (value === "trial" || value === "customer") return "qualified";
|
||||
if (value === "churned") return "disqualified";
|
||||
return value as PlatformLeadBackfillInitialStatus;
|
||||
}
|
||||
|
||||
function requireRecord(value: unknown, code: PlatformLeadBackfillRejectionCode = "INVALID_RECORD"): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new PlatformLeadBackfillRowError(code);
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function field(record: Record<string, unknown>, ...keys: string[]): unknown {
|
||||
for (const key of keys) {
|
||||
if (Object.prototype.hasOwnProperty.call(record, key)) return record[key];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function optionalObjectOrJson(value: unknown): Record<string, unknown> | null {
|
||||
if (value === null || value === undefined || value === "") return null;
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
return requireRecord(JSON.parse(value) as unknown);
|
||||
} catch {
|
||||
throw new PlatformLeadBackfillRowError("INVALID_RECORD");
|
||||
}
|
||||
}
|
||||
return requireRecord(value);
|
||||
}
|
||||
|
||||
function requireLegacyId(value: unknown, code: PlatformLeadBackfillRejectionCode = "MISSING_LEGACY_ID"): string {
|
||||
const normalized = optionalLegacyId(value);
|
||||
if (!normalized) throw new PlatformLeadBackfillRowError(code);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function optionalLegacyId(value: unknown): string | null {
|
||||
if (typeof value !== "string" && typeof value !== "number") return null;
|
||||
const normalized = String(value).trim();
|
||||
if (!normalized || normalized.length > 2_048 || hasControlCharacters(normalized)) return null;
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireString(value: unknown, code: PlatformLeadBackfillRejectionCode): string {
|
||||
const normalized = optionalString(value, code);
|
||||
if (!normalized) throw new PlatformLeadBackfillRowError(code);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function optionalString(value: unknown, code: PlatformLeadBackfillRejectionCode = "INVALID_RECORD"): string | null {
|
||||
if (value === undefined || value === null) return null;
|
||||
if (typeof value !== "string") throw new PlatformLeadBackfillRowError(code);
|
||||
const normalized = value.trim();
|
||||
if (!normalized) return null;
|
||||
if (hasControlCharacters(normalized)) throw new PlatformLeadBackfillRowError(code);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function optionalNote(value: unknown): string | null {
|
||||
if (value === undefined || value === null) return null;
|
||||
if (typeof value !== "string" || value.length > 8_000 || /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(value)) {
|
||||
throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
||||
}
|
||||
return value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function requireTimestamp(value: unknown, code: PlatformLeadBackfillRejectionCode = "INVALID_TIMESTAMP"): Date {
|
||||
if (typeof value !== "string" && typeof value !== "number") throw new PlatformLeadBackfillRowError(code);
|
||||
const date = new Date(value);
|
||||
if (!Number.isFinite(date.getTime())) throw new PlatformLeadBackfillRowError(code);
|
||||
return date;
|
||||
}
|
||||
|
||||
function optionalTimestamp(value: unknown): Date | null {
|
||||
if (value === undefined || value === null || value === "") return null;
|
||||
return requireTimestamp(value, "INVALID_ACTIVITY");
|
||||
}
|
||||
|
||||
function requireValidDate(value: Date): Date {
|
||||
if (!(value instanceof Date) || !Number.isFinite(value.getTime())) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeRunId(value: string): string {
|
||||
if (typeof value !== "string") throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
const normalized = value.trim();
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,190}$/u.test(normalized)) {
|
||||
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function capturedAtRange(records: readonly PlatformLeadBackfillImportRecord[]): PlatformLeadBackfillManifest["timeRange"] {
|
||||
if (records.length === 0) return { firstCapturedAt: null, lastCapturedAt: null };
|
||||
const timestamps = records.map(({ capturedAt }) => capturedAt.getTime());
|
||||
return {
|
||||
firstCapturedAt: new Date(Math.min(...timestamps)).toISOString(),
|
||||
lastCapturedAt: new Date(Math.max(...timestamps)).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveOutcome(mode: PlatformLeadBackfillMode, failures: readonly PlatformLeadBackfillGateFailure[]): PlatformLeadBackfillManifest["outcome"] {
|
||||
if (failures.length > 0) {
|
||||
return failures.includes("IMPORT_FAILURES_PRESENT") || failures.includes("REPLAY_CREATED_ROWS") ? "failed" : "blocked";
|
||||
}
|
||||
if (mode === "dry-run") return "validated";
|
||||
return mode === "apply" ? "applied" : "replayed";
|
||||
}
|
||||
|
||||
function incrementCount(counts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>, code: PlatformLeadBackfillRejectionCode): void {
|
||||
counts[code] = (counts[code] ?? 0) + 1;
|
||||
}
|
||||
|
||||
function sumCounts(counts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>): number {
|
||||
return Object.values(counts).reduce<number>((total, count) => total + (count ?? 0), 0);
|
||||
}
|
||||
|
||||
function toRejectionCode(error: unknown): PlatformLeadBackfillRejectionCode {
|
||||
return error instanceof PlatformLeadBackfillRowError ? error.code : "INVALID_RECORD";
|
||||
}
|
||||
|
||||
function sha256(value: Uint8Array): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function hasControlCharacters(value: string): boolean {
|
||||
return /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(value);
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { parsePlatformLeadSourceBootstrapCliArgs } from "./source-bootstrap-cli.js";
|
||||
|
||||
describe("platform lead source bootstrap CLI contract", () => {
|
||||
it("requires actor and routing operator as runtime arguments instead of hardcoding a person", () => {
|
||||
expect(
|
||||
parsePlatformLeadSourceBootstrapCliArgs([
|
||||
"--",
|
||||
"--mode=dry-run",
|
||||
"--actor-admin-user-id=operator_1",
|
||||
"--routing-admin-user-id=haris_runtime_id",
|
||||
"--manifest=/tmp/source-bootstrap.dry.json",
|
||||
"--run-id=source-dry-1",
|
||||
])
|
||||
).toEqual({
|
||||
mode: "dry-run",
|
||||
actorAdminUserId: "operator_1",
|
||||
routingAdminUserId: "haris_runtime_id",
|
||||
manifestPath: "/tmp/source-bootstrap.dry.json",
|
||||
runId: "source-dry-1",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires the dry-run plan hash before apply", () => {
|
||||
const args = [
|
||||
"--mode=apply",
|
||||
"--actor-admin-user-id=operator_1",
|
||||
"--routing-admin-user-id=haris_runtime_id",
|
||||
"--manifest=/tmp/source-bootstrap.apply.json",
|
||||
];
|
||||
expect(() => parsePlatformLeadSourceBootstrapCliArgs(args)).toThrow();
|
||||
expect(parsePlatformLeadSourceBootstrapCliArgs([...args, `--expected-plan-sha256=${"f".repeat(64)}`, "--run-id=source-apply-1"])).toMatchObject({
|
||||
mode: "apply",
|
||||
expectedPlanSha256: "f".repeat(64),
|
||||
});
|
||||
});
|
||||
});
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env tsx
|
||||
import "dotenv/config";
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
|
||||
import { createPrismaPlatformLeadService } from "@hivegrid/db/platform-leads-admin.server";
|
||||
import { disconnectDbClients, getDbClient } from "@hivegrid/db/client.server";
|
||||
import { runWithRlsContext } from "@hivegrid/db/rls-context.server";
|
||||
|
||||
import { closeSharedStores } from "../../apps/web/app/shared-store.server.js";
|
||||
import { userHasPermission } from "../../apps/web/app/utils/db/permissions/userRoles.db.server.js";
|
||||
|
||||
import {
|
||||
isSuccessfulPlatformLeadSourceBootstrapManifest,
|
||||
PlatformLeadSourceBootstrapError,
|
||||
runPlatformLeadSourceBootstrap,
|
||||
type PlatformLeadSourceBootstrapMode,
|
||||
} from "./source-bootstrap.js";
|
||||
|
||||
export type PlatformLeadSourceBootstrapCliOptions = {
|
||||
mode: PlatformLeadSourceBootstrapMode;
|
||||
actorAdminUserId: string;
|
||||
routingAdminUserId: string;
|
||||
manifestPath: string;
|
||||
runId: string;
|
||||
expectedPlanSha256?: string;
|
||||
};
|
||||
|
||||
export function parsePlatformLeadSourceBootstrapCliArgs(argv: readonly string[]): PlatformLeadSourceBootstrapCliOptions {
|
||||
const values = parseNamedArguments(argv);
|
||||
const allowed = new Set(["mode", "actor-admin-user-id", "routing-admin-user-id", "manifest", "run-id", "expected-plan-sha256"]);
|
||||
if ([...values.keys()].some((key) => !allowed.has(key))) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
const modeValue = required(values.get("mode"));
|
||||
if (modeValue !== "dry-run" && modeValue !== "apply") throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
const options: PlatformLeadSourceBootstrapCliOptions = {
|
||||
mode: modeValue,
|
||||
actorAdminUserId: identifier(values.get("actor-admin-user-id")),
|
||||
routingAdminUserId: identifier(values.get("routing-admin-user-id")),
|
||||
manifestPath: required(values.get("manifest")),
|
||||
runId: values.get("run-id")?.trim() || randomUUID(),
|
||||
};
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,190}$/u.test(options.runId)) {
|
||||
throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
}
|
||||
if (modeValue === "apply") {
|
||||
const expected = required(values.get("expected-plan-sha256")).toLowerCase();
|
||||
if (!/^[a-f0-9]{64}$/u.test(expected)) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
options.expectedPlanSha256 = expected;
|
||||
} else if (values.has("expected-plan-sha256")) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
return options;
|
||||
}
|
||||
|
||||
export async function runPlatformLeadSourceBootstrapCli(argv: readonly string[] = process.argv.slice(2)): Promise<number> {
|
||||
let options: PlatformLeadSourceBootstrapCliOptions;
|
||||
try {
|
||||
options = parsePlatformLeadSourceBootstrapCliArgs(argv);
|
||||
} catch (error) {
|
||||
writeSafeError(error);
|
||||
return 1;
|
||||
}
|
||||
|
||||
try {
|
||||
const manifest = await runWithRlsContext({ userId: options.actorAdminUserId }, async () => {
|
||||
if (!(await userHasPermission(options.actorAdminUserId, null, "admin.leads.admin"))) {
|
||||
throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
}
|
||||
const [canReadOwn, canWrite] = await Promise.all([
|
||||
userHasPermission(options.routingAdminUserId, null, "admin.leads.read_own"),
|
||||
userHasPermission(options.routingAdminUserId, null, "admin.leads.write"),
|
||||
]);
|
||||
if (!canReadOwn || !canWrite) {
|
||||
throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
}
|
||||
const db = getDbClient();
|
||||
const routingOperator = await db.adminUser.findUnique({
|
||||
where: { userId: options.routingAdminUserId },
|
||||
select: { user: { select: { active: true, pendingDeletion: true, pseudonymized: true } } },
|
||||
});
|
||||
if (
|
||||
!routingOperator?.user.active ||
|
||||
routingOperator.user.pendingDeletion ||
|
||||
routingOperator.user.pseudonymized
|
||||
) {
|
||||
throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
}
|
||||
const service = createPrismaPlatformLeadService();
|
||||
const existingSources = await service.listSources();
|
||||
return runPlatformLeadSourceBootstrap({
|
||||
mode: options.mode,
|
||||
runId: options.runId,
|
||||
actorAdminUserId: options.actorAdminUserId,
|
||||
routingAdminUserId: options.routingAdminUserId,
|
||||
existingSources,
|
||||
expectedPlanSha256: options.expectedPlanSha256,
|
||||
service,
|
||||
});
|
||||
});
|
||||
const serialized = `${JSON.stringify(manifest, null, 2)}\n`;
|
||||
await writeFile(options.manifestPath, serialized, { encoding: "utf8", flag: "wx", mode: 0o600 });
|
||||
process.stdout.write(serialized);
|
||||
return isSuccessfulPlatformLeadSourceBootstrapManifest(manifest) ? 0 : 2;
|
||||
} catch (error) {
|
||||
writeSafeError(error);
|
||||
return 1;
|
||||
} finally {
|
||||
await Promise.allSettled([closeSharedStores(), disconnectDbClients()]);
|
||||
}
|
||||
}
|
||||
|
||||
function parseNamedArguments(argv: readonly string[]): Map<string, string> {
|
||||
const values = new Map<string, string>();
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const argument = argv[index]!;
|
||||
if (argument === "--" && index === 0) continue;
|
||||
if (!argument.startsWith("--")) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
const equalsIndex = argument.indexOf("=");
|
||||
const key = argument.slice(2, equalsIndex === -1 ? undefined : equalsIndex);
|
||||
const value = equalsIndex === -1 ? argv[++index] : argument.slice(equalsIndex + 1);
|
||||
if (!key || !value || value.startsWith("--") || values.has(key)) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
values.set(key, value);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function required(value: string | undefined): string {
|
||||
const normalized = value?.trim();
|
||||
if (!normalized || /[\u0000-\u001f\u007f]/u.test(normalized)) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function identifier(value: string | undefined): string {
|
||||
const normalized = required(value);
|
||||
if (normalized.length > 191 || !/^[A-Za-z0-9_-]+$/u.test(normalized)) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function writeSafeError(error: unknown): void {
|
||||
const code = error instanceof PlatformLeadSourceBootstrapError ? error.code : "OPERATION_FAILED";
|
||||
process.stderr.write(`${JSON.stringify({ ok: false, code })}\n`);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runPlatformLeadSourceBootstrapCli()
|
||||
.then((exitCode) => {
|
||||
process.exitCode = exitCode;
|
||||
})
|
||||
.catch(() => {
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
import type { PlatformLeadSourceAdminDto } from "@hivegrid/db/platform-leads-admin.server";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
isSuccessfulPlatformLeadSourceBootstrapManifest,
|
||||
planPlatformLeadSourceBootstrap,
|
||||
runPlatformLeadSourceBootstrap,
|
||||
type PlatformLeadSourceBootstrapService,
|
||||
} from "./source-bootstrap.js";
|
||||
|
||||
function source(overrides: Partial<PlatformLeadSourceAdminDto> & Pick<PlatformLeadSourceAdminDto, "id" | "key">): PlatformLeadSourceAdminDto {
|
||||
return {
|
||||
id: overrides.id,
|
||||
key: overrides.key,
|
||||
displayName: overrides.displayName ?? overrides.key,
|
||||
createdAt: overrides.createdAt ?? "2026-07-16T00:00:00.000Z",
|
||||
updatedAt: overrides.updatedAt ?? "2026-07-16T00:00:00.000Z",
|
||||
enabled: overrides.enabled ?? false,
|
||||
defaultAssigneeAdminUserId: overrides.defaultAssigneeAdminUserId ?? null,
|
||||
notifyAdminUserIds: overrides.notifyAdminUserIds ?? [],
|
||||
rateLimitPerMinute: overrides.rateLimitPerMinute ?? 60,
|
||||
leadCount: overrides.leadCount ?? 0,
|
||||
metadata: overrides.metadata ?? null,
|
||||
credentials: overrides.credentials ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function clock(...values: string[]): () => Date {
|
||||
const dates = values.map((value) => new Date(value));
|
||||
return () => dates.shift() ?? new Date("2026-07-16T12:00:00.000Z");
|
||||
}
|
||||
|
||||
describe("platform lead source bootstrap", () => {
|
||||
it("plans all three sources and configures the current website for the supplied operator only", () => {
|
||||
const plan = planPlatformLeadSourceBootstrap([], "haris_user_id");
|
||||
|
||||
expect(plan.hash).toMatch(/^[a-f0-9]{64}$/u);
|
||||
expect(plan.actions).toEqual([
|
||||
{
|
||||
kind: "create",
|
||||
key: "hivegrid-website",
|
||||
input: expect.objectContaining({ enabled: true, defaultAssigneeAdminUserId: "haris_user_id", notifyAdminUserIds: ["haris_user_id"] }),
|
||||
},
|
||||
{
|
||||
kind: "create",
|
||||
key: "hivegrid-legacy-cloud",
|
||||
input: expect.objectContaining({ enabled: false, defaultAssigneeAdminUserId: null, notifyAdminUserIds: [] }),
|
||||
},
|
||||
{
|
||||
kind: "create",
|
||||
key: "lead-inbox-legacy",
|
||||
input: expect.objectContaining({ enabled: false, defaultAssigneeAdminUserId: null, notifyAdminUserIds: [] }),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves extra website recipients while disabling import-only legacy sources", () => {
|
||||
const existing = [
|
||||
source({
|
||||
id: "source_hivegrid",
|
||||
key: "hivegrid-website",
|
||||
enabled: false,
|
||||
defaultAssigneeAdminUserId: "other_operator",
|
||||
notifyAdminUserIds: ["other_operator"],
|
||||
}),
|
||||
source({ id: "source_cloud", key: "hivegrid-legacy-cloud", enabled: true, notifyAdminUserIds: ["other_operator"] }),
|
||||
source({ id: "source_inbox", key: "lead-inbox-legacy", enabled: false }),
|
||||
];
|
||||
|
||||
const plan = planPlatformLeadSourceBootstrap(existing, "haris");
|
||||
expect(plan.actions).toEqual([
|
||||
{
|
||||
kind: "update",
|
||||
key: "hivegrid-website",
|
||||
input: {
|
||||
sourceId: "source_hivegrid",
|
||||
enabled: true,
|
||||
defaultAssigneeAdminUserId: "haris",
|
||||
notifyAdminUserIds: ["haris", "other_operator"],
|
||||
},
|
||||
},
|
||||
{ kind: "update", key: "hivegrid-legacy-cloud", input: { sourceId: "source_cloud", enabled: false } },
|
||||
{ kind: "unchanged", key: "lead-inbox-legacy" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns a PII-free dry-run manifest and issues no credentials", async () => {
|
||||
const manifest = await runPlatformLeadSourceBootstrap({
|
||||
mode: "dry-run",
|
||||
runId: "bootstrap-dry-1",
|
||||
actorAdminUserId: "operator_admin",
|
||||
routingAdminUserId: "haris_private_identifier",
|
||||
existingSources: [],
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
|
||||
expect(manifest).toMatchObject({
|
||||
outcome: "validated",
|
||||
counts: { createPlanned: 3, updatePlanned: 0, unchanged: 0, created: 0, updated: 0, failed: 0 },
|
||||
credentialsIssued: 0,
|
||||
notificationsTriggered: 0,
|
||||
});
|
||||
expect(manifest.operatorBindingSha256).toMatch(/^[a-f0-9]{64}$/u);
|
||||
expect(JSON.stringify(manifest)).not.toContain("haris_private_identifier");
|
||||
expect(isSuccessfulPlatformLeadSourceBootstrapManifest(manifest)).toBe(true);
|
||||
});
|
||||
|
||||
it("blocks apply on plan drift before any source mutation", async () => {
|
||||
const createSource = vi.fn();
|
||||
const updateSource = vi.fn();
|
||||
const manifest = await runPlatformLeadSourceBootstrap({
|
||||
mode: "apply",
|
||||
runId: "bootstrap-apply-1",
|
||||
actorAdminUserId: "operator_admin",
|
||||
routingAdminUserId: "haris",
|
||||
existingSources: [],
|
||||
expectedPlanSha256: "0".repeat(64),
|
||||
service: { createSource, updateSource } as PlatformLeadSourceBootstrapService,
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
|
||||
expect(manifest.outcome).toBe("blocked");
|
||||
expect(createSource).not.toHaveBeenCalled();
|
||||
expect(updateSource).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("applies the confirmed plan through audited source service calls", async () => {
|
||||
const plan = planPlatformLeadSourceBootstrap([], "haris");
|
||||
const createSource = vi.fn(async (input) => source({ id: `created_${input.key}`, key: input.key, enabled: input.enabled }));
|
||||
const updateSource = vi.fn();
|
||||
const manifest = await runPlatformLeadSourceBootstrap({
|
||||
mode: "apply",
|
||||
runId: "bootstrap-apply-2",
|
||||
actorAdminUserId: "operator_admin",
|
||||
routingAdminUserId: "haris",
|
||||
existingSources: [],
|
||||
expectedPlanSha256: plan.hash,
|
||||
service: { createSource, updateSource } as PlatformLeadSourceBootstrapService,
|
||||
clock: clock("2026-07-16T10:00:00.000Z", "2026-07-16T10:00:01.000Z"),
|
||||
});
|
||||
|
||||
expect(createSource).toHaveBeenCalledTimes(3);
|
||||
expect(createSource).toHaveBeenCalledWith(expect.objectContaining({ actorAdminUserId: "operator_admin", requestId: "bootstrap-apply-2" }));
|
||||
expect(updateSource).not.toHaveBeenCalled();
|
||||
expect(manifest).toMatchObject({ outcome: "applied", counts: { created: 3, updated: 0, failed: 0 } });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,236 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import type { CreatePlatformLeadSourceInput, PlatformLeadSourceAdminDto, UpdatePlatformLeadSourceInput } from "@hivegrid/db/platform-leads-admin.server";
|
||||
|
||||
export type PlatformLeadSourceBootstrapMode = "dry-run" | "apply";
|
||||
|
||||
export const PLATFORM_LEAD_INITIAL_SOURCE_CONFIGURATION = Object.freeze([
|
||||
{ key: "hivegrid-website", displayName: "HiveGrid Website", currentWebsiteSource: true },
|
||||
{ key: "hivegrid-legacy-cloud", displayName: "HiveGrid Cloud Legacy", currentWebsiteSource: false },
|
||||
{ key: "lead-inbox-legacy", displayName: "Lead Inbox Legacy", currentWebsiteSource: false },
|
||||
] as const);
|
||||
|
||||
type CreateAction = {
|
||||
kind: "create";
|
||||
key: string;
|
||||
input: Omit<CreatePlatformLeadSourceInput, "actorAdminUserId" | "reason">;
|
||||
};
|
||||
|
||||
type UpdateAction = {
|
||||
kind: "update";
|
||||
key: string;
|
||||
input: Omit<UpdatePlatformLeadSourceInput, "actorAdminUserId" | "reason">;
|
||||
};
|
||||
|
||||
type UnchangedAction = { kind: "unchanged"; key: string };
|
||||
|
||||
export type PlatformLeadSourceBootstrapAction = CreateAction | UpdateAction | UnchangedAction;
|
||||
|
||||
export type PlatformLeadSourceBootstrapPlan = {
|
||||
hash: string;
|
||||
actions: PlatformLeadSourceBootstrapAction[];
|
||||
};
|
||||
|
||||
export type PlatformLeadSourceBootstrapService = {
|
||||
createSource(input: CreatePlatformLeadSourceInput): Promise<PlatformLeadSourceAdminDto>;
|
||||
updateSource(input: UpdatePlatformLeadSourceInput): Promise<PlatformLeadSourceAdminDto>;
|
||||
};
|
||||
|
||||
export type PlatformLeadSourceBootstrapManifest = {
|
||||
schemaVersion: 1;
|
||||
runId: string;
|
||||
mode: PlatformLeadSourceBootstrapMode;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
operatorBindingSha256: string;
|
||||
planSha256: string;
|
||||
expectedPlanSha256: string | null;
|
||||
planVerified: boolean | null;
|
||||
actions: Array<{ key: string; action: PlatformLeadSourceBootstrapAction["kind"] }>;
|
||||
counts: { createPlanned: number; updatePlanned: number; unchanged: number; created: number; updated: number; failed: number };
|
||||
outcome: "validated" | "applied" | "blocked" | "failed";
|
||||
credentialsIssued: 0;
|
||||
notificationsTriggered: 0;
|
||||
};
|
||||
|
||||
export class PlatformLeadSourceBootstrapError extends Error {
|
||||
public override readonly name = "PlatformLeadSourceBootstrapError";
|
||||
|
||||
public constructor(public readonly code: "INVALID_CONFIGURATION") {
|
||||
super(code);
|
||||
}
|
||||
}
|
||||
|
||||
export function planPlatformLeadSourceBootstrap(
|
||||
existingSources: readonly PlatformLeadSourceAdminDto[],
|
||||
routingAdminUserId: string
|
||||
): PlatformLeadSourceBootstrapPlan {
|
||||
const routingUserId = requireIdentifier(routingAdminUserId);
|
||||
const byKey = new Map<string, PlatformLeadSourceAdminDto>();
|
||||
for (const source of existingSources) {
|
||||
if (byKey.has(source.key)) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
byKey.set(source.key, source);
|
||||
}
|
||||
|
||||
const actions = PLATFORM_LEAD_INITIAL_SOURCE_CONFIGURATION.map<PlatformLeadSourceBootstrapAction>((desired) => {
|
||||
const existing = byKey.get(desired.key);
|
||||
if (!existing) {
|
||||
return {
|
||||
kind: "create",
|
||||
key: desired.key,
|
||||
input: {
|
||||
key: desired.key,
|
||||
displayName: desired.displayName,
|
||||
enabled: desired.currentWebsiteSource,
|
||||
defaultAssigneeAdminUserId: desired.currentWebsiteSource ? routingUserId : null,
|
||||
notifyAdminUserIds: desired.currentWebsiteSource ? [routingUserId] : [],
|
||||
rateLimitPerMinute: 60,
|
||||
metadata: { managedBy: "R-LEADS" },
|
||||
},
|
||||
};
|
||||
}
|
||||
if (!desired.currentWebsiteSource) {
|
||||
return existing.enabled
|
||||
? { kind: "update", key: desired.key, input: { sourceId: existing.id, enabled: false } }
|
||||
: { kind: "unchanged", key: desired.key };
|
||||
}
|
||||
|
||||
const notifyAdminUserIds = [...new Set([...existing.notifyAdminUserIds, routingUserId])].sort();
|
||||
const input: UpdateAction["input"] = { sourceId: existing.id };
|
||||
if (!existing.enabled) input.enabled = true;
|
||||
if (existing.defaultAssigneeAdminUserId !== routingUserId) input.defaultAssigneeAdminUserId = routingUserId;
|
||||
if (!sameStringList(existing.notifyAdminUserIds, notifyAdminUserIds)) input.notifyAdminUserIds = notifyAdminUserIds;
|
||||
if (Object.keys(input).length === 1) return { kind: "unchanged", key: desired.key };
|
||||
return { kind: "update", key: desired.key, input };
|
||||
});
|
||||
|
||||
return { hash: hashPlan(actions), actions };
|
||||
}
|
||||
|
||||
export async function runPlatformLeadSourceBootstrap(input: {
|
||||
mode: PlatformLeadSourceBootstrapMode;
|
||||
runId: string;
|
||||
actorAdminUserId: string;
|
||||
routingAdminUserId: string;
|
||||
existingSources: readonly PlatformLeadSourceAdminDto[];
|
||||
expectedPlanSha256?: string;
|
||||
service?: PlatformLeadSourceBootstrapService;
|
||||
clock?: () => Date;
|
||||
}): Promise<PlatformLeadSourceBootstrapManifest> {
|
||||
const clock = input.clock ?? (() => new Date());
|
||||
const startedAt = validDate(clock());
|
||||
const actorAdminUserId = requireIdentifier(input.actorAdminUserId);
|
||||
const routingAdminUserId = requireIdentifier(input.routingAdminUserId);
|
||||
const runId = requireRunId(input.runId);
|
||||
const plan = planPlatformLeadSourceBootstrap(input.existingSources, routingAdminUserId);
|
||||
const expectedPlanSha256 = input.expectedPlanSha256?.trim().toLowerCase() ?? null;
|
||||
const planVerified = expectedPlanSha256 === null ? null : plan.hash === expectedPlanSha256;
|
||||
if (input.mode === "apply" && (!expectedPlanSha256 || !/^[a-f0-9]{64}$/u.test(expectedPlanSha256) || !input.service)) {
|
||||
throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
}
|
||||
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
let failed = 0;
|
||||
if (input.mode === "apply" && planVerified) {
|
||||
for (const action of plan.actions) {
|
||||
try {
|
||||
if (action.kind === "create") {
|
||||
await input.service!.createSource({
|
||||
...action.input,
|
||||
actorAdminUserId,
|
||||
reason: "R-LEADS initial platform lead source bootstrap",
|
||||
requestId: runId,
|
||||
});
|
||||
created += 1;
|
||||
} else if (action.kind === "update") {
|
||||
await input.service!.updateSource({
|
||||
...action.input,
|
||||
actorAdminUserId,
|
||||
reason: "R-LEADS platform lead source routing bootstrap",
|
||||
requestId: runId,
|
||||
});
|
||||
updated += 1;
|
||||
}
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const createPlanned = plan.actions.filter(({ kind }) => kind === "create").length;
|
||||
const updatePlanned = plan.actions.filter(({ kind }) => kind === "update").length;
|
||||
const unchanged = plan.actions.filter(({ kind }) => kind === "unchanged").length;
|
||||
const outcome = input.mode === "dry-run" ? "validated" : planVerified === false ? "blocked" : failed > 0 ? "failed" : "applied";
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
mode: input.mode,
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: validDate(clock()).toISOString(),
|
||||
operatorBindingSha256: sha256(routingAdminUserId),
|
||||
planSha256: plan.hash,
|
||||
expectedPlanSha256,
|
||||
planVerified,
|
||||
actions: plan.actions.map(({ key, kind }) => ({ key, action: kind })),
|
||||
counts: { createPlanned, updatePlanned, unchanged, created, updated, failed },
|
||||
outcome,
|
||||
credentialsIssued: 0,
|
||||
notificationsTriggered: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function isSuccessfulPlatformLeadSourceBootstrapManifest(manifest: PlatformLeadSourceBootstrapManifest): boolean {
|
||||
return manifest.outcome === "validated" || manifest.outcome === "applied";
|
||||
}
|
||||
|
||||
function hashPlan(actions: readonly PlatformLeadSourceBootstrapAction[]): string {
|
||||
return sha256(
|
||||
JSON.stringify(
|
||||
actions.map((action) =>
|
||||
action.kind === "unchanged" ? action : { kind: action.kind, key: action.key, input: sortRecord(action.input as Record<string, unknown>) }
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function sortRecord(record: Record<string, unknown>): Record<string, unknown> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(record)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, value]) => [key, Array.isArray(value) ? [...value].sort() : value])
|
||||
);
|
||||
}
|
||||
|
||||
function sameStringList(left: readonly string[], right: readonly string[]): boolean {
|
||||
if (left.length !== right.length) return false;
|
||||
const sortedLeft = [...left].sort();
|
||||
return sortedLeft.every((value, index) => value === right[index]);
|
||||
}
|
||||
|
||||
function requireIdentifier(value: string): string {
|
||||
if (typeof value !== "string") throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
const normalized = value.trim();
|
||||
if (!normalized || normalized.length > 191 || !/^[A-Za-z0-9_-]+$/u.test(normalized)) {
|
||||
throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireRunId(value: string): string {
|
||||
if (typeof value !== "string") throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
const normalized = value.trim();
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,190}$/u.test(normalized)) {
|
||||
throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function validDate(value: Date): Date {
|
||||
if (!(value instanceof Date) || !Number.isFinite(value.getTime())) throw new PlatformLeadSourceBootstrapError("INVALID_CONFIGURATION");
|
||||
return value;
|
||||
}
|
||||
|
||||
function sha256(value: string): string {
|
||||
return createHash("sha256").update(value, "utf8").digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"include": ["./*.ts"],
|
||||
"compilerOptions": {
|
||||
"types": ["@react-router/node", "vite/client"],
|
||||
"noEmit": true
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user