699 lines
27 KiB
TypeScript
699 lines
27 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
|
|
import {
|
|
buildLegacyPlatformLeadExternalId,
|
|
normalizePlatformLeadPayload,
|
|
PLATFORM_LEAD_LEGACY_STORES,
|
|
type PlatformLeadImportInput,
|
|
type PlatformLeadIngestionResult,
|
|
type PlatformLeadLegacyActivityInput,
|
|
type PlatformLeadPayload,
|
|
} from "@hivegrid/db/platform-leads.server";
|
|
|
|
export type PlatformLeadLegacyStore = (typeof PLATFORM_LEAD_LEGACY_STORES)[number];
|
|
export type PlatformLeadBackfillMode = "dry-run" | "apply" | "replay";
|
|
export type PlatformLeadBackfillInputFormat = "jsonl";
|
|
export type PlatformLeadBackfillInitialStatus = NonNullable<PlatformLeadImportInput["initialStatus"]>;
|
|
|
|
export const PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE = Object.freeze({
|
|
"lead-inbox-postgres": "lead-inbox-legacy",
|
|
"hivegrid-website-production-sqlite": "hivegrid-website",
|
|
"hivegrid-website-staging-sqlite": "hivegrid-website",
|
|
"hivegrid-cloud-legacy": "hivegrid-legacy-cloud",
|
|
} as const satisfies Record<PlatformLeadLegacyStore, string>);
|
|
|
|
export const PLATFORM_LEAD_BACKFILL_REJECTION_CODES = [
|
|
"INVALID_JSON",
|
|
"INVALID_RECORD",
|
|
"MISSING_LEGACY_ID",
|
|
"INVALID_TIMESTAMP",
|
|
"INVALID_LEAD_PAYLOAD",
|
|
"DUPLICATE_LEGACY_ID",
|
|
"INVALID_ACTIVITY",
|
|
"UNSUPPORTED_ACTIVITY_TYPE",
|
|
"UNSUPPORTED_LEGACY_STATUS",
|
|
] as const;
|
|
|
|
export type PlatformLeadBackfillRejectionCode = (typeof PLATFORM_LEAD_BACKFILL_REJECTION_CODES)[number];
|
|
export type PlatformLeadBackfillGateFailure =
|
|
| "INPUT_HASH_MISMATCH"
|
|
| "ACCEPTED_COUNT_MISMATCH"
|
|
| "REJECTED_ROWS_PRESENT"
|
|
| "IMPORT_FAILURES_PRESENT"
|
|
| "REPLAY_CREATED_ROWS";
|
|
|
|
export type PlatformLeadBackfillImportRecord = {
|
|
legacyId: string;
|
|
capturedAt: Date;
|
|
initialStatus?: PlatformLeadBackfillInitialStatus;
|
|
payload: PlatformLeadPayload & { externalId: string };
|
|
legacyActivities: PlatformLeadLegacyActivityInput[];
|
|
};
|
|
|
|
export type PlatformLeadBackfillParseResult = {
|
|
legacyStore: PlatformLeadLegacyStore;
|
|
sourceKey: string;
|
|
inputSha256: string;
|
|
inputBytes: number;
|
|
scanned: number;
|
|
accepted: PlatformLeadBackfillImportRecord[];
|
|
rejectionCounts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>;
|
|
};
|
|
|
|
export type PlatformLeadBackfillManifest = {
|
|
schemaVersion: 1;
|
|
runId: string;
|
|
mode: PlatformLeadBackfillMode;
|
|
legacyStore: PlatformLeadLegacyStore;
|
|
sourceKey: string;
|
|
sourceId: string | null;
|
|
startedAt: string;
|
|
finishedAt: string;
|
|
input: {
|
|
sha256: string;
|
|
bytes: number;
|
|
expectedSha256: string | null;
|
|
hashVerified: boolean | null;
|
|
};
|
|
counts: {
|
|
scanned: number;
|
|
accepted: number;
|
|
expectedAccepted: number | null;
|
|
acceptedCountVerified: boolean | null;
|
|
rejected: number;
|
|
created: number;
|
|
replayed: number;
|
|
failed: number;
|
|
};
|
|
rejectionCounts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>;
|
|
timeRange: {
|
|
firstCapturedAt: string | null;
|
|
lastCapturedAt: string | null;
|
|
};
|
|
gateFailures: PlatformLeadBackfillGateFailure[];
|
|
outcome: "validated" | "applied" | "replayed" | "blocked" | "failed";
|
|
notificationsTriggered: 0;
|
|
automationsTriggered: 0;
|
|
};
|
|
|
|
export type PlatformLeadBackfillImporter = (input: PlatformLeadImportInput) => Promise<PlatformLeadIngestionResult>;
|
|
|
|
export type RunPlatformLeadBackfillInput = {
|
|
mode: PlatformLeadBackfillMode;
|
|
legacyStore: PlatformLeadLegacyStore;
|
|
format: PlatformLeadBackfillInputFormat;
|
|
bytes: Uint8Array;
|
|
runId: string;
|
|
sourceId?: string;
|
|
actorAdminUserId?: string;
|
|
expectedInputSha256?: string;
|
|
expectedAccepted?: number;
|
|
importer?: PlatformLeadBackfillImporter;
|
|
clock?: () => Date;
|
|
};
|
|
|
|
type CandidateRow = { rowNumber: number; value?: unknown; rejectionCode?: PlatformLeadBackfillRejectionCode };
|
|
|
|
class PlatformLeadBackfillRowError extends Error {
|
|
public override readonly name = "PlatformLeadBackfillRowError";
|
|
|
|
public constructor(public readonly code: PlatformLeadBackfillRejectionCode) {
|
|
super(code);
|
|
}
|
|
}
|
|
|
|
export class PlatformLeadBackfillInputError extends Error {
|
|
public override readonly name = "PlatformLeadBackfillInputError";
|
|
|
|
public constructor(public readonly code: "INVALID_INPUT" | "INVALID_CONFIGURATION") {
|
|
super(code);
|
|
}
|
|
}
|
|
|
|
export function isPlatformLeadLegacyStore(value: string): value is PlatformLeadLegacyStore {
|
|
return (PLATFORM_LEAD_LEGACY_STORES as readonly string[]).includes(value);
|
|
}
|
|
|
|
export function parsePlatformLeadBackfillInput(input: {
|
|
legacyStore: PlatformLeadLegacyStore;
|
|
format: PlatformLeadBackfillInputFormat;
|
|
bytes: Uint8Array;
|
|
}): PlatformLeadBackfillParseResult {
|
|
if (input.format !== "jsonl") throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
|
|
|
const inputBytes = Buffer.from(input.bytes);
|
|
const inputSha256 = sha256(inputBytes);
|
|
let text: string;
|
|
try {
|
|
text = new TextDecoder("utf-8", { fatal: true }).decode(inputBytes).replace(/^\uFEFF/u, "");
|
|
} catch {
|
|
throw new PlatformLeadBackfillInputError("INVALID_INPUT");
|
|
}
|
|
|
|
const candidates = parseJsonCandidates(text);
|
|
const accepted: PlatformLeadBackfillImportRecord[] = [];
|
|
const rejectionCounts: Partial<Record<PlatformLeadBackfillRejectionCode, number>> = {};
|
|
const seenExternalIds = new Set<string>();
|
|
|
|
for (const candidate of candidates) {
|
|
if (candidate.rejectionCode) {
|
|
incrementCount(rejectionCounts, candidate.rejectionCode);
|
|
continue;
|
|
}
|
|
try {
|
|
const mapped = mapCandidate(input.legacyStore, candidate.value);
|
|
if (seenExternalIds.has(mapped.payload.externalId)) {
|
|
throw new PlatformLeadBackfillRowError("DUPLICATE_LEGACY_ID");
|
|
}
|
|
seenExternalIds.add(mapped.payload.externalId);
|
|
accepted.push(mapped);
|
|
} catch (error) {
|
|
incrementCount(rejectionCounts, toRejectionCode(error));
|
|
}
|
|
}
|
|
|
|
return {
|
|
legacyStore: input.legacyStore,
|
|
sourceKey: PLATFORM_LEAD_SOURCE_KEY_BY_LEGACY_STORE[input.legacyStore],
|
|
inputSha256,
|
|
inputBytes: inputBytes.byteLength,
|
|
scanned: candidates.length,
|
|
accepted,
|
|
rejectionCounts,
|
|
};
|
|
}
|
|
|
|
export async function runPlatformLeadBackfill(input: RunPlatformLeadBackfillInput): Promise<PlatformLeadBackfillManifest> {
|
|
const clock = input.clock ?? (() => new Date());
|
|
const startedAt = requireValidDate(clock());
|
|
const runId = normalizeRunId(input.runId);
|
|
const parsed = parsePlatformLeadBackfillInput(input);
|
|
const expectedInputSha256 = input.expectedInputSha256?.trim().toLowerCase() ?? null;
|
|
const hashVerified = expectedInputSha256 === null ? null : parsed.inputSha256 === expectedInputSha256;
|
|
const acceptedCountVerified = input.expectedAccepted === undefined ? null : parsed.accepted.length === input.expectedAccepted;
|
|
const rejected = sumCounts(parsed.rejectionCounts);
|
|
const gateFailures: PlatformLeadBackfillGateFailure[] = [];
|
|
if (hashVerified === false) gateFailures.push("INPUT_HASH_MISMATCH");
|
|
if (acceptedCountVerified === false) gateFailures.push("ACCEPTED_COUNT_MISMATCH");
|
|
if (rejected > 0) gateFailures.push("REJECTED_ROWS_PRESENT");
|
|
|
|
if (input.mode !== "dry-run") {
|
|
if (!expectedInputSha256 || !/^[a-f0-9]{64}$/u.test(expectedInputSha256) || input.expectedAccepted === undefined) {
|
|
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
|
}
|
|
if (!Number.isSafeInteger(input.expectedAccepted) || input.expectedAccepted < 0) {
|
|
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
|
}
|
|
if (!input.sourceId || !input.actorAdminUserId || !input.importer) {
|
|
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
|
}
|
|
}
|
|
|
|
let created = 0;
|
|
let replayed = 0;
|
|
let failed = 0;
|
|
if (input.mode !== "dry-run" && gateFailures.length === 0) {
|
|
for (const record of parsed.accepted) {
|
|
try {
|
|
const result = await input.importer!({
|
|
channel: "import",
|
|
sourceId: input.sourceId!,
|
|
actorAdminUserId: input.actorAdminUserId!,
|
|
reason: `R-LEADS deterministic ${input.legacyStore} backfill`,
|
|
requestId: runId,
|
|
capturedAt: record.capturedAt,
|
|
initialStatus: record.initialStatus,
|
|
payload: record.payload,
|
|
legacyActivities: record.legacyActivities,
|
|
});
|
|
if (result.outcome === "created") created += 1;
|
|
else replayed += 1;
|
|
} catch {
|
|
failed += 1;
|
|
}
|
|
}
|
|
if (failed > 0) gateFailures.push("IMPORT_FAILURES_PRESENT");
|
|
if (input.mode === "replay" && created > 0) gateFailures.push("REPLAY_CREATED_ROWS");
|
|
}
|
|
|
|
const finishedAt = requireValidDate(clock());
|
|
const range = capturedAtRange(parsed.accepted);
|
|
const outcome = resolveOutcome(input.mode, gateFailures);
|
|
return {
|
|
schemaVersion: 1,
|
|
runId,
|
|
mode: input.mode,
|
|
legacyStore: parsed.legacyStore,
|
|
sourceKey: parsed.sourceKey,
|
|
sourceId: input.sourceId ?? null,
|
|
startedAt: startedAt.toISOString(),
|
|
finishedAt: finishedAt.toISOString(),
|
|
input: {
|
|
sha256: parsed.inputSha256,
|
|
bytes: parsed.inputBytes,
|
|
expectedSha256: expectedInputSha256,
|
|
hashVerified,
|
|
},
|
|
counts: {
|
|
scanned: parsed.scanned,
|
|
accepted: parsed.accepted.length,
|
|
expectedAccepted: input.expectedAccepted ?? null,
|
|
acceptedCountVerified,
|
|
rejected,
|
|
created,
|
|
replayed,
|
|
failed,
|
|
},
|
|
rejectionCounts: parsed.rejectionCounts,
|
|
timeRange: range,
|
|
gateFailures,
|
|
outcome,
|
|
notificationsTriggered: 0,
|
|
automationsTriggered: 0,
|
|
};
|
|
}
|
|
|
|
export function isSuccessfulPlatformLeadBackfillManifest(manifest: PlatformLeadBackfillManifest): boolean {
|
|
return manifest.gateFailures.length === 0 && manifest.outcome !== "blocked" && manifest.outcome !== "failed";
|
|
}
|
|
|
|
function parseJsonCandidates(text: string): CandidateRow[] {
|
|
if (!text.trim()) return [];
|
|
const trimmed = text.trim();
|
|
if (trimmed.startsWith("[")) {
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
if (!Array.isArray(parsed)) throw new PlatformLeadBackfillInputError("INVALID_INPUT");
|
|
return parsed.map((value, index) => ({ rowNumber: index + 1, value }));
|
|
} catch (error) {
|
|
if (error instanceof PlatformLeadBackfillInputError) throw error;
|
|
throw new PlatformLeadBackfillInputError("INVALID_INPUT");
|
|
}
|
|
}
|
|
return text
|
|
.split(/\r?\n/u)
|
|
.map((line, index) => ({ line: line.trim(), rowNumber: index + 1 }))
|
|
.filter(({ line }) => line.length > 0)
|
|
.map(({ line, rowNumber }) => {
|
|
try {
|
|
return { rowNumber, value: JSON.parse(line) as unknown };
|
|
} catch {
|
|
return { rowNumber, rejectionCode: "INVALID_JSON" as const };
|
|
}
|
|
});
|
|
}
|
|
|
|
function mapCandidate(legacyStore: PlatformLeadLegacyStore, value: unknown): PlatformLeadBackfillImportRecord {
|
|
const record = requireRecord(value);
|
|
switch (legacyStore) {
|
|
case "lead-inbox-postgres":
|
|
return mapLeadInboxRecord(legacyStore, record);
|
|
case "hivegrid-website-production-sqlite":
|
|
case "hivegrid-website-staging-sqlite":
|
|
return mapHiveGridWebsiteRecord(legacyStore, record);
|
|
case "hivegrid-cloud-legacy":
|
|
return mapHiveGridCloudRecord(legacyStore, record);
|
|
}
|
|
}
|
|
|
|
function mapLeadInboxRecord(legacyStore: PlatformLeadLegacyStore, record: Record<string, unknown>): PlatformLeadBackfillImportRecord {
|
|
const legacyId = requireLegacyId(field(record, "id"));
|
|
const meta = optionalObjectOrJson(field(record, "meta"));
|
|
return buildRecord({
|
|
legacyStore,
|
|
legacyId,
|
|
capturedAt: requireTimestamp(field(record, "created_at", "createdAt")),
|
|
metadata: legacyMetadata(record, meta),
|
|
payload: {
|
|
...nameFields(record),
|
|
email: optionalString(field(record, "email")),
|
|
phone: optionalString(field(record, "phone")),
|
|
company: optionalString(field(record, "company")),
|
|
message: optionalString(field(record, "message")),
|
|
sourceDetail: optionalString(field(record, "source", "site")),
|
|
...attributionFields(record, meta),
|
|
},
|
|
});
|
|
}
|
|
|
|
function mapHiveGridWebsiteRecord(legacyStore: PlatformLeadLegacyStore, record: Record<string, unknown>): PlatformLeadBackfillImportRecord {
|
|
const legacyId = requireLegacyId(field(record, "id"));
|
|
return buildRecord({
|
|
legacyStore,
|
|
legacyId,
|
|
capturedAt: requireTimestamp(field(record, "created_at", "createdAt")),
|
|
payload: {
|
|
...nameFields(record),
|
|
email: optionalString(field(record, "email")),
|
|
company: optionalString(field(record, "company")),
|
|
message: optionalString(field(record, "message")),
|
|
sourceDetail: "legacy-website-sqlite",
|
|
},
|
|
});
|
|
}
|
|
|
|
function mapHiveGridCloudRecord(legacyStore: PlatformLeadLegacyStore, record: Record<string, unknown>): PlatformLeadBackfillImportRecord {
|
|
const legacyId = requireLegacyId(field(record, "id"));
|
|
const legacyStatus = optionalLegacyStatus(field(record, "status"));
|
|
const status = mapLegacyStatus(legacyStatus);
|
|
const eventsValue = field(record, "events");
|
|
if (eventsValue !== undefined && !Array.isArray(eventsValue)) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
|
const capturedAt = requireTimestamp(field(record, "firstSeenAt", "first_seen_at", "createdAt", "created_at"));
|
|
const legacyActivities = [
|
|
...(eventsValue ?? []).map((event) => mapLegacyCloudActivity(event)),
|
|
...mapLegacyCloudNotes({ legacyId, notes: field(record, "notes"), capturedAt }),
|
|
];
|
|
return buildRecord({
|
|
legacyStore,
|
|
legacyId,
|
|
capturedAt,
|
|
initialStatus: status,
|
|
metadata: legacyStatus && legacyStatus !== status ? { legacyStatus } : undefined,
|
|
legacyActivities,
|
|
payload: {
|
|
firstName: optionalString(field(record, "firstName", "first_name")),
|
|
lastName: optionalString(field(record, "lastName", "last_name")),
|
|
email: optionalString(field(record, "email")),
|
|
phone: optionalString(field(record, "phone")),
|
|
company: optionalString(field(record, "company")),
|
|
sourceDetail: optionalString(field(record, "source")),
|
|
...attributionFields(record),
|
|
},
|
|
});
|
|
}
|
|
|
|
function mapLegacyCloudActivity(value: unknown): PlatformLeadLegacyActivityInput {
|
|
const event = requireRecord(value, "INVALID_ACTIVITY");
|
|
const rawType = requireString(field(event, "type"), "INVALID_ACTIVITY");
|
|
const type = mapLegacyActivityType(rawType);
|
|
const payload = normalizeActivityPayload(field(event, "payload"));
|
|
const actorId = optionalString(field(event, "actorId", "actor_id"), "INVALID_ACTIVITY");
|
|
if (actorId) payload.legacyActorId = actorId;
|
|
if (rawType === "re-captured" || type === "legacy_event") payload.legacyType = rawType;
|
|
return {
|
|
externalActivityId: requireLegacyId(field(event, "id"), "INVALID_ACTIVITY"),
|
|
occurredAt: requireTimestamp(field(event, "createdAt", "created_at"), "INVALID_ACTIVITY"),
|
|
type,
|
|
payload,
|
|
};
|
|
}
|
|
|
|
function mapLegacyActivityType(value: string): PlatformLeadLegacyActivityInput["type"] {
|
|
const mapping: Record<string, PlatformLeadLegacyActivityInput["type"] | undefined> = {
|
|
captured: "captured",
|
|
"re-captured": "captured",
|
|
status_changed: "status_changed",
|
|
"status-changed": "status_changed",
|
|
assigned: "assigned",
|
|
note_added: "note_added",
|
|
"note-added": "note_added",
|
|
archived: "archived",
|
|
restored: "restored",
|
|
duplicate_hint_recorded: "duplicate_hint_recorded",
|
|
"score-changed": "legacy_event",
|
|
score_changed: "legacy_event",
|
|
converted: "legacy_event",
|
|
};
|
|
const mapped = mapping[value];
|
|
if (!mapped) throw new PlatformLeadBackfillRowError("UNSUPPORTED_ACTIVITY_TYPE");
|
|
return mapped;
|
|
}
|
|
|
|
function mapLegacyCloudNotes(input: {
|
|
legacyId: string;
|
|
notes: unknown;
|
|
capturedAt: Date;
|
|
}): PlatformLeadLegacyActivityInput[] {
|
|
if (input.notes === undefined || input.notes === null) return [];
|
|
const notes = requireRecord(input.notes, "INVALID_ACTIVITY");
|
|
const mapped: PlatformLeadLegacyActivityInput[] = [];
|
|
const history = field(notes, "history");
|
|
if (history !== undefined && !Array.isArray(history)) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
|
for (const [index, value] of (history ?? []).entries()) {
|
|
const entry = requireRecord(value, "INVALID_ACTIVITY");
|
|
const note = optionalNote(field(entry, "body"));
|
|
if (!note) continue;
|
|
mapped.push({
|
|
externalActivityId: `${input.legacyId}:notes:history:${index}`,
|
|
occurredAt: optionalTimestamp(field(entry, "at")) ?? input.capturedAt,
|
|
type: "note_added",
|
|
payload: { note, legacyType: "lead.notes.history" },
|
|
});
|
|
}
|
|
const current = optionalNote(field(notes, "body"));
|
|
if (current) {
|
|
mapped.push({
|
|
externalActivityId: `${input.legacyId}:notes:current`,
|
|
occurredAt: optionalTimestamp(field(notes, "updatedAt", "updated_at")) ?? input.capturedAt,
|
|
type: "note_added",
|
|
payload: { note: current, legacyType: "lead.notes.current" },
|
|
});
|
|
}
|
|
return mapped;
|
|
}
|
|
|
|
function normalizeActivityPayload(value: unknown): Record<string, string | number | boolean | null> {
|
|
if (value === null || value === undefined) return {};
|
|
const payload = requireRecord(value, "INVALID_ACTIVITY");
|
|
const entries = Object.entries(payload);
|
|
if (entries.length > 32) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
|
const normalized: Record<string, string | number | boolean | null> = {};
|
|
for (const [key, entry] of entries) {
|
|
if (!/^[A-Za-z][A-Za-z0-9_-]{0,63}$/u.test(key)) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
|
if (entry === null || typeof entry === "boolean" || (typeof entry === "number" && Number.isFinite(entry))) {
|
|
normalized[key] = entry;
|
|
} else if (typeof entry === "string" && entry.length <= 1_000 && !hasControlCharacters(entry)) {
|
|
normalized[key] = entry;
|
|
} else throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
|
}
|
|
if (Buffer.byteLength(JSON.stringify(normalized), "utf8") > 4_096) throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
|
return normalized;
|
|
}
|
|
|
|
function buildRecord(input: {
|
|
legacyStore: PlatformLeadLegacyStore;
|
|
legacyId: string;
|
|
capturedAt: Date;
|
|
initialStatus?: PlatformLeadBackfillInitialStatus;
|
|
payload: PlatformLeadPayload;
|
|
metadata?: Record<string, string | number | boolean | null>;
|
|
legacyActivities?: PlatformLeadLegacyActivityInput[];
|
|
}): PlatformLeadBackfillImportRecord {
|
|
const externalId = buildLegacyPlatformLeadExternalId({ legacyStore: input.legacyStore, legacyId: input.legacyId });
|
|
const payload: PlatformLeadPayload & { externalId: string } = {
|
|
...input.payload,
|
|
externalId,
|
|
metadata: {
|
|
...input.metadata,
|
|
legacyId: sha256(Buffer.from(input.legacyId, "utf8")),
|
|
legacyStore: input.legacyStore,
|
|
},
|
|
};
|
|
try {
|
|
normalizePlatformLeadPayload(payload);
|
|
} catch {
|
|
throw new PlatformLeadBackfillRowError("INVALID_LEAD_PAYLOAD");
|
|
}
|
|
return {
|
|
legacyId: input.legacyId,
|
|
capturedAt: input.capturedAt,
|
|
initialStatus: input.initialStatus,
|
|
payload,
|
|
legacyActivities: input.legacyActivities ?? [],
|
|
};
|
|
}
|
|
|
|
function nameFields(record: Record<string, unknown>): Pick<PlatformLeadPayload, "firstName" | "lastName"> {
|
|
const explicitFirstName = optionalString(field(record, "firstName", "first_name"));
|
|
const explicitLastName = optionalString(field(record, "lastName", "last_name"));
|
|
if (explicitFirstName || explicitLastName) return { firstName: explicitFirstName, lastName: explicitLastName };
|
|
const fullName = optionalString(field(record, "name"));
|
|
if (!fullName) return { firstName: null, lastName: null };
|
|
const [firstName, ...rest] = fullName.split(/\s+/u);
|
|
return { firstName: firstName ?? null, lastName: rest.join(" ") || null };
|
|
}
|
|
|
|
function attributionFields(
|
|
...records: Array<Record<string, unknown> | null>
|
|
): Pick<PlatformLeadPayload, "utmSource" | "utmMedium" | "utmCampaign" | "utmTerm" | "utmContent" | "referrer" | "landingPage"> {
|
|
const find = (...keys: string[]): unknown => {
|
|
for (const record of records) {
|
|
if (!record) continue;
|
|
const value = field(record, ...keys);
|
|
if (value !== undefined && value !== null && value !== "") return value;
|
|
}
|
|
return undefined;
|
|
};
|
|
return {
|
|
utmSource: optionalString(find("utmSource", "utm_source")),
|
|
utmMedium: optionalString(find("utmMedium", "utm_medium")),
|
|
utmCampaign: optionalString(find("utmCampaign", "utm_campaign")),
|
|
utmTerm: optionalString(find("utmTerm", "utm_term")),
|
|
utmContent: optionalString(find("utmContent", "utm_content")),
|
|
referrer: optionalString(find("referrer")),
|
|
landingPage: optionalString(find("landingPage", "landing_page")),
|
|
};
|
|
}
|
|
|
|
function legacyMetadata(...records: Array<Record<string, unknown> | null>): Record<string, string | number | boolean | null> {
|
|
const aliases = {
|
|
consent: ["consent"],
|
|
formId: ["formId", "form_id"],
|
|
formName: ["formName", "form_name"],
|
|
locale: ["locale"],
|
|
productInterest: ["productInterest", "product_interest"],
|
|
submissionId: ["submissionId", "submission_id"],
|
|
} as const;
|
|
const result: Record<string, string | number | boolean | null> = {};
|
|
for (const [target, keys] of Object.entries(aliases)) {
|
|
let value: unknown;
|
|
for (const record of records) {
|
|
if (!record) continue;
|
|
value = field(record, ...keys);
|
|
if (value !== undefined) break;
|
|
}
|
|
if (value === undefined) continue;
|
|
if (value === null || typeof value === "boolean" || (typeof value === "number" && Number.isFinite(value))) result[target] = value;
|
|
else if (typeof value === "string" && value.length <= 500 && !hasControlCharacters(value)) result[target] = value;
|
|
else throw new PlatformLeadBackfillRowError("INVALID_RECORD");
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function optionalLegacyStatus(value: unknown): string | null {
|
|
if (value === undefined || value === null || value === "") return null;
|
|
if (typeof value !== "string") throw new PlatformLeadBackfillRowError("UNSUPPORTED_LEGACY_STATUS");
|
|
if (["new", "contacted", "qualified", "trial", "customer", "churned", "disqualified"].includes(value)) return value;
|
|
throw new PlatformLeadBackfillRowError("UNSUPPORTED_LEGACY_STATUS");
|
|
}
|
|
|
|
function mapLegacyStatus(value: string | null): PlatformLeadBackfillInitialStatus {
|
|
if (!value) return "new";
|
|
if (value === "trial" || value === "customer") return "qualified";
|
|
if (value === "churned") return "disqualified";
|
|
return value as PlatformLeadBackfillInitialStatus;
|
|
}
|
|
|
|
function requireRecord(value: unknown, code: PlatformLeadBackfillRejectionCode = "INVALID_RECORD"): Record<string, unknown> {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw new PlatformLeadBackfillRowError(code);
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function field(record: Record<string, unknown>, ...keys: string[]): unknown {
|
|
for (const key of keys) {
|
|
if (Object.prototype.hasOwnProperty.call(record, key)) return record[key];
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function optionalObjectOrJson(value: unknown): Record<string, unknown> | null {
|
|
if (value === null || value === undefined || value === "") return null;
|
|
if (typeof value === "string") {
|
|
try {
|
|
return requireRecord(JSON.parse(value) as unknown);
|
|
} catch {
|
|
throw new PlatformLeadBackfillRowError("INVALID_RECORD");
|
|
}
|
|
}
|
|
return requireRecord(value);
|
|
}
|
|
|
|
function requireLegacyId(value: unknown, code: PlatformLeadBackfillRejectionCode = "MISSING_LEGACY_ID"): string {
|
|
const normalized = optionalLegacyId(value);
|
|
if (!normalized) throw new PlatformLeadBackfillRowError(code);
|
|
return normalized;
|
|
}
|
|
|
|
function optionalLegacyId(value: unknown): string | null {
|
|
if (typeof value !== "string" && typeof value !== "number") return null;
|
|
const normalized = String(value).trim();
|
|
if (!normalized || normalized.length > 2_048 || hasControlCharacters(normalized)) return null;
|
|
return normalized;
|
|
}
|
|
|
|
function requireString(value: unknown, code: PlatformLeadBackfillRejectionCode): string {
|
|
const normalized = optionalString(value, code);
|
|
if (!normalized) throw new PlatformLeadBackfillRowError(code);
|
|
return normalized;
|
|
}
|
|
|
|
function optionalString(value: unknown, code: PlatformLeadBackfillRejectionCode = "INVALID_RECORD"): string | null {
|
|
if (value === undefined || value === null) return null;
|
|
if (typeof value !== "string") throw new PlatformLeadBackfillRowError(code);
|
|
const normalized = value.trim();
|
|
if (!normalized) return null;
|
|
if (hasControlCharacters(normalized)) throw new PlatformLeadBackfillRowError(code);
|
|
return normalized;
|
|
}
|
|
|
|
function optionalNote(value: unknown): string | null {
|
|
if (value === undefined || value === null) return null;
|
|
if (typeof value !== "string" || value.length > 8_000 || /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(value)) {
|
|
throw new PlatformLeadBackfillRowError("INVALID_ACTIVITY");
|
|
}
|
|
return value.trim().length > 0 ? value : null;
|
|
}
|
|
|
|
function requireTimestamp(value: unknown, code: PlatformLeadBackfillRejectionCode = "INVALID_TIMESTAMP"): Date {
|
|
if (typeof value !== "string" && typeof value !== "number") throw new PlatformLeadBackfillRowError(code);
|
|
const date = new Date(value);
|
|
if (!Number.isFinite(date.getTime())) throw new PlatformLeadBackfillRowError(code);
|
|
return date;
|
|
}
|
|
|
|
function optionalTimestamp(value: unknown): Date | null {
|
|
if (value === undefined || value === null || value === "") return null;
|
|
return requireTimestamp(value, "INVALID_ACTIVITY");
|
|
}
|
|
|
|
function requireValidDate(value: Date): Date {
|
|
if (!(value instanceof Date) || !Number.isFinite(value.getTime())) throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
|
return value;
|
|
}
|
|
|
|
function normalizeRunId(value: string): string {
|
|
if (typeof value !== "string") throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
|
const normalized = value.trim();
|
|
if (!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,190}$/u.test(normalized)) {
|
|
throw new PlatformLeadBackfillInputError("INVALID_CONFIGURATION");
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function capturedAtRange(records: readonly PlatformLeadBackfillImportRecord[]): PlatformLeadBackfillManifest["timeRange"] {
|
|
if (records.length === 0) return { firstCapturedAt: null, lastCapturedAt: null };
|
|
const timestamps = records.map(({ capturedAt }) => capturedAt.getTime());
|
|
return {
|
|
firstCapturedAt: new Date(Math.min(...timestamps)).toISOString(),
|
|
lastCapturedAt: new Date(Math.max(...timestamps)).toISOString(),
|
|
};
|
|
}
|
|
|
|
function resolveOutcome(mode: PlatformLeadBackfillMode, failures: readonly PlatformLeadBackfillGateFailure[]): PlatformLeadBackfillManifest["outcome"] {
|
|
if (failures.length > 0) {
|
|
return failures.includes("IMPORT_FAILURES_PRESENT") || failures.includes("REPLAY_CREATED_ROWS") ? "failed" : "blocked";
|
|
}
|
|
if (mode === "dry-run") return "validated";
|
|
return mode === "apply" ? "applied" : "replayed";
|
|
}
|
|
|
|
function incrementCount(counts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>, code: PlatformLeadBackfillRejectionCode): void {
|
|
counts[code] = (counts[code] ?? 0) + 1;
|
|
}
|
|
|
|
function sumCounts(counts: Partial<Record<PlatformLeadBackfillRejectionCode, number>>): number {
|
|
return Object.values(counts).reduce<number>((total, count) => total + (count ?? 0), 0);
|
|
}
|
|
|
|
function toRejectionCode(error: unknown): PlatformLeadBackfillRejectionCode {
|
|
return error instanceof PlatformLeadBackfillRowError ? error.code : "INVALID_RECORD";
|
|
}
|
|
|
|
function sha256(value: Uint8Array): string {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
|
|
function hasControlCharacters(value: string): boolean {
|
|
return /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(value);
|
|
}
|