Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+315
-38
@@ -6,26 +6,32 @@
|
||||
* inserts the AuditEvent row into Postgres.
|
||||
*
|
||||
* The hash chain is computed inside a short Prisma transaction:
|
||||
* - SELECT the most-recent eventHash for the tenant (FOR UPDATE
|
||||
* equivalent via serializable isolation)
|
||||
* - acquire a tenant/platform PostgreSQL advisory transaction lock
|
||||
* - authenticate and read the O(1) persistent chain head
|
||||
* - compute the new event's eventHash
|
||||
* - INSERT the new row
|
||||
* - insert the event and atomically advance the authenticated head
|
||||
*
|
||||
* Running this under serializable isolation ensures concurrent persists
|
||||
* for the same tenant either commit in some order with a coherent
|
||||
* chain, or one of them retries on serialization-failure. The handler
|
||||
* is therefore safe to invoke from N parallel BullMQ workers without
|
||||
* corrupting the chain.
|
||||
* The advisory lock is shared with direct writers and retention, so N
|
||||
* parallel BullMQ workers cannot fork a tenant or platform chain.
|
||||
*
|
||||
* Idempotency: the producer-side `publishAuditEventViaOutbox` writes
|
||||
* one outbox row per business-write. The dispatcher delivers each row
|
||||
* exactly once (claim+ack); on retry we may re-process the same
|
||||
* payload but the audit insert is a plain CREATE — duplicates simply
|
||||
* extend the chain by one extra row. The acceptable outcome here is
|
||||
* "at-least-once": a duplicated audit row is harmless evidence.
|
||||
* Authority and idempotency: Redis is transport only. Before inserting an
|
||||
* AuditEvent the consumer proves the exact tenant/generation/payload/dispatch
|
||||
* key/lease against the durable OutboxEvent, claims that generation with a
|
||||
* CAS, then commits the AuditEvent and terminal DONE receipt atomically.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import { computeAuditHash, type AuditHashPayload } from "@hivegrid/db/audit-hash";
|
||||
import {
|
||||
acquireAuditChainLock,
|
||||
auditChainKey,
|
||||
computeAuditChainHeadMac,
|
||||
verifyAuditChainHead,
|
||||
verifyAuditCheckpoint,
|
||||
type AuditChainHeadRecord,
|
||||
type AuditChainCheckpointRecord,
|
||||
} from "@hivegrid/db/audit-chain-checkpoint.server";
|
||||
import {
|
||||
buildAuditEventCreateData,
|
||||
type AuditEventCreateData,
|
||||
@@ -33,6 +39,7 @@ import {
|
||||
} from "@hivegrid/db/audit-events.server";
|
||||
|
||||
export const AUDIT_EVENT_OUTBOX_QUEUE = "outbox.audit" as const;
|
||||
const AUDIT_OUTBOX_CONSUMER_LEASE_MS = 5 * 60 * 1_000;
|
||||
|
||||
export interface AuditEventPersistLogger {
|
||||
info?: (record: Record<string, unknown>, message?: string) => void;
|
||||
@@ -41,23 +48,89 @@ export interface AuditEventPersistLogger {
|
||||
}
|
||||
|
||||
export interface AuditEventPersistTxClient {
|
||||
$queryRaw<T = unknown>(query: TemplateStringsArray, ...values: unknown[]): Promise<T>;
|
||||
auditEvent: {
|
||||
findFirst(args: {
|
||||
where: { tenantId: string | null; eventHash: { not: null } };
|
||||
orderBy: { createdAt: "desc" };
|
||||
select: { eventHash: true };
|
||||
}): Promise<{ eventHash: string | null } | null>;
|
||||
orderBy: Array<{ createdAt: "desc" } | { id: "desc" }>;
|
||||
select: { eventHash: true; createdAt: true };
|
||||
}): Promise<{ eventHash: string | null; createdAt: Date } | null>;
|
||||
create(args: {
|
||||
data: AuditEventCreateData & { previousHash: string | null; eventHash: string };
|
||||
data: AuditEventCreateData & { id: string; createdAt: Date; previousHash: string | null; eventHash: string };
|
||||
}): Promise<{ id: string; eventHash: string }>;
|
||||
};
|
||||
auditChainCheckpoint: {
|
||||
findFirst(args: {
|
||||
where: { chainKey: string };
|
||||
orderBy: Array<{ createdAt: "desc" } | { id: "desc" }>;
|
||||
}): Promise<AuditChainCheckpointRecord | null>;
|
||||
findUnique(args: { where: { id: string } }): Promise<AuditChainCheckpointRecord | null>;
|
||||
};
|
||||
auditChainCheckpointHead: {
|
||||
findUnique(args: {
|
||||
where: { chainKey: string };
|
||||
select: { checkpointId: true };
|
||||
}): Promise<{ checkpointId: string } | null>;
|
||||
};
|
||||
auditChainHead: {
|
||||
findUnique(args: { where: { chainKey: string } }): Promise<AuditChainHeadRecord | null>;
|
||||
upsert(args: {
|
||||
where: { chainKey: string };
|
||||
create: AuditChainHeadRecord;
|
||||
update: Omit<AuditChainHeadRecord, "chainKey">;
|
||||
}): Promise<unknown>;
|
||||
};
|
||||
outboxEvent: {
|
||||
findFirst(args: {
|
||||
where: {
|
||||
id: string;
|
||||
tenantId: string | null;
|
||||
eventType: "audit.event.create";
|
||||
deliveryGeneration: number;
|
||||
};
|
||||
select: {
|
||||
id: true;
|
||||
tenantId: true;
|
||||
eventType: true;
|
||||
payload: true;
|
||||
status: true;
|
||||
deliveryGeneration: true;
|
||||
dispatchKey: true;
|
||||
lockedUntil: true;
|
||||
};
|
||||
}): Promise<{
|
||||
id: string;
|
||||
tenantId: string | null;
|
||||
eventType: string;
|
||||
payload: unknown;
|
||||
status: "PENDING" | "PROCESSING" | "DONE" | "FAILED" | "CANCELLED";
|
||||
deliveryGeneration: number;
|
||||
dispatchKey: string | null;
|
||||
lockedUntil: Date | null;
|
||||
} | null>;
|
||||
updateMany(args: {
|
||||
where: {
|
||||
id: string;
|
||||
tenantId: string | null;
|
||||
eventType: "audit.event.create";
|
||||
deliveryGeneration: number;
|
||||
dispatchKey: string | null;
|
||||
status: "PROCESSING";
|
||||
lockedUntil: Date;
|
||||
payload: { equals: unknown };
|
||||
};
|
||||
data:
|
||||
| { lockedUntil: Date }
|
||||
| { status: "DONE"; processedAt: Date; lastError: null; lockedUntil: null };
|
||||
}): Promise<{ count: number }>;
|
||||
};
|
||||
}
|
||||
|
||||
export interface AuditEventPersistTransactionRunner {
|
||||
/**
|
||||
* Run `fn` inside a Prisma transaction. Production wiring passes
|
||||
* `prisma.$transaction` with isolationLevel="Serializable" so that
|
||||
* the prevHash lookup + insert form an atomic chain link.
|
||||
* Run `fn` inside a Prisma transaction. Production additionally uses
|
||||
* serializable isolation; the per-chain advisory lock is the primary
|
||||
* cross-writer serialization primitive.
|
||||
*/
|
||||
<T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T>;
|
||||
}
|
||||
@@ -70,11 +143,13 @@ export interface AuditEventPersistDependencies {
|
||||
* leaves this undefined so the helper reads `process.env.AUDIT_CHAIN_SECRET`.
|
||||
*/
|
||||
readonly secret?: string | null;
|
||||
/** Clock override for durable lease checks and deterministic tests. */
|
||||
readonly now?: () => Date;
|
||||
}
|
||||
|
||||
export class AuditEventPersistJobError extends Error {
|
||||
override readonly name = "AuditEventPersistJobError";
|
||||
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "MISSING_ACTION";
|
||||
readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "MISSING_ACTION" | "OUTBOX_PROOF_INVALID";
|
||||
constructor(message: string, code: AuditEventPersistJobError["code"]) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
@@ -82,9 +157,13 @@ export class AuditEventPersistJobError extends Error {
|
||||
}
|
||||
|
||||
export interface AuditEventPersistJobData {
|
||||
readonly outboxEventId?: string;
|
||||
readonly outboxEventId: string;
|
||||
readonly tenantId: string | null;
|
||||
readonly eventType: string;
|
||||
readonly eventType: "audit.event.create";
|
||||
readonly dispatchKey: string | null;
|
||||
readonly deliveryGeneration: number;
|
||||
readonly lockedUntil: Date;
|
||||
readonly rawPayload: Readonly<Record<string, unknown>>;
|
||||
readonly payload: AuditEventCreateData;
|
||||
}
|
||||
|
||||
@@ -95,7 +174,7 @@ export interface AuditEventPersistResult {
|
||||
}
|
||||
|
||||
export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobData {
|
||||
if (!data || typeof data !== "object") {
|
||||
if (!data || typeof data !== "object" || Array.isArray(data)) {
|
||||
throw new AuditEventPersistJobError("audit-event persist job missing data envelope", "INVALID_PAYLOAD");
|
||||
}
|
||||
const envelope = data as Record<string, unknown>;
|
||||
@@ -110,6 +189,9 @@ export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobDa
|
||||
if (!rawPayload || typeof rawPayload !== "object") {
|
||||
throw new AuditEventPersistJobError("audit-event persist job missing payload", "INVALID_PAYLOAD");
|
||||
}
|
||||
if (Array.isArray(rawPayload)) {
|
||||
throw new AuditEventPersistJobError("audit-event persist job payload must be an object", "INVALID_PAYLOAD");
|
||||
}
|
||||
const inner = rawPayload as Record<string, unknown>;
|
||||
const action = typeof inner.action === "string" ? inner.action.trim() : "";
|
||||
if (action.length === 0) {
|
||||
@@ -118,18 +200,67 @@ export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobDa
|
||||
// Re-run normalization in case the producer emitted an older payload
|
||||
// shape — defensive. `buildAuditEventCreateData` is idempotent.
|
||||
const normalized = buildAuditEventCreateData(inner as unknown as AuditEventInput);
|
||||
const outboxEventId = parseIdentifier(envelope.outboxEventId);
|
||||
const tenantId = parseNullableIdentifier(envelope, "tenantId");
|
||||
const dispatchKey = parseNullableDispatchKey(envelope);
|
||||
const deliveryGeneration = envelope.deliveryGeneration;
|
||||
const lockedUntil = parseLease(envelope.lockedUntil);
|
||||
if (
|
||||
!outboxEventId ||
|
||||
tenantId === undefined ||
|
||||
dispatchKey === undefined ||
|
||||
!Number.isSafeInteger(deliveryGeneration) ||
|
||||
(deliveryGeneration as number) < 0 ||
|
||||
!lockedUntil
|
||||
) {
|
||||
throw new AuditEventPersistJobError("audit-event persist job has no exact outbox identity", "INVALID_PAYLOAD");
|
||||
}
|
||||
if (normalized.tenantId !== null && normalized.tenantId !== tenantId) {
|
||||
throw new AuditEventPersistJobError("audit-event tenant does not match its outbox envelope", "INVALID_PAYLOAD");
|
||||
}
|
||||
return {
|
||||
outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
|
||||
tenantId: typeof envelope.tenantId === "string" ? envelope.tenantId : null,
|
||||
eventType,
|
||||
outboxEventId,
|
||||
tenantId,
|
||||
eventType: "audit.event.create",
|
||||
dispatchKey,
|
||||
deliveryGeneration: deliveryGeneration as number,
|
||||
lockedUntil,
|
||||
rawPayload: inner,
|
||||
payload: normalized,
|
||||
};
|
||||
}
|
||||
|
||||
function toHashPayload(data: AuditEventCreateData): AuditHashPayload {
|
||||
function parseIdentifier(value: unknown): string | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const normalized = value.trim();
|
||||
return normalized.length > 0 && normalized.length <= 191 && !normalized.includes("\u0000") ? normalized : null;
|
||||
}
|
||||
|
||||
function parseNullableIdentifier(
|
||||
envelope: Readonly<Record<string, unknown>>,
|
||||
key: "tenantId",
|
||||
): string | null | undefined {
|
||||
if (!Object.hasOwn(envelope, key)) return undefined;
|
||||
if (envelope[key] === null) return null;
|
||||
return parseIdentifier(envelope[key]) ?? undefined;
|
||||
}
|
||||
|
||||
function parseNullableDispatchKey(envelope: Readonly<Record<string, unknown>>): string | null | undefined {
|
||||
if (!Object.hasOwn(envelope, "dispatchKey")) return undefined;
|
||||
if (envelope.dispatchKey === null) return null;
|
||||
if (typeof envelope.dispatchKey !== "string") return undefined;
|
||||
return /^[A-Za-z0-9._:-]{1,256}$/u.test(envelope.dispatchKey) ? envelope.dispatchKey : undefined;
|
||||
}
|
||||
|
||||
function parseLease(value: unknown): Date | null {
|
||||
const parsed = value instanceof Date ? new Date(value.getTime()) : typeof value === "string" ? new Date(value) : null;
|
||||
return parsed && Number.isFinite(parsed.getTime()) ? parsed : null;
|
||||
}
|
||||
|
||||
function toHashPayload(data: AuditEventCreateData, id: string, createdAt: Date): AuditHashPayload {
|
||||
return {
|
||||
id: "",
|
||||
createdAt: "",
|
||||
id,
|
||||
createdAt: createdAt.toISOString(),
|
||||
tenantId: data.tenantId,
|
||||
actorType: data.actorType,
|
||||
actorId: data.actorId,
|
||||
@@ -156,20 +287,139 @@ export async function runAuditEventPersistJob(
|
||||
const payload = job.payload;
|
||||
|
||||
const result = await deps.runInTransaction(async (tx) => {
|
||||
const prior = await tx.auditEvent.findFirst({
|
||||
where: { tenantId: payload.tenantId, eventHash: { not: null } },
|
||||
orderBy: { createdAt: "desc" },
|
||||
select: { eventHash: true },
|
||||
const durable = await tx.outboxEvent.findFirst({
|
||||
where: {
|
||||
id: job.outboxEventId,
|
||||
tenantId: job.tenantId,
|
||||
eventType: job.eventType,
|
||||
deliveryGeneration: job.deliveryGeneration,
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
tenantId: true,
|
||||
eventType: true,
|
||||
payload: true,
|
||||
status: true,
|
||||
deliveryGeneration: true,
|
||||
dispatchKey: true,
|
||||
lockedUntil: true,
|
||||
},
|
||||
});
|
||||
const previousHash = prior?.eventHash ?? null;
|
||||
const observedAt = deps.now?.() ?? new Date();
|
||||
if (
|
||||
!durable ||
|
||||
durable.status !== "PROCESSING" ||
|
||||
durable.dispatchKey !== job.dispatchKey ||
|
||||
!durable.lockedUntil ||
|
||||
durable.lockedUntil.getTime() !== job.lockedUntil.getTime() ||
|
||||
durable.lockedUntil.getTime() <= observedAt.getTime() ||
|
||||
!sameCanonicalJson(durable.payload, job.rawPayload)
|
||||
) {
|
||||
throw invalidOutboxProof();
|
||||
}
|
||||
const claimedLease = new Date(
|
||||
Math.max(
|
||||
observedAt.getTime() + AUDIT_OUTBOX_CONSUMER_LEASE_MS,
|
||||
durable.lockedUntil.getTime() + 1,
|
||||
),
|
||||
);
|
||||
const claimed = await tx.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: job.outboxEventId,
|
||||
tenantId: job.tenantId,
|
||||
eventType: job.eventType,
|
||||
deliveryGeneration: job.deliveryGeneration,
|
||||
dispatchKey: job.dispatchKey,
|
||||
status: "PROCESSING",
|
||||
lockedUntil: durable.lockedUntil,
|
||||
payload: { equals: job.rawPayload },
|
||||
},
|
||||
data: { lockedUntil: claimedLease },
|
||||
});
|
||||
if (claimed.count !== 1) throw invalidOutboxProof();
|
||||
|
||||
await acquireAuditChainLock(tx, payload.tenantId);
|
||||
const key = auditChainKey(payload.tenantId);
|
||||
const head = await tx.auditChainHead.findUnique({ where: { chainKey: key } });
|
||||
if (head && !verifyAuditChainHead(head, deps.secret)) {
|
||||
throw new Error(`Audit chain-head authentication failed for ${key}.`);
|
||||
}
|
||||
let previousHash = head?.eventHash ?? null;
|
||||
let previousCreatedAt = head?.eventCreatedAt ?? null;
|
||||
if (!previousHash) {
|
||||
const prior = await tx.auditEvent.findFirst({
|
||||
where: { tenantId: payload.tenantId, eventHash: { not: null } },
|
||||
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
|
||||
select: { eventHash: true, createdAt: true },
|
||||
});
|
||||
previousHash = prior?.eventHash ?? null;
|
||||
previousCreatedAt = prior?.createdAt ?? null;
|
||||
}
|
||||
if (!previousHash) {
|
||||
const checkpointHead = await tx.auditChainCheckpointHead.findUnique({
|
||||
where: { chainKey: key },
|
||||
select: { checkpointId: true },
|
||||
});
|
||||
const checkpoint = checkpointHead
|
||||
? await tx.auditChainCheckpoint.findUnique({ where: { id: checkpointHead.checkpointId } })
|
||||
: await tx.auditChainCheckpoint.findFirst({
|
||||
where: { chainKey: key },
|
||||
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
|
||||
});
|
||||
if (checkpoint) {
|
||||
if (!verifyAuditCheckpoint(checkpoint, deps.secret)) {
|
||||
throw new Error(`Audit checkpoint authentication failed for ${checkpoint.chainKey}.`);
|
||||
}
|
||||
previousHash = checkpoint.lastDeletedEventHash;
|
||||
previousCreatedAt = checkpoint.deletedThroughAt;
|
||||
}
|
||||
}
|
||||
const id = randomUUID();
|
||||
const now = deps.now?.() ?? new Date();
|
||||
const createdAt = previousCreatedAt && now.getTime() <= previousCreatedAt.getTime()
|
||||
? new Date(previousCreatedAt.getTime() + 1)
|
||||
: now;
|
||||
const eventHash = computeAuditHash({
|
||||
previousHash,
|
||||
payload: toHashPayload(payload),
|
||||
payload: toHashPayload(payload, id, createdAt),
|
||||
secret: deps.secret,
|
||||
version: "hmac-v2",
|
||||
});
|
||||
const inserted = await tx.auditEvent.create({
|
||||
data: { ...payload, previousHash, eventHash },
|
||||
data: { ...payload, id, createdAt, previousHash, eventHash },
|
||||
});
|
||||
const headWithoutMac: Omit<AuditChainHeadRecord, "headMac"> = {
|
||||
chainKey: key,
|
||||
tenantId: payload.tenantId,
|
||||
eventId: id,
|
||||
eventHash,
|
||||
eventCreatedAt: createdAt,
|
||||
};
|
||||
const headMac = computeAuditChainHeadMac(headWithoutMac, deps.secret);
|
||||
await tx.auditChainHead.upsert({
|
||||
where: { chainKey: key },
|
||||
create: { ...headWithoutMac, headMac },
|
||||
update: { tenantId: payload.tenantId, eventId: id, eventHash, eventCreatedAt: createdAt, headMac },
|
||||
});
|
||||
const receipt = await tx.outboxEvent.updateMany({
|
||||
where: {
|
||||
id: job.outboxEventId,
|
||||
tenantId: job.tenantId,
|
||||
eventType: job.eventType,
|
||||
deliveryGeneration: job.deliveryGeneration,
|
||||
dispatchKey: job.dispatchKey,
|
||||
status: "PROCESSING",
|
||||
lockedUntil: claimedLease,
|
||||
payload: { equals: job.rawPayload },
|
||||
},
|
||||
data: {
|
||||
status: "DONE",
|
||||
processedAt: createdAt,
|
||||
lastError: null,
|
||||
lockedUntil: null,
|
||||
},
|
||||
});
|
||||
if (receipt.count !== 1) throw invalidOutboxProof();
|
||||
return { auditEventId: inserted.id, eventHash };
|
||||
});
|
||||
|
||||
@@ -186,3 +436,30 @@ export async function runAuditEventPersistJob(
|
||||
|
||||
return { persisted: true, auditEventId: result.auditEventId, eventHash: result.eventHash };
|
||||
}
|
||||
|
||||
function invalidOutboxProof(): AuditEventPersistJobError {
|
||||
return new AuditEventPersistJobError(
|
||||
"audit-event persist job does not match an active durable outbox generation",
|
||||
"OUTBOX_PROOF_INVALID",
|
||||
);
|
||||
}
|
||||
|
||||
function sameCanonicalJson(left: unknown, right: unknown): boolean {
|
||||
try {
|
||||
return JSON.stringify(canonicalJson(left)) === JSON.stringify(canonicalJson(right));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown): unknown {
|
||||
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
||||
if (typeof value === "number" && Number.isFinite(value)) return value;
|
||||
if (Array.isArray(value)) return value.map(canonicalJson);
|
||||
if (!value || typeof value !== "object") throw new TypeError("Value is not canonical JSON");
|
||||
return Object.fromEntries(
|
||||
Object.keys(value as Record<string, unknown>)
|
||||
.sort()
|
||||
.map((key) => [key, canonicalJson((value as Record<string, unknown>)[key])]),
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user