Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -0,0 +1,130 @@
/**
* Welle 4 · Deliverable 2 — Operator health aggregator.
*
* Composes the EXISTING admin snapshots (system-health, deploy, queue-monitor)
* plus a small `BillingWebhookLog` error read into the pure
* {@link buildOperatorHealthVM} input. Nothing here re-implements a snapshot;
* each source stays the single owner of its own diagnostics. Every source is
* wrapped defensively so one unreachable dependency (e.g. Redis down) degrades
* its own subsystem to `unknown`/`critical` instead of breaking the page.
*/
import {
buildDeployAdminVM,
buildQueueMonitorVM,
buildSystemHealthVM,
type OperatorHealthBillingError,
type OperatorHealthInput,
type OperatorHealthStatus,
type QueueCountsInput,
} from "@hivegrid/view-models";
import { db } from "~/utils/db.server";
import { adminPath } from "~/utils/app/CanonicalUrlUtils";
import { getDeployAdminSnapshot } from "./deploy-admin.server";
import { getQueueMonitorSnapshot } from "./queue-monitor.server";
import { getSystemHealthSnapshot } from "./system-health.server";
const BILLING_ERROR_MESSAGE_MAX = 500;
const RECENT_BILLING_ERROR_LIMIT = 5;
export type OperatorHealthDeps = {
readonly loadBillingWebhookErrors?: () => Promise<readonly OperatorHealthBillingError[]>;
readonly countWebhookFailures?: () => Promise<number>;
};
/** Maps the queue-monitor signal (ok/warn/paused) to the health status. */
function queueSignalToStatus(signal: "ok" | "warn" | "paused"): OperatorHealthStatus {
return signal === "ok" ? "ok" : "warning";
}
async function defaultLoadBillingWebhookErrors(): Promise<readonly OperatorHealthBillingError[]> {
const rows = await db.billingWebhookLog.findMany({
where: { error: { not: null } },
orderBy: { createdAt: "desc" },
take: RECENT_BILLING_ERROR_LIMIT,
select: { id: true, eventType: true, error: true, createdAt: true, relatedTenantId: true },
});
return rows.map((row) => ({
id: row.id,
eventType: row.eventType,
error: (row.error ?? "").slice(0, BILLING_ERROR_MESSAGE_MAX),
createdAt: row.createdAt.toISOString(),
relatedTenantId: row.relatedTenantId,
}));
}
async function defaultCountWebhookFailures(): Promise<number> {
return db.eventWebhookAttempt.count({ where: { success: false } });
}
export async function getOperatorHealthSnapshot(deps: OperatorHealthDeps = {}): Promise<OperatorHealthInput> {
const loadBillingWebhookErrors = deps.loadBillingWebhookErrors ?? defaultLoadBillingWebhookErrors;
const countWebhookFailures = deps.countWebhookFailures ?? defaultCountWebhookFailures;
// System health — self-contained, pings Postgres internally.
const systemVm = buildSystemHealthVM(await getSystemHealthSnapshot());
const systemStatus: OperatorHealthStatus = systemVm.status === "ready" ? systemVm.data.overallStatus : "unknown";
// Deploy / migrations — self-contained, catches migration-DB failures.
const deployVm = buildDeployAdminVM(await getDeployAdminSnapshot());
const deployStatus: OperatorHealthStatus = deployVm.status === "ready" ? deployVm.data.overallStatus : "unknown";
const deployMeta = deployVm.status === "ready" ? deployVm.data.metadata : null;
// Queues — requires Redis; a missing connection throws synchronously.
let queues: QueueCountsInput[] = [];
let queuesReachable = true;
try {
queues = await getQueueMonitorSnapshot();
} catch {
queuesReachable = false;
}
const queueVm = buildQueueMonitorVM({ queues });
const failedJobsTotal = queueVm.status === "ready" ? queueVm.data.failedTotal : 0;
const pausedQueues = queueVm.status === "ready" ? queueVm.data.pausedCount : 0;
const queuesStatus: OperatorHealthStatus = !queuesReachable ? "unknown" : queueVm.status === "ready" ? queueSignalToStatus(queueVm.data.overallSignal) : "ok";
// Webhook delivery failures (eventWebhookAttempt).
let webhookFailures = 0;
try {
webhookFailures = await countWebhookFailures();
} catch {
webhookFailures = 0;
}
const webhooksStatus: OperatorHealthStatus = webhookFailures > 0 ? "warning" : "ok";
// Billing sync failures (BillingWebhookLog.error) — previously invisible.
let billingErrors: readonly OperatorHealthBillingError[] = [];
try {
billingErrors = await loadBillingWebhookErrors();
} catch {
billingErrors = [];
}
const billingStatus: OperatorHealthStatus = billingErrors.length > 0 ? "warning" : "ok";
return {
subsystems: [
{ id: "system", status: systemStatus, href: adminPath("system/health", { mode: "host-aware" }) },
{ id: "deploy", status: deployStatus, href: adminPath("system/deploys", { mode: "host-aware" }) },
{ id: "queues", status: queuesStatus, href: adminPath("system/jobs", { mode: "host-aware" }) },
{ id: "webhooks", status: webhooksStatus, href: adminPath("system/webhooks", { mode: "host-aware" }) },
{ id: "billing", status: billingStatus, href: adminPath("system/billing", { mode: "host-aware" }) },
],
deploy: {
environment: deployMeta?.environment ?? "unknown",
release: deployMeta?.release ?? "unknown",
revision: deployMeta?.revision ?? "unknown",
branch: deployMeta?.branch ?? "unknown",
},
signals: {
failedJobsTotal,
pausedQueues,
webhookFailures,
billingErrors: billingErrors.length,
},
links: {
jobs: adminPath("system/jobs", { mode: "host-aware" }),
webhooks: adminPath("system/webhooks", { mode: "host-aware" }),
billing: adminPath("system/billing", { mode: "host-aware" }),
},
billingErrors,
};
}