Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+130
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Welle 4 · Deliverable 2 — Operator health aggregator.
|
||||
*
|
||||
* Composes the EXISTING admin snapshots (system-health, deploy, queue-monitor)
|
||||
* plus a small `BillingWebhookLog` error read into the pure
|
||||
* {@link buildOperatorHealthVM} input. Nothing here re-implements a snapshot;
|
||||
* each source stays the single owner of its own diagnostics. Every source is
|
||||
* wrapped defensively so one unreachable dependency (e.g. Redis down) degrades
|
||||
* its own subsystem to `unknown`/`critical` instead of breaking the page.
|
||||
*/
|
||||
import {
|
||||
buildDeployAdminVM,
|
||||
buildQueueMonitorVM,
|
||||
buildSystemHealthVM,
|
||||
type OperatorHealthBillingError,
|
||||
type OperatorHealthInput,
|
||||
type OperatorHealthStatus,
|
||||
type QueueCountsInput,
|
||||
} from "@hivegrid/view-models";
|
||||
import { db } from "~/utils/db.server";
|
||||
import { adminPath } from "~/utils/app/CanonicalUrlUtils";
|
||||
import { getDeployAdminSnapshot } from "./deploy-admin.server";
|
||||
import { getQueueMonitorSnapshot } from "./queue-monitor.server";
|
||||
import { getSystemHealthSnapshot } from "./system-health.server";
|
||||
|
||||
const BILLING_ERROR_MESSAGE_MAX = 500;
|
||||
const RECENT_BILLING_ERROR_LIMIT = 5;
|
||||
|
||||
export type OperatorHealthDeps = {
|
||||
readonly loadBillingWebhookErrors?: () => Promise<readonly OperatorHealthBillingError[]>;
|
||||
readonly countWebhookFailures?: () => Promise<number>;
|
||||
};
|
||||
|
||||
/** Maps the queue-monitor signal (ok/warn/paused) to the health status. */
|
||||
function queueSignalToStatus(signal: "ok" | "warn" | "paused"): OperatorHealthStatus {
|
||||
return signal === "ok" ? "ok" : "warning";
|
||||
}
|
||||
|
||||
async function defaultLoadBillingWebhookErrors(): Promise<readonly OperatorHealthBillingError[]> {
|
||||
const rows = await db.billingWebhookLog.findMany({
|
||||
where: { error: { not: null } },
|
||||
orderBy: { createdAt: "desc" },
|
||||
take: RECENT_BILLING_ERROR_LIMIT,
|
||||
select: { id: true, eventType: true, error: true, createdAt: true, relatedTenantId: true },
|
||||
});
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
eventType: row.eventType,
|
||||
error: (row.error ?? "").slice(0, BILLING_ERROR_MESSAGE_MAX),
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
relatedTenantId: row.relatedTenantId,
|
||||
}));
|
||||
}
|
||||
|
||||
async function defaultCountWebhookFailures(): Promise<number> {
|
||||
return db.eventWebhookAttempt.count({ where: { success: false } });
|
||||
}
|
||||
|
||||
export async function getOperatorHealthSnapshot(deps: OperatorHealthDeps = {}): Promise<OperatorHealthInput> {
|
||||
const loadBillingWebhookErrors = deps.loadBillingWebhookErrors ?? defaultLoadBillingWebhookErrors;
|
||||
const countWebhookFailures = deps.countWebhookFailures ?? defaultCountWebhookFailures;
|
||||
|
||||
// System health — self-contained, pings Postgres internally.
|
||||
const systemVm = buildSystemHealthVM(await getSystemHealthSnapshot());
|
||||
const systemStatus: OperatorHealthStatus = systemVm.status === "ready" ? systemVm.data.overallStatus : "unknown";
|
||||
|
||||
// Deploy / migrations — self-contained, catches migration-DB failures.
|
||||
const deployVm = buildDeployAdminVM(await getDeployAdminSnapshot());
|
||||
const deployStatus: OperatorHealthStatus = deployVm.status === "ready" ? deployVm.data.overallStatus : "unknown";
|
||||
const deployMeta = deployVm.status === "ready" ? deployVm.data.metadata : null;
|
||||
|
||||
// Queues — requires Redis; a missing connection throws synchronously.
|
||||
let queues: QueueCountsInput[] = [];
|
||||
let queuesReachable = true;
|
||||
try {
|
||||
queues = await getQueueMonitorSnapshot();
|
||||
} catch {
|
||||
queuesReachable = false;
|
||||
}
|
||||
const queueVm = buildQueueMonitorVM({ queues });
|
||||
const failedJobsTotal = queueVm.status === "ready" ? queueVm.data.failedTotal : 0;
|
||||
const pausedQueues = queueVm.status === "ready" ? queueVm.data.pausedCount : 0;
|
||||
const queuesStatus: OperatorHealthStatus = !queuesReachable ? "unknown" : queueVm.status === "ready" ? queueSignalToStatus(queueVm.data.overallSignal) : "ok";
|
||||
|
||||
// Webhook delivery failures (eventWebhookAttempt).
|
||||
let webhookFailures = 0;
|
||||
try {
|
||||
webhookFailures = await countWebhookFailures();
|
||||
} catch {
|
||||
webhookFailures = 0;
|
||||
}
|
||||
const webhooksStatus: OperatorHealthStatus = webhookFailures > 0 ? "warning" : "ok";
|
||||
|
||||
// Billing sync failures (BillingWebhookLog.error) — previously invisible.
|
||||
let billingErrors: readonly OperatorHealthBillingError[] = [];
|
||||
try {
|
||||
billingErrors = await loadBillingWebhookErrors();
|
||||
} catch {
|
||||
billingErrors = [];
|
||||
}
|
||||
const billingStatus: OperatorHealthStatus = billingErrors.length > 0 ? "warning" : "ok";
|
||||
|
||||
return {
|
||||
subsystems: [
|
||||
{ id: "system", status: systemStatus, href: adminPath("system/health", { mode: "host-aware" }) },
|
||||
{ id: "deploy", status: deployStatus, href: adminPath("system/deploys", { mode: "host-aware" }) },
|
||||
{ id: "queues", status: queuesStatus, href: adminPath("system/jobs", { mode: "host-aware" }) },
|
||||
{ id: "webhooks", status: webhooksStatus, href: adminPath("system/webhooks", { mode: "host-aware" }) },
|
||||
{ id: "billing", status: billingStatus, href: adminPath("system/billing", { mode: "host-aware" }) },
|
||||
],
|
||||
deploy: {
|
||||
environment: deployMeta?.environment ?? "unknown",
|
||||
release: deployMeta?.release ?? "unknown",
|
||||
revision: deployMeta?.revision ?? "unknown",
|
||||
branch: deployMeta?.branch ?? "unknown",
|
||||
},
|
||||
signals: {
|
||||
failedJobsTotal,
|
||||
pausedQueues,
|
||||
webhookFailures,
|
||||
billingErrors: billingErrors.length,
|
||||
},
|
||||
links: {
|
||||
jobs: adminPath("system/jobs", { mode: "host-aware" }),
|
||||
webhooks: adminPath("system/webhooks", { mode: "host-aware" }),
|
||||
billing: adminPath("system/billing", { mode: "host-aware" }),
|
||||
},
|
||||
billingErrors,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user