131 lines
5.4 KiB
TypeScript
131 lines
5.4 KiB
TypeScript
/**
|
|
* Welle 4 · Deliverable 2 — Operator health aggregator.
|
|
*
|
|
* Composes the EXISTING admin snapshots (system-health, deploy, queue-monitor)
|
|
* plus a small `BillingWebhookLog` error read into the pure
|
|
* {@link buildOperatorHealthVM} input. Nothing here re-implements a snapshot;
|
|
* each source stays the single owner of its own diagnostics. Every source is
|
|
* wrapped defensively so one unreachable dependency (e.g. Redis down) degrades
|
|
* its own subsystem to `unknown`/`critical` instead of breaking the page.
|
|
*/
|
|
import {
|
|
buildDeployAdminVM,
|
|
buildQueueMonitorVM,
|
|
buildSystemHealthVM,
|
|
type OperatorHealthBillingError,
|
|
type OperatorHealthInput,
|
|
type OperatorHealthStatus,
|
|
type QueueCountsInput,
|
|
} from "@hivegrid/view-models";
|
|
import { db } from "~/utils/db.server";
|
|
import { adminPath } from "~/utils/app/CanonicalUrlUtils";
|
|
import { getDeployAdminSnapshot } from "./deploy-admin.server";
|
|
import { getQueueMonitorSnapshot } from "./queue-monitor.server";
|
|
import { getSystemHealthSnapshot } from "./system-health.server";
|
|
|
|
const BILLING_ERROR_MESSAGE_MAX = 500;
|
|
const RECENT_BILLING_ERROR_LIMIT = 5;
|
|
|
|
export type OperatorHealthDeps = {
|
|
readonly loadBillingWebhookErrors?: () => Promise<readonly OperatorHealthBillingError[]>;
|
|
readonly countWebhookFailures?: () => Promise<number>;
|
|
};
|
|
|
|
/** Maps the queue-monitor signal (ok/warn/paused) to the health status. */
|
|
function queueSignalToStatus(signal: "ok" | "warn" | "paused"): OperatorHealthStatus {
|
|
return signal === "ok" ? "ok" : "warning";
|
|
}
|
|
|
|
async function defaultLoadBillingWebhookErrors(): Promise<readonly OperatorHealthBillingError[]> {
|
|
const rows = await db.billingWebhookLog.findMany({
|
|
where: { error: { not: null } },
|
|
orderBy: { createdAt: "desc" },
|
|
take: RECENT_BILLING_ERROR_LIMIT,
|
|
select: { id: true, eventType: true, error: true, createdAt: true, relatedTenantId: true },
|
|
});
|
|
return rows.map((row) => ({
|
|
id: row.id,
|
|
eventType: row.eventType,
|
|
error: (row.error ?? "").slice(0, BILLING_ERROR_MESSAGE_MAX),
|
|
createdAt: row.createdAt.toISOString(),
|
|
relatedTenantId: row.relatedTenantId,
|
|
}));
|
|
}
|
|
|
|
async function defaultCountWebhookFailures(): Promise<number> {
|
|
return db.eventWebhookAttempt.count({ where: { success: false } });
|
|
}
|
|
|
|
export async function getOperatorHealthSnapshot(deps: OperatorHealthDeps = {}): Promise<OperatorHealthInput> {
|
|
const loadBillingWebhookErrors = deps.loadBillingWebhookErrors ?? defaultLoadBillingWebhookErrors;
|
|
const countWebhookFailures = deps.countWebhookFailures ?? defaultCountWebhookFailures;
|
|
|
|
// System health — self-contained, pings Postgres internally.
|
|
const systemVm = buildSystemHealthVM(await getSystemHealthSnapshot());
|
|
const systemStatus: OperatorHealthStatus = systemVm.status === "ready" ? systemVm.data.overallStatus : "unknown";
|
|
|
|
// Deploy / migrations — self-contained, catches migration-DB failures.
|
|
const deployVm = buildDeployAdminVM(await getDeployAdminSnapshot());
|
|
const deployStatus: OperatorHealthStatus = deployVm.status === "ready" ? deployVm.data.overallStatus : "unknown";
|
|
const deployMeta = deployVm.status === "ready" ? deployVm.data.metadata : null;
|
|
|
|
// Queues — requires Redis; a missing connection throws synchronously.
|
|
let queues: QueueCountsInput[] = [];
|
|
let queuesReachable = true;
|
|
try {
|
|
queues = await getQueueMonitorSnapshot();
|
|
} catch {
|
|
queuesReachable = false;
|
|
}
|
|
const queueVm = buildQueueMonitorVM({ queues });
|
|
const failedJobsTotal = queueVm.status === "ready" ? queueVm.data.failedTotal : 0;
|
|
const pausedQueues = queueVm.status === "ready" ? queueVm.data.pausedCount : 0;
|
|
const queuesStatus: OperatorHealthStatus = !queuesReachable ? "unknown" : queueVm.status === "ready" ? queueSignalToStatus(queueVm.data.overallSignal) : "ok";
|
|
|
|
// Webhook delivery failures (eventWebhookAttempt).
|
|
let webhookFailures = 0;
|
|
try {
|
|
webhookFailures = await countWebhookFailures();
|
|
} catch {
|
|
webhookFailures = 0;
|
|
}
|
|
const webhooksStatus: OperatorHealthStatus = webhookFailures > 0 ? "warning" : "ok";
|
|
|
|
// Billing sync failures (BillingWebhookLog.error) — previously invisible.
|
|
let billingErrors: readonly OperatorHealthBillingError[] = [];
|
|
try {
|
|
billingErrors = await loadBillingWebhookErrors();
|
|
} catch {
|
|
billingErrors = [];
|
|
}
|
|
const billingStatus: OperatorHealthStatus = billingErrors.length > 0 ? "warning" : "ok";
|
|
|
|
return {
|
|
subsystems: [
|
|
{ id: "system", status: systemStatus, href: adminPath("system/health", { mode: "host-aware" }) },
|
|
{ id: "deploy", status: deployStatus, href: adminPath("system/deploys", { mode: "host-aware" }) },
|
|
{ id: "queues", status: queuesStatus, href: adminPath("system/jobs", { mode: "host-aware" }) },
|
|
{ id: "webhooks", status: webhooksStatus, href: adminPath("system/webhooks", { mode: "host-aware" }) },
|
|
{ id: "billing", status: billingStatus, href: adminPath("system/billing", { mode: "host-aware" }) },
|
|
],
|
|
deploy: {
|
|
environment: deployMeta?.environment ?? "unknown",
|
|
release: deployMeta?.release ?? "unknown",
|
|
revision: deployMeta?.revision ?? "unknown",
|
|
branch: deployMeta?.branch ?? "unknown",
|
|
},
|
|
signals: {
|
|
failedJobsTotal,
|
|
pausedQueues,
|
|
webhookFailures,
|
|
billingErrors: billingErrors.length,
|
|
},
|
|
links: {
|
|
jobs: adminPath("system/jobs", { mode: "host-aware" }),
|
|
webhooks: adminPath("system/webhooks", { mode: "host-aware" }),
|
|
billing: adminPath("system/billing", { mode: "host-aware" }),
|
|
},
|
|
billingErrors,
|
|
};
|
|
}
|