/** * Welle 4 · Deliverable 2 — Operator health aggregator. * * Composes the EXISTING admin snapshots (system-health, deploy, queue-monitor) * plus a small `BillingWebhookLog` error read into the pure * {@link buildOperatorHealthVM} input. Nothing here re-implements a snapshot; * each source stays the single owner of its own diagnostics. Every source is * wrapped defensively so one unreachable dependency (e.g. Redis down) degrades * its own subsystem to `unknown`/`critical` instead of breaking the page. */ import { buildDeployAdminVM, buildQueueMonitorVM, buildSystemHealthVM, type OperatorHealthBillingError, type OperatorHealthInput, type OperatorHealthStatus, type QueueCountsInput, } from "@hivegrid/view-models"; import { db } from "~/utils/db.server"; import { adminPath } from "~/utils/app/CanonicalUrlUtils"; import { getDeployAdminSnapshot } from "./deploy-admin.server"; import { getQueueMonitorSnapshot } from "./queue-monitor.server"; import { getSystemHealthSnapshot } from "./system-health.server"; const BILLING_ERROR_MESSAGE_MAX = 500; const RECENT_BILLING_ERROR_LIMIT = 5; export type OperatorHealthDeps = { readonly loadBillingWebhookErrors?: () => Promise; readonly countWebhookFailures?: () => Promise; }; /** Maps the queue-monitor signal (ok/warn/paused) to the health status. */ function queueSignalToStatus(signal: "ok" | "warn" | "paused"): OperatorHealthStatus { return signal === "ok" ? "ok" : "warning"; } async function defaultLoadBillingWebhookErrors(): Promise { const rows = await db.billingWebhookLog.findMany({ where: { error: { not: null } }, orderBy: { createdAt: "desc" }, take: RECENT_BILLING_ERROR_LIMIT, select: { id: true, eventType: true, error: true, createdAt: true, relatedTenantId: true }, }); return rows.map((row) => ({ id: row.id, eventType: row.eventType, error: (row.error ?? "").slice(0, BILLING_ERROR_MESSAGE_MAX), createdAt: row.createdAt.toISOString(), relatedTenantId: row.relatedTenantId, })); } async function defaultCountWebhookFailures(): Promise { return db.eventWebhookAttempt.count({ where: { success: false } }); } export async function getOperatorHealthSnapshot(deps: OperatorHealthDeps = {}): Promise { const loadBillingWebhookErrors = deps.loadBillingWebhookErrors ?? defaultLoadBillingWebhookErrors; const countWebhookFailures = deps.countWebhookFailures ?? defaultCountWebhookFailures; // System health — self-contained, pings Postgres internally. const systemVm = buildSystemHealthVM(await getSystemHealthSnapshot()); const systemStatus: OperatorHealthStatus = systemVm.status === "ready" ? systemVm.data.overallStatus : "unknown"; // Deploy / migrations — self-contained, catches migration-DB failures. const deployVm = buildDeployAdminVM(await getDeployAdminSnapshot()); const deployStatus: OperatorHealthStatus = deployVm.status === "ready" ? deployVm.data.overallStatus : "unknown"; const deployMeta = deployVm.status === "ready" ? deployVm.data.metadata : null; // Queues — requires Redis; a missing connection throws synchronously. let queues: QueueCountsInput[] = []; let queuesReachable = true; try { queues = await getQueueMonitorSnapshot(); } catch { queuesReachable = false; } const queueVm = buildQueueMonitorVM({ queues }); const failedJobsTotal = queueVm.status === "ready" ? queueVm.data.failedTotal : 0; const pausedQueues = queueVm.status === "ready" ? queueVm.data.pausedCount : 0; const queuesStatus: OperatorHealthStatus = !queuesReachable ? "unknown" : queueVm.status === "ready" ? queueSignalToStatus(queueVm.data.overallSignal) : "ok"; // Webhook delivery failures (eventWebhookAttempt). let webhookFailures = 0; try { webhookFailures = await countWebhookFailures(); } catch { webhookFailures = 0; } const webhooksStatus: OperatorHealthStatus = webhookFailures > 0 ? "warning" : "ok"; // Billing sync failures (BillingWebhookLog.error) — previously invisible. let billingErrors: readonly OperatorHealthBillingError[] = []; try { billingErrors = await loadBillingWebhookErrors(); } catch { billingErrors = []; } const billingStatus: OperatorHealthStatus = billingErrors.length > 0 ? "warning" : "ok"; return { subsystems: [ { id: "system", status: systemStatus, href: adminPath("system/health", { mode: "host-aware" }) }, { id: "deploy", status: deployStatus, href: adminPath("system/deploys", { mode: "host-aware" }) }, { id: "queues", status: queuesStatus, href: adminPath("system/jobs", { mode: "host-aware" }) }, { id: "webhooks", status: webhooksStatus, href: adminPath("system/webhooks", { mode: "host-aware" }) }, { id: "billing", status: billingStatus, href: adminPath("system/billing", { mode: "host-aware" }) }, ], deploy: { environment: deployMeta?.environment ?? "unknown", release: deployMeta?.release ?? "unknown", revision: deployMeta?.revision ?? "unknown", branch: deployMeta?.branch ?? "unknown", }, signals: { failedJobsTotal, pausedQueues, webhookFailures, billingErrors: billingErrors.length, }, links: { jobs: adminPath("system/jobs", { mode: "host-aware" }), webhooks: adminPath("system/webhooks", { mode: "host-aware" }), billing: adminPath("system/billing", { mode: "host-aware" }), }, billingErrors, }; }