156 lines
5.7 KiB
TypeScript
156 lines
5.7 KiB
TypeScript
/**
|
|
* Periodic refresher for gauge-style worker metrics.
|
|
*
|
|
* Counters and histograms are updated synchronously on the hot path
|
|
* (see queue runtime + business hooks). Gauges, on the other hand, model
|
|
* "current state" — they need a poller. Two cadences:
|
|
*
|
|
* - **30s**: BullMQ queue depth (`queue_jobs_active`, `queue_jobs_waiting`).
|
|
* Reads `getJobCounts()` per queue.
|
|
* - **60s**: business gauges (`active_tenants`, `active_users_24h`,
|
|
* `subscription_active`) loaded from the DB.
|
|
*
|
|
* Both jobs are best-effort: errors are logged and metrics simply retain
|
|
* their previous value until the next tick. The scheduler is started
|
|
* from the worker entrypoint when `METRICS_ENABLED=true`.
|
|
*/
|
|
import type { DefaultMetrics } from "@hivegrid/metrics";
|
|
|
|
export type QueueDepthSample = {
|
|
queue: string;
|
|
active: number;
|
|
waiting: number;
|
|
};
|
|
|
|
export type BusinessGaugeSample = {
|
|
activeTenants: number;
|
|
activeUsers24h: number;
|
|
subscriptionActiveByPlanId: Record<string, number>;
|
|
};
|
|
|
|
export type MetricsSchedulerLogger = {
|
|
error?: (fields: Record<string, unknown>, message: string) => void;
|
|
info?: (fields: Record<string, unknown>, message: string) => void;
|
|
};
|
|
|
|
export type MetricsSchedulerDependencies = {
|
|
metrics: Pick<DefaultMetrics, "queueJobsActive" | "queueJobsWaiting" | "activeTenants" | "activeUsers24h" | "subscriptionActive">;
|
|
/**
|
|
* Polls BullMQ queue depth. Returns an array of `{ queue, active, waiting }`
|
|
* for every queue this worker hosts. The default production wiring uses
|
|
* `Queue.getJobCounts()`; tests inject a fake.
|
|
*/
|
|
fetchQueueDepth: () => Promise<readonly QueueDepthSample[]>;
|
|
/**
|
|
* Polls DB business stats. Returns active tenant/user counts plus a flat
|
|
* map of `plan_id => count` for the subscription gauge.
|
|
*/
|
|
fetchBusinessGauges: () => Promise<BusinessGaugeSample>;
|
|
logger?: MetricsSchedulerLogger;
|
|
/**
|
|
* Optional setInterval shim — tests inject a manual clock to drive each
|
|
* tick deterministically. Defaults to the global `setInterval`.
|
|
*/
|
|
setInterval?: (handler: () => void, ms: number) => unknown;
|
|
/**
|
|
* Optional clearInterval shim matching `setInterval`. Production uses the
|
|
* global `clearInterval`.
|
|
*/
|
|
clearInterval?: (handle: unknown) => void;
|
|
/**
|
|
* Override poll cadences (in ms). Defaults: queue=30_000, business=60_000.
|
|
*/
|
|
queueIntervalMs?: number;
|
|
businessIntervalMs?: number;
|
|
};
|
|
|
|
export type MetricsScheduler = {
|
|
/** Trigger one queue-depth refresh immediately. Useful for tests + on-startup priming. */
|
|
refreshQueueDepth: () => Promise<void>;
|
|
/** Trigger one business-gauge refresh immediately. */
|
|
refreshBusinessGauges: () => Promise<void>;
|
|
/** Stop both timers. Idempotent. */
|
|
stop: () => void;
|
|
};
|
|
|
|
const DEFAULT_QUEUE_INTERVAL_MS = 30_000;
|
|
const DEFAULT_BUSINESS_INTERVAL_MS = 60_000;
|
|
|
|
export function startMetricsScheduler(deps: MetricsSchedulerDependencies): MetricsScheduler {
|
|
const setIntervalImpl = deps.setInterval ?? ((h: () => void, ms: number) => setInterval(h, ms));
|
|
const clearIntervalImpl = deps.clearInterval ?? ((handle: unknown) => clearInterval(handle as ReturnType<typeof setInterval>));
|
|
const queueIntervalMs = deps.queueIntervalMs ?? DEFAULT_QUEUE_INTERVAL_MS;
|
|
const businessIntervalMs = deps.businessIntervalMs ?? DEFAULT_BUSINESS_INTERVAL_MS;
|
|
const logger = deps.logger;
|
|
let queueRefreshInFlight: Promise<void> | undefined;
|
|
let businessRefreshInFlight: Promise<void> | undefined;
|
|
|
|
function refreshQueueDepth(): Promise<void> {
|
|
if (!queueRefreshInFlight) {
|
|
queueRefreshInFlight = (async () => {
|
|
try {
|
|
const samples = await deps.fetchQueueDepth();
|
|
for (const sample of samples) {
|
|
deps.metrics.queueJobsActive.labels({ queue: sample.queue }).set(sample.active);
|
|
deps.metrics.queueJobsWaiting.labels({ queue: sample.queue }).set(sample.waiting);
|
|
}
|
|
} catch (error) {
|
|
logger?.error?.({ error: redactError(error) }, "metrics-scheduler queue-depth refresh failed");
|
|
}
|
|
})().finally(() => {
|
|
queueRefreshInFlight = undefined;
|
|
});
|
|
}
|
|
return queueRefreshInFlight;
|
|
}
|
|
|
|
function refreshBusinessGauges(): Promise<void> {
|
|
if (!businessRefreshInFlight) {
|
|
businessRefreshInFlight = (async () => {
|
|
try {
|
|
const sample = await deps.fetchBusinessGauges();
|
|
deps.metrics.activeTenants.set(sample.activeTenants);
|
|
deps.metrics.activeUsers24h.set(sample.activeUsers24h);
|
|
// Reset before re-setting so a previously-tracked plan that disappears
|
|
// doesn't keep emitting a stale value.
|
|
deps.metrics.subscriptionActive.reset();
|
|
for (const [planId, count] of Object.entries(sample.subscriptionActiveByPlanId)) {
|
|
deps.metrics.subscriptionActive.labels({ plan_id: planId }).set(count);
|
|
}
|
|
} catch (error) {
|
|
logger?.error?.({ error: redactError(error) }, "metrics-scheduler business-gauge refresh failed");
|
|
}
|
|
})().finally(() => {
|
|
businessRefreshInFlight = undefined;
|
|
});
|
|
}
|
|
return businessRefreshInFlight;
|
|
}
|
|
|
|
const queueHandle = setIntervalImpl(() => {
|
|
void refreshQueueDepth();
|
|
}, queueIntervalMs);
|
|
const businessHandle = setIntervalImpl(() => {
|
|
void refreshBusinessGauges();
|
|
}, businessIntervalMs);
|
|
|
|
let stopped = false;
|
|
return {
|
|
refreshQueueDepth,
|
|
refreshBusinessGauges,
|
|
stop: () => {
|
|
if (stopped) return;
|
|
stopped = true;
|
|
clearIntervalImpl(queueHandle);
|
|
clearIntervalImpl(businessHandle);
|
|
},
|
|
};
|
|
}
|
|
|
|
function redactError(error: unknown): { name: string; message: string } {
|
|
if (error instanceof Error) {
|
|
return { name: error.name, message: error.message.slice(0, 200) };
|
|
}
|
|
return { name: "UnknownError", message: String(error).slice(0, 200) };
|
|
}
|