Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/packages/plugin-runtime/src/crash-tracker.ts
T

182 lines
6.3 KiB
TypeScript

/**
* Phase 06 #22 — Plugin crash tracker with auto-disable.
*
* Backed by SharedStore counters: each failure for
* `(pluginId, tenantId)` increments a key that expires after a
* sliding-window TTL. When the count crosses the configured threshold
* the tracker:
* 1. flips the Plugin row's `enabled` flag to `false` via the
* injected disable seam,
* 2. emits an audit-event-shaped notification through the injected
* `notify` seam,
* 3. resets the counter so a re-enable does not immediately
* auto-disable again.
*
* Implementation notes:
* - Successful invocations call `recordSuccess`, which decrements
* (floored at 0) so flaky-but-mostly-OK plugins don't accumulate
* stale failures forever.
* - The tracker is intentionally seam-driven: it accepts a Redis-
* shaped `incr / expire / get / set / del` surface so the worker
* side can use the real shared-store adapter and tests can pass
* an in-memory fake.
* - Default window / threshold come from environment variables
* PLUGIN_AUTO_DISABLE_THRESHOLD (default 10) and
* PLUGIN_AUTO_DISABLE_WINDOW_MS (default 600000).
*/
export type CrashTrackerStore = {
incr(key: string, by?: number): Promise<number>;
/** Set TTL on an existing key. Returns true when the key existed. */
expire(key: string, ttlSeconds: number): Promise<boolean>;
get(key: string): Promise<string | null>;
set(key: string, value: string, opts?: { ttlSeconds?: number }): Promise<boolean>;
del(key: string): Promise<number>;
};
export type CrashTrackerNotifier = (args: {
pluginId: string;
tenantId: string;
failureCount: number;
reason: string;
correlationId: string;
}) => Promise<void>;
export type CrashTrackerDisableFn = (args: {
pluginId: string;
tenantId: string;
failureCount: number;
reason: string;
correlationId: string;
}) => Promise<void>;
export type CrashTrackerOptions = {
/** Failure count required to flip the kill switch. Default 10. */
threshold?: number;
/** Window during which failures count. Default 10 minutes. */
windowMs?: number;
/** Optional logger for crash + auto-disable events. */
logger?: {
info: (fields: Record<string, unknown>, msg: string) => void;
warn?: (fields: Record<string, unknown>, msg: string) => void;
error?: (fields: Record<string, unknown>, msg: string) => void;
};
};
export type CrashTrackerDeps = {
store: CrashTrackerStore;
disablePlugin: CrashTrackerDisableFn;
notify?: CrashTrackerNotifier;
options?: CrashTrackerOptions;
};
export type CrashTracker = {
recordFailure(args: { pluginId: string; tenantId: string; reason: string; correlationId?: string }): Promise<{
failureCount: number;
autoDisabled: boolean;
}>;
recordSuccess(args: { pluginId: string; tenantId: string }): Promise<{ failureCount: number }>;
getFailureCount(args: { pluginId: string; tenantId: string }): Promise<number>;
};
const DEFAULT_THRESHOLD = 10;
const DEFAULT_WINDOW_MS = 600_000;
function key(pluginId: string, tenantId: string): string {
return `plugin:crash:${pluginId}:${tenantId}`;
}
function getAutoDisableCorrelationId(correlationId: string | undefined, failureCount: number): string {
return correlationId ?? `missing-invocation:${failureCount}`;
}
/**
* Create a tracker bound to a SharedStore-shaped backend. The returned
* object is stateless — multiple workers can share the same store keys
* because all operations are atomic (`incr`, `expire`).
*/
export function createCrashTracker(deps: CrashTrackerDeps): CrashTracker {
const threshold = deps.options?.threshold ?? DEFAULT_THRESHOLD;
const windowMs = deps.options?.windowMs ?? DEFAULT_WINDOW_MS;
const ttlSeconds = Math.max(1, Math.ceil(windowMs / 1000));
return {
async recordFailure({ pluginId, tenantId, reason, correlationId }) {
const k = key(pluginId, tenantId);
const count = await deps.store.incr(k, 1);
// On first failure (count == 1) set the TTL so the window slides
// from the FIRST observed failure, not the latest.
if (count === 1) {
await deps.store.expire(k, ttlSeconds);
}
deps.options?.logger?.warn?.(
{ pluginId, tenantId, failureCount: count, reason },
"plugin.crash.recorded"
);
if (count >= threshold) {
const autoDisableCorrelationId = getAutoDisableCorrelationId(correlationId, count);
try {
await deps.disablePlugin({
pluginId,
tenantId,
failureCount: count,
reason,
correlationId: autoDisableCorrelationId,
});
} catch (err) {
deps.options?.logger?.error?.(
{ pluginId, tenantId, error: err instanceof Error ? err.message : String(err) },
"plugin.crash.disable_failed"
);
return { failureCount: count, autoDisabled: false };
}
if (deps.notify) {
try {
await deps.notify({
pluginId,
tenantId,
failureCount: count,
reason,
correlationId: autoDisableCorrelationId,
});
} catch (err) {
deps.options?.logger?.error?.(
{ pluginId, tenantId, error: err instanceof Error ? err.message : String(err) },
"plugin.crash.notify_failed"
);
}
}
// Reset the counter so a re-enable starts from zero.
await deps.store.del(k);
deps.options?.logger?.info(
{ pluginId, tenantId, failureCount: count },
"plugin.crash.auto_disabled"
);
return { failureCount: count, autoDisabled: true };
}
return { failureCount: count, autoDisabled: false };
},
async recordSuccess({ pluginId, tenantId }) {
const k = key(pluginId, tenantId);
const raw = await deps.store.get(k);
const current = raw === null ? 0 : Number.parseInt(raw, 10) || 0;
if (current <= 0) {
return { failureCount: 0 };
}
const next = current - 1;
if (next <= 0) {
await deps.store.del(k);
return { failureCount: 0 };
}
await deps.store.set(k, String(next), { ttlSeconds });
return { failureCount: next };
},
async getFailureCount({ pluginId, tenantId }) {
const raw = await deps.store.get(key(pluginId, tenantId));
return raw === null ? 0 : Number.parseInt(raw, 10) || 0;
},
};
}