/** * Phase 06 #22 — Plugin crash tracker with auto-disable. * * Backed by SharedStore counters: each failure for * `(pluginId, tenantId)` increments a key that expires after a * sliding-window TTL. When the count crosses the configured threshold * the tracker: * 1. flips the Plugin row's `enabled` flag to `false` via the * injected disable seam, * 2. emits an audit-event-shaped notification through the injected * `notify` seam, * 3. resets the counter so a re-enable does not immediately * auto-disable again. * * Implementation notes: * - Successful invocations call `recordSuccess`, which decrements * (floored at 0) so flaky-but-mostly-OK plugins don't accumulate * stale failures forever. * - The tracker is intentionally seam-driven: it accepts a Redis- * shaped `incr / expire / get / set / del` surface so the worker * side can use the real shared-store adapter and tests can pass * an in-memory fake. * - Default window / threshold come from environment variables * PLUGIN_AUTO_DISABLE_THRESHOLD (default 10) and * PLUGIN_AUTO_DISABLE_WINDOW_MS (default 600000). */ export type CrashTrackerStore = { incr(key: string, by?: number): Promise; /** Set TTL on an existing key. Returns true when the key existed. */ expire(key: string, ttlSeconds: number): Promise; get(key: string): Promise; set(key: string, value: string, opts?: { ttlSeconds?: number }): Promise; del(key: string): Promise; }; export type CrashTrackerNotifier = (args: { pluginId: string; tenantId: string; failureCount: number; reason: string; correlationId: string; }) => Promise; export type CrashTrackerDisableFn = (args: { pluginId: string; tenantId: string; failureCount: number; reason: string; correlationId: string; }) => Promise; export type CrashTrackerOptions = { /** Failure count required to flip the kill switch. Default 10. */ threshold?: number; /** Window during which failures count. Default 10 minutes. */ windowMs?: number; /** Optional logger for crash + auto-disable events. */ logger?: { info: (fields: Record, msg: string) => void; warn?: (fields: Record, msg: string) => void; error?: (fields: Record, msg: string) => void; }; }; export type CrashTrackerDeps = { store: CrashTrackerStore; disablePlugin: CrashTrackerDisableFn; notify?: CrashTrackerNotifier; options?: CrashTrackerOptions; }; export type CrashTracker = { recordFailure(args: { pluginId: string; tenantId: string; reason: string; correlationId?: string }): Promise<{ failureCount: number; autoDisabled: boolean; }>; recordSuccess(args: { pluginId: string; tenantId: string }): Promise<{ failureCount: number }>; getFailureCount(args: { pluginId: string; tenantId: string }): Promise; }; const DEFAULT_THRESHOLD = 10; const DEFAULT_WINDOW_MS = 600_000; function key(pluginId: string, tenantId: string): string { return `plugin:crash:${pluginId}:${tenantId}`; } function getAutoDisableCorrelationId(correlationId: string | undefined, failureCount: number): string { return correlationId ?? `missing-invocation:${failureCount}`; } /** * Create a tracker bound to a SharedStore-shaped backend. The returned * object is stateless — multiple workers can share the same store keys * because all operations are atomic (`incr`, `expire`). */ export function createCrashTracker(deps: CrashTrackerDeps): CrashTracker { const threshold = deps.options?.threshold ?? DEFAULT_THRESHOLD; const windowMs = deps.options?.windowMs ?? DEFAULT_WINDOW_MS; const ttlSeconds = Math.max(1, Math.ceil(windowMs / 1000)); return { async recordFailure({ pluginId, tenantId, reason, correlationId }) { const k = key(pluginId, tenantId); const count = await deps.store.incr(k, 1); // On first failure (count == 1) set the TTL so the window slides // from the FIRST observed failure, not the latest. if (count === 1) { await deps.store.expire(k, ttlSeconds); } deps.options?.logger?.warn?.( { pluginId, tenantId, failureCount: count, reason }, "plugin.crash.recorded" ); if (count >= threshold) { const autoDisableCorrelationId = getAutoDisableCorrelationId(correlationId, count); try { await deps.disablePlugin({ pluginId, tenantId, failureCount: count, reason, correlationId: autoDisableCorrelationId, }); } catch (err) { deps.options?.logger?.error?.( { pluginId, tenantId, error: err instanceof Error ? err.message : String(err) }, "plugin.crash.disable_failed" ); return { failureCount: count, autoDisabled: false }; } if (deps.notify) { try { await deps.notify({ pluginId, tenantId, failureCount: count, reason, correlationId: autoDisableCorrelationId, }); } catch (err) { deps.options?.logger?.error?.( { pluginId, tenantId, error: err instanceof Error ? err.message : String(err) }, "plugin.crash.notify_failed" ); } } // Reset the counter so a re-enable starts from zero. await deps.store.del(k); deps.options?.logger?.info( { pluginId, tenantId, failureCount: count }, "plugin.crash.auto_disabled" ); return { failureCount: count, autoDisabled: true }; } return { failureCount: count, autoDisabled: false }; }, async recordSuccess({ pluginId, tenantId }) { const k = key(pluginId, tenantId); const raw = await deps.store.get(k); const current = raw === null ? 0 : Number.parseInt(raw, 10) || 0; if (current <= 0) { return { failureCount: 0 }; } const next = current - 1; if (next <= 0) { await deps.store.del(k); return { failureCount: 0 }; } await deps.store.set(k, String(next), { ttlSeconds }); return { failureCount: next }; }, async getFailureCount({ pluginId, tenantId }) { const raw = await deps.store.get(key(pluginId, tenantId)); return raw === null ? 0 : Number.parseInt(raw, 10) || 0; }, }; }