182 lines
6.3 KiB
TypeScript
182 lines
6.3 KiB
TypeScript
/**
|
|
* Phase 06 #22 — Plugin crash tracker with auto-disable.
|
|
*
|
|
* Backed by SharedStore counters: each failure for
|
|
* `(pluginId, tenantId)` increments a key that expires after a
|
|
* sliding-window TTL. When the count crosses the configured threshold
|
|
* the tracker:
|
|
* 1. flips the Plugin row's `enabled` flag to `false` via the
|
|
* injected disable seam,
|
|
* 2. emits an audit-event-shaped notification through the injected
|
|
* `notify` seam,
|
|
* 3. resets the counter so a re-enable does not immediately
|
|
* auto-disable again.
|
|
*
|
|
* Implementation notes:
|
|
* - Successful invocations call `recordSuccess`, which decrements
|
|
* (floored at 0) so flaky-but-mostly-OK plugins don't accumulate
|
|
* stale failures forever.
|
|
* - The tracker is intentionally seam-driven: it accepts a Redis-
|
|
* shaped `incr / expire / get / set / del` surface so the worker
|
|
* side can use the real shared-store adapter and tests can pass
|
|
* an in-memory fake.
|
|
* - Default window / threshold come from environment variables
|
|
* PLUGIN_AUTO_DISABLE_THRESHOLD (default 10) and
|
|
* PLUGIN_AUTO_DISABLE_WINDOW_MS (default 600000).
|
|
*/
|
|
|
|
export type CrashTrackerStore = {
|
|
incr(key: string, by?: number): Promise<number>;
|
|
/** Set TTL on an existing key. Returns true when the key existed. */
|
|
expire(key: string, ttlSeconds: number): Promise<boolean>;
|
|
get(key: string): Promise<string | null>;
|
|
set(key: string, value: string, opts?: { ttlSeconds?: number }): Promise<boolean>;
|
|
del(key: string): Promise<number>;
|
|
};
|
|
|
|
export type CrashTrackerNotifier = (args: {
|
|
pluginId: string;
|
|
tenantId: string;
|
|
failureCount: number;
|
|
reason: string;
|
|
correlationId: string;
|
|
}) => Promise<void>;
|
|
|
|
export type CrashTrackerDisableFn = (args: {
|
|
pluginId: string;
|
|
tenantId: string;
|
|
failureCount: number;
|
|
reason: string;
|
|
correlationId: string;
|
|
}) => Promise<void>;
|
|
|
|
export type CrashTrackerOptions = {
|
|
/** Failure count required to flip the kill switch. Default 10. */
|
|
threshold?: number;
|
|
/** Window during which failures count. Default 10 minutes. */
|
|
windowMs?: number;
|
|
/** Optional logger for crash + auto-disable events. */
|
|
logger?: {
|
|
info: (fields: Record<string, unknown>, msg: string) => void;
|
|
warn?: (fields: Record<string, unknown>, msg: string) => void;
|
|
error?: (fields: Record<string, unknown>, msg: string) => void;
|
|
};
|
|
};
|
|
|
|
export type CrashTrackerDeps = {
|
|
store: CrashTrackerStore;
|
|
disablePlugin: CrashTrackerDisableFn;
|
|
notify?: CrashTrackerNotifier;
|
|
options?: CrashTrackerOptions;
|
|
};
|
|
|
|
export type CrashTracker = {
|
|
recordFailure(args: { pluginId: string; tenantId: string; reason: string; correlationId?: string }): Promise<{
|
|
failureCount: number;
|
|
autoDisabled: boolean;
|
|
}>;
|
|
recordSuccess(args: { pluginId: string; tenantId: string }): Promise<{ failureCount: number }>;
|
|
getFailureCount(args: { pluginId: string; tenantId: string }): Promise<number>;
|
|
};
|
|
|
|
const DEFAULT_THRESHOLD = 10;
|
|
const DEFAULT_WINDOW_MS = 600_000;
|
|
|
|
function key(pluginId: string, tenantId: string): string {
|
|
return `plugin:crash:${pluginId}:${tenantId}`;
|
|
}
|
|
|
|
function getAutoDisableCorrelationId(correlationId: string | undefined, failureCount: number): string {
|
|
return correlationId ?? `missing-invocation:${failureCount}`;
|
|
}
|
|
|
|
/**
|
|
* Create a tracker bound to a SharedStore-shaped backend. The returned
|
|
* object is stateless — multiple workers can share the same store keys
|
|
* because all operations are atomic (`incr`, `expire`).
|
|
*/
|
|
export function createCrashTracker(deps: CrashTrackerDeps): CrashTracker {
|
|
const threshold = deps.options?.threshold ?? DEFAULT_THRESHOLD;
|
|
const windowMs = deps.options?.windowMs ?? DEFAULT_WINDOW_MS;
|
|
const ttlSeconds = Math.max(1, Math.ceil(windowMs / 1000));
|
|
|
|
return {
|
|
async recordFailure({ pluginId, tenantId, reason, correlationId }) {
|
|
const k = key(pluginId, tenantId);
|
|
const count = await deps.store.incr(k, 1);
|
|
// On first failure (count == 1) set the TTL so the window slides
|
|
// from the FIRST observed failure, not the latest.
|
|
if (count === 1) {
|
|
await deps.store.expire(k, ttlSeconds);
|
|
}
|
|
deps.options?.logger?.warn?.(
|
|
{ pluginId, tenantId, failureCount: count, reason },
|
|
"plugin.crash.recorded"
|
|
);
|
|
if (count >= threshold) {
|
|
const autoDisableCorrelationId = getAutoDisableCorrelationId(correlationId, count);
|
|
try {
|
|
await deps.disablePlugin({
|
|
pluginId,
|
|
tenantId,
|
|
failureCount: count,
|
|
reason,
|
|
correlationId: autoDisableCorrelationId,
|
|
});
|
|
} catch (err) {
|
|
deps.options?.logger?.error?.(
|
|
{ pluginId, tenantId, error: err instanceof Error ? err.message : String(err) },
|
|
"plugin.crash.disable_failed"
|
|
);
|
|
return { failureCount: count, autoDisabled: false };
|
|
}
|
|
if (deps.notify) {
|
|
try {
|
|
await deps.notify({
|
|
pluginId,
|
|
tenantId,
|
|
failureCount: count,
|
|
reason,
|
|
correlationId: autoDisableCorrelationId,
|
|
});
|
|
} catch (err) {
|
|
deps.options?.logger?.error?.(
|
|
{ pluginId, tenantId, error: err instanceof Error ? err.message : String(err) },
|
|
"plugin.crash.notify_failed"
|
|
);
|
|
}
|
|
}
|
|
// Reset the counter so a re-enable starts from zero.
|
|
await deps.store.del(k);
|
|
deps.options?.logger?.info(
|
|
{ pluginId, tenantId, failureCount: count },
|
|
"plugin.crash.auto_disabled"
|
|
);
|
|
return { failureCount: count, autoDisabled: true };
|
|
}
|
|
return { failureCount: count, autoDisabled: false };
|
|
},
|
|
|
|
async recordSuccess({ pluginId, tenantId }) {
|
|
const k = key(pluginId, tenantId);
|
|
const raw = await deps.store.get(k);
|
|
const current = raw === null ? 0 : Number.parseInt(raw, 10) || 0;
|
|
if (current <= 0) {
|
|
return { failureCount: 0 };
|
|
}
|
|
const next = current - 1;
|
|
if (next <= 0) {
|
|
await deps.store.del(k);
|
|
return { failureCount: 0 };
|
|
}
|
|
await deps.store.set(k, String(next), { ttlSeconds });
|
|
return { failureCount: next };
|
|
},
|
|
|
|
async getFailureCount({ pluginId, tenantId }) {
|
|
const raw = await deps.store.get(key(pluginId, tenantId));
|
|
return raw === null ? 0 : Number.parseInt(raw, 10) || 0;
|
|
},
|
|
};
|
|
}
|