Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
+181
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* Phase 06 #22 — Plugin crash tracker with auto-disable.
|
||||
*
|
||||
* Backed by SharedStore counters: each failure for
|
||||
* `(pluginId, tenantId)` increments a key that expires after a
|
||||
* sliding-window TTL. When the count crosses the configured threshold
|
||||
* the tracker:
|
||||
* 1. flips the Plugin row's `enabled` flag to `false` via the
|
||||
* injected disable seam,
|
||||
* 2. emits an audit-event-shaped notification through the injected
|
||||
* `notify` seam,
|
||||
* 3. resets the counter so a re-enable does not immediately
|
||||
* auto-disable again.
|
||||
*
|
||||
* Implementation notes:
|
||||
* - Successful invocations call `recordSuccess`, which decrements
|
||||
* (floored at 0) so flaky-but-mostly-OK plugins don't accumulate
|
||||
* stale failures forever.
|
||||
* - The tracker is intentionally seam-driven: it accepts a Redis-
|
||||
* shaped `incr / expire / get / set / del` surface so the worker
|
||||
* side can use the real shared-store adapter and tests can pass
|
||||
* an in-memory fake.
|
||||
* - Default window / threshold come from environment variables
|
||||
* PLUGIN_AUTO_DISABLE_THRESHOLD (default 10) and
|
||||
* PLUGIN_AUTO_DISABLE_WINDOW_MS (default 600000).
|
||||
*/
|
||||
|
||||
export type CrashTrackerStore = {
|
||||
incr(key: string, by?: number): Promise<number>;
|
||||
/** Set TTL on an existing key. Returns true when the key existed. */
|
||||
expire(key: string, ttlSeconds: number): Promise<boolean>;
|
||||
get(key: string): Promise<string | null>;
|
||||
set(key: string, value: string, opts?: { ttlSeconds?: number }): Promise<boolean>;
|
||||
del(key: string): Promise<number>;
|
||||
};
|
||||
|
||||
export type CrashTrackerNotifier = (args: {
|
||||
pluginId: string;
|
||||
tenantId: string;
|
||||
failureCount: number;
|
||||
reason: string;
|
||||
correlationId: string;
|
||||
}) => Promise<void>;
|
||||
|
||||
export type CrashTrackerDisableFn = (args: {
|
||||
pluginId: string;
|
||||
tenantId: string;
|
||||
failureCount: number;
|
||||
reason: string;
|
||||
correlationId: string;
|
||||
}) => Promise<void>;
|
||||
|
||||
export type CrashTrackerOptions = {
|
||||
/** Failure count required to flip the kill switch. Default 10. */
|
||||
threshold?: number;
|
||||
/** Window during which failures count. Default 10 minutes. */
|
||||
windowMs?: number;
|
||||
/** Optional logger for crash + auto-disable events. */
|
||||
logger?: {
|
||||
info: (fields: Record<string, unknown>, msg: string) => void;
|
||||
warn?: (fields: Record<string, unknown>, msg: string) => void;
|
||||
error?: (fields: Record<string, unknown>, msg: string) => void;
|
||||
};
|
||||
};
|
||||
|
||||
export type CrashTrackerDeps = {
|
||||
store: CrashTrackerStore;
|
||||
disablePlugin: CrashTrackerDisableFn;
|
||||
notify?: CrashTrackerNotifier;
|
||||
options?: CrashTrackerOptions;
|
||||
};
|
||||
|
||||
export type CrashTracker = {
|
||||
recordFailure(args: { pluginId: string; tenantId: string; reason: string; correlationId?: string }): Promise<{
|
||||
failureCount: number;
|
||||
autoDisabled: boolean;
|
||||
}>;
|
||||
recordSuccess(args: { pluginId: string; tenantId: string }): Promise<{ failureCount: number }>;
|
||||
getFailureCount(args: { pluginId: string; tenantId: string }): Promise<number>;
|
||||
};
|
||||
|
||||
const DEFAULT_THRESHOLD = 10;
|
||||
const DEFAULT_WINDOW_MS = 600_000;
|
||||
|
||||
function key(pluginId: string, tenantId: string): string {
|
||||
return `plugin:crash:${pluginId}:${tenantId}`;
|
||||
}
|
||||
|
||||
function getAutoDisableCorrelationId(correlationId: string | undefined, failureCount: number): string {
|
||||
return correlationId ?? `missing-invocation:${failureCount}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a tracker bound to a SharedStore-shaped backend. The returned
|
||||
* object is stateless — multiple workers can share the same store keys
|
||||
* because all operations are atomic (`incr`, `expire`).
|
||||
*/
|
||||
export function createCrashTracker(deps: CrashTrackerDeps): CrashTracker {
|
||||
const threshold = deps.options?.threshold ?? DEFAULT_THRESHOLD;
|
||||
const windowMs = deps.options?.windowMs ?? DEFAULT_WINDOW_MS;
|
||||
const ttlSeconds = Math.max(1, Math.ceil(windowMs / 1000));
|
||||
|
||||
return {
|
||||
async recordFailure({ pluginId, tenantId, reason, correlationId }) {
|
||||
const k = key(pluginId, tenantId);
|
||||
const count = await deps.store.incr(k, 1);
|
||||
// On first failure (count == 1) set the TTL so the window slides
|
||||
// from the FIRST observed failure, not the latest.
|
||||
if (count === 1) {
|
||||
await deps.store.expire(k, ttlSeconds);
|
||||
}
|
||||
deps.options?.logger?.warn?.(
|
||||
{ pluginId, tenantId, failureCount: count, reason },
|
||||
"plugin.crash.recorded"
|
||||
);
|
||||
if (count >= threshold) {
|
||||
const autoDisableCorrelationId = getAutoDisableCorrelationId(correlationId, count);
|
||||
try {
|
||||
await deps.disablePlugin({
|
||||
pluginId,
|
||||
tenantId,
|
||||
failureCount: count,
|
||||
reason,
|
||||
correlationId: autoDisableCorrelationId,
|
||||
});
|
||||
} catch (err) {
|
||||
deps.options?.logger?.error?.(
|
||||
{ pluginId, tenantId, error: err instanceof Error ? err.message : String(err) },
|
||||
"plugin.crash.disable_failed"
|
||||
);
|
||||
return { failureCount: count, autoDisabled: false };
|
||||
}
|
||||
if (deps.notify) {
|
||||
try {
|
||||
await deps.notify({
|
||||
pluginId,
|
||||
tenantId,
|
||||
failureCount: count,
|
||||
reason,
|
||||
correlationId: autoDisableCorrelationId,
|
||||
});
|
||||
} catch (err) {
|
||||
deps.options?.logger?.error?.(
|
||||
{ pluginId, tenantId, error: err instanceof Error ? err.message : String(err) },
|
||||
"plugin.crash.notify_failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Reset the counter so a re-enable starts from zero.
|
||||
await deps.store.del(k);
|
||||
deps.options?.logger?.info(
|
||||
{ pluginId, tenantId, failureCount: count },
|
||||
"plugin.crash.auto_disabled"
|
||||
);
|
||||
return { failureCount: count, autoDisabled: true };
|
||||
}
|
||||
return { failureCount: count, autoDisabled: false };
|
||||
},
|
||||
|
||||
async recordSuccess({ pluginId, tenantId }) {
|
||||
const k = key(pluginId, tenantId);
|
||||
const raw = await deps.store.get(k);
|
||||
const current = raw === null ? 0 : Number.parseInt(raw, 10) || 0;
|
||||
if (current <= 0) {
|
||||
return { failureCount: 0 };
|
||||
}
|
||||
const next = current - 1;
|
||||
if (next <= 0) {
|
||||
await deps.store.del(k);
|
||||
return { failureCount: 0 };
|
||||
}
|
||||
await deps.store.set(k, String(next), { ttlSeconds });
|
||||
return { failureCount: next };
|
||||
},
|
||||
|
||||
async getFailureCount({ pluginId, tenantId }) {
|
||||
const raw = await deps.store.get(key(pluginId, tenantId));
|
||||
return raw === null ? 0 : Number.parseInt(raw, 10) || 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user