Track HiveGrid Cloud planning reference
This commit is contained in:
1 parent
b3d5ea6764
commit
e341c5938a
5784 files changed
+1104929
-3
No files matched your search
@@ -0,0 +1,650 @@
|
||||
/**
|
||||
* Phase 06 #21 — Production wiring for the plugin-invoke worker handler.
|
||||
*
|
||||
* `handlePluginInvokeJob` lives in `./plugin-invoke.ts` and is fully
|
||||
* dependency-injected. This module composes the real production seams:
|
||||
*
|
||||
* - `loadPlugin` → Prisma `Plugin.findUnique`
|
||||
* - `loadBinary` → filesystem read from `PLUGIN_BINARY_BASE_PATH`
|
||||
* - `verifySignature` → `verifyPluginSignature` from plugin-runtime
|
||||
* - `consumeRateLimit` → SharedStore sliding-window counter
|
||||
* - `invokeInSandbox` → worker-thread sandbox + capability gateway
|
||||
* - `recordAudit` → Prisma `PluginInvocation.create` + structured logger
|
||||
* - `trustedPublicKeysHex` → parsed once from `PLUGIN_TRUSTED_PUBLIC_KEYS`
|
||||
* - `recordOutcomeForCrashTracking` → `createCrashTracker` adapter
|
||||
*
|
||||
* Closes F-015 (Critical) — before this bootstrap, `handlePluginInvokeJob`
|
||||
* was not wired into production; the `plugin-invoke` queue accepted jobs
|
||||
* but the handler at `handlers.ts:752` was a silent no-op stub.
|
||||
*
|
||||
* Out of scope for this bootstrap (tracked in plugin-runtime deviations):
|
||||
* - host-fn implementations for higher-risk capabilities. The safe
|
||||
* `log` and `audit` imports are wired; network, storage, secret,
|
||||
* tenant.info and DB bindings remain fail-closed until reviewed.
|
||||
* - hot-reload of binaries / manifests
|
||||
*/
|
||||
|
||||
import { promises as fs } from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
import {
|
||||
PluginCapabilityUseAuditTruncatedError,
|
||||
createCrashTracker,
|
||||
createPluginCapabilityUseRecorder,
|
||||
assertPluginRuntimeProductionEnabled,
|
||||
parsePluginManifest,
|
||||
parseTrustedPublicKeys,
|
||||
verifyPluginSignature,
|
||||
type CrashTrackerStore,
|
||||
type PluginCapabilityGrant,
|
||||
type PluginCapabilityUseBatch,
|
||||
type PluginInvocationResult,
|
||||
} from "@toolctrl/plugin-runtime";
|
||||
import { AUDIT_EVENT_OUTBOX_TYPE, buildAuditEventCreateData } from "@toolctrl/db/audit-events.server";
|
||||
import type { SharedStore } from "@toolctrl/shared-store";
|
||||
|
||||
import type {
|
||||
PluginInvokeDeps,
|
||||
PluginInvokeOutcome,
|
||||
PluginInvokeRow,
|
||||
RateLimitDecision,
|
||||
} from "./plugin-invoke.js";
|
||||
import { PLUGIN_CAPABILITY_USE_EVENT_TYPE } from "./plugin-capability-use.js";
|
||||
import { invokePluginSandboxInWorker } from "./plugin-sandbox-isolated.js";
|
||||
|
||||
export type PluginInvokeBootstrapLogger = {
|
||||
info: (fields: Record<string, unknown>, msg: string) => void;
|
||||
warn?: (fields: Record<string, unknown>, msg?: string) => void;
|
||||
error?: (fields: Record<string, unknown>, msg?: string) => void;
|
||||
};
|
||||
|
||||
export type PluginInvokeBootstrapPrismaSurface = {
|
||||
plugin: {
|
||||
findUnique: (args: {
|
||||
where: { tenantId_pluginId: { tenantId: string; pluginId: string } };
|
||||
}) => Promise<{
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
version: string;
|
||||
manifest: unknown;
|
||||
wasmSha256: string;
|
||||
signaturePublicKey: string;
|
||||
enabled: boolean;
|
||||
} | null>;
|
||||
update: (args: {
|
||||
where: { tenantId_pluginId: { tenantId: string; pluginId: string } };
|
||||
data: { enabled: boolean };
|
||||
}) => Promise<unknown>;
|
||||
};
|
||||
$transaction?: (fn: (tx: PluginInvokeBootstrapPrismaSurface) => Promise<unknown>) => Promise<unknown>;
|
||||
outboxEvent: {
|
||||
create: (args: {
|
||||
data: {
|
||||
tenantId: string | null;
|
||||
eventType: string;
|
||||
payload: unknown;
|
||||
dispatchKey: string;
|
||||
priority?: number;
|
||||
};
|
||||
}) => Promise<unknown>;
|
||||
findUnique: (args: {
|
||||
where: { dispatchKey: string };
|
||||
}) => Promise<unknown | null>;
|
||||
};
|
||||
pluginInvocation: {
|
||||
create: (args: {
|
||||
data: {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
version: string;
|
||||
hookName: string;
|
||||
outcome: string;
|
||||
failureReason?: string | null;
|
||||
message?: string | null;
|
||||
durationMs: number;
|
||||
fuelUsed?: number | null;
|
||||
};
|
||||
}) => Promise<unknown>;
|
||||
};
|
||||
tenantPluginCapabilityGrant: {
|
||||
findMany: (args: {
|
||||
where: { tenantId: string; pluginId: string };
|
||||
select: { capabilityName: true; scopeJson: true };
|
||||
}) => Promise<Array<{ capabilityName: string; scopeJson: unknown }>>;
|
||||
};
|
||||
};
|
||||
|
||||
export type PluginInvokeBootstrapOptions = {
|
||||
env: NodeJS.ProcessEnv;
|
||||
prisma: PluginInvokeBootstrapPrismaSurface;
|
||||
store: SharedStore;
|
||||
logger: PluginInvokeBootstrapLogger;
|
||||
};
|
||||
|
||||
/** Default base path for plugin binaries on disk. */
|
||||
const DEFAULT_PLUGIN_BINARY_BASE_PATH = "/app/var/plugins";
|
||||
const DEFAULT_RATE_LIMIT_PER_MINUTE = 100;
|
||||
const DEFAULT_RATE_LIMIT_WINDOW_SECONDS = 60;
|
||||
const DEFAULT_PLUGIN_MEMORY_LIMIT_MB = 64;
|
||||
const DEFAULT_PLUGIN_FUEL_LIMIT = 1_000_000;
|
||||
const DEFAULT_PLUGIN_WALL_CLOCK_MS = 5_000;
|
||||
const DEFAULT_PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES = 2_048;
|
||||
const DEFAULT_PLUGIN_AUTO_DISABLE_THRESHOLD = 10;
|
||||
const DEFAULT_PLUGIN_AUTO_DISABLE_WINDOW_MS = 600_000;
|
||||
|
||||
function resolveIntEnv(env: NodeJS.ProcessEnv, key: string, fallback: number): number {
|
||||
const raw = env[key];
|
||||
if (typeof raw !== "string" || raw.length === 0) return fallback;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin pass-through from SharedStore (`incr`, `expire`, `get`, `set`,
|
||||
* `del`) to the CrashTracker's expected store contract. The shapes are
|
||||
* already aligned; the wrapper exists so callers can swap in a fake
|
||||
* store in tests without dragging SharedStore in.
|
||||
*/
|
||||
function adaptCrashTrackerStore(store: SharedStore): CrashTrackerStore {
|
||||
return {
|
||||
incr: (key, by = 1) => store.incr(key, by),
|
||||
expire: (key, ttlSeconds) => store.expire(key, ttlSeconds),
|
||||
get: (key) => store.get(key),
|
||||
set: async (key, value, opts) => store.set(key, value, opts?.ttlSeconds ? { ttlSeconds: opts.ttlSeconds } : undefined),
|
||||
del: (key) => store.del(key),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sliding-window rate limiter keyed by `(pluginId, tenantId)`. Uses the
|
||||
* SharedStore counter primitive — atomic incr + ttl on first hit.
|
||||
*/
|
||||
function makeRateLimiter(args: {
|
||||
store: SharedStore;
|
||||
perMinute: number;
|
||||
windowSeconds: number;
|
||||
}): PluginInvokeDeps["consumeRateLimit"] {
|
||||
return async ({ pluginId, tenantId }): Promise<RateLimitDecision> => {
|
||||
const key = `plugin-ratelimit-${pluginId}-${tenantId}`;
|
||||
const count = await args.store.incr(key, 1);
|
||||
if (count === 1) {
|
||||
await args.store.expire(key, args.windowSeconds);
|
||||
}
|
||||
const allowed = count <= args.perMinute;
|
||||
return { allowed, remaining: Math.max(0, args.perMinute - count) };
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Filesystem loader. Reads `<base>/<pluginId>/<version>/plugin.wasm`.
|
||||
* Returns null on ENOENT so the handler can route to `binary_load_failed`
|
||||
* cleanly; any other error propagates (BullMQ retries).
|
||||
*/
|
||||
function makeBinaryLoader(args: {
|
||||
basePath: string;
|
||||
logger: PluginInvokeBootstrapLogger;
|
||||
}): PluginInvokeDeps["loadBinary"] {
|
||||
const basePath = path.resolve(args.basePath);
|
||||
return async ({ pluginId, version }) => {
|
||||
const filePath = path.resolve(basePath, pluginId, version, "plugin.wasm");
|
||||
const relativePath = path.relative(basePath, filePath);
|
||||
if (relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
|
||||
args.logger.warn?.({ pluginId, version }, "plugin.invoke.binary_path_escape_blocked");
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const buf = await fs.readFile(filePath);
|
||||
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
args.logger.warn?.({ pluginId, version, filePath }, "plugin.invoke.binary_missing_on_disk");
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Prisma adapter. Reads the Plugin row + JSON manifest. Returns null on
|
||||
* miss; the handler then short-circuits to `plugin_not_found`.
|
||||
*/
|
||||
function makePluginLoader(
|
||||
prisma: PluginInvokeBootstrapPrismaSurface,
|
||||
logger: PluginInvokeBootstrapLogger
|
||||
): PluginInvokeDeps["loadPlugin"] {
|
||||
return async ({ tenantId, pluginId, version }) => {
|
||||
const row = await prisma.plugin.findUnique({
|
||||
where: { tenantId_pluginId: { tenantId, pluginId } },
|
||||
});
|
||||
if (!row) return null;
|
||||
if (row.version !== version) return null;
|
||||
const parsed = parsePluginManifest(row.manifest);
|
||||
if (!parsed.ok) {
|
||||
logger.warn?.({ tenantId, pluginId, version, issues: parsed.issues }, "plugin.invoke.stored_manifest_invalid");
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
tenantId: row.tenantId,
|
||||
pluginId: row.pluginId,
|
||||
version: row.version,
|
||||
manifest: parsed.manifest,
|
||||
wasmSha256: row.wasmSha256,
|
||||
signaturePublicKey: row.signaturePublicKey,
|
||||
enabled: row.enabled,
|
||||
} satisfies PluginInvokeRow;
|
||||
};
|
||||
}
|
||||
|
||||
function makeCapabilityGrantLoader(
|
||||
prisma: PluginInvokeBootstrapPrismaSurface
|
||||
): PluginInvokeDeps["loadCapabilityGrants"] {
|
||||
return async ({ tenantId, pluginId }): Promise<PluginCapabilityGrant[]> => {
|
||||
const rows = await prisma.tenantPluginCapabilityGrant.findMany({
|
||||
where: { tenantId, pluginId },
|
||||
select: { capabilityName: true, scopeJson: true },
|
||||
});
|
||||
return rows.map((row) => ({
|
||||
capabilityName: row.capabilityName,
|
||||
scopeJson: row.scopeJson,
|
||||
}));
|
||||
};
|
||||
}
|
||||
|
||||
function makeCapabilityUseSink(args: {
|
||||
prisma: PluginInvokeBootstrapPrismaSurface;
|
||||
logger: PluginInvokeBootstrapLogger;
|
||||
}) {
|
||||
return {
|
||||
emit: async (batch: PluginCapabilityUseBatch): Promise<void> => {
|
||||
const dispatchKey = `plugin-capability-use:${batch.pluginInvocationId}`;
|
||||
try {
|
||||
await args.prisma.outboxEvent.create({
|
||||
data: {
|
||||
tenantId: batch.tenantId,
|
||||
eventType: PLUGIN_CAPABILITY_USE_EVENT_TYPE,
|
||||
payload: batch,
|
||||
dispatchKey,
|
||||
priority: 10,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (!isUniqueConstraintViolation(error)) {
|
||||
throw error;
|
||||
}
|
||||
const existing = await args.prisma.outboxEvent.findUnique({
|
||||
where: { dispatchKey },
|
||||
});
|
||||
if (!existing) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
args.logger.info(
|
||||
{
|
||||
tenantId: batch.tenantId,
|
||||
pluginId: batch.pluginId,
|
||||
pluginInvocationId: batch.pluginInvocationId,
|
||||
entryCount: batch.entries.length,
|
||||
},
|
||||
"plugin.invoke.capability_use_enqueued"
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type JsonPrimitive = string | number | boolean | null;
|
||||
type JsonSafeValue = JsonPrimitive | JsonSafeValue[] | { [key: string]: JsonSafeValue };
|
||||
|
||||
const MAX_AUDIT_METADATA_DEPTH = 8;
|
||||
|
||||
function toJsonSafeValue(value: unknown, depth = 0): JsonSafeValue {
|
||||
if (depth > MAX_AUDIT_METADATA_DEPTH) {
|
||||
return "[TRUNCATED]";
|
||||
}
|
||||
if (value === null || typeof value === "string" || typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "number") {
|
||||
return Number.isFinite(value) ? value : String(value);
|
||||
}
|
||||
if (typeof value === "bigint") {
|
||||
return value.toString();
|
||||
}
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((entry) => toJsonSafeValue(entry, depth + 1));
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.filter(([, entry]) => typeof entry !== "undefined" && typeof entry !== "function" && typeof entry !== "symbol")
|
||||
.map(([key, entry]) => [key, toJsonSafeValue(entry, depth + 1)])
|
||||
);
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
async function enqueuePluginHostAuditEvent(args: {
|
||||
prisma: PluginInvokeBootstrapPrismaSurface;
|
||||
logger: PluginInvokeBootstrapLogger;
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
version: string;
|
||||
hookName: string;
|
||||
pluginInvocationId: string;
|
||||
sequence: number;
|
||||
hostRecord: Record<string, unknown>;
|
||||
}): Promise<void> {
|
||||
const payload = buildAuditEventCreateData({
|
||||
tenantId: args.tenantId,
|
||||
actorId: args.pluginId,
|
||||
actorType: "plugin",
|
||||
action: "plugin.audit.recorded",
|
||||
targetType: "plugin",
|
||||
targetId: args.pluginId,
|
||||
requestId: args.pluginInvocationId,
|
||||
sourceModule: "worker.plugin-runtime",
|
||||
outcome: "success",
|
||||
dataSensitivity: "internal",
|
||||
regulatoryScope: ["soc2"],
|
||||
metadata: {
|
||||
pluginId: args.pluginId,
|
||||
pluginManifestId: typeof args.hostRecord.pluginId === "string" ? args.hostRecord.pluginId : args.pluginId,
|
||||
pluginInvocationId: args.pluginInvocationId,
|
||||
version: args.version,
|
||||
hookName: args.hookName,
|
||||
hostBinding: "audit",
|
||||
sequence: args.sequence,
|
||||
value: toJsonSafeValue(args.hostRecord.value),
|
||||
},
|
||||
});
|
||||
const dispatchKey = `plugin-host-audit:${args.pluginInvocationId}:${args.sequence}`;
|
||||
try {
|
||||
await args.prisma.outboxEvent.create({
|
||||
data: {
|
||||
tenantId: args.tenantId,
|
||||
eventType: AUDIT_EVENT_OUTBOX_TYPE,
|
||||
payload,
|
||||
dispatchKey,
|
||||
priority: 20,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (!isUniqueConstraintViolation(error)) {
|
||||
throw error;
|
||||
}
|
||||
const existing = await args.prisma.outboxEvent.findUnique({
|
||||
where: { dispatchKey },
|
||||
});
|
||||
if (!existing) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
args.logger.info(
|
||||
{
|
||||
tenantId: args.tenantId,
|
||||
pluginId: args.pluginId,
|
||||
version: args.version,
|
||||
hookName: args.hookName,
|
||||
pluginInvocationId: args.pluginInvocationId,
|
||||
sequence: args.sequence,
|
||||
},
|
||||
"plugin.invoke.host.audit_enqueued"
|
||||
);
|
||||
}
|
||||
|
||||
async function enqueuePluginAutoDisableAudit(args: {
|
||||
prisma: PluginInvokeBootstrapPrismaSurface;
|
||||
logger: PluginInvokeBootstrapLogger;
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
failureCount: number;
|
||||
reason: string;
|
||||
correlationId: string;
|
||||
}): Promise<void> {
|
||||
const payload = buildAuditEventCreateData({
|
||||
tenantId: args.tenantId,
|
||||
actorType: "system",
|
||||
action: "plugin.auto_disabled",
|
||||
targetType: "plugin",
|
||||
targetId: args.pluginId,
|
||||
sourceModule: "worker.plugin-runtime",
|
||||
outcome: "blocked",
|
||||
dataSensitivity: "internal",
|
||||
regulatoryScope: ["soc2"],
|
||||
metadata: {
|
||||
pluginId: args.pluginId,
|
||||
failureCount: args.failureCount,
|
||||
reason: args.reason,
|
||||
pluginInvocationId: args.correlationId,
|
||||
trigger: "crash-threshold",
|
||||
},
|
||||
});
|
||||
const dispatchKey = `plugin-auto-disable-audit:${args.tenantId}:${args.pluginId}:${args.correlationId}`;
|
||||
try {
|
||||
await args.prisma.outboxEvent.create({
|
||||
data: {
|
||||
tenantId: args.tenantId,
|
||||
eventType: AUDIT_EVENT_OUTBOX_TYPE,
|
||||
payload,
|
||||
dispatchKey,
|
||||
priority: 20,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (!isUniqueConstraintViolation(error)) {
|
||||
throw error;
|
||||
}
|
||||
const existing = await args.prisma.outboxEvent.findUnique({
|
||||
where: { dispatchKey },
|
||||
});
|
||||
if (!existing) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
args.logger.info(
|
||||
{
|
||||
tenantId: args.tenantId,
|
||||
pluginId: args.pluginId,
|
||||
failureCount: args.failureCount,
|
||||
reason: args.reason,
|
||||
},
|
||||
"plugin.crash.auto_disable_audit_enqueued"
|
||||
);
|
||||
}
|
||||
|
||||
function isUniqueConstraintViolation(error: unknown): boolean {
|
||||
return typeof error === "object" && error !== null && (error as { code?: unknown }).code === "P2002";
|
||||
}
|
||||
|
||||
/**
|
||||
* Sandbox adapter. Runs V8 WebAssembly in a killable worker thread and replays
|
||||
* safe sync `log` and `audit` host imports back into the parent process for
|
||||
* logging + capability-use auditing. Higher-risk host bindings remain absent
|
||||
* and fail closed with `no_host_implementation`.
|
||||
*/
|
||||
function makeSandboxInvoker(args: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
prisma: PluginInvokeBootstrapPrismaSurface;
|
||||
logger: PluginInvokeBootstrapLogger;
|
||||
capabilityUseSink: ReturnType<typeof makeCapabilityUseSink>;
|
||||
}): PluginInvokeDeps["invokeInSandbox"] {
|
||||
const memoryLimitMb = resolveIntEnv(args.env, "PLUGIN_MAX_MEMORY_MB", DEFAULT_PLUGIN_MEMORY_LIMIT_MB);
|
||||
const fuelLimit = resolveIntEnv(args.env, "PLUGIN_MAX_FUEL", DEFAULT_PLUGIN_FUEL_LIMIT);
|
||||
const wallClockMs = resolveIntEnv(args.env, "PLUGIN_MAX_WALL_CLOCK_MS", DEFAULT_PLUGIN_WALL_CLOCK_MS);
|
||||
const maxCapabilityCallEntries = resolveIntEnv(
|
||||
args.env,
|
||||
"PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES",
|
||||
DEFAULT_PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES
|
||||
);
|
||||
|
||||
return async ({ pluginInvocationId, tenantId, pluginId, wasmBuffer, manifest, capabilityGrants, hookName, payload }): Promise<PluginInvocationResult> => {
|
||||
const capabilityRecorder = createPluginCapabilityUseRecorder({
|
||||
pluginInvocationId,
|
||||
tenantId,
|
||||
pluginId,
|
||||
sink: args.capabilityUseSink,
|
||||
logger: args.logger,
|
||||
failClosed: true,
|
||||
});
|
||||
try {
|
||||
const result = await invokePluginSandboxInWorker({
|
||||
wasmBuffer,
|
||||
manifest,
|
||||
capabilityGrants,
|
||||
hookName,
|
||||
payload,
|
||||
memoryLimitMb,
|
||||
fuelLimit,
|
||||
maxCapabilityCallEntries,
|
||||
wallClockMs,
|
||||
});
|
||||
if (result.droppedCapabilityCallCount > 0) {
|
||||
throw new PluginCapabilityUseAuditTruncatedError(pluginInvocationId, result.droppedCapabilityCallCount, result.capabilityCalls.length);
|
||||
}
|
||||
for (const info of result.capabilityCalls) {
|
||||
capabilityRecorder.record(info);
|
||||
}
|
||||
let auditSequence = 0;
|
||||
for (const entry of result.hostLogs) {
|
||||
if (entry.level === "warn") {
|
||||
args.logger.warn?.(entry.record, entry.message);
|
||||
} else {
|
||||
args.logger.info(entry.record, entry.message);
|
||||
}
|
||||
if (entry.message === "plugin.invoke.host.audit") {
|
||||
await enqueuePluginHostAuditEvent({
|
||||
prisma: args.prisma,
|
||||
logger: args.logger,
|
||||
tenantId,
|
||||
pluginId,
|
||||
version: manifest.version,
|
||||
hookName,
|
||||
pluginInvocationId,
|
||||
sequence: auditSequence,
|
||||
hostRecord: entry.record,
|
||||
});
|
||||
auditSequence += 1;
|
||||
}
|
||||
}
|
||||
return result.result;
|
||||
} finally {
|
||||
await capabilityRecorder.flush();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose the production PluginInvokeDeps from env + Prisma + SharedStore
|
||||
* + logger. Caller (`apps/worker/src/index.ts`) injects the returned
|
||||
* object into `extraHandlerDependencies.pluginInvokeDeps`.
|
||||
*/
|
||||
export function bootstrapPluginInvokeDeps(options: PluginInvokeBootstrapOptions): PluginInvokeDeps {
|
||||
const { env, prisma, store, logger } = options;
|
||||
assertPluginRuntimeProductionEnabled(env);
|
||||
const basePath = env.PLUGIN_BINARY_BASE_PATH?.trim() || DEFAULT_PLUGIN_BINARY_BASE_PATH;
|
||||
const perMinute = resolveIntEnv(env, "PLUGIN_RATE_LIMIT_PER_MINUTE", DEFAULT_RATE_LIMIT_PER_MINUTE);
|
||||
const windowSeconds = resolveIntEnv(env, "PLUGIN_RATE_LIMIT_WINDOW_SECONDS", DEFAULT_RATE_LIMIT_WINDOW_SECONDS);
|
||||
const autoDisableThreshold = resolveIntEnv(env, "PLUGIN_AUTO_DISABLE_THRESHOLD", DEFAULT_PLUGIN_AUTO_DISABLE_THRESHOLD);
|
||||
const autoDisableWindowMs = resolveIntEnv(env, "PLUGIN_AUTO_DISABLE_WINDOW_MS", DEFAULT_PLUGIN_AUTO_DISABLE_WINDOW_MS);
|
||||
const trustedPublicKeysHex = parseTrustedPublicKeys(env.PLUGIN_TRUSTED_PUBLIC_KEYS);
|
||||
const capabilityUseSink = makeCapabilityUseSink({ prisma, logger });
|
||||
const crashTracker = createCrashTracker({
|
||||
store: adaptCrashTrackerStore(store),
|
||||
disablePlugin: async ({ pluginId, tenantId, failureCount, reason, correlationId }) => {
|
||||
logger.warn?.({ pluginId, tenantId, failureCount, reason }, "plugin.crash.auto_disable_requested");
|
||||
const disableAndAudit = async (client: PluginInvokeBootstrapPrismaSurface) => {
|
||||
await client.plugin.update({
|
||||
where: { tenantId_pluginId: { tenantId, pluginId } },
|
||||
data: { enabled: false },
|
||||
});
|
||||
await enqueuePluginAutoDisableAudit({
|
||||
prisma: client,
|
||||
logger,
|
||||
tenantId,
|
||||
pluginId,
|
||||
failureCount,
|
||||
reason,
|
||||
correlationId,
|
||||
});
|
||||
};
|
||||
if (typeof prisma.$transaction === "function") {
|
||||
await prisma.$transaction(disableAndAudit);
|
||||
} else {
|
||||
await disableAndAudit(prisma);
|
||||
}
|
||||
},
|
||||
notify: async ({ pluginId, tenantId, failureCount, reason }) => {
|
||||
logger.error?.({ pluginId, tenantId, failureCount, reason }, "plugin.crash.threshold_exceeded");
|
||||
},
|
||||
options: {
|
||||
logger,
|
||||
threshold: autoDisableThreshold,
|
||||
windowMs: autoDisableWindowMs,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
loadPlugin: makePluginLoader(prisma, logger),
|
||||
loadCapabilityGrants: makeCapabilityGrantLoader(prisma),
|
||||
loadBinary: makeBinaryLoader({ basePath, logger }),
|
||||
verifySignature: ({ wasmBuffer, manifest, trustedPublicKeysHex: trusted }) =>
|
||||
verifyPluginSignature({
|
||||
wasmBuffer,
|
||||
expectedWasmSha256: manifest.wasmSha256,
|
||||
signatureHex: manifest.signature,
|
||||
publicKeyHex: manifest.signedBy,
|
||||
trustedPublicKeysHex: trusted,
|
||||
manifest,
|
||||
}),
|
||||
consumeRateLimit: makeRateLimiter({ store, perMinute, windowSeconds }),
|
||||
invokeInSandbox: makeSandboxInvoker({ env, prisma, logger, capabilityUseSink }),
|
||||
recordAudit: async ({ tenantId, pluginId, version, hookName, pluginInvocationId, outcome }) => {
|
||||
await prisma.pluginInvocation.create({
|
||||
data: {
|
||||
id: pluginInvocationId,
|
||||
tenantId,
|
||||
pluginId,
|
||||
version,
|
||||
hookName,
|
||||
outcome: outcome.ok ? "success" : "failure",
|
||||
failureReason: outcome.ok ? null : outcome.reason,
|
||||
message: outcome.ok ? null : outcome.message ?? null,
|
||||
durationMs: outcome.durationMs,
|
||||
fuelUsed: outcome.ok ? outcome.fuelUsed : null,
|
||||
},
|
||||
});
|
||||
if (outcome.ok) {
|
||||
logger.info(
|
||||
{ tenantId, pluginId, version, hookName, pluginInvocationId, fuelUsed: outcome.fuelUsed, durationMs: outcome.durationMs },
|
||||
"plugin.invoke.audit.success"
|
||||
);
|
||||
} else {
|
||||
logger.error?.(
|
||||
{ tenantId, pluginId, version, hookName, pluginInvocationId, reason: outcome.reason, message: outcome.message, durationMs: outcome.durationMs },
|
||||
"plugin.invoke.audit.failure"
|
||||
);
|
||||
}
|
||||
},
|
||||
trustedPublicKeysHex,
|
||||
recordOutcomeForCrashTracking: async ({ tenantId, pluginId, outcome }) => {
|
||||
if (outcome.ok) {
|
||||
await crashTracker.recordSuccess({ pluginId, tenantId });
|
||||
} else {
|
||||
await crashTracker.recordFailure({
|
||||
pluginId,
|
||||
tenantId,
|
||||
reason: outcome.reason,
|
||||
correlationId: outcome.pluginInvocationId,
|
||||
});
|
||||
}
|
||||
},
|
||||
logger,
|
||||
} satisfies PluginInvokeDeps;
|
||||
}
|
||||
|
||||
export type { PluginInvokeOutcome };
|
||||
Reference in new issue
Block a user