/** * Phase 06 #21 — Production wiring for the plugin-invoke worker handler. * * `handlePluginInvokeJob` lives in `./plugin-invoke.ts` and is fully * dependency-injected. This module composes the real production seams: * * - `loadPlugin` → Prisma `Plugin.findUnique` * - `loadBinary` → filesystem read from `PLUGIN_BINARY_BASE_PATH` * - `verifySignature` → `verifyPluginSignature` from plugin-runtime * - `consumeRateLimit` → SharedStore sliding-window counter * - `invokeInSandbox` → worker-thread sandbox + capability gateway * - `recordAudit` → Prisma `PluginInvocation.create` + structured logger * - `trustedPublicKeysHex` → parsed once from `PLUGIN_TRUSTED_PUBLIC_KEYS` * - `recordOutcomeForCrashTracking` → `createCrashTracker` adapter * * Closes F-015 (Critical) — before this bootstrap, `handlePluginInvokeJob` * was not wired into production; the `plugin-invoke` queue accepted jobs * but the handler at `handlers.ts:752` was a silent no-op stub. * * Out of scope for this bootstrap (tracked in plugin-runtime deviations): * - host-fn implementations for higher-risk capabilities. The safe * `log` and `audit` imports are wired; network, storage, secret, * tenant.info and DB bindings remain fail-closed until reviewed. * - hot-reload of binaries / manifests */ import { promises as fs } from "node:fs"; import * as path from "node:path"; import { PluginCapabilityUseAuditTruncatedError, createCrashTracker, createPluginCapabilityUseRecorder, assertPluginRuntimeProductionEnabled, parsePluginManifest, parseTrustedPublicKeys, verifyPluginSignature, type CrashTrackerStore, type PluginCapabilityGrant, type PluginCapabilityUseBatch, type PluginInvocationResult, } from "@toolctrl/plugin-runtime"; import { AUDIT_EVENT_OUTBOX_TYPE, buildAuditEventCreateData } from "@toolctrl/db/audit-events.server"; import type { SharedStore } from "@toolctrl/shared-store"; import type { PluginInvokeDeps, PluginInvokeOutcome, PluginInvokeRow, RateLimitDecision, } from "./plugin-invoke.js"; import { PLUGIN_CAPABILITY_USE_EVENT_TYPE } from "./plugin-capability-use.js"; import { invokePluginSandboxInWorker } from "./plugin-sandbox-isolated.js"; export type PluginInvokeBootstrapLogger = { info: (fields: Record, msg: string) => void; warn?: (fields: Record, msg?: string) => void; error?: (fields: Record, msg?: string) => void; }; export type PluginInvokeBootstrapPrismaSurface = { plugin: { findUnique: (args: { where: { tenantId_pluginId: { tenantId: string; pluginId: string } }; }) => Promise<{ tenantId: string; pluginId: string; version: string; manifest: unknown; wasmSha256: string; signaturePublicKey: string; enabled: boolean; } | null>; update: (args: { where: { tenantId_pluginId: { tenantId: string; pluginId: string } }; data: { enabled: boolean }; }) => Promise; }; $transaction?: (fn: (tx: PluginInvokeBootstrapPrismaSurface) => Promise) => Promise; outboxEvent: { create: (args: { data: { tenantId: string | null; eventType: string; payload: unknown; dispatchKey: string; priority?: number; }; }) => Promise; findUnique: (args: { where: { dispatchKey: string }; }) => Promise; }; pluginInvocation: { create: (args: { data: { id: string; tenantId: string; pluginId: string; version: string; hookName: string; outcome: string; failureReason?: string | null; message?: string | null; durationMs: number; fuelUsed?: number | null; }; }) => Promise; }; tenantPluginCapabilityGrant: { findMany: (args: { where: { tenantId: string; pluginId: string }; select: { capabilityName: true; scopeJson: true }; }) => Promise>; }; }; export type PluginInvokeBootstrapOptions = { env: NodeJS.ProcessEnv; prisma: PluginInvokeBootstrapPrismaSurface; store: SharedStore; logger: PluginInvokeBootstrapLogger; }; /** Default base path for plugin binaries on disk. */ const DEFAULT_PLUGIN_BINARY_BASE_PATH = "/app/var/plugins"; const DEFAULT_RATE_LIMIT_PER_MINUTE = 100; const DEFAULT_RATE_LIMIT_WINDOW_SECONDS = 60; const DEFAULT_PLUGIN_MEMORY_LIMIT_MB = 64; const DEFAULT_PLUGIN_FUEL_LIMIT = 1_000_000; const DEFAULT_PLUGIN_WALL_CLOCK_MS = 5_000; const DEFAULT_PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES = 2_048; const DEFAULT_PLUGIN_AUTO_DISABLE_THRESHOLD = 10; const DEFAULT_PLUGIN_AUTO_DISABLE_WINDOW_MS = 600_000; function resolveIntEnv(env: NodeJS.ProcessEnv, key: string, fallback: number): number { const raw = env[key]; if (typeof raw !== "string" || raw.length === 0) return fallback; const parsed = Number.parseInt(raw, 10); if (!Number.isFinite(parsed) || parsed <= 0) return fallback; return parsed; } /** * Thin pass-through from SharedStore (`incr`, `expire`, `get`, `set`, * `del`) to the CrashTracker's expected store contract. The shapes are * already aligned; the wrapper exists so callers can swap in a fake * store in tests without dragging SharedStore in. */ function adaptCrashTrackerStore(store: SharedStore): CrashTrackerStore { return { incr: (key, by = 1) => store.incr(key, by), expire: (key, ttlSeconds) => store.expire(key, ttlSeconds), get: (key) => store.get(key), set: async (key, value, opts) => store.set(key, value, opts?.ttlSeconds ? { ttlSeconds: opts.ttlSeconds } : undefined), del: (key) => store.del(key), }; } /** * Sliding-window rate limiter keyed by `(pluginId, tenantId)`. Uses the * SharedStore counter primitive — atomic incr + ttl on first hit. */ function makeRateLimiter(args: { store: SharedStore; perMinute: number; windowSeconds: number; }): PluginInvokeDeps["consumeRateLimit"] { return async ({ pluginId, tenantId }): Promise => { const key = `plugin-ratelimit-${pluginId}-${tenantId}`; const count = await args.store.incr(key, 1); if (count === 1) { await args.store.expire(key, args.windowSeconds); } const allowed = count <= args.perMinute; return { allowed, remaining: Math.max(0, args.perMinute - count) }; }; } /** * Filesystem loader. Reads `///plugin.wasm`. * Returns null on ENOENT so the handler can route to `binary_load_failed` * cleanly; any other error propagates (BullMQ retries). */ function makeBinaryLoader(args: { basePath: string; logger: PluginInvokeBootstrapLogger; }): PluginInvokeDeps["loadBinary"] { const basePath = path.resolve(args.basePath); return async ({ pluginId, version }) => { const filePath = path.resolve(basePath, pluginId, version, "plugin.wasm"); const relativePath = path.relative(basePath, filePath); if (relativePath.startsWith("..") || path.isAbsolute(relativePath)) { args.logger.warn?.({ pluginId, version }, "plugin.invoke.binary_path_escape_blocked"); return null; } try { const buf = await fs.readFile(filePath); return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT") { args.logger.warn?.({ pluginId, version, filePath }, "plugin.invoke.binary_missing_on_disk"); return null; } throw err; } }; } /** * Prisma adapter. Reads the Plugin row + JSON manifest. Returns null on * miss; the handler then short-circuits to `plugin_not_found`. */ function makePluginLoader( prisma: PluginInvokeBootstrapPrismaSurface, logger: PluginInvokeBootstrapLogger ): PluginInvokeDeps["loadPlugin"] { return async ({ tenantId, pluginId, version }) => { const row = await prisma.plugin.findUnique({ where: { tenantId_pluginId: { tenantId, pluginId } }, }); if (!row) return null; if (row.version !== version) return null; const parsed = parsePluginManifest(row.manifest); if (!parsed.ok) { logger.warn?.({ tenantId, pluginId, version, issues: parsed.issues }, "plugin.invoke.stored_manifest_invalid"); return null; } return { tenantId: row.tenantId, pluginId: row.pluginId, version: row.version, manifest: parsed.manifest, wasmSha256: row.wasmSha256, signaturePublicKey: row.signaturePublicKey, enabled: row.enabled, } satisfies PluginInvokeRow; }; } function makeCapabilityGrantLoader( prisma: PluginInvokeBootstrapPrismaSurface ): PluginInvokeDeps["loadCapabilityGrants"] { return async ({ tenantId, pluginId }): Promise => { const rows = await prisma.tenantPluginCapabilityGrant.findMany({ where: { tenantId, pluginId }, select: { capabilityName: true, scopeJson: true }, }); return rows.map((row) => ({ capabilityName: row.capabilityName, scopeJson: row.scopeJson, })); }; } function makeCapabilityUseSink(args: { prisma: PluginInvokeBootstrapPrismaSurface; logger: PluginInvokeBootstrapLogger; }) { return { emit: async (batch: PluginCapabilityUseBatch): Promise => { const dispatchKey = `plugin-capability-use:${batch.pluginInvocationId}`; try { await args.prisma.outboxEvent.create({ data: { tenantId: batch.tenantId, eventType: PLUGIN_CAPABILITY_USE_EVENT_TYPE, payload: batch, dispatchKey, priority: 10, }, }); } catch (error) { if (!isUniqueConstraintViolation(error)) { throw error; } const existing = await args.prisma.outboxEvent.findUnique({ where: { dispatchKey }, }); if (!existing) { throw error; } } args.logger.info( { tenantId: batch.tenantId, pluginId: batch.pluginId, pluginInvocationId: batch.pluginInvocationId, entryCount: batch.entries.length, }, "plugin.invoke.capability_use_enqueued" ); }, }; } type JsonPrimitive = string | number | boolean | null; type JsonSafeValue = JsonPrimitive | JsonSafeValue[] | { [key: string]: JsonSafeValue }; const MAX_AUDIT_METADATA_DEPTH = 8; function toJsonSafeValue(value: unknown, depth = 0): JsonSafeValue { if (depth > MAX_AUDIT_METADATA_DEPTH) { return "[TRUNCATED]"; } if (value === null || typeof value === "string" || typeof value === "boolean") { return value; } if (typeof value === "number") { return Number.isFinite(value) ? value : String(value); } if (typeof value === "bigint") { return value.toString(); } if (value instanceof Date) { return value.toISOString(); } if (Array.isArray(value)) { return value.map((entry) => toJsonSafeValue(entry, depth + 1)); } if (value && typeof value === "object") { return Object.fromEntries( Object.entries(value as Record) .filter(([, entry]) => typeof entry !== "undefined" && typeof entry !== "function" && typeof entry !== "symbol") .map(([key, entry]) => [key, toJsonSafeValue(entry, depth + 1)]) ); } return String(value); } async function enqueuePluginHostAuditEvent(args: { prisma: PluginInvokeBootstrapPrismaSurface; logger: PluginInvokeBootstrapLogger; tenantId: string; pluginId: string; version: string; hookName: string; pluginInvocationId: string; sequence: number; hostRecord: Record; }): Promise { const payload = buildAuditEventCreateData({ tenantId: args.tenantId, actorId: args.pluginId, actorType: "plugin", action: "plugin.audit.recorded", targetType: "plugin", targetId: args.pluginId, requestId: args.pluginInvocationId, sourceModule: "worker.plugin-runtime", outcome: "success", dataSensitivity: "internal", regulatoryScope: ["soc2"], metadata: { pluginId: args.pluginId, pluginManifestId: typeof args.hostRecord.pluginId === "string" ? args.hostRecord.pluginId : args.pluginId, pluginInvocationId: args.pluginInvocationId, version: args.version, hookName: args.hookName, hostBinding: "audit", sequence: args.sequence, value: toJsonSafeValue(args.hostRecord.value), }, }); const dispatchKey = `plugin-host-audit:${args.pluginInvocationId}:${args.sequence}`; try { await args.prisma.outboxEvent.create({ data: { tenantId: args.tenantId, eventType: AUDIT_EVENT_OUTBOX_TYPE, payload, dispatchKey, priority: 20, }, }); } catch (error) { if (!isUniqueConstraintViolation(error)) { throw error; } const existing = await args.prisma.outboxEvent.findUnique({ where: { dispatchKey }, }); if (!existing) { throw error; } } args.logger.info( { tenantId: args.tenantId, pluginId: args.pluginId, version: args.version, hookName: args.hookName, pluginInvocationId: args.pluginInvocationId, sequence: args.sequence, }, "plugin.invoke.host.audit_enqueued" ); } async function enqueuePluginAutoDisableAudit(args: { prisma: PluginInvokeBootstrapPrismaSurface; logger: PluginInvokeBootstrapLogger; tenantId: string; pluginId: string; failureCount: number; reason: string; correlationId: string; }): Promise { const payload = buildAuditEventCreateData({ tenantId: args.tenantId, actorType: "system", action: "plugin.auto_disabled", targetType: "plugin", targetId: args.pluginId, sourceModule: "worker.plugin-runtime", outcome: "blocked", dataSensitivity: "internal", regulatoryScope: ["soc2"], metadata: { pluginId: args.pluginId, failureCount: args.failureCount, reason: args.reason, pluginInvocationId: args.correlationId, trigger: "crash-threshold", }, }); const dispatchKey = `plugin-auto-disable-audit:${args.tenantId}:${args.pluginId}:${args.correlationId}`; try { await args.prisma.outboxEvent.create({ data: { tenantId: args.tenantId, eventType: AUDIT_EVENT_OUTBOX_TYPE, payload, dispatchKey, priority: 20, }, }); } catch (error) { if (!isUniqueConstraintViolation(error)) { throw error; } const existing = await args.prisma.outboxEvent.findUnique({ where: { dispatchKey }, }); if (!existing) { throw error; } } args.logger.info( { tenantId: args.tenantId, pluginId: args.pluginId, failureCount: args.failureCount, reason: args.reason, }, "plugin.crash.auto_disable_audit_enqueued" ); } function isUniqueConstraintViolation(error: unknown): boolean { return typeof error === "object" && error !== null && (error as { code?: unknown }).code === "P2002"; } /** * Sandbox adapter. Runs V8 WebAssembly in a killable worker thread and replays * safe sync `log` and `audit` host imports back into the parent process for * logging + capability-use auditing. Higher-risk host bindings remain absent * and fail closed with `no_host_implementation`. */ function makeSandboxInvoker(args: { env: NodeJS.ProcessEnv; prisma: PluginInvokeBootstrapPrismaSurface; logger: PluginInvokeBootstrapLogger; capabilityUseSink: ReturnType; }): PluginInvokeDeps["invokeInSandbox"] { const memoryLimitMb = resolveIntEnv(args.env, "PLUGIN_MAX_MEMORY_MB", DEFAULT_PLUGIN_MEMORY_LIMIT_MB); const fuelLimit = resolveIntEnv(args.env, "PLUGIN_MAX_FUEL", DEFAULT_PLUGIN_FUEL_LIMIT); const wallClockMs = resolveIntEnv(args.env, "PLUGIN_MAX_WALL_CLOCK_MS", DEFAULT_PLUGIN_WALL_CLOCK_MS); const maxCapabilityCallEntries = resolveIntEnv( args.env, "PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES", DEFAULT_PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES ); return async ({ pluginInvocationId, tenantId, pluginId, wasmBuffer, manifest, capabilityGrants, hookName, payload }): Promise => { const capabilityRecorder = createPluginCapabilityUseRecorder({ pluginInvocationId, tenantId, pluginId, sink: args.capabilityUseSink, logger: args.logger, failClosed: true, }); try { const result = await invokePluginSandboxInWorker({ wasmBuffer, manifest, capabilityGrants, hookName, payload, memoryLimitMb, fuelLimit, maxCapabilityCallEntries, wallClockMs, }); if (result.droppedCapabilityCallCount > 0) { throw new PluginCapabilityUseAuditTruncatedError(pluginInvocationId, result.droppedCapabilityCallCount, result.capabilityCalls.length); } for (const info of result.capabilityCalls) { capabilityRecorder.record(info); } let auditSequence = 0; for (const entry of result.hostLogs) { if (entry.level === "warn") { args.logger.warn?.(entry.record, entry.message); } else { args.logger.info(entry.record, entry.message); } if (entry.message === "plugin.invoke.host.audit") { await enqueuePluginHostAuditEvent({ prisma: args.prisma, logger: args.logger, tenantId, pluginId, version: manifest.version, hookName, pluginInvocationId, sequence: auditSequence, hostRecord: entry.record, }); auditSequence += 1; } } return result.result; } finally { await capabilityRecorder.flush(); } }; } /** * Compose the production PluginInvokeDeps from env + Prisma + SharedStore * + logger. Caller (`apps/worker/src/index.ts`) injects the returned * object into `extraHandlerDependencies.pluginInvokeDeps`. */ export function bootstrapPluginInvokeDeps(options: PluginInvokeBootstrapOptions): PluginInvokeDeps { const { env, prisma, store, logger } = options; assertPluginRuntimeProductionEnabled(env); const basePath = env.PLUGIN_BINARY_BASE_PATH?.trim() || DEFAULT_PLUGIN_BINARY_BASE_PATH; const perMinute = resolveIntEnv(env, "PLUGIN_RATE_LIMIT_PER_MINUTE", DEFAULT_RATE_LIMIT_PER_MINUTE); const windowSeconds = resolveIntEnv(env, "PLUGIN_RATE_LIMIT_WINDOW_SECONDS", DEFAULT_RATE_LIMIT_WINDOW_SECONDS); const autoDisableThreshold = resolveIntEnv(env, "PLUGIN_AUTO_DISABLE_THRESHOLD", DEFAULT_PLUGIN_AUTO_DISABLE_THRESHOLD); const autoDisableWindowMs = resolveIntEnv(env, "PLUGIN_AUTO_DISABLE_WINDOW_MS", DEFAULT_PLUGIN_AUTO_DISABLE_WINDOW_MS); const trustedPublicKeysHex = parseTrustedPublicKeys(env.PLUGIN_TRUSTED_PUBLIC_KEYS); const capabilityUseSink = makeCapabilityUseSink({ prisma, logger }); const crashTracker = createCrashTracker({ store: adaptCrashTrackerStore(store), disablePlugin: async ({ pluginId, tenantId, failureCount, reason, correlationId }) => { logger.warn?.({ pluginId, tenantId, failureCount, reason }, "plugin.crash.auto_disable_requested"); const disableAndAudit = async (client: PluginInvokeBootstrapPrismaSurface) => { await client.plugin.update({ where: { tenantId_pluginId: { tenantId, pluginId } }, data: { enabled: false }, }); await enqueuePluginAutoDisableAudit({ prisma: client, logger, tenantId, pluginId, failureCount, reason, correlationId, }); }; if (typeof prisma.$transaction === "function") { await prisma.$transaction(disableAndAudit); } else { await disableAndAudit(prisma); } }, notify: async ({ pluginId, tenantId, failureCount, reason }) => { logger.error?.({ pluginId, tenantId, failureCount, reason }, "plugin.crash.threshold_exceeded"); }, options: { logger, threshold: autoDisableThreshold, windowMs: autoDisableWindowMs, }, }); return { loadPlugin: makePluginLoader(prisma, logger), loadCapabilityGrants: makeCapabilityGrantLoader(prisma), loadBinary: makeBinaryLoader({ basePath, logger }), verifySignature: ({ wasmBuffer, manifest, trustedPublicKeysHex: trusted }) => verifyPluginSignature({ wasmBuffer, expectedWasmSha256: manifest.wasmSha256, signatureHex: manifest.signature, publicKeyHex: manifest.signedBy, trustedPublicKeysHex: trusted, manifest, }), consumeRateLimit: makeRateLimiter({ store, perMinute, windowSeconds }), invokeInSandbox: makeSandboxInvoker({ env, prisma, logger, capabilityUseSink }), recordAudit: async ({ tenantId, pluginId, version, hookName, pluginInvocationId, outcome }) => { await prisma.pluginInvocation.create({ data: { id: pluginInvocationId, tenantId, pluginId, version, hookName, outcome: outcome.ok ? "success" : "failure", failureReason: outcome.ok ? null : outcome.reason, message: outcome.ok ? null : outcome.message ?? null, durationMs: outcome.durationMs, fuelUsed: outcome.ok ? outcome.fuelUsed : null, }, }); if (outcome.ok) { logger.info( { tenantId, pluginId, version, hookName, pluginInvocationId, fuelUsed: outcome.fuelUsed, durationMs: outcome.durationMs }, "plugin.invoke.audit.success" ); } else { logger.error?.( { tenantId, pluginId, version, hookName, pluginInvocationId, reason: outcome.reason, message: outcome.message, durationMs: outcome.durationMs }, "plugin.invoke.audit.failure" ); } }, trustedPublicKeysHex, recordOutcomeForCrashTracking: async ({ tenantId, pluginId, outcome }) => { if (outcome.ok) { await crashTracker.recordSuccess({ pluginId, tenantId }); } else { await crashTracker.recordFailure({ pluginId, tenantId, reason: outcome.reason, correlationId: outcome.pluginInvocationId, }); } }, logger, } satisfies PluginInvokeDeps; } export type { PluginInvokeOutcome };