/** * Phase 06 #21 — Plugin-invoke worker job. * * Handles one job from the `toolctrl.plugin-invoke` queue: * 1. Loads the Plugin row + .wasm binary for the requested * (tenantId, pluginId, version) tuple. * 2. Verifies the binary's SHA-256 matches the stored hash and the * manifest's Ed25519 signature is valid against the trusted-keys * allowlist. * 3. Enforces a per-tenant sliding-window rate limit on plugin calls. * 4. Builds a fresh WASM sandbox with memory + fuel limits, invokes * the requested hook, and writes an audit event with the outcome. * * The handler is host-agnostic — every dependency (DB lookup, binary * loader, signature verify, rate limiter, audit writer, logger) is * passed in as a seam. The worker bootstrap wires real implementations; * tests inject fakes to assert behaviour without spinning a database. * * Deferred (see docs/improvement-plan/06-plugin-system/wasm-sandbox.md): * - hot-reload of plugin binaries (operator restarts pod) * - rich structured payload marshalling (covered by #22's hook lifecycle) * - Sentry / OTel span wiring (covered by #23's audit + docs) */ import { randomUUID } from "node:crypto"; import { getMissingCapabilityGrantNames, getPluginHook, sanitizePluginOutput, type PluginCapabilityGrant, type PluginInvocationResult, type PluginManifest, type SanitizationStrictness, type VerifyPluginSignatureResult, validatePluginHookPayload, } from "@toolctrl/plugin-runtime"; /** * Resolve the sanitization strictness from env. Default is `moderate` * (redact PANs/JWTs/IBANs/API-keys but keep emails — emails are often * legitimate plugin output). Unknown values fall back to `moderate`. */ function resolveSanitizationStrictness(): SanitizationStrictness { const raw = process.env.PLUGIN_OUTPUT_SANITIZATION_STRICTNESS?.trim().toLowerCase(); if (raw === "strict" || raw === "moderate" || raw === "off") { return raw; } return "moderate"; } function validateStoredPluginVerificationMetadata(row: PluginInvokeRow): string | null { if (row.wasmSha256 !== row.manifest.wasmSha256) { return "stored_wasm_sha256_mismatch"; } if (row.signaturePublicKey.toLowerCase() !== row.manifest.signedBy.toLowerCase()) { return "stored_signature_public_key_mismatch"; } return null; } export type PluginInvokeJobPayload = { tenantId: string; pluginId: string; version: string; hookName: string; input?: unknown; metadata?: Record; }; export type PluginInvokeRow = { tenantId: string; pluginId: string; version: string; manifest: PluginManifest; wasmSha256: string; signaturePublicKey: string; enabled: boolean; }; export type PluginInvokeOutcome = | { ok: true; output: unknown; fuelUsed: number; durationMs: number; pluginInvocationId: string } | { ok: false; reason: PluginInvokeFailure; message?: string; durationMs: number; pluginInvocationId: string }; export type PluginInvokeFailure = | "plugin_not_found" | "plugin_disabled" | "hook_not_declared" | "capability_grant_missing" | "binary_load_failed" | "invalid_hook_payload" | "signature_invalid" | "rate_limit_exceeded" | "sandbox_failed"; export type RateLimitDecision = { allowed: boolean; remaining: number }; export type PluginInvokeDeps = { /** Resolve the Plugin row for a tenant+pluginId+version tuple. */ loadPlugin: (args: { tenantId: string; pluginId: string; version: string }) => Promise; /** Resolve operator-approved capability grants for this tenant+plugin pair. */ loadCapabilityGrants: (args: { tenantId: string; pluginId: string }) => Promise>; /** Load the raw .wasm bytes from disk / object storage. */ loadBinary: (args: { pluginId: string; version: string }) => Promise; /** Verify the binary signature against the trusted-keys allowlist. */ verifySignature: (args: { wasmBuffer: Uint8Array; manifest: PluginManifest; trustedPublicKeysHex: ReadonlyArray }) => VerifyPluginSignatureResult; /** Per-tenant sliding-window rate limit (returns allowed flag). */ consumeRateLimit: (args: { tenantId: string; pluginId: string }) => Promise; /** Invoke the plugin inside the worker-owned sandbox boundary. */ invokeInSandbox: (args: { pluginInvocationId: string; tenantId: string; pluginId: string; wasmBuffer: Uint8Array; manifest: PluginManifest; hookName: string; payload: unknown; capabilityGrants: ReadonlyArray; }) => Promise; /** Write an audit row for this invocation (success or failure). */ recordAudit: (args: { tenantId: string; pluginId: string; version: string; hookName: string; pluginInvocationId: string; outcome: PluginInvokeOutcome; }) => Promise; /** Trusted Ed25519 public keys (hex). Loaded once from env at boot. */ trustedPublicKeysHex: ReadonlyArray; /** Pino-compatible structured logger seam. */ logger?: { info: (fields: Record, msg: string) => void; warn?: (fields: Record, msg: string) => void; error?: (fields: Record, msg: string) => void; }; /** * Phase 06 #22 — crash-tracker seam. Receives every outcome so the * sliding-window counter can auto-disable a misbehaving plugin. The * worker bootstrap wires this to a SharedStore-backed tracker; tests * leave it `undefined` and the handler short-circuits. */ recordOutcomeForCrashTracking?: (args: { tenantId: string; pluginId: string; outcome: PluginInvokeOutcome }) => Promise; /** Injected clock; defaults to Date.now. */ now?: () => number; }; /** * Process exactly one plugin-invoke job. Returns the outcome (which * is also persisted via `recordAudit`). Runtime failures are encoded as * the discriminated `ok=false` shape. Durable audit failures deliberately * throw so BullMQ can retry/dead-letter instead of acknowledging an * invocation with no forensic trail. */ export async function handlePluginInvokeJob(payload: PluginInvokeJobPayload, deps: PluginInvokeDeps): Promise { const now = deps.now ?? Date.now; const startedAt = now(); const pluginInvocationId = randomUUID(); const finalise = async (outcome: PluginInvokeOutcome): Promise => { await recordInvocationAudit(deps, payload, outcome); // Phase 06 #22 — feed outcome to crash tracker. Some failure modes // (plugin_not_found, plugin_disabled) are configuration issues, not // plugin misbehaviour, so we deliberately skip them in the counter. if (deps.recordOutcomeForCrashTracking) { const trackable = outcome.ok || outcome.reason === "sandbox_failed" || outcome.reason === "signature_invalid" || outcome.reason === "binary_load_failed"; if (trackable) { try { await deps.recordOutcomeForCrashTracking({ tenantId: payload.tenantId, pluginId: payload.pluginId, outcome, }); } catch (err) { deps.logger?.warn?.({ ...payload, error: err instanceof Error ? err.message : String(err) }, "plugin.invoke.crash_tracker_failed"); } } } return outcome; }; const row = await deps.loadPlugin({ tenantId: payload.tenantId, pluginId: payload.pluginId, version: payload.version, }); if (!row) { deps.logger?.warn?.({ ...payload }, "plugin.invoke.not_found"); return finalise({ ok: false, reason: "plugin_not_found", durationMs: now() - startedAt, pluginInvocationId }); } if (!row.enabled) { deps.logger?.warn?.({ ...payload }, "plugin.invoke.disabled"); return finalise({ ok: false, reason: "plugin_disabled", durationMs: now() - startedAt, pluginInvocationId }); } const declaredHook = getDeclaredHookForPayload(row.manifest, payload); if (!declaredHook) { deps.logger?.warn?.({ ...payload, declaredHooks: getDeclaredHookExports(row.manifest) }, "plugin.invoke.hook_not_declared"); return finalise({ ok: false, reason: "hook_not_declared", message: `Hook export "${payload.hookName}" is not declared by plugin manifest.`, durationMs: now() - startedAt, pluginInvocationId, }); } const capabilityGrants = row.manifest.capabilities.length > 0 || row.manifest.hooks.length > 0 ? await deps.loadCapabilityGrants({ tenantId: payload.tenantId, pluginId: payload.pluginId }) : []; const missingCapabilityGrants = [ ...getMissingCapabilityGrantNames({ manifest: row.manifest, grants: capabilityGrants, }), ...getMissingHookGrantNames({ declaredHook, grants: capabilityGrants }), ]; if (missingCapabilityGrants.length > 0) { deps.logger?.warn?.({ ...payload, capabilities: missingCapabilityGrants }, "plugin.invoke.capability_grant_missing"); return finalise({ ok: false, reason: "capability_grant_missing", message: `Missing approved capability grants: ${missingCapabilityGrants.join(", ")}`, durationMs: now() - startedAt, pluginInvocationId, }); } let sandboxPayload = payload.input; const registeredHook = getPluginHook(declaredHook.name); if (registeredHook) { const validation = validatePluginHookPayload(registeredHook, payload.input); if (!validation.ok) { deps.logger?.warn?.({ ...payload, hookName: declaredHook.name, issues: validation.issues }, "plugin.invoke.invalid_hook_payload"); return finalise({ ok: false, reason: "invalid_hook_payload", message: `Invalid payload for plugin hook "${declaredHook.name}".`, durationMs: now() - startedAt, pluginInvocationId, }); } sandboxPayload = validation.payload; } const wasmBuffer = await deps.loadBinary({ pluginId: payload.pluginId, version: payload.version }); if (!wasmBuffer) { deps.logger?.error?.({ ...payload }, "plugin.invoke.binary_missing"); return finalise({ ok: false, reason: "binary_load_failed", durationMs: now() - startedAt, pluginInvocationId }); } const storedMetadataError = validateStoredPluginVerificationMetadata(row); if (storedMetadataError) { deps.logger?.error?.({ ...payload, reason: storedMetadataError }, "plugin.invoke.stored_metadata_invalid"); return finalise({ ok: false, reason: "signature_invalid", message: storedMetadataError, durationMs: now() - startedAt, pluginInvocationId, }); } const verify = deps.verifySignature({ wasmBuffer, manifest: row.manifest, trustedPublicKeysHex: deps.trustedPublicKeysHex, }); if (!verify.ok) { deps.logger?.error?.({ ...payload, reason: verify.reason }, "plugin.invoke.signature_invalid"); return finalise({ ok: false, reason: "signature_invalid", message: verify.reason, durationMs: now() - startedAt, pluginInvocationId, }); } const rate = await deps.consumeRateLimit({ tenantId: payload.tenantId, pluginId: payload.pluginId }); if (!rate.allowed) { deps.logger?.warn?.({ ...payload, remaining: rate.remaining }, "plugin.invoke.rate_limited"); return finalise({ ok: false, reason: "rate_limit_exceeded", durationMs: now() - startedAt, pluginInvocationId }); } let result: PluginInvocationResult; try { result = await deps.invokeInSandbox({ pluginInvocationId, tenantId: payload.tenantId, pluginId: payload.pluginId, wasmBuffer, manifest: row.manifest, hookName: payload.hookName, payload: sandboxPayload, capabilityGrants, }); } catch (err) { const message = err instanceof Error ? err.message : String(err); deps.logger?.error?.({ ...payload, error: message }, "plugin.invoke.sandbox_threw"); return finalise({ ok: false, reason: "sandbox_failed", message, durationMs: now() - startedAt, pluginInvocationId, }); } const durationMs = now() - startedAt; if (!result.ok) { deps.logger?.warn?.({ ...payload, reason: result.reason, message: result.message }, "plugin.invoke.sandbox_failed"); return finalise({ ok: false, reason: "sandbox_failed", message: result.message ?? result.reason, durationMs, pluginInvocationId, }); } deps.logger?.info({ ...payload, fuelUsed: result.fuelUsed, durationMs }, "plugin.invoke.completed"); // Phase 06 #23 — sanitize plugin output before it lands in the audit // trail or worker logs. Strictness is env-driven (default moderate). const sanitizedOutput = sanitizePluginOutput(result.output, resolveSanitizationStrictness()); return finalise({ ok: true, output: sanitizedOutput, fuelUsed: result.fuelUsed, durationMs, pluginInvocationId }); } function getDeclaredHookForPayload(manifest: PluginManifest, payload: PluginInvokeJobPayload): PluginManifest["hooks"][number] | null { if (!Array.isArray(manifest.hooks)) { return null; } const candidates = manifest.hooks.filter((hook) => hook.export === payload.hookName); if (candidates.length === 0) { return null; } const logicalHookName = typeof payload.metadata?.logicalHookName === "string" ? payload.metadata.logicalHookName : null; return candidates.find((hook) => hook.name === logicalHookName) ?? candidates[0] ?? null; } function getDeclaredHookExports(manifest: PluginManifest): string[] { if (!Array.isArray(manifest.hooks)) { return []; } return manifest.hooks.map((hook) => hook.export); } function getMissingHookGrantNames({ declaredHook, grants, }: { declaredHook: PluginManifest["hooks"][number]; grants: ReadonlyArray; }): string[] { const hookGrantName = `hook:${declaredHook.name}`; return grants.some((grant) => grant.capabilityName === hookGrantName) ? [] : [hookGrantName]; } async function recordInvocationAudit(deps: PluginInvokeDeps, payload: PluginInvokeJobPayload, outcome: PluginInvokeOutcome): Promise { try { await deps.recordAudit({ tenantId: payload.tenantId, pluginId: payload.pluginId, version: payload.version, hookName: payload.hookName, pluginInvocationId: outcome.pluginInvocationId, outcome, }); } catch (err) { const error = err instanceof Error ? err.message : String(err); deps.logger?.error?.({ ...payload, pluginInvocationId: outcome.pluginInvocationId, error }, "plugin.invoke.audit_failed"); throw err; } }