375 lines
14 KiB
TypeScript
375 lines
14 KiB
TypeScript
/**
|
|
* Phase 06 #21 — Plugin-invoke worker job.
|
|
*
|
|
* Handles one job from the `toolctrl.plugin-invoke` queue:
|
|
* 1. Loads the Plugin row + .wasm binary for the requested
|
|
* (tenantId, pluginId, version) tuple.
|
|
* 2. Verifies the binary's SHA-256 matches the stored hash and the
|
|
* manifest's Ed25519 signature is valid against the trusted-keys
|
|
* allowlist.
|
|
* 3. Enforces a per-tenant sliding-window rate limit on plugin calls.
|
|
* 4. Builds a fresh WASM sandbox with memory + fuel limits, invokes
|
|
* the requested hook, and writes an audit event with the outcome.
|
|
*
|
|
* The handler is host-agnostic — every dependency (DB lookup, binary
|
|
* loader, signature verify, rate limiter, audit writer, logger) is
|
|
* passed in as a seam. The worker bootstrap wires real implementations;
|
|
* tests inject fakes to assert behaviour without spinning a database.
|
|
*
|
|
* Deferred (see docs/improvement-plan/06-plugin-system/wasm-sandbox.md):
|
|
* - hot-reload of plugin binaries (operator restarts pod)
|
|
* - rich structured payload marshalling (covered by #22's hook lifecycle)
|
|
* - Sentry / OTel span wiring (covered by #23's audit + docs)
|
|
*/
|
|
|
|
import { randomUUID } from "node:crypto";
|
|
|
|
import {
|
|
getMissingCapabilityGrantNames,
|
|
getPluginHook,
|
|
sanitizePluginOutput,
|
|
type PluginCapabilityGrant,
|
|
type PluginInvocationResult,
|
|
type PluginManifest,
|
|
type SanitizationStrictness,
|
|
type VerifyPluginSignatureResult,
|
|
validatePluginHookPayload,
|
|
} from "@toolctrl/plugin-runtime";
|
|
|
|
/**
|
|
* Resolve the sanitization strictness from env. Default is `moderate`
|
|
* (redact PANs/JWTs/IBANs/API-keys but keep emails — emails are often
|
|
* legitimate plugin output). Unknown values fall back to `moderate`.
|
|
*/
|
|
function resolveSanitizationStrictness(): SanitizationStrictness {
|
|
const raw = process.env.PLUGIN_OUTPUT_SANITIZATION_STRICTNESS?.trim().toLowerCase();
|
|
if (raw === "strict" || raw === "moderate" || raw === "off") {
|
|
return raw;
|
|
}
|
|
return "moderate";
|
|
}
|
|
|
|
function validateStoredPluginVerificationMetadata(row: PluginInvokeRow): string | null {
|
|
if (row.wasmSha256 !== row.manifest.wasmSha256) {
|
|
return "stored_wasm_sha256_mismatch";
|
|
}
|
|
if (row.signaturePublicKey.toLowerCase() !== row.manifest.signedBy.toLowerCase()) {
|
|
return "stored_signature_public_key_mismatch";
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export type PluginInvokeJobPayload = {
|
|
tenantId: string;
|
|
pluginId: string;
|
|
version: string;
|
|
hookName: string;
|
|
input?: unknown;
|
|
metadata?: Record<string, unknown>;
|
|
};
|
|
|
|
export type PluginInvokeRow = {
|
|
tenantId: string;
|
|
pluginId: string;
|
|
version: string;
|
|
manifest: PluginManifest;
|
|
wasmSha256: string;
|
|
signaturePublicKey: string;
|
|
enabled: boolean;
|
|
};
|
|
|
|
export type PluginInvokeOutcome =
|
|
| { ok: true; output: unknown; fuelUsed: number; durationMs: number; pluginInvocationId: string }
|
|
| { ok: false; reason: PluginInvokeFailure; message?: string; durationMs: number; pluginInvocationId: string };
|
|
|
|
export type PluginInvokeFailure =
|
|
| "plugin_not_found"
|
|
| "plugin_disabled"
|
|
| "hook_not_declared"
|
|
| "capability_grant_missing"
|
|
| "binary_load_failed"
|
|
| "invalid_hook_payload"
|
|
| "signature_invalid"
|
|
| "rate_limit_exceeded"
|
|
| "sandbox_failed";
|
|
|
|
export type RateLimitDecision = { allowed: boolean; remaining: number };
|
|
|
|
export type PluginInvokeDeps = {
|
|
/** Resolve the Plugin row for a tenant+pluginId+version tuple. */
|
|
loadPlugin: (args: { tenantId: string; pluginId: string; version: string }) => Promise<PluginInvokeRow | null>;
|
|
/** Resolve operator-approved capability grants for this tenant+plugin pair. */
|
|
loadCapabilityGrants: (args: { tenantId: string; pluginId: string }) => Promise<ReadonlyArray<PluginCapabilityGrant>>;
|
|
/** Load the raw .wasm bytes from disk / object storage. */
|
|
loadBinary: (args: { pluginId: string; version: string }) => Promise<Uint8Array | null>;
|
|
/** Verify the binary signature against the trusted-keys allowlist. */
|
|
verifySignature: (args: { wasmBuffer: Uint8Array; manifest: PluginManifest; trustedPublicKeysHex: ReadonlyArray<string> }) => VerifyPluginSignatureResult;
|
|
/** Per-tenant sliding-window rate limit (returns allowed flag). */
|
|
consumeRateLimit: (args: { tenantId: string; pluginId: string }) => Promise<RateLimitDecision>;
|
|
/** Invoke the plugin inside the worker-owned sandbox boundary. */
|
|
invokeInSandbox: (args: {
|
|
pluginInvocationId: string;
|
|
tenantId: string;
|
|
pluginId: string;
|
|
wasmBuffer: Uint8Array;
|
|
manifest: PluginManifest;
|
|
hookName: string;
|
|
payload: unknown;
|
|
capabilityGrants: ReadonlyArray<PluginCapabilityGrant>;
|
|
}) => Promise<PluginInvocationResult>;
|
|
/** Write an audit row for this invocation (success or failure). */
|
|
recordAudit: (args: {
|
|
tenantId: string;
|
|
pluginId: string;
|
|
version: string;
|
|
hookName: string;
|
|
pluginInvocationId: string;
|
|
outcome: PluginInvokeOutcome;
|
|
}) => Promise<void>;
|
|
/** Trusted Ed25519 public keys (hex). Loaded once from env at boot. */
|
|
trustedPublicKeysHex: ReadonlyArray<string>;
|
|
/** Pino-compatible structured logger seam. */
|
|
logger?: {
|
|
info: (fields: Record<string, unknown>, msg: string) => void;
|
|
warn?: (fields: Record<string, unknown>, msg: string) => void;
|
|
error?: (fields: Record<string, unknown>, msg: string) => void;
|
|
};
|
|
/**
|
|
* Phase 06 #22 — crash-tracker seam. Receives every outcome so the
|
|
* sliding-window counter can auto-disable a misbehaving plugin. The
|
|
* worker bootstrap wires this to a SharedStore-backed tracker; tests
|
|
* leave it `undefined` and the handler short-circuits.
|
|
*/
|
|
recordOutcomeForCrashTracking?: (args: { tenantId: string; pluginId: string; outcome: PluginInvokeOutcome }) => Promise<void>;
|
|
/** Injected clock; defaults to Date.now. */
|
|
now?: () => number;
|
|
};
|
|
|
|
/**
|
|
* Process exactly one plugin-invoke job. Returns the outcome (which
|
|
* is also persisted via `recordAudit`). Runtime failures are encoded as
|
|
* the discriminated `ok=false` shape. Durable audit failures deliberately
|
|
* throw so BullMQ can retry/dead-letter instead of acknowledging an
|
|
* invocation with no forensic trail.
|
|
*/
|
|
export async function handlePluginInvokeJob(payload: PluginInvokeJobPayload, deps: PluginInvokeDeps): Promise<PluginInvokeOutcome> {
|
|
const now = deps.now ?? Date.now;
|
|
const startedAt = now();
|
|
const pluginInvocationId = randomUUID();
|
|
|
|
const finalise = async (outcome: PluginInvokeOutcome): Promise<PluginInvokeOutcome> => {
|
|
await recordInvocationAudit(deps, payload, outcome);
|
|
// Phase 06 #22 — feed outcome to crash tracker. Some failure modes
|
|
// (plugin_not_found, plugin_disabled) are configuration issues, not
|
|
// plugin misbehaviour, so we deliberately skip them in the counter.
|
|
if (deps.recordOutcomeForCrashTracking) {
|
|
const trackable = outcome.ok || outcome.reason === "sandbox_failed" || outcome.reason === "signature_invalid" || outcome.reason === "binary_load_failed";
|
|
if (trackable) {
|
|
try {
|
|
await deps.recordOutcomeForCrashTracking({
|
|
tenantId: payload.tenantId,
|
|
pluginId: payload.pluginId,
|
|
outcome,
|
|
});
|
|
} catch (err) {
|
|
deps.logger?.warn?.({ ...payload, error: err instanceof Error ? err.message : String(err) }, "plugin.invoke.crash_tracker_failed");
|
|
}
|
|
}
|
|
}
|
|
return outcome;
|
|
};
|
|
|
|
const row = await deps.loadPlugin({
|
|
tenantId: payload.tenantId,
|
|
pluginId: payload.pluginId,
|
|
version: payload.version,
|
|
});
|
|
if (!row) {
|
|
deps.logger?.warn?.({ ...payload }, "plugin.invoke.not_found");
|
|
return finalise({ ok: false, reason: "plugin_not_found", durationMs: now() - startedAt, pluginInvocationId });
|
|
}
|
|
if (!row.enabled) {
|
|
deps.logger?.warn?.({ ...payload }, "plugin.invoke.disabled");
|
|
return finalise({ ok: false, reason: "plugin_disabled", durationMs: now() - startedAt, pluginInvocationId });
|
|
}
|
|
const declaredHook = getDeclaredHookForPayload(row.manifest, payload);
|
|
if (!declaredHook) {
|
|
deps.logger?.warn?.({ ...payload, declaredHooks: getDeclaredHookExports(row.manifest) }, "plugin.invoke.hook_not_declared");
|
|
return finalise({
|
|
ok: false,
|
|
reason: "hook_not_declared",
|
|
message: `Hook export "${payload.hookName}" is not declared by plugin manifest.`,
|
|
durationMs: now() - startedAt,
|
|
pluginInvocationId,
|
|
});
|
|
}
|
|
|
|
const capabilityGrants =
|
|
row.manifest.capabilities.length > 0 || row.manifest.hooks.length > 0
|
|
? await deps.loadCapabilityGrants({ tenantId: payload.tenantId, pluginId: payload.pluginId })
|
|
: [];
|
|
const missingCapabilityGrants = [
|
|
...getMissingCapabilityGrantNames({
|
|
manifest: row.manifest,
|
|
grants: capabilityGrants,
|
|
}),
|
|
...getMissingHookGrantNames({ declaredHook, grants: capabilityGrants }),
|
|
];
|
|
if (missingCapabilityGrants.length > 0) {
|
|
deps.logger?.warn?.({ ...payload, capabilities: missingCapabilityGrants }, "plugin.invoke.capability_grant_missing");
|
|
return finalise({
|
|
ok: false,
|
|
reason: "capability_grant_missing",
|
|
message: `Missing approved capability grants: ${missingCapabilityGrants.join(", ")}`,
|
|
durationMs: now() - startedAt,
|
|
pluginInvocationId,
|
|
});
|
|
}
|
|
|
|
let sandboxPayload = payload.input;
|
|
const registeredHook = getPluginHook(declaredHook.name);
|
|
if (registeredHook) {
|
|
const validation = validatePluginHookPayload(registeredHook, payload.input);
|
|
if (!validation.ok) {
|
|
deps.logger?.warn?.({ ...payload, hookName: declaredHook.name, issues: validation.issues }, "plugin.invoke.invalid_hook_payload");
|
|
return finalise({
|
|
ok: false,
|
|
reason: "invalid_hook_payload",
|
|
message: `Invalid payload for plugin hook "${declaredHook.name}".`,
|
|
durationMs: now() - startedAt,
|
|
pluginInvocationId,
|
|
});
|
|
}
|
|
sandboxPayload = validation.payload;
|
|
}
|
|
|
|
const wasmBuffer = await deps.loadBinary({ pluginId: payload.pluginId, version: payload.version });
|
|
if (!wasmBuffer) {
|
|
deps.logger?.error?.({ ...payload }, "plugin.invoke.binary_missing");
|
|
return finalise({ ok: false, reason: "binary_load_failed", durationMs: now() - startedAt, pluginInvocationId });
|
|
}
|
|
|
|
const storedMetadataError = validateStoredPluginVerificationMetadata(row);
|
|
if (storedMetadataError) {
|
|
deps.logger?.error?.({ ...payload, reason: storedMetadataError }, "plugin.invoke.stored_metadata_invalid");
|
|
return finalise({
|
|
ok: false,
|
|
reason: "signature_invalid",
|
|
message: storedMetadataError,
|
|
durationMs: now() - startedAt,
|
|
pluginInvocationId,
|
|
});
|
|
}
|
|
|
|
const verify = deps.verifySignature({
|
|
wasmBuffer,
|
|
manifest: row.manifest,
|
|
trustedPublicKeysHex: deps.trustedPublicKeysHex,
|
|
});
|
|
if (!verify.ok) {
|
|
deps.logger?.error?.({ ...payload, reason: verify.reason }, "plugin.invoke.signature_invalid");
|
|
return finalise({
|
|
ok: false,
|
|
reason: "signature_invalid",
|
|
message: verify.reason,
|
|
durationMs: now() - startedAt,
|
|
pluginInvocationId,
|
|
});
|
|
}
|
|
|
|
const rate = await deps.consumeRateLimit({ tenantId: payload.tenantId, pluginId: payload.pluginId });
|
|
if (!rate.allowed) {
|
|
deps.logger?.warn?.({ ...payload, remaining: rate.remaining }, "plugin.invoke.rate_limited");
|
|
return finalise({ ok: false, reason: "rate_limit_exceeded", durationMs: now() - startedAt, pluginInvocationId });
|
|
}
|
|
|
|
let result: PluginInvocationResult;
|
|
try {
|
|
result = await deps.invokeInSandbox({
|
|
pluginInvocationId,
|
|
tenantId: payload.tenantId,
|
|
pluginId: payload.pluginId,
|
|
wasmBuffer,
|
|
manifest: row.manifest,
|
|
hookName: payload.hookName,
|
|
payload: sandboxPayload,
|
|
capabilityGrants,
|
|
});
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
deps.logger?.error?.({ ...payload, error: message }, "plugin.invoke.sandbox_threw");
|
|
return finalise({
|
|
ok: false,
|
|
reason: "sandbox_failed",
|
|
message,
|
|
durationMs: now() - startedAt,
|
|
pluginInvocationId,
|
|
});
|
|
}
|
|
|
|
const durationMs = now() - startedAt;
|
|
if (!result.ok) {
|
|
deps.logger?.warn?.({ ...payload, reason: result.reason, message: result.message }, "plugin.invoke.sandbox_failed");
|
|
return finalise({
|
|
ok: false,
|
|
reason: "sandbox_failed",
|
|
message: result.message ?? result.reason,
|
|
durationMs,
|
|
pluginInvocationId,
|
|
});
|
|
}
|
|
|
|
deps.logger?.info({ ...payload, fuelUsed: result.fuelUsed, durationMs }, "plugin.invoke.completed");
|
|
// Phase 06 #23 — sanitize plugin output before it lands in the audit
|
|
// trail or worker logs. Strictness is env-driven (default moderate).
|
|
const sanitizedOutput = sanitizePluginOutput(result.output, resolveSanitizationStrictness());
|
|
return finalise({ ok: true, output: sanitizedOutput, fuelUsed: result.fuelUsed, durationMs, pluginInvocationId });
|
|
}
|
|
|
|
function getDeclaredHookForPayload(manifest: PluginManifest, payload: PluginInvokeJobPayload): PluginManifest["hooks"][number] | null {
|
|
if (!Array.isArray(manifest.hooks)) {
|
|
return null;
|
|
}
|
|
const candidates = manifest.hooks.filter((hook) => hook.export === payload.hookName);
|
|
if (candidates.length === 0) {
|
|
return null;
|
|
}
|
|
const logicalHookName = typeof payload.metadata?.logicalHookName === "string" ? payload.metadata.logicalHookName : null;
|
|
return candidates.find((hook) => hook.name === logicalHookName) ?? candidates[0] ?? null;
|
|
}
|
|
|
|
function getDeclaredHookExports(manifest: PluginManifest): string[] {
|
|
if (!Array.isArray(manifest.hooks)) {
|
|
return [];
|
|
}
|
|
return manifest.hooks.map((hook) => hook.export);
|
|
}
|
|
|
|
function getMissingHookGrantNames({
|
|
declaredHook,
|
|
grants,
|
|
}: {
|
|
declaredHook: PluginManifest["hooks"][number];
|
|
grants: ReadonlyArray<PluginCapabilityGrant>;
|
|
}): string[] {
|
|
const hookGrantName = `hook:${declaredHook.name}`;
|
|
return grants.some((grant) => grant.capabilityName === hookGrantName) ? [] : [hookGrantName];
|
|
}
|
|
|
|
async function recordInvocationAudit(deps: PluginInvokeDeps, payload: PluginInvokeJobPayload, outcome: PluginInvokeOutcome): Promise<void> {
|
|
try {
|
|
await deps.recordAudit({
|
|
tenantId: payload.tenantId,
|
|
pluginId: payload.pluginId,
|
|
version: payload.version,
|
|
hookName: payload.hookName,
|
|
pluginInvocationId: outcome.pluginInvocationId,
|
|
outcome,
|
|
});
|
|
} catch (err) {
|
|
const error = err instanceof Error ? err.message : String(err);
|
|
deps.logger?.error?.({ ...payload, pluginInvocationId: outcome.pluginInvocationId, error }, "plugin.invoke.audit_failed");
|
|
throw err;
|
|
}
|
|
}
|