Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/worker/src/jobs/plugin-invoke.ts
T

375 lines
14 KiB
TypeScript

/**
* Phase 06 #21 — Plugin-invoke worker job.
*
* Handles one job from the `toolctrl.plugin-invoke` queue:
* 1. Loads the Plugin row + .wasm binary for the requested
* (tenantId, pluginId, version) tuple.
* 2. Verifies the binary's SHA-256 matches the stored hash and the
* manifest's Ed25519 signature is valid against the trusted-keys
* allowlist.
* 3. Enforces a per-tenant sliding-window rate limit on plugin calls.
* 4. Builds a fresh WASM sandbox with memory + fuel limits, invokes
* the requested hook, and writes an audit event with the outcome.
*
* The handler is host-agnostic — every dependency (DB lookup, binary
* loader, signature verify, rate limiter, audit writer, logger) is
* passed in as a seam. The worker bootstrap wires real implementations;
* tests inject fakes to assert behaviour without spinning a database.
*
* Deferred (see docs/improvement-plan/06-plugin-system/wasm-sandbox.md):
* - hot-reload of plugin binaries (operator restarts pod)
* - rich structured payload marshalling (covered by #22's hook lifecycle)
* - Sentry / OTel span wiring (covered by #23's audit + docs)
*/
import { randomUUID } from "node:crypto";
import {
getMissingCapabilityGrantNames,
getPluginHook,
sanitizePluginOutput,
type PluginCapabilityGrant,
type PluginInvocationResult,
type PluginManifest,
type SanitizationStrictness,
type VerifyPluginSignatureResult,
validatePluginHookPayload,
} from "@toolctrl/plugin-runtime";
/**
* Resolve the sanitization strictness from env. Default is `moderate`
* (redact PANs/JWTs/IBANs/API-keys but keep emails — emails are often
* legitimate plugin output). Unknown values fall back to `moderate`.
*/
function resolveSanitizationStrictness(): SanitizationStrictness {
const raw = process.env.PLUGIN_OUTPUT_SANITIZATION_STRICTNESS?.trim().toLowerCase();
if (raw === "strict" || raw === "moderate" || raw === "off") {
return raw;
}
return "moderate";
}
function validateStoredPluginVerificationMetadata(row: PluginInvokeRow): string | null {
if (row.wasmSha256 !== row.manifest.wasmSha256) {
return "stored_wasm_sha256_mismatch";
}
if (row.signaturePublicKey.toLowerCase() !== row.manifest.signedBy.toLowerCase()) {
return "stored_signature_public_key_mismatch";
}
return null;
}
export type PluginInvokeJobPayload = {
tenantId: string;
pluginId: string;
version: string;
hookName: string;
input?: unknown;
metadata?: Record<string, unknown>;
};
export type PluginInvokeRow = {
tenantId: string;
pluginId: string;
version: string;
manifest: PluginManifest;
wasmSha256: string;
signaturePublicKey: string;
enabled: boolean;
};
export type PluginInvokeOutcome =
| { ok: true; output: unknown; fuelUsed: number; durationMs: number; pluginInvocationId: string }
| { ok: false; reason: PluginInvokeFailure; message?: string; durationMs: number; pluginInvocationId: string };
export type PluginInvokeFailure =
| "plugin_not_found"
| "plugin_disabled"
| "hook_not_declared"
| "capability_grant_missing"
| "binary_load_failed"
| "invalid_hook_payload"
| "signature_invalid"
| "rate_limit_exceeded"
| "sandbox_failed";
export type RateLimitDecision = { allowed: boolean; remaining: number };
export type PluginInvokeDeps = {
/** Resolve the Plugin row for a tenant+pluginId+version tuple. */
loadPlugin: (args: { tenantId: string; pluginId: string; version: string }) => Promise<PluginInvokeRow | null>;
/** Resolve operator-approved capability grants for this tenant+plugin pair. */
loadCapabilityGrants: (args: { tenantId: string; pluginId: string }) => Promise<ReadonlyArray<PluginCapabilityGrant>>;
/** Load the raw .wasm bytes from disk / object storage. */
loadBinary: (args: { pluginId: string; version: string }) => Promise<Uint8Array | null>;
/** Verify the binary signature against the trusted-keys allowlist. */
verifySignature: (args: { wasmBuffer: Uint8Array; manifest: PluginManifest; trustedPublicKeysHex: ReadonlyArray<string> }) => VerifyPluginSignatureResult;
/** Per-tenant sliding-window rate limit (returns allowed flag). */
consumeRateLimit: (args: { tenantId: string; pluginId: string }) => Promise<RateLimitDecision>;
/** Invoke the plugin inside the worker-owned sandbox boundary. */
invokeInSandbox: (args: {
pluginInvocationId: string;
tenantId: string;
pluginId: string;
wasmBuffer: Uint8Array;
manifest: PluginManifest;
hookName: string;
payload: unknown;
capabilityGrants: ReadonlyArray<PluginCapabilityGrant>;
}) => Promise<PluginInvocationResult>;
/** Write an audit row for this invocation (success or failure). */
recordAudit: (args: {
tenantId: string;
pluginId: string;
version: string;
hookName: string;
pluginInvocationId: string;
outcome: PluginInvokeOutcome;
}) => Promise<void>;
/** Trusted Ed25519 public keys (hex). Loaded once from env at boot. */
trustedPublicKeysHex: ReadonlyArray<string>;
/** Pino-compatible structured logger seam. */
logger?: {
info: (fields: Record<string, unknown>, msg: string) => void;
warn?: (fields: Record<string, unknown>, msg: string) => void;
error?: (fields: Record<string, unknown>, msg: string) => void;
};
/**
* Phase 06 #22 — crash-tracker seam. Receives every outcome so the
* sliding-window counter can auto-disable a misbehaving plugin. The
* worker bootstrap wires this to a SharedStore-backed tracker; tests
* leave it `undefined` and the handler short-circuits.
*/
recordOutcomeForCrashTracking?: (args: { tenantId: string; pluginId: string; outcome: PluginInvokeOutcome }) => Promise<void>;
/** Injected clock; defaults to Date.now. */
now?: () => number;
};
/**
* Process exactly one plugin-invoke job. Returns the outcome (which
* is also persisted via `recordAudit`). Runtime failures are encoded as
* the discriminated `ok=false` shape. Durable audit failures deliberately
* throw so BullMQ can retry/dead-letter instead of acknowledging an
* invocation with no forensic trail.
*/
export async function handlePluginInvokeJob(payload: PluginInvokeJobPayload, deps: PluginInvokeDeps): Promise<PluginInvokeOutcome> {
const now = deps.now ?? Date.now;
const startedAt = now();
const pluginInvocationId = randomUUID();
const finalise = async (outcome: PluginInvokeOutcome): Promise<PluginInvokeOutcome> => {
await recordInvocationAudit(deps, payload, outcome);
// Phase 06 #22 — feed outcome to crash tracker. Some failure modes
// (plugin_not_found, plugin_disabled) are configuration issues, not
// plugin misbehaviour, so we deliberately skip them in the counter.
if (deps.recordOutcomeForCrashTracking) {
const trackable = outcome.ok || outcome.reason === "sandbox_failed" || outcome.reason === "signature_invalid" || outcome.reason === "binary_load_failed";
if (trackable) {
try {
await deps.recordOutcomeForCrashTracking({
tenantId: payload.tenantId,
pluginId: payload.pluginId,
outcome,
});
} catch (err) {
deps.logger?.warn?.({ ...payload, error: err instanceof Error ? err.message : String(err) }, "plugin.invoke.crash_tracker_failed");
}
}
}
return outcome;
};
const row = await deps.loadPlugin({
tenantId: payload.tenantId,
pluginId: payload.pluginId,
version: payload.version,
});
if (!row) {
deps.logger?.warn?.({ ...payload }, "plugin.invoke.not_found");
return finalise({ ok: false, reason: "plugin_not_found", durationMs: now() - startedAt, pluginInvocationId });
}
if (!row.enabled) {
deps.logger?.warn?.({ ...payload }, "plugin.invoke.disabled");
return finalise({ ok: false, reason: "plugin_disabled", durationMs: now() - startedAt, pluginInvocationId });
}
const declaredHook = getDeclaredHookForPayload(row.manifest, payload);
if (!declaredHook) {
deps.logger?.warn?.({ ...payload, declaredHooks: getDeclaredHookExports(row.manifest) }, "plugin.invoke.hook_not_declared");
return finalise({
ok: false,
reason: "hook_not_declared",
message: `Hook export "${payload.hookName}" is not declared by plugin manifest.`,
durationMs: now() - startedAt,
pluginInvocationId,
});
}
const capabilityGrants =
row.manifest.capabilities.length > 0 || row.manifest.hooks.length > 0
? await deps.loadCapabilityGrants({ tenantId: payload.tenantId, pluginId: payload.pluginId })
: [];
const missingCapabilityGrants = [
...getMissingCapabilityGrantNames({
manifest: row.manifest,
grants: capabilityGrants,
}),
...getMissingHookGrantNames({ declaredHook, grants: capabilityGrants }),
];
if (missingCapabilityGrants.length > 0) {
deps.logger?.warn?.({ ...payload, capabilities: missingCapabilityGrants }, "plugin.invoke.capability_grant_missing");
return finalise({
ok: false,
reason: "capability_grant_missing",
message: `Missing approved capability grants: ${missingCapabilityGrants.join(", ")}`,
durationMs: now() - startedAt,
pluginInvocationId,
});
}
let sandboxPayload = payload.input;
const registeredHook = getPluginHook(declaredHook.name);
if (registeredHook) {
const validation = validatePluginHookPayload(registeredHook, payload.input);
if (!validation.ok) {
deps.logger?.warn?.({ ...payload, hookName: declaredHook.name, issues: validation.issues }, "plugin.invoke.invalid_hook_payload");
return finalise({
ok: false,
reason: "invalid_hook_payload",
message: `Invalid payload for plugin hook "${declaredHook.name}".`,
durationMs: now() - startedAt,
pluginInvocationId,
});
}
sandboxPayload = validation.payload;
}
const wasmBuffer = await deps.loadBinary({ pluginId: payload.pluginId, version: payload.version });
if (!wasmBuffer) {
deps.logger?.error?.({ ...payload }, "plugin.invoke.binary_missing");
return finalise({ ok: false, reason: "binary_load_failed", durationMs: now() - startedAt, pluginInvocationId });
}
const storedMetadataError = validateStoredPluginVerificationMetadata(row);
if (storedMetadataError) {
deps.logger?.error?.({ ...payload, reason: storedMetadataError }, "plugin.invoke.stored_metadata_invalid");
return finalise({
ok: false,
reason: "signature_invalid",
message: storedMetadataError,
durationMs: now() - startedAt,
pluginInvocationId,
});
}
const verify = deps.verifySignature({
wasmBuffer,
manifest: row.manifest,
trustedPublicKeysHex: deps.trustedPublicKeysHex,
});
if (!verify.ok) {
deps.logger?.error?.({ ...payload, reason: verify.reason }, "plugin.invoke.signature_invalid");
return finalise({
ok: false,
reason: "signature_invalid",
message: verify.reason,
durationMs: now() - startedAt,
pluginInvocationId,
});
}
const rate = await deps.consumeRateLimit({ tenantId: payload.tenantId, pluginId: payload.pluginId });
if (!rate.allowed) {
deps.logger?.warn?.({ ...payload, remaining: rate.remaining }, "plugin.invoke.rate_limited");
return finalise({ ok: false, reason: "rate_limit_exceeded", durationMs: now() - startedAt, pluginInvocationId });
}
let result: PluginInvocationResult;
try {
result = await deps.invokeInSandbox({
pluginInvocationId,
tenantId: payload.tenantId,
pluginId: payload.pluginId,
wasmBuffer,
manifest: row.manifest,
hookName: payload.hookName,
payload: sandboxPayload,
capabilityGrants,
});
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
deps.logger?.error?.({ ...payload, error: message }, "plugin.invoke.sandbox_threw");
return finalise({
ok: false,
reason: "sandbox_failed",
message,
durationMs: now() - startedAt,
pluginInvocationId,
});
}
const durationMs = now() - startedAt;
if (!result.ok) {
deps.logger?.warn?.({ ...payload, reason: result.reason, message: result.message }, "plugin.invoke.sandbox_failed");
return finalise({
ok: false,
reason: "sandbox_failed",
message: result.message ?? result.reason,
durationMs,
pluginInvocationId,
});
}
deps.logger?.info({ ...payload, fuelUsed: result.fuelUsed, durationMs }, "plugin.invoke.completed");
// Phase 06 #23 — sanitize plugin output before it lands in the audit
// trail or worker logs. Strictness is env-driven (default moderate).
const sanitizedOutput = sanitizePluginOutput(result.output, resolveSanitizationStrictness());
return finalise({ ok: true, output: sanitizedOutput, fuelUsed: result.fuelUsed, durationMs, pluginInvocationId });
}
function getDeclaredHookForPayload(manifest: PluginManifest, payload: PluginInvokeJobPayload): PluginManifest["hooks"][number] | null {
if (!Array.isArray(manifest.hooks)) {
return null;
}
const candidates = manifest.hooks.filter((hook) => hook.export === payload.hookName);
if (candidates.length === 0) {
return null;
}
const logicalHookName = typeof payload.metadata?.logicalHookName === "string" ? payload.metadata.logicalHookName : null;
return candidates.find((hook) => hook.name === logicalHookName) ?? candidates[0] ?? null;
}
function getDeclaredHookExports(manifest: PluginManifest): string[] {
if (!Array.isArray(manifest.hooks)) {
return [];
}
return manifest.hooks.map((hook) => hook.export);
}
function getMissingHookGrantNames({
declaredHook,
grants,
}: {
declaredHook: PluginManifest["hooks"][number];
grants: ReadonlyArray<PluginCapabilityGrant>;
}): string[] {
const hookGrantName = `hook:${declaredHook.name}`;
return grants.some((grant) => grant.capabilityName === hookGrantName) ? [] : [hookGrantName];
}
async function recordInvocationAudit(deps: PluginInvokeDeps, payload: PluginInvokeJobPayload, outcome: PluginInvokeOutcome): Promise<void> {
try {
await deps.recordAudit({
tenantId: payload.tenantId,
pluginId: payload.pluginId,
version: payload.version,
hookName: payload.hookName,
pluginInvocationId: outcome.pluginInvocationId,
outcome,
});
} catch (err) {
const error = err instanceof Error ? err.message : String(err);
deps.logger?.error?.({ ...payload, pluginInvocationId: outcome.pluginInvocationId, error }, "plugin.invoke.audit_failed");
throw err;
}
}