Reorganize repo structure and standards references
This commit is contained in:
1 parent
3cb93a81be
commit
be2054274d
5885 files changed
+2588
-195
No files matched your search
@@ -0,0 +1,374 @@
|
||||
/**
|
||||
* Phase 06 #21 — Plugin-invoke worker job.
|
||||
*
|
||||
* Handles one job from the `toolctrl.plugin-invoke` queue:
|
||||
* 1. Loads the Plugin row + .wasm binary for the requested
|
||||
* (tenantId, pluginId, version) tuple.
|
||||
* 2. Verifies the binary's SHA-256 matches the stored hash and the
|
||||
* manifest's Ed25519 signature is valid against the trusted-keys
|
||||
* allowlist.
|
||||
* 3. Enforces a per-tenant sliding-window rate limit on plugin calls.
|
||||
* 4. Builds a fresh WASM sandbox with memory + fuel limits, invokes
|
||||
* the requested hook, and writes an audit event with the outcome.
|
||||
*
|
||||
* The handler is host-agnostic — every dependency (DB lookup, binary
|
||||
* loader, signature verify, rate limiter, audit writer, logger) is
|
||||
* passed in as a seam. The worker bootstrap wires real implementations;
|
||||
* tests inject fakes to assert behaviour without spinning a database.
|
||||
*
|
||||
* Deferred (see docs/improvement-plan/06-plugin-system/wasm-sandbox.md):
|
||||
* - hot-reload of plugin binaries (operator restarts pod)
|
||||
* - rich structured payload marshalling (covered by #22's hook lifecycle)
|
||||
* - Sentry / OTel span wiring (covered by #23's audit + docs)
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import {
|
||||
getMissingCapabilityGrantNames,
|
||||
getPluginHook,
|
||||
sanitizePluginOutput,
|
||||
type PluginCapabilityGrant,
|
||||
type PluginInvocationResult,
|
||||
type PluginManifest,
|
||||
type SanitizationStrictness,
|
||||
type VerifyPluginSignatureResult,
|
||||
validatePluginHookPayload,
|
||||
} from "@toolctrl/plugin-runtime";
|
||||
|
||||
/**
|
||||
* Resolve the sanitization strictness from env. Default is `moderate`
|
||||
* (redact PANs/JWTs/IBANs/API-keys but keep emails — emails are often
|
||||
* legitimate plugin output). Unknown values fall back to `moderate`.
|
||||
*/
|
||||
function resolveSanitizationStrictness(): SanitizationStrictness {
|
||||
const raw = process.env.PLUGIN_OUTPUT_SANITIZATION_STRICTNESS?.trim().toLowerCase();
|
||||
if (raw === "strict" || raw === "moderate" || raw === "off") {
|
||||
return raw;
|
||||
}
|
||||
return "moderate";
|
||||
}
|
||||
|
||||
function validateStoredPluginVerificationMetadata(row: PluginInvokeRow): string | null {
|
||||
if (row.wasmSha256 !== row.manifest.wasmSha256) {
|
||||
return "stored_wasm_sha256_mismatch";
|
||||
}
|
||||
if (row.signaturePublicKey.toLowerCase() !== row.manifest.signedBy.toLowerCase()) {
|
||||
return "stored_signature_public_key_mismatch";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export type PluginInvokeJobPayload = {
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
version: string;
|
||||
hookName: string;
|
||||
input?: unknown;
|
||||
metadata?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type PluginInvokeRow = {
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
version: string;
|
||||
manifest: PluginManifest;
|
||||
wasmSha256: string;
|
||||
signaturePublicKey: string;
|
||||
enabled: boolean;
|
||||
};
|
||||
|
||||
export type PluginInvokeOutcome =
|
||||
| { ok: true; output: unknown; fuelUsed: number; durationMs: number; pluginInvocationId: string }
|
||||
| { ok: false; reason: PluginInvokeFailure; message?: string; durationMs: number; pluginInvocationId: string };
|
||||
|
||||
export type PluginInvokeFailure =
|
||||
| "plugin_not_found"
|
||||
| "plugin_disabled"
|
||||
| "hook_not_declared"
|
||||
| "capability_grant_missing"
|
||||
| "binary_load_failed"
|
||||
| "invalid_hook_payload"
|
||||
| "signature_invalid"
|
||||
| "rate_limit_exceeded"
|
||||
| "sandbox_failed";
|
||||
|
||||
export type RateLimitDecision = { allowed: boolean; remaining: number };
|
||||
|
||||
export type PluginInvokeDeps = {
|
||||
/** Resolve the Plugin row for a tenant+pluginId+version tuple. */
|
||||
loadPlugin: (args: { tenantId: string; pluginId: string; version: string }) => Promise<PluginInvokeRow | null>;
|
||||
/** Resolve operator-approved capability grants for this tenant+plugin pair. */
|
||||
loadCapabilityGrants: (args: { tenantId: string; pluginId: string }) => Promise<ReadonlyArray<PluginCapabilityGrant>>;
|
||||
/** Load the raw .wasm bytes from disk / object storage. */
|
||||
loadBinary: (args: { pluginId: string; version: string }) => Promise<Uint8Array | null>;
|
||||
/** Verify the binary signature against the trusted-keys allowlist. */
|
||||
verifySignature: (args: { wasmBuffer: Uint8Array; manifest: PluginManifest; trustedPublicKeysHex: ReadonlyArray<string> }) => VerifyPluginSignatureResult;
|
||||
/** Per-tenant sliding-window rate limit (returns allowed flag). */
|
||||
consumeRateLimit: (args: { tenantId: string; pluginId: string }) => Promise<RateLimitDecision>;
|
||||
/** Invoke the plugin inside the worker-owned sandbox boundary. */
|
||||
invokeInSandbox: (args: {
|
||||
pluginInvocationId: string;
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
wasmBuffer: Uint8Array;
|
||||
manifest: PluginManifest;
|
||||
hookName: string;
|
||||
payload: unknown;
|
||||
capabilityGrants: ReadonlyArray<PluginCapabilityGrant>;
|
||||
}) => Promise<PluginInvocationResult>;
|
||||
/** Write an audit row for this invocation (success or failure). */
|
||||
recordAudit: (args: {
|
||||
tenantId: string;
|
||||
pluginId: string;
|
||||
version: string;
|
||||
hookName: string;
|
||||
pluginInvocationId: string;
|
||||
outcome: PluginInvokeOutcome;
|
||||
}) => Promise<void>;
|
||||
/** Trusted Ed25519 public keys (hex). Loaded once from env at boot. */
|
||||
trustedPublicKeysHex: ReadonlyArray<string>;
|
||||
/** Pino-compatible structured logger seam. */
|
||||
logger?: {
|
||||
info: (fields: Record<string, unknown>, msg: string) => void;
|
||||
warn?: (fields: Record<string, unknown>, msg: string) => void;
|
||||
error?: (fields: Record<string, unknown>, msg: string) => void;
|
||||
};
|
||||
/**
|
||||
* Phase 06 #22 — crash-tracker seam. Receives every outcome so the
|
||||
* sliding-window counter can auto-disable a misbehaving plugin. The
|
||||
* worker bootstrap wires this to a SharedStore-backed tracker; tests
|
||||
* leave it `undefined` and the handler short-circuits.
|
||||
*/
|
||||
recordOutcomeForCrashTracking?: (args: { tenantId: string; pluginId: string; outcome: PluginInvokeOutcome }) => Promise<void>;
|
||||
/** Injected clock; defaults to Date.now. */
|
||||
now?: () => number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Process exactly one plugin-invoke job. Returns the outcome (which
|
||||
* is also persisted via `recordAudit`). Runtime failures are encoded as
|
||||
* the discriminated `ok=false` shape. Durable audit failures deliberately
|
||||
* throw so BullMQ can retry/dead-letter instead of acknowledging an
|
||||
* invocation with no forensic trail.
|
||||
*/
|
||||
export async function handlePluginInvokeJob(payload: PluginInvokeJobPayload, deps: PluginInvokeDeps): Promise<PluginInvokeOutcome> {
|
||||
const now = deps.now ?? Date.now;
|
||||
const startedAt = now();
|
||||
const pluginInvocationId = randomUUID();
|
||||
|
||||
const finalise = async (outcome: PluginInvokeOutcome): Promise<PluginInvokeOutcome> => {
|
||||
await recordInvocationAudit(deps, payload, outcome);
|
||||
// Phase 06 #22 — feed outcome to crash tracker. Some failure modes
|
||||
// (plugin_not_found, plugin_disabled) are configuration issues, not
|
||||
// plugin misbehaviour, so we deliberately skip them in the counter.
|
||||
if (deps.recordOutcomeForCrashTracking) {
|
||||
const trackable = outcome.ok || outcome.reason === "sandbox_failed" || outcome.reason === "signature_invalid" || outcome.reason === "binary_load_failed";
|
||||
if (trackable) {
|
||||
try {
|
||||
await deps.recordOutcomeForCrashTracking({
|
||||
tenantId: payload.tenantId,
|
||||
pluginId: payload.pluginId,
|
||||
outcome,
|
||||
});
|
||||
} catch (err) {
|
||||
deps.logger?.warn?.({ ...payload, error: err instanceof Error ? err.message : String(err) }, "plugin.invoke.crash_tracker_failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
return outcome;
|
||||
};
|
||||
|
||||
const row = await deps.loadPlugin({
|
||||
tenantId: payload.tenantId,
|
||||
pluginId: payload.pluginId,
|
||||
version: payload.version,
|
||||
});
|
||||
if (!row) {
|
||||
deps.logger?.warn?.({ ...payload }, "plugin.invoke.not_found");
|
||||
return finalise({ ok: false, reason: "plugin_not_found", durationMs: now() - startedAt, pluginInvocationId });
|
||||
}
|
||||
if (!row.enabled) {
|
||||
deps.logger?.warn?.({ ...payload }, "plugin.invoke.disabled");
|
||||
return finalise({ ok: false, reason: "plugin_disabled", durationMs: now() - startedAt, pluginInvocationId });
|
||||
}
|
||||
const declaredHook = getDeclaredHookForPayload(row.manifest, payload);
|
||||
if (!declaredHook) {
|
||||
deps.logger?.warn?.({ ...payload, declaredHooks: getDeclaredHookExports(row.manifest) }, "plugin.invoke.hook_not_declared");
|
||||
return finalise({
|
||||
ok: false,
|
||||
reason: "hook_not_declared",
|
||||
message: `Hook export "${payload.hookName}" is not declared by plugin manifest.`,
|
||||
durationMs: now() - startedAt,
|
||||
pluginInvocationId,
|
||||
});
|
||||
}
|
||||
|
||||
const capabilityGrants =
|
||||
row.manifest.capabilities.length > 0 || row.manifest.hooks.length > 0
|
||||
? await deps.loadCapabilityGrants({ tenantId: payload.tenantId, pluginId: payload.pluginId })
|
||||
: [];
|
||||
const missingCapabilityGrants = [
|
||||
...getMissingCapabilityGrantNames({
|
||||
manifest: row.manifest,
|
||||
grants: capabilityGrants,
|
||||
}),
|
||||
...getMissingHookGrantNames({ declaredHook, grants: capabilityGrants }),
|
||||
];
|
||||
if (missingCapabilityGrants.length > 0) {
|
||||
deps.logger?.warn?.({ ...payload, capabilities: missingCapabilityGrants }, "plugin.invoke.capability_grant_missing");
|
||||
return finalise({
|
||||
ok: false,
|
||||
reason: "capability_grant_missing",
|
||||
message: `Missing approved capability grants: ${missingCapabilityGrants.join(", ")}`,
|
||||
durationMs: now() - startedAt,
|
||||
pluginInvocationId,
|
||||
});
|
||||
}
|
||||
|
||||
let sandboxPayload = payload.input;
|
||||
const registeredHook = getPluginHook(declaredHook.name);
|
||||
if (registeredHook) {
|
||||
const validation = validatePluginHookPayload(registeredHook, payload.input);
|
||||
if (!validation.ok) {
|
||||
deps.logger?.warn?.({ ...payload, hookName: declaredHook.name, issues: validation.issues }, "plugin.invoke.invalid_hook_payload");
|
||||
return finalise({
|
||||
ok: false,
|
||||
reason: "invalid_hook_payload",
|
||||
message: `Invalid payload for plugin hook "${declaredHook.name}".`,
|
||||
durationMs: now() - startedAt,
|
||||
pluginInvocationId,
|
||||
});
|
||||
}
|
||||
sandboxPayload = validation.payload;
|
||||
}
|
||||
|
||||
const wasmBuffer = await deps.loadBinary({ pluginId: payload.pluginId, version: payload.version });
|
||||
if (!wasmBuffer) {
|
||||
deps.logger?.error?.({ ...payload }, "plugin.invoke.binary_missing");
|
||||
return finalise({ ok: false, reason: "binary_load_failed", durationMs: now() - startedAt, pluginInvocationId });
|
||||
}
|
||||
|
||||
const storedMetadataError = validateStoredPluginVerificationMetadata(row);
|
||||
if (storedMetadataError) {
|
||||
deps.logger?.error?.({ ...payload, reason: storedMetadataError }, "plugin.invoke.stored_metadata_invalid");
|
||||
return finalise({
|
||||
ok: false,
|
||||
reason: "signature_invalid",
|
||||
message: storedMetadataError,
|
||||
durationMs: now() - startedAt,
|
||||
pluginInvocationId,
|
||||
});
|
||||
}
|
||||
|
||||
const verify = deps.verifySignature({
|
||||
wasmBuffer,
|
||||
manifest: row.manifest,
|
||||
trustedPublicKeysHex: deps.trustedPublicKeysHex,
|
||||
});
|
||||
if (!verify.ok) {
|
||||
deps.logger?.error?.({ ...payload, reason: verify.reason }, "plugin.invoke.signature_invalid");
|
||||
return finalise({
|
||||
ok: false,
|
||||
reason: "signature_invalid",
|
||||
message: verify.reason,
|
||||
durationMs: now() - startedAt,
|
||||
pluginInvocationId,
|
||||
});
|
||||
}
|
||||
|
||||
const rate = await deps.consumeRateLimit({ tenantId: payload.tenantId, pluginId: payload.pluginId });
|
||||
if (!rate.allowed) {
|
||||
deps.logger?.warn?.({ ...payload, remaining: rate.remaining }, "plugin.invoke.rate_limited");
|
||||
return finalise({ ok: false, reason: "rate_limit_exceeded", durationMs: now() - startedAt, pluginInvocationId });
|
||||
}
|
||||
|
||||
let result: PluginInvocationResult;
|
||||
try {
|
||||
result = await deps.invokeInSandbox({
|
||||
pluginInvocationId,
|
||||
tenantId: payload.tenantId,
|
||||
pluginId: payload.pluginId,
|
||||
wasmBuffer,
|
||||
manifest: row.manifest,
|
||||
hookName: payload.hookName,
|
||||
payload: sandboxPayload,
|
||||
capabilityGrants,
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
deps.logger?.error?.({ ...payload, error: message }, "plugin.invoke.sandbox_threw");
|
||||
return finalise({
|
||||
ok: false,
|
||||
reason: "sandbox_failed",
|
||||
message,
|
||||
durationMs: now() - startedAt,
|
||||
pluginInvocationId,
|
||||
});
|
||||
}
|
||||
|
||||
const durationMs = now() - startedAt;
|
||||
if (!result.ok) {
|
||||
deps.logger?.warn?.({ ...payload, reason: result.reason, message: result.message }, "plugin.invoke.sandbox_failed");
|
||||
return finalise({
|
||||
ok: false,
|
||||
reason: "sandbox_failed",
|
||||
message: result.message ?? result.reason,
|
||||
durationMs,
|
||||
pluginInvocationId,
|
||||
});
|
||||
}
|
||||
|
||||
deps.logger?.info({ ...payload, fuelUsed: result.fuelUsed, durationMs }, "plugin.invoke.completed");
|
||||
// Phase 06 #23 — sanitize plugin output before it lands in the audit
|
||||
// trail or worker logs. Strictness is env-driven (default moderate).
|
||||
const sanitizedOutput = sanitizePluginOutput(result.output, resolveSanitizationStrictness());
|
||||
return finalise({ ok: true, output: sanitizedOutput, fuelUsed: result.fuelUsed, durationMs, pluginInvocationId });
|
||||
}
|
||||
|
||||
function getDeclaredHookForPayload(manifest: PluginManifest, payload: PluginInvokeJobPayload): PluginManifest["hooks"][number] | null {
|
||||
if (!Array.isArray(manifest.hooks)) {
|
||||
return null;
|
||||
}
|
||||
const candidates = manifest.hooks.filter((hook) => hook.export === payload.hookName);
|
||||
if (candidates.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const logicalHookName = typeof payload.metadata?.logicalHookName === "string" ? payload.metadata.logicalHookName : null;
|
||||
return candidates.find((hook) => hook.name === logicalHookName) ?? candidates[0] ?? null;
|
||||
}
|
||||
|
||||
function getDeclaredHookExports(manifest: PluginManifest): string[] {
|
||||
if (!Array.isArray(manifest.hooks)) {
|
||||
return [];
|
||||
}
|
||||
return manifest.hooks.map((hook) => hook.export);
|
||||
}
|
||||
|
||||
function getMissingHookGrantNames({
|
||||
declaredHook,
|
||||
grants,
|
||||
}: {
|
||||
declaredHook: PluginManifest["hooks"][number];
|
||||
grants: ReadonlyArray<PluginCapabilityGrant>;
|
||||
}): string[] {
|
||||
const hookGrantName = `hook:${declaredHook.name}`;
|
||||
return grants.some((grant) => grant.capabilityName === hookGrantName) ? [] : [hookGrantName];
|
||||
}
|
||||
|
||||
async function recordInvocationAudit(deps: PluginInvokeDeps, payload: PluginInvokeJobPayload, outcome: PluginInvokeOutcome): Promise<void> {
|
||||
try {
|
||||
await deps.recordAudit({
|
||||
tenantId: payload.tenantId,
|
||||
pluginId: payload.pluginId,
|
||||
version: payload.version,
|
||||
hookName: payload.hookName,
|
||||
pluginInvocationId: outcome.pluginInvocationId,
|
||||
outcome,
|
||||
});
|
||||
} catch (err) {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
deps.logger?.error?.({ ...payload, pluginInvocationId: outcome.pluginInvocationId, error }, "plugin.invoke.audit_failed");
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user