Track HiveGrid Cloud planning reference

hivegrid-cloud/apps/worker/README.md

@@ -0,0 +1,53 @@
+# @toolctrl/worker
+
+Opt-in BullMQ worker runtime for queue-backed jobs.
+
+The worker is disabled by default and is not imported by the web runtime. Running it directly without `WORKER_ENABLED=true` exits immediately.
+
+## Local execution
+
+Check worker readiness and Redis configuration without starting processors:
+
+```sh
+pnpm --filter @toolctrl/worker health
+```
+
+Run the disabled-by-default worker locally:
+
+```sh
+pnpm --filter @toolctrl/worker dev
+```
+
+Start the enabled worker against a local Redis instance and database:
+
+```sh
+WORKER_ENABLED=true WORKER_QUEUE_GROUP=general BULLMQ_REDIS_URL=redis://localhost:6379/0 DATABASE_URL=postgresql://localhost/app pnpm --filter @toolctrl/worker dev
+```
+
+The worker stays disabled unless `WORKER_ENABLED=true`. When enabled, `DATABASE_URL` is required, and Redis is resolved from `BULLMQ_REDIS_URL` first; `REDIS_URL` is fallback only when `BULLMQ_REDIS_URL` is absent for deployments that already provide a generic Redis binding.
+
+## Docker Compose
+
+Build the worker image and start a disabled smoke container with local Redis:
+
+```sh
+docker compose up redis worker-general
+```
+
+The compose topology builds the Dockerfile `worker` target for five grouped services: `worker-general`, `worker-mail`, `worker-import`, `worker-webhook`, and `worker-plugin`. Each service leaves processors disabled by default and wires Redis with `BULLMQ_REDIS_URL=redis://redis:6379/0` using the compose service name. Enable real processors only with a valid database URL and the group you want to exercise, for example with `WORKER_ENABLED=true DATABASE_URL=postgresql://... docker compose up worker-mail` or an override file.
+
+Manual E2E note: real queued job processing requires a running Redis/container smoke test outside Auto-Mode; Auto-Mode verification for this task is limited to static configuration and documentation checks.
+
+## Current processors
+
+The worker has concrete handler paths for mail, webhooks, notifications, maintenance, storage scans, exports, imports, and guarded AI/browser job descriptors. Some queues remain disabled unless feature flags and provider services are configured.
+
+- `toolctrl.email`
+- `toolctrl.notifications`
+- `toolctrl.maintenance`
+- `toolctrl.ai-chatbot`
+- `toolctrl.browser-automation`
+- `toolctrl.file-scan`
+- `toolctrl.exports`
+- `toolctrl.imports`
+- `toolctrl.plugin-invoke`

hivegrid-cloud/apps/worker/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@toolctrl/worker",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "sideEffects": [
+    "./src/otel.ts"
+  ],
+  "scripts": {
+    "build": "node scripts/build.mjs",
+    "dev": "tsx src/index.ts",
+    "health": "node dist/health.js",
+    "start": "node dist/index.js",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "3.1045.0",
+    "@aws-sdk/lib-storage": "3.1045.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.76.0",
+    "@opentelemetry/core": "^2.7.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.218.0",
+    "@opentelemetry/resources": "^2.7.1",
+    "@opentelemetry/sdk-node": "^0.218.0",
+    "@opentelemetry/sdk-trace-base": "^2.7.1",
+    "@opentelemetry/semantic-conventions": "^1.40.0",
+    "@sentry/node": "^10.53.1",
+    "@toolctrl/config": "workspace:*",
+    "@toolctrl/db": "workspace:*",
+    "@toolctrl/email-sequences": "workspace:*",
+    "@toolctrl/email-templates": "workspace:*",
+    "@toolctrl/entitlements": "workspace:*",
+    "@toolctrl/error-reporter": "workspace:*",
+    "@toolctrl/exports": "workspace:*",
+    "@toolctrl/form-widget": "workspace:*",
+    "@toolctrl/foundation": "workspace:*",
+    "@toolctrl/imports": "workspace:*",
+    "@toolctrl/logger": "workspace:*",
+    "@toolctrl/mail": "workspace:*",
+    "@toolctrl/marketing-outbound": "workspace:*",
+    "@toolctrl/metrics": "workspace:*",
+    "@toolctrl/observability": "workspace:*",
+    "@toolctrl/plugin-runtime": "workspace:*",
+    "@toolctrl/queue": "workspace:*",
+    "@toolctrl/search": "workspace:*",
+    "@toolctrl/shared-store": "workspace:*",
+    "@toolctrl/storage": "workspace:*",
+    "bullmq": "^5.76.5",
+    "clamscan": "^2.4.0",
+    "heic-convert": "^2.1.0",
+    "ioredis": "5.10.1",
+    "meilisearch": "0.45.0",
+    "pdf-to-png-converter": "^4.1.1",
+    "pino": "^9.0.0",
+    "prom-client": "^15.1.3",
+    "semver": "^7.7.3",
+    "sharp": "^0.35.2",
+    "stripe": "^19.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.0",
+    "tsx": "^4.6.2",
+    "typescript": "^5.3.2"
+  }
+}

hivegrid-cloud/apps/worker/scripts/build.mjs

@@ -0,0 +1,35 @@
+import { build } from "esbuild";
+
+const sourcemap = process.env.SOURCEMAP === "true" || process.env.BUILD_SOURCEMAPS === "true";
+
+const externalBarePackagesPlugin = {
+  name: "external-bare-packages",
+  setup(builder) {
+    builder.onResolve({ filter: /^[^./]|^\.[^./]|^\.\.[^/]/ }, (args) => {
+      if (args.path === "@toolctrl/queue/runtime" || args.path.startsWith("@toolctrl/queue/runtime/")) {
+        return { path: args.path, external: true };
+      }
+
+      if (args.path.startsWith("@toolctrl/")) {
+        return undefined;
+      }
+
+      return { path: args.path, external: true };
+    });
+  },
+};
+
+await build({
+  entryPoints: ["src/index.ts", "src/health.ts", "src/jobs/plugin-sandbox-worker.ts"],
+  outdir: "dist",
+  bundle: true,
+  platform: "node",
+  format: "esm",
+  target: "node20",
+  sourcemap,
+  banner: {
+    js: 'import { createRequire as __toolctrlCreateRequire } from "node:module"; const require = __toolctrlCreateRequire(import.meta.url);',
+  },
+  logLevel: "warning",
+  plugins: [externalBarePackagesPlugin],
+});

hivegrid-cloud/apps/worker/src/admin-server.test.ts

@@ -0,0 +1,102 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { createMetricsRegistry, createDefaultMetrics, type MetricsRegistry, type DefaultMetrics } from "@toolctrl/metrics";
+import { createWorkerAdminHandler, type WorkerStatusReport } from "./admin-server.js";
+
+let registry: MetricsRegistry;
+let metrics: DefaultMetrics;
+
+beforeEach(() => {
+  registry = createMetricsRegistry({ serviceName: "worker-test", collectDefaultProcessMetrics: false });
+  metrics = createDefaultMetrics(registry);
+  metrics.queueJobsActive.labels({ queue: "email" }).set(1);
+});
+
+function fakeReqRes(opts: { url: string; authorization?: string }) {
+  const writes: string[] = [];
+  const headers: Record<string, string | number | string[]> = {};
+  const req = {
+    url: opts.url,
+    headers: opts.authorization ? { authorization: opts.authorization } : {},
+  } as unknown as Parameters<ReturnType<typeof createWorkerAdminHandler>>[0];
+  const res = {
+    statusCode: 200,
+    headersSent: false,
+    setHeader: (k: string, v: string | number | string[]) => {
+      headers[k.toLowerCase()] = v;
+    },
+    end: (chunk?: string) => {
+      if (chunk !== undefined) writes.push(chunk);
+    },
+  } as unknown as Parameters<ReturnType<typeof createWorkerAdminHandler>>[1];
+  return { req, res, writes, headers };
+}
+
+function okStatus(): WorkerStatusReport {
+  return {
+    status: "ok",
+    service: "worker",
+    checks: { runtime: "ok" },
+    version: "abc",
+    uptime_seconds: 1,
+    region: "eu-central-1",
+  };
+}
+
+const TOKEN = "x".repeat(64);
+
+describe("createWorkerAdminHandler", () => {
+  it("returns 200 + JSON on /status", async () => {
+    const handler = createWorkerAdminHandler({
+      enabled: true,
+      token: TOKEN,
+      registry,
+      buildStatus: () => okStatus(),
+    });
+    const { req, res, writes } = fakeReqRes({ url: "/status" });
+    await handler(req, res);
+    expect((res as unknown as { statusCode: number }).statusCode).toBe(200);
+    expect(JSON.parse(writes[0]!)).toMatchObject({ status: "ok", service: "worker" });
+  });
+
+  it("returns 503 when status report is down", async () => {
+    const handler = createWorkerAdminHandler({
+      enabled: true,
+      token: TOKEN,
+      registry,
+      buildStatus: () => ({ ...okStatus(), status: "down", checks: { runtime: "down" } }),
+    });
+    const { req, res } = fakeReqRes({ url: "/status" });
+    await handler(req, res);
+    expect((res as unknown as { statusCode: number }).statusCode).toBe(503);
+  });
+
+  it("returns 404 on /metrics when disabled", async () => {
+    const handler = createWorkerAdminHandler({ enabled: false, token: TOKEN, registry, buildStatus: () => okStatus() });
+    const { req, res } = fakeReqRes({ url: "/metrics" });
+    await handler(req, res);
+    expect((res as unknown as { statusCode: number }).statusCode).toBe(404);
+  });
+
+  it("returns 401 on /metrics without bearer token", async () => {
+    const handler = createWorkerAdminHandler({ enabled: true, token: TOKEN, registry, buildStatus: () => okStatus() });
+    const { req, res } = fakeReqRes({ url: "/metrics" });
+    await handler(req, res);
+    expect((res as unknown as { statusCode: number }).statusCode).toBe(401);
+  });
+
+  it("returns 200 + prom text on /metrics with valid token", async () => {
+    const handler = createWorkerAdminHandler({ enabled: true, token: TOKEN, registry, buildStatus: () => okStatus() });
+    const { req, res, writes, headers } = fakeReqRes({ url: "/metrics", authorization: `Bearer ${TOKEN}` });
+    await handler(req, res);
+    expect((res as unknown as { statusCode: number }).statusCode).toBe(200);
+    expect(headers["content-type"]).toContain("text/plain");
+    expect(writes.join("")).toContain("queue_jobs_active");
+  });
+
+  it("returns 404 on unknown route", async () => {
+    const handler = createWorkerAdminHandler({ enabled: true, token: TOKEN, registry, buildStatus: () => okStatus() });
+    const { req, res } = fakeReqRes({ url: "/health" });
+    await handler(req, res);
+    expect((res as unknown as { statusCode: number }).statusCode).toBe(404);
+  });
+});

hivegrid-cloud/apps/worker/src/admin-server.ts

@@ -0,0 +1,190 @@
+/**
+ * Tiny HTTP admin server for the worker process.
+ *
+ * Exposes `/status` (public, JSON, 200/503) and `/metrics` (token-protected,
+ * Prom text-format) so Prometheus + Uptime monitors can scrape the worker
+ * the same way they scrape the API. The worker has no Hono app today and we
+ * don't want to pull in one just for two endpoints — node's built-in
+ * `http.createServer` is plenty.
+ *
+ * The server only listens when `METRICS_ENABLED=true`. Otherwise this module
+ * is a no-op so dev / Sentry-less staging environments keep a single open
+ * file descriptor budget.
+ */
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import { isMetricsEnabled, validateMetricsTokenAuth, type MetricsRegistry } from "@toolctrl/metrics";
+
+export type WorkerStatusCheckOutcome = "ok" | "down" | "disabled";
+
+export type WorkerStatusReport = {
+  status: "ok" | "down";
+  service: "worker";
+  checks: {
+    runtime: WorkerStatusCheckOutcome;
+    outboxDispatcher?: WorkerStatusCheckOutcome;
+    workflowTrigger?: WorkerStatusCheckOutcome;
+    pluginHookFanout?: WorkerStatusCheckOutcome;
+    transactionalMailOutbox?: WorkerStatusCheckOutcome;
+    emailSequences?: WorkerStatusCheckOutcome;
+  };
+  // Resolved WORKFLOW_TRIGGER_EXECUTOR_MODE for this worker. Only present when the
+  // workflow-trigger subscriber runs on this worker (i.e. the check is not "disabled"),
+  // so operators can see whether queued workflows execute over the HTTP bridge, the
+  // native runner, or the hybrid fallback path.
+  workflowTriggerExecutorMode?: "http" | "native" | "hybrid";
+  version: string;
+  uptime_seconds: number;
+  region: string;
+};
+
+export type WorkerAdminServerOptions = {
+  enabled: boolean;
+  token: string | undefined;
+  registry: MetricsRegistry;
+  port: number;
+  host?: string;
+  logger?: { error?: (fields: Record<string, unknown>, msg: string) => void; info?: (fields: Record<string, unknown>, msg: string) => void };
+  buildStatus: () => Promise<WorkerStatusReport> | WorkerStatusReport;
+};
+
+export type WorkerAdminServerHandle = {
+  close: () => Promise<void>;
+  port: number;
+};
+
+// Health-server (apps/worker/src/health-server.ts) defaults to 9100. Admin
+// metrics-server must default to a different port to avoid EADDRINUSE in the
+// container — both servers bind to 0.0.0.0 inside the same process.
+export const DEFAULT_WORKER_ADMIN_PORT = 9101;
+
+export function createWorkerAdminHandler(deps: Omit<WorkerAdminServerOptions, "port" | "host">) {
+  return async function handler(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    const url = req.url ?? "/";
+    if (url === "/status" || url === "/status/") {
+      await respondStatus(res, deps);
+      return;
+    }
+    if (url === "/metrics" || url === "/metrics/") {
+      await respondMetrics(req, res, deps);
+      return;
+    }
+    res.statusCode = 404;
+    res.setHeader("content-type", "application/json");
+    res.end(JSON.stringify({ error: { code: "not_found", message: "Endpoint not found." } }));
+  };
+}
+
+export function startWorkerAdminServer(options: WorkerAdminServerOptions): WorkerAdminServerHandle | undefined {
+  if (!options.enabled) {
+    return undefined;
+  }
+  const handler = createWorkerAdminHandler({
+    enabled: options.enabled,
+    token: options.token,
+    registry: options.registry,
+    buildStatus: options.buildStatus,
+    ...(options.logger ? { logger: options.logger } : {}),
+  });
+  const server: Server = createServer((req, res) => {
+    void handler(req, res).catch((error: unknown) => {
+      options.logger?.error?.({ error: serialiseError(error) }, "worker admin server handler failed");
+      if (!res.headersSent) {
+        res.statusCode = 500;
+        res.setHeader("content-type", "application/json");
+        res.end(JSON.stringify({ error: { code: "internal", message: "Internal server error." } }));
+      } else {
+        res.end();
+      }
+    });
+  });
+  server.listen(options.port, options.host ?? "0.0.0.0");
+  options.logger?.info?.({ port: options.port, host: options.host ?? "0.0.0.0" }, "worker admin server listening");
+  return {
+    port: options.port,
+    close: () =>
+      new Promise<void>((resolve) => {
+        server.close(() => {
+          resolve();
+        });
+      }),
+  };
+}
+
+async function respondStatus(res: ServerResponse, deps: Omit<WorkerAdminServerOptions, "port" | "host">): Promise<void> {
+  let report: WorkerStatusReport;
+  try {
+    report = await deps.buildStatus();
+  } catch (error) {
+    deps.logger?.error?.({ error: serialiseError(error) }, "worker admin status build failed");
+    report = {
+      status: "down",
+      service: "worker",
+      checks: { runtime: "down" },
+      version: process.env.APP_REVISION?.trim() || "unknown",
+      uptime_seconds: Math.round(process.uptime()),
+      region: process.env.APP_REGION?.trim() || "unknown",
+    };
+  }
+  res.statusCode = report.status === "ok" ? 200 : 503;
+  res.setHeader("content-type", "application/json");
+  res.end(JSON.stringify(report));
+}
+
+async function respondMetrics(req: IncomingMessage, res: ServerResponse, deps: Omit<WorkerAdminServerOptions, "port" | "host">): Promise<void> {
+  if (!deps.enabled) {
+    res.statusCode = 404;
+    res.setHeader("content-type", "application/json");
+    res.end(JSON.stringify({ error: { code: "not_found", message: "Endpoint not enabled." } }));
+    return;
+  }
+  const auth = validateMetricsTokenAuth({
+    metricsEnabled: true,
+    configuredToken: deps.token,
+    authorizationHeader: pickAuthorizationHeader(req),
+  });
+  if (!auth.ok) {
+    if (auth.status === 500) {
+      res.statusCode = 500;
+      res.setHeader("content-type", "application/json");
+      res.end(JSON.stringify({ error: { code: "token_misconfigured", message: "Metrics token not configured." } }));
+      return;
+    }
+    if (auth.status === 403) {
+      res.statusCode = 404;
+      res.setHeader("content-type", "application/json");
+      res.end(JSON.stringify({ error: { code: "not_found", message: "Endpoint not enabled." } }));
+      return;
+    }
+    res.statusCode = 401;
+    res.setHeader("content-type", "application/json");
+    res.end(JSON.stringify({ error: { code: auth.code, message: "Authentication required." } }));
+    return;
+  }
+
+  const text = await deps.registry.metrics();
+  res.statusCode = 200;
+  res.setHeader("content-type", deps.registry.contentType);
+  res.setHeader("cache-control", "no-store");
+  res.end(text);
+}
+
+function pickAuthorizationHeader(req: IncomingMessage): string | undefined {
+  const raw = req.headers.authorization;
+  if (Array.isArray(raw)) {
+    return raw[0];
+  }
+  return raw;
+}
+
+function serialiseError(error: unknown): { name: string; message: string } {
+  if (error instanceof Error) {
+    return { name: error.name, message: error.message.slice(0, 200) };
+  }
+  return { name: "UnknownError", message: String(error).slice(0, 200) };
+}
+
+/**
+ * Reads `METRICS_ENABLED` for the worker. Re-exported so callers don't import
+ * the env helper from a different package than the admin server.
+ */
+export const isWorkerMetricsEnabled = isMetricsEnabled;

hivegrid-cloud/apps/worker/src/connection.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, test } from "vitest";
+import { DEFAULT_GROUP_POOL_SIZE, createWorkerConnection } from "./connection.js";
+
+const REDIS_URL = "redis://user:pass@redis.example.com:6380/3";
+
+describe("createWorkerConnection", () => {
+  test("uses BULLMQ_REDIS_URL when present and tags the connection with group + pid", () => {
+    const result = createWorkerConnection("mail", {
+      env: { BULLMQ_REDIS_URL: REDIS_URL },
+      pid: 9999,
+    });
+
+    expect(result.host).toBe("redis.example.com");
+    expect(result.port).toBe(6380);
+    expect(result.db).toBe(3);
+    expect(result.connectionName).toBe("worker-mail-9999");
+    expect(result.pool).toBe(DEFAULT_GROUP_POOL_SIZE.mail);
+  });
+
+  test("falls back to REDIS_URL when BULLMQ_REDIS_URL is missing", () => {
+    const result = createWorkerConnection("general", {
+      env: { REDIS_URL },
+      pid: 1,
+    });
+
+    expect(result.host).toBe("redis.example.com");
+    expect(result.connectionName).toBe("worker-general-1");
+  });
+
+  test("rejects missing Redis URL with a non-leaking message", () => {
+    expect(() => createWorkerConnection("import", { env: {} })).toThrow(
+      "Worker Redis URL is required: set BULLMQ_REDIS_URL or REDIS_URL."
+    );
+  });
+
+  test("honours WORKER_REDIS_POOL_SIZE override", () => {
+    const result = createWorkerConnection("webhook", {
+      env: { BULLMQ_REDIS_URL: REDIS_URL, WORKER_REDIS_POOL_SIZE: "16" },
+      pid: 42,
+    });
+
+    expect(result.pool).toBe(16);
+  });
+
+  test("ignores zero/negative/non-numeric pool overrides", () => {
+    for (const bogus of ["0", "-3", "abc", ""]) {
+      const result = createWorkerConnection("webhook", {
+        env: { BULLMQ_REDIS_URL: REDIS_URL, WORKER_REDIS_POOL_SIZE: bogus },
+        pid: 1,
+      });
+      expect(result.pool).toBe(DEFAULT_GROUP_POOL_SIZE.webhook);
+    }
+  });
+
+  test("defaults differ between webhook (8) and import (2)", () => {
+    const webhook = createWorkerConnection("webhook", { env: { BULLMQ_REDIS_URL: REDIS_URL }, pid: 1 });
+    const importer = createWorkerConnection("import", { env: { BULLMQ_REDIS_URL: REDIS_URL }, pid: 1 });
+    expect(webhook.pool).toBe(8);
+    expect(importer.pool).toBe(2);
+  });
+
+  test("connectionName uses runtime pid when not supplied", () => {
+    const result = createWorkerConnection("general", { env: { BULLMQ_REDIS_URL: REDIS_URL } });
+    expect(result.connectionName).toMatch(/^worker-general-\d+$/);
+  });
+});

hivegrid-cloud/apps/worker/src/connection.ts

@@ -0,0 +1,94 @@
+import type { EnvRecord } from "@toolctrl/config";
+import { parseRedisUrl, type ConnectionOptions, type WorkerQueueGroup } from "@toolctrl/queue";
+
+/**
+ * Per-group default Redis connection pool size. BullMQ uses ioredis under
+ * the hood — `family`/`tls`/pool tuning happen at ioredis level. We expose
+ * `pool` as an opt-in field BullMQ does not consume so the value can be
+ * read by integrations (custom factories, ioredis-cluster wrappers); the
+ * default fields stay 1:1 compatible with BullMQ's `WorkerOptions.connection`.
+ *
+ * Rationale per group:
+ *  - general  : 4  — mixed I/O, modest connection demand
+ *  - mail     : 4  — SMTP/HTTP, moderate
+ *  - import   : 2  — long-running but low connection turnover
+ *  - webhook  : 8  — high parallelism (concurrency 20+) needs more sockets
+ *  - plugin   : 4  — sandbox calls are short-lived; Redis is mostly BullMQ
+ *                 command traffic plus sliding-window rate limits
+ */
+export const DEFAULT_GROUP_POOL_SIZE: Readonly<Record<WorkerQueueGroup, number>> = {
+  general: 4,
+  mail: 4,
+  import: 2,
+  webhook: 8,
+  plugin: 4,
+};
+
+const POOL_ENV_VAR = "WORKER_REDIS_POOL_SIZE";
+
+export type WorkerConnectionOptions = ConnectionOptions & {
+  /**
+   * Human-readable name BullMQ/ioredis tags the connection with. Visible in
+   * `CLIENT LIST` so operators can attribute Redis traffic to a worker
+   * group + PID. Format: `worker-${group}-${pid}`.
+   */
+  connectionName: string;
+  /**
+   * Advisory pool size (not consumed by BullMQ directly). Read by callers
+   * that wrap BullMQ in a custom connection factory.
+   */
+  pool: number;
+};
+
+export type CreateWorkerConnectionOptions = {
+  env?: EnvRecord;
+  pid?: number;
+};
+
+/**
+ * Builds the BullMQ connection options for a single worker process.
+ * Resolves the Redis URL from BULLMQ_REDIS_URL/REDIS_URL via the queue
+ * package's shared parser, then layers on a connection name + pool size
+ * derived from the group.
+ *
+ * Throws if neither BULLMQ_REDIS_URL nor REDIS_URL is configured — a
+ * worker without a Redis target can't function and should fail loud.
+ */
+export function createWorkerConnection(group: WorkerQueueGroup, options: CreateWorkerConnectionOptions = {}): WorkerConnectionOptions {
+  const env = options.env ?? (process.env as EnvRecord);
+  const pid = options.pid ?? process.pid;
+  const redisUrl = pickRedisUrl(env);
+  if (!redisUrl) {
+    throw new Error("Worker Redis URL is required: set BULLMQ_REDIS_URL or REDIS_URL.");
+  }
+  const base = parseRedisUrl(redisUrl.name, redisUrl.value);
+  return {
+    ...base,
+    connectionName: `worker-${group}-${pid}`,
+    pool: resolvePoolSize(group, env),
+  };
+}
+
+function resolvePoolSize(group: WorkerQueueGroup, env: EnvRecord): number {
+  const raw = env[POOL_ENV_VAR];
+  if (typeof raw === "string" && raw.length > 0) {
+    const parsed = Number.parseInt(raw, 10);
+    if (Number.isFinite(parsed) && parsed > 0) {
+      return parsed;
+    }
+  }
+  return DEFAULT_GROUP_POOL_SIZE[group];
+}
+
+function pickRedisUrl(env: EnvRecord): { name: "BULLMQ_REDIS_URL" | "REDIS_URL"; value: string } | undefined {
+  const bullmq = trim(env.BULLMQ_REDIS_URL);
+  if (bullmq) return { name: "BULLMQ_REDIS_URL", value: bullmq };
+  const redis = trim(env.REDIS_URL);
+  if (redis) return { name: "REDIS_URL", value: redis };
+  return undefined;
+}
+
+function trim(value: string | undefined): string | undefined {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed : undefined;
+}

hivegrid-cloud/apps/worker/src/db/tenantRls.ts

@@ -0,0 +1,41 @@
+// AK-P4a — worker-side tenant RLS adoption, mirroring R-P2b's web helper
+// (apps/web/app/utils/db/.server/tenantRls.server.ts). The worker cannot import
+// the web helper (it is bound to the web `db` singleton), so it gets its own thin
+// wrapper that takes the Prisma client as a parameter (the worker uses
+// getDbClient() from @toolctrl/db/client.server).
+//
+// Sets the tenant context TRANSACTION-LOCALLY (set_config(..., true) == SET LOCAL,
+// PgBouncer transaction-pooling safe) and runs the queries inside that same
+// transaction, so the RLS policies (USING/WITH CHECK on
+// current_setting('app.tenant_id')) bind. Without the context an RLS-enabled
+// table returns no rows for the restricted app role.
+//
+// The helper is generic over the transaction client so callers can keep narrow,
+// string-typed seam interfaces (decoupled from the generated Prisma enum types,
+// which keeps the dynamic-import jobs building before native libs are installed).
+// The only requirement is that the tx exposes `$executeRaw` (a real Prisma
+// TransactionClient does).
+
+/** Minimal surface the helper needs on the transaction client to set the GUC. */
+export interface RlsTransactionExecutor {
+  $executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
+}
+
+/** A client whose `$transaction` yields a tx of type `Tx`. */
+export interface TransactionCapableClient<Tx> {
+  $transaction<T>(fn: (tx: Tx) => Promise<T>): Promise<T>;
+}
+
+export async function withTenantRLS<Tx extends RlsTransactionExecutor, T>(
+  client: TransactionCapableClient<Tx>,
+  tenantId: string,
+  callback: (tx: Tx) => Promise<T>,
+): Promise<T> {
+  if (!tenantId) {
+    throw new Error("withTenantRLS requires a non-empty tenantId");
+  }
+  return client.$transaction(async (tx) => {
+    await tx.$executeRaw`SELECT set_config('app.tenant_id', ${tenantId}, true)`;
+    return callback(tx);
+  });
+}

hivegrid-cloud/apps/worker/src/group-runtime.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, test } from "vitest";
+import { queueNames } from "@toolctrl/queue";
+import {
+  DEFAULT_GROUP_CONCURRENCY,
+  InvalidWorkerQueueGroupError,
+  bootstrapWorkerGroup,
+  resolveGroupConcurrency,
+  resolveQueueNamesForGroup,
+  resolveWorkerQueueGroup,
+} from "./group-runtime.js";
+
+describe("resolveWorkerQueueGroup", () => {
+  test("returns the validated group when env is set", () => {
+    expect(resolveWorkerQueueGroup({ WORKER_QUEUE_GROUP: "mail" })).toBe("mail");
+  });
+
+  test("throws when env is missing", () => {
+    expect(() => resolveWorkerQueueGroup({})).toThrow(InvalidWorkerQueueGroupError);
+  });
+
+  test("throws when env is unknown", () => {
+    expect(() => resolveWorkerQueueGroup({ WORKER_QUEUE_GROUP: "all" })).toThrow(InvalidWorkerQueueGroupError);
+  });
+
+  test("error message lists every valid group", () => {
+    try {
+      resolveWorkerQueueGroup({ WORKER_QUEUE_GROUP: "bogus" });
+      throw new Error("should have thrown");
+    } catch (error) {
+      expect(error).toBeInstanceOf(InvalidWorkerQueueGroupError);
+      const message = (error as Error).message;
+      for (const group of ["general", "mail", "import", "webhook", "plugin"]) {
+        expect(message).toContain(group);
+      }
+      expect(message).toContain("docs/ops/worker-queues.md");
+    }
+  });
+});
+
+describe("resolveQueueNamesForGroup", () => {
+  test("mail group resolves to email queue", () => {
+    expect(resolveQueueNamesForGroup("mail")).toEqual([queueNames.email]);
+  });
+
+  test("general group includes notifications + maintenance + fileScan", () => {
+    expect(resolveQueueNamesForGroup("general")).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan]);
+  });
+
+  test("import group includes imports + exports", () => {
+    expect(resolveQueueNamesForGroup("import")).toEqual([queueNames.imports, queueNames.exports]);
+  });
+
+  test("webhook group covers AI + browser automation", () => {
+    expect(resolveQueueNamesForGroup("webhook")).toEqual([queueNames.aiChatbot, queueNames.browserAutomation]);
+  });
+
+  test("plugin group resolves to plugin-invoke queue", () => {
+    expect(resolveQueueNamesForGroup("plugin")).toEqual([queueNames.pluginInvoke]);
+  });
+});
+
+describe("resolveGroupConcurrency", () => {
+  test("falls back to documented defaults when env is unset", () => {
+    expect(resolveGroupConcurrency("general", {})).toBe(DEFAULT_GROUP_CONCURRENCY.general);
+    expect(resolveGroupConcurrency("mail", {})).toBe(DEFAULT_GROUP_CONCURRENCY.mail);
+    expect(resolveGroupConcurrency("import", {})).toBe(DEFAULT_GROUP_CONCURRENCY.import);
+    expect(resolveGroupConcurrency("webhook", {})).toBe(DEFAULT_GROUP_CONCURRENCY.webhook);
+    expect(resolveGroupConcurrency("plugin", {})).toBe(DEFAULT_GROUP_CONCURRENCY.plugin);
+  });
+
+  test("honours WORKER_CONCURRENCY_PLUGIN override", () => {
+    expect(resolveGroupConcurrency("plugin", { WORKER_CONCURRENCY_PLUGIN: "12" })).toBe(12);
+  });
+
+  test("honours explicit WORKER_CONCURRENCY_<GROUP> override", () => {
+    expect(resolveGroupConcurrency("webhook", { WORKER_CONCURRENCY_WEBHOOK: "40" })).toBe(40);
+    expect(resolveGroupConcurrency("mail", { WORKER_CONCURRENCY_MAIL: "8" })).toBe(8);
+  });
+
+  test("ignores non-numeric or zero/negative overrides and uses default", () => {
+    expect(resolveGroupConcurrency("mail", { WORKER_CONCURRENCY_MAIL: "abc" })).toBe(DEFAULT_GROUP_CONCURRENCY.mail);
+    expect(resolveGroupConcurrency("mail", { WORKER_CONCURRENCY_MAIL: "0" })).toBe(DEFAULT_GROUP_CONCURRENCY.mail);
+    expect(resolveGroupConcurrency("mail", { WORKER_CONCURRENCY_MAIL: "-5" })).toBe(DEFAULT_GROUP_CONCURRENCY.mail);
+  });
+});
+
+describe("bootstrapWorkerGroup", () => {
+  test("returns a fully-resolved record for valid env", () => {
+    expect(
+      bootstrapWorkerGroup({ WORKER_QUEUE_GROUP: "import", WORKER_CONCURRENCY_IMPORT: "2" })
+    ).toEqual({
+      group: "import",
+      queueNames: [queueNames.imports, queueNames.exports],
+      concurrency: 2,
+      concurrencyEnvVar: "WORKER_CONCURRENCY_IMPORT",
+    });
+  });
+
+  test("rejects invalid group", () => {
+    expect(() => bootstrapWorkerGroup({ WORKER_QUEUE_GROUP: "" })).toThrow(InvalidWorkerQueueGroupError);
+  });
+});

hivegrid-cloud/apps/worker/src/group-runtime.ts

@@ -0,0 +1,113 @@
+import type { EnvRecord } from "@toolctrl/config";
+import {
+  WORKER_QUEUE_GROUP_NAMES,
+  getQueueNamesForGroup,
+  isWorkerQueueGroup,
+  type QueueName,
+  type WorkerQueueGroup,
+} from "@toolctrl/queue";
+
+/**
+ * Default per-group concurrency. These are deliberate — they reflect the
+ * workload profile of each group, not a guess:
+ *  - general : 10 (mixed I/O, short jobs)
+ *  - mail    :  5 (SMTP/API throughput-bound)
+ *  - import  :  1 (CPU/memory-bound long-running)
+ *  - webhook : 20 (I/O-bound external HTTP)
+ *  - plugin  :  5 (WASM-sandboxed third-party code; tight memory cap)
+ *
+ * Operators override via `WORKER_CONCURRENCY_<GROUP>` (uppercase, e.g.
+ * `WORKER_CONCURRENCY_WEBHOOK=40`).
+ */
+export const DEFAULT_GROUP_CONCURRENCY: Readonly<Record<WorkerQueueGroup, number>> = {
+  general: 10,
+  mail: 5,
+  import: 1,
+  webhook: 20,
+  plugin: 5,
+};
+
+const CONCURRENCY_ENV_VAR: Readonly<Record<WorkerQueueGroup, string>> = {
+  general: "WORKER_CONCURRENCY_GENERAL",
+  mail: "WORKER_CONCURRENCY_MAIL",
+  import: "WORKER_CONCURRENCY_IMPORT",
+  webhook: "WORKER_CONCURRENCY_WEBHOOK",
+  plugin: "WORKER_CONCURRENCY_PLUGIN",
+};
+
+export class InvalidWorkerQueueGroupError extends Error {
+  override readonly name = "InvalidWorkerQueueGroupError";
+  constructor(value: string) {
+    super(
+      `WORKER_QUEUE_GROUP must be one of [${WORKER_QUEUE_GROUP_NAMES.join(", ")}]; received ${JSON.stringify(value)}. ` +
+        "See docs/ops/worker-queues.md."
+    );
+  }
+}
+
+/**
+ * Reads `WORKER_QUEUE_GROUP` from the given env and returns the validated
+ * group. Throws `InvalidWorkerQueueGroupError` (a thin subclass of Error)
+ * if the value is missing or unknown — caller is expected to log the
+ * message and `process.exit(1)`.
+ */
+export function resolveWorkerQueueGroup(env: EnvRecord = process.env as EnvRecord): WorkerQueueGroup {
+  const raw = env.WORKER_QUEUE_GROUP;
+  if (typeof raw !== "string" || raw.length === 0) {
+    throw new InvalidWorkerQueueGroupError(raw === undefined ? "" : String(raw));
+  }
+  if (!isWorkerQueueGroup(raw)) {
+    throw new InvalidWorkerQueueGroupError(raw);
+  }
+  return raw;
+}
+
+/**
+ * Returns the BullMQ queue names this worker process should subscribe to,
+ * given its group. Wrapper over `getQueueNamesForGroup` kept as a separate
+ * module-level helper so callers don't have to import from the queue
+ * package directly when they already depend on the group runtime.
+ */
+export function resolveQueueNamesForGroup(group: WorkerQueueGroup): readonly QueueName[] {
+  return getQueueNamesForGroup(group);
+}
+
+/**
+ * Reads `WORKER_CONCURRENCY_<GROUP>` from env (e.g. `WORKER_CONCURRENCY_MAIL`).
+ * Falls back to the documented default for the group. Non-numeric or non-
+ * positive overrides fall back to the default rather than throwing — a
+ * misconfigured env var should not knock the worker offline.
+ */
+export function resolveGroupConcurrency(group: WorkerQueueGroup, env: EnvRecord = process.env as EnvRecord): number {
+  const raw = env[CONCURRENCY_ENV_VAR[group]];
+  if (typeof raw === "string" && raw.length > 0) {
+    const parsed = Number.parseInt(raw, 10);
+    if (Number.isFinite(parsed) && parsed > 0) {
+      return parsed;
+    }
+  }
+  return DEFAULT_GROUP_CONCURRENCY[group];
+}
+
+export type WorkerGroupResolution = {
+  group: WorkerQueueGroup;
+  queueNames: readonly QueueName[];
+  concurrency: number;
+  concurrencyEnvVar: string;
+};
+
+/**
+ * One-shot bootstrap helper for the worker entry point. Validates env and
+ * returns the group + its queues + the resolved concurrency in a single
+ * record — that record is enough for `startWorkerRuntime` to register the
+ * right BullMQ workers without re-reading env.
+ */
+export function bootstrapWorkerGroup(env: EnvRecord = process.env as EnvRecord): WorkerGroupResolution {
+  const group = resolveWorkerQueueGroup(env);
+  return {
+    group,
+    queueNames: resolveQueueNamesForGroup(group),
+    concurrency: resolveGroupConcurrency(group, env),
+    concurrencyEnvVar: CONCURRENCY_ENV_VAR[group],
+  };
+}

hivegrid-cloud/apps/worker/src/health-server.test.ts

@@ -0,0 +1,167 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { describe, expect, test, vi } from "vitest";
+import { createWorkerHealthHandler } from "./health-server.js";
+
+function createMockResponse() {
+  const captured = {
+    statusCode: 200,
+    headers: {} as Record<string, string>,
+    body: "" as string,
+    ended: false,
+    headersSent: false,
+  };
+  const res = {
+    get statusCode() {
+      return captured.statusCode;
+    },
+    set statusCode(code: number) {
+      captured.statusCode = code;
+    },
+    get headersSent() {
+      return captured.headersSent;
+    },
+    setHeader(name: string, value: string) {
+      captured.headers[name.toLowerCase()] = value;
+    },
+    end(body?: string) {
+      captured.headersSent = true;
+      if (body !== undefined) captured.body = body;
+      captured.ended = true;
+    },
+  } as unknown as ServerResponse;
+  return { res, captured };
+}
+
+function createMockReq(url: string): IncomingMessage {
+  return { url } as IncomingMessage;
+}
+
+describe("worker health handler", () => {
+  test("/healthz returns 200 alive without invoking probes", async () => {
+    const redisProbe = vi.fn();
+    const databaseProbe = vi.fn();
+    const handler = createWorkerHealthHandler({ redisProbe, databaseProbe, workerGroup: "general" });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/healthz"), res);
+
+    expect(captured.statusCode).toBe(200);
+    expect(JSON.parse(captured.body)).toEqual({ status: "alive", workerGroup: "general" });
+    expect(redisProbe).not.toHaveBeenCalled();
+    expect(databaseProbe).not.toHaveBeenCalled();
+  });
+
+  test("/readyz returns 200 when both probes succeed", async () => {
+    const handler = createWorkerHealthHandler({
+      redisProbe: async () => ({ ok: true } as const),
+      databaseProbe: async () => ({ ok: true } as const),
+      workerGroup: "mail",
+    });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/readyz"), res);
+
+    expect(captured.statusCode).toBe(200);
+    const body = JSON.parse(captured.body);
+    expect(body.status).toBe("ready");
+    expect(body.workerGroup).toBe("mail");
+    expect(body.checks).toEqual({ redis: "ok", database: "ok" });
+  });
+
+  test("/readyz returns 503 when redis probe fails", async () => {
+    const handler = createWorkerHealthHandler({
+      redisProbe: async () => ({ ok: false, error: "ECONNREFUSED" } as const),
+      databaseProbe: async () => ({ ok: true } as const),
+    });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/readyz"), res);
+
+    expect(captured.statusCode).toBe(503);
+    const body = JSON.parse(captured.body);
+    expect(body.status).toBe("not_ready");
+    expect(body.checks.redis).toBe("error");
+    expect(body.checks.database).toBe("ok");
+  });
+
+  test("/readyz returns 503 when database probe throws", async () => {
+    const handler = createWorkerHealthHandler({
+      redisProbe: async () => ({ ok: true } as const),
+      databaseProbe: async () => {
+        throw new Error("pg disconnected");
+      },
+    });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/readyz"), res);
+
+    expect(captured.statusCode).toBe(503);
+    expect(JSON.parse(captured.body).checks.database).toBe("error");
+  });
+
+  test("/readyz includes required runtime probes in readiness status", async () => {
+    const handler = createWorkerHealthHandler({
+      redisProbe: async () => ({ ok: true } as const),
+      databaseProbe: async () => ({ ok: true } as const),
+      readinessProbes: {
+        outboxDispatcher: async () => ({ ok: true } as const),
+        workflowTrigger: async () => ({ ok: false, error: "subscriber disabled" } as const),
+      },
+      workerGroup: "general",
+    });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/readyz"), res);
+
+    expect(captured.statusCode).toBe(503);
+    expect(JSON.parse(captured.body)).toMatchObject({
+      status: "not_ready",
+      workerGroup: "general",
+      checks: {
+        redis: "ok",
+        database: "ok",
+        outboxDispatcher: "ok",
+        workflowTrigger: "error",
+      },
+    });
+  });
+
+  test("/readyz times out a hanging probe", async () => {
+    const handler = createWorkerHealthHandler({
+      redisProbe: () => new Promise(() => undefined),
+      databaseProbe: async () => ({ ok: true } as const),
+      probeTimeoutMs: 20,
+    });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/readyz"), res);
+
+    expect(captured.statusCode).toBe(503);
+    expect(JSON.parse(captured.body).checks.redis).toBe("error");
+  });
+
+  test("unknown routes return 404 JSON", async () => {
+    const handler = createWorkerHealthHandler({
+      redisProbe: async () => ({ ok: true } as const),
+      databaseProbe: async () => ({ ok: true } as const),
+    });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/wat"), res);
+
+    expect(captured.statusCode).toBe(404);
+    expect(JSON.parse(captured.body).error.code).toBe("not_found");
+  });
+
+  test("/healthz omits workerGroup when none is configured", async () => {
+    const handler = createWorkerHealthHandler({
+      redisProbe: async () => ({ ok: true } as const),
+      databaseProbe: async () => ({ ok: true } as const),
+    });
+    const { res, captured } = createMockResponse();
+
+    await handler(createMockReq("/healthz"), res);
+
+    expect(JSON.parse(captured.body)).toEqual({ status: "alive" });
+  });
+});

hivegrid-cloud/apps/worker/src/health-server.ts

@@ -0,0 +1,162 @@
+/**
+ * Per-worker HTTP health server.
+ *
+ * Exposes two Kubernetes/Coolify-style probes on a dedicated port so a
+ * platform can verify the worker process is alive (`/healthz`) and that
+ * its critical dependencies are reachable (`/readyz`).
+ *
+ * Separate from `admin-server.ts` (which serves /status and /metrics
+ * gated by METRICS_ENABLED + a bearer token). Health probes must work
+ * even when metrics are off and even before a bearer is provisioned.
+ *
+ *  - `/healthz` — process liveness. Returns 200 as long as the event
+ *                 loop is responsive. No external dependency calls.
+ *  - `/readyz`  — readiness. Issues a BullMQ Redis ping and a Prisma
+ *                 `SELECT 1` to confirm both backends are reachable.
+ *                 Returns 200 only when every probe succeeds; 503
+ *                 otherwise. Each check is bounded by a timeout so a
+ *                 hung dependency cannot stall the probe.
+ *
+ * The handler is intentionally framework-free — node's built-in
+ * `http.createServer` keeps the bundle size and startup cost minimal.
+ */
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import type { WorkerQueueGroup } from "@toolctrl/queue";
+
+export const DEFAULT_WORKER_HEALTH_PORT = 9100;
+
+export type HealthProbeResult = { ok: true } | { ok: false; error: string };
+
+export type HealthProbe = () => Promise<HealthProbeResult> | HealthProbeResult;
+
+export type ReadyResponseBody = {
+  status: "ready" | "not_ready";
+  workerGroup?: WorkerQueueGroup;
+  checks: Record<string, "ok" | "error">;
+};
+
+export type WorkerHealthServerOptions = {
+  port?: number;
+  host?: string;
+  workerGroup?: WorkerQueueGroup;
+  redisProbe: HealthProbe;
+  databaseProbe: HealthProbe;
+  readinessProbes?: Record<string, HealthProbe>;
+  probeTimeoutMs?: number;
+  logger?: {
+    error?: (fields: Record<string, unknown>, msg: string) => void;
+    info?: (fields: Record<string, unknown>, msg: string) => void;
+  };
+};
+
+export type WorkerHealthServerHandle = {
+  port: number;
+  close: () => Promise<void>;
+};
+
+const DEFAULT_PROBE_TIMEOUT_MS = 2_000;
+
+/**
+ * Pure handler factory — exposed so tests can hit the request handler
+ * directly without binding a TCP port.
+ */
+export function createWorkerHealthHandler(deps: Omit<WorkerHealthServerOptions, "port" | "host" | "logger">) {
+  const timeoutMs = deps.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
+  return async function handler(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    const url = req.url ?? "/";
+    if (url === "/healthz" || url === "/healthz/") {
+      respondJson(res, 200, { status: "alive", ...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}) });
+      return;
+    }
+    if (url === "/readyz" || url === "/readyz/") {
+      const additionalProbes = Object.entries(deps.readinessProbes ?? {});
+      const [redis, database, ...additionalResults] = await Promise.all([
+        runWithTimeout(deps.redisProbe, timeoutMs, "redis"),
+        runWithTimeout(deps.databaseProbe, timeoutMs, "database"),
+        ...additionalProbes.map(([label, probe]) => runWithTimeout(probe, timeoutMs, label)),
+      ]);
+      const checks: ReadyResponseBody["checks"] = {
+        redis: redis.ok ? "ok" : "error",
+        database: database.ok ? "ok" : "error",
+      };
+      for (const [index, [label]] of additionalProbes.entries()) {
+        checks[label] = additionalResults[index]?.ok ? "ok" : "error";
+      }
+      const allOk = redis.ok && database.ok && additionalResults.every((result) => result.ok);
+      respondJson(res, allOk ? 200 : 503, {
+        status: allOk ? "ready" : "not_ready",
+        ...(deps.workerGroup ? { workerGroup: deps.workerGroup } : {}),
+        checks,
+      });
+      return;
+    }
+    respondJson(res, 404, { error: { code: "not_found", message: "Endpoint not found." } });
+  };
+}
+
+/**
+ * Binds the health handler to a TCP port and returns a handle whose
+ * `close()` shuts the server cleanly during SIGTERM/SIGINT.
+ */
+export function startWorkerHealthServer(options: WorkerHealthServerOptions): WorkerHealthServerHandle {
+  const handler = createWorkerHealthHandler({
+    redisProbe: options.redisProbe,
+    databaseProbe: options.databaseProbe,
+    ...(options.readinessProbes ? { readinessProbes: options.readinessProbes } : {}),
+    ...(options.workerGroup ? { workerGroup: options.workerGroup } : {}),
+    ...(options.probeTimeoutMs !== undefined ? { probeTimeoutMs: options.probeTimeoutMs } : {}),
+  });
+
+  const port = options.port ?? DEFAULT_WORKER_HEALTH_PORT;
+  const host = options.host ?? "0.0.0.0";
+
+  const server: Server = createServer((req, res) => {
+    void handler(req, res).catch((error: unknown) => {
+      options.logger?.error?.({ error: serialiseError(error) }, "worker health server handler failed");
+      if (!res.headersSent) {
+        respondJson(res, 500, { error: { code: "internal", message: "Internal server error." } });
+      } else {
+        res.end();
+      }
+    });
+  });
+
+  server.listen(port, host);
+  options.logger?.info?.({ port, host, workerGroup: options.workerGroup }, "worker health server listening");
+
+  return {
+    port,
+    close: () =>
+      new Promise<void>((resolve) => {
+        server.close(() => resolve());
+      }),
+  };
+}
+
+async function runWithTimeout(probe: HealthProbe, timeoutMs: number, label: string): Promise<HealthProbeResult> {
+  try {
+    const result = await Promise.race([
+      Promise.resolve().then(() => probe()),
+      new Promise<HealthProbeResult>((resolve) =>
+        setTimeout(() => resolve({ ok: false, error: `${label} probe timed out after ${timeoutMs}ms` }), timeoutMs)
+      ),
+    ]);
+    return result;
+  } catch (error) {
+    return { ok: false, error: serialiseError(error).message };
+  }
+}
+
+function respondJson(res: ServerResponse, statusCode: number, body: unknown): void {
+  res.statusCode = statusCode;
+  res.setHeader("content-type", "application/json");
+  res.setHeader("cache-control", "no-store");
+  res.end(JSON.stringify(body));
+}
+
+function serialiseError(error: unknown): { name: string; message: string } {
+  if (error instanceof Error) {
+    return { name: error.name, message: error.message.slice(0, 200) };
+  }
+  return { name: "UnknownError", message: String(error).slice(0, 200) };
+}

hivegrid-cloud/apps/worker/src/health.test.ts

@@ -0,0 +1,726 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, resolve } from "node:path";
+import { describe, expect, test, vi } from "vitest";
+import { queueNames } from "@toolctrl/queue";
+import { buildWorkerHealthReport, runWorkerHealthCli } from "./health.js";
+
+const packageJsonPath = resolve(dirname(fileURLToPath(import.meta.url)), "../package.json");
+const mailjetEnv = {
+  MAILJET_API_KEY: "fixture-mailjet-key",
+  MAILJET_SECRET_KEY: "fixture-mailjet-secret",
+  MAILJET_FROM_EMAIL: "sender@example.com",
+};
+
+function readPackageScripts(): Record<string, string> {
+  return JSON.parse(readFileSync(packageJsonPath, "utf8")).scripts;
+}
+
+function createIo() {
+  let output = "";
+
+  return {
+    io: {
+      stdout: {
+        write(chunk: string) {
+          output += chunk;
+          return true;
+        },
+      },
+    },
+    readOutput: () => output,
+  };
+}
+
+describe("worker health", () => {
+  test("production health script runs compiled Node output without tsx", () => {
+    const scripts = readPackageScripts();
+
+    expect(scripts.health).toBe("node dist/health.js");
+    expect(scripts.health).not.toContain("tsx");
+  });
+
+  test("reports disabled without Redis as a successful config-only health state and skips DB", async () => {
+    const databaseProbe = vi.fn();
+    const report = await buildWorkerHealthReport({ WORKER_ENABLED: "false" }, { databaseProbe });
+
+    expect(report).toEqual({
+      databaseConfigured: false,
+      databaseStatus: "skipped",
+      errors: [],
+      importConfigured: false,
+      importStatus: "skipped",
+      mailjetConfigured: false,
+      mailjetStatus: "skipped",
+      pluginRuntimeConfigured: false,
+      pluginRuntimeStatus: "skipped",
+      queues: [queueNames.email, queueNames.notifications],
+      redisConfigured: false,
+      redisParseStatus: "not_configured",
+      redisSource: "none",
+      redisStatus: "skipped",
+      revision: "unknown",
+      status: "disabled",
+      storageConfigured: false,
+      storageStatus: "skipped",
+      workerEnabled: false,
+      workerGroup: null,
+    });
+    expect(databaseProbe).not.toHaveBeenCalled();
+  });
+
+  test("reports ready with BULLMQ Redis and ready DB configured", async () => {
+    const databaseProbe = vi.fn();
+    const redisProbe = vi.fn();
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://user:db-secret@localhost:5432/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe, redisProbe }
+    );
+
+    expect(report).toMatchObject({
+      databaseConfigured: true,
+      databaseStatus: "ready",
+      errors: [],
+      redisConfigured: true,
+      redisParseStatus: "valid",
+      redisSource: "BULLMQ_REDIS_URL",
+      redisStatus: "ready",
+      status: "ready",
+      importConfigured: false,
+      importStatus: "skipped",
+      mailjetConfigured: true,
+      mailjetStatus: "ready",
+      storageConfigured: false,
+      storageStatus: "skipped",
+      workerEnabled: true,
+      workerGroup: "general",
+    });
+    expect(report.queues).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan]);
+    expect(redisProbe).toHaveBeenCalledWith("redis://localhost:6379/0");
+    expect(databaseProbe).toHaveBeenCalledTimes(1);
+  });
+
+  test("probes Redis and database readiness concurrently", async () => {
+    const events: string[] = [];
+    let releaseRedis: (() => void) | undefined;
+    let databaseStartedBeforeRedisFinished = false;
+    const redisProbe = vi.fn(
+      () =>
+        new Promise<void>((resolve) => {
+          events.push("redis:start");
+          releaseRedis = () => {
+            events.push("redis:end");
+            resolve();
+          };
+        })
+    );
+    const databaseProbe = vi.fn(async () => {
+      databaseStartedBeforeRedisFinished = !events.includes("redis:end");
+      events.push("db:start");
+      releaseRedis?.();
+      events.push("db:end");
+    });
+
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://user:db-secret@localhost:5432/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe, redisProbe }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(databaseStartedBeforeRedisFinished).toBe(true);
+    expect(events).toEqual(["redis:start", "db:start", "redis:end", "db:end"]);
+  });
+
+  test("includes the injected image revision without probing dependencies", async () => {
+    const report = await buildWorkerHealthReport({ APP_REVISION: "abc1234", WORKER_ENABLED: "false" }, { databaseProbe: vi.fn() });
+
+    expect(report.revision).toBe("abc1234");
+  });
+
+  test("does not block webhook-capable worker readiness when Mailjet delivery config is absent", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "webhook",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(report.mailjetConfigured).toBe(false);
+    expect(report.mailjetStatus).toBe("skipped");
+    expect(report.errors).toEqual([]);
+  });
+
+  test("reports queue-group-scoped queues for the mail worker", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "mail",
+        ...mailjetEnv,
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(report.workerGroup).toBe("mail");
+    expect(report.queues).toEqual([queueNames.email]);
+  });
+
+  test("reports general required-mode gaps as redacted health diagnostics", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_REQUIRED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.errors).toEqual(
+      expect.arrayContaining([
+        "OUTBOX_DISPATCHER_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and OUTBOX_DISPATCHER_REQUIRED=true.",
+        "WORKFLOW_TRIGGER_OUTBOX_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true.",
+        "API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true with http or hybrid executor mode.",
+        "WORKFLOW_TRIGGER_EXECUTOR_URL:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true with http or hybrid executor mode.",
+      ])
+    );
+    expect(JSON.stringify(report)).not.toContain("postgresql://localhost/app");
+  });
+
+  test("does not accept the legacy workflow executor token without the web API token", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "true",
+        OUTBOX_DISPATCHER_REQUIRED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_EXECUTOR_TOKEN: "legacy-worker-only",
+        WORKFLOW_TRIGGER_EXECUTOR_URL: "https://app.example.com",
+        WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.errors).toContain(
+      "API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true with http or hybrid executor mode."
+    );
+  });
+
+  test("reports workflow required-mode outbox dispatcher enablement gaps explicitly", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        API_ACCESS_TOKEN: "redacted-test-token",
+        APP_URL: "https://app.example.com",
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.errors).toEqual([
+      "OUTBOX_DISPATCHER_ENABLED:missing-workflow-trigger-dispatcher:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true.",
+    ]);
+    expect(JSON.stringify(report)).not.toContain("redacted-test-token");
+    expect(JSON.stringify(report)).not.toContain("https://app.example.com");
+  });
+
+  test("reports explicitly enabled workflow outbox bridge gaps even when not required", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "true",
+        OUTBOX_DISPATCHER_REQUIRED: "false",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "false",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.errors).toEqual([
+      "API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_ENABLED=true with http or hybrid executor mode.",
+      "WORKFLOW_TRIGGER_EXECUTOR_URL:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_ENABLED=true with http or hybrid executor mode.",
+    ]);
+    expect(JSON.stringify(report)).not.toContain("postgresql://localhost/app");
+  });
+
+  test("accepts general required-mode workflow fanout when native executor mode is explicit", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "true",
+        OUTBOX_DISPATCHER_REQUIRED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_EXECUTOR_MODE: "native",
+        WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(report.errors).toEqual([]);
+  });
+
+  test("accepts general required-mode workflow fanout when native executor mode has whitespace", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "true",
+        OUTBOX_DISPATCHER_REQUIRED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_EXECUTOR_MODE: " native ",
+        WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(report.errors).toEqual([]);
+  });
+
+  test("reports invalid workflow executor mode before required-mode bootstrap fails later", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        API_ACCESS_TOKEN: "redacted-test-token",
+        APP_URL: "https://app.example.com",
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "true",
+        OUTBOX_DISPATCHER_REQUIRED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_EXECUTOR_MODE: "sidecar",
+        WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.errors).toEqual([
+      "WORKFLOW_TRIGGER_EXECUTOR_MODE:invalid-workflow-executor-mode:Expected http, native, or hybrid when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true.",
+    ]);
+    expect(JSON.stringify(report)).not.toContain("redacted-test-token");
+    expect(JSON.stringify(report)).not.toContain("https://app.example.com");
+  });
+
+  test("accepts general required-mode workflow fanout with signed HTTP bridge config", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        API_ACCESS_TOKEN: "redacted-test-token",
+        APP_URL: "https://app.example.com",
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "true",
+        OUTBOX_DISPATCHER_REQUIRED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+        WORKFLOW_TRIGGER_OUTBOX_ENABLED: "true",
+        WORKFLOW_TRIGGER_OUTBOX_REQUIRED: "true",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(JSON.stringify(report)).not.toContain("redacted-test-token");
+    expect(JSON.stringify(report)).not.toContain("https://app.example.com");
+  });
+
+  test("reports production plugin runtime gaps on the general worker", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "false",
+        PLUGIN_CAPABILITY_USE_ENABLED: "false",
+        PLUGIN_HOOK_FANOUT_ENABLED: "false",
+        PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.pluginRuntimeConfigured).toBe(true);
+    expect(report.pluginRuntimeStatus).toBe("misconfigured");
+    expect(report.errors).toEqual(
+      expect.arrayContaining([
+        "PLUGIN_TRUSTED_PUBLIC_KEYS:missing-plugin-trust-root:Required when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.",
+        "OUTBOX_DISPATCHER_ENABLED:missing-plugin-runtime-dispatcher:Required on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.",
+        "PLUGIN_HOOK_FANOUT_ENABLED:disabled-plugin-runtime-fanout:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.",
+        "PLUGIN_CAPABILITY_USE_ENABLED:disabled-plugin-capability-use:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.",
+      ])
+    );
+  });
+
+  test("reports production plugin runtime ready when trust root and general subscribers are configured", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        OUTBOX_DISPATCHER_ENABLED: "true",
+        PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
+        PLUGIN_TRUSTED_PUBLIC_KEYS: "aa".repeat(32),
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(report.pluginRuntimeConfigured).toBe(true);
+    expect(report.pluginRuntimeStatus).toBe("ready");
+    expect(report.errors).toEqual([]);
+  });
+
+  test("reports plugin worker production runtime trust-root readiness", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "plugin",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.pluginRuntimeConfigured).toBe(true);
+    expect(report.pluginRuntimeStatus).toBe("misconfigured");
+    expect(report.errors).toEqual(["PLUGIN_TRUSTED_PUBLIC_KEYS:missing-plugin-trust-root:Required when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."]);
+  });
+
+  test("reports required mail worker misconfigured when Mailjet env is absent", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "mail",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.mailjetConfigured).toBe(false);
+    expect(report.mailjetStatus).toBe("misconfigured");
+    expect(report.errors).toEqual(
+      expect.arrayContaining([
+        "MAILJET_API_KEY:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.",
+        "MAILJET_SECRET_KEY:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.",
+        "MAILJET_FROM_EMAIL:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.",
+      ])
+    );
+  });
+
+  test("reports misconfigured and skips DB probe when enabled worker lacks DATABASE_URL", async () => {
+    const databaseProbe = vi.fn();
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe, redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.databaseConfigured).toBe(false);
+    expect(report.databaseStatus).toBe("skipped");
+    expect(report.errors).toEqual(expect.arrayContaining(["DATABASE_URL:missing-required:DATABASE_URL is required when WORKER_ENABLED is true."]));
+    expect(databaseProbe).not.toHaveBeenCalled();
+  });
+
+  test("reports misconfigured and skips dependency probes when enabled worker lacks WORKER_QUEUE_GROUP", async () => {
+    const databaseProbe = vi.fn();
+    const redisProbe = vi.fn();
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+      },
+      { databaseProbe, redisProbe }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.workerEnabled).toBe(true);
+    expect(report.workerGroup).toBe(null);
+    expect(report.errors).toEqual(expect.arrayContaining(["WORKER_QUEUE_GROUP:missing-required:WORKER_QUEUE_GROUP is required when WORKER_ENABLED is true."]));
+    expect(databaseProbe).not.toHaveBeenCalled();
+    expect(redisProbe).not.toHaveBeenCalled();
+  });
+
+  test("reports the maintenance queue when export retention processing is enabled", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        EXPORT_RETENTION_QUEUE_ENABLED: "true",
+        ...mailjetEnv,
+        STORAGE_ACCESS_KEY_ID: "access-key",
+        STORAGE_BUCKET: "exports",
+        STORAGE_ENDPOINT: "https://storage.example.com",
+        STORAGE_REGION: "us-east-1",
+        STORAGE_SECRET_ACCESS_KEY: "secret-key",
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(report.queues).toEqual([queueNames.notifications, queueNames.maintenance, queueNames.fileScan]);
+    expect(report.storageStatus).toBe("ready");
+  });
+
+  test("keeps the import worker group ready but idle until import runtime gates are enabled", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "import",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("ready");
+    expect(report.queues).toEqual([]);
+    expect(report.importStatus).toBe("skipped");
+    expect(report.errors).not.toEqual(expect.arrayContaining([expect.stringMatching(/missing-import/u)]));
+  });
+
+  test("reports misconfigured when import worker group is partially enabled", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        IMPORT_QUEUE_ENABLED: "true",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "import",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.queues).toEqual([]);
+    expect(report.importStatus).toBe("misconfigured");
+    expect(report.errors).toEqual(
+      expect.arrayContaining([
+        "IMPORT_QUEUE_RUNTIME_READY:missing-import-runtime-gate:Required with IMPORT_QUEUE_ENABLED=true after import, export, and scan services are wired.",
+      ])
+    );
+  });
+
+  test("reports misconfigured when export retention is enabled without storage delete config", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://localhost/app",
+        EXPORT_RETENTION_QUEUE_ENABLED: "true",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report.status).toBe("misconfigured");
+    expect(report.storageStatus).toBe("misconfigured");
+    expect(report.errors).toEqual(expect.arrayContaining(["STORAGE_ENDPOINT:missing-storage-config:Required when EXPORT_RETENTION_QUEUE_ENABLED=true."]));
+  });
+
+  test("falls back to REDIS_URL when BULLMQ_REDIS_URL is absent", async () => {
+    const report = await buildWorkerHealthReport(
+      {
+        DATABASE_URL: "postgresql://localhost/app",
+        REDIS_URL: "rediss://localhost:6380/1",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(report).toMatchObject({
+      databaseStatus: "ready",
+      errors: [],
+      redisConfigured: true,
+      redisParseStatus: "valid",
+      redisSource: "REDIS_URL",
+      redisStatus: "ready",
+      status: "ready",
+      workerEnabled: true,
+    });
+  });
+
+  test("returns non-zero for enabled worker with invalid Redis protocol and skips DB", async () => {
+    const databaseProbe = vi.fn();
+    const { io, readOutput } = createIo();
+    const secretRedisUrl = "http://user:redis-secret@localhost:6379";
+    const exitCode = await runWorkerHealthCli(
+      {
+        BULLMQ_REDIS_URL: secretRedisUrl,
+        DATABASE_URL: "postgresql://db-secret@localhost/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      io,
+      { databaseProbe }
+    );
+    const output = readOutput();
+    const report = JSON.parse(output);
+
+    expect(exitCode).toBe(1);
+    expect(report).toMatchObject({
+      databaseConfigured: true,
+      databaseStatus: "skipped",
+      redisConfigured: false,
+      redisParseStatus: "invalid",
+      redisSource: "BULLMQ_REDIS_URL",
+      redisStatus: "skipped",
+      status: "misconfigured",
+      workerEnabled: true,
+    });
+    expect(report.errors).toEqual(["BULLMQ_REDIS_URL:invalid-redis-protocol:Configured Redis URL protocol must be redis: or rediss:."]);
+    expect(databaseProbe).not.toHaveBeenCalled();
+    expect(output).not.toContain(secretRedisUrl);
+    expect(output).not.toContain("redis-secret");
+    expect(output).not.toContain("db-secret");
+  });
+
+  test("returns non-zero with redacted DB code when DB readiness fails", async () => {
+    const { io, readOutput } = createIo();
+    const exitCode = await runWorkerHealthCli(
+      {
+        BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+        DATABASE_URL: "postgresql://user:database-secret@localhost:5432/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      io,
+      {
+        databaseProbe: vi.fn().mockRejectedValue(
+          Object.assign(new Error("postgresql://user:database-secret@localhost:5432/app failed with token abc123"), {
+            code: "P1001",
+            stack: "secret stack trace",
+          })
+        ),
+        redisProbe: vi.fn(),
+      }
+    );
+    const output = readOutput();
+    const report = JSON.parse(output);
+
+    expect(exitCode).toBe(1);
+    expect(report).toMatchObject({
+      databaseConfigured: true,
+      databaseErrorCode: "P1001",
+      databaseStatus: "misconfigured",
+      status: "misconfigured",
+    });
+    expect(output).not.toContain("database-secret");
+    expect(output).not.toContain("abc123");
+    expect(output).not.toContain("postgresql://user:database-secret@localhost:5432/app");
+    expect(output).not.toContain("secret stack trace");
+  });
+
+  test("returns non-zero with redacted Redis code when Redis readiness fails", async () => {
+    const { io, readOutput } = createIo();
+    const exitCode = await runWorkerHealthCli(
+      {
+        BULLMQ_REDIS_URL: "redis://user:redis-secret@localhost:6379/0",
+        DATABASE_URL: "postgresql://user:db-secret@localhost:5432/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      io,
+      {
+        databaseProbe: vi.fn(),
+        redisProbe: vi.fn().mockRejectedValue(Object.assign(new Error("redis://user:redis-secret@localhost failed"), { code: "ECONNREFUSED" })),
+      }
+    );
+    const output = readOutput();
+    const report = JSON.parse(output);
+
+    expect(exitCode).toBe(1);
+    expect(report).toMatchObject({
+      redisErrorCode: "ECONNREFUSED",
+      redisStatus: "misconfigured",
+      status: "misconfigured",
+    });
+    expect(output).not.toContain("redis-secret");
+    expect(output).not.toContain("db-secret");
+  });
+
+  test("does not leak password-bearing Redis URLs in JSON output", async () => {
+    const { io, readOutput } = createIo();
+    const exitCode = await runWorkerHealthCli(
+      {
+        BULLMQ_REDIS_URL: "redis://user:super-secret@localhost:6379/0",
+        DATABASE_URL: "postgresql://user:db-secret@localhost:5432/app",
+        ...mailjetEnv,
+        WORKER_ENABLED: "true",
+        WORKER_QUEUE_GROUP: "general",
+      },
+      io,
+      { databaseProbe: vi.fn(), redisProbe: vi.fn() }
+    );
+
+    expect(exitCode).toBe(0);
+    expect(readOutput()).not.toContain("super-secret");
+    expect(readOutput()).not.toContain("db-secret");
+    expect(readOutput()).not.toContain("redis://user:super-secret@localhost:6379/0");
+    expect(JSON.parse(readOutput())).toMatchObject({ databaseStatus: "ready", redisSource: "BULLMQ_REDIS_URL", redisStatus: "ready", status: "ready" });
+  });
+});

hivegrid-cloud/apps/worker/src/health.ts

@@ -0,0 +1,461 @@
+import { loadWorkerConfig, type EnvRecord, type WorkerConfigDiagnostic, type WorkerConfigRedisParseStatus } from "@toolctrl/config";
+import { getDbClient } from "@toolctrl/db/client.server";
+import { isWorkerQueueGroup, parseRedisUrl, type QueueName, type WorkerQueueGroup } from "@toolctrl/queue";
+import { Queue } from "bullmq";
+import { getWorkerQueueNames } from "./queues.js";
+import { getMissingWorkerStorageEnv } from "./storage.js";
+
+export type WorkerHealthEnv = EnvRecord;
+
+export type WorkerHealthStatus = "disabled" | "ready" | "misconfigured";
+export type WorkerHealthRedisSource = "BULLMQ_REDIS_URL" | "REDIS_URL" | "none";
+
+export type WorkerHealthDatabaseStatus = "skipped" | "ready" | "misconfigured";
+export type WorkerHealthRedisStatus = "skipped" | "ready" | "misconfigured";
+export type WorkerHealthStorageStatus = "skipped" | "ready" | "misconfigured";
+export type WorkerHealthImportStatus = "skipped" | "ready" | "misconfigured";
+export type WorkerHealthMailStatus = "skipped" | "ready" | "misconfigured";
+export type WorkerHealthPluginRuntimeStatus = "skipped" | "ready" | "misconfigured";
+export type WorkerHealthRequiredModeStatus = "skipped" | "ready" | "misconfigured";
+
+export type WorkerHealthReport = {
+  status: WorkerHealthStatus;
+  revision: string;
+  workerEnabled: boolean;
+  workerGroup: WorkerQueueGroup | null;
+  redisConfigured: boolean;
+  redisSource: WorkerHealthRedisSource;
+  redisParseStatus: "not_configured" | "valid" | "invalid";
+  redisStatus: WorkerHealthRedisStatus;
+  redisErrorCode?: string;
+  databaseConfigured: boolean;
+  databaseStatus: WorkerHealthDatabaseStatus;
+  databaseErrorCode?: string;
+  storageConfigured: boolean;
+  storageStatus: WorkerHealthStorageStatus;
+  mailjetConfigured: boolean;
+  mailjetStatus: WorkerHealthMailStatus;
+  importConfigured: boolean;
+  importStatus: WorkerHealthImportStatus;
+  pluginRuntimeConfigured: boolean;
+  pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus;
+  queues: readonly QueueName[];
+  errors: readonly string[];
+};
+
+export type WorkerHealthCliIo = {
+  stderr?: Pick<NodeJS.WriteStream, "write">;
+  stdout: Pick<NodeJS.WriteStream, "write">;
+};
+
+export type WorkerDatabaseProbe = () => Promise<void> | void;
+export type WorkerRedisProbe = (redisUrl: string) => Promise<void> | void;
+
+export type WorkerHealthDependencies = {
+  databaseProbe?: WorkerDatabaseProbe;
+  redisProbe?: WorkerRedisProbe;
+};
+
+export async function buildWorkerHealthReport(
+  env: WorkerHealthEnv = process.env as WorkerHealthEnv,
+  dependencies: WorkerHealthDependencies = {}
+): Promise<WorkerHealthReport> {
+  const result = loadWorkerConfig(env);
+  const { config } = result;
+  const shouldProbeDatabase = config.enabled && result.ok;
+  const shouldProbeRedis = config.enabled && result.ok;
+  const databaseConfigured = Boolean(env.DATABASE_URL);
+  const [redisReadiness, databaseReadiness] = await Promise.all([
+    checkRedisReadiness(shouldProbeRedis, config.redis.url, dependencies.redisProbe),
+    checkDatabaseReadiness(shouldProbeDatabase, dependencies.databaseProbe),
+  ]);
+  const storageReadiness = checkStorageReadiness(config.enabled && env.EXPORT_RETENTION_QUEUE_ENABLED === "true", env);
+  const mailReadiness = checkMailReadiness(config.enabled, config.mailjet.configured, isMailProviderRequired(env), env);
+  const importReadiness = checkImportReadiness(config.enabled && result.ok && getWorkerGroup(env) === "import", env);
+  const pluginRuntimeReadiness = checkPluginRuntimeReadiness(config.enabled && result.ok, getWorkerGroup(env), env);
+  const requiredModeReadiness = checkRequiredGeneralWorkerReadiness(config.enabled && result.ok && getWorkerGroup(env) === "general", env);
+  const status = toWorkerHealthStatus(
+    config.enabled,
+    result.ok,
+    redisReadiness.status,
+    databaseReadiness.status,
+    storageReadiness.status,
+    mailReadiness.status,
+    importReadiness.status,
+    pluginRuntimeReadiness.status,
+    requiredModeReadiness.status
+  );
+
+  return {
+    status,
+    revision: env.APP_REVISION || "unknown",
+    workerEnabled: config.enabled,
+    workerGroup: getWorkerGroup(env),
+    redisConfigured: config.redis.configured,
+    redisSource: config.redis.source,
+    redisParseStatus: toHealthRedisParseStatus(config.redis.parseStatus),
+    redisStatus: redisReadiness.status,
+    ...(redisReadiness.errorCode ? { redisErrorCode: redisReadiness.errorCode } : {}),
+    databaseConfigured,
+    databaseStatus: databaseReadiness.status,
+    ...(databaseReadiness.errorCode ? { databaseErrorCode: databaseReadiness.errorCode } : {}),
+    storageConfigured: storageReadiness.configured,
+    storageStatus: storageReadiness.status,
+    mailjetConfigured: mailReadiness.configured,
+    mailjetStatus: mailReadiness.status,
+    importConfigured: importReadiness.configured,
+    importStatus: importReadiness.status,
+    pluginRuntimeConfigured: pluginRuntimeReadiness.configured,
+    pluginRuntimeStatus: pluginRuntimeReadiness.status,
+    queues: getWorkerQueueNames(env),
+    errors: [
+      ...result.diagnostics.map(formatWorkerConfigDiagnostic),
+      ...storageReadiness.errors,
+      ...mailReadiness.errors,
+      ...importReadiness.errors,
+      ...pluginRuntimeReadiness.errors,
+      ...requiredModeReadiness.errors,
+    ],
+  };
+}
+
+function getWorkerGroup(env: WorkerHealthEnv): WorkerQueueGroup | null {
+  const group = env.WORKER_QUEUE_GROUP;
+  return typeof group === "string" && isWorkerQueueGroup(group) ? group : null;
+}
+
+export async function runWorkerHealthCli(
+  env: WorkerHealthEnv = process.env as WorkerHealthEnv,
+  io: WorkerHealthCliIo = process,
+  dependencies: WorkerHealthDependencies = {}
+): Promise<number> {
+  const report = await buildWorkerHealthReport(env, dependencies);
+  io.stdout.write(`${JSON.stringify(report)}\n`);
+  return report.status === "misconfigured" ? 1 : 0;
+}
+
+function toWorkerHealthStatus(
+  workerEnabled: boolean,
+  redisReady: boolean,
+  redisStatus: WorkerHealthRedisStatus,
+  databaseStatus: WorkerHealthDatabaseStatus,
+  storageStatus: WorkerHealthStorageStatus,
+  mailStatus: WorkerHealthMailStatus,
+  importStatus: WorkerHealthImportStatus,
+  pluginRuntimeStatus: WorkerHealthPluginRuntimeStatus,
+  requiredModeStatus: WorkerHealthRequiredModeStatus
+): WorkerHealthStatus {
+  if (!workerEnabled) {
+    return "disabled";
+  }
+
+  if (
+    !redisReady ||
+    redisStatus === "misconfigured" ||
+    databaseStatus === "misconfigured" ||
+    storageStatus === "misconfigured" ||
+    mailStatus === "misconfigured" ||
+    importStatus === "misconfigured" ||
+    pluginRuntimeStatus === "misconfigured" ||
+    requiredModeStatus === "misconfigured"
+  ) {
+    return "misconfigured";
+  }
+
+  return "ready";
+}
+
+function checkPluginRuntimeReadiness(
+  shouldCheck: boolean,
+  workerGroup: WorkerQueueGroup | null,
+  env: WorkerHealthEnv
+): { configured: boolean; errors: string[]; status: WorkerHealthPluginRuntimeStatus } {
+  const productionRuntimeEnabled = env.PLUGIN_RUNTIME_PRODUCTION_ENABLED === "true";
+  if (!shouldCheck || !productionRuntimeEnabled || (workerGroup !== "general" && workerGroup !== "plugin")) {
+    return { configured: false, errors: [], status: "skipped" };
+  }
+
+  const errors: string[] = [];
+  if (!hasTrimmedValue(env.PLUGIN_TRUSTED_PUBLIC_KEYS)) {
+    errors.push("PLUGIN_TRUSTED_PUBLIC_KEYS:missing-plugin-trust-root:Required when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true.");
+  }
+
+  if (workerGroup === "general") {
+    if (!hasTrimmedValue(env.BULLMQ_REDIS_URL) && !hasTrimmedValue(env.REDIS_URL)) {
+      errors.push(
+        "BULLMQ_REDIS_URL:missing-plugin-runtime-redis:Required for plugin hook fanout and capability-use subscribers when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
+      );
+    }
+    if (env.OUTBOX_DISPATCHER_ENABLED !== "true") {
+      errors.push(
+        "OUTBOX_DISPATCHER_ENABLED:missing-plugin-runtime-dispatcher:Required on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
+      );
+    }
+    if (env.PLUGIN_HOOK_FANOUT_ENABLED === "false") {
+      errors.push(
+        "PLUGIN_HOOK_FANOUT_ENABLED:disabled-plugin-runtime-fanout:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
+      );
+    }
+    if (env.PLUGIN_CAPABILITY_USE_ENABLED === "false") {
+      errors.push(
+        "PLUGIN_CAPABILITY_USE_ENABLED:disabled-plugin-capability-use:Must not be false on WORKER_QUEUE_GROUP=general when PLUGIN_RUNTIME_PRODUCTION_ENABLED=true."
+      );
+    }
+  }
+
+  return { configured: true, errors, status: errors.length > 0 ? "misconfigured" : "ready" };
+}
+
+function isMailProviderRequired(env: WorkerHealthEnv): boolean {
+  return env.WORKER_QUEUE_GROUP === "mail" && env.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED === "true";
+}
+
+function checkRequiredGeneralWorkerReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { errors: string[]; status: WorkerHealthRequiredModeStatus } {
+  if (!shouldCheck) {
+    return { errors: [], status: "skipped" };
+  }
+
+  const errors: string[] = [];
+
+  if (env.OUTBOX_DISPATCHER_REQUIRED === "true" && env.OUTBOX_DISPATCHER_ENABLED !== "true") {
+    errors.push("OUTBOX_DISPATCHER_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and OUTBOX_DISPATCHER_REQUIRED=true.");
+  }
+
+  const workflowTriggerRequired = env.WORKFLOW_TRIGGER_OUTBOX_REQUIRED === "true";
+  const workflowTriggerEnabled = env.WORKFLOW_TRIGGER_OUTBOX_ENABLED === "true";
+  const workflowTriggerModeLabel = workflowTriggerRequired ? "WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true" : "WORKFLOW_TRIGGER_OUTBOX_ENABLED=true";
+
+  if (workflowTriggerRequired || workflowTriggerEnabled) {
+    if (env.OUTBOX_DISPATCHER_ENABLED !== "true" && env.OUTBOX_DISPATCHER_REQUIRED !== "true") {
+      errors.push(`OUTBOX_DISPATCHER_ENABLED:missing-workflow-trigger-dispatcher:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`);
+    }
+
+    if (workflowTriggerRequired && !workflowTriggerEnabled) {
+      errors.push(
+        "WORKFLOW_TRIGGER_OUTBOX_ENABLED:missing-required-general-worker:Required when WORKER_QUEUE_GROUP=general and WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true."
+      );
+    }
+
+    const executorMode = parseWorkflowTriggerExecutorModeForHealth(env.WORKFLOW_TRIGGER_EXECUTOR_MODE);
+    if (!executorMode) {
+      errors.push(
+        `WORKFLOW_TRIGGER_EXECUTOR_MODE:invalid-workflow-executor-mode:Expected http, native, or hybrid when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel}.`
+      );
+    } else if (workflowTriggerExecutorNeedsHttpBridge(executorMode)) {
+      if (!hasTrimmedValue(env.API_ACCESS_TOKEN)) {
+        errors.push(
+          `API_ACCESS_TOKEN:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
+        );
+      }
+
+      if (!hasTrimmedValue(env.WORKFLOW_TRIGGER_EXECUTOR_URL) && !hasTrimmedValue(env.APP_URL) && !hasTrimmedValue(env.WEB_ORIGIN)) {
+        errors.push(
+          `WORKFLOW_TRIGGER_EXECUTOR_URL:missing-workflow-executor-bridge:Required when WORKER_QUEUE_GROUP=general and ${workflowTriggerModeLabel} with http or hybrid executor mode.`
+        );
+      }
+    }
+  }
+
+  return { errors, status: errors.length > 0 ? "misconfigured" : "ready" };
+}
+
+function parseWorkflowTriggerExecutorModeForHealth(value: unknown): "http" | "native" | "hybrid" | null {
+  if (typeof value !== "string" || value.trim().length === 0) {
+    return "http";
+  }
+  const normalized = value.trim().toLowerCase();
+  return normalized === "http" || normalized === "native" || normalized === "hybrid" ? normalized : null;
+}
+
+function workflowTriggerExecutorNeedsHttpBridge(mode: "http" | "native" | "hybrid"): boolean {
+  return mode !== "native";
+}
+
+function hasTrimmedValue(value: string | undefined): boolean {
+  return typeof value === "string" && value.trim().length > 0;
+}
+
+function checkMailReadiness(
+  shouldCheck: boolean,
+  configured: boolean,
+  required: boolean,
+  env: WorkerHealthEnv
+): { configured: boolean; errors: string[]; status: WorkerHealthMailStatus } {
+  if (!shouldCheck) {
+    return { configured: false, errors: [], status: "skipped" };
+  }
+
+  if (!configured) {
+    if (required) {
+      const missing = ["MAILJET_API_KEY", "MAILJET_SECRET_KEY", "MAILJET_FROM_EMAIL"].filter((name) => !env[name]?.trim());
+      return {
+        configured: false,
+        errors: missing.map(
+          (name) => `${name}:missing-mail-provider-config:Required when WORKER_QUEUE_GROUP=mail and TRANSACTIONAL_MAIL_OUTBOX_REQUIRED=true.`
+        ),
+        status: "misconfigured",
+      };
+    }
+    return { configured: false, errors: [], status: "skipped" };
+  }
+
+  return { configured: true, errors: [], status: "ready" };
+}
+
+function checkImportReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthImportStatus } {
+  if (!shouldCheck) {
+    return { configured: false, errors: [], status: "skipped" };
+  }
+
+  const importEnabled = env.IMPORT_QUEUE_ENABLED === "true";
+  const runtimeReady = env.IMPORT_QUEUE_RUNTIME_READY === "true";
+  if (!importEnabled && !runtimeReady) {
+    return { configured: false, errors: [], status: "skipped" };
+  }
+
+  if (!importEnabled || !runtimeReady) {
+    const errors: string[] = [];
+    if (!importEnabled) {
+      errors.push(
+        "IMPORT_QUEUE_ENABLED:missing-import-runtime-gate:Required with IMPORT_QUEUE_RUNTIME_READY=true before WORKER_QUEUE_GROUP=import can consume toolctrl.imports/toolctrl.exports."
+      );
+    }
+    if (!runtimeReady) {
+      errors.push(
+        "IMPORT_QUEUE_RUNTIME_READY:missing-import-runtime-gate:Required with IMPORT_QUEUE_ENABLED=true after import, export, and scan services are wired."
+      );
+    }
+    return {
+      configured: false,
+      errors,
+      status: "misconfigured",
+    };
+  }
+
+  return { configured: true, errors: [], status: "ready" };
+}
+
+function checkStorageReadiness(shouldCheck: boolean, env: WorkerHealthEnv): { configured: boolean; errors: string[]; status: WorkerHealthStorageStatus } {
+  if (!shouldCheck) {
+    return { configured: false, errors: [], status: "skipped" };
+  }
+
+  const missing = getMissingWorkerStorageEnv(env);
+  if (missing.length > 0) {
+    return {
+      configured: false,
+      errors: missing.map((name) => `${name}:missing-storage-config:Required when EXPORT_RETENTION_QUEUE_ENABLED=true.`),
+      status: "misconfigured",
+    };
+  }
+
+  return { configured: true, errors: [], status: "ready" };
+}
+
+async function checkRedisReadiness(
+  shouldProbe: boolean,
+  redisUrl: string | undefined,
+  redisProbe: WorkerRedisProbe = defaultRedisProbe
+): Promise<{ status: WorkerHealthRedisStatus; errorCode?: string }> {
+  if (!shouldProbe || !redisUrl) {
+    return { status: "skipped" };
+  }
+
+  try {
+    await redisProbe(redisUrl);
+    return { status: "ready" };
+  } catch (error) {
+    return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error, "REDIS_READINESS_FAILED") };
+  }
+}
+
+async function checkDatabaseReadiness(
+  shouldProbe: boolean,
+  databaseProbe: WorkerDatabaseProbe = defaultDatabaseProbe
+): Promise<{ status: WorkerHealthDatabaseStatus; errorCode?: string }> {
+  if (!shouldProbe) {
+    return { status: "skipped" };
+  }
+
+  try {
+    await databaseProbe();
+    return { status: "ready" };
+  } catch (error) {
+    return { status: "misconfigured", errorCode: toRedactedDependencyErrorCode(error) };
+  }
+}
+
+async function defaultDatabaseProbe(): Promise<void> {
+  const db = getDbClient();
+
+  try {
+    await db.$queryRaw`SELECT 1`;
+  } finally {
+    await db.$disconnect();
+  }
+}
+
+async function defaultRedisProbe(redisUrl: string): Promise<void> {
+  const queue = new Queue("__toolctrl_worker_health__", {
+    connection: parseRedisUrl(redisUrl),
+  });
+
+  try {
+    await withTimeout(queue.waitUntilReady(), 2_000);
+  } finally {
+    await queue.close();
+  }
+}
+
+function toHealthRedisParseStatus(status: WorkerConfigRedisParseStatus): WorkerHealthReport["redisParseStatus"] {
+  if (status === "valid") {
+    return "valid";
+  }
+
+  if (status === "not-provided") {
+    return "not_configured";
+  }
+
+  return "invalid";
+}
+
+function toRedactedDependencyErrorCode(error: unknown, fallback = "DATABASE_READINESS_FAILED"): string {
+  if (typeof error === "object" && error !== null && "code" in error) {
+    const code = (error as { code?: unknown }).code;
+    if (typeof code === "string" && /^[A-Z0-9_-]{1,64}$/i.test(code)) {
+      return code;
+    }
+  }
+
+  return fallback;
+}
+
+async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  let timeout: NodeJS.Timeout | undefined;
+  try {
+    return await Promise.race([
+      promise,
+      new Promise<never>((_, reject) => {
+        timeout = setTimeout(() => reject(Object.assign(new Error("Redis readiness timed out."), { code: "REDIS_TIMEOUT" })), timeoutMs);
+      }),
+    ]);
+  } finally {
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+  }
+}
+
+function formatWorkerConfigDiagnostic(diagnostic: WorkerConfigDiagnostic): string {
+  return `${diagnostic.name}:${diagnostic.code}:${diagnostic.message}`;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  runWorkerHealthCli()
+    .then((exitCode) => {
+      process.exitCode = exitCode;
+    })
+    .catch(() => {
+      process.exitCode = 1;
+    });
+}

hivegrid-cloud/apps/worker/src/jobs/audit-event-persist-bootstrap.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  bootstrapAuditEventPersist,
+  type AuditEventPersistBootstrapDeps,
+  type AuditEventPersistWorkerHandle,
+} from "./audit-event-persist-bootstrap.js";
+
+type WorkerImpl = NonNullable<AuditEventPersistBootstrapDeps["createWorkerImpl"]>;
+
+function fakeWorker(): AuditEventPersistWorkerHandle {
+  return { name: "outbox.audit", async close() {} };
+}
+
+describe("bootstrapAuditEventPersist", () => {
+  it("disables itself when WORKER_QUEUE_GROUP is not general", () => {
+    const result = bootstrapAuditEventPersist({ WORKER_QUEUE_GROUP: "mail" }, { runInTransaction: async (fn) => fn({} as never) });
+    expect(result).toEqual({ status: "disabled", reason: "non-general worker group" });
+  });
+
+  it("disables itself when AUDIT_EVENT_PERSIST_ENABLED=false", () => {
+    const result = bootstrapAuditEventPersist(
+      { WORKER_QUEUE_GROUP: "general", AUDIT_EVENT_PERSIST_ENABLED: "false" },
+      { runInTransaction: async (fn) => fn({} as never) }
+    );
+    expect(result).toEqual({ status: "disabled", reason: "AUDIT_EVENT_PERSIST_ENABLED=false" });
+  });
+
+  it("disables itself when no transaction runner is wired", () => {
+    const result = bootstrapAuditEventPersist({ WORKER_QUEUE_GROUP: "general" }, {});
+    expect(result.status).toBe("disabled");
+    if (result.status === "disabled") {
+      expect(result.reason).toContain("no transaction runner wired");
+    }
+  });
+
+  it("subscribes a worker to outbox.audit with the default concurrency", () => {
+    const inner = vi.fn(() => fakeWorker());
+    const createWorkerImpl = inner as unknown as WorkerImpl;
+    const result = bootstrapAuditEventPersist(
+      { WORKER_QUEUE_GROUP: "general" },
+      { runInTransaction: async (fn) => fn({} as never), createWorkerImpl }
+    );
+    expect(result.status).toBe("started");
+    expect(inner).toHaveBeenCalledTimes(1);
+    const args = inner.mock.calls[0] as unknown as [string, unknown, { concurrency?: number }];
+    expect(args[0]).toBe("outbox.audit");
+    expect(args[2]).toEqual({ concurrency: 5 });
+  });
+
+  it("honours AUDIT_EVENT_PERSIST_CONCURRENCY override", () => {
+    const inner = vi.fn(() => fakeWorker());
+    const createWorkerImpl = inner as unknown as WorkerImpl;
+    bootstrapAuditEventPersist(
+      { WORKER_QUEUE_GROUP: "general", AUDIT_EVENT_PERSIST_CONCURRENCY: "12" },
+      { runInTransaction: async (fn) => fn({} as never), createWorkerImpl }
+    );
+    const args = inner.mock.calls[0] as unknown as [string, unknown, { concurrency?: number }];
+    expect(args[2]).toEqual({ concurrency: 12 });
+  });
+
+  it("invokes the underlying processor when BullMQ delivers a job", async () => {
+    let captured: ((job: { data: unknown }) => Promise<unknown>) | null = null;
+    const inner = vi.fn((_queue: unknown, processor: unknown) => {
+      captured = processor as (job: { data: unknown }) => Promise<unknown>;
+      return fakeWorker();
+    });
+    const createWorkerImpl = inner as unknown as WorkerImpl;
+    const runInTransaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) =>
+      fn({
+        auditEvent: {
+          findFirst: async () => null,
+          create: async () => ({ id: "audit_1", eventHash: "h1" }),
+        },
+      })
+    );
+    bootstrapAuditEventPersist({ WORKER_QUEUE_GROUP: "general" }, { runInTransaction: runInTransaction as never, createWorkerImpl });
+
+    expect(captured).not.toBeNull();
+    if (captured) {
+      await (captured as (job: { data: unknown }) => Promise<unknown>)({
+        data: {
+          outboxEventId: "outbox_1",
+          tenantId: "tenant_1",
+          eventType: "audit.event.create",
+          payload: { action: "x.do", actorType: "system" },
+        },
+      });
+    }
+    expect(runInTransaction).toHaveBeenCalledTimes(1);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/audit-event-persist-bootstrap.ts

@@ -0,0 +1,105 @@
+/**
+ * Bootstrap helper for the audit-event outbox subscriber.
+ *
+ * Subscribes a BullMQ worker to the `outbox.audit` queue inside the
+ * `general` worker group. Each delivered job carries an
+ * `audit.event.create` outbox payload; the handler inserts the
+ * AuditEvent with a freshly-computed chain hash inside a serializable
+ * Prisma transaction (see runAuditEventPersistJob).
+ *
+ * Gating:
+ *   - WORKER_QUEUE_GROUP === "general" (the audit subscriber co-locates
+ *     with the outbox dispatcher to keep DB connections concentrated)
+ *   - AUDIT_EVENT_PERSIST_ENABLED !== "false" (escape hatch)
+ *
+ * Tests inject a fake `createWorkerImpl` to assert subscription
+ * semantics without spinning up BullMQ.
+ */
+
+import type { EnvRecord } from "@toolctrl/config";
+
+import {
+  AUDIT_EVENT_OUTBOX_QUEUE,
+  runAuditEventPersistJob,
+  type AuditEventPersistDependencies,
+  type AuditEventPersistLogger,
+  type AuditEventPersistTransactionRunner,
+} from "./audit-event-persist.js";
+
+export type AuditEventPersistBootstrapEnv = EnvRecord & {
+  WORKER_QUEUE_GROUP?: string;
+  AUDIT_EVENT_PERSIST_ENABLED?: string;
+  AUDIT_EVENT_PERSIST_CONCURRENCY?: string;
+};
+
+export interface AuditEventPersistWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface AuditEventPersistBootstrapDeps {
+  readonly logger?: AuditEventPersistLogger;
+  /** Override the transaction runner. Production wires Prisma $transaction. */
+  readonly runInTransaction?: AuditEventPersistTransactionRunner;
+  /** BullMQ worker factory. Production builds a real Worker; tests fake it. */
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number }
+  ) => AuditEventPersistWorkerHandle;
+}
+
+export type AuditEventPersistBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: AuditEventPersistWorkerHandle };
+
+const DEFAULT_CONCURRENCY = 5;
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export function bootstrapAuditEventPersist(
+  env: AuditEventPersistBootstrapEnv,
+  deps: AuditEventPersistBootstrapDeps = {}
+): AuditEventPersistBootstrapResult {
+  const logger = deps.logger;
+
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
+    return { status: "disabled", reason: "non-general worker group" };
+  }
+  if (env.AUDIT_EVENT_PERSIST_ENABLED === "false") {
+    return { status: "disabled", reason: "AUDIT_EVENT_PERSIST_ENABLED=false" };
+  }
+  if (!deps.runInTransaction) {
+    return { status: "disabled", reason: "no transaction runner wired (worker host must supply prisma.$transaction)" };
+  }
+
+  const runInTransaction = deps.runInTransaction;
+  const concurrency = parsePositiveInt(env.AUDIT_EVENT_PERSIST_CONCURRENCY, DEFAULT_CONCURRENCY);
+
+  const processor = async (job: { data: unknown }): Promise<unknown> => {
+    const handlerDeps: AuditEventPersistDependencies = {
+      runInTransaction,
+      ...(logger ? { logger } : {}),
+    };
+    return runAuditEventPersistJob(job.data, handlerDeps);
+  };
+
+  const createWorkerImpl = deps.createWorkerImpl ?? defaultCreateWorker;
+  const handle = createWorkerImpl(AUDIT_EVENT_OUTBOX_QUEUE, processor, { concurrency });
+
+  logger?.info?.(
+    { component: "audit-event-persist", queue: AUDIT_EVENT_OUTBOX_QUEUE, concurrency },
+    "audit-event-persist worker started"
+  );
+  return { status: "started", handle };
+}
+
+const defaultCreateWorker: NonNullable<AuditEventPersistBootstrapDeps["createWorkerImpl"]> = () => {
+  throw new Error(
+    "bootstrapAuditEventPersist requires deps.createWorkerImpl in this build; the worker entry must wire the BullMQ Worker constructor."
+  );
+};

hivegrid-cloud/apps/worker/src/jobs/audit-event-persist.test.ts

@@ -0,0 +1,218 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { computeAuditHash } from "@toolctrl/db/audit-hash";
+import {
+  AUDIT_EVENT_OUTBOX_QUEUE,
+  AuditEventPersistJobError,
+  parseAuditEventPersistJob,
+  runAuditEventPersistJob,
+  type AuditEventPersistTransactionRunner,
+  type AuditEventPersistTxClient,
+} from "./audit-event-persist.js";
+
+type PersistedRow = {
+  id: string;
+  tenantId: string | null;
+  action: string;
+  eventHash: string;
+  previousHash: string | null;
+  createdAt: Date;
+};
+
+function makeStore(): {
+  rows: PersistedRow[];
+  runInTransaction: AuditEventPersistTransactionRunner;
+} {
+  const rows: PersistedRow[] = [];
+  let idSeq = 0;
+  let chainSeq = 0;
+  const tx: AuditEventPersistTxClient = {
+    auditEvent: {
+      async findFirst({ where }) {
+        const tenantRows = rows.filter((r) => r.tenantId === where.tenantId && r.eventHash !== null);
+        if (tenantRows.length === 0) return null;
+        // newest first
+        tenantRows.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
+        return { eventHash: tenantRows[0]?.eventHash ?? null };
+      },
+      async create({ data }) {
+        idSeq += 1;
+        chainSeq += 1;
+        const row: PersistedRow = {
+          id: `audit_${idSeq}`,
+          tenantId: data.tenantId,
+          action: data.action,
+          eventHash: data.eventHash,
+          previousHash: data.previousHash,
+          createdAt: new Date(2026, 4, 17, 0, 0, 0, chainSeq),
+        };
+        rows.push(row);
+        return { id: row.id, eventHash: row.eventHash };
+      },
+    },
+  };
+  const runInTransaction: AuditEventPersistTransactionRunner = async <T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T> => fn(tx);
+  return { rows, runInTransaction };
+}
+
+const baseJob = {
+  outboxEventId: "outbox_1",
+  tenantId: "tenant_1",
+  eventType: "audit.event.create",
+  payload: {
+    tenantId: "tenant_1",
+    actorId: "user_1",
+    actorType: "user",
+    actorEmail: null,
+    actorIp: null,
+    actorUserAgent: null,
+    action: "widget.created",
+    targetType: "widget",
+    targetId: "widget_1",
+    requestId: null,
+    sourceModule: "crm",
+    outcome: "success",
+    dataSensitivity: "internal",
+    regulatoryScope: ["soc2"],
+    metadata: null,
+  },
+};
+
+describe("AUDIT_EVENT_OUTBOX_QUEUE", () => {
+  it("matches the outbox audit queue name", () => {
+    expect(AUDIT_EVENT_OUTBOX_QUEUE).toBe("outbox.audit");
+  });
+});
+
+describe("parseAuditEventPersistJob", () => {
+  it("throws on missing data envelope", () => {
+    expect(() => parseAuditEventPersistJob(null)).toThrowError(AuditEventPersistJobError);
+  });
+
+  it("throws on wrong eventType", () => {
+    expect(() => parseAuditEventPersistJob({ eventType: "marketing.lead.persist", payload: { action: "x" } })).toThrowError(AuditEventPersistJobError);
+  });
+
+  it("throws on missing payload", () => {
+    expect(() => parseAuditEventPersistJob({ eventType: "audit.event.create" })).toThrowError(AuditEventPersistJobError);
+  });
+
+  it("throws on missing action", () => {
+    expect(() =>
+      parseAuditEventPersistJob({ eventType: "audit.event.create", payload: { actorType: "user" } })
+    ).toThrowError(AuditEventPersistJobError);
+  });
+
+  it("normalizes payload through buildAuditEventCreateData", () => {
+    const parsed = parseAuditEventPersistJob({
+      eventType: "audit.event.create",
+      tenantId: "tenant_1",
+      payload: { action: "x.do", actorType: "  ", regulatoryScope: ["SOC2"] },
+    });
+    expect(parsed.payload.actorType).toBe("system");
+    expect(parsed.payload.regulatoryScope).toEqual(["soc2"]);
+    expect(parsed.payload.outcome).toBe("success");
+    expect(parsed.payload.dataSensitivity).toBe("internal");
+  });
+});
+
+describe("runAuditEventPersistJob", () => {
+  it("persists the event with prevHash=null on chain root", async () => {
+    const { rows, runInTransaction } = makeStore();
+    const result = await runAuditEventPersistJob(baseJob, { runInTransaction });
+    expect(result).toEqual({ persisted: true, auditEventId: "audit_1", eventHash: expect.stringMatching(/^[0-9a-f]{64}$/) });
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.previousHash).toBeNull();
+  });
+
+  it("links the chain when a prior event exists for the same tenant", async () => {
+    const { rows, runInTransaction } = makeStore();
+    await runAuditEventPersistJob(baseJob, { runInTransaction });
+    const second = await runAuditEventPersistJob(
+      { ...baseJob, payload: { ...baseJob.payload, action: "widget.updated" } },
+      { runInTransaction }
+    );
+    expect(rows).toHaveLength(2);
+    expect(rows[1]?.previousHash).toBe(rows[0]?.eventHash);
+    expect(second.eventHash).toBe(rows[1]?.eventHash);
+  });
+
+  it("isolates chains per tenant", async () => {
+    const { rows, runInTransaction } = makeStore();
+    await runAuditEventPersistJob(baseJob, { runInTransaction });
+    await runAuditEventPersistJob(
+      { ...baseJob, tenantId: "tenant_2", payload: { ...baseJob.payload, tenantId: "tenant_2", action: "x.do" } },
+      { runInTransaction }
+    );
+    expect(rows[0]?.tenantId).toBe("tenant_1");
+    expect(rows[1]?.tenantId).toBe("tenant_2");
+    expect(rows[1]?.previousHash).toBeNull();
+  });
+
+  it("computes an eventHash consistent with computeAuditHash", async () => {
+    const { rows, runInTransaction } = makeStore();
+    await runAuditEventPersistJob(baseJob, { runInTransaction, secret: "test-secret" });
+    const recomputed = computeAuditHash({
+      previousHash: null,
+      payload: {
+        id: "",
+        createdAt: "",
+        tenantId: "tenant_1",
+        actorType: "user",
+        actorId: "user_1",
+        actorEmail: null,
+        actorIp: null,
+        actorUserAgent: null,
+        action: "widget.created",
+        targetType: "widget",
+        targetId: "widget_1",
+        requestId: null,
+        sourceModule: "crm",
+        outcome: "success",
+        dataSensitivity: "internal",
+        regulatoryScope: ["soc2"],
+        metadata: null,
+      },
+      secret: "test-secret",
+    });
+    expect(rows[0]?.eventHash).toBe(recomputed);
+  });
+
+  it("handles concurrent persists for the same tenant without breaking the chain when the tx runner serializes", async () => {
+    // The production runner sets isolationLevel="Serializable". This
+    // test simulates that by serializing the underlying mock store
+    // through a single chained Promise. Each call sees the previously
+    // committed row, so the chain stays consistent under parallel
+    // dispatch.
+    const { rows, runInTransaction: serialRunner } = makeStore();
+    let gate: Promise<void> = Promise.resolve();
+    const queuedRunner: AuditEventPersistTransactionRunner = async <T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T> => {
+      const prev = gate;
+      let release!: () => void;
+      gate = new Promise<void>((r) => (release = r));
+      await prev;
+      try {
+        return await serialRunner(fn);
+      } finally {
+        release();
+      }
+    };
+    const logger = { error: vi.fn() };
+    await Promise.all([
+      runAuditEventPersistJob(baseJob, { runInTransaction: queuedRunner, logger }),
+      runAuditEventPersistJob(
+        { ...baseJob, payload: { ...baseJob.payload, action: "widget.updated" } },
+        { runInTransaction: queuedRunner, logger }
+      ),
+      runAuditEventPersistJob(
+        { ...baseJob, payload: { ...baseJob.payload, action: "widget.deleted" } },
+        { runInTransaction: queuedRunner, logger }
+      ),
+    ]);
+    expect(rows).toHaveLength(3);
+    expect(rows[0]?.previousHash).toBeNull();
+    expect(rows[1]?.previousHash).toBe(rows[0]?.eventHash);
+    expect(rows[2]?.previousHash).toBe(rows[1]?.eventHash);
+    expect(logger.error).not.toHaveBeenCalled();
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/audit-event-persist.ts

@@ -0,0 +1,188 @@
+/**
+ * Phase 10 Task #67 — Audit-event outbox handler.
+ *
+ * Subscribes to the `outbox.audit` BullMQ queue (populated by the
+ * outbox dispatcher in apps/worker/src/outbox/dispatcher.ts) and
+ * inserts the AuditEvent row into Postgres.
+ *
+ * The hash chain is computed inside a short Prisma transaction:
+ *   - SELECT the most-recent eventHash for the tenant (FOR UPDATE
+ *     equivalent via serializable isolation)
+ *   - compute the new event's eventHash
+ *   - INSERT the new row
+ *
+ * Running this under serializable isolation ensures concurrent persists
+ * for the same tenant either commit in some order with a coherent
+ * chain, or one of them retries on serialization-failure. The handler
+ * is therefore safe to invoke from N parallel BullMQ workers without
+ * corrupting the chain.
+ *
+ * Idempotency: the producer-side `publishAuditEventViaOutbox` writes
+ * one outbox row per business-write. The dispatcher delivers each row
+ * exactly once (claim+ack); on retry we may re-process the same
+ * payload but the audit insert is a plain CREATE — duplicates simply
+ * extend the chain by one extra row. The acceptable outcome here is
+ * "at-least-once": a duplicated audit row is harmless evidence.
+ */
+
+import { computeAuditHash, type AuditHashPayload } from "@toolctrl/db/audit-hash";
+import {
+  buildAuditEventCreateData,
+  type AuditEventCreateData,
+  type AuditEventInput,
+} from "@toolctrl/db/audit-events.server";
+
+export const AUDIT_EVENT_OUTBOX_QUEUE = "outbox.audit" as const;
+
+export interface AuditEventPersistLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface AuditEventPersistTxClient {
+  auditEvent: {
+    findFirst(args: {
+      where: { tenantId: string | null; eventHash: { not: null } };
+      orderBy: { createdAt: "desc" };
+      select: { eventHash: true };
+    }): Promise<{ eventHash: string | null } | null>;
+    create(args: {
+      data: AuditEventCreateData & { previousHash: string | null; eventHash: string };
+    }): Promise<{ id: string; eventHash: string }>;
+  };
+}
+
+export interface AuditEventPersistTransactionRunner {
+  /**
+   * Run `fn` inside a Prisma transaction. Production wiring passes
+   * `prisma.$transaction` with isolationLevel="Serializable" so that
+   * the prevHash lookup + insert form an atomic chain link.
+   */
+  <T>(fn: (tx: AuditEventPersistTxClient) => Promise<T>): Promise<T>;
+}
+
+export interface AuditEventPersistDependencies {
+  readonly runInTransaction: AuditEventPersistTransactionRunner;
+  readonly logger?: AuditEventPersistLogger;
+  /**
+   * Override the hash-chain secret resolution for tests. Production
+   * leaves this undefined so the helper reads `process.env.AUDIT_CHAIN_SECRET`.
+   */
+  readonly secret?: string | null;
+}
+
+export class AuditEventPersistJobError extends Error {
+  override readonly name = "AuditEventPersistJobError";
+  readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "MISSING_ACTION";
+  constructor(message: string, code: AuditEventPersistJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export interface AuditEventPersistJobData {
+  readonly outboxEventId?: string;
+  readonly tenantId: string | null;
+  readonly eventType: string;
+  readonly payload: AuditEventCreateData;
+}
+
+export interface AuditEventPersistResult {
+  readonly persisted: true;
+  readonly auditEventId: string;
+  readonly eventHash: string;
+}
+
+export function parseAuditEventPersistJob(data: unknown): AuditEventPersistJobData {
+  if (!data || typeof data !== "object") {
+    throw new AuditEventPersistJobError("audit-event persist job missing data envelope", "INVALID_PAYLOAD");
+  }
+  const envelope = data as Record<string, unknown>;
+  const eventType = typeof envelope.eventType === "string" ? envelope.eventType : "";
+  if (eventType !== "audit.event.create") {
+    throw new AuditEventPersistJobError(
+      `audit-event persist job rejected eventType ${JSON.stringify(eventType)}`,
+      "INVALID_EVENT_TYPE"
+    );
+  }
+  const rawPayload = envelope.payload;
+  if (!rawPayload || typeof rawPayload !== "object") {
+    throw new AuditEventPersistJobError("audit-event persist job missing payload", "INVALID_PAYLOAD");
+  }
+  const inner = rawPayload as Record<string, unknown>;
+  const action = typeof inner.action === "string" ? inner.action.trim() : "";
+  if (action.length === 0) {
+    throw new AuditEventPersistJobError("audit-event persist job missing action", "MISSING_ACTION");
+  }
+  // Re-run normalization in case the producer emitted an older payload
+  // shape — defensive. `buildAuditEventCreateData` is idempotent.
+  const normalized = buildAuditEventCreateData(inner as unknown as AuditEventInput);
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId: typeof envelope.tenantId === "string" ? envelope.tenantId : null,
+    eventType,
+    payload: normalized,
+  };
+}
+
+function toHashPayload(data: AuditEventCreateData): AuditHashPayload {
+  return {
+    id: "",
+    createdAt: "",
+    tenantId: data.tenantId,
+    actorType: data.actorType,
+    actorId: data.actorId,
+    actorEmail: data.actorEmail,
+    actorIp: data.actorIp,
+    actorUserAgent: data.actorUserAgent,
+    action: data.action,
+    targetType: data.targetType,
+    targetId: data.targetId,
+    requestId: data.requestId,
+    sourceModule: data.sourceModule,
+    outcome: data.outcome,
+    dataSensitivity: data.dataSensitivity,
+    regulatoryScope: data.regulatoryScope,
+    metadata: data.metadata,
+  };
+}
+
+export async function runAuditEventPersistJob(
+  data: unknown,
+  deps: AuditEventPersistDependencies
+): Promise<AuditEventPersistResult> {
+  const job = parseAuditEventPersistJob(data);
+  const payload = job.payload;
+
+  const result = await deps.runInTransaction(async (tx) => {
+    const prior = await tx.auditEvent.findFirst({
+      where: { tenantId: payload.tenantId, eventHash: { not: null } },
+      orderBy: { createdAt: "desc" },
+      select: { eventHash: true },
+    });
+    const previousHash = prior?.eventHash ?? null;
+    const eventHash = computeAuditHash({
+      previousHash,
+      payload: toHashPayload(payload),
+      secret: deps.secret,
+    });
+    const inserted = await tx.auditEvent.create({
+      data: { ...payload, previousHash, eventHash },
+    });
+    return { auditEventId: inserted.id, eventHash };
+  });
+
+  deps.logger?.info?.(
+    {
+      component: "audit-event-persist",
+      tenantId: payload.tenantId,
+      action: payload.action,
+      auditEventId: result.auditEventId,
+      outboxEventId: job.outboxEventId,
+    },
+    "audit-event persisted via outbox"
+  );
+
+  return { persisted: true, auditEventId: result.auditEventId, eventHash: result.eventHash };
+}

hivegrid-cloud/apps/worker/src/jobs/audit-retention.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  DEFAULT_AUDIT_RETENTION_DAYS,
+  GDPR_AUDIT_RETENTION_DAYS,
+  SOC2_AUDIT_RETENTION_DAYS,
+  resolveAuditRetentionPolicy,
+  runAuditRetentionJob,
+} from "./audit-retention.js";
+
+describe("resolveAuditRetentionPolicy", () => {
+  it("returns 7y for SOC2 + HIPAA tenants (HIPAA wins on label)", () => {
+    expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["soc2"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "soc2" });
+    expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["hipaa"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "hipaa" });
+    expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["soc2", "hipaa"] })).toEqual({ retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: "hipaa" });
+  });
+
+  it("returns 6y for GDPR-only tenants", () => {
+    expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: ["gdpr"] })).toEqual({ retentionDays: GDPR_AUDIT_RETENTION_DAYS, source: "gdpr" });
+  });
+
+  it("returns 1y default for tenants with no regulatory scope", () => {
+    expect(resolveAuditRetentionPolicy({ tenantId: "t1", regulatoryScopes: [] })).toEqual({ retentionDays: DEFAULT_AUDIT_RETENTION_DAYS, source: "default" });
+  });
+});
+
+describe("runAuditRetentionJob", () => {
+  it("walks tenants, deletes rows, and emits a meta-audit for each non-empty sweep", async () => {
+    const now = new Date("2026-05-17T00:00:00.000Z");
+    const tenants = [
+      { tenantId: "t1", regulatoryScopes: ["soc2"] },
+      { tenantId: "t2", regulatoryScopes: [] },
+    ];
+    const eventDeleter = vi.fn(async (tenantId: string, threshold: Date) => {
+      void threshold;
+      return tenantId === "t1" ? 42 : 0;
+    });
+    const auditWriter = vi.fn(async () => {});
+    const result = await runAuditRetentionJob({
+      now,
+      tenantLister: async () => tenants,
+      eventDeleter,
+      auditWriter,
+    });
+    expect(result).toEqual({ inspectedTenants: 2, deletedRows: 42, failures: [] });
+    expect(eventDeleter).toHaveBeenCalledTimes(2);
+    // SOC2 -> 7y threshold
+    const soc2Threshold = eventDeleter.mock.calls[0]?.[1];
+    expect(soc2Threshold).toBeInstanceOf(Date);
+    expect(soc2Threshold.toISOString()).toBe(new Date(now.getTime() - SOC2_AUDIT_RETENTION_DAYS * 86_400_000).toISOString());
+    // Only t1 had deletions -> only one meta-audit
+    expect(auditWriter).toHaveBeenCalledTimes(1);
+  });
+
+  it("records failures without aborting other tenants", async () => {
+    const now = new Date("2026-05-17T00:00:00.000Z");
+    const tenants = [
+      { tenantId: "t-fail", regulatoryScopes: [] },
+      { tenantId: "t-ok", regulatoryScopes: [] },
+    ];
+    const eventDeleter = vi.fn(async (tenantId: string) => {
+      if (tenantId === "t-fail") throw new Error("boom");
+      return 5;
+    });
+    const result = await runAuditRetentionJob({ now, tenantLister: async () => tenants, eventDeleter });
+    expect(result.deletedRows).toBe(5);
+    expect(result.failures).toEqual([{ tenantId: "t-fail", error: "boom" }]);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/audit-retention.ts

@@ -0,0 +1,94 @@
+/**
+ * Phase 10 Task #67 — Audit-Event retention worker.
+ *
+ * Walks every tenant on a daily cron, looks up the tenant's retention
+ * window (plan-based default; configurable per-tenant via an override
+ * JSON in a future task), and deletes AuditEvent rows older than the
+ * window. Deletions are themselves audited as a system-action event so
+ * the cleanup itself is traceable — see
+ * docs/features/compliance-audit.md.
+ *
+ * Deliberately host-agnostic: the worker bootstrap (apps/worker/src/index.ts)
+ * supplies the seams (tenantLister, eventDeleter, auditWriter) so this
+ * file is trivially unit-testable without a real database.
+ */
+
+export const DEFAULT_AUDIT_RETENTION_DAYS = 365; // baseline / "none" scope
+export const SOC2_AUDIT_RETENTION_DAYS = 7 * 365; // SOC2 + HIPAA: 7y
+export const GDPR_AUDIT_RETENTION_DAYS = 6 * 365; // GDPR: 6y
+
+export type AuditRetentionPolicy = {
+  /** Hard cap in days. Events older than this are pruned. */
+  retentionDays: number;
+  /** Free-form label for log lines (`gdpr` | `soc2` | `plan:enterprise` | …). */
+  source: string;
+};
+
+export type AuditRetentionTenant = {
+  tenantId: string;
+  /** Highest regulatoryScope used by events for this tenant. */
+  regulatoryScopes: ReadonlyArray<string>;
+};
+
+export type AuditRetentionInput = {
+  /** Reference "now" — defaults to `new Date()`. Injected for tests. */
+  now?: Date;
+  /** List tenants (each with the regulatoryScopes ever used by their events). */
+  tenantLister: () => Promise<ReadonlyArray<AuditRetentionTenant>>;
+  /** Delete events with createdAt < threshold for tenantId. Returns row count. */
+  eventDeleter: (tenantId: string, threshold: Date) => Promise<number>;
+  /** Optional audit writer for the meta-audit (cleanup itself). */
+  auditWriter?: (input: { tenantId: string; deletedCount: number; threshold: Date; policy: AuditRetentionPolicy }) => Promise<void>;
+  /** Logger seam. */
+  logger?: { info: (fields: Record<string, unknown>, msg: string) => void; error?: (fields: Record<string, unknown>, msg: string) => void };
+  /** Optional override for plan-resolved policy per tenant. */
+  policyResolver?: (tenant: AuditRetentionTenant) => AuditRetentionPolicy;
+};
+
+export function resolveAuditRetentionPolicy(tenant: AuditRetentionTenant): AuditRetentionPolicy {
+  const scopes = new Set(tenant.regulatoryScopes.map((s) => s.toLowerCase()));
+  if (scopes.has("soc2") || scopes.has("hipaa")) {
+    return { retentionDays: SOC2_AUDIT_RETENTION_DAYS, source: scopes.has("hipaa") ? "hipaa" : "soc2" };
+  }
+  if (scopes.has("gdpr")) {
+    return { retentionDays: GDPR_AUDIT_RETENTION_DAYS, source: "gdpr" };
+  }
+  return { retentionDays: DEFAULT_AUDIT_RETENTION_DAYS, source: "default" };
+}
+
+export type AuditRetentionResult = {
+  inspectedTenants: number;
+  deletedRows: number;
+  failures: Array<{ tenantId: string; error: string }>;
+};
+
+/**
+ * Run the retention sweep across every tenant. Caller (cron bootstrap)
+ * decides scheduling — this function is a single pass.
+ */
+export async function runAuditRetentionJob(input: AuditRetentionInput): Promise<AuditRetentionResult> {
+  const now = input.now ?? new Date();
+  const tenants = await input.tenantLister();
+  let deletedRows = 0;
+  const failures: AuditRetentionResult["failures"] = [];
+
+  for (const tenant of tenants) {
+    try {
+      const policy = (input.policyResolver ?? resolveAuditRetentionPolicy)(tenant);
+      const threshold = new Date(now.getTime() - policy.retentionDays * 86_400_000);
+      const count = await input.eventDeleter(tenant.tenantId, threshold);
+      deletedRows += count;
+      input.logger?.info({ tenantId: tenant.tenantId, deletedCount: count, retentionDays: policy.retentionDays, policySource: policy.source }, "audit.retention.tenant.swept");
+      if (count > 0 && input.auditWriter) {
+        await input.auditWriter({ tenantId: tenant.tenantId, deletedCount: count, threshold, policy });
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      failures.push({ tenantId: tenant.tenantId, error: message });
+      input.logger?.error?.({ tenantId: tenant.tenantId, error: message }, "audit.retention.tenant.failed");
+    }
+  }
+
+  input.logger?.info({ inspected: tenants.length, deletedRows, failures: failures.length }, "audit.retention.complete");
+  return { inspectedTenants: tenants.length, deletedRows, failures };
+}

hivegrid-cloud/apps/worker/src/jobs/dsgvo-deletion.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { USER_FK_MAP } from "@toolctrl/db/dsgvo/user-fk-map";
+
+import { runDsgvoDeletion, type DsgvoDeletionDependencies, type DsgvoDeletionJobInput } from "./dsgvo-deletion.js";
+
+function makeDeps(overrides?: Partial<DsgvoDeletionDependencies>): {
+  deps: DsgvoDeletionDependencies;
+  audits: Array<Parameters<DsgvoDeletionDependencies["writeAuditEvent"]>[0]>;
+  hardDeletes: string[];
+  pseudoTables: Array<{ model: string; column: string }>;
+  tombstones: Array<{ model: string; column: string }>;
+  hardDeleteUserCalls: string[];
+  pseudonymizeUserCalls: string[];
+} {
+  const audits: Array<Parameters<DsgvoDeletionDependencies["writeAuditEvent"]>[0]> = [];
+  const hardDeletes: string[] = [];
+  const pseudoTables: Array<{ model: string; column: string }> = [];
+  const tombstones: Array<{ model: string; column: string }> = [];
+  const hardDeleteUserCalls: string[] = [];
+  const pseudonymizeUserCalls: string[] = [];
+
+  const deps: DsgvoDeletionDependencies = {
+    tableClient: {
+      hardDelete: vi.fn().mockImplementation(async (entry: { model: string; column: string }) => {
+        hardDeletes.push(`${entry.model}.${entry.column}`);
+        return 1;
+      }),
+      pseudonymize: vi.fn().mockImplementation(async (entry: { model: string; column: string }) => {
+        pseudoTables.push({ model: entry.model, column: entry.column });
+        return 2;
+      }),
+      retainTombstone: vi.fn().mockImplementation(async (entry: { model: string; column: string }) => {
+        tombstones.push({ model: entry.model, column: entry.column });
+        return 3;
+      }),
+    },
+    userClient: {
+      hardDeleteUser: vi.fn().mockImplementation(async (userId: string) => {
+        hardDeleteUserCalls.push(userId);
+      }),
+      pseudonymizeUser: vi.fn().mockImplementation(async (userId: string) => {
+        pseudonymizeUserCalls.push(userId);
+      }),
+    },
+    writeAuditEvent: vi.fn().mockImplementation(async (input) => {
+      audits.push(input);
+    }),
+    logger: { info: vi.fn(), error: vi.fn() },
+    ...overrides,
+  };
+
+  return { deps, audits, hardDeletes, pseudoTables, tombstones, hardDeleteUserCalls, pseudonymizeUserCalls };
+}
+
+const baseJob: DsgvoDeletionJobInput = {
+  userId: "u1",
+  tenantId: "t1",
+  requestId: "req-1",
+  actorType: "system",
+  actorId: "system",
+  mode: "delete",
+};
+
+describe("runDsgvoDeletion", () => {
+  it("applies hard-delete, pseudonymize and retain-tombstone strategies across the user-fk-map", async () => {
+    const { deps, audits, hardDeletes, pseudoTables, tombstones, hardDeleteUserCalls } = makeDeps();
+    const result = await runDsgvoDeletion(deps, baseJob);
+
+    const expectedHardDelete = USER_FK_MAP.filter((e) => e.strategy === "hard-delete").length;
+    const expectedPseudo = USER_FK_MAP.filter((e) => e.strategy === "pseudonymize").length;
+    const expectedTombstone = USER_FK_MAP.filter((e) => e.strategy === "retain-tombstone").length;
+
+    expect(hardDeletes).toHaveLength(expectedHardDelete);
+    expect(pseudoTables).toHaveLength(expectedPseudo);
+    expect(tombstones).toHaveLength(expectedTombstone);
+    expect(hardDeleteUserCalls).toEqual(["u1"]);
+
+    expect(result.tables).toHaveLength(USER_FK_MAP.length);
+    expect(result.userOutcome).toBe("deleted");
+
+    expect(audits).toHaveLength(1);
+    expect(audits[0]?.action).toBe("DELETE_USER");
+    expect(audits[0]?.outcome).toBe("success");
+    expect(audits[0]?.regulatoryScope).toEqual(["gdpr"]);
+  });
+
+  it("pseudonymizes the user when mode=pseudonymize", async () => {
+    const { deps, audits, pseudonymizeUserCalls, hardDeleteUserCalls } = makeDeps();
+    const result = await runDsgvoDeletion(deps, { ...baseJob, mode: "pseudonymize", actorType: "admin", actorId: "admin-1", reason: "court order" });
+
+    expect(pseudonymizeUserCalls).toEqual(["u1"]);
+    expect(hardDeleteUserCalls).toEqual([]);
+    expect(result.userOutcome).toBe("pseudonymized");
+    expect(audits[0]?.action).toBe("PSEUDONYMIZE_USER");
+    expect(audits[0]?.metadata).toMatchObject({ mode: "pseudonymize", reason: "court order" });
+  });
+
+  it("audits failure and rethrows when a table strategy errors mid-run", async () => {
+    const { deps, audits } = makeDeps({
+      tableClient: {
+        hardDelete: vi.fn().mockRejectedValue(new Error("db lock")),
+        pseudonymize: vi.fn().mockResolvedValue(0),
+        retainTombstone: vi.fn().mockResolvedValue(0),
+      },
+    });
+    await expect(runDsgvoDeletion(deps, baseJob)).rejects.toThrow(/db lock/);
+    expect(audits.at(-1)?.outcome).toBe("failure");
+    expect(audits.at(-1)?.metadata).toMatchObject({ error: expect.stringContaining("db lock") });
+  });
+
+  it("never deletes the AuditEvent rows even when mode=delete (SOC2-CC6.1 audit-trail integrity)", async () => {
+    const { deps, hardDeletes, pseudoTables } = makeDeps();
+    await runDsgvoDeletion(deps, baseJob);
+    expect(hardDeletes).not.toContain("AuditEvent.actorId");
+    expect(pseudoTables.some((t) => t.model === "AuditEvent" && t.column === "actorId")).toBe(true);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/dsgvo-deletion.ts

@@ -0,0 +1,170 @@
+/**
+ * Phase 10 Task #68 — DSGVO deletion-cascade worker.
+ *
+ * Walks every entry in `USER_FK_MAP` and applies its declared strategy
+ * to the rows that reference the deletion target:
+ *
+ *   • hard-delete     → `model.deleteMany({ where: { column: userId } })`.
+ *   • pseudonymize    → `model.updateMany(..., { column: pseudonym.id })`
+ *                       with a deterministic pseudonym (see
+ *                       packages/db/src/dsgvo/pseudonymize.ts). When the
+ *                       model also stores an email or other PII column,
+ *                       caller-supplied `pseudonymizeRow` may rewrite it.
+ *   • retain-tombstone → `model.updateMany(..., { column: null })` —
+ *                       the column is nullable per schema; the row
+ *                       outlives the user but loses the reference.
+ *
+ * Once every entry is processed the canonical `User` row itself is
+ * deleted (hard delete) unless `mode === 'pseudonymize'` in which case
+ * the row is kept and the deterministic-pseudonym field set is written
+ * via `buildUserPseudonymizationUpdate`.
+ *
+ * The worker is host-agnostic — the bootstrap supplies the table
+ * client + audit writer so this file is fully unit-testable.
+ */
+
+import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
+import { buildUserPseudonymizationUpdate, pseudonymize } from "@toolctrl/db/dsgvo/pseudonymize";
+
+export type DsgvoDeletionMode = "delete" | "pseudonymize";
+
+export type DsgvoDeletionJobInput = {
+  userId: string;
+  tenantId: string | null;
+  requestId: string;
+  actorType: "user" | "admin" | "system";
+  actorId: string;
+  mode: DsgvoDeletionMode;
+  reason?: string | null;
+};
+
+export type DsgvoDeletionTableResult = {
+  model: string;
+  column: string;
+  strategy: UserFkMapEntry["strategy"];
+  affectedRows: number;
+};
+
+export type DsgvoDeletionResult = {
+  userId: string;
+  requestId: string;
+  mode: DsgvoDeletionMode;
+  tables: DsgvoDeletionTableResult[];
+  userOutcome: "deleted" | "pseudonymized";
+};
+
+export type DsgvoDeletionTableClient = {
+  hardDelete: (entry: UserFkMapEntry, userId: string) => Promise<number>;
+  pseudonymize: (entry: UserFkMapEntry, userId: string, pseudonymUserId: string) => Promise<number>;
+  retainTombstone: (entry: UserFkMapEntry, userId: string) => Promise<number>;
+};
+
+export type DsgvoDeletionUserClient = {
+  hardDeleteUser: (userId: string) => Promise<void>;
+  pseudonymizeUser: (userId: string, update: ReturnType<typeof buildUserPseudonymizationUpdate>) => Promise<void>;
+};
+
+export type DsgvoDeletionDependencies = {
+  tableClient: DsgvoDeletionTableClient;
+  userClient: DsgvoDeletionUserClient;
+  writeAuditEvent: (input: {
+    tenantId: string | null;
+    actorId: string;
+    actorType: string;
+    action: string;
+    targetType: string;
+    targetId: string;
+    regulatoryScope: string[];
+    dataSensitivity: "pii";
+    outcome: "success" | "failure";
+    metadata: Record<string, unknown>;
+  }) => Promise<void>;
+  logger: { info: (fields: Record<string, unknown>, msg: string) => void; error: (fields: Record<string, unknown>, msg: string) => void };
+};
+
+export async function runDsgvoDeletion(
+  deps: DsgvoDeletionDependencies,
+  job: DsgvoDeletionJobInput,
+): Promise<DsgvoDeletionResult> {
+  const pseudo = pseudonymize(job.userId);
+  const tables: DsgvoDeletionTableResult[] = [];
+
+  try {
+    for (const entry of USER_FK_MAP) {
+      let affectedRows = 0;
+      switch (entry.strategy) {
+        case "hard-delete":
+          // In `mode=pseudonymize` we *still* hard-delete from join/RBAC
+          // tables (TenantUser, UserRole, GroupUser): keeping these rows
+          // would let the pseudonymized identity stay attached to active
+          // workspaces, which contradicts the user's deletion intent. The
+          // pseudonymization mode only protects retention-bound rows.
+          affectedRows = await deps.tableClient.hardDelete(entry, job.userId);
+          break;
+        case "pseudonymize":
+          affectedRows = await deps.tableClient.pseudonymize(entry, job.userId, pseudo.pseudoId);
+          break;
+        case "retain-tombstone":
+          affectedRows = await deps.tableClient.retainTombstone(entry, job.userId);
+          break;
+      }
+      tables.push({ model: entry.model, column: entry.column, strategy: entry.strategy, affectedRows });
+    }
+
+    let userOutcome: "deleted" | "pseudonymized";
+    if (job.mode === "pseudonymize") {
+      await deps.userClient.pseudonymizeUser(job.userId, buildUserPseudonymizationUpdate(job.userId));
+      userOutcome = "pseudonymized";
+    } else {
+      await deps.userClient.hardDeleteUser(job.userId);
+      userOutcome = "deleted";
+    }
+
+    await deps.writeAuditEvent({
+      tenantId: job.tenantId,
+      actorId: job.actorId,
+      actorType: job.actorType,
+      action: job.mode === "pseudonymize" ? "PSEUDONYMIZE_USER" : "DELETE_USER",
+      targetType: "User",
+      targetId: job.userId,
+      regulatoryScope: ["gdpr"],
+      dataSensitivity: "pii",
+      outcome: "success",
+      metadata: {
+        requestId: job.requestId,
+        mode: job.mode,
+        tablesProcessed: tables.length,
+        rowsAffected: tables.reduce((sum, t) => sum + t.affectedRows, 0),
+        reason: job.reason ?? null,
+      },
+    });
+
+    deps.logger.info(
+      { userId: job.userId, requestId: job.requestId, mode: job.mode, tables: tables.length },
+      "dsgvo-deletion.complete",
+    );
+
+    return { userId: job.userId, requestId: job.requestId, mode: job.mode, tables, userOutcome };
+  } catch (error) {
+    deps.logger.error({ userId: job.userId, requestId: job.requestId, err: String(error) }, "dsgvo-deletion.failed");
+    await deps.writeAuditEvent({
+      tenantId: job.tenantId,
+      actorId: job.actorId,
+      actorType: job.actorType,
+      action: job.mode === "pseudonymize" ? "PSEUDONYMIZE_USER" : "DELETE_USER",
+      targetType: "User",
+      targetId: job.userId,
+      regulatoryScope: ["gdpr"],
+      dataSensitivity: "pii",
+      outcome: "failure",
+      metadata: {
+        requestId: job.requestId,
+        mode: job.mode,
+        tablesProcessed: tables.length,
+        error: String(error),
+        reason: job.reason ?? null,
+      },
+    });
+    throw error;
+  }
+}

hivegrid-cloud/apps/worker/src/jobs/dsgvo-export.test.ts

@@ -0,0 +1,138 @@
+import { PassThrough } from "node:stream";
+
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  runDsgvoExport,
+  type DsgvoExportDependencies,
+  type DsgvoExportJobInput,
+  type DsgvoExportSink,
+} from "./dsgvo-export.js";
+
+function makeSink(): { sink: DsgvoExportSink; collected: () => string } {
+  const writeStream = new PassThrough();
+  const chunks: Buffer[] = [];
+  writeStream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
+  // Swallow stream-destroy errors so the "iterator throws" test does not
+  // surface as an uncaught exception on the runner. The job rethrows the
+  // original error via its own catch block — that's what the test asserts.
+  writeStream.on("error", () => undefined);
+  const completionPromise = new Promise<{ bucket: string; key: string; sizeBytes: number }>((resolve) => {
+    writeStream.on("end", () => {
+      const buf = Buffer.concat(chunks);
+      resolve({ bucket: "test-bucket", key: "user-data/test.json", sizeBytes: buf.byteLength });
+    });
+  });
+  return {
+    sink: { writeStream, completionPromise },
+    collected: () => Buffer.concat(chunks).toString("utf8"),
+  };
+}
+
+function makeDeps(overrides?: Partial<DsgvoExportDependencies>): {
+  deps: DsgvoExportDependencies;
+  audits: Array<Parameters<DsgvoExportDependencies["writeAuditEvent"]>[0]>;
+  mails: Array<Parameters<DsgvoExportDependencies["sendExportReadyEmail"]>[0]>;
+  collected: () => string;
+} {
+  const { sink, collected } = makeSink();
+  const audits: Array<Parameters<DsgvoExportDependencies["writeAuditEvent"]>[0]> = [];
+  const mails: Array<Parameters<DsgvoExportDependencies["sendExportReadyEmail"]>[0]> = [];
+
+  const deps: DsgvoExportDependencies = {
+    loadUser: vi.fn().mockResolvedValue({
+      id: "u1",
+      email: "alice@example.com",
+      firstName: "Alice",
+      lastName: "Anderson",
+      createdAt: new Date("2026-01-01T00:00:00Z"),
+    }),
+    tableSource: {
+      async *iterate(_userId, entry) {
+        // Return one row per "Log.userId" section so we can assert order.
+        if (entry.model === "Log") {
+          yield { id: "log1", message: "hello" };
+        }
+      },
+    },
+    openSink: () => sink,
+    buildSignedUrl: vi.fn().mockResolvedValue({ url: "https://s3/signed", expiresAt: new Date("2026-02-01T00:00:00Z") }),
+    writeAuditEvent: vi.fn().mockImplementation(async (input) => {
+      audits.push(input);
+    }),
+    sendExportReadyEmail: vi.fn().mockImplementation(async (input) => {
+      mails.push(input);
+    }),
+    buildObjectKey: () => ({ bucket: "test-bucket", key: "user-data/test.json" }),
+    logger: { info: vi.fn(), error: vi.fn() },
+    ...overrides,
+  };
+
+  return { deps, audits, mails, collected };
+}
+
+const baseJob: DsgvoExportJobInput = {
+  userId: "u1",
+  tenantId: "t1",
+  requestId: "req-1",
+  actorType: "user",
+  actorId: "u1",
+};
+
+describe("runDsgvoExport", () => {
+  it("streams a JSON bundle, audits success, and sends the export-ready email", async () => {
+    const { deps, audits, mails, collected } = makeDeps();
+    const result = await runDsgvoExport(deps, baseJob);
+
+    expect(result.bucket).toBe("test-bucket");
+    expect(result.key).toBe("user-data/test.json");
+    expect(result.rowCount).toBeGreaterThan(0);
+    expect(result.signedUrl).toBe("https://s3/signed");
+
+    const body = collected();
+    const parsed = JSON.parse(body);
+    expect(parsed.bundleVersion).toBe("1.0");
+    expect(parsed.requestId).toBe("req-1");
+    expect(parsed.user.email).toBe("alice@example.com");
+    expect(parsed.sections["Log.userId"]).toHaveLength(1);
+
+    expect(audits).toHaveLength(1);
+    expect(audits[0]?.action).toBe("EXPORT_PERSONAL_DATA");
+    expect(audits[0]?.outcome).toBe("success");
+    expect(audits[0]?.regulatoryScope).toEqual(["gdpr"]);
+    expect(audits[0]?.dataSensitivity).toBe("pii");
+
+    expect(mails).toHaveLength(1);
+    expect(mails[0]?.toEmail).toBe("alice@example.com");
+    expect(mails[0]?.signedUrl).toBe("https://s3/signed");
+  });
+
+  it("audits a failure when the user cannot be loaded", async () => {
+    const { deps, audits } = makeDeps({
+      loadUser: vi.fn().mockResolvedValue(null),
+    });
+    await expect(runDsgvoExport(deps, baseJob)).rejects.toThrow(/not found/);
+    expect(audits).toHaveLength(1);
+    expect(audits[0]?.outcome).toBe("failure");
+    expect(audits[0]?.metadata).toMatchObject({ error: "user_not_found" });
+  });
+
+  it("audits a failure and rethrows when a table iterator throws", async () => {
+    const { deps, audits } = makeDeps({
+      tableSource: {
+        async *iterate() {
+          throw new Error("db boom");
+        },
+      },
+    });
+    await expect(runDsgvoExport(deps, baseJob)).rejects.toThrow(/db boom/);
+    expect(audits.at(-1)?.outcome).toBe("failure");
+  });
+
+  it("forwards the actorType and reason into the audit metadata for admin-initiated exports", async () => {
+    const { deps, audits } = makeDeps();
+    await runDsgvoExport(deps, { ...baseJob, actorType: "admin", actorId: "admin-1", reason: "user requested via support ticket #123" });
+    expect(audits[0]?.actorType).toBe("admin");
+    expect(audits[0]?.metadata).toMatchObject({ reason: "user requested via support ticket #123" });
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/dsgvo-export.ts

@@ -0,0 +1,244 @@
+/**
+ * Phase 10 Task #68 — DSGVO personal-data export worker.
+ *
+ * Aggregates every row that references `userId` across the
+ * USER_FK_MAP (see packages/db/src/dsgvo/user-fk-map.ts) plus the
+ * canonical `User` row, serialises the result as a single
+ * UTF-8 JSON bundle, and streams it to S3 via the multipart sink
+ * shared with Phase 03 #46.
+ *
+ * The worker is host-agnostic: the bootstrap (apps/worker/src/index.ts)
+ * passes in the dependencies (table iterator, sink factory, audit
+ * writer, signed-URL builder, mailer) so this module can be unit-tested
+ * in isolation without S3 / Prisma / SES.
+ *
+ * Job lifecycle:
+ *   1. Resolve the user (loadUser).
+ *   2. For each entry in USER_FK_MAP with includeInExport=true, stream
+ *      its rows into the JSON bundle under a section keyed by model.
+ *   3. Finalise the multipart upload, mint a signed URL with the
+ *      configured validity, send the "export ready" notification email,
+ *      and emit an AuditEvent with action=EXPORT_PERSONAL_DATA,
+ *      regulatoryScope=[gdpr], dataSensitivity=pii.
+ *
+ * Errors short-circuit the pipeline; the caller (BullMQ worker) is
+ * responsible for retry semantics. The sink aborts its multipart upload
+ * automatically on stream destruction so no orphan parts linger in S3.
+ */
+
+import type { Writable } from "node:stream";
+
+import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
+
+export type DsgvoExportJobInput = {
+  userId: string;
+  /** Tenant scope for the AuditEvent + S3 metadata. */
+  tenantId: string | null;
+  /** Idempotency token (mirrors User.deletionRequestId for repeats). */
+  requestId: string;
+  /** Initiator: 'user' (self-service) or 'admin' (operator-on-behalf). */
+  actorType: "user" | "admin";
+  /** Actor identity for the AuditEvent. */
+  actorId: string;
+  /** Optional admin reason (admin path only). */
+  reason?: string | null;
+};
+
+export type DsgvoExportSection = {
+  model: string;
+  column: string;
+  rowCount: number;
+};
+
+export type DsgvoExportResult = {
+  bucket: string;
+  key: string;
+  rowCount: number;
+  sizeBytes: number;
+  signedUrl: string;
+  signedUrlExpiresAt: Date;
+  sections: DsgvoExportSection[];
+};
+
+export type DsgvoExportSink = {
+  writeStream: Writable;
+  completionPromise: Promise<{ bucket: string; key: string; sizeBytes: number }>;
+};
+
+export type DsgvoExportTableSource = {
+  /**
+   * Async-iterate every row in `entry.model` filtered by
+   * `${entry.column} = userId`. Implementations typically use a Prisma
+   * cursor; the worker buffers nothing.
+   */
+  iterate(userId: string, entry: UserFkMapEntry): AsyncIterable<Record<string, unknown>>;
+};
+
+export type DsgvoExportUserRecord = {
+  id: string;
+  email: string;
+  firstName: string;
+  lastName: string;
+  createdAt: Date;
+  phone?: string | null;
+  avatar?: string | null;
+  locale?: string | null;
+};
+
+export type DsgvoExportDependencies = {
+  /** Load the canonical User row (header section of the bundle). */
+  loadUser: (userId: string) => Promise<DsgvoExportUserRecord | null>;
+  /** Iterate every user-fk-map entry that opts into export. */
+  tableSource: DsgvoExportTableSource;
+  /** Open the streaming sink (typically createS3MultipartUploadSink). */
+  openSink: (objectKey: string, contentType: string, metadata: Record<string, string>) => DsgvoExportSink;
+  /** Mint a presigned URL pointing at the finished artifact. */
+  buildSignedUrl: (input: { bucket: string; key: string }) => Promise<{ url: string; expiresAt: Date }>;
+  /** Persist the AuditEvent (createAuditEvent). */
+  writeAuditEvent: (input: {
+    tenantId: string | null;
+    actorId: string;
+    actorType: string;
+    action: string;
+    targetType: string;
+    targetId: string;
+    regulatoryScope: string[];
+    dataSensitivity: "pii";
+    outcome: "success" | "failure";
+    metadata: Record<string, unknown>;
+  }) => Promise<void>;
+  /** Notify the user that the export is ready. */
+  sendExportReadyEmail: (input: {
+    userId: string;
+    toEmail: string;
+    signedUrl: string;
+    expiresAt: Date;
+    sizeBytes: number;
+    rowCount: number;
+  }) => Promise<void>;
+  /** Object-key builder — caller controls bucket + path layout. */
+  buildObjectKey: (input: { userId: string; requestId: string }) => { bucket: string; key: string };
+  /** Logger seam — no console.* in production code. */
+  logger: { info: (fields: Record<string, unknown>, msg: string) => void; error: (fields: Record<string, unknown>, msg: string) => void };
+};
+
+const EXPORT_CONTENT_TYPE = "application/json; charset=utf-8";
+const BUNDLE_VERSION = "1.0";
+
+export async function runDsgvoExport(deps: DsgvoExportDependencies, job: DsgvoExportJobInput): Promise<DsgvoExportResult> {
+  const user = await deps.loadUser(job.userId);
+  if (!user) {
+    await deps.writeAuditEvent({
+      tenantId: job.tenantId,
+      actorId: job.actorId,
+      actorType: job.actorType,
+      action: "EXPORT_PERSONAL_DATA",
+      targetType: "User",
+      targetId: job.userId,
+      regulatoryScope: ["gdpr"],
+      dataSensitivity: "pii",
+      outcome: "failure",
+      metadata: { requestId: job.requestId, error: "user_not_found", reason: job.reason ?? null },
+    });
+    throw new Error(`dsgvo-export: user ${job.userId} not found`);
+  }
+
+  const exportEntries = USER_FK_MAP.filter((e) => e.includeInExport);
+  const sections: DsgvoExportSection[] = [];
+  let totalRows = 0;
+  const { key } = deps.buildObjectKey({ userId: job.userId, requestId: job.requestId });
+  const sink = deps.openSink(key, EXPORT_CONTENT_TYPE, {
+    "x-tc-user-id": job.userId,
+    "x-tc-request-id": job.requestId,
+  });
+
+  try {
+    sink.writeStream.write(`{\n  "bundleVersion": ${JSON.stringify(BUNDLE_VERSION)},\n`);
+    sink.writeStream.write(`  "exportedAt": ${JSON.stringify(new Date().toISOString())},\n`);
+    sink.writeStream.write(`  "requestId": ${JSON.stringify(job.requestId)},\n`);
+    sink.writeStream.write(`  "user": ${JSON.stringify(user)},\n`);
+    sink.writeStream.write(`  "sections": {\n`);
+
+    let firstSection = true;
+    for (const entry of exportEntries) {
+      if (!firstSection) sink.writeStream.write(",\n");
+      firstSection = false;
+      sink.writeStream.write(`    ${JSON.stringify(`${entry.model}.${entry.column}`)}: [`);
+      let rowCount = 0;
+      let firstRow = true;
+      for await (const row of deps.tableSource.iterate(job.userId, entry)) {
+        if (!firstRow) sink.writeStream.write(",");
+        firstRow = false;
+        sink.writeStream.write(JSON.stringify(row));
+        rowCount += 1;
+      }
+      sink.writeStream.write("]");
+      sections.push({ model: entry.model, column: entry.column, rowCount });
+      totalRows += rowCount;
+    }
+
+    sink.writeStream.write("\n  }\n}\n");
+    sink.writeStream.end();
+    const completion = await sink.completionPromise;
+
+    const { url, expiresAt } = await deps.buildSignedUrl({ bucket: completion.bucket, key: completion.key });
+
+    await deps.sendExportReadyEmail({
+      userId: job.userId,
+      toEmail: user.email,
+      signedUrl: url,
+      expiresAt,
+      sizeBytes: completion.sizeBytes,
+      rowCount: totalRows,
+    });
+
+    await deps.writeAuditEvent({
+      tenantId: job.tenantId,
+      actorId: job.actorId,
+      actorType: job.actorType,
+      action: "EXPORT_PERSONAL_DATA",
+      targetType: "User",
+      targetId: job.userId,
+      regulatoryScope: ["gdpr"],
+      dataSensitivity: "pii",
+      outcome: "success",
+      metadata: {
+        requestId: job.requestId,
+        rowCount: totalRows,
+        sizeBytes: completion.sizeBytes,
+        sectionCount: sections.length,
+        bucket: completion.bucket,
+        key: completion.key,
+        reason: job.reason ?? null,
+      },
+    });
+
+    deps.logger.info({ userId: job.userId, requestId: job.requestId, rowCount: totalRows }, "dsgvo-export.complete");
+
+    return {
+      bucket: completion.bucket,
+      key: completion.key,
+      rowCount: totalRows,
+      sizeBytes: completion.sizeBytes,
+      signedUrl: url,
+      signedUrlExpiresAt: expiresAt,
+      sections,
+    };
+  } catch (error) {
+    sink.writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
+    deps.logger.error({ userId: job.userId, requestId: job.requestId, err: String(error) }, "dsgvo-export.failed");
+    await deps.writeAuditEvent({
+      tenantId: job.tenantId,
+      actorId: job.actorId,
+      actorType: job.actorType,
+      action: "EXPORT_PERSONAL_DATA",
+      targetType: "User",
+      targetId: job.userId,
+      regulatoryScope: ["gdpr"],
+      dataSensitivity: "pii",
+      outcome: "failure",
+      metadata: { requestId: job.requestId, error: String(error), reason: job.reason ?? null },
+    });
+    throw error;
+  }
+}

hivegrid-cloud/apps/worker/src/jobs/dsgvo-grace-cron.test.ts

@@ -0,0 +1,99 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  DSGVO_GRACE_CRON_EVERY_MS,
+  DSGVO_GRACE_CRON_JOB_ID,
+  registerDsgvoGraceCron,
+  runDsgvoGraceCron,
+  type DsgvoGraceCandidate,
+} from "./dsgvo-grace-cron.js";
+
+function makeLogger() {
+  return { info: vi.fn(), error: vi.fn() };
+}
+
+describe("dsgvo-grace-cron", () => {
+  it("dispatches one deletion per overdue user", async () => {
+    const now = new Date("2026-06-17T00:00:00Z");
+    const candidates: DsgvoGraceCandidate[] = [
+      { userId: "u1", tenantId: "t1", scheduledDeletionAt: new Date("2026-06-01T00:00:00Z"), deletionRequestId: "req-1" },
+      { userId: "u2", tenantId: null, scheduledDeletionAt: new Date("2026-06-10T00:00:00Z"), deletionRequestId: null },
+    ];
+    const enqueue = vi.fn().mockResolvedValue({ jobId: "job-x" });
+    const logger = makeLogger();
+    const result = await runDsgvoGraceCron({
+      listOverdueUsers: async () => candidates,
+      enqueueDeletion: enqueue,
+      now: () => now,
+      logger,
+    });
+    expect(result.scanned).toBe(2);
+    expect(result.dispatched).toBe(2);
+    expect(result.failed).toBe(0);
+    expect(enqueue).toHaveBeenCalledTimes(2);
+    expect(enqueue.mock.calls[0][0].requestId).toBe("req-1");
+    expect(enqueue.mock.calls[0][0].mode).toBe("delete");
+    expect(enqueue.mock.calls[0][0].actorType).toBe("system");
+    expect(enqueue.mock.calls[0][0].reason).toBe("dsgvo-grace-period-elapsed");
+    // u2 had no deletionRequestId — synthesize a grace-derived one.
+    expect(enqueue.mock.calls[1][0].requestId.startsWith("grace-u2-")).toBe(true);
+    expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ userId: "u1" }), "dsgvo-grace-cron.dispatched");
+  });
+
+  it("isolates per-candidate failures", async () => {
+    const candidates: DsgvoGraceCandidate[] = [
+      { userId: "ok", tenantId: null, scheduledDeletionAt: new Date(), deletionRequestId: null },
+      { userId: "bad", tenantId: null, scheduledDeletionAt: new Date(), deletionRequestId: null },
+      { userId: "ok2", tenantId: null, scheduledDeletionAt: new Date(), deletionRequestId: null },
+    ];
+    const enqueue = vi
+      .fn()
+      .mockResolvedValueOnce({ jobId: "j1" })
+      .mockRejectedValueOnce(new Error("redis down"))
+      .mockResolvedValueOnce({ jobId: "j3" });
+    const logger = makeLogger();
+    const result = await runDsgvoGraceCron({
+      listOverdueUsers: async () => candidates,
+      enqueueDeletion: enqueue,
+      logger,
+    });
+    expect(result.scanned).toBe(3);
+    expect(result.dispatched).toBe(2);
+    expect(result.failed).toBe(1);
+    expect(result.failures[0]).toEqual({ userId: "bad", error: "redis down" });
+    expect(logger.error).toHaveBeenCalledWith(
+      expect.objectContaining({ userId: "bad", err: "redis down" }),
+      "dsgvo-grace-cron.dispatch-failed",
+    );
+  });
+
+  it("returns zero when no candidates", async () => {
+    const enqueue = vi.fn();
+    const logger = makeLogger();
+    const result = await runDsgvoGraceCron({
+      listOverdueUsers: async () => [],
+      enqueueDeletion: enqueue,
+      logger,
+    });
+    expect(result.scanned).toBe(0);
+    expect(result.dispatched).toBe(0);
+    expect(result.failed).toBe(0);
+    expect(enqueue).not.toHaveBeenCalled();
+  });
+
+  it("registerDsgvoGraceCron upserts a 24h scheduler with discriminator-shaped data", async () => {
+    const upsert = vi.fn().mockResolvedValue(undefined);
+    await registerDsgvoGraceCron({ upsertJobScheduler: upsert });
+    expect(upsert).toHaveBeenCalledTimes(1);
+    expect(upsert.mock.calls[0][0]).toBe(DSGVO_GRACE_CRON_JOB_ID);
+    expect(upsert.mock.calls[0][1]).toEqual({ every: DSGVO_GRACE_CRON_EVERY_MS });
+    // The scheduled-job template MUST carry the CleanupJob discriminator
+    // so the maintenance handler can dispatch on payload.task. Without
+    // this, every tick raised UnsupportedWorkerJobError in staging.
+    expect(upsert.mock.calls[0][2]).toEqual({
+      name: DSGVO_GRACE_CRON_JOB_ID,
+      data: { kind: "cleanup.run", task: "dsgvo-grace-period-scan" },
+    });
+    expect(DSGVO_GRACE_CRON_EVERY_MS).toBe(24 * 60 * 60 * 1000);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/dsgvo-grace-cron.ts

@@ -0,0 +1,145 @@
+/**
+ * Phase 10 Task #68 — DSGVO grace-period cron.
+ *
+ * Daily job that scans `User` rows with `pendingDeletion=true` and
+ * `scheduledDeletionAt <= now()`, then dispatches a per-user
+ * dsgvo-deletion job. The cron itself never deletes data — it merely
+ * fans out work to the existing cascade worker (apps/worker/src/jobs/
+ * dsgvo-deletion.ts) so all the audit + per-table-strategy invariants
+ * stay in one place.
+ *
+ * Host-agnostic: callers supply the user-reader, the dispatcher, and
+ * the audit/logger shims. This keeps the file fully unit-testable
+ * without hauling in BullMQ or Prisma. The worker bootstrap wires it
+ * to the real Prisma + BullMQ implementations.
+ *
+ * Cadence is once per 24h. We deliberately do not run more often:
+ *  - Deletion is irreversible past the grace window, so a slightly
+ *    delayed dispatch is acceptable (and indeed safer than a tight
+ *    loop).
+ *  - One row per overdue user per day is enough — re-runs are
+ *    idempotent because the dsgvo-deletion job clears the User row.
+ *
+ * Repeat-job registration is exported as `registerDsgvoGraceCron` and
+ * called from `apps/worker/src/index.ts` on the `general` worker
+ * group. We use BullMQ's `Queue.upsertJobScheduler` API so re-deploys
+ * do not stack duplicate schedules.
+ */
+
+export const DSGVO_GRACE_CRON_JOB_ID = "dsgvo-grace-period-scan";
+export const DSGVO_GRACE_CRON_EVERY_MS = 24 * 60 * 60 * 1000;
+
+export type DsgvoGraceCandidate = {
+  userId: string;
+  tenantId: string | null;
+  scheduledDeletionAt: Date;
+  deletionRequestId: string | null;
+};
+
+export type DsgvoGraceCronDependencies = {
+  /** Returns every User row currently overdue (pendingDeletion=true AND scheduledDeletionAt<=now). */
+  listOverdueUsers: (now: Date) => Promise<DsgvoGraceCandidate[]>;
+  /**
+   * Enqueue one dsgvo-deletion cascade for the given candidate. Resolves
+   * with a job id for logging. May throw — failures are isolated per
+   * candidate so one bad row does not block the rest.
+   */
+  enqueueDeletion: (input: {
+    userId: string;
+    tenantId: string | null;
+    requestId: string;
+    actorType: "system";
+    actorId: string;
+    mode: "delete";
+    reason: string;
+  }) => Promise<{ jobId?: string }>;
+  /** Optional. Defaults to () => new Date(). */
+  now?: () => Date;
+  logger: {
+    info: (fields: Record<string, unknown>, msg: string) => void;
+    error: (fields: Record<string, unknown>, msg: string) => void;
+  };
+  /** Stable actor id stamped on dispatched-cascade audit rows. Default "dsgvo-grace-cron". */
+  systemActorId?: string;
+};
+
+export type DsgvoGraceCronResult = {
+  scanned: number;
+  dispatched: number;
+  failed: number;
+  failures: Array<{ userId: string; error: string }>;
+};
+
+/**
+ * Run one pass of the grace-period scan.
+ *
+ * Failure-isolation: if one candidate's dispatch throws, log + continue.
+ * Returning a non-zero `failed` count gives the BullMQ wrapper a clear
+ * signal to surface in metrics without blackholing the rest of the batch.
+ */
+export async function runDsgvoGraceCron(deps: DsgvoGraceCronDependencies): Promise<DsgvoGraceCronResult> {
+  const now = deps.now ? deps.now() : new Date();
+  const systemActorId = deps.systemActorId ?? "dsgvo-grace-cron";
+  const candidates = await deps.listOverdueUsers(now);
+  let dispatched = 0;
+  const failures: Array<{ userId: string; error: string }> = [];
+
+  for (const candidate of candidates) {
+    try {
+      // Reuse the deletion-request-id from the original user request so
+      // the dsgvo-deletion cascade's AuditEvent.metadata.requestId
+      // correlates back to the self-service POST that started it.
+      const requestId = candidate.deletionRequestId ?? `grace-${candidate.userId}-${now.getTime()}`;
+      const result = await deps.enqueueDeletion({
+        userId: candidate.userId,
+        tenantId: candidate.tenantId,
+        requestId,
+        actorType: "system",
+        actorId: systemActorId,
+        mode: "delete",
+        reason: "dsgvo-grace-period-elapsed",
+      });
+      dispatched += 1;
+      deps.logger.info(
+        { userId: candidate.userId, requestId, jobId: result.jobId ?? null, scheduledDeletionAt: candidate.scheduledDeletionAt.toISOString() },
+        "dsgvo-grace-cron.dispatched",
+      );
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      failures.push({ userId: candidate.userId, error: message });
+      deps.logger.error({ userId: candidate.userId, err: message }, "dsgvo-grace-cron.dispatch-failed");
+    }
+  }
+
+  return {
+    scanned: candidates.length,
+    dispatched,
+    failed: failures.length,
+    failures,
+  };
+}
+
+/**
+ * Helper for the worker bootstrap. Given a BullMQ-like queue with the
+ * `upsertJobScheduler` API, install (or replace) the 24h repeat schedule.
+ * Pinned to the queue's default job name + a stable scheduler key so a
+ * second call is a no-op (idempotent on redeploys).
+ */
+export async function registerDsgvoGraceCron(queue: {
+  upsertJobScheduler: (
+    schedulerId: string,
+    options: { every: number },
+    template: { name: string; data?: Record<string, unknown> },
+  ) => Promise<unknown>;
+}): Promise<void> {
+  // The job-template `data` must satisfy the maintenance CleanupJob
+  // discriminator so the registry-level handler can route on
+  // `payload.task` (see apps/worker/src/handlers.ts maintenance branch).
+  // Prior shape `{ kind: "dsgvo-grace-cron" }` had no `task` field and
+  // tripped UnsupportedWorkerJobError on every tick.
+  await queue.upsertJobScheduler(
+    DSGVO_GRACE_CRON_JOB_ID,
+    { every: DSGVO_GRACE_CRON_EVERY_MS },
+    { name: DSGVO_GRACE_CRON_JOB_ID, data: { kind: "cleanup.run", task: "dsgvo-grace-period-scan" } },
+  );
+}

hivegrid-cloud/apps/worker/src/jobs/dsgvo-verify.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { USER_FK_MAP } from "@toolctrl/db/dsgvo/user-fk-map";
+
+import { runDsgvoVerify, type DsgvoVerifyDependencies } from "./dsgvo-verify.js";
+
+function makeDeps(countRemainingImpl: (model: string, column: string) => number): {
+  deps: DsgvoVerifyDependencies;
+  audits: Array<Parameters<DsgvoVerifyDependencies["writeAuditEvent"]>[0]>;
+  escalations: Array<Parameters<NonNullable<DsgvoVerifyDependencies["raiseEscalation"]>>[0]>;
+} {
+  const audits: Array<Parameters<DsgvoVerifyDependencies["writeAuditEvent"]>[0]> = [];
+  const escalations: Array<Parameters<NonNullable<DsgvoVerifyDependencies["raiseEscalation"]>>[0]> = [];
+  const deps: DsgvoVerifyDependencies = {
+    client: {
+      countRemaining: vi.fn().mockImplementation(async (entry: { model: string; column: string }) =>
+        countRemainingImpl(entry.model, entry.column),
+      ),
+    },
+    writeAuditEvent: vi.fn().mockImplementation(async (input) => {
+      audits.push(input);
+    }),
+    raiseEscalation: vi.fn().mockImplementation(async (input) => {
+      escalations.push(input);
+    }),
+    logger: { info: vi.fn(), warn: vi.fn() },
+  };
+  return { deps, audits, escalations };
+}
+
+describe("runDsgvoVerify", () => {
+  it("emits a success AuditEvent when every hard-delete table is empty", async () => {
+    const { deps, audits, escalations } = makeDeps(() => 0);
+    const result = await runDsgvoVerify(deps, { userId: "u1", tenantId: "t1", requestId: "req-1" });
+    expect(result.ok).toBe(true);
+    expect(result.failures).toHaveLength(0);
+    expect(audits[0]?.outcome).toBe("success");
+    expect(escalations).toHaveLength(0);
+  });
+
+  it("only checks hard-delete tables — pseudonymized / tombstoned tables are skipped", async () => {
+    const inspected = new Set<string>();
+    const { deps } = makeDeps((model, column) => {
+      inspected.add(`${model}.${column}`);
+      return 0;
+    });
+    await runDsgvoVerify(deps, { userId: "u1", tenantId: null, requestId: "req-1" });
+    const hardDeleteKeys = new Set(USER_FK_MAP.filter((e) => e.strategy === "hard-delete").map((e) => `${e.model}.${e.column}`));
+    expect(inspected).toEqual(hardDeleteKeys);
+  });
+
+  it("emits a failure AuditEvent and escalates when stragglers remain", async () => {
+    const { deps, audits, escalations } = makeDeps((model) => (model === "TenantUser" ? 1 : 0));
+    const result = await runDsgvoVerify(deps, { userId: "u1", tenantId: "t1", requestId: "req-1" });
+    expect(result.ok).toBe(false);
+    expect(result.failures.some((f) => f.model === "TenantUser")).toBe(true);
+    expect(audits[0]?.outcome).toBe("failure");
+    expect(escalations).toHaveLength(1);
+    expect(escalations[0]?.failures.some((f) => f.model === "TenantUser")).toBe(true);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/dsgvo-verify.ts

@@ -0,0 +1,112 @@
+/**
+ * Phase 10 Task #68 — DSGVO deletion-verifier worker.
+ *
+ * Runs ~1 hour after the deletion cascade. For every USER_FK_MAP entry
+ * with strategy=`hard-delete`, queries the table for any straggler row
+ * still pointing at the deleted user. Tables in the `pseudonymize` and
+ * `retain-tombstone` buckets are skipped — these intentionally keep
+ * rows.
+ *
+ * A clean run emits an AuditEvent with outcome=success and metadata
+ * `{ tablesChecked, straggler: 0 }`. A run with any leftover rows emits
+ * an AuditEvent with outcome=failure and the per-table breakdown so the
+ * operator can escalate the incident.
+ */
+
+import { USER_FK_MAP, type UserFkMapEntry } from "@toolctrl/db/dsgvo/user-fk-map";
+
+export type DsgvoVerifyJobInput = {
+  userId: string;
+  tenantId: string | null;
+  requestId: string;
+};
+
+export type DsgvoVerifyTableResult = {
+  model: string;
+  column: string;
+  remainingRows: number;
+};
+
+export type DsgvoVerifyResult = {
+  userId: string;
+  requestId: string;
+  ok: boolean;
+  tablesChecked: number;
+  failures: DsgvoVerifyTableResult[];
+};
+
+export type DsgvoVerifyClient = {
+  countRemaining: (entry: UserFkMapEntry, userId: string) => Promise<number>;
+};
+
+export type DsgvoVerifyDependencies = {
+  client: DsgvoVerifyClient;
+  writeAuditEvent: (input: {
+    tenantId: string | null;
+    actorId: string;
+    actorType: string;
+    action: string;
+    targetType: string;
+    targetId: string;
+    regulatoryScope: string[];
+    dataSensitivity: "pii";
+    outcome: "success" | "failure";
+    metadata: Record<string, unknown>;
+  }) => Promise<void>;
+  raiseEscalation?: (input: { userId: string; failures: DsgvoVerifyTableResult[] }) => Promise<void>;
+  logger: { info: (fields: Record<string, unknown>, msg: string) => void; warn?: (fields: Record<string, unknown>, msg: string) => void };
+};
+
+export async function runDsgvoVerify(
+  deps: DsgvoVerifyDependencies,
+  job: DsgvoVerifyJobInput,
+): Promise<DsgvoVerifyResult> {
+  const hardDeleteEntries = USER_FK_MAP.filter((e) => e.strategy === "hard-delete");
+  const failures: DsgvoVerifyTableResult[] = [];
+
+  for (const entry of hardDeleteEntries) {
+    const remaining = await deps.client.countRemaining(entry, job.userId);
+    if (remaining > 0) {
+      failures.push({ model: entry.model, column: entry.column, remainingRows: remaining });
+    }
+  }
+
+  const ok = failures.length === 0;
+
+  await deps.writeAuditEvent({
+    tenantId: job.tenantId,
+    actorId: "system",
+    actorType: "system",
+    action: "VERIFY_USER_DELETION",
+    targetType: "User",
+    targetId: job.userId,
+    regulatoryScope: ["gdpr"],
+    dataSensitivity: "pii",
+    outcome: ok ? "success" : "failure",
+    metadata: {
+      requestId: job.requestId,
+      tablesChecked: hardDeleteEntries.length,
+      stragglerTables: failures,
+    },
+  });
+
+  if (!ok) {
+    deps.logger.warn?.(
+      { userId: job.userId, requestId: job.requestId, failureCount: failures.length },
+      "dsgvo-verify.stragglers-found",
+    );
+    if (deps.raiseEscalation) {
+      await deps.raiseEscalation({ userId: job.userId, failures });
+    }
+  } else {
+    deps.logger.info({ userId: job.userId, requestId: job.requestId }, "dsgvo-verify.clean");
+  }
+
+  return {
+    userId: job.userId,
+    requestId: job.requestId,
+    ok,
+    tablesChecked: hardDeleteEntries.length,
+    failures,
+  };
+}

hivegrid-cloud/apps/worker/src/jobs/email-sequence-bootstrap.test.ts

@@ -0,0 +1,99 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { bootstrapEmailSequenceWorkers } from "./email-sequence-bootstrap.js";
+import { EMAIL_SEQUENCE_TRIGGER_QUEUE } from "./email-sequence-trigger.js";
+import { EMAIL_SEQUENCE_QUEUE } from "./email-sequence-scheduler.js";
+
+function fakeWorker() {
+  const close = vi.fn(async () => undefined);
+  const calls: Array<{ queue: string; concurrency?: number }> = [];
+  const handles: Array<{ name: string; close: typeof close }> = [];
+  const createWorkerImpl = vi.fn((queueName: string, _processor: unknown, options: { concurrency?: number }) => {
+    calls.push({ queue: queueName, concurrency: options.concurrency });
+    const handle = { name: `fake-${queueName}`, close };
+    handles.push(handle);
+    return handle;
+  });
+  return { createWorkerImpl, calls, handles };
+}
+
+afterEach(() => {
+  delete process.env.MARKETING_SEQUENCES_ENABLED;
+});
+
+describe("bootstrapEmailSequenceWorkers", () => {
+  const triggerProcessor = async () => undefined;
+  const schedulerProcessor = async () => undefined;
+
+  it("returns disabled when WORKER_QUEUE_GROUP is not mail", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapEmailSequenceWorkers(
+      { WORKER_QUEUE_GROUP: "general" },
+      { createWorkerImpl, triggerProcessor, schedulerProcessor }
+    );
+    expect(result.status).toBe("disabled");
+    if (result.status === "disabled") expect(result.reason).toMatch(/non-mail/);
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  it("returns disabled when MARKETING_SEQUENCES_ENABLED is not explicitly true", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const unset = bootstrapEmailSequenceWorkers(
+      { WORKER_QUEUE_GROUP: "mail" },
+      { createWorkerImpl, triggerProcessor, schedulerProcessor }
+    );
+    expect(unset.status).toBe("disabled");
+
+    const result = bootstrapEmailSequenceWorkers(
+      { WORKER_QUEUE_GROUP: "mail", MARKETING_SEQUENCES_ENABLED: "false" },
+      { createWorkerImpl, triggerProcessor, schedulerProcessor }
+    );
+    expect(result.status).toBe("disabled");
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  it("starts both workers when explicitly enabled on mail group", () => {
+    const { createWorkerImpl, calls } = fakeWorker();
+    const result = bootstrapEmailSequenceWorkers(
+      { WORKER_QUEUE_GROUP: "mail", MARKETING_SEQUENCES_ENABLED: "true" },
+      { createWorkerImpl, triggerProcessor, schedulerProcessor }
+    );
+    expect(result.status).toBe("started");
+    expect(calls.map((c) => c.queue).sort()).toEqual([EMAIL_SEQUENCE_QUEUE, EMAIL_SEQUENCE_TRIGGER_QUEUE].sort());
+  });
+
+  it("honours concurrency env overrides", () => {
+    const { createWorkerImpl, calls } = fakeWorker();
+    bootstrapEmailSequenceWorkers(
+      {
+        WORKER_QUEUE_GROUP: "mail",
+        MARKETING_SEQUENCES_ENABLED: "true",
+        EMAIL_SEQUENCE_TRIGGER_CONCURRENCY: "20",
+        EMAIL_SEQUENCE_SCHEDULER_CONCURRENCY: "40",
+      },
+      { createWorkerImpl, triggerProcessor, schedulerProcessor }
+    );
+    const trigger = calls.find((c) => c.queue === EMAIL_SEQUENCE_TRIGGER_QUEUE);
+    const scheduler = calls.find((c) => c.queue === EMAIL_SEQUENCE_QUEUE);
+    expect(trigger?.concurrency).toBe(20);
+    expect(scheduler?.concurrency).toBe(40);
+  });
+
+  it("defaults concurrency to documented values when env unset", () => {
+    const { createWorkerImpl, calls } = fakeWorker();
+    bootstrapEmailSequenceWorkers(
+      { WORKER_QUEUE_GROUP: "mail", MARKETING_SEQUENCES_ENABLED: "true" },
+      { createWorkerImpl, triggerProcessor, schedulerProcessor }
+    );
+    expect(calls.find((c) => c.queue === EMAIL_SEQUENCE_TRIGGER_QUEUE)?.concurrency).toBe(5);
+    expect(calls.find((c) => c.queue === EMAIL_SEQUENCE_QUEUE)?.concurrency).toBe(10);
+  });
+
+  it("throws when createWorkerImpl is omitted", () => {
+    expect(() =>
+      bootstrapEmailSequenceWorkers(
+        { WORKER_QUEUE_GROUP: "mail", MARKETING_SEQUENCES_ENABLED: "true" },
+        { triggerProcessor, schedulerProcessor } as never
+      )
+    ).toThrow(/createWorkerImpl/);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/email-sequence-bootstrap.ts

@@ -0,0 +1,122 @@
+/**
+ * Marketing-Funnel Phase 3 (Phase 05 #80) — bootstrap helpers wiring
+ * the BullMQ subscribers for the email-sequence trigger handler +
+ * scheduler step-runner.
+ *
+ * Two BullMQ Workers are started inside the `mail` worker group:
+ *   - `outbox.marketing` — invokes the trigger handler. Outbox-dispatcher
+ *     publishes `marketing.lead.*` events here. We also accept bare
+ *     `lead.*` and `tenant.*` event-types when the publisher routes
+ *     them via a future fan-out.
+ *   - `email-sequence.step` — wakes up at the delayed-job's `runAt` and
+ *     invokes the step-runner.
+ *
+ * Both bootstrap helpers are gated on:
+ *   - WORKER_QUEUE_GROUP === "mail" (mail-bound outbound work)
+ *   - MARKETING_SEQUENCES_ENABLED === "true"
+ *
+ * Production wiring lives in `apps/worker/src/index.ts`; tests pass a
+ * fake `createWorkerImpl` to assert subscription semantics without
+ * spinning up BullMQ.
+ */
+
+import type { EnvRecord } from "@toolctrl/config";
+import { EMAIL_SEQUENCE_TRIGGER_QUEUE } from "./email-sequence-trigger.js";
+import { EMAIL_SEQUENCE_QUEUE } from "./email-sequence-scheduler.js";
+
+export type EmailSequenceBootstrapEnv = EnvRecord & {
+  MARKETING_SEQUENCES_ENABLED?: string;
+  WORKER_QUEUE_GROUP?: string;
+  EMAIL_SEQUENCE_TRIGGER_CONCURRENCY?: string;
+  EMAIL_SEQUENCE_SCHEDULER_CONCURRENCY?: string;
+};
+
+export interface EmailSequenceWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface EmailSequenceBootstrapLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface EmailSequenceBootstrapDeps {
+  readonly logger?: EmailSequenceBootstrapLogger;
+  /** Factory that returns a BullMQ-like Worker handle. */
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number }
+  ) => EmailSequenceWorkerHandle;
+  /** Trigger-job processor. Production wires Prisma-backed deps. */
+  readonly triggerProcessor: (job: { data: unknown }) => Promise<unknown>;
+  /** Scheduler-job processor. Production wires Prisma-backed deps. */
+  readonly schedulerProcessor: (job: { data: unknown }) => Promise<unknown>;
+}
+
+export type EmailSequenceBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | {
+      readonly status: "started";
+      readonly triggerHandle: EmailSequenceWorkerHandle;
+      readonly schedulerHandle: EmailSequenceWorkerHandle;
+    };
+
+const DEFAULT_TRIGGER_CONCURRENCY = 5;
+const DEFAULT_SCHEDULER_CONCURRENCY = 10;
+
+export function bootstrapEmailSequenceWorkers(
+  env: EmailSequenceBootstrapEnv,
+  deps: EmailSequenceBootstrapDeps
+): EmailSequenceBootstrapResult {
+  const logger = deps.logger;
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "mail") {
+    return { status: "disabled", reason: "non-mail worker group" };
+  }
+  if (env.MARKETING_SEQUENCES_ENABLED !== "true") {
+    logger?.info?.(
+      { component: "email-sequences", configured: env.MARKETING_SEQUENCES_ENABLED === "true" },
+      "email-sequence workers disabled: MARKETING_SEQUENCES_ENABLED must be true"
+    );
+    return { status: "disabled", reason: "MARKETING_SEQUENCES_ENABLED must be true" };
+  }
+  if (!deps.createWorkerImpl) {
+    throw new Error(
+      "bootstrapEmailSequenceWorkers requires deps.createWorkerImpl; the worker entry must wire the BullMQ Worker constructor."
+    );
+  }
+
+  const triggerConcurrency = parsePositiveInt(env.EMAIL_SEQUENCE_TRIGGER_CONCURRENCY, DEFAULT_TRIGGER_CONCURRENCY);
+  const schedulerConcurrency = parsePositiveInt(
+    env.EMAIL_SEQUENCE_SCHEDULER_CONCURRENCY,
+    DEFAULT_SCHEDULER_CONCURRENCY
+  );
+
+  const triggerHandle = deps.createWorkerImpl(EMAIL_SEQUENCE_TRIGGER_QUEUE, deps.triggerProcessor, {
+    concurrency: triggerConcurrency,
+  });
+  const schedulerHandle = deps.createWorkerImpl(EMAIL_SEQUENCE_QUEUE, deps.schedulerProcessor, {
+    concurrency: schedulerConcurrency,
+  });
+
+  logger?.info?.(
+    {
+      component: "email-sequences",
+      triggerQueue: EMAIL_SEQUENCE_TRIGGER_QUEUE,
+      schedulerQueue: EMAIL_SEQUENCE_QUEUE,
+      triggerConcurrency,
+      schedulerConcurrency,
+    },
+    "email-sequence workers started"
+  );
+
+  return { status: "started", triggerHandle, schedulerHandle };
+}
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}

hivegrid-cloud/apps/worker/src/jobs/email-sequence-prisma-store.test.ts

@@ -0,0 +1,238 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  createPrismaSequenceLoader,
+  createPrismaSequenceStore,
+  type PrismaLikeForSequences,
+} from "./email-sequence-prisma-store.js";
+
+const STEPS = [
+  {
+    stepIndex: 0,
+    delayHours: 0,
+    template: "welcome",
+    subject: "Welcome",
+  },
+] as const;
+
+function createPrismaFake(overrides: Partial<PrismaLikeForSequences> = {}): PrismaLikeForSequences {
+  return {
+    emailSequence: {
+      findMany: vi.fn(async () => []),
+      findUnique: vi.fn(async () => null),
+      ...overrides.emailSequence,
+    },
+    emailSequenceRun: {
+      create: vi.fn(async (args) => ({
+        id: "run_1",
+        sequenceId: String(args.data.sequenceId),
+        leadId: args.data.leadId,
+        tenantId: args.data.tenantId,
+        triggerEventId: args.data.triggerEventId,
+        status: String(args.data.status),
+        startedAt: new Date("2026-05-27T06:00:00Z"),
+        completedAt: null,
+        stepStateJson: "stepStateJson" in args.data ? args.data.stepStateJson : null,
+      })),
+      findFirst: vi.fn(async () => null),
+      update: vi.fn(async () => ({})),
+      ...overrides.emailSequenceRun,
+    },
+    emailSequenceStepDelivery: {
+      create: vi.fn(async (args) => ({
+        id: "delivery_1",
+        runId: args.data.runId,
+        stepIndex: args.data.stepIndex,
+        scheduledAt: args.data.scheduledAt,
+        status: args.data.status,
+      })),
+      updateMany: vi.fn(async () => ({})),
+      ...overrides.emailSequenceStepDelivery,
+    },
+  };
+}
+
+describe("createPrismaSequenceLoader", () => {
+  it("returns enabled sequence definitions parsed from Prisma rows", async () => {
+    const prisma = createPrismaFake({
+      emailSequence: {
+        findMany: vi.fn(async () => [
+          {
+            id: "seq_1",
+            tenantId: "tenant_1",
+            name: "Welcome",
+            description: "First touch",
+            triggerEvent: "lead.captured",
+            stepsJson: STEPS,
+            conditionsJson: [{ path: "lead.status", op: "eq", value: "new" }],
+            isEnabled: true,
+          },
+        ]),
+        findUnique: vi.fn(async () => null),
+      },
+    });
+    const loader = createPrismaSequenceLoader(prisma);
+
+    await expect(loader.listEnabledForEvent("lead.captured")).resolves.toEqual([
+      {
+        id: "seq_1",
+        tenantId: "tenant_1",
+        name: "Welcome",
+        description: "First touch",
+        triggerEvent: "lead.captured",
+        steps: STEPS,
+        conditions: [{ path: "lead.status", op: "eq", value: "new" }],
+        isEnabled: true,
+      },
+    ]);
+    expect(prisma.emailSequence.findMany).toHaveBeenCalledWith({
+      where: { triggerEvent: "lead.captured", isEnabled: true },
+      select: {
+        id: true,
+        tenantId: true,
+        name: true,
+        description: true,
+        triggerEvent: true,
+        stepsJson: true,
+        conditionsJson: true,
+        isEnabled: true,
+      },
+    });
+  });
+});
+
+describe("createPrismaSequenceStore", () => {
+  it("persists remembered trigger context with newly created runs", async () => {
+    const prisma = createPrismaFake();
+    const store = createPrismaSequenceStore(prisma);
+
+    store.rememberTrigger({
+      triggerEventId: "evt_1",
+      leadId: "lead_1",
+      tenantId: "tenant_1",
+      email: "alice@example.com",
+      recipientName: "Alice",
+      payload: { lead: { status: "new" } },
+    });
+
+    const run = await store.createRun({
+      sequenceId: "seq_1",
+      leadId: "lead_1",
+      tenantId: "tenant_1",
+      triggerEventId: "evt_1",
+    });
+
+    expect(run.stepStateJson).toEqual({
+      trigger: {
+        triggerEventId: "evt_1",
+        leadId: "lead_1",
+        tenantId: "tenant_1",
+        email: "alice@example.com",
+        recipientName: "Alice",
+        payload: { lead: { status: "new" } },
+      },
+    });
+    expect(prisma.emailSequenceRun.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        sequenceId: "seq_1",
+        status: "running",
+        stepStateJson: expect.objectContaining({
+          trigger: expect.objectContaining({ email: "alice@example.com" }),
+        }),
+      }),
+    });
+  });
+
+  it("rebuilds trigger context from the stored run state", async () => {
+    const prisma = createPrismaFake({
+      emailSequenceRun: {
+        create: vi.fn(),
+        update: vi.fn(),
+        findFirst: vi.fn(async () => ({
+          id: "run_1",
+          sequenceId: "seq_1",
+          leadId: "lead_1",
+          tenantId: "tenant_1",
+          triggerEventId: "evt_1",
+          status: "running",
+          startedAt: new Date("2026-05-27T06:00:00Z"),
+          completedAt: null,
+          stepStateJson: {
+            trigger: {
+              email: "alice@example.com",
+              recipientName: "Alice",
+              payload: { source: "form" },
+            },
+          },
+        })),
+      },
+    });
+    const store = createPrismaSequenceStore(prisma);
+
+    await expect(store.loadTriggerContext("run_1")).resolves.toEqual({
+      triggerEventId: "evt_1",
+      leadId: "lead_1",
+      tenantId: "tenant_1",
+      email: "alice@example.com",
+      recipientName: "Alice",
+      payload: { source: "form" },
+    });
+  });
+
+  it("loads steps from the related sequence row when the run query has no include", async () => {
+    const prisma = createPrismaFake({
+      emailSequence: {
+        findMany: vi.fn(async () => []),
+        findUnique: vi.fn(async () => ({
+          stepsJson: STEPS,
+        })),
+      },
+      emailSequenceRun: {
+        create: vi.fn(),
+        update: vi.fn(),
+        findFirst: vi.fn(async () => ({
+          id: "run_1",
+          sequenceId: "seq_1",
+          leadId: null,
+          tenantId: null,
+          triggerEventId: null,
+          status: "running",
+          startedAt: new Date("2026-05-27T06:00:00Z"),
+          completedAt: null,
+          stepStateJson: null,
+        })),
+      },
+    });
+    const store = createPrismaSequenceStore(prisma);
+
+    await expect(store.loadSteps("run_1")).resolves.toEqual(STEPS);
+    expect(prisma.emailSequence.findUnique).toHaveBeenCalledWith({
+      where: { id: "seq_1" },
+      select: { stepsJson: true },
+    });
+    expect(prisma.emailSequence.findMany).not.toHaveBeenCalled();
+  });
+
+  it("marks deliveries with only present delivery outcome fields", async () => {
+    const prisma = createPrismaFake();
+    const store = createPrismaSequenceStore(prisma);
+    const sentAt = new Date("2026-05-27T06:10:00Z");
+
+    await store.markDelivery({
+      runId: "run_1",
+      stepIndex: 0,
+      status: "sent",
+      sentAt,
+      providerMessageId: "msg_1",
+    });
+
+    expect(prisma.emailSequenceStepDelivery.updateMany).toHaveBeenCalledWith({
+      where: { runId: "run_1", stepIndex: 0 },
+      data: {
+        status: "sent",
+        sentAt,
+        providerMessageId: "msg_1",
+      },
+    });
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/email-sequence-prisma-store.ts

@@ -0,0 +1,312 @@
+/**
+ * Marketing-Funnel Phase 3 (Phase 05 #80) — Prisma-backed adapter for
+ * the email-sequence engine + step-runner storage surface.
+ *
+ * The pure-logic package `@toolctrl/email-sequences` accepts an
+ * injected `StepRunnerStore` so this module is the production wiring:
+ * EmailSequence + EmailSequenceRun + EmailSequenceStepDelivery rows
+ * mapped to/from the engine types.
+ *
+ * Loaded lazily from `email-sequence-bootstrap.ts` so a misconfigured
+ * Prisma client cannot break worker boot for other groups.
+ */
+
+import type {
+  SequenceDefinition,
+  SequenceRunRow,
+  SequenceTriggerContext,
+  StepRunnerStore,
+  SequenceRunStatus,
+  StepDeliveryStatus,
+  SequenceStep,
+} from "@toolctrl/email-sequences";
+import { parseSequenceSteps } from "@toolctrl/email-sequences";
+
+/**
+ * Narrow Prisma surface used by the adapter. Kept structural so tests
+ * can pass a fake without depending on the generated client. Methods
+ * mirror the real client one-for-one.
+ */
+export interface PrismaLikeForSequences {
+  emailSequence: {
+    findMany(args: { where: Record<string, unknown>; select: EmailSequenceDefinitionSelect }): Promise<Array<{
+      id: string;
+      tenantId: string | null;
+      name: string;
+      description: string | null;
+      triggerEvent: string;
+      stepsJson: unknown;
+      conditionsJson: unknown;
+      isEnabled: boolean;
+    }>>;
+    findUnique(args: { where: { id: string }; select: { stepsJson: true } }): Promise<{
+      stepsJson: unknown;
+    } | null>;
+  };
+  emailSequenceRun: {
+    create(args: {
+      data: {
+        sequenceId: string;
+        leadId: string | null;
+        tenantId: string | null;
+        triggerEventId: string | null;
+        status: string;
+      };
+    }): Promise<{
+      id: string;
+      sequenceId: string;
+      leadId: string | null;
+      tenantId: string | null;
+      triggerEventId: string | null;
+      status: string;
+      startedAt: Date;
+      completedAt: Date | null;
+      stepStateJson: unknown;
+    }>;
+    findFirst(args: { where: Record<string, unknown> }): Promise<{
+      id: string;
+      sequenceId: string;
+      leadId: string | null;
+      tenantId: string | null;
+      triggerEventId: string | null;
+      status: string;
+      startedAt: Date;
+      completedAt: Date | null;
+      stepStateJson: unknown;
+      sequence?: {
+        stepsJson: unknown;
+        conditionsJson: unknown;
+      };
+    } | null>;
+    update(args: {
+      where: { id: string };
+      data: Record<string, unknown>;
+    }): Promise<unknown>;
+  };
+  emailSequenceStepDelivery: {
+    create(args: {
+      data: { runId: string; stepIndex: number; scheduledAt: Date; status: string };
+    }): Promise<{ id: string; runId: string; stepIndex: number; scheduledAt: Date; status: string }>;
+    updateMany(args: {
+      where: { runId: string; stepIndex: number };
+      data: Record<string, unknown>;
+    }): Promise<unknown>;
+  };
+}
+
+const emailSequenceDefinitionSelect = {
+  id: true,
+  tenantId: true,
+  name: true,
+  description: true,
+  triggerEvent: true,
+  stepsJson: true,
+  conditionsJson: true,
+  isEnabled: true,
+} as const;
+
+type EmailSequenceDefinitionSelect = typeof emailSequenceDefinitionSelect;
+
+function rowToRun(row: {
+  id: string;
+  sequenceId: string;
+  leadId: string | null;
+  tenantId: string | null;
+  triggerEventId: string | null;
+  status: string;
+  startedAt: Date;
+  completedAt: Date | null;
+  stepStateJson: unknown;
+}): SequenceRunRow {
+  return {
+    id: row.id,
+    sequenceId: row.sequenceId,
+    leadId: row.leadId,
+    tenantId: row.tenantId,
+    triggerEventId: row.triggerEventId,
+    status: row.status as SequenceRunStatus,
+    startedAt: row.startedAt,
+    completedAt: row.completedAt,
+    stepStateJson:
+      row.stepStateJson && typeof row.stepStateJson === "object"
+        ? (row.stepStateJson as Record<string, unknown>)
+        : null,
+  };
+}
+
+function rowToDefinition(row: {
+  id: string;
+  tenantId: string | null;
+  name: string;
+  description: string | null;
+  triggerEvent: string;
+  stepsJson: unknown;
+  conditionsJson: unknown;
+  isEnabled: boolean;
+}): SequenceDefinition {
+  const steps = parseSequenceSteps(row.stepsJson);
+  const conditions =
+    row.conditionsJson && typeof row.conditionsJson === "object" && Array.isArray(row.conditionsJson)
+      ? (row.conditionsJson as SequenceDefinition["conditions"])
+      : undefined;
+  return {
+    id: row.id,
+    tenantId: row.tenantId,
+    name: row.name,
+    description: row.description,
+    triggerEvent: row.triggerEvent,
+    steps,
+    ...(conditions ? { conditions } : {}),
+    isEnabled: row.isEnabled,
+  };
+}
+
+/**
+ * The trigger handler keeps the `SequenceTriggerContext.payload`
+ * inside the run's stepStateJson so the step-runner can rebuild it
+ * when the delayed job fires hours/days later.
+ */
+function readTriggerFromStepState(
+  run: SequenceRunRow,
+  fallbackEmail?: string
+): SequenceTriggerContext | null {
+  const state = run.stepStateJson;
+  if (!state || typeof state !== "object") return null;
+  const trigger = (state as Record<string, unknown>)["trigger"];
+  if (!trigger || typeof trigger !== "object") return null;
+  const t = trigger as Record<string, unknown>;
+  const email = typeof t["email"] === "string" ? (t["email"] as string) : fallbackEmail;
+  if (!email) return null;
+  const payload = t["payload"] && typeof t["payload"] === "object" ? (t["payload"] as Record<string, unknown>) : {};
+  return {
+    triggerEventId: run.triggerEventId,
+    leadId: run.leadId,
+    tenantId: run.tenantId,
+    email,
+    recipientName: typeof t["recipientName"] === "string" ? (t["recipientName"] as string) : null,
+    payload,
+  };
+}
+
+export function createPrismaSequenceLoader(prisma: PrismaLikeForSequences) {
+  return {
+    async listEnabledForEvent(eventType: string): Promise<readonly SequenceDefinition[]> {
+      const rows = await prisma.emailSequence.findMany({
+        where: { triggerEvent: eventType, isEnabled: true },
+        select: emailSequenceDefinitionSelect,
+      });
+      return rows.map(rowToDefinition);
+    },
+  };
+}
+
+export interface PrismaSequenceStoreOptions {
+  /** Persist the trigger context inside the run row so the step-runner can rebuild it. */
+  attachTriggerContext?: SequenceTriggerContext | null;
+}
+
+export function createPrismaSequenceStore(prisma: PrismaLikeForSequences): StepRunnerStore & {
+  /** Cache the trigger context for subsequent step-runner invocations. */
+  rememberTrigger(trigger: SequenceTriggerContext | null): void;
+} {
+  let triggerCache: SequenceTriggerContext | null = null;
+
+  return {
+    rememberTrigger(trigger) {
+      triggerCache = trigger;
+    },
+
+    async createRun(input) {
+      const stepStateJson = triggerCache
+        ? ({ trigger: triggerCache } as unknown as Record<string, unknown>)
+        : null;
+      const row = await prisma.emailSequenceRun.create({
+        data: {
+          sequenceId: input.sequenceId,
+          leadId: input.leadId,
+          tenantId: input.tenantId,
+          triggerEventId: input.triggerEventId,
+          status: "running",
+          ...(stepStateJson ? { stepStateJson } : {}),
+        } as never,
+      });
+      return rowToRun(row);
+    },
+
+    async findRunByTrigger(sequenceId, triggerEventId) {
+      if (!triggerEventId) return null;
+      const row = await prisma.emailSequenceRun.findFirst({
+        where: { sequenceId, triggerEventId },
+      });
+      return row ? rowToRun(row) : null;
+    },
+
+    async updateRunStatus(runId, status, completedAt) {
+      await prisma.emailSequenceRun.update({
+        where: { id: runId },
+        data: completedAt ? { status, completedAt } : { status },
+      });
+    },
+
+    async createDelivery(input) {
+      const row = await prisma.emailSequenceStepDelivery.create({
+        data: {
+          runId: input.runId,
+          stepIndex: input.stepIndex,
+          scheduledAt: input.scheduledAt,
+          status: "pending",
+        },
+      });
+      return {
+        id: row.id,
+        runId: row.runId,
+        stepIndex: row.stepIndex,
+        status: row.status as StepDeliveryStatus,
+        scheduledAt: row.scheduledAt,
+      };
+    },
+
+    async loadRun(runId) {
+      const row = await prisma.emailSequenceRun.findFirst({ where: { id: runId } });
+      return row ? rowToRun(row) : null;
+    },
+
+    async loadSteps(runId): Promise<readonly SequenceStep[]> {
+      const row = await prisma.emailSequenceRun.findFirst({
+        where: { id: runId },
+      });
+      if (!row?.sequence) {
+        // findFirst without include returns the run only; the bootstrap
+        // wires a richer adapter when needed. Fall back to fetching the
+        // sequence directly.
+        const seqRun = row;
+        if (!seqRun) return [];
+        const seq = await prisma.emailSequence.findUnique({
+          where: { id: seqRun.sequenceId },
+          select: { stepsJson: true },
+        });
+        return seq ? parseSequenceSteps(seq.stepsJson) : [];
+      }
+      return parseSequenceSteps(row.sequence.stepsJson);
+    },
+
+    async loadTriggerContext(runId) {
+      const row = await prisma.emailSequenceRun.findFirst({ where: { id: runId } });
+      if (!row) return null;
+      const run = rowToRun(row);
+      return readTriggerFromStepState(run);
+    },
+
+    async markDelivery(args) {
+      const data: Record<string, unknown> = { status: args.status };
+      if (args.sentAt) data["sentAt"] = args.sentAt;
+      if (args.bouncedAt) data["bouncedAt"] = args.bouncedAt;
+      if (args.errorMessage) data["errorMessage"] = args.errorMessage;
+      if (args.providerMessageId) data["providerMessageId"] = args.providerMessageId;
+      await prisma.emailSequenceStepDelivery.updateMany({
+        where: { runId: args.runId, stepIndex: args.stepIndex },
+        data,
+      });
+    },
+  };
+}

hivegrid-cloud/apps/worker/src/jobs/email-sequence-runtime.test.ts

@@ -0,0 +1,336 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  buildEmailSequenceStepJobId,
+  createEmailSequenceSchedulerDeps,
+  createEmailSequenceTriggerDeps,
+} from "./email-sequence-runtime.js";
+import { EMAIL_SEQUENCE_STEP_JOB_NAME, runEmailSequenceStepJob } from "./email-sequence-scheduler.js";
+import type { SequenceRunRow, SequenceStep, SequenceTriggerContext } from "@toolctrl/email-sequences";
+import { verifyUnsubscribeToken } from "@toolctrl/email-sequences/unsubscribe-token";
+
+const loader = { load: vi.fn() };
+const store = {
+  save: vi.fn(),
+  loadRun: vi.fn<(runId: string) => Promise<SequenceRunRow | null>>(async () => null),
+  loadSteps: vi.fn<(runId: string) => Promise<readonly SequenceStep[]>>(async () => []),
+  loadTriggerContext: vi.fn<(runId: string) => Promise<SequenceTriggerContext | null>>(async () => null),
+  markDelivery: vi.fn(),
+  createRun: vi.fn(),
+  findRunByTrigger: vi.fn(async () => null),
+  updateRunStatus: vi.fn(),
+  createDelivery: vi.fn(),
+};
+const prisma = { emailSequence: {} };
+const TOKEN_SECRET = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+const mailMocks = vi.hoisted(() => ({
+  provider: {
+    id: "mailjet" as const,
+    send: vi.fn(),
+  },
+  createMailjetProviderFromEnv: vi.fn(),
+}));
+
+vi.mock("@toolctrl/db/client.server", () => ({
+  getDbClient: vi.fn(() => prisma),
+}));
+
+vi.mock("@toolctrl/mail", () => ({
+  createMailjetProviderFromEnv: mailMocks.createMailjetProviderFromEnv,
+}));
+
+vi.mock("./email-sequence-prisma-store.js", () => ({
+  createPrismaSequenceLoader: vi.fn(() => loader),
+  createPrismaSequenceStore: vi.fn(() => store),
+}));
+
+function logger() {
+  return {
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  };
+}
+
+describe("email sequence runtime deps", () => {
+  beforeEach(() => {
+    store.loadRun.mockResolvedValue(null);
+    store.loadSteps.mockResolvedValue([]);
+    store.loadTriggerContext.mockResolvedValue(null);
+    mailMocks.provider.send.mockResolvedValue({ provider: "mailjet", status: "queued", messageId: "msg_1" });
+    mailMocks.createMailjetProviderFromEnv.mockReturnValue(mailMocks.provider);
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+    delete process.env.APP_URL;
+    delete process.env.LEAD_CONVERSION_SECRET;
+    delete process.env.MARKETING_TOKEN_SECRET;
+    delete process.env.MARKETING_SEQUENCES_ENABLED;
+    delete process.env.MAILJET_API_KEY;
+    delete process.env.MAILJET_FROM_EMAIL;
+    delete process.env.MAILJET_FROM_NAME;
+  });
+
+  it("builds trigger deps that enqueue delayed step jobs idempotently", async () => {
+    const log = logger();
+    const stepQueue = { add: vi.fn(async () => ({ id: "step_job_1" })) };
+    const now = () => new Date("2026-01-01T00:00:00.000Z");
+    const deps = await createEmailSequenceTriggerDeps({ logger: log, now, stepQueue });
+
+    expect(deps).toMatchObject({ loader, store });
+    await deps?.queue.scheduleStep({ runId: "run_1", stepIndex: 2, runAt: new Date("2026-01-01T00:10:00.000Z") });
+    expect(stepQueue.add).toHaveBeenCalledWith(
+      EMAIL_SEQUENCE_STEP_JOB_NAME,
+      {
+        runId: "run_1",
+        stepIndex: 2,
+        scheduledFor: "2026-01-01T00:10:00.000Z",
+      },
+      {
+        attempts: 3,
+        backoff: { type: "exponential", delay: 1_000 },
+        delay: 600_000,
+        jobId: "email-sequence-step:run_1:2",
+        removeOnComplete: false,
+        removeOnFail: 1_000,
+      }
+    );
+    expect(log.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        op: "sequence.queue.scheduleStep.enqueued",
+        queue: "email-sequence.step",
+        jobId: "step_job_1",
+        runId: "run_1",
+        stepIndex: 2,
+      }),
+      "email sequence step enqueued"
+    );
+  });
+
+  it("keeps scheduleStep fail-closed when Redis is not configured", async () => {
+    const log = logger();
+    const deps = await createEmailSequenceTriggerDeps({ logger: log, env: {} });
+
+    await deps?.queue.scheduleStep({ runId: "run_1", stepIndex: 0, runAt: new Date("2026-01-01T00:00:00.000Z") });
+    expect(log.error).toHaveBeenCalledWith(
+      { op: "sequence.queue.scheduleStep.missing_redis", runId: "run_1", stepIndex: 0 },
+      "cannot schedule email sequence step: Redis URL is not configured"
+    );
+  });
+
+  it("builds stable email-sequence step job ids", () => {
+    expect(buildEmailSequenceStepJobId("run_1", 2)).toBe("email-sequence-step:run_1:2");
+  });
+
+  it("builds scheduler deps that fail closed without mail configuration", async () => {
+    const log = logger();
+    const stepQueue = { add: vi.fn(async () => ({ id: "next_step_job" })) };
+    const deps = await createEmailSequenceSchedulerDeps({
+      logger: log,
+      stepQueue,
+      env: {
+        APP_URL: "https://app.example.test/",
+        MARKETING_TOKEN_SECRET: TOKEN_SECRET,
+      },
+      now: () => new Date("2026-01-01T00:00:00.000Z"),
+    });
+
+    expect(deps).toMatchObject({ store });
+    await deps?.queue.scheduleStep({ runId: "run_2", stepIndex: 1, runAt: new Date("2026-01-01T00:00:00.000Z") });
+    expect(stepQueue.add).toHaveBeenCalledWith(
+      EMAIL_SEQUENCE_STEP_JOB_NAME,
+      expect.objectContaining({ runId: "run_2", stepIndex: 1 }),
+      expect.objectContaining({ jobId: "email-sequence-step:run_2:1" })
+    );
+    const rendered = await deps!.renderTemplate(
+      {
+        stepIndex: 0,
+        delayHours: 0,
+        template: "welcome",
+        subject: "Welcome {{trigger.recipientName}}",
+        variables: { ctaUrl: "https://app.example.test/start", productName: "ToolCTRL Test" },
+      },
+      {
+        triggerEventId: "evt_1",
+        leadId: "lead_1",
+        tenantId: "tenant_1",
+        email: "user@example.test",
+        recipientName: "User",
+        payload: {},
+      },
+      "run_render_1"
+    );
+    expect(rendered).toMatchObject({
+      subject: "Welcome User",
+    });
+    expect(rendered.html).toContain("ToolCTRL Test");
+    expect(rendered.html).toContain("https://app.example.test/start");
+    expect(rendered.html).toContain("/unsubscribe/u1.");
+    await expect(
+      deps?.sendMail({
+        to: "user@example.test",
+        subject: "Subject",
+        html: "<p>Hello</p>",
+        text: "Hello",
+        unsubscribeUrl: "https://app.example.test/unsubscribe/u1.token.sig",
+      })
+    ).resolves.toEqual({
+      ok: false,
+      errorTag: "email_sequence_mail_sender_unavailable",
+    });
+    expect(log.error).toHaveBeenCalledWith(
+      { op: "sequence.step.sendMail.missing_config", recipientCount: 1 },
+      "email sequence mail sender unavailable: MAILJET_API_KEY, MAILJET_SECRET_KEY and MAILJET_FROM_EMAIL are required"
+    );
+    const url = deps?.buildUnsubscribeUrl(
+      {
+        triggerEventId: "evt_1",
+        leadId: "lead_1",
+        tenantId: "tenant_1",
+        email: "user@example.test",
+        recipientName: "User",
+        payload: {},
+      },
+      "run_1"
+    );
+    expect(url?.startsWith("https://app.example.test/unsubscribe/u1.")).toBe(true);
+    const token = url?.split("/unsubscribe/")[1];
+    expect(verifyUnsubscribeToken(token, { secret: TOKEN_SECRET })).toMatchObject({
+      runId: "run_1",
+      leadId: "lead_1",
+      tenantId: "tenant_1",
+    });
+  });
+
+  it("sends sequence mail through @toolctrl/mail when Mailjet env is configured", async () => {
+    const log = logger();
+    const deps = await createEmailSequenceSchedulerDeps({
+      logger: log,
+      stepQueue: { add: vi.fn() },
+      env: {
+        APP_URL: "https://app.example.test",
+        MARKETING_TOKEN_SECRET: TOKEN_SECRET,
+        MAILJET_API_KEY: "mailer-key",
+        MAILJET_SECRET_KEY: "mailer-secret",
+        MAILJET_FROM_EMAIL: "sender@example.test",
+        MAILJET_FROM_NAME: "Sender",
+      },
+    });
+
+    await expect(
+      deps?.sendMail({
+        to: "user@example.test",
+        subject: "Subject",
+        html: "<p>Hello</p>",
+        text: "Hello",
+        unsubscribeUrl: "https://app.example.test/unsubscribe/u1.token.sig",
+        listUnsubscribeMailto: "mailto:unsubscribe@example.test",
+      })
+    ).resolves.toEqual({ ok: true, messageId: "msg_1" });
+    expect(mailMocks.createMailjetProviderFromEnv).toHaveBeenCalledWith({
+      APP_URL: "https://app.example.test",
+      MARKETING_TOKEN_SECRET: TOKEN_SECRET,
+      MAILJET_API_KEY: "mailer-key",
+      MAILJET_SECRET_KEY: "mailer-secret",
+      MAILJET_FROM_EMAIL: "sender@example.test",
+      MAILJET_FROM_NAME: "Sender",
+    });
+    expect(mailMocks.provider.send).toHaveBeenCalledWith({
+      to: ["user@example.test"],
+      subject: "Subject",
+      html: "<p>Hello</p>",
+      text: "Hello",
+      tags: ["email-sequence"],
+      precedenceBulk: true,
+      listUnsubscribe: "<https://app.example.test/unsubscribe/u1.token.sig>, <mailto:unsubscribe@example.test>",
+      headers: [{ name: "List-Unsubscribe-Post", value: "List-Unsubscribe=One-Click" }],
+      priorityOverride: "low",
+    });
+    expect(log.info).toHaveBeenCalledWith(
+      { op: "sequence.step.sendMail.sent", provider: "mailjet", recipientCount: 1, messageId: "msg_1" },
+      "email sequence mail accepted by provider"
+    );
+  });
+
+  it("maps @toolctrl/mail provider failures to stable sequence error tags", async () => {
+    const log = logger();
+    mailMocks.provider.send.mockRejectedValue(Object.assign(new Error("provider rejected"), { statusCode: 429 }));
+    const deps = await createEmailSequenceSchedulerDeps({
+      logger: log,
+      stepQueue: { add: vi.fn() },
+      env: {
+        MAILJET_API_KEY: "mailer-key",
+        MAILJET_SECRET_KEY: "mailer-secret",
+        MAILJET_FROM_EMAIL: "sender@example.test",
+      },
+    });
+
+    await expect(
+      deps?.sendMail({
+        to: "user@example.test",
+        subject: "Subject",
+        html: "<p>Hello</p>",
+        text: "Hello",
+        unsubscribeUrl: "https://app.example.test/unsubscribe/u1.token.sig",
+      })
+    ).resolves.toEqual({ ok: false, errorTag: "email_sequence_mail_provider_4xx" });
+    expect(log.error).toHaveBeenCalledWith(
+      {
+        op: "sequence.step.sendMail.failed",
+        provider: "mailjet",
+        recipientCount: 1,
+        errorTag: "email_sequence_mail_provider_4xx",
+        statusCode: 429,
+      },
+      "email sequence mail provider send failed"
+    );
+  });
+
+  it("keeps production step jobs fail-closed while mail delivery is unavailable", async () => {
+    process.env.MARKETING_SEQUENCES_ENABLED = "true";
+    const log = logger();
+    const deps = await createEmailSequenceSchedulerDeps({
+      logger: log,
+      stepQueue: { add: vi.fn() },
+      env: {
+        APP_URL: "https://app.example.test",
+        MARKETING_TOKEN_SECRET: TOKEN_SECRET,
+      },
+    });
+
+    store.loadRun.mockResolvedValue({
+      id: "run_mail_unavailable",
+      sequenceId: "seq_1",
+      leadId: "lead_1",
+      tenantId: null,
+      triggerEventId: "evt_1",
+      status: "running",
+      startedAt: new Date("2026-01-01T00:00:00.000Z"),
+      completedAt: null,
+      stepStateJson: null,
+    });
+    store.loadSteps.mockResolvedValue([{ stepIndex: 0, delayHours: 0, template: "welcome", subject: "Welcome" }]);
+    store.loadTriggerContext.mockResolvedValue({
+      triggerEventId: "evt_1",
+      leadId: "lead_1",
+      tenantId: null,
+      email: "user@example.test",
+      recipientName: "User",
+      payload: {},
+    });
+
+    await expect(
+      runEmailSequenceStepJob({ runId: "run_mail_unavailable", stepIndex: 0 }, deps!)
+    ).resolves.toEqual({ action: "failed", completed: false });
+    expect(store.markDelivery).toHaveBeenCalledWith({
+      runId: "run_mail_unavailable",
+      stepIndex: 0,
+      status: "failed",
+      errorMessage: "email_sequence_mail_sender_unavailable",
+    });
+    expect(log.error).toHaveBeenCalledWith(
+      expect.objectContaining({ op: "sequence.step.sendMail.missing_config" }),
+      expect.any(String)
+    );
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/email-sequence-runtime.ts

@@ -0,0 +1,440 @@
+/**
+ * Marketing-Funnel Phase 3 (Phase 05 #80) — production runtime-deps
+ * builder for the email-sequence trigger handler + scheduler.
+ *
+ * Returns the injected dependency bundles consumed by
+ * `runSequenceTriggerJob` + `runEmailSequenceStepJob`. Loaded lazily
+ * inside the BullMQ subscribers so a misconfigured optional adapter
+ * (e.g. mail provider unavailable) cannot crash worker boot for other
+ * groups.
+ *
+ * Returns `null` when the deps cannot be built — the BullMQ subscriber
+ * treats that as a no-op so the queue drains cleanly while operators
+ * fix the upstream config.
+ */
+
+import { parseRedisUrl, type RedisConnectionEnv } from "@toolctrl/queue";
+import { resolvePath, type StepRunnerDeps, type SequenceEngineQueue } from "@toolctrl/email-sequences";
+import { buildUnsubscribeUrl as buildSignedUnsubscribeUrl } from "@toolctrl/email-sequences/unsubscribe-token";
+import type { MailProvider } from "@toolctrl/mail";
+import {
+  GenericStepEmail,
+  ReActivationEmail,
+  renderEmail,
+  TrialReminderEmail,
+  WelcomeEmail,
+} from "@toolctrl/email-templates";
+import {
+  EMAIL_SEQUENCE_QUEUE,
+  EMAIL_SEQUENCE_STEP_JOB_NAME,
+  type EmailSequenceStepJobData,
+} from "./email-sequence-scheduler.js";
+import type { SequenceTriggerDeps } from "./email-sequence-trigger.js";
+
+export interface EmailSequenceRuntimeLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface EmailSequenceRuntimeOptions {
+  readonly logger: EmailSequenceRuntimeLogger;
+  readonly env?: EmailSequenceRuntimeEnv;
+  readonly now?: () => Date;
+  readonly stepQueue?: EmailSequenceStepQueue;
+  readonly mailProvider?: EmailSequenceMailProvider;
+}
+
+type EmailSequenceRuntimeEnv = RedisConnectionEnv & {
+  APP_URL?: string;
+  APP_NAME?: string;
+  LEAD_CONVERSION_SECRET?: string;
+  MARKETING_TOKEN_SECRET?: string;
+  MAILJET_API_KEY?: string;
+  MAILJET_SECRET_KEY?: string;
+  MAILJET_FROM_EMAIL?: string;
+  MAILJET_FROM_NAME?: string;
+};
+
+type EmailSequenceMailProvider = Pick<MailProvider, "id" | "send">;
+
+export interface EmailSequenceStepQueue {
+  add(
+    name: typeof EMAIL_SEQUENCE_STEP_JOB_NAME,
+    data: EmailSequenceStepJobData,
+    options: EmailSequenceStepQueueOptions
+  ): Promise<{ id?: string }>;
+  close?(): Promise<void>;
+}
+
+export type EmailSequenceStepQueueOptions = {
+  attempts: number;
+  backoff: { type: "exponential"; delay: number };
+  delay: number;
+  jobId: string;
+  removeOnComplete: boolean | number;
+  removeOnFail: boolean | number;
+};
+
+const EMAIL_SEQUENCE_STEP_ATTEMPTS = 3;
+const EMAIL_SEQUENCE_STEP_BACKOFF_MS = 1_000;
+const EMAIL_SEQUENCE_STEP_REMOVE_ON_FAIL = 1_000;
+
+/**
+ * Build the trigger-handler dependency bundle. Returns null if Prisma
+ * is not available in this build.
+ */
+export async function createEmailSequenceTriggerDeps(
+  opts: EmailSequenceRuntimeOptions
+): Promise<SequenceTriggerDeps | null> {
+  try {
+    const [{ getDbClient }, { createPrismaSequenceLoader, createPrismaSequenceStore }] = await Promise.all([
+      import("@toolctrl/db/client.server"),
+      import("./email-sequence-prisma-store.js"),
+    ]);
+    const prisma = getDbClient() as unknown as Parameters<typeof createPrismaSequenceLoader>[0];
+    const loader = createPrismaSequenceLoader(prisma);
+    const store = createPrismaSequenceStore(prisma);
+    return {
+      loader,
+      store,
+      queue: {
+        scheduleStep: (args) => scheduleEmailSequenceStep(args, opts),
+      },
+      logger: opts.logger,
+    };
+  } catch (error) {
+    opts.logger.error({ error }, "failed to build email-sequence trigger deps");
+    return null;
+  }
+}
+
+/**
+ * Build the scheduler-handler dependency bundle. Returns null if the
+ * Prisma client, mail sender or template renderer are unavailable.
+ */
+export async function createEmailSequenceSchedulerDeps(
+  opts: EmailSequenceRuntimeOptions
+): Promise<StepRunnerDeps | null> {
+  try {
+    const [{ getDbClient }, { createPrismaSequenceStore }] = await Promise.all([
+      import("@toolctrl/db/client.server"),
+      import("./email-sequence-prisma-store.js"),
+    ]);
+    const prisma = getDbClient() as unknown as Parameters<typeof createPrismaSequenceStore>[0];
+    const store = createPrismaSequenceStore(prisma);
+    const sendMail = await createEmailSequenceMailSender(opts);
+    return {
+      store,
+      queue: {
+        scheduleStep: (args) => scheduleEmailSequenceStep(args, opts),
+      },
+      logger: opts.logger,
+      renderTemplate: (step, trigger, runId) => renderSequenceTemplate(step, trigger, runId, opts),
+      sendMail,
+      buildUnsubscribeUrl: (trigger, runId) => {
+        const env = opts.env ?? process.env;
+        const baseUrl = env.APP_URL?.trim() || "https://app.example.com";
+        return buildSignedUnsubscribeUrl({
+          baseUrl,
+          runId,
+          leadId: trigger.leadId,
+          tenantId: trigger.tenantId,
+          secret: env.MARKETING_TOKEN_SECRET ?? env.LEAD_CONVERSION_SECRET,
+          now: opts.now,
+        });
+      },
+    };
+  } catch (error) {
+    opts.logger.error({ error }, "failed to build email-sequence scheduler deps");
+    return null;
+  }
+}
+
+async function createEmailSequenceMailSender(opts: EmailSequenceRuntimeOptions): Promise<StepRunnerDeps["sendMail"]> {
+  const provider = opts.mailProvider ?? (await createEmailSequenceMailProviderFromEnv(opts));
+  if (!provider) {
+    return async () => {
+      opts.logger.error(
+        { op: "sequence.step.sendMail.missing_config", recipientCount: 1 },
+        "email sequence mail sender unavailable: MAILJET_API_KEY, MAILJET_SECRET_KEY and MAILJET_FROM_EMAIL are required"
+      );
+      return { ok: false, errorTag: "email_sequence_mail_sender_unavailable" };
+    };
+  }
+
+  return async (input) => {
+    try {
+      const result = await provider.send({
+        to: [input.to],
+        subject: input.subject,
+        html: input.html,
+        text: input.text,
+        tags: ["email-sequence"],
+        precedenceBulk: true,
+        listUnsubscribe: formatListUnsubscribe(input.unsubscribeUrl, input.listUnsubscribeMailto),
+        headers: [{ name: "List-Unsubscribe-Post", value: "List-Unsubscribe=One-Click" }],
+        priorityOverride: "low",
+      });
+      opts.logger.info?.(
+        { op: "sequence.step.sendMail.sent", provider: result.provider, recipientCount: 1, messageId: result.messageId },
+        "email sequence mail accepted by provider"
+      );
+      return { ok: true, messageId: result.messageId };
+    } catch (error) {
+      const statusCode = readStatusCode(error);
+      const errorTag = statusCode ? `email_sequence_mail_provider_${Math.floor(statusCode / 100)}xx` : "email_sequence_mail_provider_failed";
+      opts.logger.error(
+        {
+          op: "sequence.step.sendMail.failed",
+          provider: provider.id,
+          recipientCount: 1,
+          errorTag,
+          ...(statusCode ? { statusCode } : {}),
+        },
+        "email sequence mail provider send failed"
+      );
+      return { ok: false, errorTag };
+    }
+  };
+}
+
+async function createEmailSequenceMailProviderFromEnv(opts: EmailSequenceRuntimeOptions): Promise<EmailSequenceMailProvider | null> {
+  const env = opts.env ?? process.env;
+  if (!env.MAILJET_API_KEY?.trim() || !env.MAILJET_SECRET_KEY?.trim() || !env.MAILJET_FROM_EMAIL?.trim()) {
+    return null;
+  }
+
+  try {
+    const { createMailjetProviderFromEnv } = await import("@toolctrl/mail");
+    return createMailjetProviderFromEnv(env);
+  } catch (error) {
+    opts.logger.error(
+      { op: "sequence.step.sendMail.provider_init_failed", error },
+      "failed to initialize email sequence mail provider"
+    );
+    return null;
+  }
+}
+
+function formatListUnsubscribe(unsubscribeUrl: string, mailto: string | undefined): string {
+  const urlHeader = `<${unsubscribeUrl}>`;
+  return mailto ? `${urlHeader}, <${mailto}>` : urlHeader;
+}
+
+function readStatusCode(error: unknown): number | undefined {
+  return error && typeof error === "object" && typeof (error as { statusCode?: unknown }).statusCode === "number"
+    ? (error as { statusCode: number }).statusCode
+    : undefined;
+}
+
+function renderSequenceTemplate(
+  step: Parameters<StepRunnerDeps["renderTemplate"]>[0],
+  trigger: Parameters<StepRunnerDeps["renderTemplate"]>[1],
+  runId: string,
+  opts: EmailSequenceRuntimeOptions
+) {
+  const vars = step.variables ?? {};
+  const env = opts.env ?? process.env;
+  const appUrl = (env.APP_URL?.trim() || "https://app.example.com").replace(/\/+$/, "");
+  const defaultProductName = env.APP_NAME?.trim() || "ToolCTRL";
+  const productName = stringVariable(vars, trigger, "productName", defaultProductName) ?? defaultProductName;
+  const recipientName = stringVariable(vars, trigger, "recipientName", trigger.recipientName ?? undefined);
+  const unsubscribeUrl = buildSignedUnsubscribeUrl({
+    baseUrl: appUrl,
+    runId,
+    leadId: trigger.leadId,
+    tenantId: trigger.tenantId,
+    secret: env.MARKETING_TOKEN_SECRET ?? env.LEAD_CONVERSION_SECRET,
+    now: opts.now,
+  });
+  const subject = interpolateTemplate(step.subject, vars, trigger);
+
+  switch (step.template) {
+    case "welcome": {
+      const ctaUrl = stringVariable(vars, trigger, "ctaUrl", appUrl) ?? appUrl;
+      const ctaLabel = stringVariable(vars, trigger, "ctaLabel");
+      const senderName = stringVariable(vars, trigger, "senderName");
+      const email = renderEmail(WelcomeEmail({
+        productName,
+        ctaUrl,
+        unsubscribeUrl,
+        ...(recipientName ? { recipientName } : {}),
+        ...(ctaLabel ? { ctaLabel } : {}),
+        ...(senderName ? { senderName } : {}),
+      }));
+      return { ...email, subject };
+    }
+    case "trial-reminder": {
+      const upgradeUrl = stringVariable(vars, trigger, "upgradeUrl", `${appUrl}/settings/subscription`);
+      const email = renderEmail(TrialReminderEmail({
+        productName,
+        daysRemaining: numberVariable(vars, trigger, "daysRemaining", 3),
+        upgradeUrl: upgradeUrl ?? `${appUrl}/settings/subscription`,
+        unsubscribeUrl,
+        ...(recipientName ? { recipientName } : {}),
+      }));
+      return { ...email, subject };
+    }
+    case "re-activation": {
+      const highlight = stringVariable(vars, trigger, "highlight");
+      const returnUrl = stringVariable(vars, trigger, "returnUrl", appUrl);
+      const email = renderEmail(ReActivationEmail({
+        productName,
+        returnUrl: returnUrl ?? appUrl,
+        unsubscribeUrl,
+        ...(recipientName ? { recipientName } : {}),
+        ...(highlight ? { highlight } : {}),
+      }));
+      return { ...email, subject };
+    }
+    case "generic-step": {
+      const headline = stringVariable(vars, trigger, "headline", subject) ?? subject;
+      const ctaUrl = stringVariable(vars, trigger, "ctaUrl");
+      const ctaLabel = stringVariable(vars, trigger, "ctaLabel");
+      const email = renderEmail(GenericStepEmail({
+        productName,
+        headline,
+        bodyParagraphs: stringArrayVariable(vars, trigger, "bodyParagraphs", [headline]),
+        unsubscribeUrl,
+        ...(recipientName ? { recipientName } : {}),
+        ...(ctaUrl ? { ctaUrl } : {}),
+        ...(ctaLabel ? { ctaLabel } : {}),
+      }));
+      return { ...email, subject };
+    }
+    default:
+      throw new Error(`email_sequence_template_unknown:${step.template}`);
+  }
+}
+
+function variableValue(
+  vars: Readonly<Record<string, unknown>>,
+  trigger: Parameters<StepRunnerDeps["renderTemplate"]>[1],
+  key: string
+) {
+  const value = vars[key];
+  if (typeof value === "string" && value.startsWith("$.")) {
+    return resolvePath({ trigger, payload: trigger.payload }, value.slice(2));
+  }
+  return value;
+}
+
+function stringVariable(
+  vars: Readonly<Record<string, unknown>>,
+  trigger: Parameters<StepRunnerDeps["renderTemplate"]>[1],
+  key: string,
+  fallback?: string
+): string | undefined {
+  const value = variableValue(vars, trigger, key);
+  if (typeof value === "string" && value.trim().length > 0) return interpolateTemplate(value, vars, trigger);
+  if (typeof value === "number" || typeof value === "boolean") return String(value);
+  return fallback;
+}
+
+function numberVariable(
+  vars: Readonly<Record<string, unknown>>,
+  trigger: Parameters<StepRunnerDeps["renderTemplate"]>[1],
+  key: string,
+  fallback: number
+): number {
+  const value = variableValue(vars, trigger, key);
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "string") {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return fallback;
+}
+
+function stringArrayVariable(
+  vars: Readonly<Record<string, unknown>>,
+  trigger: Parameters<StepRunnerDeps["renderTemplate"]>[1],
+  key: string,
+  fallback: readonly string[]
+): readonly string[] {
+  const value = variableValue(vars, trigger, key);
+  if (Array.isArray(value)) {
+    const lines = value.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
+    return lines.length > 0 ? lines.map((line) => interpolateTemplate(line, vars, trigger)) : fallback;
+  }
+  if (typeof value === "string" && value.trim().length > 0) {
+    return [interpolateTemplate(value, vars, trigger)];
+  }
+  return fallback;
+}
+
+function interpolateTemplate(
+  template: string,
+  vars: Readonly<Record<string, unknown>>,
+  trigger: Parameters<StepRunnerDeps["renderTemplate"]>[1]
+) {
+  return template.replace(/\{\{\s*([^}]+?)\s*\}\}/gu, (_match, rawPath: string) => {
+    const path = rawPath.trim();
+    const direct = variableValue(vars, trigger, path);
+    const value = direct === undefined ? resolvePath({ trigger, payload: trigger.payload }, path) : direct;
+    return typeof value === "string" || typeof value === "number" || typeof value === "boolean" ? String(value) : "";
+  });
+}
+
+async function scheduleEmailSequenceStep(
+  args: Parameters<SequenceEngineQueue["scheduleStep"]>[0],
+  opts: EmailSequenceRuntimeOptions
+): Promise<void> {
+  const queue = opts.stepQueue ?? (await createEmailSequenceStepQueue(opts.env ?? process.env));
+  if (!queue) {
+    opts.logger.error(
+      { op: "sequence.queue.scheduleStep.missing_redis", runId: args.runId, stepIndex: args.stepIndex },
+      "cannot schedule email sequence step: Redis URL is not configured"
+    );
+    return;
+  }
+
+  const now = (opts.now ?? (() => new Date()))();
+  const delay = Math.max(0, args.runAt.getTime() - now.getTime());
+  const payload: EmailSequenceStepJobData = {
+    runId: args.runId,
+    stepIndex: args.stepIndex,
+    scheduledFor: args.runAt.toISOString(),
+  };
+  const jobId = buildEmailSequenceStepJobId(args.runId, args.stepIndex);
+
+  try {
+    const job = await queue.add(EMAIL_SEQUENCE_STEP_JOB_NAME, payload, {
+      attempts: EMAIL_SEQUENCE_STEP_ATTEMPTS,
+      backoff: { type: "exponential", delay: EMAIL_SEQUENCE_STEP_BACKOFF_MS },
+      delay,
+      jobId,
+      removeOnComplete: false,
+      removeOnFail: EMAIL_SEQUENCE_STEP_REMOVE_ON_FAIL,
+    });
+    opts.logger.info?.(
+      {
+        op: "sequence.queue.scheduleStep.enqueued",
+        queue: EMAIL_SEQUENCE_QUEUE,
+        jobId: job.id ?? jobId,
+        runId: args.runId,
+        stepIndex: args.stepIndex,
+        delay,
+      },
+      "email sequence step enqueued"
+    );
+  } finally {
+    if (!opts.stepQueue) {
+      await queue.close?.().catch((error) => {
+        opts.logger.warn?.({ error, queue: EMAIL_SEQUENCE_QUEUE }, "failed to close email sequence step queue");
+      });
+    }
+  }
+}
+
+export function buildEmailSequenceStepJobId(runId: string, stepIndex: number): string {
+  return `email-sequence-step:${runId}:${stepIndex}`;
+}
+
+async function createEmailSequenceStepQueue(env: RedisConnectionEnv): Promise<EmailSequenceStepQueue | null> {
+  const redisUrl = env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim();
+  if (!redisUrl) return null;
+  const { Queue } = await import("bullmq");
+  return new Queue(EMAIL_SEQUENCE_QUEUE, { connection: parseRedisUrl(redisUrl) }) as EmailSequenceStepQueue;
+}

hivegrid-cloud/apps/worker/src/jobs/email-sequence-scheduler.test.ts

@@ -0,0 +1,74 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  EMAIL_SEQUENCE_QUEUE,
+  EmailSequenceStepJobError,
+  parseEmailSequenceStepJob,
+  runEmailSequenceStepJob,
+} from "./email-sequence-scheduler.js";
+import type { StepRunnerDeps } from "@toolctrl/email-sequences";
+
+function buildDeps(): StepRunnerDeps {
+  return {
+    store: {
+      loadRun: vi.fn(async () => null),
+      loadSteps: vi.fn(async () => []),
+      loadTriggerContext: vi.fn(async () => null),
+      markDelivery: vi.fn(),
+      createRun: vi.fn(),
+      findRunByTrigger: vi.fn(async () => null),
+      updateRunStatus: vi.fn(),
+      createDelivery: vi.fn(),
+    },
+    queue: { scheduleStep: vi.fn() },
+    renderTemplate: () => ({ html: "<p>x</p>", text: "x", subject: "s" }),
+    sendMail: async () => ({ ok: true }),
+    buildUnsubscribeUrl: () => "https://app.example.com/unsubscribe/tok",
+  };
+}
+
+afterEach(() => {
+  delete process.env.MARKETING_SEQUENCES_ENABLED;
+});
+
+beforeEach(() => {
+  process.env.MARKETING_SEQUENCES_ENABLED = "true";
+});
+
+describe("EMAIL_SEQUENCE_QUEUE", () => {
+  it("exposes the queue name", () => {
+    expect(EMAIL_SEQUENCE_QUEUE).toBe("email-sequence.step");
+  });
+});
+
+describe("parseEmailSequenceStepJob", () => {
+  it("accepts a well-formed envelope", () => {
+    const job = parseEmailSequenceStepJob({ runId: "run_1", stepIndex: 2 });
+    expect(job.runId).toBe("run_1");
+    expect(job.stepIndex).toBe(2);
+  });
+  it("rejects missing runId", () => {
+    expect(() => parseEmailSequenceStepJob({ stepIndex: 0 })).toThrow(EmailSequenceStepJobError);
+  });
+  it("rejects negative stepIndex", () => {
+    expect(() => parseEmailSequenceStepJob({ runId: "run_1", stepIndex: -1 })).toThrow(EmailSequenceStepJobError);
+  });
+  it("rejects non-integer stepIndex", () => {
+    expect(() => parseEmailSequenceStepJob({ runId: "run_1", stepIndex: 0.5 })).toThrow(EmailSequenceStepJobError);
+  });
+});
+
+describe("runEmailSequenceStepJob", () => {
+  it("returns cancelled when killswitch active", async () => {
+    process.env.MARKETING_SEQUENCES_ENABLED = "false";
+    const result = await runEmailSequenceStepJob({ runId: "run_1", stepIndex: 0 }, buildDeps());
+    expect(result.action).toBe("cancelled");
+  });
+  it("invokes runStep when enabled", async () => {
+    const deps = buildDeps();
+    const result = await runEmailSequenceStepJob({ runId: "run_1", stepIndex: 0 }, deps);
+    // runStep cancels because loadRun returns null in our fake — that is
+    // the right deterministic outcome here. We just verify the wiring.
+    expect(result.action).toBe("cancelled");
+    expect(deps.store.loadRun).toHaveBeenCalledWith("run_1");
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/email-sequence-scheduler.ts

@@ -0,0 +1,62 @@
+/**
+ * Marketing-Funnel Phase 3 (Phase 05 #80) — sequence step scheduler.
+ *
+ * BullMQ job handler for the `email-sequence.step` queue. The trigger
+ * handler (and the step engine itself) enqueues a delayed job with
+ * `{ runId, stepIndex, runAt }`. BullMQ wakes the worker at `runAt`
+ * and this handler invokes the pure {@link runStep} routine with the
+ * injected store + mail-sender + template renderer.
+ *
+ * Storage / mail / template surfaces are bound by the worker bootstrap
+ * in production; tests pass in-memory fakes.
+ */
+
+import { runStep, type StepRunnerDeps, type StepRunResult } from "@toolctrl/email-sequences";
+
+export const EMAIL_SEQUENCE_QUEUE = "email-sequence.step" as const;
+export const EMAIL_SEQUENCE_STEP_JOB_NAME = "run-email-sequence-step" as const;
+
+export interface EmailSequenceStepJobData {
+  readonly runId: string;
+  readonly stepIndex: number;
+  /** ISO timestamp the job was scheduled for — for tracing only. */
+  readonly scheduledFor?: string;
+}
+
+export class EmailSequenceStepJobError extends Error {
+  override readonly name = "EmailSequenceStepJobError";
+  readonly code: "INVALID_PAYLOAD";
+  constructor(message: string, code: EmailSequenceStepJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export function parseEmailSequenceStepJob(data: unknown): EmailSequenceStepJobData {
+  if (!data || typeof data !== "object") {
+    throw new EmailSequenceStepJobError("scheduler job missing data envelope", "INVALID_PAYLOAD");
+  }
+  const e = data as Record<string, unknown>;
+  if (typeof e.runId !== "string" || e.runId.length === 0) {
+    throw new EmailSequenceStepJobError("scheduler job missing runId", "INVALID_PAYLOAD");
+  }
+  if (typeof e.stepIndex !== "number" || !Number.isInteger(e.stepIndex) || e.stepIndex < 0) {
+    throw new EmailSequenceStepJobError("scheduler job stepIndex invalid", "INVALID_PAYLOAD");
+  }
+  return {
+    runId: e.runId,
+    stepIndex: e.stepIndex,
+    scheduledFor: typeof e.scheduledFor === "string" ? e.scheduledFor : undefined,
+  };
+}
+
+export async function runEmailSequenceStepJob(
+  data: unknown,
+  deps: StepRunnerDeps
+): Promise<StepRunResult> {
+  if (process.env.MARKETING_SEQUENCES_ENABLED !== "true") {
+    return { action: "cancelled", completed: true };
+  }
+  const job = parseEmailSequenceStepJob(data);
+  return runStep({ runId: job.runId, stepIndex: job.stepIndex }, deps);
+}

hivegrid-cloud/apps/worker/src/jobs/email-sequence-trigger.test.ts

@@ -0,0 +1,244 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  EMAIL_SEQUENCE_TRIGGER_QUEUE,
+  SEQUENCE_TRIGGER_EVENT_TYPES,
+  SequenceTriggerJobError,
+  normaliseTriggerEventType,
+  parseSequenceTriggerJob,
+  runSequenceTriggerJob,
+  type SequenceTriggerDeps,
+} from "./email-sequence-trigger.js";
+import type { SequenceDefinition } from "@toolctrl/email-sequences";
+
+function buildSequence(overrides: Partial<SequenceDefinition> = {}): SequenceDefinition {
+  return {
+    id: "seq_1",
+    tenantId: null,
+    name: "Welcome",
+    description: null,
+    triggerEvent: "lead.captured",
+    isEnabled: true,
+    steps: [{ stepIndex: 0, delayHours: 0, template: "welcome", subject: "Welcome" }],
+    ...overrides,
+  };
+}
+
+function buildDeps(sequences: SequenceDefinition[]): SequenceTriggerDeps & {
+  scheduled: Array<{ runId: string; stepIndex: number; runAt: Date }>;
+  createdRuns: Array<{ sequenceId: string }>;
+  rememberedTriggers: unknown[];
+} {
+  const scheduled: Array<{ runId: string; stepIndex: number; runAt: Date }> = [];
+  const createdRuns: Array<{ sequenceId: string }> = [];
+  const rememberedTriggers: unknown[] = [];
+  return {
+    scheduled,
+    createdRuns,
+    rememberedTriggers,
+    loader: {
+      listEnabledForEvent: vi.fn(async (eventType: string) =>
+        sequences.filter((s) => s.triggerEvent === eventType && s.isEnabled)
+      ),
+    },
+    store: {
+      createRun: vi.fn(async ({ sequenceId, leadId, tenantId, triggerEventId }) => {
+        createdRuns.push({ sequenceId });
+        return {
+          id: `run_${createdRuns.length}`,
+          sequenceId,
+          leadId,
+          tenantId,
+          triggerEventId,
+          status: "running" as const,
+          startedAt: new Date(),
+          completedAt: null,
+          stepStateJson: null,
+        };
+      }),
+      findRunByTrigger: vi.fn(async () => null),
+      updateRunStatus: vi.fn(),
+      createDelivery: vi.fn(async ({ runId, stepIndex, scheduledAt }) => ({
+        id: `del_${runId}_${stepIndex}`,
+        runId,
+        stepIndex,
+        status: "pending" as const,
+        scheduledAt,
+      })),
+      rememberTrigger: vi.fn((trigger) => {
+        rememberedTriggers.push(trigger);
+      }),
+    },
+    queue: {
+      scheduleStep: vi.fn(async (args) => {
+        scheduled.push(args);
+      }),
+    },
+    logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn() },
+    now: () => new Date("2026-05-17T12:00:00Z"),
+  };
+}
+
+afterEach(() => {
+  delete process.env.MARKETING_SEQUENCES_ENABLED;
+});
+
+beforeEach(() => {
+  process.env.MARKETING_SEQUENCES_ENABLED = "true";
+});
+
+describe("EMAIL_SEQUENCE_TRIGGER_QUEUE", () => {
+  it("subscribes to the outbox.marketing queue", () => {
+    expect(EMAIL_SEQUENCE_TRIGGER_QUEUE).toBe("outbox.marketing");
+  });
+});
+
+describe("SEQUENCE_TRIGGER_EVENT_TYPES", () => {
+  it("covers marketing.lead.* + bare lead.* + tenant.* events", () => {
+    expect(SEQUENCE_TRIGGER_EVENT_TYPES).toContain("marketing.lead.captured");
+    expect(SEQUENCE_TRIGGER_EVENT_TYPES).toContain("lead.captured");
+    expect(SEQUENCE_TRIGGER_EVENT_TYPES).toContain("tenant.signup");
+    expect(SEQUENCE_TRIGGER_EVENT_TYPES).toContain("tenant.trial_expiring");
+  });
+});
+
+describe("normaliseTriggerEventType", () => {
+  it("strips marketing prefix", () => {
+    expect(normaliseTriggerEventType("marketing.lead.captured")).toBe("lead.captured");
+  });
+  it("maps dispatcher-emitted convertedtotrial to canonical converted_to_trial", () => {
+    expect(normaliseTriggerEventType("marketing.lead.convertedtotrial")).toBe("lead.converted_to_trial");
+    expect(normaliseTriggerEventType("marketing.lead.convertedtocustomer")).toBe("lead.converted_to_customer");
+  });
+  it("leaves tenant.* events untouched", () => {
+    expect(normaliseTriggerEventType("tenant.signup")).toBe("tenant.signup");
+  });
+});
+
+describe("parseSequenceTriggerJob", () => {
+  it("accepts well-formed envelope", () => {
+    const job = parseSequenceTriggerJob({
+      eventType: "marketing.lead.captured",
+      payload: { email: "alice@example.com" },
+      outboxEventId: "evt_1",
+    });
+    expect(job.eventType).toBe("marketing.lead.captured");
+    expect(job.outboxEventId).toBe("evt_1");
+  });
+  it("rejects missing eventType", () => {
+    expect(() => parseSequenceTriggerJob({ payload: {} })).toThrow(SequenceTriggerJobError);
+  });
+  it("rejects unknown eventType", () => {
+    expect(() => parseSequenceTriggerJob({ eventType: "billing.invoice.paid", payload: {} })).toThrow(
+      SequenceTriggerJobError
+    );
+  });
+  it("rejects missing payload", () => {
+    expect(() => parseSequenceTriggerJob({ eventType: "lead.captured" })).toThrow(SequenceTriggerJobError);
+  });
+});
+
+describe("runSequenceTriggerJob", () => {
+  const DATA = {
+    eventType: "marketing.lead.captured",
+    outboxEventId: "evt_1",
+    payload: { email: "alice@example.com", lead: { id: "lead_1", status: "new" } },
+  };
+
+  it("returns disabled-shape when killswitch active", async () => {
+    process.env.MARKETING_SEQUENCES_ENABLED = "false";
+    const deps = buildDeps([buildSequence()]);
+    const result = await runSequenceTriggerJob(DATA, deps);
+    expect(result.matched).toBe(0);
+    expect(result.normalisedEventType).toBe("(disabled)");
+  });
+
+  it("returns 0/0/0 when payload has no recipient email", async () => {
+    const deps = buildDeps([buildSequence()]);
+    const result = await runSequenceTriggerJob({ ...DATA, payload: { lead: { id: "lead_1" } } }, deps);
+    expect(result.matched).toBe(0);
+    expect(result.started).toBe(0);
+  });
+
+  it("returns no-match when zero sequences enabled for event", async () => {
+    const deps = buildDeps([buildSequence({ triggerEvent: "tenant.signup" })]);
+    const result = await runSequenceTriggerJob(DATA, deps);
+    expect(result.matched).toBe(0);
+    expect(result.started).toBe(0);
+  });
+
+  it("starts a run for each matching sequence", async () => {
+    const deps = buildDeps([
+      buildSequence({ id: "seq_1" }),
+      buildSequence({ id: "seq_2" }),
+      buildSequence({ id: "seq_off", isEnabled: false }),
+    ]);
+    const result = await runSequenceTriggerJob(DATA, deps);
+    expect(result.matched).toBe(2);
+    expect(result.started).toBe(2);
+    expect(deps.createdRuns.map((r) => r.sequenceId).sort()).toEqual(["seq_1", "seq_2"]);
+    expect(deps.scheduled).toHaveLength(2);
+    expect(deps.store.rememberTrigger).toHaveBeenCalledTimes(2);
+    expect(deps.rememberedTriggers).toEqual([
+      expect.objectContaining({
+        triggerEventId: "evt_1",
+        leadId: "lead_1",
+        email: "alice@example.com",
+        payload: DATA.payload,
+      }),
+      expect.objectContaining({
+        triggerEventId: "evt_1",
+        leadId: "lead_1",
+        email: "alice@example.com",
+        payload: DATA.payload,
+      }),
+    ]);
+  });
+
+  it("uses outboxEventId as triggerEventId for idempotency", async () => {
+    const deps = buildDeps([buildSequence()]);
+    await runSequenceTriggerJob(DATA, deps);
+    expect(deps.store.findRunByTrigger).toHaveBeenCalledWith("seq_1", "evt_1");
+  });
+
+  it("isolates per-sequence failures so one bad sequence does not block others", async () => {
+    const deps = buildDeps([buildSequence({ id: "seq_1" }), buildSequence({ id: "seq_2" })]);
+    // Make sequence 1 fail at createRun, sequence 2 should still start.
+    let firstCall = true;
+    deps.store.createRun = vi.fn(async (args) => {
+      if (firstCall) {
+        firstCall = false;
+        throw new Error("simulated failure");
+      }
+      return {
+        id: "run_ok",
+        sequenceId: args.sequenceId,
+        leadId: args.leadId,
+        tenantId: args.tenantId,
+        triggerEventId: args.triggerEventId,
+        status: "running" as const,
+        startedAt: new Date(),
+        completedAt: null,
+        stepStateJson: null,
+      };
+    });
+    const result = await runSequenceTriggerJob(DATA, deps);
+    expect(result.matched).toBe(2);
+    // One started, one failed (counted as skipped to keep the metric simple).
+    expect(result.started + result.skipped).toBe(2);
+    expect(deps.logger?.error).toHaveBeenCalled();
+  });
+
+  it("extracts tenantId from top-level field when not on payload.tenant", async () => {
+    const deps = buildDeps([buildSequence({ triggerEvent: "tenant.signup" })]);
+    await runSequenceTriggerJob(
+      {
+        eventType: "tenant.signup",
+        outboxEventId: "evt_2",
+        tenantId: "tenant_z",
+        payload: { email: "ops@tenant.com" },
+      },
+      deps
+    );
+    expect(deps.store.createRun).toHaveBeenCalledWith(expect.objectContaining({ tenantId: "tenant_z" }));
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/email-sequence-trigger.ts

@@ -0,0 +1,263 @@
+/**
+ * Marketing-Funnel Phase 3 (Phase 05 #80) — sequence trigger handler.
+ *
+ * Subscribed to BullMQ queue `outbox.marketing` (and a fan-out for
+ * `tenant.*` lifecycle events). For every received outbox-event the
+ * handler:
+ *   1. Normalises the event-type (strip `marketing.` prefix so
+ *      `marketing.lead.captured` matches sequences with trigger
+ *      `lead.captured`).
+ *   2. Looks up all enabled EmailSequences with matching `triggerEvent`.
+ *   3. Builds a `SequenceTriggerContext` from the outbox payload
+ *      (lead-id, tenant-id, recipient email + name, payload bag for
+ *      condition evaluation).
+ *   4. Invokes {@link startSequenceRun} once per sequence. Idempotency
+ *      is provided by `(sequenceId, triggerEventId)` unique constraint
+ *      on EmailSequenceRun — duplicate deliveries are no-ops.
+ *
+ * Storage + queue + sequence-loader surfaces are injected so the
+ * handler is fully unit-testable. Production wiring lives in
+ * `email-sequence-trigger-bootstrap.ts`.
+ */
+
+import {
+  startSequenceRun,
+  type SequenceDefinition,
+  type SequenceEngineQueue,
+  type SequenceEngineStore,
+  type SequenceTriggerContext,
+} from "@toolctrl/email-sequences";
+
+export const EMAIL_SEQUENCE_TRIGGER_QUEUE = "outbox.marketing" as const;
+
+/** Set of outbox event-types this handler will react to. */
+export const SEQUENCE_TRIGGER_EVENT_TYPES: ReadonlyArray<string> = [
+  "marketing.lead.captured",
+  "marketing.lead.qualified",
+  "marketing.lead.convertedtotrial",
+  "marketing.lead.convertedtocustomer",
+  "marketing.lead.stale_60d",
+  "lead.captured",
+  "lead.qualified",
+  "lead.converted_to_trial",
+  "lead.converted_to_customer",
+  "lead.stale_60d",
+  "tenant.signup",
+  "tenant.trial_expiring",
+  "tenant.cancelled",
+];
+
+export interface SequenceTriggerJobData {
+  /** Outbox row id — used for idempotency in startSequenceRun. */
+  readonly outboxEventId?: string;
+  readonly tenantId?: string | null;
+  readonly eventType: string;
+  readonly payload: Readonly<Record<string, unknown>>;
+}
+
+export class SequenceTriggerJobError extends Error {
+  override readonly name = "SequenceTriggerJobError";
+  readonly code: "INVALID_PAYLOAD" | "UNKNOWN_EVENT_TYPE";
+  constructor(message: string, code: SequenceTriggerJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export interface SequenceTriggerLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+/**
+ * Read-side surface needed to find sequences matching an event. The
+ * production implementation queries Prisma; tests provide an in-memory
+ * fake.
+ */
+export interface SequenceTriggerLoader {
+  listEnabledForEvent(normalisedEventType: string): Promise<readonly SequenceDefinition[]>;
+}
+
+export interface SequenceTriggerDeps {
+  readonly loader: SequenceTriggerLoader;
+  readonly store: SequenceEngineStore & {
+    rememberTrigger?: (trigger: SequenceTriggerContext | null) => void;
+  };
+  readonly queue: SequenceEngineQueue;
+  readonly logger?: SequenceTriggerLogger;
+  readonly now?: () => Date;
+}
+
+export interface SequenceTriggerJobResult {
+  readonly normalisedEventType: string;
+  readonly matched: number;
+  readonly started: number;
+  readonly skipped: number;
+}
+
+/**
+ * Strip the `marketing.` prefix so an outbox event-type like
+ * `marketing.lead.captured` matches a sequence whose `triggerEvent` is
+ * the bare `lead.captured`. We also unify the dashed
+ * `marketing.lead.convertedtotrial` (no underscore — that is what the
+ * dispatcher emits today) to the documented `lead.converted_to_trial`.
+ */
+export function normaliseTriggerEventType(eventType: string): string {
+  const trimmed = eventType.startsWith("marketing.") ? eventType.slice("marketing.".length) : eventType;
+  switch (trimmed) {
+    case "lead.convertedtotrial":
+      return "lead.converted_to_trial";
+    case "lead.convertedtocustomer":
+      return "lead.converted_to_customer";
+    default:
+      return trimmed;
+  }
+}
+
+export function parseSequenceTriggerJob(data: unknown): SequenceTriggerJobData {
+  if (!data || typeof data !== "object") {
+    throw new SequenceTriggerJobError("trigger job missing data envelope", "INVALID_PAYLOAD");
+  }
+  const e = data as Record<string, unknown>;
+  const eventType = typeof e.eventType === "string" ? e.eventType : "";
+  if (eventType.length === 0) {
+    throw new SequenceTriggerJobError("trigger job missing eventType", "INVALID_PAYLOAD");
+  }
+  if (!SEQUENCE_TRIGGER_EVENT_TYPES.includes(eventType)) {
+    throw new SequenceTriggerJobError(`unsupported event-type: ${eventType}`, "UNKNOWN_EVENT_TYPE");
+  }
+  const payload = e.payload;
+  if (!payload || typeof payload !== "object") {
+    throw new SequenceTriggerJobError("trigger job missing payload", "INVALID_PAYLOAD");
+  }
+  return {
+    outboxEventId: typeof e.outboxEventId === "string" ? e.outboxEventId : undefined,
+    tenantId: typeof e.tenantId === "string" ? e.tenantId : null,
+    eventType,
+    payload: payload as Readonly<Record<string, unknown>>,
+  };
+}
+
+function extractEmail(payload: Readonly<Record<string, unknown>>): string | null {
+  const direct = payload["email"];
+  if (typeof direct === "string" && direct.includes("@")) return direct;
+  const lead = payload["lead"];
+  if (lead && typeof lead === "object") {
+    const leadEmail = (lead as Record<string, unknown>)["email"];
+    if (typeof leadEmail === "string" && leadEmail.includes("@")) return leadEmail;
+  }
+  const tenant = payload["tenant"];
+  if (tenant && typeof tenant === "object") {
+    const tenantEmail = (tenant as Record<string, unknown>)["email"];
+    if (typeof tenantEmail === "string" && tenantEmail.includes("@")) return tenantEmail;
+  }
+  return null;
+}
+
+function extractLeadId(payload: Readonly<Record<string, unknown>>): string | null {
+  const direct = payload["leadId"];
+  if (typeof direct === "string" && direct) return direct;
+  const lead = payload["lead"];
+  if (lead && typeof lead === "object") {
+    const id = (lead as Record<string, unknown>)["id"];
+    if (typeof id === "string" && id) return id;
+  }
+  return null;
+}
+
+function extractTenantId(payload: Readonly<Record<string, unknown>>, fallback: string | null): string | null {
+  const direct = payload["tenantId"];
+  if (typeof direct === "string" && direct) return direct;
+  const tenant = payload["tenant"];
+  if (tenant && typeof tenant === "object") {
+    const id = (tenant as Record<string, unknown>)["id"];
+    if (typeof id === "string" && id) return id;
+  }
+  return fallback;
+}
+
+function extractRecipientName(payload: Readonly<Record<string, unknown>>): string | null {
+  const direct = payload["recipientName"] ?? payload["firstName"];
+  if (typeof direct === "string" && direct) return direct;
+  const lead = payload["lead"];
+  if (lead && typeof lead === "object") {
+    const firstName = (lead as Record<string, unknown>)["firstName"];
+    if (typeof firstName === "string" && firstName) return firstName;
+  }
+  return null;
+}
+
+/**
+ * Main handler entry point. Looks up matching sequences and starts a
+ * run for each. Per-sequence failures are isolated so one bad
+ * definition cannot block the others.
+ */
+export async function runSequenceTriggerJob(
+  data: unknown,
+  deps: SequenceTriggerDeps
+): Promise<SequenceTriggerJobResult> {
+  if (process.env.MARKETING_SEQUENCES_ENABLED !== "true") {
+    deps.logger?.info?.({ op: "sequence.trigger.skipped" }, "email sequences disabled by env");
+    return { normalisedEventType: "(disabled)", matched: 0, started: 0, skipped: 0 };
+  }
+  const job = parseSequenceTriggerJob(data);
+  const normalised = normaliseTriggerEventType(job.eventType);
+  const email = extractEmail(job.payload);
+  if (!email) {
+    deps.logger?.warn?.(
+      { op: "sequence.trigger.no_email", eventType: job.eventType, outboxEventId: job.outboxEventId },
+      "trigger payload missing recipient email — cannot enrol any sequence"
+    );
+    return { normalisedEventType: normalised, matched: 0, started: 0, skipped: 0 };
+  }
+
+  const sequences = await deps.loader.listEnabledForEvent(normalised);
+  if (sequences.length === 0) {
+    deps.logger?.info?.({ op: "sequence.trigger.no_match", eventType: normalised }, "no enabled sequences match event");
+    return { normalisedEventType: normalised, matched: 0, started: 0, skipped: 0 };
+  }
+
+  const trigger: SequenceTriggerContext = {
+    triggerEventId: job.outboxEventId ?? null,
+    leadId: extractLeadId(job.payload),
+    tenantId: extractTenantId(job.payload, job.tenantId ?? null),
+    email,
+    recipientName: extractRecipientName(job.payload),
+    payload: job.payload,
+  };
+
+  let started = 0;
+  let skipped = 0;
+  for (const sequence of sequences) {
+    try {
+      deps.store.rememberTrigger?.(trigger);
+      const result = await startSequenceRun(
+        { sequence, trigger },
+        { store: deps.store, queue: deps.queue, logger: deps.logger, now: deps.now }
+      );
+      if (result.scheduled) started += 1;
+      else skipped += 1;
+      deps.logger?.info?.(
+        {
+          op: result.scheduled ? "sequence.trigger.started" : "sequence.trigger.skipped",
+          sequenceId: sequence.id,
+          runId: result.run.id || null,
+          alreadyExisted: result.alreadyExisted,
+        },
+        result.scheduled ? "sequence run started" : "sequence run not scheduled"
+      );
+    } catch (error) {
+      skipped += 1;
+      deps.logger?.error(
+        {
+          op: "sequence.trigger.error",
+          sequenceId: sequence.id,
+          error: error instanceof Error ? error.message : String(error),
+        },
+        "sequence run failed to start"
+      );
+    }
+  }
+  return { normalisedEventType: normalised, matched: sequences.length, started, skipped };
+}

hivegrid-cloud/apps/worker/src/jobs/export.test.ts

@@ -0,0 +1,199 @@
+import { PassThrough } from "node:stream";
+
+import { describe, expect, it, vi } from "vitest";
+
+import type { TenantPlanTier } from "@toolctrl/exports/limits";
+import type { ExportRecord } from "@toolctrl/exports/types";
+import type { ExportJob } from "@toolctrl/queue";
+import type { SharedStore } from "@toolctrl/shared-store";
+
+import { createExportJobRunner, type ExportRowSource, type ExportStreamerRegistry } from "./export.js";
+
+function buildPayload(overrides: Partial<ExportJob> = {}): ExportJob {
+  return {
+    kind: "export.run",
+    exportJobId: "job-1",
+    tenantId: "tenant-1",
+    requestedByUserId: "user-1",
+    exportType: "users",
+    format: "csv",
+    ...overrides,
+  } as ExportJob;
+}
+
+function buildRowSource(rows: ExportRecord[], columns: readonly string[] = ["id", "name"], estimated?: number): ExportRowSource {
+  return {
+    estimateRowCount: vi.fn().mockResolvedValue(estimated),
+    resolveColumns: vi.fn().mockResolvedValue(columns),
+    iterateRows: () => (async function* () {
+      for (const row of rows) yield row;
+    })(),
+  } satisfies ExportRowSource;
+}
+
+interface SharedStoreStub {
+  store: SharedStore;
+  set: ReturnType<typeof vi.fn>;
+}
+
+function buildSharedStoreStub(): SharedStoreStub {
+  const set = vi.fn().mockResolvedValue(true);
+  const stub = {
+    set,
+    get: vi.fn(),
+    del: vi.fn(),
+    incr: vi.fn(),
+    expire: vi.fn(),
+    mget: vi.fn(),
+    scan: vi.fn(),
+    publish: vi.fn(),
+    subscribe: vi.fn(),
+    zAdd: vi.fn(),
+    zRemRangeByScore: vi.fn(),
+    zCard: vi.fn(),
+    eval: vi.fn(),
+    close: vi.fn(),
+    key: vi.fn(),
+    namespace: "x",
+    ttlDefault: undefined,
+    adapter: {} as never,
+  };
+  return { store: stub as unknown as SharedStore, set };
+}
+
+function buildCountingRegistry(): { registry: ExportStreamerRegistry; written: PassThrough; calls: { csv: number; json: number; xlsx: number; pdf: number } } {
+  const written = new PassThrough();
+  written.on("data", () => undefined);
+  const calls = { csv: 0, json: 0, xlsx: 0, pdf: 0 };
+
+  const consume: ExportStreamerRegistry["csv"] = async (writeStream, _columns, rows) => {
+    let count = 0;
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    for await (const _row of rows) {
+      count += 1;
+    }
+    writeStream.end();
+    return { rowCount: count };
+  };
+
+  return {
+    written,
+    calls,
+    registry: {
+      csv: async (...args) => { calls.csv += 1; return consume(...args); },
+      json: async (...args) => { calls.json += 1; return consume(...args); },
+      xlsx: async (...args) => { calls.xlsx += 1; return consume(...args); },
+      pdf: async (...args) => { calls.pdf += 1; return consume(...args); },
+    },
+  };
+}
+
+function buildUploadStub(written: PassThrough): { Upload: unknown; done: ReturnType<typeof vi.fn> } {
+  const done = vi.fn().mockResolvedValue({ ETag: '"abc"' });
+  const Upload = vi.fn().mockImplementation((args: { params: { Body: PassThrough } }) => {
+    args.params.Body.pipe(written);
+    return { done };
+  });
+  return { Upload, done };
+}
+
+describe("createExportJobRunner", () => {
+  it("returns the resolved row count and artifact metadata on success", async () => {
+    const { registry, written } = buildCountingRegistry();
+    const upload = buildUploadStub(written);
+
+    const runner = createExportJobRunner({
+      s3Client: {} as never,
+      rowSource: buildRowSource([{ id: 1 }, { id: 2 }, { id: 3 }], ["id"]),
+      resolveTenantPlan: async (): Promise<TenantPlanTier> => "pro",
+      buildArtifactKey: () => ({ bucket: "exports", key: "tenants/t/job.csv", contentType: "text/csv", expiresAt: new Date("2026-01-01T00:00:00Z") }),
+      streamerRegistry: registry,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      uploadCtor: upload.Upload as any,
+    });
+
+    const result = await runner(buildPayload());
+    expect(result).toEqual({
+      artifactBucket: "exports",
+      artifactKey: "tenants/t/job.csv",
+      rowCount: 3,
+      expiresAt: new Date("2026-01-01T00:00:00Z"),
+    });
+    expect(upload.done).toHaveBeenCalled();
+  });
+
+  it("rejects up front when estimateRowCount exceeds the resolved plan cap", async () => {
+    const { registry, written } = buildCountingRegistry();
+    const upload = buildUploadStub(written);
+    const runner = createExportJobRunner({
+      s3Client: {} as never,
+      rowSource: buildRowSource([{ id: 1 }], ["id"], 20_000),
+      resolveTenantPlan: async () => "free",
+      buildArtifactKey: () => ({ bucket: "b", key: "k", contentType: "text/csv", expiresAt: new Date() }),
+      streamerRegistry: registry,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      uploadCtor: upload.Upload as any,
+    });
+
+    await expect(runner(buildPayload())).rejects.toThrow(/EXPORT_ROW_LIMIT_PLAN|exceeds the free/);
+    expect(registry.csv).not.toHaveBeenCalled;
+  });
+
+  it("publishes progress to the shared store on every flushInterval rows", async () => {
+    const rows: ExportRecord[] = Array.from({ length: 12 }, (_, i) => ({ id: i }));
+    const { registry, written } = buildCountingRegistry();
+    const upload = buildUploadStub(written);
+    const sharedStore = buildSharedStoreStub();
+
+    const runner = createExportJobRunner({
+      s3Client: {} as never,
+      sharedStore: sharedStore.store,
+      rowSource: buildRowSource(rows, ["id"], 12),
+      resolveTenantPlan: async () => "pro",
+      buildArtifactKey: () => ({ bucket: "b", key: "k", contentType: "text/csv", expiresAt: new Date("2026-01-01") }),
+      streamerRegistry: registry,
+      progressFlushInterval: 5,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      uploadCtor: upload.Upload as any,
+    });
+
+    const result = await runner(buildPayload());
+    expect(result.rowCount).toBe(12);
+    // Expect at least the row-5, row-10 progress flushes plus the final
+    // success flush.
+    expect(sharedStore.set).toHaveBeenCalledWith(
+      "export:progress:job-1",
+      expect.stringContaining('"processedRows":5'),
+      expect.objectContaining({ ttlSeconds: 3600 }),
+    );
+    expect(sharedStore.set).toHaveBeenCalledWith(
+      "export:progress:job-1",
+      expect.stringContaining('"processedRows":12'),
+      expect.objectContaining({ ttlSeconds: 3600 }),
+    );
+  });
+
+  it("destroys the write stream when serialisation throws", async () => {
+    const written = new PassThrough();
+    written.on("data", () => undefined);
+    const failingRegistry: ExportStreamerRegistry = {
+      csv: async () => { throw new Error("boom"); },
+      json: async () => { throw new Error("boom"); },
+      xlsx: async () => { throw new Error("boom"); },
+      pdf: async () => { throw new Error("boom"); },
+    };
+    const upload = buildUploadStub(written);
+
+    const runner = createExportJobRunner({
+      s3Client: {} as never,
+      rowSource: buildRowSource([{ id: 1 }], ["id"]),
+      resolveTenantPlan: async () => "pro",
+      buildArtifactKey: () => ({ bucket: "b", key: "k", contentType: "text/csv", expiresAt: new Date() }),
+      streamerRegistry: failingRegistry,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      uploadCtor: upload.Upload as any,
+    });
+
+    await expect(runner(buildPayload())).rejects.toThrow("boom");
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/export.ts

@@ -0,0 +1,220 @@
+import type { Writable } from "node:stream";
+
+import type { Upload } from "@aws-sdk/lib-storage";
+import { S3Client } from "@aws-sdk/client-s3";
+import type { EnvRecord } from "@toolctrl/config";
+import { serializeExportTableToCsvStream } from "@toolctrl/exports/csv";
+import { serializeExportTableToJsonStream } from "@toolctrl/exports/json";
+import { ExportRowCapExceededError, assertExportRowCap, resolveExportRowCap, type ExportFormat, type TenantPlanTier } from "@toolctrl/exports/limits";
+import { serializeExportTableToPdfStream } from "@toolctrl/exports/pdf";
+import { createS3MultipartUploadSink } from "@toolctrl/exports/s3-multipart";
+import type { ExportRecord } from "@toolctrl/exports/types";
+import { serializeExportTableToXlsxStream } from "@toolctrl/exports/xlsx";
+import type { SharedStore } from "@toolctrl/shared-store";
+import type { ExportJob } from "@toolctrl/queue";
+
+import { workerLogger } from "../logger.js";
+
+export interface WorkerExportRunResult {
+  artifactBucket: string;
+  artifactKey: string;
+  rowCount: number;
+  expiresAt: Date;
+}
+
+/**
+ * Async row-source contract — implementations stream the rows of the
+ * target export in chunks (typically a Prisma cursor) without buffering
+ * the full dataset. The job pipeline orchestrates serialisation,
+ * progress-tracking and S3 multipart upload around this iterator.
+ */
+export interface ExportRowSourceContext {
+  payload: ExportJob;
+  plan: TenantPlanTier;
+  /** Effective row cap (min of plan and format hard cap). */
+  effectiveRowCap: number;
+}
+
+export interface ExportRowSource {
+  /** Best-effort row-count estimate, used for XLSX strategy selection. */
+  estimateRowCount(context: ExportRowSourceContext): Promise<number | undefined>;
+  /** Column names in export order. */
+  resolveColumns(context: ExportRowSourceContext): Promise<readonly string[]>;
+  /** Async iterator over the row stream. */
+  iterateRows(context: ExportRowSourceContext): AsyncIterable<ExportRecord>;
+}
+
+export interface ExportTenantPlanResolver {
+  (tenantId: string): Promise<TenantPlanTier>;
+}
+
+export interface ExportArtifactKeyBuilder {
+  (payload: ExportJob): { bucket: string; key: string; contentType: string; expiresAt: Date };
+}
+
+export interface ExportJobDependencies {
+  s3Client: S3Client;
+  sharedStore?: SharedStore;
+  rowSource: ExportRowSource;
+  resolveTenantPlan: ExportTenantPlanResolver;
+  buildArtifactKey: ExportArtifactKeyBuilder;
+  /** Test seam — overrides the per-format streamer registry. */
+  streamerRegistry?: ExportStreamerRegistry;
+  /** Progress-publish frequency in rows; defaults to 5_000. */
+  progressFlushInterval?: number;
+  /** TTL applied to the progress key in seconds; defaults to 3600 (1h). */
+  progressTtlSeconds?: number;
+  /** Test seam — overrides the @aws-sdk/lib-storage Upload class. */
+  uploadCtor?: typeof Upload;
+}
+
+export type ExportFormatNarrow = ExportFormat;
+
+export interface ExportStreamerRegistry {
+  csv: (writeStream: Writable, columns: readonly string[], rows: AsyncIterable<ExportRecord>, estimatedRowCount: number | undefined) => Promise<{ rowCount: number }>;
+  json: (writeStream: Writable, columns: readonly string[], rows: AsyncIterable<ExportRecord>, estimatedRowCount: number | undefined) => Promise<{ rowCount: number }>;
+  xlsx: (writeStream: Writable, columns: readonly string[], rows: AsyncIterable<ExportRecord>, estimatedRowCount: number | undefined) => Promise<{ rowCount: number }>;
+  pdf: (writeStream: Writable, columns: readonly string[], rows: AsyncIterable<ExportRecord>, estimatedRowCount: number | undefined) => Promise<{ rowCount: number }>;
+}
+
+const DEFAULT_REGISTRY: ExportStreamerRegistry = {
+  csv: async (writeStream, columns, rows) =>
+    serializeExportTableToCsvStream(writeStream, { columns: [...columns], rows }),
+  json: async (writeStream, columns, rows) =>
+    serializeExportTableToJsonStream(writeStream, { columns: [...columns], rows }),
+  xlsx: async (writeStream, columns, rows, estimatedRowCount) =>
+    serializeExportTableToXlsxStream(writeStream, {
+      columns: [...columns],
+      rows,
+      ...(estimatedRowCount !== undefined ? { estimatedRowCount } : {}),
+    }),
+  pdf: async (writeStream, columns, rows) => {
+    // PDF needs a buffered table because react-pdf renders the document
+    // tree synchronously; we cap below the format hard limit so heap
+    // usage stays bounded.
+    const buffered: ExportRecord[] = [];
+    for await (const row of rows) {
+      buffered.push(row);
+    }
+    return serializeExportTableToPdfStream(writeStream, { columns: [...columns], rows: buffered });
+  },
+};
+
+export function createExportJobRunner(deps: ExportJobDependencies): (payload: ExportJob) => Promise<WorkerExportRunResult> {
+  const flushInterval = deps.progressFlushInterval ?? 5_000;
+  const progressTtl = deps.progressTtlSeconds ?? 3_600;
+  const registry = deps.streamerRegistry ?? DEFAULT_REGISTRY;
+
+  return async function runExport(payload: ExportJob): Promise<WorkerExportRunResult> {
+    const format = normaliseFormat(payload.format);
+    const plan = await deps.resolveTenantPlan(payload.tenantId);
+    const effectiveRowCap = resolveExportRowCap(plan, format);
+    const context: ExportRowSourceContext = { payload, plan, effectiveRowCap };
+
+    const estimate = await deps.rowSource.estimateRowCount(context);
+    if (estimate !== undefined) {
+      assertExportRowCap(estimate, plan, format);
+    }
+
+    const columns = await deps.rowSource.resolveColumns(context);
+    const artifact = deps.buildArtifactKey(payload);
+    const { writeStream, completionPromise } = createS3MultipartUploadSink({
+      client: deps.s3Client,
+      bucket: artifact.bucket,
+      key: artifact.key,
+      contentType: artifact.contentType,
+      metadata: { tenantId: payload.tenantId, exportJobId: payload.exportJobId, exportType: payload.exportType },
+      ...(deps.uploadCtor ? { uploadCtor: deps.uploadCtor } : {}),
+    });
+
+    let processedRows = 0;
+    const guardedRows = (async function* () {
+      for await (const row of deps.rowSource.iterateRows(context)) {
+        if (processedRows >= effectiveRowCap) {
+          throw new ExportRowCapExceededError(processedRows + 1, effectiveRowCap, plan, format);
+        }
+        processedRows += 1;
+        if (deps.sharedStore && processedRows % flushInterval === 0) {
+          // Best-effort progress publish — failure here must not abort
+          // the export so we swallow + log only.
+          await publishProgress(deps.sharedStore, payload.exportJobId, processedRows, estimate, progressTtl);
+        }
+        yield row;
+      }
+    })();
+
+    try {
+      const result = await registry[format](writeStream, columns, guardedRows, estimate);
+      writeStream.end();
+      const completion = await completionPromise;
+
+      if (deps.sharedStore) {
+        await publishProgress(deps.sharedStore, payload.exportJobId, result.rowCount, estimate, progressTtl);
+      }
+
+      return {
+        artifactBucket: completion.bucket,
+        artifactKey: completion.key,
+        rowCount: result.rowCount,
+        expiresAt: artifact.expiresAt,
+      };
+    } catch (error) {
+      writeStream.destroy(error instanceof Error ? error : new Error(String(error)));
+      throw error;
+    }
+  };
+}
+
+async function publishProgress(
+  store: SharedStore,
+  exportJobId: string,
+  processedRows: number,
+  totalEstimate: number | undefined,
+  ttlSeconds: number,
+): Promise<void> {
+  try {
+    await store.set(
+      `export:progress:${exportJobId}`,
+      JSON.stringify({ processedRows, totalEstimate: totalEstimate ?? null, updatedAt: new Date().toISOString() }),
+      { ttlSeconds },
+    );
+  } catch (error) {
+    workerLogger.warn({ error, exportJobId }, "Failed to publish export progress to shared store.");
+  }
+}
+
+function normaliseFormat(format: ExportJob["format"]): ExportFormat {
+  switch (format) {
+    case "csv":
+    case "json":
+      return format;
+    default: {
+      // The queue payload only declares csv/json today, but the worker is
+      // ready for xlsx/pdf as soon as the producer side widens the union.
+      const widened = format as unknown as ExportFormat;
+      if (widened === "xlsx" || widened === "pdf") return widened;
+      throw new Error(`Unsupported export format: ${String(format)}`);
+    }
+  }
+}
+
+/**
+ * Resolve the S3Client used by the worker for export multipart uploads.
+ * Mirrors the scanService env shape so operators configure a single set of
+ * STORAGE_* variables for both the scanner and the exporter.
+ */
+export function createWorkerExportS3Client(env: EnvRecord = process.env as EnvRecord): S3Client | undefined {
+  if (!env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
+    return undefined;
+  }
+  return new S3Client({
+    endpoint: env.STORAGE_ENDPOINT,
+    region: env.STORAGE_REGION,
+    forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === "true",
+    credentials: {
+      accessKeyId: env.STORAGE_ACCESS_KEY_ID,
+      secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY,
+      ...(env.STORAGE_SESSION_TOKEN ? { sessionToken: env.STORAGE_SESSION_TOKEN } : {}),
+    },
+  });
+}

hivegrid-cloud/apps/worker/src/jobs/file-preview.ts

@@ -0,0 +1,257 @@
+// AK-P4a §4 — file-preview service (BullMQ `toolctrl.file-preview`).
+//
+// Mirrors scanService.ts: a dedicated AWS S3 client built from STORAGE_* env
+// GETs the original, renders the preview and PUTs preview/thumb into the same
+// private bucket (AD-23). previewStatus is written back onto the File row
+// (pending -> generating -> ready|failed|not_supported).
+//
+// Format matrix v1 (designer brief 2026-06-12):
+//   image/heic|heif         -> heic-convert -> JPG  -> preview.jpg + thumb.webp
+//   image/jpeg|png|gif|webp -> sharp        -> WebP -> preview.webp + thumb.webp
+//   application/pdf         -> pdf-to-png   -> PNG  -> preview.png + thumb.webp
+//   text/plain | markdown   -> ready (inline render from bytes, no file)
+//   anything else           -> not_supported
+//
+// The native render libs (sharp, heic-convert, pdf-to-png-converter) are loaded
+// via dynamic string import (same pattern as qdrant/sentry in index.ts) so
+// typecheck/build stay green before they are installed.
+// ACTIVATION: `pnpm add sharp heic-convert pdf-to-png-converter -F @toolctrl/worker`.
+// If a lib is missing at runtime the service falls back cleanly to
+// previewStatus=failed (no hard crash) and the user can regenerate from detail.
+
+import { GetObjectCommand, PutObjectCommand, S3Client as AwsS3Client } from "@aws-sdk/client-s3";
+import type { EnvRecord } from "@toolctrl/config";
+import type { FilePreviewJob } from "@toolctrl/queue";
+import { buildFilePreviewKey, buildFileStorageKeys, type FilePreviewExtension } from "@toolctrl/storage";
+
+import { withTenantRLS } from "../db/tenantRls.js";
+import type { FilePreviewServiceResult } from "../handlers.js";
+
+export interface FilePreviewServiceEnv {
+  STORAGE_ENDPOINT?: string;
+  STORAGE_REGION?: string;
+  STORAGE_ACCESS_KEY_ID?: string;
+  STORAGE_SECRET_ACCESS_KEY?: string;
+  STORAGE_SESSION_TOKEN?: string;
+  STORAGE_FORCE_PATH_STYLE?: string;
+}
+
+type PreviewLogger = {
+  info: (fields: Record<string, unknown>, msg: string) => void;
+  warn: (fields: Record<string, unknown>, msg: string) => void;
+  error: (fields: Record<string, unknown>, msg: string) => void;
+};
+
+/**
+ * The File write surface the preview status write-back uses. `$executeRaw` is
+ * required so the helper can set `app.tenant_id` on the transaction (AK-P4a RLS
+ * prep). `data` stays string-typed to avoid coupling to the generated Prisma
+ * enum types (keeps the dynamic-import build green before native libs install).
+ */
+interface FilePreviewTx {
+  $executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
+  file: {
+    update(args: {
+      where: { id: string };
+      data: {
+        previewStatus?: string;
+        previewStorageKey?: string | null;
+        thumbnailStorageKey?: string | null;
+      };
+    }): Promise<unknown>;
+  };
+}
+
+/** Minimal Prisma surface the service needs (tenant transaction). */
+export interface FilePreviewPrisma {
+  $transaction<T>(fn: (tx: FilePreviewTx) => Promise<T>): Promise<T>;
+}
+
+export interface FilePreviewService {
+  generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult>;
+}
+
+const THUMBNAIL_MAX_DIMENSION = 480;
+const PREVIEW_MAX_DIMENSION = 2048;
+
+const RASTER_IMAGE_TYPES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
+const HEIC_TYPES = new Set(["image/heic", "image/heif"]);
+const TEXT_TYPES = new Set(["text/plain", "text/markdown", "text/csv"]);
+
+function normalizeMime(mimeType: string): string {
+  return mimeType.split(";")[0]?.trim().toLowerCase() ?? "";
+}
+
+/**
+ * Builds the preview service from STORAGE_* env. Returns undefined when the S3
+ * credentials are missing (the bootstrap wiring then skips it and the handler
+ * refuses jobs with a config error instead of running blind).
+ */
+export function createWorkerFilePreviewService(
+  env: EnvRecord & FilePreviewServiceEnv,
+  options: { prisma: FilePreviewPrisma; logger: PreviewLogger },
+): FilePreviewService | undefined {
+  if (!env.STORAGE_REGION || !env.STORAGE_ACCESS_KEY_ID || !env.STORAGE_SECRET_ACCESS_KEY) {
+    return undefined;
+  }
+
+  const { prisma, logger } = options;
+  const s3 = new AwsS3Client({
+    endpoint: env.STORAGE_ENDPOINT,
+    region: env.STORAGE_REGION,
+    forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === "true",
+    credentials: {
+      accessKeyId: env.STORAGE_ACCESS_KEY_ID,
+      secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY,
+      sessionToken: env.STORAGE_SESSION_TOKEN,
+    },
+  });
+
+  async function loadOriginal(bucket: string, key: string): Promise<Buffer> {
+    const result = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+    const body = result.Body;
+    if (!body) {
+      throw new Error(`S3 GET ${bucket}/${key} returned an empty body`);
+    }
+    // The AWS SDK exposes transformToByteArray() on the Node stream body.
+    const bytes = await (body as { transformToByteArray(): Promise<Uint8Array> }).transformToByteArray();
+    return Buffer.from(bytes);
+  }
+
+  async function putRendered(bucket: string, key: string, body: Buffer, contentType: string): Promise<void> {
+    await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, ContentType: contentType }));
+  }
+
+  async function writeStatus(
+    tenantId: string,
+    fileId: string,
+    data: { previewStatus: string; previewStorageKey?: string | null; thumbnailStorageKey?: string | null },
+  ): Promise<void> {
+    // Set the tenant GUC so the write satisfies the File RLS policy once it is
+    // activated (P5 cutover). Preview jobs always carry a tenantId.
+    await withTenantRLS(prisma, tenantId, (tx) => tx.file.update({ where: { id: fileId }, data }));
+  }
+
+  return {
+    async generatePreview(payload: FilePreviewJob): Promise<FilePreviewServiceResult> {
+      const mime = normalizeMime(payload.mimeType);
+      const keys = buildFileStorageKeys(payload.tenantId, payload.fileId);
+
+      // Plain text / markdown: detail view renders inline from bytes, no
+      // rendered preview file needed.
+      if (TEXT_TYPES.has(mime)) {
+        await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "ready", previewStorageKey: null, thumbnailStorageKey: null });
+        return { previewStatus: "ready" };
+      }
+
+      const isHeic = HEIC_TYPES.has(mime);
+      const isRaster = RASTER_IMAGE_TYPES.has(mime);
+      const isPdf = mime === "application/pdf";
+
+      if (!isHeic && !isRaster && !isPdf) {
+        await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "not_supported", previewStorageKey: null, thumbnailStorageKey: null });
+        return { previewStatus: "not_supported" };
+      }
+
+      await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "generating" });
+
+      try {
+        const original = await loadOriginal(payload.bucket, payload.storageKey);
+        let previewBuffer: Buffer;
+        let previewExt: FilePreviewExtension;
+        let previewContentType: string;
+
+        if (isHeic) {
+          // HEIC -> JPG (mandatory for iPhone field-worker photos).
+          const jpg = await heicToJpeg(original);
+          previewBuffer = await rasterToFormat(jpg, "jpeg", PREVIEW_MAX_DIMENSION);
+          previewExt = "jpg";
+          previewContentType = "image/jpeg";
+        } else if (isPdf) {
+          const png = await pdfFirstPageToPng(original);
+          previewBuffer = await rasterToFormat(png, "png", PREVIEW_MAX_DIMENSION);
+          previewExt = "png";
+          previewContentType = "image/png";
+        } else {
+          previewBuffer = await rasterToFormat(original, "webp", PREVIEW_MAX_DIMENSION);
+          previewExt = "webp";
+          previewContentType = "image/webp";
+        }
+
+        const thumbnail = await rasterToFormat(previewBuffer, "webp", THUMBNAIL_MAX_DIMENSION);
+
+        const previewKey = buildFilePreviewKey(payload.tenantId, payload.fileId, previewExt);
+        const thumbnailKey = keys.thumbnail;
+
+        await putRendered(payload.bucket, previewKey, previewBuffer, previewContentType);
+        await putRendered(payload.bucket, thumbnailKey, thumbnail, "image/webp");
+
+        await writeStatus(payload.tenantId, payload.fileId, {
+          previewStatus: "ready",
+          previewStorageKey: previewKey,
+          thumbnailStorageKey: thumbnailKey,
+        });
+
+        return { previewStatus: "ready", previewStorageKey: previewKey, thumbnailStorageKey: thumbnailKey };
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        logger.warn({ fileId: payload.fileId, tenantId: payload.tenantId, mime }, "file preview generation failed");
+        await writeStatus(payload.tenantId, payload.fileId, { previewStatus: "failed" });
+        return { previewStatus: "failed", reason };
+      }
+    },
+  };
+}
+
+// --- dynamic native-lib loaders (kept untyped so build stays green pre-install) ---
+
+type SharpFactory = (input: Buffer) => {
+  rotate(): ReturnType<SharpFactory>;
+  resize(opts: { width: number; height: number; fit: "inside"; withoutEnlargement: boolean }): ReturnType<SharpFactory>;
+  jpeg(opts: { quality: number }): ReturnType<SharpFactory>;
+  png(): ReturnType<SharpFactory>;
+  webp(opts: { quality: number }): ReturnType<SharpFactory>;
+  toBuffer(): Promise<Buffer>;
+};
+
+let sharpModule: SharpFactory | undefined;
+async function loadSharp(): Promise<SharpFactory> {
+  if (sharpModule) return sharpModule;
+  const moduleName = "sharp";
+  const loaded = (await import(/* @vite-ignore */ moduleName)) as { default?: SharpFactory } & SharpFactory;
+  sharpModule = (loaded.default ?? loaded) as SharpFactory;
+  return sharpModule;
+}
+
+async function rasterToFormat(input: Buffer, format: "jpeg" | "png" | "webp", maxDimension: number): Promise<Buffer> {
+  const sharp = await loadSharp();
+  const pipeline = sharp(input)
+    .rotate()
+    .resize({ width: maxDimension, height: maxDimension, fit: "inside", withoutEnlargement: true });
+  if (format === "jpeg") return pipeline.jpeg({ quality: 82 }).toBuffer();
+  if (format === "png") return pipeline.png().toBuffer();
+  return pipeline.webp({ quality: 80 }).toBuffer();
+}
+
+async function heicToJpeg(input: Buffer): Promise<Buffer> {
+  const moduleName = "heic-convert";
+  const loaded = (await import(/* @vite-ignore */ moduleName)) as
+    | { default: (opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer> }
+    | ((opts: { buffer: Buffer; format: "JPEG"; quality: number }) => Promise<ArrayBuffer | Buffer>);
+  const convert = typeof loaded === "function" ? loaded : loaded.default;
+  const out = await convert({ buffer: input, format: "JPEG", quality: 0.85 });
+  return Buffer.from(out as ArrayBuffer);
+}
+
+async function pdfFirstPageToPng(input: Buffer): Promise<Buffer> {
+  const moduleName = "pdf-to-png-converter";
+  const loaded = (await import(/* @vite-ignore */ moduleName)) as {
+    pdfToPng: (input: Buffer, opts: Record<string, unknown>) => Promise<Array<{ content: Buffer }>>;
+  };
+  const pages = await loaded.pdfToPng(input, { pagesToProcess: [1], viewportScale: 2.0 });
+  const first = pages[0];
+  if (!first) {
+    throw new Error("PDF rendered no pages");
+  }
+  return Buffer.from(first.content);
+}

hivegrid-cloud/apps/worker/src/jobs/file-scan-status.ts

@@ -0,0 +1,80 @@
+// AK-P4a §5 — scan-verdict write-back for the BullMQ `toolctrl.file-scan` queue.
+//
+// The scan handler (handlers.ts) calls deps.scanService.scanObject() which runs
+// ClamAV INSTREAM and quarantines infected objects, but it does NOT touch the
+// database. Without a write-back the File row stays at scanStatus=pending and
+// the fail-closed media proxy (AD-23) never serves it. This thin service writes
+// the verdict (clean|infected) back onto File.scanStatus.
+//
+// Mirrors createWorkerFilePreviewService: a Prisma seam is injected at the
+// general-group bootstrap (index.ts) so handlers.ts stays free of direct DB
+// imports. When the seam is unset the handler simply skips the write-back
+// (legacy import-path scans carry no fileId either).
+
+import { withTenantRLS } from "../db/tenantRls.js";
+
+type ScanStatusLogger = {
+  info: (fields: Record<string, unknown>, msg: string) => void;
+  warn: (fields: Record<string, unknown>, msg: string) => void;
+};
+
+/**
+ * The File write surface the write-back uses. `$executeRaw` is required so the
+ * helper can set `app.tenant_id` on the transaction (AK-P4a RLS prep).
+ */
+interface FileScanStatusTx {
+  $executeRaw(query: TemplateStringsArray, ...values: unknown[]): Promise<number>;
+  file: {
+    updateMany(args: {
+      where: { id: string; tenantId?: string };
+      data: { scanStatus: string };
+    }): Promise<{ count: number }>;
+  };
+}
+
+/** Minimal Prisma surface the write-back needs (direct write + tenant tx). */
+export interface FileScanStatusPrisma {
+  $transaction<T>(fn: (tx: FileScanStatusTx) => Promise<T>): Promise<T>;
+  file: {
+    updateMany(args: {
+      where: { id: string; tenantId?: string };
+      data: { scanStatus: string };
+    }): Promise<{ count: number }>;
+  };
+}
+
+export type FileScanVerdict = "clean" | "infected";
+
+export interface FileScanStatusWriter {
+  recordVerdict(input: { fileId: string; tenantId?: string; verdict: FileScanVerdict }): Promise<void>;
+}
+
+export function createWorkerFileScanStatusWriter(options: {
+  prisma: FileScanStatusPrisma;
+  logger: ScanStatusLogger;
+}): FileScanStatusWriter {
+  const { prisma, logger } = options;
+  return {
+    async recordVerdict({ fileId, tenantId, verdict }) {
+      // updateMany (not update) so a tenant-scoped/RLS-filtered or already
+      // deleted row is a logged no-op instead of a throw that dead-letters
+      // the job after the scan already ran.
+      const write = (db: Pick<FileScanStatusTx, "file">) =>
+        db.file.updateMany({
+          where: tenantId ? { id: fileId, tenantId } : { id: fileId },
+          data: { scanStatus: verdict },
+        });
+      // Set the tenant GUC so the write satisfies the File RLS policy once it is
+      // activated (P5 cutover). Legacy import-path scans carry no tenantId and
+      // fall back to the direct write (no RLS context to set).
+      const result = tenantId
+        ? await withTenantRLS(prisma, tenantId, (tx) => write(tx))
+        : await write(prisma);
+      if (result.count === 0) {
+        logger.warn({ fileId, tenantId, verdict }, "scan verdict write-back matched no File row");
+      } else {
+        logger.info({ fileId, verdict }, "scan verdict written to File.scanStatus");
+      }
+    },
+  };
+}

hivegrid-cloud/apps/worker/src/jobs/form-submission-followups.test.ts

@@ -0,0 +1,96 @@
+import { describe, expect, it } from "vitest";
+import { planFormSubmissionFollowUps } from "./form-submission-followups.js";
+
+function baseInput(overrides: Record<string, unknown> = {}) {
+  return {
+    submissionId: "sub_1",
+    formId: "form_1",
+    tenantId: "t_1",
+    targetType: "crm-contact" as const,
+    target: { id: "contact_1" },
+    webhookUrl: null as string | null,
+    notifyEmails: [] as readonly string[],
+    values: { email: "a@b.co" },
+    ...overrides,
+  };
+}
+
+describe("planFormSubmissionFollowUps", () => {
+  it("emits no follow-ups when neither webhook nor notifyEmails are set", () => {
+    expect(planFormSubmissionFollowUps(baseInput())).toEqual([]);
+  });
+
+  it("emits a webhook follow-up when webhookUrl is set", () => {
+    const events = planFormSubmissionFollowUps(
+      baseInput({ webhookUrl: "https://example.com/hook" }),
+    );
+    expect(events).toHaveLength(1);
+    expect(events[0]!.type).toBe("webhook.form-submission.deliver");
+    expect(events[0]!.dispatchKey).toBe("form-submission-webhook:sub_1");
+    expect(events[0]!.payload.webhookUrl).toBe("https://example.com/hook");
+    expect(events[0]!.payload.targetId).toBe("contact_1");
+  });
+
+  it("skips webhook follow-up when webhookUrl is empty / whitespace", () => {
+    expect(planFormSubmissionFollowUps(baseInput({ webhookUrl: "   " }))).toEqual([]);
+  });
+
+  it("emits a mail follow-up when notifyEmails contains valid addresses", () => {
+    const events = planFormSubmissionFollowUps(
+      baseInput({ notifyEmails: ["ops@example.com", "alerts@example.com"] }),
+    );
+    expect(events).toHaveLength(1);
+    expect(events[0]!.type).toBe("mail.transactional.form-submission");
+    expect(events[0]!.dispatchKey).toBe("form-submission-mail:sub_1");
+    expect(events[0]!.payload.recipients).toEqual(["ops@example.com", "alerts@example.com"]);
+  });
+
+  it("filters out invalid email entries", () => {
+    const events = planFormSubmissionFollowUps(
+      baseInput({ notifyEmails: ["valid@example.com", "not-an-email", ""] }),
+    );
+    expect(events[0]!.payload.recipients).toEqual(["valid@example.com"]);
+  });
+
+  it("skips mail follow-up when no valid recipients remain after filtering", () => {
+    const events = planFormSubmissionFollowUps(
+      baseInput({ notifyEmails: ["not-an-email", ""] }),
+    );
+    expect(events).toEqual([]);
+  });
+
+  it("emits both webhook AND mail follow-ups when both are configured", () => {
+    const events = planFormSubmissionFollowUps(
+      baseInput({
+        webhookUrl: "https://example.com/hook",
+        notifyEmails: ["ops@example.com"],
+      }),
+    );
+    expect(events.map((e) => e.type)).toEqual([
+      "webhook.form-submission.deliver",
+      "mail.transactional.form-submission",
+    ]);
+  });
+
+  it("uses stable dispatch keys for idempotent retries", () => {
+    const a = planFormSubmissionFollowUps(
+      baseInput({ webhookUrl: "https://example.com/hook", notifyEmails: ["ops@example.com"] }),
+    );
+    const b = planFormSubmissionFollowUps(
+      baseInput({ webhookUrl: "https://example.com/hook", notifyEmails: ["ops@example.com"] }),
+    );
+    expect(a.map((e) => e.dispatchKey)).toEqual(b.map((e) => e.dispatchKey));
+  });
+
+  it("includes targetId in payload so downstream knows what was created", () => {
+    const events = planFormSubmissionFollowUps(
+      baseInput({
+        webhookUrl: "https://example.com/hook",
+        targetType: "entity-row",
+        target: { id: "row_99" },
+      }),
+    );
+    expect(events[0]!.payload.targetType).toBe("entity-row");
+    expect(events[0]!.payload.targetId).toBe("row_99");
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/form-submission-followups.ts

@@ -0,0 +1,90 @@
+/**
+ * Form-Widget post-routing follow-ups (Phase 05 #58).
+ *
+ * After {@link runFormSubmissionRouteJob} writes a submission to its
+ * target (CRM contact / entity row / generic), this module publishes
+ * two optional outbox events:
+ *
+ *  - `form.submission.webhook` — picked up by the existing webhook
+ *    delivery worker; only emitted when the form has `webhookUrl` set.
+ *  - `mail.transactional.form-submission` — picked up by the mail
+ *    worker; only emitted when the form has `notifyEmails` non-empty.
+ *
+ * Both publishes go through the outbox so they enjoy the same
+ * at-least-once delivery guarantees + Sentry visibility as the rest of
+ * the system. The helper is pure / dependency-free so it can be tested
+ * without spinning Prisma; the worker bootstrap wires a thin adapter
+ * that calls `publishOutboxEvent` inside the host's transaction.
+ */
+
+export type FormSubmissionFollowUpEventType =
+  | "webhook.form-submission.deliver"
+  | "mail.transactional.form-submission";
+
+export interface FormSubmissionFollowUpInput {
+  readonly submissionId: string;
+  readonly formId: string;
+  readonly tenantId: string;
+  readonly targetType: "generic" | "crm-contact" | "entity-row";
+  readonly target: { id: string | null };
+  readonly webhookUrl: string | null;
+  readonly notifyEmails: readonly string[];
+  readonly values: Record<string, unknown>;
+}
+
+export interface FollowUpPublishArgs {
+  readonly tenantId: string;
+  readonly type: FormSubmissionFollowUpEventType;
+  readonly payload: Record<string, unknown>;
+  readonly dispatchKey: string;
+}
+
+/**
+ * Pure planner: returns the list of outbox events the host should
+ * publish for a given follow-up input. Returns an empty array when
+ * neither webhook nor notify-email is configured.
+ */
+export function planFormSubmissionFollowUps(
+  input: FormSubmissionFollowUpInput,
+): readonly FollowUpPublishArgs[] {
+  const events: FollowUpPublishArgs[] = [];
+
+  if (typeof input.webhookUrl === "string" && input.webhookUrl.trim().length > 0) {
+    events.push({
+      tenantId: input.tenantId,
+      type: "webhook.form-submission.deliver",
+      payload: {
+        submissionId: input.submissionId,
+        formId: input.formId,
+        tenantId: input.tenantId,
+        webhookUrl: input.webhookUrl,
+        targetType: input.targetType,
+        targetId: input.target.id,
+        values: input.values,
+      },
+      dispatchKey: `form-submission-webhook:${input.submissionId}`,
+    });
+  }
+
+  const recipients = input.notifyEmails.filter(
+    (email): email is string => typeof email === "string" && email.includes("@"),
+  );
+  if (recipients.length > 0) {
+    events.push({
+      tenantId: input.tenantId,
+      type: "mail.transactional.form-submission",
+      payload: {
+        submissionId: input.submissionId,
+        formId: input.formId,
+        tenantId: input.tenantId,
+        recipients,
+        targetType: input.targetType,
+        targetId: input.target.id,
+        values: input.values,
+      },
+      dispatchKey: `form-submission-mail:${input.submissionId}`,
+    });
+  }
+
+  return events;
+}

hivegrid-cloud/apps/worker/src/jobs/form-submission-route-bootstrap.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it, vi } from "vitest";
+import { bootstrapFormSubmissionRoute } from "./form-submission-route-bootstrap.js";
+import { FORM_SUBMISSION_ROUTE_QUEUE } from "./form-submission-route.js";
+
+function buildHandlerDeps() {
+  return {
+    store: {
+      markRouted: vi.fn(async () => undefined),
+      load: vi.fn(async () => ({ status: "received", tenantId: "t_1" })),
+    },
+  };
+}
+
+describe("bootstrapFormSubmissionRoute", () => {
+  it("disables itself when the worker group is not general", () => {
+    const res = bootstrapFormSubmissionRoute({ WORKER_QUEUE_GROUP: "mail" } as Record<string, string>);
+    expect(res.status).toBe("disabled");
+    if (res.status === "disabled") expect(res.reason).toMatch(/non-general/);
+  });
+
+  it("disables itself when the env flag is set to false", () => {
+    const res = bootstrapFormSubmissionRoute({
+      WORKER_QUEUE_GROUP: "general",
+      FORM_SUBMISSION_ROUTE_ENABLED: "false",
+    } as Record<string, string>);
+    expect(res.status).toBe("disabled");
+    if (res.status === "disabled") expect(res.reason).toMatch(/FORM_SUBMISSION_ROUTE_ENABLED/);
+  });
+
+  it("disables itself when no handlerDeps are supplied", () => {
+    const res = bootstrapFormSubmissionRoute({ WORKER_QUEUE_GROUP: "general" } as Record<string, string>);
+    expect(res.status).toBe("disabled");
+    if (res.status === "disabled") expect(res.reason).toMatch(/handler deps/);
+  });
+
+  it("subscribes to the form-submission-route queue when enabled", () => {
+    const create = vi.fn(() => ({ name: FORM_SUBMISSION_ROUTE_QUEUE, close: async () => undefined }));
+    const res = bootstrapFormSubmissionRoute(
+      { WORKER_QUEUE_GROUP: "general" } as Record<string, string>,
+      { handlerDeps: buildHandlerDeps(), createWorkerImpl: create },
+    );
+    expect(res.status).toBe("started");
+    expect(create).toHaveBeenCalledWith(
+      FORM_SUBMISSION_ROUTE_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 5 }),
+    );
+  });
+
+  it("honours FORM_SUBMISSION_ROUTE_CONCURRENCY", () => {
+    const create = vi.fn(() => ({ name: FORM_SUBMISSION_ROUTE_QUEUE, close: async () => undefined }));
+    bootstrapFormSubmissionRoute(
+      {
+        WORKER_QUEUE_GROUP: "general",
+        FORM_SUBMISSION_ROUTE_CONCURRENCY: "12",
+      } as Record<string, string>,
+      { handlerDeps: buildHandlerDeps(), createWorkerImpl: create },
+    );
+    expect(create).toHaveBeenCalledWith(
+      FORM_SUBMISSION_ROUTE_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 12 }),
+    );
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/form-submission-route-bootstrap.ts

@@ -0,0 +1,118 @@
+/**
+ * Bootstrap helper for the form-submission-route outbox subscriber.
+ *
+ * Subscribes a BullMQ worker to the `outbox.form-submission-route`
+ * queue inside the `general` worker group. Each delivered job carries
+ * a `form.submission.process` outbox payload; the handler routes the
+ * submission to CRM-contact / entity-row / generic targets and updates
+ * `FormSubmission.status` with the outcome.
+ *
+ * Gating:
+ *   - WORKER_QUEUE_GROUP === "general" (the route subscriber co-locates
+ *     with the outbox dispatcher to keep DB connections concentrated)
+ *   - FORM_SUBMISSION_ROUTE_ENABLED !== "false" (escape hatch)
+ *
+ * Tests inject a fake `createWorkerImpl` to assert subscription
+ * semantics without spinning up BullMQ.
+ */
+
+import type { EnvRecord } from "@toolctrl/config";
+
+import {
+  FORM_SUBMISSION_ROUTE_QUEUE,
+  runFormSubmissionRouteJob,
+  type FormSubmissionRouteDependencies,
+  type FormSubmissionRouteLogger,
+} from "./form-submission-route.js";
+
+export type FormSubmissionRouteBootstrapEnv = EnvRecord & {
+  WORKER_QUEUE_GROUP?: string;
+  FORM_SUBMISSION_ROUTE_ENABLED?: string;
+  FORM_SUBMISSION_ROUTE_CONCURRENCY?: string;
+};
+
+export interface FormSubmissionRouteWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface FormSubmissionRouteBootstrapDeps {
+  readonly logger?: FormSubmissionRouteLogger;
+  /** Pre-built handler deps (store + crm + entity + reporter + follow-ups). */
+  readonly handlerDeps?: FormSubmissionRouteDependencies;
+  /** BullMQ worker factory. Production builds a real Worker; tests fake it. */
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number },
+  ) => FormSubmissionRouteWorkerHandle;
+}
+
+export type FormSubmissionRouteBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: FormSubmissionRouteWorkerHandle };
+
+const DEFAULT_CONCURRENCY = 5;
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function defaultCreateWorker(
+  queueName: string,
+  processor: (job: { data: unknown }) => Promise<unknown>,
+  options: { concurrency?: number },
+): FormSubmissionRouteWorkerHandle {
+  // Real BullMQ wiring lives in `apps/worker/src/index.ts` where the Redis
+  // connection + Sentry tracer + DB client are in scope. The bootstrap
+  // accepts a `createWorkerImpl` so the host can inject the real worker;
+  // when nothing is provided we return an inert handle so tests can run.
+  void processor;
+  void options;
+  return {
+    name: queueName,
+    close: async () => undefined,
+  };
+}
+
+export function bootstrapFormSubmissionRoute(
+  env: FormSubmissionRouteBootstrapEnv,
+  deps: FormSubmissionRouteBootstrapDeps = {},
+): FormSubmissionRouteBootstrapResult {
+  const logger = deps.logger;
+
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
+    return { status: "disabled", reason: "non-general worker group" };
+  }
+  if (env.FORM_SUBMISSION_ROUTE_ENABLED === "false") {
+    return { status: "disabled", reason: "FORM_SUBMISSION_ROUTE_ENABLED=false" };
+  }
+  if (!deps.handlerDeps) {
+    return {
+      status: "disabled",
+      reason: "no handler deps wired (host must supply store + adapters)",
+    };
+  }
+
+  const handlerDeps = deps.handlerDeps;
+  const concurrency = parsePositiveInt(env.FORM_SUBMISSION_ROUTE_CONCURRENCY, DEFAULT_CONCURRENCY);
+
+  const processor = async (job: { data: unknown }): Promise<unknown> => {
+    return runFormSubmissionRouteJob(job.data, handlerDeps);
+  };
+
+  const createWorkerImpl = deps.createWorkerImpl ?? defaultCreateWorker;
+  const handle = createWorkerImpl(FORM_SUBMISSION_ROUTE_QUEUE, processor, { concurrency });
+
+  logger?.info?.(
+    {
+      component: "form-submission-route",
+      queue: FORM_SUBMISSION_ROUTE_QUEUE,
+      concurrency,
+    },
+    "form-submission-route worker started",
+  );
+  return { status: "started", handle };
+}

hivegrid-cloud/apps/worker/src/jobs/form-submission-route.test.ts

@@ -0,0 +1,261 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+  FORM_SUBMISSION_ROUTE_QUEUE,
+  FormSubmissionRouteJobError,
+  parseFormSubmissionRouteJob,
+  runFormSubmissionRouteJob,
+  type CrmContactCreator,
+  type EntityRowCreator,
+  type FormSubmissionStore,
+} from "./form-submission-route.js";
+
+function makeStore(initialStatus = "received"): FormSubmissionStore & {
+  markRouted: ReturnType<typeof vi.fn>;
+  load: ReturnType<typeof vi.fn>;
+} {
+  return {
+    markRouted: vi.fn(async () => undefined),
+    load: vi.fn(async () => ({ status: initialStatus, tenantId: "t_1" })),
+  };
+}
+
+function makeCrm(): CrmContactCreator & { createContact: ReturnType<typeof vi.fn> } {
+  return { createContact: vi.fn(async () => ({ id: "contact_1" })) };
+}
+
+function makeEntity(): EntityRowCreator & { createRow: ReturnType<typeof vi.fn> } {
+  return { createRow: vi.fn(async () => ({ id: "row_1" })) };
+}
+
+function buildJob(overrides: Record<string, unknown> = {}) {
+  return {
+    outboxEventId: "ob_1",
+    tenantId: "t_1",
+    eventType: "form.submission.process",
+    payload: {
+      submissionId: "sub_1",
+      formId: "form_1",
+      tenantId: "t_1",
+      targetType: "generic",
+      targetEntityId: null,
+      fieldMapping: {},
+      webhookUrl: null,
+      values: { email: "a@b.co", name: "Alice" },
+      ...overrides,
+    },
+  };
+}
+
+describe("FORM_SUBMISSION_ROUTE_QUEUE", () => {
+  it("matches the outbox form-submission-route queue name", () => {
+    expect(FORM_SUBMISSION_ROUTE_QUEUE).toBe("outbox.form-submission-route");
+  });
+});
+
+describe("parseFormSubmissionRouteJob", () => {
+  it("accepts a valid envelope", () => {
+    const parsed = parseFormSubmissionRouteJob(buildJob());
+    expect(parsed.payload.submissionId).toBe("sub_1");
+    expect(parsed.eventType).toBe("form.submission.process");
+  });
+
+  it("rejects envelopes with foreign event-type prefix", () => {
+    expect(() =>
+      parseFormSubmissionRouteJob({ ...buildJob(), eventType: "marketing.lead.captured" }),
+    ).toThrow(FormSubmissionRouteJobError);
+  });
+
+  it("rejects payloads missing submissionId", () => {
+    expect(() => parseFormSubmissionRouteJob(buildJob({ submissionId: 42 }))).toThrow(
+      FormSubmissionRouteJobError,
+    );
+  });
+
+  it("rejects payloads with invalid targetType", () => {
+    expect(() => parseFormSubmissionRouteJob(buildJob({ targetType: "rocket-launch" }))).toThrow(
+      FormSubmissionRouteJobError,
+    );
+  });
+
+  it("rejects envelope and payload tenant mismatches", () => {
+    expect(() => parseFormSubmissionRouteJob(buildJob({ tenantId: "t_2" }))).toThrow(
+      /tenantId mismatch/,
+    );
+  });
+
+  it("rejects null data", () => {
+    expect(() => parseFormSubmissionRouteJob(null)).toThrow(FormSubmissionRouteJobError);
+  });
+});
+
+describe("runFormSubmissionRouteJob", () => {
+  it("no-ops for targetType=generic and marks the submission processed", async () => {
+    const store = makeStore();
+    const result = await runFormSubmissionRouteJob(buildJob(), { store });
+    expect(result.status).toBe("processed");
+    expect(result.target.id).toBeNull();
+    expect(store.markRouted).toHaveBeenCalledWith(
+      expect.objectContaining({ submissionId: "sub_1", status: "processed" }),
+    );
+  });
+
+  it("creates a CRM contact for targetType=crm-contact using the field mapping", async () => {
+    const store = makeStore();
+    const crm = makeCrm();
+    const result = await runFormSubmissionRouteJob(
+      buildJob({
+        targetType: "crm-contact",
+        fieldMapping: { firstName: "firstName", email: "email" },
+        values: { firstName: "Alice", email: "a@b.co", honeypot: "" },
+      }),
+      { store, crm },
+    );
+    expect(result.status).toBe("routed-to-crm");
+    expect(result.target.id).toBe("contact_1");
+    expect(crm.createContact).toHaveBeenCalledWith({
+      tenantId: "t_1",
+      values: { firstName: "Alice", email: "a@b.co" },
+    });
+    expect(store.markRouted).toHaveBeenCalledWith(
+      expect.objectContaining({ status: "routed-to-crm" }),
+    );
+  });
+
+  it("creates an entity row for targetType=entity-row", async () => {
+    const store = makeStore();
+    const entity = makeEntity();
+    const result = await runFormSubmissionRouteJob(
+      buildJob({
+        targetType: "entity-row",
+        targetEntityId: "ent_42",
+        fieldMapping: { name: "title" },
+        values: { name: "My Lead" },
+      }),
+      { store, entity },
+    );
+    expect(result.status).toBe("routed-to-entity");
+    expect(result.target.id).toBe("row_1");
+    expect(entity.createRow).toHaveBeenCalledWith({
+      tenantId: "t_1",
+      entityId: "ent_42",
+      values: { title: "My Lead" },
+    });
+  });
+
+  it("throws and marks route-failed when the CRM adapter is missing", async () => {
+    const store = makeStore();
+    await expect(
+      runFormSubmissionRouteJob(buildJob({ targetType: "crm-contact" }), { store }),
+    ).rejects.toBeInstanceOf(FormSubmissionRouteJobError);
+    expect(store.markRouted).toHaveBeenCalledWith(
+      expect.objectContaining({ status: "route-failed" }),
+    );
+  });
+
+  it("throws and marks route-failed when entity-row routing has no entityId", async () => {
+    const store = makeStore();
+    const entity = makeEntity();
+    await expect(
+      runFormSubmissionRouteJob(
+        buildJob({ targetType: "entity-row", targetEntityId: null }),
+        { store, entity },
+      ),
+    ).rejects.toBeInstanceOf(FormSubmissionRouteJobError);
+    expect(store.markRouted).toHaveBeenCalledWith(
+      expect.objectContaining({ status: "route-failed" }),
+    );
+  });
+
+  it("propagates and marks route-failed when the CRM adapter throws", async () => {
+    const store = makeStore();
+    const crm: CrmContactCreator = {
+      createContact: async () => {
+        throw new Error("crm 503");
+      },
+    };
+    const reporter = { captureException: vi.fn() };
+    await expect(
+      runFormSubmissionRouteJob(
+        buildJob({ targetType: "crm-contact", fieldMapping: { email: "email" } }),
+        { store, crm, reporter },
+      ),
+    ).rejects.toThrow(/crm 503/);
+    expect(store.markRouted).toHaveBeenCalledWith(
+      expect.objectContaining({ status: "route-failed" }),
+    );
+    expect(reporter.captureException).toHaveBeenCalled();
+  });
+
+  it("skips routing for replays where the submission is already routed", async () => {
+    const store = makeStore("routed-to-crm");
+    const crm = makeCrm();
+    const result = await runFormSubmissionRouteJob(
+      buildJob({ targetType: "crm-contact" }),
+      { store, crm },
+    );
+    expect(result.status).toBe("routed-to-crm");
+    expect(crm.createContact).not.toHaveBeenCalled();
+    expect(store.markRouted).not.toHaveBeenCalled();
+  });
+
+  it("skips silently when the submission row no longer exists", async () => {
+    const store: FormSubmissionStore = {
+      load: async () => null,
+      markRouted: vi.fn(async () => undefined),
+    };
+    const result = await runFormSubmissionRouteJob(buildJob(), { store });
+    expect(result.status).toBe("processed");
+    expect((store.markRouted as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();
+  });
+
+  it("rejects cross-tenant submission id replays before routing", async () => {
+    const store: FormSubmissionStore = {
+      load: vi.fn(async () => ({ status: "received", tenantId: "t_2" })),
+      markRouted: vi.fn(async () => undefined),
+    };
+    const crm = makeCrm();
+    await expect(runFormSubmissionRouteJob(buildJob({ targetType: "crm-contact" }), { store, crm })).rejects.toMatchObject({
+      code: "TENANT_MISMATCH",
+    });
+    expect(crm.createContact).not.toHaveBeenCalled();
+    expect((store.markRouted as ReturnType<typeof vi.fn>)).not.toHaveBeenCalled();
+  });
+
+  it("fires the emitFollowUps callback after a successful CRM route", async () => {
+    const store = makeStore();
+    const crm = makeCrm();
+    const emitFollowUps = vi.fn(async () => undefined);
+    await runFormSubmissionRouteJob(
+      buildJob({
+        targetType: "crm-contact",
+        fieldMapping: { email: "email" },
+        webhookUrl: "https://example.com/hook",
+        notifyEmails: ["ops@example.com"],
+      }),
+      { store, crm, emitFollowUps },
+    );
+    expect(emitFollowUps).toHaveBeenCalledWith(
+      expect.objectContaining({
+        submissionId: "sub_1",
+        webhookUrl: "https://example.com/hook",
+        notifyEmails: ["ops@example.com"],
+        target: { id: "contact_1" },
+      }),
+    );
+  });
+
+  it("does not fail the job when emitFollowUps throws", async () => {
+    const store = makeStore();
+    const crm = makeCrm();
+    const emitFollowUps = vi.fn(async () => {
+      throw new Error("queue down");
+    });
+    const reporter = { captureException: vi.fn() };
+    const result = await runFormSubmissionRouteJob(
+      buildJob({ targetType: "crm-contact", fieldMapping: { email: "email" } }),
+      { store, crm, emitFollowUps, reporter },
+    );
+    expect(result.status).toBe("routed-to-crm");
+    expect(reporter.captureException).toHaveBeenCalled();
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/form-submission-route.ts

@@ -0,0 +1,408 @@
+/**
+ * Form-Widget submission-routing worker handler (Phase 05 #58).
+ *
+ * Subscribes to the `outbox.form-submission-route` BullMQ queue (populated
+ * by the outbox dispatcher when `eventType` starts with `form.submission.`).
+ *
+ * The submission API ({@link
+ * apps/web/app/routes/api.embed.forms.$formKey.submit.tsx}) already
+ * persists every payload as a generic `FormSubmission` row. This worker
+ * runs the *follow-up* side-effect — depending on `targetType`:
+ *
+ *  - `generic`      → no-op (the row is already stored)
+ *  - `crm-contact`  → create a CRM contact via `CrmService.createContact`
+ *                     using the form's `fieldMapping`
+ *  - `entity-row`   → create a row in `targetEntityId` via `RowsApi.create`
+ *
+ * On success the worker:
+ *   - updates `FormSubmission.status` to `routed-to-crm` /
+ *     `routed-to-entity` / `processed` (for generic), and
+ *   - writes the target reference (e.g. created row id) to
+ *     `FormSubmission.result`.
+ *
+ * Failures are NOT silently swallowed:
+ *   - `FormSubmission.status` becomes `route-failed`
+ *   - `FormSubmission.result.error` carries the redacted message
+ *   - the error is re-thrown so the BullMQ retry machinery + Sentry
+ *     reporter both see it.
+ *
+ * Idempotency: routing relies on the outbox dispatch-key
+ * (`form-submission:<id>`) to avoid duplicate delivery. The worker
+ * itself short-circuits when it sees a non-`received` status, which
+ * makes manual replays safe.
+ */
+
+import { applyFieldMapping } from "@toolctrl/form-widget";
+
+export const FORM_SUBMISSION_ROUTE_QUEUE = "outbox.form-submission-route" as const;
+
+/** Routing-status values written back to `FormSubmission.status`. */
+export type FormSubmissionRoutingStatus =
+  | "received"
+  | "processed"
+  | "routed-to-crm"
+  | "routed-to-entity"
+  | "route-failed";
+
+export interface FormSubmissionRouteJobData {
+  readonly outboxEventId?: string;
+  readonly tenantId: string;
+  readonly eventType: string;
+  readonly payload: {
+    readonly submissionId: string;
+    readonly formId: string;
+    readonly tenantId: string;
+    readonly targetType: "generic" | "crm-contact" | "entity-row";
+    readonly targetEntityId: string | null;
+    readonly fieldMapping: Record<string, string>;
+    readonly webhookUrl: string | null;
+    readonly notifyEmails?: readonly string[];
+    readonly values: Record<string, unknown>;
+  };
+}
+
+export interface FormSubmissionStore {
+  /**
+   * Mark a submission row with routing result. Implementations are
+   * expected to issue a single Prisma `update` keyed by submissionId.
+   */
+  markRouted(input: {
+    submissionId: string;
+    tenantId: string;
+    status: FormSubmissionRoutingStatus;
+    result: Record<string, unknown>;
+  }): Promise<void>;
+  /**
+   * Look up the current row so we can short-circuit replays / re-runs.
+   * `null` ⇒ submission was deleted; we no-op and log.
+   */
+  load(submissionId: string): Promise<{ status: string; tenantId: string } | null>;
+}
+
+/**
+ * CRM adapter — accepts a flat key/value record produced by
+ * `applyFieldMapping`. We don't bind to a specific CRM signature here
+ * (Phase 05's CRM lives in apps/web). The bootstrap wires a thin
+ * adapter that forwards to `CrmService.createContact`.
+ */
+export interface CrmContactCreator {
+  createContact(input: {
+    tenantId: string;
+    values: Record<string, unknown>;
+  }): Promise<{ id: string }>;
+}
+
+export interface EntityRowCreator {
+  createRow(input: {
+    tenantId: string;
+    entityId: string;
+    values: Record<string, unknown>;
+  }): Promise<{ id: string }>;
+}
+
+export interface FormSubmissionRouteLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface FormSubmissionRouteReporter {
+  captureException?: (error: unknown, context?: Record<string, unknown>) => void;
+}
+
+export interface FormSubmissionRouteDependencies {
+  readonly store: FormSubmissionStore;
+  readonly crm?: CrmContactCreator;
+  readonly entity?: EntityRowCreator;
+  readonly logger?: FormSubmissionRouteLogger;
+  readonly reporter?: FormSubmissionRouteReporter;
+  /**
+   * Optional callback fired after routing succeeds, used by the bootstrap
+   * to publish webhook + notify-email follow-up outbox events. Kept as a
+   * dependency so this handler stays free of DB / Prisma imports.
+   */
+  readonly emitFollowUps?: (followUp: {
+    submissionId: string;
+    formId: string;
+    tenantId: string;
+    targetType: FormSubmissionRouteJobData["payload"]["targetType"];
+    target: { id: string | null };
+    webhookUrl: string | null;
+    notifyEmails: readonly string[];
+    values: Record<string, unknown>;
+  }) => Promise<void>;
+}
+
+export interface FormSubmissionRouteJobResult {
+  readonly submissionId: string;
+  readonly status: FormSubmissionRoutingStatus;
+  readonly target: { id: string | null };
+}
+
+export class FormSubmissionRouteJobError extends Error {
+  override readonly name = "FormSubmissionRouteJobError";
+  readonly code:
+    | "INVALID_PAYLOAD"
+    | "INVALID_EVENT_TYPE"
+    | "TENANT_MISMATCH"
+    | "MISSING_CRM_ADAPTER"
+    | "MISSING_ENTITY_ADAPTER"
+    | "MISSING_TARGET_ENTITY";
+  constructor(message: string, code: FormSubmissionRouteJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export function parseFormSubmissionRouteJob(data: unknown): FormSubmissionRouteJobData {
+  if (!data || typeof data !== "object") {
+    throw new FormSubmissionRouteJobError(
+      "form-submission-route job missing data envelope",
+      "INVALID_PAYLOAD",
+    );
+  }
+  const envelope = data as Record<string, unknown>;
+  const eventType = typeof envelope.eventType === "string" ? envelope.eventType : "";
+  if (!eventType.startsWith("form.submission.")) {
+    throw new FormSubmissionRouteJobError(
+      `form-submission-route rejected eventType ${JSON.stringify(eventType)}`,
+      "INVALID_EVENT_TYPE",
+    );
+  }
+  const tenantId = typeof envelope.tenantId === "string" ? envelope.tenantId : "";
+  if (!tenantId) {
+    throw new FormSubmissionRouteJobError(
+      "form-submission-route job missing tenantId",
+      "INVALID_PAYLOAD",
+    );
+  }
+  const rawPayload = envelope.payload;
+  if (!rawPayload || typeof rawPayload !== "object") {
+    throw new FormSubmissionRouteJobError(
+      "form-submission-route job missing payload",
+      "INVALID_PAYLOAD",
+    );
+  }
+  const p = rawPayload as Record<string, unknown>;
+  const submissionId = typeof p.submissionId === "string" ? p.submissionId : "";
+  const formId = typeof p.formId === "string" ? p.formId : "";
+  const targetType = p.targetType;
+  if (!submissionId || !formId) {
+    throw new FormSubmissionRouteJobError(
+      "form-submission-route payload missing submissionId/formId",
+      "INVALID_PAYLOAD",
+    );
+  }
+  if (targetType !== "generic" && targetType !== "crm-contact" && targetType !== "entity-row") {
+    throw new FormSubmissionRouteJobError(
+      `form-submission-route payload has invalid targetType ${JSON.stringify(targetType)}`,
+      "INVALID_PAYLOAD",
+    );
+  }
+  const fieldMapping =
+    p.fieldMapping && typeof p.fieldMapping === "object"
+      ? (p.fieldMapping as Record<string, string>)
+      : {};
+  const values =
+    p.values && typeof p.values === "object" ? (p.values as Record<string, unknown>) : {};
+  const notifyEmails = Array.isArray(p.notifyEmails)
+    ? (p.notifyEmails.filter((v) => typeof v === "string") as string[])
+    : [];
+  const payloadTenantId = typeof p.tenantId === "string" ? p.tenantId : tenantId;
+  if (payloadTenantId !== tenantId) {
+    throw new FormSubmissionRouteJobError(
+      "form-submission-route tenantId mismatch between envelope and payload",
+      "TENANT_MISMATCH",
+    );
+  }
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId,
+    eventType,
+    payload: {
+      submissionId,
+      formId,
+      tenantId: payloadTenantId,
+      targetType,
+      targetEntityId: typeof p.targetEntityId === "string" ? p.targetEntityId : null,
+      fieldMapping,
+      webhookUrl: typeof p.webhookUrl === "string" ? p.webhookUrl : null,
+      notifyEmails,
+      values,
+    },
+  };
+}
+
+function redactErrorMessage(message: string): string {
+  // Cap length to 500 chars and strip obvious secret-shaped substrings.
+  const capped = message.length > 500 ? `${message.slice(0, 480)}… (truncated)` : message;
+  return capped
+    .replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
+    .replace(/(password|secret|token|api[_-]?key)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]");
+}
+
+export async function runFormSubmissionRouteJob(
+  data: unknown,
+  deps: FormSubmissionRouteDependencies,
+): Promise<FormSubmissionRouteJobResult> {
+  const job = parseFormSubmissionRouteJob(data);
+  const { submissionId, formId, tenantId, targetType, targetEntityId, fieldMapping, values } =
+    job.payload;
+
+  const existing = await deps.store.load(submissionId);
+  if (!existing) {
+    deps.logger?.warn?.(
+      { op: "form-submission-route.missing", submissionId, formId },
+      "submission row not found — skipping",
+    );
+    return { submissionId, status: "processed", target: { id: null } };
+  }
+  if (existing.status !== "received") {
+    deps.logger?.info?.(
+      { op: "form-submission-route.skipped", submissionId, status: existing.status },
+      "submission already routed — skipping replay",
+    );
+    return {
+      submissionId,
+      status: existing.status as FormSubmissionRoutingStatus,
+      target: { id: null },
+    };
+  }
+  if (existing.tenantId !== tenantId) {
+    throw new FormSubmissionRouteJobError(
+      "form-submission-route loaded a submission for a different tenant",
+      "TENANT_MISMATCH",
+    );
+  }
+
+  let routedTarget: { id: string | null } = { id: null };
+  let newStatus: FormSubmissionRoutingStatus = "processed";
+
+  try {
+    if (targetType === "generic") {
+      newStatus = "processed";
+    } else if (targetType === "crm-contact") {
+      if (!deps.crm) {
+        throw new FormSubmissionRouteJobError(
+          "CRM adapter not configured",
+          "MISSING_CRM_ADAPTER",
+        );
+      }
+      const mapped = applyFieldMapping({ payload: values, fieldMapping });
+      const contact = await deps.crm.createContact({ tenantId, values: mapped });
+      routedTarget = { id: contact.id };
+      newStatus = "routed-to-crm";
+    } else if (targetType === "entity-row") {
+      if (!targetEntityId) {
+        throw new FormSubmissionRouteJobError(
+          "entity-row routing requires targetEntityId",
+          "MISSING_TARGET_ENTITY",
+        );
+      }
+      if (!deps.entity) {
+        throw new FormSubmissionRouteJobError(
+          "Entity adapter not configured",
+          "MISSING_ENTITY_ADAPTER",
+        );
+      }
+      const mapped = applyFieldMapping({ payload: values, fieldMapping });
+      const row = await deps.entity.createRow({
+        tenantId,
+        entityId: targetEntityId,
+        values: mapped,
+      });
+      routedTarget = { id: row.id };
+      newStatus = "routed-to-entity";
+    }
+
+    await deps.store.markRouted({
+      submissionId,
+      tenantId,
+      status: newStatus,
+      result: { target: targetType, targetId: routedTarget.id },
+    });
+
+    deps.logger?.info?.(
+      {
+        op: "form-submission-route.routed",
+        submissionId,
+        formId,
+        targetType,
+        targetId: routedTarget.id,
+        status: newStatus,
+      },
+      "submission routed",
+    );
+
+    if (deps.emitFollowUps) {
+      try {
+        await deps.emitFollowUps({
+          submissionId,
+          formId,
+          tenantId,
+          targetType,
+          target: routedTarget,
+          webhookUrl: job.payload.webhookUrl,
+          notifyEmails: job.payload.notifyEmails ?? [],
+          values,
+        });
+      } catch (err) {
+        // Follow-up emission MUST NOT fail the route job; the row is
+        // already routed. We log + capture so ops can backfill.
+        const msg = err instanceof Error ? err.message : String(err);
+        deps.logger?.warn?.(
+          {
+            op: "form-submission-route.followup-failed",
+            submissionId,
+            err: redactErrorMessage(msg),
+          },
+          "follow-up emission failed (non-fatal)",
+        );
+        deps.reporter?.captureException?.(err, {
+          component: "form-submission-route",
+          op: "emit-follow-ups",
+          submissionId,
+        });
+      }
+    }
+
+    return { submissionId, status: newStatus, target: routedTarget };
+  } catch (err) {
+    const message = redactErrorMessage(err instanceof Error ? err.message : String(err));
+    deps.logger?.error(
+      {
+        op: "form-submission-route.failed",
+        submissionId,
+        formId,
+        targetType,
+        err: message,
+      },
+      "submission routing failed",
+    );
+    deps.reporter?.captureException?.(err, {
+      component: "form-submission-route",
+      op: "route",
+      submissionId,
+      formId,
+      targetType,
+    });
+    try {
+      await deps.store.markRouted({
+        submissionId,
+        tenantId,
+        status: "route-failed",
+        result: { target: targetType, error: message },
+      });
+    } catch (markErr) {
+      deps.logger?.error(
+        {
+          op: "form-submission-route.mark-failed-error",
+          submissionId,
+          err: markErr instanceof Error ? redactErrorMessage(markErr.message) : String(markErr),
+        },
+        "could not mark submission as failed",
+      );
+    }
+    throw err;
+  }
+}

hivegrid-cloud/apps/worker/src/jobs/form-submission-webhook-delivery-bootstrap.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it, vi } from "vitest";
+import { bootstrapFormSubmissionWebhookDelivery } from "./form-submission-webhook-delivery-bootstrap.js";
+import { FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE } from "./form-submission-webhook-delivery.js";
+
+function buildHandlerDeps() {
+  return {
+    enqueueWebhook: vi.fn(async () => undefined),
+  };
+}
+
+describe("bootstrapFormSubmissionWebhookDelivery", () => {
+  it("disables itself outside the mail worker group", () => {
+    const res = bootstrapFormSubmissionWebhookDelivery({ WORKER_QUEUE_GROUP: "general" } as Record<string, string>);
+    expect(res.status).toBe("disabled");
+    if (res.status === "disabled") expect(res.reason).toMatch(/non-mail/);
+  });
+
+  it("disables itself when the env flag is false", () => {
+    const res = bootstrapFormSubmissionWebhookDelivery({
+      WORKER_QUEUE_GROUP: "mail",
+      FORM_SUBMISSION_WEBHOOK_DELIVERY_ENABLED: "false",
+    } as Record<string, string>);
+    expect(res.status).toBe("disabled");
+    if (res.status === "disabled") expect(res.reason).toMatch(/FORM_SUBMISSION_WEBHOOK_DELIVERY_ENABLED/);
+  });
+
+  it("subscribes to outbox.webhook on the mail worker group", () => {
+    const create = vi.fn(() => ({ name: FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE, close: async () => undefined }));
+    const res = bootstrapFormSubmissionWebhookDelivery(
+      { WORKER_QUEUE_GROUP: "mail" } as Record<string, string>,
+      { handlerDeps: buildHandlerDeps(), createWorkerImpl: create },
+    );
+    expect(res.status).toBe("started");
+    expect(create).toHaveBeenCalledWith(
+      FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 5 }),
+    );
+  });
+
+  it("honours FORM_SUBMISSION_WEBHOOK_DELIVERY_CONCURRENCY", () => {
+    const create = vi.fn(() => ({ name: FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE, close: async () => undefined }));
+    bootstrapFormSubmissionWebhookDelivery(
+      {
+        WORKER_QUEUE_GROUP: "mail",
+        FORM_SUBMISSION_WEBHOOK_DELIVERY_CONCURRENCY: "9",
+      } as Record<string, string>,
+      { handlerDeps: buildHandlerDeps(), createWorkerImpl: create },
+    );
+    expect(create).toHaveBeenCalledWith(
+      FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 9 }),
+    );
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/form-submission-webhook-delivery-bootstrap.ts

@@ -0,0 +1,74 @@
+import type { EnvRecord } from "@toolctrl/config";
+
+import {
+  FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE,
+  runFormSubmissionWebhookDeliveryJob,
+  type FormSubmissionWebhookDeliveryDeps,
+} from "./form-submission-webhook-delivery.js";
+import type { FormSubmissionRouteLogger } from "./form-submission-route.js";
+
+export type FormSubmissionWebhookDeliveryBootstrapEnv = EnvRecord & {
+  WORKER_QUEUE_GROUP?: string;
+  FORM_SUBMISSION_WEBHOOK_DELIVERY_ENABLED?: string;
+  FORM_SUBMISSION_WEBHOOK_DELIVERY_CONCURRENCY?: string;
+};
+
+export interface FormSubmissionWebhookDeliveryWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface FormSubmissionWebhookDeliveryBootstrapDeps {
+  readonly logger?: FormSubmissionRouteLogger;
+  readonly handlerDeps?: FormSubmissionWebhookDeliveryDeps;
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number },
+  ) => FormSubmissionWebhookDeliveryWorkerHandle;
+}
+
+export type FormSubmissionWebhookDeliveryBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: FormSubmissionWebhookDeliveryWorkerHandle };
+
+const DEFAULT_CONCURRENCY = 5;
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export function bootstrapFormSubmissionWebhookDelivery(
+  env: FormSubmissionWebhookDeliveryBootstrapEnv,
+  deps: FormSubmissionWebhookDeliveryBootstrapDeps = {},
+): FormSubmissionWebhookDeliveryBootstrapResult {
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "mail") {
+    return { status: "disabled", reason: "non-mail worker group" };
+  }
+  if (env.FORM_SUBMISSION_WEBHOOK_DELIVERY_ENABLED === "false") {
+    return { status: "disabled", reason: "FORM_SUBMISSION_WEBHOOK_DELIVERY_ENABLED=false" };
+  }
+  if (!deps.handlerDeps) {
+    return { status: "disabled", reason: "no handler deps wired (host must supply webhook enqueue)" };
+  }
+  if (!deps.createWorkerImpl) {
+    return { status: "disabled", reason: "no worker factory wired" };
+  }
+
+  const concurrency = parsePositiveInt(env.FORM_SUBMISSION_WEBHOOK_DELIVERY_CONCURRENCY, DEFAULT_CONCURRENCY);
+  const handlerDeps = deps.handlerDeps;
+  const processor = (job: { data: unknown }) => runFormSubmissionWebhookDeliveryJob(job.data, handlerDeps);
+  const handle = deps.createWorkerImpl(FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE, processor, { concurrency });
+
+  deps.logger?.info?.(
+    {
+      component: "form-submission-webhook-delivery",
+      queue: FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE,
+      concurrency,
+    },
+    "form-submission webhook delivery bridge started",
+  );
+  return { status: "started", handle };
+}

hivegrid-cloud/apps/worker/src/jobs/form-submission-webhook-delivery.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it, vi } from "vitest";
+import { queueNames } from "@toolctrl/queue";
+import {
+  FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE,
+  FormSubmissionWebhookDeliveryJobError,
+  parseFormSubmissionWebhookDeliveryJob,
+  runFormSubmissionWebhookDeliveryJob,
+} from "./form-submission-webhook-delivery.js";
+
+function buildJob(overrides: Record<string, unknown> = {}) {
+  return {
+    outboxEventId: "outbox_1",
+    tenantId: "tenant_1",
+    eventType: "webhook.form-submission.deliver",
+    dispatchKey: "form-submission-webhook:sub_1",
+    payload: {
+      submissionId: "sub_1",
+      formId: "form_1",
+      tenantId: "tenant_1",
+      webhookUrl: "https://example.com/hook",
+      targetType: "crm-contact",
+      targetId: "contact_1",
+      values: { email: "ada@example.com" },
+      ...overrides,
+    },
+  };
+}
+
+describe("FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE", () => {
+  it("matches the outbox webhook queue", () => {
+    expect(FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE).toBe("outbox.webhook");
+  });
+});
+
+describe("parseFormSubmissionWebhookDeliveryJob", () => {
+  it("accepts the exact form-submission webhook event", () => {
+    const parsed = parseFormSubmissionWebhookDeliveryJob(buildJob());
+    expect(parsed.eventType).toBe("webhook.form-submission.deliver");
+    expect(parsed.payload.webhookUrl).toBe("https://example.com/hook");
+  });
+
+  it("rejects other webhook event types", () => {
+    expect(() => parseFormSubmissionWebhookDeliveryJob({ ...buildJob(), eventType: "webhook.other.deliver" })).toThrow(FormSubmissionWebhookDeliveryJobError);
+  });
+
+  it("rejects envelope and payload tenant mismatches", () => {
+    expect(() => parseFormSubmissionWebhookDeliveryJob(buildJob({ tenantId: "tenant_2" }))).toThrow(/tenantId mismatch/);
+  });
+});
+
+describe("runFormSubmissionWebhookDeliveryJob", () => {
+  it("enqueues a deliver-webhook job with outbox idempotency", async () => {
+    const enqueueWebhook = vi.fn(async () => undefined);
+
+    const result = await runFormSubmissionWebhookDeliveryJob(buildJob(), { enqueueWebhook });
+
+    expect(result).toEqual({ enqueued: true, queueName: queueNames.email, jobName: "deliver-webhook" });
+    expect(enqueueWebhook).toHaveBeenCalledWith(
+      expect.objectContaining({
+        queueName: queueNames.email,
+        jobName: "deliver-webhook",
+        payload: expect.objectContaining({
+          kind: "webhook.deliver",
+          url: "https://example.com/hook",
+          method: "POST",
+          eventName: "form.submission",
+          tenantId: "tenant_1",
+          body: expect.objectContaining({ submissionId: "sub_1", tenantId: "tenant_1" }),
+          metadata: expect.objectContaining({ source: "form-submission", outboxEventId: "outbox_1" }),
+        }),
+      }),
+      expect.objectContaining({ tenantId: "tenant_1", idempotencyKey: "outbox_1" }),
+    );
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/form-submission-webhook-delivery.ts

@@ -0,0 +1,119 @@
+/**
+ * Bridges form-submission webhook outbox events onto the existing
+ * `toolctrl.email` webhook delivery processor.
+ */
+
+import {
+  buildWebhookDeliveryJob,
+  type QueueEnqueueOptions,
+  type WebhookDeliveryJobDescriptor,
+} from "@toolctrl/queue";
+
+export const FORM_SUBMISSION_WEBHOOK_OUTBOX_QUEUE = "outbox.webhook" as const;
+
+export interface FormSubmissionWebhookDeliveryJobData {
+  readonly outboxEventId?: string;
+  readonly tenantId: string;
+  readonly eventType: "webhook.form-submission.deliver";
+  readonly dispatchKey?: string | null;
+  readonly payload: {
+    readonly submissionId: string;
+    readonly formId: string;
+    readonly tenantId: string;
+    readonly webhookUrl: string;
+    readonly targetType?: string;
+    readonly targetId?: string | null;
+    readonly values?: Record<string, unknown>;
+  };
+}
+
+export interface FormSubmissionWebhookDeliveryDeps {
+  readonly enqueueWebhook: (
+    descriptor: WebhookDeliveryJobDescriptor,
+    options: QueueEnqueueOptions,
+  ) => Promise<unknown>;
+}
+
+export class FormSubmissionWebhookDeliveryJobError extends Error {
+  override readonly name = "FormSubmissionWebhookDeliveryJobError";
+  readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "TENANT_MISMATCH";
+
+  constructor(message: string, code: FormSubmissionWebhookDeliveryJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export function parseFormSubmissionWebhookDeliveryJob(data: unknown): FormSubmissionWebhookDeliveryJobData {
+  if (!data || typeof data !== "object") {
+    throw new FormSubmissionWebhookDeliveryJobError("form-submission webhook job missing envelope", "INVALID_PAYLOAD");
+  }
+  const envelope = data as Record<string, unknown>;
+  if (envelope.eventType !== "webhook.form-submission.deliver") {
+    throw new FormSubmissionWebhookDeliveryJobError(
+      `form-submission webhook rejected eventType ${JSON.stringify(envelope.eventType)}`,
+      "INVALID_EVENT_TYPE",
+    );
+  }
+  const tenantId = typeof envelope.tenantId === "string" ? envelope.tenantId : "";
+  if (tenantId.length === 0) {
+    throw new FormSubmissionWebhookDeliveryJobError("form-submission webhook job missing tenantId", "INVALID_PAYLOAD");
+  }
+  const rawPayload = envelope.payload;
+  if (!rawPayload || typeof rawPayload !== "object") {
+    throw new FormSubmissionWebhookDeliveryJobError("form-submission webhook job missing payload", "INVALID_PAYLOAD");
+  }
+  const payload = rawPayload as Record<string, unknown>;
+  const payloadTenantId = typeof payload.tenantId === "string" ? payload.tenantId : "";
+  if (payloadTenantId !== tenantId) {
+    throw new FormSubmissionWebhookDeliveryJobError("form-submission webhook tenantId mismatch", "TENANT_MISMATCH");
+  }
+  const submissionId = typeof payload.submissionId === "string" ? payload.submissionId : "";
+  const formId = typeof payload.formId === "string" ? payload.formId : "";
+  const webhookUrl = typeof payload.webhookUrl === "string" ? payload.webhookUrl.trim() : "";
+  if (!submissionId || !formId || !webhookUrl) {
+    throw new FormSubmissionWebhookDeliveryJobError("form-submission webhook payload missing required fields", "INVALID_PAYLOAD");
+  }
+
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId,
+    eventType: "webhook.form-submission.deliver",
+    dispatchKey: typeof envelope.dispatchKey === "string" ? envelope.dispatchKey : null,
+    payload: {
+      submissionId,
+      formId,
+      tenantId: payloadTenantId,
+      webhookUrl,
+      targetType: typeof payload.targetType === "string" ? payload.targetType : undefined,
+      targetId: typeof payload.targetId === "string" ? payload.targetId : null,
+      values: payload.values && typeof payload.values === "object" ? (payload.values as Record<string, unknown>) : {},
+    },
+  };
+}
+
+export async function runFormSubmissionWebhookDeliveryJob(
+  data: unknown,
+  deps: FormSubmissionWebhookDeliveryDeps,
+): Promise<{ enqueued: true; queueName: WebhookDeliveryJobDescriptor["queueName"]; jobName: WebhookDeliveryJobDescriptor["jobName"] }> {
+  const job = parseFormSubmissionWebhookDeliveryJob(data);
+  const descriptor = buildWebhookDeliveryJob({
+    url: job.payload.webhookUrl,
+    method: "POST",
+    eventName: "form.submission",
+    tenantId: job.tenantId,
+    body: job.payload,
+    metadata: {
+      source: "form-submission",
+      ...(job.outboxEventId ? { outboxEventId: job.outboxEventId } : {}),
+      ...(job.dispatchKey ? { dispatchKey: job.dispatchKey } : {}),
+    },
+  });
+
+  await deps.enqueueWebhook(descriptor, {
+    tenantId: job.tenantId,
+    idempotencyKey: job.outboxEventId ?? job.dispatchKey ?? `form-submission-webhook:${job.payload.submissionId}`,
+  });
+
+  return { enqueued: true, queueName: descriptor.queueName, jobName: descriptor.jobName };
+}

hivegrid-cloud/apps/worker/src/jobs/lead-stale-trigger.test.ts

@@ -0,0 +1,154 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { runLeadStaleTriggerCron, type LeadStaleCronDeps, type LeadStaleRow } from "./lead-stale-trigger.js";
+
+function buildDeps(leads: LeadStaleRow[]): LeadStaleCronDeps & {
+  published: Array<{ type: string; payload: Record<string, unknown>; dispatchKey: string }>;
+} {
+  const published: Array<{ type: string; payload: Record<string, unknown>; dispatchKey: string }> = [];
+  return {
+    store: {
+      findStaleLeads: vi.fn(async () => leads),
+    },
+    publisher: {
+      publish: vi.fn(async (args) => {
+        published.push({ type: args.type, payload: args.payload, dispatchKey: args.dispatchKey });
+        return { ok: true };
+      }),
+    },
+    logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn() },
+    now: () => new Date("2026-05-18T00:00:00Z"),
+    published,
+  };
+}
+
+const LEADS: LeadStaleRow[] = [
+  {
+    id: "lead_1",
+    email: "alice@example.com",
+    firstName: "Alice",
+    status: "new",
+    firstSeenAt: new Date("2026-02-01T00:00:00Z"),
+  },
+  {
+    id: "lead_2",
+    email: "bob@example.com",
+    firstName: null,
+    status: "contacted",
+    firstSeenAt: new Date("2026-01-15T00:00:00Z"),
+  },
+];
+
+function deferred() {
+  let resolve!: () => void;
+  const promise = new Promise<void>((res) => {
+    resolve = res;
+  });
+  return { promise, resolve };
+}
+
+afterEach(() => {
+  delete process.env.MARKETING_SEQUENCES_ENABLED;
+});
+
+beforeEach(() => {
+  process.env.MARKETING_SEQUENCES_ENABLED = "true";
+});
+
+describe("runLeadStaleTriggerCron", () => {
+  it("emits one lead.stale_60d outbox event per stale lead", async () => {
+    const deps = buildDeps(LEADS);
+    const result = await runLeadStaleTriggerCron(deps);
+    expect(result.scanned).toBe(2);
+    expect(result.emitted).toBe(2);
+    expect(deps.published.map((p) => p.type)).toEqual(["lead.stale_60d", "lead.stale_60d"]);
+  });
+
+  it("uses a date-keyed dispatchKey for idempotency", async () => {
+    const deps = buildDeps([LEADS[0]!]);
+    await runLeadStaleTriggerCron(deps);
+    expect(deps.published[0]?.dispatchKey).toBe("lead.stale_60d.lead_1.2026-05-18");
+  });
+
+  it("returns zero work when no stale leads found", async () => {
+    const deps = buildDeps([]);
+    const result = await runLeadStaleTriggerCron(deps);
+    expect(result.scanned).toBe(0);
+    expect(result.emitted).toBe(0);
+  });
+
+  it("short-circuits when killswitch active", async () => {
+    process.env.MARKETING_SEQUENCES_ENABLED = "false";
+    const deps = buildDeps(LEADS);
+    const result = await runLeadStaleTriggerCron(deps);
+    expect(result.scanned).toBe(0);
+    expect(deps.store.findStaleLeads).not.toHaveBeenCalled();
+  });
+
+  it("forwards thresholdDays and statuses to the store", async () => {
+    const deps = buildDeps(LEADS);
+    await runLeadStaleTriggerCron({ ...deps, thresholdDays: 30, statuses: ["new"] });
+    expect(deps.store.findStaleLeads).toHaveBeenCalledWith(expect.objectContaining({ statuses: ["new"] }));
+    const call = vi.mocked(deps.store.findStaleLeads).mock.calls[0]?.[0];
+    expect(call?.thresholdDate.toISOString()).toBe("2026-04-18T00:00:00.000Z");
+  });
+
+  it("counts publisher failures as skipped without aborting the scan", async () => {
+    const deps = buildDeps(LEADS);
+    deps.publisher.publish = vi.fn().mockResolvedValueOnce({ ok: true }).mockRejectedValueOnce(new Error("simulated"));
+    const result = await runLeadStaleTriggerCron(deps);
+    expect(result.scanned).toBe(2);
+    expect(result.emitted).toBe(1);
+    expect(result.skipped).toBe(1);
+    expect(deps.logger?.error).toHaveBeenCalled();
+  });
+
+  it("caps publish concurrency", async () => {
+    const leads = Array.from(
+      { length: 5 },
+      (_, index): LeadStaleRow => ({
+        id: `lead_${index + 1}`,
+        email: `lead-${index + 1}@example.com`,
+        firstName: null,
+        status: "new",
+        firstSeenAt: new Date("2026-01-01T00:00:00Z"),
+      })
+    );
+    const deps = buildDeps(leads);
+    let active = 0;
+    let maxActive = 0;
+    let releaseIndex = 0;
+    const releases = Array.from({ length: 5 }, () => deferred());
+    deps.publisher.publish = vi.fn(async () => {
+      const release = releases[releaseIndex++]!;
+      active += 1;
+      maxActive = Math.max(maxActive, active);
+      await release.promise;
+      active -= 1;
+      return { ok: true };
+    });
+
+    const resultPromise = runLeadStaleTriggerCron({ ...deps, concurrency: 2 });
+    await Promise.resolve();
+    releases[0]!.resolve();
+    releases[1]!.resolve();
+    await Promise.resolve();
+    releases[2]!.resolve();
+    releases[3]!.resolve();
+    await Promise.resolve();
+    releases[4]!.resolve();
+
+    const result = await resultPromise;
+    expect(result).toMatchObject({ scanned: 5, emitted: 5, skipped: 0 });
+    expect(maxActive).toBeLessThanOrEqual(2);
+  });
+
+  it("payload contains the canonical envelope fields", async () => {
+    const deps = buildDeps([LEADS[0]!]);
+    await runLeadStaleTriggerCron(deps);
+    expect(deps.published[0]?.payload).toMatchObject({
+      leadId: "lead_1",
+      email: "alice@example.com",
+      recipientName: "Alice",
+    });
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/lead-stale-trigger.ts

@@ -0,0 +1,120 @@
+import { mapWithConcurrency } from "@toolctrl/foundation";
+
+/**
+ * Marketing-Funnel Phase 3 (Phase 05 #80) — stale-lead trigger cron.
+ *
+ * Daily cron worker that scans `Lead` for rows older than the
+ * configured threshold (default 60 days) whose `status` is still
+ * `new` or `contacted` and which have not converted. For every match
+ * a `lead.stale_60d` outbox event is emitted; downstream the
+ * email-sequence trigger handler enrols re-activation sequences.
+ *
+ * Storage + clock + outbox publisher are injected so the handler is
+ * fully unit-testable. Intended production wiring is a daily BullMQ
+ * scheduled-job from `apps/worker/src/index.ts`; as of this slice the
+ * runner is not registered by the worker entrypoint.
+ *
+ * Idempotency: the dispatchKey is `lead.stale_60d.<leadId>.<YYYY-MM-DD>`
+ * so re-running the cron on the same day is a no-op even if the worker
+ * restarts mid-scan.
+ */
+
+export interface LeadStaleRow {
+  readonly id: string;
+  readonly email: string;
+  readonly firstName: string | null;
+  readonly status: string;
+  readonly firstSeenAt: Date;
+}
+
+export interface LeadStaleStore {
+  findStaleLeads(args: { thresholdDate: Date; statuses: readonly string[]; limit: number }): Promise<readonly LeadStaleRow[]>;
+}
+
+export interface LeadStalePublisherTx {
+  outboxEvent: {
+    create(args: { data: Record<string, unknown> }): Promise<unknown>;
+    findUnique(args: { where: { dispatchKey: string } }): Promise<unknown | null>;
+  };
+}
+
+export interface LeadStalePublisher {
+  publish(args: { type: string; payload: Record<string, unknown>; dispatchKey: string; tenantId?: string | null }): Promise<{ ok: boolean }>;
+}
+
+export interface LeadStaleLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface LeadStaleCronDeps {
+  readonly store: LeadStaleStore;
+  readonly publisher: LeadStalePublisher;
+  readonly logger?: LeadStaleLogger;
+  readonly now?: () => Date;
+  /** Days-since-firstSeenAt cutoff. Default 60. */
+  readonly thresholdDays?: number;
+  /** Statuses to consider stale. Default `["new","contacted"]`. */
+  readonly statuses?: readonly string[];
+  /** Hard cap per cron tick — defends against unbounded scans. */
+  readonly limit?: number;
+  /** Max outbox publishes at once. Default 10. */
+  readonly concurrency?: number;
+}
+
+export interface LeadStaleCronResult {
+  readonly scanned: number;
+  readonly emitted: number;
+  readonly skipped: number;
+}
+
+const DEFAULT_THRESHOLD_DAYS = 60;
+const DEFAULT_STATUSES = ["new", "contacted"] as const;
+const DEFAULT_LIMIT = 500;
+
+function isoDate(d: Date): string {
+  return d.toISOString().slice(0, 10);
+}
+
+export async function runLeadStaleTriggerCron(deps: LeadStaleCronDeps): Promise<LeadStaleCronResult> {
+  if (process.env.MARKETING_SEQUENCES_ENABLED !== "true") {
+    deps.logger?.info?.({ op: "lead.stale.skipped" }, "marketing sequences disabled by env");
+    return { scanned: 0, emitted: 0, skipped: 0 };
+  }
+  const now = (deps.now ?? (() => new Date()))();
+  const thresholdDays = deps.thresholdDays ?? DEFAULT_THRESHOLD_DAYS;
+  const statuses = deps.statuses ?? DEFAULT_STATUSES;
+  const limit = deps.limit ?? DEFAULT_LIMIT;
+  const thresholdDate = new Date(now.getTime() - thresholdDays * 24 * 60 * 60 * 1000);
+  const today = isoDate(now);
+
+  const leads = await deps.store.findStaleLeads({ thresholdDate, statuses, limit });
+  let emitted = 0;
+  let skipped = 0;
+  await mapWithConcurrency(leads, deps.concurrency ?? 10, async (lead) => {
+    const dispatchKey = `lead.stale_60d.${lead.id}.${today}`;
+    try {
+      await deps.publisher.publish({
+        type: "lead.stale_60d",
+        payload: {
+          leadId: lead.id,
+          email: lead.email,
+          recipientName: lead.firstName,
+          lead: { id: lead.id, email: lead.email, firstName: lead.firstName, status: lead.status, firstSeenAt: lead.firstSeenAt.toISOString() },
+          thresholdDays,
+        },
+        dispatchKey,
+      });
+      emitted += 1;
+    } catch (error) {
+      skipped += 1;
+      deps.logger?.error(
+        { op: "lead.stale.publish_failed", leadId: lead.id, error: error instanceof Error ? error.message : String(error) },
+        "stale-lead outbox publish failed"
+      );
+    }
+  });
+  deps.logger?.info?.({ op: "lead.stale.done", scanned: leads.length, emitted, skipped, thresholdDays }, "stale-lead trigger cron complete");
+  return { scanned: leads.length, emitted, skipped };
+}

hivegrid-cloud/apps/worker/src/jobs/legacy-event-receiver-bootstrap.ts

@@ -0,0 +1,97 @@
+/**
+ * Bootstrap helper for the legacy-event outbox receiver.
+ *
+ * Subscribes a BullMQ worker to the `outbox.notification` queue (which
+ * the dispatcher routes `billing.subscription.*` events into) and feeds
+ * each job through {@link runLegacyEventReceiverJob}.
+ *
+ * Gating:
+ *   - WORKER_QUEUE_GROUP === "general" (co-located with the outbox
+ *     dispatcher to keep DB connection pools concentrated)
+ *   - LEGACY_EVENT_RECEIVER_ENABLED !== "false" (escape hatch)
+ */
+
+import type { EnvRecord } from "@toolctrl/config";
+
+import {
+  LEGACY_EVENT_OUTBOX_QUEUE,
+  runLegacyEventReceiverJob,
+  type LegacyEventReceiverDependencies,
+  type LegacyEventReceiverLogger,
+  type LegacyEventReceiverStore,
+} from "./legacy-event-receiver.js";
+
+export type LegacyEventReceiverBootstrapEnv = EnvRecord & {
+  WORKER_QUEUE_GROUP?: string;
+  LEGACY_EVENT_RECEIVER_ENABLED?: string;
+  LEGACY_EVENT_RECEIVER_CONCURRENCY?: string;
+};
+
+export interface LegacyEventReceiverWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface LegacyEventReceiverBootstrapDeps {
+  readonly logger?: LegacyEventReceiverLogger;
+  readonly store?: LegacyEventReceiverStore;
+  readonly idempotency?: LegacyEventReceiverDependencies["idempotency"];
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number }
+  ) => LegacyEventReceiverWorkerHandle;
+}
+
+export type LegacyEventReceiverBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: LegacyEventReceiverWorkerHandle };
+
+const DEFAULT_CONCURRENCY = 5;
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export function bootstrapLegacyEventReceiver(
+  env: LegacyEventReceiverBootstrapEnv,
+  deps: LegacyEventReceiverBootstrapDeps = {}
+): LegacyEventReceiverBootstrapResult {
+  const logger = deps.logger;
+
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
+    return { status: "disabled", reason: "non-general worker group" };
+  }
+  if (env.LEGACY_EVENT_RECEIVER_ENABLED === "false") {
+    return { status: "disabled", reason: "LEGACY_EVENT_RECEIVER_ENABLED=false" };
+  }
+  if (!deps.store) {
+    return { status: "disabled", reason: "no store wired (worker host must supply createLegacyEvent)" };
+  }
+  if (!deps.createWorkerImpl) {
+    return { status: "disabled", reason: "no createWorkerImpl wired (worker host must supply BullMQ Worker factory)" };
+  }
+
+  const concurrency = parsePositiveInt(env.LEGACY_EVENT_RECEIVER_CONCURRENCY, DEFAULT_CONCURRENCY);
+  const store = deps.store;
+  const idempotency = deps.idempotency;
+
+  const processor = async (job: { data: unknown }): Promise<unknown> => {
+    const handlerDeps: LegacyEventReceiverDependencies = {
+      store,
+      ...(idempotency ? { idempotency } : {}),
+      ...(logger ? { logger } : {}),
+    };
+    return runLegacyEventReceiverJob(job.data, handlerDeps);
+  };
+
+  const handle = deps.createWorkerImpl(LEGACY_EVENT_OUTBOX_QUEUE, processor, { concurrency });
+
+  logger?.info?.(
+    { component: "legacy-event-receiver", queue: LEGACY_EVENT_OUTBOX_QUEUE, concurrency },
+    "legacy-event receiver worker started"
+  );
+  return { status: "started", handle };
+}

hivegrid-cloud/apps/worker/src/jobs/legacy-event-receiver.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  LEGACY_EVENT_OUTBOX_QUEUE,
+  LEGACY_EVENT_OUTBOX_TYPES,
+  LegacyEventReceiverJobError,
+  parseLegacyEventReceiverJob,
+  runLegacyEventReceiverJob,
+  type LegacyEventReceiverStore,
+} from "./legacy-event-receiver.js";
+
+function buildJob(overrides: Record<string, unknown> = {}): unknown {
+  return {
+    outboxEventId: "evt-1",
+    tenantId: "tenant-a",
+    eventType: "billing.subscription.ended",
+    payload: {
+      name: "subscription.ended",
+      tenantId: "tenant-a",
+      userId: null,
+      data: JSON.stringify({ tenant: { id: "tenant-a" }, product: { title: "Pro" } }),
+      description: "Acme subscription ended: Pro",
+    },
+    ...overrides,
+  };
+}
+
+function inMemoryStore(): LegacyEventReceiverStore & { calls: number; last: unknown } {
+  const state = { calls: 0, last: null as unknown };
+  return {
+    calls: 0,
+    last: null,
+    async create(data) {
+      state.calls += 1;
+      state.last = data;
+      this.calls = state.calls;
+      this.last = state.last;
+      return { id: `evt-row-${state.calls}` };
+    },
+  };
+}
+
+function inMemoryIdempotency() {
+  const claimed = new Set<string>();
+  const completed = new Map<string, unknown>();
+  return {
+    async claim(key: string): Promise<boolean> {
+      if (claimed.has(key)) return false;
+      claimed.add(key);
+      return true;
+    },
+    async complete(key: string, result: unknown): Promise<void> {
+      completed.set(key, result);
+    },
+    async get(key: string) {
+      if (completed.has(key)) {
+        return { state: "complete" as const, result: completed.get(key) };
+      }
+      if (claimed.has(key)) {
+        return { state: "pending" as const };
+      }
+      return null;
+    },
+    async forget(key: string): Promise<void> {
+      claimed.delete(key);
+    },
+    _claimed: claimed,
+  };
+}
+
+describe("legacy-event-receiver constants", () => {
+  it("declares the documented outbox queue and event types", () => {
+    expect(LEGACY_EVENT_OUTBOX_QUEUE).toBe("outbox.notification");
+    expect([...LEGACY_EVENT_OUTBOX_TYPES]).toEqual([
+      "billing.subscription.ended",
+      "billing.subscription.cancelled",
+    ]);
+  });
+});
+
+describe("parseLegacyEventReceiverJob", () => {
+  it("parses a well-formed envelope", () => {
+    const parsed = parseLegacyEventReceiverJob(buildJob());
+    expect(parsed.outboxEventId).toBe("evt-1");
+    expect(parsed.eventType).toBe("billing.subscription.ended");
+    expect(parsed.payload.name).toBe("subscription.ended");
+  });
+
+  it("rejects when outboxEventId is missing", () => {
+    expect(() => parseLegacyEventReceiverJob(buildJob({ outboxEventId: undefined }))).toThrow(
+      LegacyEventReceiverJobError
+    );
+  });
+
+  it("rejects unknown event types", () => {
+    expect(() => parseLegacyEventReceiverJob(buildJob({ eventType: "mail.welcome.send" }))).toThrow(
+      /unsupported eventType/
+    );
+  });
+
+  it("rejects when payload.name is empty", () => {
+    expect(() =>
+      parseLegacyEventReceiverJob(buildJob({ payload: { name: "", data: "{}", description: null, userId: null, tenantId: "tenant-a" } }))
+    ).toThrow(/payload\.name/);
+  });
+
+  it("rejects non-object data", () => {
+    expect(() => parseLegacyEventReceiverJob(null)).toThrow(LegacyEventReceiverJobError);
+    expect(() => parseLegacyEventReceiverJob("nope" as unknown)).toThrow(LegacyEventReceiverJobError);
+  });
+});
+
+describe("runLegacyEventReceiverJob", () => {
+  it("persists the event when no idempotency store is wired", async () => {
+    const store = inMemoryStore();
+    const result = await runLegacyEventReceiverJob(buildJob(), { store });
+    expect(result.status).toBe("persisted");
+    expect(store.calls).toBe(1);
+  });
+
+  it("guards against duplicate delivery via the idempotency store", async () => {
+    const store = inMemoryStore();
+    const idempotency = inMemoryIdempotency();
+    const first = await runLegacyEventReceiverJob(buildJob(), { store, idempotency });
+    const second = await runLegacyEventReceiverJob(buildJob(), { store, idempotency });
+    expect(first.status).toBe("persisted");
+    expect(second.status).toBe("skipped");
+    expect(store.calls).toBe(1);
+  });
+
+  it("does not ack a redelivery while the previous receiver claim is still pending", async () => {
+    const store = inMemoryStore();
+    const idempotency = inMemoryIdempotency();
+    await idempotency.claim("outbox:evt-1");
+
+    await expect(runLegacyEventReceiverJob(buildJob(), { store, idempotency })).rejects.toThrow(/already pending/);
+    expect(store.calls).toBe(0);
+  });
+
+  it("propagates store failures without poisoning the idempotency claim", async () => {
+    const store: LegacyEventReceiverStore = {
+      create: vi.fn(async () => {
+        throw new Error("DB went away");
+      }),
+    };
+    const idempotency = inMemoryIdempotency();
+    await expect(runLegacyEventReceiverJob(buildJob(), { store, idempotency })).rejects.toThrow(/DB went away/);
+    // Claim must be released so the retry can re-attempt.
+    expect(idempotency._claimed.size).toBe(0);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/legacy-event-receiver.ts

@@ -0,0 +1,194 @@
+/**
+ * Outbox receiver for "legacy" Event-table inserts.
+ *
+ * Phase 03 #42 — Stripe-webhook migration. The Stripe webhook used to call
+ * `createLegacyEvent` directly after updating a TenantSubscriptionProduct,
+ * meaning the DB write + the legacy-Event row were committed in two
+ * separate transactions; a worker crash between the two would silently
+ * drop the downstream notification.
+ *
+ * The webhook now wraps both writes in a single Prisma `$transaction` and
+ * the legacy-Event insert is published as an outbox event instead. This
+ * handler is the BullMQ consumer on the resulting `outbox.notification`
+ * queue: it receives `billing.subscription.ended` and
+ * `billing.subscription.cancelled` payloads and writes the legacy Event
+ * row.
+ *
+ * Idempotency: every job is wrapped with `withOutboxIdempotency` against
+ * the producer's outbox event id, so re-delivery (BullMQ retry, worker
+ * restart between ACK and downstream effect) is a no-op.
+ */
+
+import {
+  withOutboxIdempotency,
+  type OutboxReceiverIdempotencyStore,
+} from "@toolctrl/db/outbox-receiver";
+
+export const LEGACY_EVENT_OUTBOX_QUEUE = "outbox.notification" as const;
+
+/**
+ * The full set of event types this receiver knows how to write into the
+ * legacy `Event` table. New billing/subscription event types should be
+ * registered here.
+ */
+export const LEGACY_EVENT_OUTBOX_TYPES = [
+  "billing.subscription.ended",
+  "billing.subscription.cancelled",
+] as const;
+
+export type LegacyEventReceiverType = (typeof LEGACY_EVENT_OUTBOX_TYPES)[number];
+
+export interface LegacyEventReceiverLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface LegacyEventReceiverPayload {
+  readonly name: string;
+  readonly tenantId: string | null;
+  readonly userId: string | null;
+  readonly data: string;
+  readonly description: string | null;
+}
+
+export interface LegacyEventReceiverJobData {
+  readonly outboxEventId?: string;
+  readonly tenantId: string | null;
+  readonly eventType: string;
+  readonly payload: LegacyEventReceiverPayload;
+}
+
+export class LegacyEventReceiverJobError extends Error {
+  override readonly name = "LegacyEventReceiverJobError";
+  readonly code:
+    | "INVALID_PAYLOAD"
+    | "MISSING_OUTBOX_EVENT_ID"
+    | "UNKNOWN_EVENT_TYPE";
+  constructor(message: string, code: LegacyEventReceiverJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export interface LegacyEventReceiverStore {
+  /**
+   * Insert the Event row. Production wires `createLegacyEvent` from
+   * `@toolctrl/db/events.server`; tests pass an in-memory recorder.
+   */
+  create(data: LegacyEventReceiverPayload): Promise<{ id: string }>;
+}
+
+export interface LegacyEventReceiverDependencies {
+  readonly store: LegacyEventReceiverStore;
+  readonly idempotency?: OutboxReceiverIdempotencyStore & {
+    forget?(key: string): Promise<void>;
+  };
+  readonly logger?: LegacyEventReceiverLogger;
+}
+
+export interface LegacyEventReceiverResult {
+  readonly outboxEventId: string;
+  readonly status: "persisted" | "skipped";
+}
+
+function isLegacyType(value: string): value is LegacyEventReceiverType {
+  return (LEGACY_EVENT_OUTBOX_TYPES as readonly string[]).includes(value);
+}
+
+export function parseLegacyEventReceiverJob(data: unknown): LegacyEventReceiverJobData {
+  if (!data || typeof data !== "object") {
+    throw new LegacyEventReceiverJobError(
+      "legacy-event receiver job missing data envelope",
+      "INVALID_PAYLOAD"
+    );
+  }
+  const envelope = data as Record<string, unknown>;
+  const outboxEventId = typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined;
+  if (!outboxEventId) {
+    throw new LegacyEventReceiverJobError(
+      "legacy-event receiver job missing outboxEventId",
+      "MISSING_OUTBOX_EVENT_ID"
+    );
+  }
+  const eventType = typeof envelope.eventType === "string" ? envelope.eventType : "";
+  if (!isLegacyType(eventType)) {
+    throw new LegacyEventReceiverJobError(
+      `legacy-event receiver job has unsupported eventType ${JSON.stringify(eventType)}`,
+      "UNKNOWN_EVENT_TYPE"
+    );
+  }
+  const payload = envelope.payload;
+  if (!payload || typeof payload !== "object") {
+    throw new LegacyEventReceiverJobError(
+      "legacy-event receiver job missing payload",
+      "INVALID_PAYLOAD"
+    );
+  }
+  const inner = payload as Record<string, unknown>;
+  const name = typeof inner.name === "string" ? inner.name : "";
+  const data2 = typeof inner.data === "string" ? inner.data : "";
+  if (name.length === 0) {
+    throw new LegacyEventReceiverJobError(
+      "legacy-event receiver job missing payload.name",
+      "INVALID_PAYLOAD"
+    );
+  }
+  return {
+    outboxEventId,
+    tenantId: typeof envelope.tenantId === "string" ? envelope.tenantId : null,
+    eventType,
+    payload: {
+      name,
+      tenantId: typeof inner.tenantId === "string" ? inner.tenantId : null,
+      userId: typeof inner.userId === "string" ? inner.userId : null,
+      data: data2,
+      description: typeof inner.description === "string" ? inner.description : null,
+    },
+  };
+}
+
+export async function runLegacyEventReceiverJob(
+  data: unknown,
+  deps: LegacyEventReceiverDependencies
+): Promise<LegacyEventReceiverResult> {
+  const job = parseLegacyEventReceiverJob(data);
+
+  const persist = async (): Promise<{ status: "persisted" }> => {
+    await deps.store.create(job.payload);
+    deps.logger?.info?.(
+      {
+        component: "legacy-event-receiver",
+        outboxEventId: job.outboxEventId,
+        eventType: job.eventType,
+        tenantId: job.tenantId,
+        name: job.payload.name,
+      },
+      "legacy-event persisted from outbox"
+    );
+    return { status: "persisted" };
+  };
+
+  if (!deps.idempotency) {
+    await persist();
+    return { outboxEventId: job.outboxEventId!, status: "persisted" };
+  }
+
+  const outcome = await withOutboxIdempotency(
+    deps.idempotency,
+    job.outboxEventId!,
+    persist
+  );
+  if (outcome.status === "executed") {
+    return { outboxEventId: job.outboxEventId!, status: "persisted" };
+  }
+  deps.logger?.info?.(
+    {
+      component: "legacy-event-receiver",
+      outboxEventId: job.outboxEventId,
+      eventType: job.eventType,
+    },
+    "legacy-event skipped (idempotent re-delivery)"
+  );
+  return { outboxEventId: job.outboxEventId!, status: "skipped" };
+}

hivegrid-cloud/apps/worker/src/jobs/marketing-outbound.test.ts

@@ -0,0 +1,287 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  MARKETING_LEAD_EVENT_TYPES,
+  MARKETING_OUTBOUND_ACTIVE_SUBSCRIBER,
+  MarketingOutboundJobError,
+  parseMarketingOutboundJob,
+  runMarketingOutboundJob,
+  type MarketingIntegrationRow,
+} from "./marketing-outbound.js";
+
+const BASE_PAYLOAD = {
+  leadId: "lead_1",
+  email: "test@example.com",
+  firstName: "Test",
+  lastName: "User",
+  company: "Acme",
+  utm: { source: "google", campaign: "q2" },
+};
+
+function buildJobData(extra: Record<string, unknown> = {}) {
+  return {
+    outboxEventId: "ob_1",
+    tenantId: "tenant_1",
+    eventType: "marketing.lead.captured",
+    payload: BASE_PAYLOAD,
+    ...extra,
+  };
+}
+
+function buildStore(integrations: MarketingIntegrationRow[] = []) {
+  return {
+    listEnabledForEvent: vi.fn().mockResolvedValue(integrations),
+    recordDelivery: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+function mockFetch(status = 200): typeof fetch {
+  return vi.fn().mockResolvedValue(new Response("ok", { status })) as unknown as typeof fetch;
+}
+
+function buildIdempotency() {
+  const claimed = new Set<string>();
+  const completed = new Map<string, unknown>();
+  return {
+    _claimed: claimed,
+    _completed: completed,
+    claim: vi.fn(async (key: string) => {
+      if (claimed.has(key) || completed.has(key)) return false;
+      claimed.add(key);
+      return true;
+    }),
+    complete: vi.fn(async (key: string, result: unknown) => {
+      claimed.delete(key);
+      completed.set(key, result);
+    }),
+    get: vi.fn(async (key: string) => {
+      if (completed.has(key)) return { state: "complete" as const, result: completed.get(key) };
+      if (claimed.has(key)) return { state: "pending" as const };
+      return null;
+    }),
+    forget: vi.fn(async (key: string) => {
+      claimed.delete(key);
+      completed.delete(key);
+    }),
+  };
+}
+
+describe("constants", () => {
+  it("exposes handler-only status + recognised event types", () => {
+    expect(MARKETING_OUTBOUND_ACTIVE_SUBSCRIBER).toBe(false);
+    expect(MARKETING_LEAD_EVENT_TYPES).toContain("marketing.lead.captured");
+    expect(MARKETING_LEAD_EVENT_TYPES).toContain("marketing.lead.convertedtocustomer");
+  });
+});
+
+describe("parseMarketingOutboundJob", () => {
+  it("accepts a valid envelope", () => {
+    const job = parseMarketingOutboundJob(buildJobData());
+    expect(job.eventType).toBe("marketing.lead.captured");
+    expect(job.payload.leadId).toBe("lead_1");
+  });
+
+  it("rejects unknown event-type", () => {
+    expect(() => parseMarketingOutboundJob(buildJobData({ eventType: "marketing.lead.unknown" }))).toThrow(MarketingOutboundJobError);
+  });
+
+  it("rejects payload missing leadId/email", () => {
+    expect(() => parseMarketingOutboundJob(buildJobData({ payload: { email: "x@y" } }))).toThrow(MarketingOutboundJobError);
+  });
+});
+
+describe("runMarketingOutboundJob", () => {
+  const originalEnv = process.env.MARKETING_WEBHOOKS_ENABLED;
+  beforeEach(() => {
+    delete process.env.MARKETING_WEBHOOKS_ENABLED;
+  });
+  afterEach(() => {
+    if (originalEnv === undefined) delete process.env.MARKETING_WEBHOOKS_ENABLED;
+    else process.env.MARKETING_WEBHOOKS_ENABLED = originalEnv;
+  });
+
+  it("dispatches Slack integration successfully", async () => {
+    const fetchImpl = mockFetch(200);
+    const store = buildStore([
+      {
+        id: "int_1",
+        type: "slack",
+        name: "Sales channel",
+        config: { webhookUrl: "https://hooks.slack.com/services/X/Y/Z" },
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+    ]);
+    const res = await runMarketingOutboundJob(buildJobData(), { store, fetch: fetchImpl });
+    expect(res.dispatched).toBe(1);
+    expect(res.succeeded).toBe(1);
+    expect(res.failed).toBe(0);
+    expect(store.recordDelivery).toHaveBeenCalledWith("int_1", { ok: true, errorTag: undefined });
+  });
+
+  it("uses outbox receiver idempotency to skip duplicate webhook redelivery", async () => {
+    const fetchImpl = mockFetch(200);
+    const idempotency = buildIdempotency();
+    const store = buildStore([
+      {
+        id: "int_1",
+        type: "slack",
+        name: "Sales channel",
+        config: { webhookUrl: "https://hooks.slack.com/services/X/Y/Z" },
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+    ]);
+
+    await expect(runMarketingOutboundJob(buildJobData(), { store, fetch: fetchImpl, idempotency })).resolves.toMatchObject({
+      dispatched: 1,
+      succeeded: 1,
+      failed: 0,
+    });
+    await expect(runMarketingOutboundJob(buildJobData(), { store, fetch: fetchImpl, idempotency })).resolves.toMatchObject({
+      dispatched: 1,
+      succeeded: 1,
+      failed: 0,
+    });
+
+    expect(idempotency.claim).toHaveBeenCalledWith("outbox:ob_1");
+    expect(idempotency.complete).toHaveBeenCalledWith("outbox:ob_1", expect.objectContaining({ succeeded: 1 }));
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+    expect(store.recordDelivery).toHaveBeenCalledTimes(1);
+  });
+
+  it("releases outbox receiver idempotency when the store fails before delivery", async () => {
+    const idempotency = buildIdempotency();
+    const store = buildStore();
+    store.listEnabledForEvent.mockRejectedValueOnce(new Error("database unavailable"));
+
+    await expect(runMarketingOutboundJob(buildJobData(), { store, idempotency })).rejects.toThrow("database unavailable");
+
+    expect(idempotency.claim).toHaveBeenCalledWith("outbox:ob_1");
+    expect(idempotency.forget).toHaveBeenCalledWith("outbox:ob_1");
+    expect(idempotency._claimed.size).toBe(0);
+    await expect(runMarketingOutboundJob(buildJobData(), { store, idempotency })).resolves.toMatchObject({ dispatched: 0 });
+  });
+
+  it("propagates pending outbox receiver idempotency so BullMQ can retry", async () => {
+    const idempotency = buildIdempotency();
+    await idempotency.claim("outbox:ob_1");
+    const store = buildStore();
+
+    await expect(runMarketingOutboundJob(buildJobData(), { store, idempotency })).rejects.toMatchObject({
+      name: "OutboxReceiverPendingError",
+      key: "outbox:ob_1",
+    });
+    expect(store.listEnabledForEvent).not.toHaveBeenCalled();
+  });
+
+  it("runs without receiver idempotency when outboxEventId is absent", async () => {
+    const fetchImpl = mockFetch(200);
+    const idempotency = buildIdempotency();
+    const store = buildStore([
+      {
+        id: "int_1",
+        type: "slack",
+        name: "Sales channel",
+        config: { webhookUrl: "https://hooks.slack.com/services/X/Y/Z" },
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+    ]);
+
+    await expect(runMarketingOutboundJob(buildJobData({ outboxEventId: undefined }), { store, fetch: fetchImpl, idempotency })).resolves.toMatchObject({
+      dispatched: 1,
+      succeeded: 1,
+      failed: 0,
+    });
+    expect(idempotency.claim).not.toHaveBeenCalled();
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+  });
+
+  it("isolates failures across integrations", async () => {
+    let calls = 0;
+    const fetchImpl = vi.fn().mockImplementation(async () => {
+      calls += 1;
+      return new Response("err", { status: calls === 1 ? 500 : 200 });
+    }) as unknown as typeof fetch;
+    const store = buildStore([
+      {
+        id: "int_slack",
+        type: "slack",
+        name: "Slack",
+        config: { webhookUrl: "https://hooks.slack.com/services/X" },
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+      {
+        id: "int_discord",
+        type: "discord",
+        name: "Discord",
+        config: { webhookUrl: "https://discord.com/api/webhooks/x/y" },
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+    ]);
+    const res = await runMarketingOutboundJob(buildJobData(), { store, fetch: fetchImpl });
+    expect(res.dispatched).toBe(2);
+    expect(res.succeeded).toBe(1);
+    expect(res.failed).toBe(1);
+  });
+
+  it("records unknown_adapter_type for missing adapter", async () => {
+    const store = buildStore([
+      {
+        id: "int_x",
+        type: "nope",
+        name: "Bad",
+        config: {},
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+    ]);
+    const res = await runMarketingOutboundJob(buildJobData(), { store });
+    expect(res.failed).toBe(1);
+    expect(store.recordDelivery).toHaveBeenCalledWith("int_x", expect.objectContaining({ errorTag: "unknown_adapter_type" }));
+  });
+
+  it("short-circuits when MARKETING_WEBHOOKS_ENABLED=false", async () => {
+    process.env.MARKETING_WEBHOOKS_ENABLED = "false";
+    const store = buildStore([
+      {
+        id: "int_1",
+        type: "slack",
+        name: "Slack",
+        config: { webhookUrl: "https://hooks.slack.com/services/X" },
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+    ]);
+    const res = await runMarketingOutboundJob(buildJobData(), { store });
+    expect(res.dispatched).toBe(0);
+    expect(store.listEnabledForEvent).not.toHaveBeenCalled();
+  });
+
+  it("no-ops when zero integrations subscribe", async () => {
+    const store = buildStore([]);
+    const res = await runMarketingOutboundJob(buildJobData(), { store });
+    expect(res.dispatched).toBe(0);
+    expect(res.succeeded).toBe(0);
+  });
+
+  it("catches adapter parseConfig errors and records adapter_error", async () => {
+    const fetchImpl = mockFetch();
+    const store = buildStore([
+      {
+        id: "int_bad",
+        type: "slack",
+        // Missing webhookUrl => parseConfig throws
+        config: {},
+        name: "Bad Slack",
+        enabled: true,
+        triggerEvents: ["marketing.lead.captured"],
+      },
+    ]);
+    const res = await runMarketingOutboundJob(buildJobData(), { store, fetch: fetchImpl });
+    expect(res.failed).toBe(1);
+    expect(store.recordDelivery).toHaveBeenCalledWith("int_bad", expect.objectContaining({ ok: false }));
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/marketing-outbound.ts

@@ -0,0 +1,211 @@
+/**
+ * Marketing-funnel Phase 2 outbound webhook handler foundation (Phase 05 #79).
+ *
+ * This module is intentionally a pure handler: it does not claim or
+ * bootstrap the `outbox.marketing` BullMQ queue. That queue is actively
+ * owned by the email-sequence trigger subscriber. A future marketing
+ * integration subscriber must introduce its own bootstrap module and
+ * ownership tests before wiring this handler to a queue.
+ *
+ * The handler looks up every enabled `MarketingIntegration` whose
+ * `triggerEvents` contains the event-type, decrypts its config, and
+ * dispatches the payload through the matching adapter (Slack, Discord,
+ * Mailchimp, HubSpot, Brevo, custom webhook).
+ *
+ * Failures per integration are isolated — one bad integration cannot
+ * starve the others. Final-failures persist `MarketingIntegration.lastError`
+ * and emit `marketing.integration.failed` audit events; the BullMQ
+ * retry machinery handles transient 5xx/network failures by re-queueing
+ * the outbox row.
+ */
+
+import { resolveAdapter, type MarketingLeadEventType, type MarketingLeadPayload } from "@toolctrl/marketing-outbound";
+import { withOutboxIdempotency, type OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
+
+export const MARKETING_OUTBOUND_ACTIVE_SUBSCRIBER = false as const;
+
+/** Set of event-type strings the handler recognises. */
+export const MARKETING_LEAD_EVENT_TYPES: readonly MarketingLeadEventType[] = [
+  "marketing.lead.captured",
+  "marketing.lead.qualified",
+  "marketing.lead.convertedtotrial",
+  "marketing.lead.convertedtocustomer",
+];
+
+export interface MarketingIntegrationRow {
+  id: string;
+  type: string;
+  name: string;
+  /** Decrypted config blob. The DB row stores `configJson` encrypted; the
+   *  store adapter is responsible for decryption before this handler
+   *  sees the values. */
+  config: unknown;
+  enabled: boolean;
+  triggerEvents: string[];
+}
+
+export interface MarketingOutboundJobData {
+  outboxEventId?: string;
+  tenantId?: string | null;
+  eventType: string;
+  payload: MarketingLeadPayload;
+}
+
+export interface MarketingOutboundStore {
+  /** Fetch all enabled integrations whose triggerEvents contain `eventType`. */
+  listEnabledForEvent(eventType: string): Promise<MarketingIntegrationRow[]>;
+  /** Update lastDeliveredAt / lastError diagnostics on the integration row. */
+  recordDelivery(integrationId: string, result: { ok: boolean; errorTag?: string }): Promise<void>;
+}
+
+export interface MarketingOutboundLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface MarketingOutboundDependencies {
+  readonly store: MarketingOutboundStore;
+  readonly logger?: MarketingOutboundLogger;
+  readonly idempotency?: OutboxReceiverIdempotencyStore & { forget?(key: string): Promise<void> };
+  readonly fetch?: typeof fetch;
+  /** Override the default 10s adapter timeout (test seam). */
+  readonly timeoutMs?: number;
+}
+
+export interface MarketingOutboundJobResult {
+  readonly eventType: string;
+  readonly dispatched: number;
+  readonly succeeded: number;
+  readonly failed: number;
+}
+
+export class MarketingOutboundJobError extends Error {
+  override readonly name = "MarketingOutboundJobError";
+  readonly code: "INVALID_PAYLOAD" | "UNKNOWN_EVENT_TYPE";
+  constructor(message: string, code: MarketingOutboundJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export function parseMarketingOutboundJob(data: unknown): MarketingOutboundJobData {
+  if (!data || typeof data !== "object") {
+    throw new MarketingOutboundJobError("marketing-outbound job missing data envelope", "INVALID_PAYLOAD");
+  }
+  const envelope = data as Record<string, unknown>;
+  const eventType = typeof envelope.eventType === "string" ? envelope.eventType : "";
+  if (!MARKETING_LEAD_EVENT_TYPES.includes(eventType as MarketingLeadEventType)) {
+    throw new MarketingOutboundJobError(`unknown marketing event-type: ${eventType}`, "UNKNOWN_EVENT_TYPE");
+  }
+  const payload = envelope.payload;
+  if (!payload || typeof payload !== "object") {
+    throw new MarketingOutboundJobError("marketing-outbound job missing payload", "INVALID_PAYLOAD");
+  }
+  const p = payload as Record<string, unknown>;
+  if (typeof p.leadId !== "string" || typeof p.email !== "string") {
+    throw new MarketingOutboundJobError("marketing-outbound payload missing leadId/email", "INVALID_PAYLOAD");
+  }
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId: typeof envelope.tenantId === "string" ? envelope.tenantId : null,
+    eventType,
+    payload: payload as MarketingLeadPayload,
+  };
+}
+
+export async function runMarketingOutboundJob(
+  data: unknown,
+  deps: MarketingOutboundDependencies
+): Promise<MarketingOutboundJobResult> {
+  // Allow operator to short-circuit all outbound webhooks (rollback) by
+  // setting MARKETING_WEBHOOKS_ENABLED=false. Outbox events still
+  // accumulate so the queue can be re-played after a fix.
+  if (process.env.MARKETING_WEBHOOKS_ENABLED === "false") {
+    deps.logger?.info?.({ op: "marketing-outbound.skipped" }, "marketing webhooks disabled by env");
+    return { eventType: "(disabled)", dispatched: 0, succeeded: 0, failed: 0 };
+  }
+  const job = parseMarketingOutboundJob(data);
+  if (deps.idempotency && job.outboxEventId) {
+    const outcome = await withOutboxIdempotency(deps.idempotency, job.outboxEventId, () => dispatchMarketingOutboundJob(job, deps));
+    if (outcome.status === "executed" && isMarketingOutboundJobResult(outcome.result)) {
+      return outcome.result;
+    }
+    deps.logger?.info?.(
+      { op: "marketing-outbound.idempotent_skip", outboxEventId: job.outboxEventId, eventType: job.eventType },
+      "marketing outbound skipped idempotent redelivery"
+    );
+    return isMarketingOutboundJobResult(outcome.result)
+      ? outcome.result
+      : { eventType: job.eventType, dispatched: 0, succeeded: 0, failed: 0 };
+  }
+
+  return dispatchMarketingOutboundJob(job, deps);
+}
+
+async function dispatchMarketingOutboundJob(
+  job: MarketingOutboundJobData,
+  deps: MarketingOutboundDependencies
+): Promise<MarketingOutboundJobResult> {
+  const integrations = await deps.store.listEnabledForEvent(job.eventType);
+  let succeeded = 0;
+  let failed = 0;
+  for (const integration of integrations) {
+    const adapter = resolveAdapter(integration.type);
+    if (!adapter) {
+      failed += 1;
+      await deps.store.recordDelivery(integration.id, { ok: false, errorTag: "unknown_adapter_type" });
+      deps.logger?.warn?.(
+        { op: "marketing-outbound.unknown_adapter", integrationId: integration.id, type: integration.type },
+        "no adapter registered for integration type"
+      );
+      continue;
+    }
+    let result;
+    try {
+      const config = adapter.parseConfig(integration.config);
+      result = await adapter.send({
+        eventType: job.eventType as MarketingLeadEventType,
+        payload: job.payload,
+        config,
+        ctx: { fetch: deps.fetch, timeoutMs: deps.timeoutMs },
+      });
+    } catch (err) {
+      result = {
+        ok: false,
+        errorTag: err instanceof Error ? `${err.name}` : "adapter_error",
+      } as const;
+    }
+    if (result.ok) succeeded += 1;
+    else failed += 1;
+    await deps.store.recordDelivery(integration.id, { ok: result.ok, errorTag: result.errorTag });
+    deps.logger?.info?.(
+      {
+        op: result.ok ? "marketing-outbound.delivered" : "marketing-outbound.failed",
+        integrationId: integration.id,
+        type: integration.type,
+        eventType: job.eventType,
+        leadId: job.payload.leadId,
+        errorTag: result.errorTag,
+      },
+      result.ok ? "marketing webhook delivered" : "marketing webhook delivery failed"
+    );
+  }
+  return {
+    eventType: job.eventType,
+    dispatched: integrations.length,
+    succeeded,
+    failed,
+  };
+}
+
+function isMarketingOutboundJobResult(value: unknown): value is MarketingOutboundJobResult {
+  if (!value || typeof value !== "object") return false;
+  const record = value as Record<string, unknown>;
+  return (
+    typeof record.eventType === "string" &&
+    typeof record.dispatched === "number" &&
+    typeof record.succeeded === "number" &&
+    typeof record.failed === "number"
+  );
+}

hivegrid-cloud/apps/worker/src/jobs/outbox-done-retention-cron.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  OUTBOX_DONE_RETENTION_DEFAULT_BATCH_SIZE,
+  OUTBOX_DONE_RETENTION_DEFAULT_HOURS,
+  OUTBOX_DONE_RETENTION_EVERY_MS,
+  OUTBOX_DONE_RETENTION_JOB_ID,
+  registerOutboxDoneRetentionCron,
+  resolveOutboxDoneRetentionCronConfig,
+} from "./outbox-done-retention-cron.js";
+
+describe("outbox-done-retention-cron", () => {
+  it("resolves retention defaults and clamps invalid env", () => {
+    expect(resolveOutboxDoneRetentionCronConfig({})).toEqual({
+      olderThanHours: OUTBOX_DONE_RETENTION_DEFAULT_HOURS,
+      take: OUTBOX_DONE_RETENTION_DEFAULT_BATCH_SIZE,
+    });
+    expect(
+      resolveOutboxDoneRetentionCronConfig({
+        OUTBOX_DONE_RETENTION_HOURS: "168.9",
+        OUTBOX_DONE_RETENTION_BATCH_SIZE: "2500.5",
+      } as NodeJS.ProcessEnv)
+    ).toEqual({ olderThanHours: 168, take: 2500 });
+    expect(
+      resolveOutboxDoneRetentionCronConfig({
+        OUTBOX_DONE_RETENTION_HOURS: "-1",
+        OUTBOX_DONE_RETENTION_BATCH_SIZE: "nope",
+      } as NodeJS.ProcessEnv)
+    ).toEqual({
+      olderThanHours: OUTBOX_DONE_RETENTION_DEFAULT_HOURS,
+      take: OUTBOX_DONE_RETENTION_DEFAULT_BATCH_SIZE,
+    });
+  });
+
+  it("registers a 24h scheduler with cleanup.run payload", async () => {
+    const upsert = vi.fn().mockResolvedValue(undefined);
+    await registerOutboxDoneRetentionCron(
+      { upsertJobScheduler: upsert },
+      {
+        OUTBOX_DONE_RETENTION_HOURS: "720",
+        OUTBOX_DONE_RETENTION_BATCH_SIZE: "1000",
+      } as NodeJS.ProcessEnv,
+    );
+
+    expect(upsert).toHaveBeenCalledTimes(1);
+    expect(upsert.mock.calls[0][0]).toBe(OUTBOX_DONE_RETENTION_JOB_ID);
+    expect(upsert.mock.calls[0][1]).toEqual({ every: OUTBOX_DONE_RETENTION_EVERY_MS });
+    expect(upsert.mock.calls[0][2]).toEqual({
+      name: OUTBOX_DONE_RETENTION_JOB_ID,
+      data: {
+        kind: "cleanup.run",
+        task: "outbox-done-retention",
+        olderThanHours: 720,
+        take: 1000,
+      },
+    });
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/outbox-done-retention-cron.ts

@@ -0,0 +1,47 @@
+export const OUTBOX_DONE_RETENTION_JOB_ID = "outbox-done-retention" as const;
+export const OUTBOX_DONE_RETENTION_EVERY_MS = 24 * 60 * 60 * 1000;
+export const OUTBOX_DONE_RETENTION_DEFAULT_HOURS = 720;
+export const OUTBOX_DONE_RETENTION_DEFAULT_BATCH_SIZE = 1000;
+
+export function resolveOutboxDoneRetentionCronConfig(env: NodeJS.ProcessEnv = process.env): {
+  olderThanHours: number;
+  take: number;
+} {
+  return {
+    olderThanHours: parsePositiveInteger(env.OUTBOX_DONE_RETENTION_HOURS, OUTBOX_DONE_RETENTION_DEFAULT_HOURS),
+    take: parsePositiveInteger(env.OUTBOX_DONE_RETENTION_BATCH_SIZE, OUTBOX_DONE_RETENTION_DEFAULT_BATCH_SIZE),
+  };
+}
+
+export async function registerOutboxDoneRetentionCron(
+  queue: {
+    upsertJobScheduler: (
+      schedulerId: string,
+      options: { every: number },
+      template: { name: string; data?: Record<string, unknown> },
+    ) => Promise<unknown>;
+  },
+  env: NodeJS.ProcessEnv = process.env,
+): Promise<void> {
+  const config = resolveOutboxDoneRetentionCronConfig(env);
+  await queue.upsertJobScheduler(
+    OUTBOX_DONE_RETENTION_JOB_ID,
+    { every: OUTBOX_DONE_RETENTION_EVERY_MS },
+    {
+      name: OUTBOX_DONE_RETENTION_JOB_ID,
+      data: {
+        kind: "cleanup.run",
+        task: OUTBOX_DONE_RETENTION_JOB_ID,
+        olderThanHours: config.olderThanHours,
+        take: config.take,
+      },
+    },
+  );
+}
+
+function parsePositiveInteger(value: string | undefined, fallback: number): number {
+  if (!value) return fallback;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return Math.floor(parsed);
+}

hivegrid-cloud/apps/worker/src/jobs/plugin-audit-retention.test.ts

@@ -0,0 +1,337 @@
+import { describe, expect, test, vi } from "vitest";
+
+import {
+  PLUGIN_AUDIT_RETENTION_BATCH_SIZE_DEFAULT,
+  PLUGIN_AUDIT_RETENTION_CRON_DEFAULT,
+  PLUGIN_AUDIT_RETENTION_DAYS_DEFAULT,
+  PLUGIN_AUDIT_RETENTION_JOB_ID,
+  PLUGIN_AUDIT_RETENTION_MAX_BATCHES_DEFAULT,
+  PLUGIN_AUDIT_RETENTION_QUEUE,
+  registerPluginAuditRetentionCron,
+  resolvePluginAuditRetentionConfig,
+  runPluginAuditRetention,
+} from "./plugin-audit-retention.js";
+
+const MS_PER_DAY = 86_400_000;
+
+type SilentLogger = {
+  info: ReturnType<typeof vi.fn>;
+  warn: ReturnType<typeof vi.fn>;
+  error: ReturnType<typeof vi.fn>;
+};
+
+function makeLogger(): SilentLogger {
+  return { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+}
+
+describe("queue + scheduler constants", () => {
+  test("exports stable queue and job-id names", () => {
+    expect(PLUGIN_AUDIT_RETENTION_QUEUE).toBe("plugin-audit-retention");
+    expect(PLUGIN_AUDIT_RETENTION_JOB_ID).toBe("plugin-audit-retention");
+    expect(PLUGIN_AUDIT_RETENTION_CRON_DEFAULT).toBe("0 3 * * *");
+  });
+
+  test("exports documented defaults", () => {
+    expect(PLUGIN_AUDIT_RETENTION_DAYS_DEFAULT).toBe(90);
+    expect(PLUGIN_AUDIT_RETENTION_BATCH_SIZE_DEFAULT).toBe(1000);
+    expect(PLUGIN_AUDIT_RETENTION_MAX_BATCHES_DEFAULT).toBe(100);
+  });
+});
+
+describe("resolvePluginAuditRetentionConfig", () => {
+  test("returns defaults when env is empty", () => {
+    expect(resolvePluginAuditRetentionConfig({})).toEqual({
+      retentionDays: PLUGIN_AUDIT_RETENTION_DAYS_DEFAULT,
+      batchSize: PLUGIN_AUDIT_RETENTION_BATCH_SIZE_DEFAULT,
+      maxBatches: PLUGIN_AUDIT_RETENTION_MAX_BATCHES_DEFAULT,
+    });
+  });
+
+  test("returns defaults when env is undefined", () => {
+    expect(resolvePluginAuditRetentionConfig(undefined)).toEqual({
+      retentionDays: PLUGIN_AUDIT_RETENTION_DAYS_DEFAULT,
+      batchSize: PLUGIN_AUDIT_RETENTION_BATCH_SIZE_DEFAULT,
+      maxBatches: PLUGIN_AUDIT_RETENTION_MAX_BATCHES_DEFAULT,
+    });
+  });
+
+  test("parses overrides from env", () => {
+    expect(
+      resolvePluginAuditRetentionConfig({
+        PLUGIN_AUDIT_RETENTION_DAYS: "30",
+        PLUGIN_AUDIT_RETENTION_BATCH_SIZE: "250",
+        PLUGIN_AUDIT_RETENTION_MAX_BATCHES: "8",
+      })
+    ).toEqual({ retentionDays: 30, batchSize: 250, maxBatches: 8 });
+  });
+
+  test("falls back to defaults on garbage / non-positive values", () => {
+    expect(
+      resolvePluginAuditRetentionConfig({
+        PLUGIN_AUDIT_RETENTION_DAYS: "not-a-number",
+        PLUGIN_AUDIT_RETENTION_BATCH_SIZE: "0",
+        PLUGIN_AUDIT_RETENTION_MAX_BATCHES: "-5",
+      })
+    ).toEqual({
+      retentionDays: PLUGIN_AUDIT_RETENTION_DAYS_DEFAULT,
+      batchSize: PLUGIN_AUDIT_RETENTION_BATCH_SIZE_DEFAULT,
+      maxBatches: PLUGIN_AUDIT_RETENTION_MAX_BATCHES_DEFAULT,
+    });
+  });
+});
+
+describe("runPluginAuditRetention — cutoff math", () => {
+  test("cutoff is exactly retentionDays before `now`", async () => {
+    const now = new Date("2026-05-19T12:00:00.000Z");
+    const cutoffs: Date[] = [];
+    const deleteBatch = vi.fn(async (cutoff: Date) => {
+      cutoffs.push(cutoff);
+      return 0;
+    });
+
+    const result = await runPluginAuditRetention({
+      now,
+      config: { retentionDays: 90, batchSize: 1000, maxBatches: 10 },
+      deleteBatch,
+    });
+
+    expect(cutoffs).toHaveLength(1);
+    const expected = new Date(now.getTime() - 90 * MS_PER_DAY);
+    expect(cutoffs[0]!.toISOString()).toBe(expected.toISOString());
+    expect(result.cutoffIso).toBe(expected.toISOString());
+    // 2026-05-19 minus 90 days = 2026-02-18
+    expect(result.cutoffIso).toBe("2026-02-18T12:00:00.000Z");
+  });
+
+  test("cutoff respects a non-default retentionDays value", async () => {
+    const now = new Date("2026-05-19T00:00:00.000Z");
+    const deleteBatch = vi.fn(async (cutoff: Date, batchSize: number) => {
+      void cutoff;
+      void batchSize;
+      return 0;
+    });
+    const result = await runPluginAuditRetention({
+      now,
+      config: { retentionDays: 7, batchSize: 100, maxBatches: 5 },
+      deleteBatch,
+    });
+    expect(result.cutoffIso).toBe("2026-05-12T00:00:00.000Z");
+  });
+});
+
+describe("runPluginAuditRetention — batching", () => {
+  test("loops until deleteBatch returns 0", async () => {
+    // Simulate: 3 full batches of 1000, then 250, then 0.
+    const sequence = [1000, 1000, 1000, 250, 0];
+    const deleteBatch = vi.fn(async () => sequence.shift() ?? 0);
+
+    const result = await runPluginAuditRetention({
+      now: new Date("2026-05-19T03:00:00.000Z"),
+      config: { retentionDays: 90, batchSize: 1000, maxBatches: 10 },
+      deleteBatch,
+    });
+
+    expect(deleteBatch).toHaveBeenCalledTimes(5);
+    expect(result.deletedRows).toBe(3250);
+    expect(result.batches).toBe(5);
+  });
+
+  test("invokes deleteBatch with the configured batchSize", async () => {
+    const deleteBatch = vi.fn(async (_cutoff: Date, batchSize: number) => (batchSize === 500 ? 0 : 0));
+    await runPluginAuditRetention({
+      now: new Date("2026-05-19T03:00:00.000Z"),
+      config: { retentionDays: 90, batchSize: 500, maxBatches: 10 },
+      deleteBatch,
+    });
+    expect(deleteBatch).toHaveBeenCalledWith(expect.any(Date), 500);
+  });
+});
+
+describe("runPluginAuditRetention — max-batches guard", () => {
+  test("stops at maxBatches even if more rows remain", async () => {
+    // Every call returns a full batch — without the cap this would be
+    // an infinite loop. With maxBatches=3 we must stop after 3 calls.
+    const deleteBatch = vi.fn(async () => 1000);
+    const logger = makeLogger();
+
+    const result = await runPluginAuditRetention({
+      now: new Date("2026-05-19T03:00:00.000Z"),
+      config: { retentionDays: 90, batchSize: 1000, maxBatches: 3 },
+      deleteBatch,
+      logger,
+    });
+
+    expect(deleteBatch).toHaveBeenCalledTimes(3);
+    expect(result.batches).toBe(3);
+    expect(result.deletedRows).toBe(3000);
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.objectContaining({ batches: 3, deletedRows: 3000, maxBatches: 3 }),
+      "plugin-audit-retention.max-batches-reached"
+    );
+  });
+});
+
+describe("runPluginAuditRetention — empty / soft-fail", () => {
+  test("returns { deletedRows: 0, batches: 1 } when no rows are eligible", async () => {
+    const deleteBatch = vi.fn(async (cutoff: Date, batchSize: number) => {
+      void cutoff;
+      void batchSize;
+      return 0;
+    });
+    const result = await runPluginAuditRetention({
+      now: new Date("2026-05-19T03:00:00.000Z"),
+      config: { retentionDays: 90, batchSize: 1000, maxBatches: 100 },
+      deleteBatch,
+    });
+    expect(deleteBatch).toHaveBeenCalledTimes(1);
+    expect(result.deletedRows).toBe(0);
+    expect(result.batches).toBe(1);
+  });
+
+  test("soft-fails on DB error: logs, terminates, returns accumulated count", async () => {
+    let call = 0;
+    const deleteBatch = vi.fn(async () => {
+      call += 1;
+      if (call === 1) return 1000;
+      if (call === 2) return 500;
+      throw new Error("connection reset");
+    });
+    const logger = makeLogger();
+
+    const result = await runPluginAuditRetention({
+      now: new Date("2026-05-19T03:00:00.000Z"),
+      config: { retentionDays: 90, batchSize: 1000, maxBatches: 100 },
+      deleteBatch,
+      logger,
+    });
+
+    expect(deleteBatch).toHaveBeenCalledTimes(3);
+    expect(result.deletedRows).toBe(1500);
+    expect(result.batches).toBe(2);
+    expect(logger.error).toHaveBeenCalledWith(
+      expect.objectContaining({ deletedRows: 1500, err: "connection reset" }),
+      "plugin-audit-retention.batch-failed"
+    );
+  });
+
+  test("reads config from env when `config` is omitted", async () => {
+    const deleteBatch = vi.fn(async (cutoff: Date, batchSize: number) => {
+      void cutoff;
+      void batchSize;
+      return 0;
+    });
+    const sizesSeen: number[] = [];
+    await runPluginAuditRetention({
+      now: new Date("2026-05-19T03:00:00.000Z"),
+      env: {
+        PLUGIN_AUDIT_RETENTION_DAYS: "30",
+        PLUGIN_AUDIT_RETENTION_BATCH_SIZE: "42",
+        PLUGIN_AUDIT_RETENTION_MAX_BATCHES: "3",
+      },
+      deleteBatch: async (cutoff, batchSize) => {
+        sizesSeen.push(batchSize);
+        return deleteBatch(cutoff, batchSize);
+      },
+    });
+    expect(sizesSeen).toEqual([42]);
+  });
+});
+
+describe("registerPluginAuditRetentionCron", () => {
+  test("upserts the job-scheduler with default cron pattern", async () => {
+    const upsertJobScheduler = vi.fn(async () => ({}));
+    const queue = { upsertJobScheduler };
+
+    const result = await registerPluginAuditRetentionCron(queue, {});
+
+    expect(upsertJobScheduler).toHaveBeenCalledTimes(1);
+    expect(upsertJobScheduler).toHaveBeenCalledWith(
+      PLUGIN_AUDIT_RETENTION_JOB_ID,
+      { pattern: PLUGIN_AUDIT_RETENTION_CRON_DEFAULT },
+      { name: PLUGIN_AUDIT_RETENTION_JOB_ID, data: { task: PLUGIN_AUDIT_RETENTION_JOB_ID } }
+    );
+    expect(result.pattern).toBe(PLUGIN_AUDIT_RETENTION_CRON_DEFAULT);
+  });
+
+  test("honours PLUGIN_AUDIT_RETENTION_CRON env override", async () => {
+    const upsertJobScheduler = vi.fn(async () => ({}));
+    const queue = { upsertJobScheduler };
+
+    const result = await registerPluginAuditRetentionCron(queue, {
+      PLUGIN_AUDIT_RETENTION_CRON: "15 4 * * *",
+    });
+
+    expect(upsertJobScheduler).toHaveBeenCalledWith(
+      PLUGIN_AUDIT_RETENTION_JOB_ID,
+      { pattern: "15 4 * * *" },
+      expect.any(Object)
+    );
+    expect(result.pattern).toBe("15 4 * * *");
+  });
+});
+
+// F-020 — surviving-rows count + size warning.
+describe("runPluginAuditRetention — post-sweep size check (F-020)", () => {
+  test("emits a warn line when surviving rows exceed the threshold", async () => {
+    const logger = makeLogger();
+    const deleteBatch = vi.fn(async () => 0);
+    const countRows = vi.fn(async () => 12_345_678);
+    await runPluginAuditRetention({
+      config: { retentionDays: 90, batchSize: 100, maxBatches: 1 },
+      deleteBatch,
+      countRows,
+      warnRowCountThreshold: 10_000_000,
+      logger,
+    });
+    const warns = logger.warn.mock.calls.map(([, msg]) => msg);
+    expect(warns).toContain("plugin-audit-retention.table-size-warning");
+  });
+
+  test("emits an info line when surviving rows are below the threshold", async () => {
+    const logger = makeLogger();
+    const deleteBatch = vi.fn(async () => 0);
+    const countRows = vi.fn(async () => 100);
+    await runPluginAuditRetention({
+      config: { retentionDays: 90, batchSize: 100, maxBatches: 1 },
+      deleteBatch,
+      countRows,
+      warnRowCountThreshold: 10_000_000,
+      logger,
+    });
+    const infos = logger.info.mock.calls.map(([, msg]) => msg);
+    expect(infos).toContain("plugin-audit-retention.table-size");
+  });
+
+  test("a countRows failure does not crash the run", async () => {
+    const logger = makeLogger();
+    const deleteBatch = vi.fn(async () => 0);
+    const countRows = vi.fn(async () => {
+      throw new Error("DB connection lost");
+    });
+    const result = await runPluginAuditRetention({
+      config: { retentionDays: 90, batchSize: 100, maxBatches: 1 },
+      deleteBatch,
+      countRows,
+      logger,
+    });
+    expect(result.deletedRows).toBe(0);
+    const warns = logger.warn.mock.calls.map(([, msg]) => msg);
+    expect(warns).toContain("plugin-audit-retention.count-failed");
+  });
+
+  test("countRows is skipped silently when not supplied", async () => {
+    const logger = makeLogger();
+    const deleteBatch = vi.fn(async () => 0);
+    await runPluginAuditRetention({
+      config: { retentionDays: 90, batchSize: 100, maxBatches: 1 },
+      deleteBatch,
+      logger,
+    });
+    const allMessages = [
+      ...logger.info.mock.calls.map(([, msg]) => msg),
+      ...logger.warn.mock.calls.map(([, msg]) => msg),
+    ];
+    expect(allMessages).not.toContain("plugin-audit-retention.table-size");
+    expect(allMessages).not.toContain("plugin-audit-retention.table-size-warning");
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-audit-retention.ts

@@ -0,0 +1,228 @@
+/**
+ * Phase 06 #23 — PluginCapabilityUse audit retention nightly job.
+ *
+ * Walks the `PluginCapabilityUse` table once per day and deletes rows
+ * whose `timestamp` column is older than `PLUGIN_AUDIT_RETENTION_DAYS`
+ * (default 90). DSGVO + storage-bloat concern: the table is
+ * append-only at runtime, so without this sweep it grows unbounded.
+ *
+ * Deletion is batched (default 1000 rows per iteration) and capped at
+ * a max number of iterations per run (default 100) so the cron never
+ * locks the table for an arbitrarily long window. Anything that does
+ * not get cleaned up tonight will be picked up tomorrow.
+ *
+ * Host-agnostic on purpose: the worker bootstrap (apps/worker/src/
+ * index.ts) supplies the seams (`deleteBatch`, `logger`) so this file
+ * is unit-testable without spinning up Prisma or BullMQ.
+ *
+ * Spec: docs/improvement-plan/06-plugin-system/audit-docs.md (Step 5).
+ */
+
+export const PLUGIN_AUDIT_RETENTION_QUEUE = "plugin-audit-retention" as const;
+export const PLUGIN_AUDIT_RETENTION_JOB_ID = "plugin-audit-retention" as const;
+/** Nightly at 03:00 UTC — matches the spec. */
+export const PLUGIN_AUDIT_RETENTION_CRON_DEFAULT = "0 3 * * *" as const;
+
+export const PLUGIN_AUDIT_RETENTION_DAYS_DEFAULT = 90;
+export const PLUGIN_AUDIT_RETENTION_BATCH_SIZE_DEFAULT = 1000;
+export const PLUGIN_AUDIT_RETENTION_MAX_BATCHES_DEFAULT = 100;
+
+const MS_PER_DAY = 86_400_000;
+
+export interface PluginAuditRetentionLogger {
+  info: (fields: Record<string, unknown>, msg: string) => void;
+  warn?: (fields: Record<string, unknown>, msg: string) => void;
+  error: (fields: Record<string, unknown>, msg: string) => void;
+}
+
+export interface PluginAuditRetentionConfig {
+  /** Days to keep. Rows with `timestamp < now - days` are deleted. */
+  retentionDays: number;
+  /** Max rows to delete per `deleteBatch` call. */
+  batchSize: number;
+  /** Hard cap on iterations per run (back-pressure / lock-window guard). */
+  maxBatches: number;
+}
+
+export type PluginAuditRetentionEnv = Readonly<Record<string, string | undefined>>;
+
+export interface PluginAuditRetentionInput {
+  /** Reference "now" — defaults to `new Date()`. Injected for tests. */
+  now?: Date;
+  /** Resolved config. If absent, read from `env`. */
+  config?: PluginAuditRetentionConfig;
+  /** Env (process.env in production). Used only when `config` is omitted. */
+  env?: PluginAuditRetentionEnv;
+  /**
+   * Delete up to `batchSize` rows where `timestamp < cutoff`. Returns the
+   * exact number of rows removed. Production wires this to
+   * `prismaWrite.pluginCapabilityUse.deleteMany` with a `where.id in
+   * (subselect)` trick to bound the batch (Postgres does not natively
+   * support LIMIT on DELETE).
+   */
+  deleteBatch: (cutoff: Date, batchSize: number) => Promise<number>;
+  /**
+   * F-020 — optional post-sweep row counter. When supplied, the
+   * retention job logs the surviving row count at the end of the run.
+   * Production wires this to `prismaWrite.pluginCapabilityUse.count()`.
+   * When omitted (tests, dev) the size-warning step is skipped silently.
+   */
+  countRows?: () => Promise<number>;
+  /**
+   * F-020 — soft warning threshold. When the surviving row count after
+   * the sweep exceeds this number, the job emits a `warn` log line so
+   * the operator notices unbounded growth (e.g. retention is too lax
+   * for the current invocation volume). Default 10 million rows.
+   */
+  warnRowCountThreshold?: number;
+  logger?: PluginAuditRetentionLogger;
+}
+
+export interface PluginAuditRetentionResult {
+  readonly deletedRows: number;
+  readonly batches: number;
+  readonly cutoffIso: string;
+}
+
+function parsePositiveInt(raw: string | undefined, fallback: number): number {
+  if (typeof raw !== "string" || raw.trim().length === 0) return fallback;
+  const parsed = Number.parseInt(raw.trim(), 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export function resolvePluginAuditRetentionConfig(
+  env: PluginAuditRetentionEnv | undefined
+): PluginAuditRetentionConfig {
+  const source = env ?? {};
+  return {
+    retentionDays: parsePositiveInt(source.PLUGIN_AUDIT_RETENTION_DAYS, PLUGIN_AUDIT_RETENTION_DAYS_DEFAULT),
+    batchSize: parsePositiveInt(source.PLUGIN_AUDIT_RETENTION_BATCH_SIZE, PLUGIN_AUDIT_RETENTION_BATCH_SIZE_DEFAULT),
+    maxBatches: parsePositiveInt(source.PLUGIN_AUDIT_RETENTION_MAX_BATCHES, PLUGIN_AUDIT_RETENTION_MAX_BATCHES_DEFAULT),
+  };
+}
+
+/**
+ * Run one retention sweep. Resolves with a structured result so the
+ * BullMQ wrapper can log/metricise without needing to throw on the
+ * "0 rows" common case.
+ *
+ * Soft-fail per batch: if `deleteBatch` throws, the error is logged and
+ * the loop terminates with the rows accumulated so far. The next nightly
+ * run will retry — we deliberately do not throw because BullMQ would
+ * then either (a) retry immediately (hammering a sick DB) or (b)
+ * dead-letter a perfectly resumable cron.
+ */
+export async function runPluginAuditRetention(
+  input: PluginAuditRetentionInput
+): Promise<PluginAuditRetentionResult> {
+  const now = input.now ?? new Date();
+  const config = input.config ?? resolvePluginAuditRetentionConfig(input.env);
+  const cutoff = new Date(now.getTime() - config.retentionDays * MS_PER_DAY);
+  const cutoffIso = cutoff.toISOString();
+
+  let deletedRows = 0;
+  let batches = 0;
+
+  input.logger?.info(
+    {
+      retentionDays: config.retentionDays,
+      batchSize: config.batchSize,
+      maxBatches: config.maxBatches,
+      cutoff: cutoffIso,
+    },
+    "plugin-audit-retention.start"
+  );
+
+  while (batches < config.maxBatches) {
+    let removed: number;
+    try {
+      removed = await input.deleteBatch(cutoff, config.batchSize);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      input.logger?.error(
+        { batch: batches + 1, cutoff: cutoffIso, deletedRows, err: message },
+        "plugin-audit-retention.batch-failed"
+      );
+      // Soft-fail: stop the loop but return what we successfully deleted.
+      return { deletedRows, batches, cutoffIso };
+    }
+
+    batches += 1;
+    deletedRows += removed;
+
+    input.logger?.info(
+      { batch: batches, removed, deletedRowsRunning: deletedRows },
+      "plugin-audit-retention.batch-complete"
+    );
+
+    if (removed === 0) {
+      break;
+    }
+  }
+
+  if (batches >= config.maxBatches && deletedRows > 0) {
+    input.logger?.warn?.(
+      { batches, deletedRows, cutoff: cutoffIso, maxBatches: config.maxBatches },
+      "plugin-audit-retention.max-batches-reached"
+    );
+  }
+
+  // F-020 — optional surviving-rows count + soft warning when the
+  // table is unusually large. Soft-fail: a count() error never aborts
+  // the retention result the caller already earned.
+  if (input.countRows) {
+    const threshold = input.warnRowCountThreshold ?? 10_000_000;
+    try {
+      const surviving = await input.countRows();
+      if (surviving > threshold) {
+        input.logger?.warn?.(
+          { survivingRows: surviving, threshold, cutoff: cutoffIso, deletedRows },
+          "plugin-audit-retention.table-size-warning"
+        );
+      } else {
+        input.logger?.info(
+          { survivingRows: surviving, cutoff: cutoffIso },
+          "plugin-audit-retention.table-size"
+        );
+      }
+    } catch (err) {
+      input.logger?.warn?.(
+        { err: err instanceof Error ? err.message : String(err) },
+        "plugin-audit-retention.count-failed"
+      );
+    }
+  }
+
+  input.logger?.info(
+    { deletedRows, batches, cutoff: cutoffIso },
+    "plugin-audit-retention.complete"
+  );
+
+  return { deletedRows, batches, cutoffIso };
+}
+
+/**
+ * Helper for the worker bootstrap. Given a BullMQ-like queue with the
+ * `upsertJobScheduler` API, install (or replace) the nightly schedule.
+ * Pinned to a stable scheduler key so a second call is a no-op
+ * (idempotent on redeploys). The cron pattern is read from
+ * `PLUGIN_AUDIT_RETENTION_CRON` (default `0 3 * * *`).
+ */
+export async function registerPluginAuditRetentionCron(
+  queue: {
+    upsertJobScheduler: (
+      schedulerId: string,
+      options: { pattern: string },
+      template: { name: string; data?: Record<string, unknown> }
+    ) => Promise<unknown>;
+  },
+  env: PluginAuditRetentionEnv | undefined = process.env
+): Promise<{ pattern: string }> {
+  const pattern = env?.PLUGIN_AUDIT_RETENTION_CRON?.trim() || PLUGIN_AUDIT_RETENTION_CRON_DEFAULT;
+  await queue.upsertJobScheduler(
+    PLUGIN_AUDIT_RETENTION_JOB_ID,
+    { pattern },
+    { name: PLUGIN_AUDIT_RETENTION_JOB_ID, data: { task: PLUGIN_AUDIT_RETENTION_JOB_ID } }
+  );
+  return { pattern };
+}

hivegrid-cloud/apps/worker/src/jobs/plugin-capability-use-bootstrap.test.ts

@@ -0,0 +1,152 @@
+import { describe, expect, test, vi } from "vitest";
+
+import { PLUGIN_CAPABILITY_USE_EVENT_TYPE, PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE, type PluginCapabilityUseStore } from "./plugin-capability-use.js";
+import {
+  bootstrapPluginCapabilityUse,
+  isPluginCapabilityUseRequired,
+  PluginCapabilityUseRequiredBootstrapError,
+  type PluginCapabilityUseBootstrapDeps,
+  type PluginCapabilityUseWorkerHandle,
+} from "./plugin-capability-use-bootstrap.js";
+
+type WorkerProcessor = (job: { data: unknown }) => Promise<unknown>;
+type WorkerOptions = { concurrency?: number };
+
+function fakeWorker() {
+  const close = vi.fn(async () => undefined);
+  const handle: PluginCapabilityUseWorkerHandle = { name: "fake-plugin-capability-use", close };
+  const calls: Array<[string, WorkerProcessor, WorkerOptions]> = [];
+  const createWorkerImpl = vi.fn((queueName: string, processor: WorkerProcessor, options: WorkerOptions) => {
+    calls.push([queueName, processor, options]);
+    return handle;
+  });
+  return { close, handle, createWorkerImpl, calls };
+}
+
+function fakeStore(): PluginCapabilityUseStore {
+  return {
+    async insertMany(rows) {
+      return { inserted: rows.length };
+    },
+  };
+}
+
+function validJobData() {
+  return {
+    outboxEventId: "evt-1",
+    tenantId: "t1",
+    eventType: PLUGIN_CAPABILITY_USE_EVENT_TYPE,
+    payload: {
+      pluginInvocationId: "inv-1",
+      tenantId: "t1",
+      pluginId: "de.acme.slack",
+      entries: [
+        {
+          id: "row-1",
+          capability: "log",
+          outcome: "success",
+          timestamp: "2026-05-17T20:00:00.000Z",
+        },
+      ],
+    },
+  };
+}
+
+describe("bootstrapPluginCapabilityUse", () => {
+  test("disables on non-general worker groups", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapPluginCapabilityUse({ WORKER_QUEUE_GROUP: "mail" }, { store: fakeStore(), createWorkerImpl });
+
+    expect(result).toEqual({ status: "disabled", reason: "non-general worker group" });
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("disables when explicitly turned off", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapPluginCapabilityUse(
+      { WORKER_QUEUE_GROUP: "general", PLUGIN_CAPABILITY_USE_ENABLED: "false" },
+      { store: fakeStore(), createWorkerImpl }
+    );
+
+    expect(result).toEqual({ status: "disabled", reason: "PLUGIN_CAPABILITY_USE_ENABLED=false" });
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("treats capability-use persistence as required when production plugin runtime is enabled", () => {
+    expect(
+      isPluginCapabilityUseRequired({
+        NODE_ENV: "production",
+        WORKER_QUEUE_GROUP: "general",
+        PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
+      })
+    ).toBe(true);
+  });
+
+  test("fails closed when required capability-use persistence is explicitly disabled", () => {
+    const { createWorkerImpl } = fakeWorker();
+
+    expect(() =>
+      bootstrapPluginCapabilityUse(
+        {
+          NODE_ENV: "production",
+          WORKER_QUEUE_GROUP: "general",
+          PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
+          PLUGIN_CAPABILITY_USE_ENABLED: "false",
+        },
+        { store: fakeStore(), createWorkerImpl }
+      )
+    ).toThrow(PluginCapabilityUseRequiredBootstrapError);
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("disables when no store is wired", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapPluginCapabilityUse({ WORKER_QUEUE_GROUP: "general" }, { createWorkerImpl } as PluginCapabilityUseBootstrapDeps);
+
+    expect(result).toEqual({ status: "disabled", reason: "no capability-use store wired" });
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("fails closed when required capability-use persistence has no store wired", () => {
+    const { createWorkerImpl } = fakeWorker();
+
+    expect(() =>
+      bootstrapPluginCapabilityUse(
+        {
+          NODE_ENV: "production",
+          WORKER_QUEUE_GROUP: "general",
+          PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
+        },
+        { createWorkerImpl } as PluginCapabilityUseBootstrapDeps
+      )
+    ).toThrow("no capability-use store wired");
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("subscribes to the plugin capability-use outbox queue", async () => {
+    const { createWorkerImpl, handle, calls } = fakeWorker();
+    const result = bootstrapPluginCapabilityUse(
+      { WORKER_QUEUE_GROUP: "general", PLUGIN_CAPABILITY_USE_CONCURRENCY: "3" },
+      { store: fakeStore(), createWorkerImpl }
+    );
+
+    expect(result.status).toBe("started");
+    expect(result.status === "started" ? result.handle : null).toBe(handle);
+    expect(createWorkerImpl).toHaveBeenCalledWith(PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE, expect.any(Function), expect.objectContaining({ concurrency: 3 }));
+
+    const processor = calls[0]?.[1];
+    expect(processor).toBeDefined();
+    if (!processor) throw new Error("missing processor");
+    await expect(processor({ data: validJobData() })).resolves.toEqual({
+      outcome: "persisted",
+      inserted: 1,
+    });
+  });
+
+  test("serializes capability-use batches by default to keep the audit hash-chain linear", () => {
+    const { createWorkerImpl } = fakeWorker();
+    bootstrapPluginCapabilityUse({ WORKER_QUEUE_GROUP: "general" }, { store: fakeStore(), createWorkerImpl });
+
+    expect(createWorkerImpl).toHaveBeenCalledWith(PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE, expect.any(Function), expect.objectContaining({ concurrency: 1 }));
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-capability-use-bootstrap.ts

@@ -0,0 +1,90 @@
+import type { EnvRecord } from "@toolctrl/config";
+import { describePluginRuntimeProductionGate } from "@toolctrl/plugin-runtime";
+
+import {
+  PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE,
+  handlePluginCapabilityUseJob,
+  type PluginCapabilityUseLogger,
+  type PluginCapabilityUseStore,
+} from "./plugin-capability-use.js";
+
+export type PluginCapabilityUseBootstrapEnv = EnvRecord & {
+  WORKER_QUEUE_GROUP?: string;
+  PLUGIN_CAPABILITY_USE_ENABLED?: string;
+  PLUGIN_CAPABILITY_USE_CONCURRENCY?: string;
+};
+
+export interface PluginCapabilityUseWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface PluginCapabilityUseBootstrapDeps {
+  readonly logger?: PluginCapabilityUseLogger;
+  readonly store?: PluginCapabilityUseStore;
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number }
+  ) => PluginCapabilityUseWorkerHandle;
+}
+
+export type PluginCapabilityUseBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: PluginCapabilityUseWorkerHandle };
+
+const DEFAULT_CONCURRENCY = 1;
+
+export class PluginCapabilityUseRequiredBootstrapError extends Error {
+  constructor(readonly reason: string) {
+    super(`Plugin capability-use persistence is required but disabled: ${reason}`);
+    this.name = "PluginCapabilityUseRequiredBootstrapError";
+  }
+}
+
+export function isPluginCapabilityUseRequired(env: PluginCapabilityUseBootstrapEnv, workerGroup = env.WORKER_QUEUE_GROUP): boolean {
+  return workerGroup === "general" && env.NODE_ENV === "production" && describePluginRuntimeProductionGate(env) === null;
+}
+
+export function bootstrapPluginCapabilityUse(
+  env: PluginCapabilityUseBootstrapEnv,
+  deps: PluginCapabilityUseBootstrapDeps = {}
+): PluginCapabilityUseBootstrapResult {
+  const logger = deps.logger;
+  const required = isPluginCapabilityUseRequired(env);
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
+    return { status: "disabled", reason: "non-general worker group" };
+  }
+  if (env.PLUGIN_CAPABILITY_USE_ENABLED === "false") {
+    return disabledOrRequired("PLUGIN_CAPABILITY_USE_ENABLED=false", required);
+  }
+  if (!deps.store) {
+    return disabledOrRequired("no capability-use store wired", required);
+  }
+
+  const store = deps.store;
+  const concurrency = parsePositiveInt(env.PLUGIN_CAPABILITY_USE_CONCURRENCY, DEFAULT_CONCURRENCY);
+  const processor = (job: { data: unknown }): Promise<unknown> => handlePluginCapabilityUseJob(job.data, { store, logger });
+  const createWorkerImpl = deps.createWorkerImpl ?? defaultCreateWorker;
+  const handle = createWorkerImpl(PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE, processor, { concurrency });
+
+  logger?.info?.({ component: "plugin-capability-use", queue: PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE, concurrency }, "plugin capability-use worker started");
+  return { status: "started", handle };
+}
+
+function disabledOrRequired(reason: string, required: boolean): { readonly status: "disabled"; readonly reason: string } {
+  if (required) {
+    throw new PluginCapabilityUseRequiredBootstrapError(reason);
+  }
+  return { status: "disabled", reason };
+}
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+const defaultCreateWorker: NonNullable<PluginCapabilityUseBootstrapDeps["createWorkerImpl"]> = () => {
+  throw new Error("bootstrapPluginCapabilityUse requires deps.createWorkerImpl in this build; the worker entry must wire the BullMQ Worker constructor.");
+};

hivegrid-cloud/apps/worker/src/jobs/plugin-capability-use.test.ts

@@ -0,0 +1,347 @@
+import { describe, expect, test } from "vitest";
+import {
+  PLUGIN_CAPABILITY_USE_EVENT_TYPE,
+  PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE,
+  createPrismaPluginCapabilityUseStore,
+  handlePluginCapabilityUseJob,
+  parsePluginCapabilityUseJob,
+  PluginCapabilityUseJobError,
+  type PluginCapabilityUseRow,
+  type PluginCapabilityUseStorePrisma,
+  type PluginCapabilityUseStore,
+} from "./plugin-capability-use.js";
+
+function makeStore(): {
+  store: PluginCapabilityUseStore;
+  inserts: ReadonlyArray<PluginCapabilityUseRow>[];
+} {
+  const inserts: ReadonlyArray<PluginCapabilityUseRow>[] = [];
+  return {
+    inserts,
+    store: {
+      async insertMany(rows) {
+        inserts.push(rows);
+        return { inserted: rows.length };
+      },
+    },
+  };
+}
+
+function validBatch(overrides?: Partial<Record<string, unknown>>) {
+  return {
+    outboxEventId: "evt-1",
+    tenantId: "t1",
+    eventType: PLUGIN_CAPABILITY_USE_EVENT_TYPE,
+    payload: {
+      pluginInvocationId: "inv-1",
+      tenantId: "t1",
+      pluginId: "de.acme.slack",
+      entries: [
+        {
+          id: "row-1",
+          pluginInvocationId: "inv-1",
+          tenantId: "t1",
+          pluginId: "de.acme.slack",
+          capability: "http.outbound",
+          outcome: "success",
+          payloadHash: "a".repeat(64),
+          durationMs: 12,
+          timestamp: "2026-05-17T20:00:00.000Z",
+        },
+        {
+          id: "row-2",
+          pluginInvocationId: "inv-1",
+          tenantId: "t1",
+          pluginId: "de.acme.slack",
+          capability: "log",
+          outcome: "success",
+          durationMs: 1,
+          timestamp: "2026-05-17T20:00:00.100Z",
+        },
+      ],
+      ...overrides,
+    },
+  };
+}
+
+describe("queue / event-type constants", () => {
+  test("exports stable queue + event-type names", () => {
+    expect(PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE).toBe("outbox.plugin-capability-use");
+    expect(PLUGIN_CAPABILITY_USE_EVENT_TYPE).toBe("plugin.capability-use.batch");
+  });
+});
+
+describe("parsePluginCapabilityUseJob", () => {
+  test("parses a well-formed batch", () => {
+    const parsed = parsePluginCapabilityUseJob(validBatch());
+    expect(parsed.payload.entries).toHaveLength(2);
+    expect(parsed.payload.pluginId).toBe("de.acme.slack");
+  });
+
+  test("rejects missing payload", () => {
+    expect(() => parsePluginCapabilityUseJob({})).toThrow(PluginCapabilityUseJobError);
+  });
+
+  test("rejects empty entries array", () => {
+    expect(() => parsePluginCapabilityUseJob(validBatch({ entries: [] }))).toThrow(/EMPTY_BATCH|no entries/);
+  });
+
+  test("rejects entry missing required fields", () => {
+    expect(() =>
+      parsePluginCapabilityUseJob(
+        validBatch({ entries: [{ capability: "log", outcome: "success" }] })
+      )
+    ).toThrow(PluginCapabilityUseJobError);
+  });
+
+  test.each([
+    ["tenantId", "other-tenant"],
+    ["pluginId", "de.acme.other"],
+    ["pluginInvocationId", "inv-other"],
+  ])("rejects entries whose %s differs from the batch envelope", (field, value) => {
+    expect(() =>
+      parsePluginCapabilityUseJob(
+        validBatch({
+          entries: [
+            {
+              id: "row-x",
+              pluginInvocationId: "inv-1",
+              tenantId: "t1",
+              pluginId: "de.acme.slack",
+              capability: "log",
+              outcome: "success",
+              timestamp: "2026-05-17T20:00:00.000Z",
+              [field]: value,
+            },
+          ],
+        })
+      )
+    ).toThrow(PluginCapabilityUseJobError);
+  });
+
+  test("rejects an outbox envelope whose tenant differs from the payload tenant", () => {
+    expect(() => parsePluginCapabilityUseJob({ ...validBatch(), tenantId: "other-tenant" })).toThrow(PluginCapabilityUseJobError);
+  });
+
+  test("inherits invocation defaults when entry omits them", () => {
+    const parsed = parsePluginCapabilityUseJob(
+      validBatch({
+        entries: [
+          {
+            id: "row-x",
+            capability: "log",
+            outcome: "success",
+            timestamp: "2026-05-17T20:00:00.000Z",
+          },
+        ],
+      })
+    );
+    expect(parsed.payload.entries[0]?.tenantId).toBe("t1");
+    expect(parsed.payload.entries[0]?.pluginId).toBe("de.acme.slack");
+    expect(parsed.payload.entries[0]?.pluginInvocationId).toBe("inv-1");
+  });
+});
+
+describe("handlePluginCapabilityUseJob", () => {
+  test("persists every entry from a valid batch", async () => {
+    const { store, inserts } = makeStore();
+    const result = await handlePluginCapabilityUseJob(validBatch(), { store });
+    expect(result.outcome).toBe("persisted");
+    expect(result.inserted).toBe(2);
+    expect(inserts).toHaveLength(1);
+    expect(inserts[0]).toHaveLength(2);
+    expect(inserts[0]?.[0]?.timestamp).toBeInstanceOf(Date);
+  });
+
+  test("normalises undefined optional fields to null", async () => {
+    const { store, inserts } = makeStore();
+    await handlePluginCapabilityUseJob(validBatch(), { store });
+    const row = inserts[0]?.[1];
+    expect(row?.payloadHash).toBeNull();
+    expect(row?.errorCode).toBeNull();
+  });
+
+  test("rejects garbage payloads without retry", async () => {
+    const { store } = makeStore();
+    const warns: Record<string, unknown>[] = [];
+    const result = await handlePluginCapabilityUseJob({}, {
+      store,
+      logger: {
+        warn: (record) => warns.push(record),
+        error: () => undefined,
+      },
+    });
+    expect(result.outcome).toBe("rejected");
+    expect(result.reason).toBe("INVALID_PAYLOAD");
+    expect(warns).toHaveLength(1);
+  });
+
+  test("rejects scoped-entry mismatches without inserting audit rows", async () => {
+    const { store, inserts } = makeStore();
+    const result = await handlePluginCapabilityUseJob(
+      validBatch({
+        entries: [
+          {
+            id: "row-x",
+            pluginInvocationId: "inv-other",
+            tenantId: "t1",
+            pluginId: "de.acme.slack",
+            capability: "log",
+            outcome: "success",
+            timestamp: "2026-05-17T20:00:00.000Z",
+          },
+        ],
+      }),
+      { store }
+    );
+    expect(result).toEqual({ outcome: "rejected", inserted: 0, reason: "INVALID_PAYLOAD" });
+    expect(inserts).toHaveLength(0);
+  });
+
+  test("propagates transient DB errors so dispatcher retries", async () => {
+    const failingStore: PluginCapabilityUseStore = {
+      async insertMany() {
+        throw new Error("connection refused");
+      },
+    };
+    await expect(
+      handlePluginCapabilityUseJob(validBatch(), { store: failingStore })
+    ).rejects.toThrow(/connection refused/);
+  });
+
+  test("treats duplicate receiver delivery as an idempotent zero-insert", async () => {
+    const inserts: ReadonlyArray<PluginCapabilityUseRow>[] = [];
+    const store: PluginCapabilityUseStore = {
+      async insertMany(rows) {
+        inserts.push(rows);
+        return { inserted: 0 };
+      },
+    };
+
+    const result = await handlePluginCapabilityUseJob(validBatch(), { store });
+
+    expect(result).toEqual({ outcome: "persisted", inserted: 0 });
+    expect(inserts).toHaveLength(1);
+  });
+
+  // F-003 — hash chain stamping.
+  describe("hash chain stamping (F-003)", () => {
+    test("stamps every inserted row with prevHash + rowHash", async () => {
+      const { store, inserts } = makeStore();
+      const result = await handlePluginCapabilityUseJob(validBatch(), { store });
+      expect(result.outcome).toBe("persisted");
+      const rows = inserts[0]!;
+      expect(rows.length).toBeGreaterThan(0);
+      for (const row of rows) {
+        expect(row.rowHash).toMatch(/^[0-9a-f]{64}$/);
+      }
+      // Within a batch each row's prevHash must equal the previous row's rowHash.
+      for (let i = 1; i < rows.length; i++) {
+        expect(rows[i]!.prevHash).toBe(rows[i - 1]!.rowHash);
+      }
+      // The genesis row's prevHash is null (no chain head supplied).
+      expect(rows[0]!.prevHash).toBeNull();
+    });
+
+    test("seeds prevHash from getChainHead when supplied", async () => {
+      const { inserts } = makeStore();
+      const fakeChainHead = "f".repeat(64);
+      const store: PluginCapabilityUseStore = {
+        async insertMany(rows) {
+          inserts.push(rows);
+          return { inserted: rows.length };
+        },
+        async getChainHead() {
+          return fakeChainHead;
+        },
+      };
+      await handlePluginCapabilityUseJob(validBatch(), { store });
+      const rows = inserts[0]!;
+      expect(rows[0]!.prevHash).toBe(fakeChainHead);
+    });
+
+    test("falls back to null seed when getChainHead throws", async () => {
+      const { inserts } = makeStore();
+      const store: PluginCapabilityUseStore = {
+        async insertMany(rows) {
+          inserts.push(rows);
+          return { inserted: rows.length };
+        },
+        async getChainHead() {
+          throw new Error("DB unavailable");
+        },
+      };
+      const result = await handlePluginCapabilityUseJob(validBatch(), { store });
+      expect(result.outcome).toBe("persisted");
+      expect(inserts[0]![0]!.prevHash).toBeNull();
+    });
+  });
+});
+
+describe("createPrismaPluginCapabilityUseStore", () => {
+  test("uses createMany with skipDuplicates so outbox redelivery is idempotent", async () => {
+    const calls: unknown[] = [];
+    const prisma: PluginCapabilityUseStorePrisma = {
+      pluginCapabilityUse: {
+        async createMany(args) {
+          calls.push(args);
+          return { count: args.data.length };
+        },
+        async findFirst() {
+          return null;
+        },
+      },
+    };
+    const store = createPrismaPluginCapabilityUseStore(prisma);
+
+    const result = await store.insertMany([
+      {
+        id: "row-1",
+        pluginInvocationId: "inv-1",
+        tenantId: "t1",
+        pluginId: "de.acme.slack",
+        capability: "log",
+        outcome: "success",
+        payloadHash: null,
+        errorCode: null,
+        durationMs: null,
+        timestamp: new Date("2026-05-17T20:00:00.000Z"),
+        prevHash: null,
+        rowHash: "a".repeat(64),
+      },
+    ]);
+
+    expect(result).toEqual({ inserted: 1 });
+    expect(calls).toEqual([
+      {
+        data: expect.arrayContaining([expect.objectContaining({ id: "row-1" })]),
+        skipDuplicates: true,
+      },
+    ]);
+  });
+
+  test("reads the latest tenant chain head for cross-batch hash continuity", async () => {
+    const calls: unknown[] = [];
+    const prisma: PluginCapabilityUseStorePrisma = {
+      pluginCapabilityUse: {
+        async createMany() {
+          return { count: 0 };
+        },
+        async findFirst(args) {
+          calls.push(args);
+          return { rowHash: "f".repeat(64) };
+        },
+      },
+    };
+    const store = createPrismaPluginCapabilityUseStore(prisma);
+
+    await expect(store.getChainHead?.("t1")).resolves.toBe("f".repeat(64));
+    expect(calls).toEqual([
+      {
+        where: { tenantId: "t1", rowHash: { not: null } },
+        select: { rowHash: true },
+        orderBy: [{ timestamp: "desc" }, { id: "desc" }],
+      },
+    ]);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-capability-use.ts

@@ -0,0 +1,306 @@
+/**
+ * Plugin capability-use batch insert — Phase 06 #23.
+ *
+ * Subscribes to the `outbox.plugin-capability-use` BullMQ queue
+ * (populated by the outbox dispatcher in apps/worker/src/outbox/
+ * dispatcher.ts) and writes each batch into the `PluginCapabilityUse`
+ * table. Batches are bounded by `PluginCapabilityUseRecorder.maxEntries`
+ * (default 1024) so a single `createMany` call is always safe.
+ *
+ * Failure mode: the handler returns a structured result instead of
+ * throwing on payload validation issues so the dispatcher can dead-
+ * letter without retrying garbage payloads. Transient DB errors
+ * propagate so BullMQ retries with backoff.
+ */
+
+export const PLUGIN_CAPABILITY_USE_OUTBOX_QUEUE = "outbox.plugin-capability-use" as const;
+export const PLUGIN_CAPABILITY_USE_EVENT_TYPE = "plugin.capability-use.batch" as const;
+
+export interface PluginCapabilityUseLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface PluginCapabilityUseRow {
+  readonly id: string;
+  readonly pluginInvocationId: string;
+  readonly tenantId: string;
+  readonly pluginId: string;
+  readonly capability: string;
+  readonly outcome: string;
+  readonly payloadHash: string | null;
+  readonly errorCode: string | null;
+  readonly durationMs: number | null;
+  readonly timestamp: Date;
+  /** F-003 — hash chain. Null on rows persisted before F-003 landed. */
+  readonly prevHash?: string | null;
+  readonly rowHash?: string | null;
+}
+
+export interface PluginCapabilityUseStore {
+  /**
+   * Persist rows idempotently. Production stores must ignore duplicate
+   * primary keys so an outbox redelivery does not turn an already-written
+   * audit batch into a retry loop.
+   */
+  insertMany(rows: ReadonlyArray<PluginCapabilityUseRow>): Promise<{ inserted: number }>;
+  /**
+   * F-003 — return the `rowHash` of the most recent row for a tenant,
+   * or null when this is the genesis batch (no prior hashed rows).
+   * Production implementations point this at a SELECT…ORDER BY
+   * timestamp DESC, id DESC LIMIT 1 WHERE rowHash IS NOT NULL.
+   *
+   * Optional for back-compat with stores written before F-003. When
+   * omitted, the handler still computes hashes (genesis chain per
+   * batch) but cannot detect tampering across batches — visible in
+   * `verifyPluginCapabilityUseChain` as a `prev_mismatch` on every
+   * batch boundary.
+   */
+  getChainHead?: (tenantId: string) => Promise<string | null>;
+}
+
+export type PluginCapabilityUseStorePrisma = {
+  pluginCapabilityUse: {
+    createMany(args: {
+      data: ReadonlyArray<PluginCapabilityUseRow>;
+      skipDuplicates: true;
+    }): Promise<{ count: number }>;
+    findFirst(args: {
+      where: { tenantId: string; rowHash: { not: null } };
+      select: { rowHash: true };
+      orderBy: [{ timestamp: "desc" }, { id: "desc" }];
+    }): Promise<{ rowHash: string | null } | null>;
+  };
+};
+
+export function createPrismaPluginCapabilityUseStore(
+  prisma: PluginCapabilityUseStorePrisma
+): PluginCapabilityUseStore {
+  return {
+    async insertMany(rows) {
+      const result = await prisma.pluginCapabilityUse.createMany({
+        data: rows,
+        skipDuplicates: true,
+      });
+      return { inserted: result.count };
+    },
+    async getChainHead(tenantId) {
+      const row = await prisma.pluginCapabilityUse.findFirst({
+        where: { tenantId, rowHash: { not: null } },
+        select: { rowHash: true },
+        orderBy: [{ timestamp: "desc" }, { id: "desc" }],
+      });
+      return row?.rowHash ?? null;
+    },
+  };
+}
+
+export interface PluginCapabilityUseJobData {
+  readonly outboxEventId?: string;
+  readonly tenantId: string | null;
+  readonly eventType: string;
+  readonly payload: PluginCapabilityUseBatchPayload;
+}
+
+export interface PluginCapabilityUseBatchPayload {
+  readonly pluginInvocationId: string;
+  readonly tenantId: string;
+  readonly pluginId: string;
+  readonly entries: ReadonlyArray<PluginCapabilityUseEntryPayload>;
+}
+
+export interface PluginCapabilityUseEntryPayload {
+  readonly id: string;
+  readonly pluginInvocationId: string;
+  readonly tenantId: string;
+  readonly pluginId: string;
+  readonly capability: string;
+  readonly outcome: string;
+  readonly payloadHash?: string;
+  readonly errorCode?: string;
+  readonly durationMs?: number;
+  readonly timestamp: string;
+}
+
+export class PluginCapabilityUseJobError extends Error {
+  override readonly name = "PluginCapabilityUseJobError";
+  readonly code: "INVALID_PAYLOAD" | "EMPTY_BATCH";
+  constructor(message: string, code: PluginCapabilityUseJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export function parsePluginCapabilityUseJob(data: unknown): PluginCapabilityUseJobData {
+  if (!data || typeof data !== "object") {
+    throw new PluginCapabilityUseJobError("missing data envelope", "INVALID_PAYLOAD");
+  }
+  const envelope = data as Record<string, unknown>;
+  const payload = envelope.payload;
+  if (!payload || typeof payload !== "object") {
+    throw new PluginCapabilityUseJobError("missing payload", "INVALID_PAYLOAD");
+  }
+  const inner = payload as Record<string, unknown>;
+  const pluginInvocationId = typeof inner.pluginInvocationId === "string" ? inner.pluginInvocationId : "";
+  const tenantId = typeof inner.tenantId === "string" ? inner.tenantId : "";
+  const pluginId = typeof inner.pluginId === "string" ? inner.pluginId : "";
+  if (pluginInvocationId.length === 0 || tenantId.length === 0 || pluginId.length === 0) {
+    throw new PluginCapabilityUseJobError(
+      "missing pluginInvocationId / tenantId / pluginId",
+      "INVALID_PAYLOAD"
+    );
+  }
+  if (typeof envelope.tenantId === "string" && envelope.tenantId !== tenantId) {
+    throw new PluginCapabilityUseJobError("envelope tenantId does not match payload tenantId", "INVALID_PAYLOAD");
+  }
+  const rawEntries = inner.entries;
+  if (!Array.isArray(rawEntries) || rawEntries.length === 0) {
+    throw new PluginCapabilityUseJobError("batch has no entries", "EMPTY_BATCH");
+  }
+  const entries = rawEntries.map((entry, index) => normaliseEntry(entry, index, {
+    pluginInvocationId,
+    tenantId,
+    pluginId,
+  }));
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId: typeof envelope.tenantId === "string" ? envelope.tenantId : null,
+    eventType: typeof envelope.eventType === "string" ? envelope.eventType : "",
+    payload: {
+      pluginInvocationId,
+      tenantId,
+      pluginId,
+      entries,
+    },
+  };
+}
+
+function normaliseEntry(
+  raw: unknown,
+  index: number,
+  defaults: { pluginInvocationId: string; tenantId: string; pluginId: string }
+): PluginCapabilityUseEntryPayload {
+  if (!raw || typeof raw !== "object") {
+    throw new PluginCapabilityUseJobError(`entry[${index}] is not an object`, "INVALID_PAYLOAD");
+  }
+  const e = raw as Record<string, unknown>;
+  const id = typeof e.id === "string" ? e.id : "";
+  const capability = typeof e.capability === "string" ? e.capability : "";
+  const outcome = typeof e.outcome === "string" ? e.outcome : "";
+  const timestamp = typeof e.timestamp === "string" ? e.timestamp : "";
+  if (id.length === 0 || capability.length === 0 || outcome.length === 0 || timestamp.length === 0) {
+    throw new PluginCapabilityUseJobError(
+      `entry[${index}] missing id / capability / outcome / timestamp`,
+      "INVALID_PAYLOAD"
+    );
+  }
+  return {
+    id,
+    pluginInvocationId: normaliseScopedEntryValue(e.pluginInvocationId, defaults.pluginInvocationId, `entry[${index}].pluginInvocationId`),
+    tenantId: normaliseScopedEntryValue(e.tenantId, defaults.tenantId, `entry[${index}].tenantId`),
+    pluginId: normaliseScopedEntryValue(e.pluginId, defaults.pluginId, `entry[${index}].pluginId`),
+    capability,
+    outcome,
+    payloadHash: typeof e.payloadHash === "string" ? e.payloadHash : undefined,
+    errorCode: typeof e.errorCode === "string" ? e.errorCode : undefined,
+    durationMs:
+      typeof e.durationMs === "number" && Number.isFinite(e.durationMs) ? Math.max(0, e.durationMs) : undefined,
+    timestamp,
+  };
+}
+
+function normaliseScopedEntryValue(raw: unknown, expected: string, path: string): string {
+  if (raw === undefined || raw === null) {
+    return expected;
+  }
+  if (raw !== expected) {
+    throw new PluginCapabilityUseJobError(`${path} does not match the batch envelope`, "INVALID_PAYLOAD");
+  }
+  return expected;
+}
+
+export interface PluginCapabilityUseHandlerDeps {
+  readonly store: PluginCapabilityUseStore;
+  readonly logger?: PluginCapabilityUseLogger;
+}
+
+export interface PluginCapabilityUseHandlerResult {
+  readonly outcome: "persisted" | "rejected";
+  readonly inserted: number;
+  readonly reason?: string;
+}
+
+/**
+ * Process one batch. Resolves with a structured outcome — the dispatcher
+ * decides whether to ack (outcome=persisted/rejected) or retry (thrown
+ * exception, e.g. transient DB error).
+ */
+export async function handlePluginCapabilityUseJob(
+  data: unknown,
+  deps: PluginCapabilityUseHandlerDeps
+): Promise<PluginCapabilityUseHandlerResult> {
+  let parsed: PluginCapabilityUseJobData;
+  try {
+    parsed = parsePluginCapabilityUseJob(data);
+  } catch (err) {
+    if (err instanceof PluginCapabilityUseJobError) {
+      deps.logger?.warn?.(
+        { code: err.code, message: err.message },
+        "plugin capability-use job rejected"
+      );
+      return { outcome: "rejected", inserted: 0, reason: err.code };
+    }
+    throw err;
+  }
+  const rawRows = parsed.payload.entries.map((entry) => ({
+    id: entry.id,
+    pluginInvocationId: entry.pluginInvocationId,
+    tenantId: entry.tenantId,
+    pluginId: entry.pluginId,
+    capability: entry.capability,
+    outcome: entry.outcome,
+    payloadHash: entry.payloadHash ?? null,
+    errorCode: entry.errorCode ?? null,
+    durationMs: entry.durationMs ?? null,
+    timestamp: new Date(entry.timestamp),
+  }));
+
+  // F-003 — stamp every row with prevHash + rowHash. The chain is
+  // per-tenant; the store optionally exposes getChainHead so the
+  // chain survives across batches (otherwise each batch is a fresh
+  // mini-chain — still useful for intra-batch integrity).
+  const { stampPluginCapabilityUseChain } = await import("@toolctrl/plugin-runtime");
+  const seedPrevHash = deps.store.getChainHead
+    ? await deps.store.getChainHead(parsed.payload.tenantId).catch(() => null)
+    : null;
+  const chainInput = rawRows.map((r) => ({
+    id: r.id,
+    pluginInvocationId: r.pluginInvocationId,
+    tenantId: r.tenantId,
+    pluginId: r.pluginId,
+    capability: r.capability,
+    payloadHash: r.payloadHash,
+    outcome: r.outcome,
+    errorCode: r.errorCode,
+    durationMs: r.durationMs,
+    timestamp: r.timestamp.toISOString(),
+  }));
+  const { stamped } = stampPluginCapabilityUseChain({ seedPrevHash, rows: chainInput });
+  const rows: ReadonlyArray<PluginCapabilityUseRow> = rawRows.map((raw, i) => ({
+    ...raw,
+    prevHash: stamped[i]?.prevHash ?? null,
+    rowHash: stamped[i]?.rowHash ?? null,
+  }));
+  const result = await deps.store.insertMany(rows);
+  deps.logger?.info?.(
+    {
+      tenantId: parsed.payload.tenantId,
+      pluginId: parsed.payload.pluginId,
+      pluginInvocationId: parsed.payload.pluginInvocationId,
+      inserted: result.inserted,
+    },
+    "plugin capability-use batch persisted"
+  );
+  return { outcome: "persisted", inserted: result.inserted };
+}

hivegrid-cloud/apps/worker/src/jobs/plugin-hook-fanout-bootstrap.test.ts

@@ -0,0 +1,231 @@
+import { describe, expect, test, vi } from "vitest";
+
+import {
+  bootstrapPluginHookFanout,
+  isPluginHookFanoutRequired,
+  PluginHookFanoutRequiredBootstrapError,
+} from "./plugin-hook-fanout-bootstrap.js";
+import { PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE } from "./plugin-hook-fanout.js";
+
+type WorkerProcessor = (job: { data: unknown }) => Promise<unknown>;
+type WorkerOptions = { concurrency?: number };
+
+function validManifest(hooks: Array<{ name: string; export: string }>) {
+  return {
+    schemaVersion: 1 as const,
+    id: "com.acme.audit",
+    version: "1.0.0",
+    name: "Audit",
+    entry: "plugin.wasm",
+    wasmSha256: "a".repeat(64),
+    signature: "b".repeat(128),
+    signedBy: "c".repeat(64),
+    allowedHosts: [],
+    capabilities: [],
+    hooks,
+  };
+}
+
+function fakeWorker() {
+  const close = vi.fn(async () => undefined);
+  const handle = { name: "fake-plugin-hook-fanout", close };
+  const calls: Array<[string, WorkerProcessor, WorkerOptions]> = [];
+  const createWorkerImpl = vi.fn((queueName: string, processor: WorkerProcessor, options: WorkerOptions) => {
+    calls.push([queueName, processor, options]);
+    return handle;
+  });
+  return { close, handle, createWorkerImpl, calls };
+}
+
+describe("bootstrapPluginHookFanout", () => {
+  test("disables on non-general worker groups", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapPluginHookFanout({ WORKER_QUEUE_GROUP: "plugin" }, { createWorkerImpl });
+    expect(result.status).toBe("disabled");
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("disables when explicitly turned off", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapPluginHookFanout({ WORKER_QUEUE_GROUP: "general", PLUGIN_HOOK_FANOUT_ENABLED: "false" }, { createWorkerImpl });
+    expect(result.status).toBe("disabled");
+  });
+
+  test("treats production plugin runtime as required even when the optional toggle is disabled", () => {
+    expect(
+      isPluginHookFanoutRequired(
+        {
+          NODE_ENV: "production",
+          WORKER_QUEUE_GROUP: "general",
+          BULLMQ_REDIS_URL: "redis://localhost:6379/0",
+          PLUGIN_RUNTIME_PRODUCTION_ENABLED: "true",
+          PLUGIN_HOOK_FANOUT_ENABLED: "false",
+        },
+        "general"
+      )
+    ).toBe(true);
+  });
+
+  test("fails closed when required fanout is explicitly disabled", () => {
+    const { createWorkerImpl } = fakeWorker();
+
+    expect(() =>
+      bootstrapPluginHookFanout({ WORKER_QUEUE_GROUP: "general", PLUGIN_HOOK_FANOUT_REQUIRED: "true", PLUGIN_HOOK_FANOUT_ENABLED: "false" }, { createWorkerImpl })
+    ).toThrow(PluginHookFanoutRequiredBootstrapError);
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("disables in production until the plugin runtime rollout gate is enabled", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapPluginHookFanout({ NODE_ENV: "production", WORKER_QUEUE_GROUP: "general" }, { createWorkerImpl });
+
+    expect(result).toMatchObject({
+      status: "disabled",
+      reason: expect.stringContaining("PLUGIN_RUNTIME_PRODUCTION_ENABLED=true"),
+    });
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("fails closed when required dependencies are not wired", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const env = { WORKER_QUEUE_GROUP: "general", PLUGIN_HOOK_FANOUT_REQUIRED: "true" };
+
+    expect(() => bootstrapPluginHookFanout(env, { createWorkerImpl })).toThrow("no prisma surface wired");
+    expect(() =>
+      bootstrapPluginHookFanout(env, {
+        createWorkerImpl,
+        prisma: { plugin: { findMany: vi.fn(async () => []) } },
+      })
+    ).toThrow("no plugin invoke queue wired");
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  test("starts the outbox plugin-hook subscriber and targets plugin invoke queue", async () => {
+    const { createWorkerImpl, handle, calls } = fakeWorker();
+    const addCalls: Array<[string, unknown, Record<string, unknown>]> = [];
+    const add = vi.fn(async (jobName: string, payload: unknown, options: Record<string, unknown>) => {
+      addCalls.push([jobName, payload, options]);
+      return undefined;
+    });
+    const prisma = {
+      plugin: {
+        findMany: vi.fn(async () => [
+          {
+            pluginId: "com.acme.audit",
+            version: "1.0.0",
+            manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+            capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+          },
+        ]),
+      },
+    };
+    const result = bootstrapPluginHookFanout(
+      { WORKER_QUEUE_GROUP: "general", PLUGIN_HOOK_FANOUT_CONCURRENCY: "3" },
+      {
+        createWorkerImpl,
+        prisma,
+        idempotency: {
+          claim: vi.fn(async () => true),
+          complete: vi.fn(async () => undefined),
+          get: vi.fn(async () => null),
+        },
+        pluginInvokeQueue: { add } as never,
+      }
+    );
+
+    expect(result.status).toBe("started");
+    expect(result.status === "started" ? result.handle : null).toBe(handle);
+    expect(createWorkerImpl).toHaveBeenCalledWith(
+      PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 3 })
+    );
+
+    const processor = calls[0]?.[1];
+    expect(processor).toBeDefined();
+    if (!processor) throw new Error("missing processor");
+    await processor({
+      data: {
+        outboxEventId: "evt_1",
+        tenantId: "tenant_1",
+        eventType: "plugin.hook.tenant.created",
+        payload: {
+          tenantId: "tenant_1",
+          slug: "acme",
+          ownerEmail: "owner@example.test",
+          createdAt: "2026-05-26T20:00:00.000Z",
+        },
+      },
+    });
+
+    expect(add).toHaveBeenCalledWith(
+      "invoke-plugin",
+      expect.objectContaining({ kind: "plugin.invoke", pluginId: "com.acme.audit" }),
+      expect.objectContaining({ jobId: "plugin-hook:evt_1:com.acme.audit:1.0.0" })
+    );
+    expect(prisma.plugin.findMany).toHaveBeenCalledWith({
+      where: { tenantId: "tenant_1", enabled: true },
+      select: { pluginId: true, version: true, manifest: true, capabilityGrants: { select: { capabilityName: true } } },
+    });
+    expect(addCalls[0]?.[1]).toMatchObject({ kind: "plugin.invoke" });
+    expect(addCalls[0]?.[2]).toMatchObject({ attempts: 3 });
+    expect(addCalls[0]?.[1]).not.toMatchObject({ queueName: expect.any(String) });
+  });
+
+  test("loads per-user plugin config when the hook payload has userId", async () => {
+    const { createWorkerImpl, calls } = fakeWorker();
+    const add = vi.fn(async () => undefined);
+    const prisma = {
+      plugin: {
+        findMany: vi.fn(async () => [
+          {
+            pluginId: "com.acme.audit",
+            version: "1.0.0",
+            manifest: validManifest([{ name: "user.created", export: "onUserCreated" }]),
+            capabilityGrants: [{ capabilityName: "hook:user.created" }],
+            userConfigs: [{ enabled: false }],
+          },
+        ]),
+      },
+    };
+
+    bootstrapPluginHookFanout(
+      { WORKER_QUEUE_GROUP: "general" },
+      {
+        createWorkerImpl,
+        prisma,
+        pluginInvokeQueue: { add } as never,
+      }
+    );
+
+    const processor = calls[0]?.[1];
+    expect(processor).toBeDefined();
+    if (!processor) throw new Error("missing processor");
+    await processor({
+      data: {
+        outboxEventId: "evt_1",
+        tenantId: "tenant_1",
+        eventType: "plugin.hook.user.created",
+        payload: {
+          userId: "user_1",
+          tenantId: "tenant_1",
+          email: "user@example.test",
+          locale: "en",
+          createdAt: "2026-05-26T20:00:00.000Z",
+        },
+      },
+    });
+
+    expect(prisma.plugin.findMany).toHaveBeenCalledWith({
+      where: { tenantId: "tenant_1", enabled: true },
+      select: {
+        pluginId: true,
+        version: true,
+        manifest: true,
+        capabilityGrants: { select: { capabilityName: true } },
+        userConfigs: { where: { userId: "user_1" }, select: { enabled: true } },
+      },
+    });
+    expect(add).not.toHaveBeenCalled();
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-hook-fanout-bootstrap.ts

@@ -0,0 +1,163 @@
+import type { EnvRecord } from "@toolctrl/config";
+import type { OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
+import { buildPluginInvokeJob, enqueueJob, queueNames, type QueueEnqueueOptions, type TypedQueue } from "@toolctrl/queue";
+import { describePluginRuntimeProductionGate } from "@toolctrl/plugin-runtime";
+
+import {
+  PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE,
+  runPluginHookFanoutJob,
+  type PluginHookFanoutLogger,
+  type PluginHookFanoutStore,
+} from "./plugin-hook-fanout.js";
+
+export type PluginHookFanoutBootstrapEnv = EnvRecord & {
+  WORKER_QUEUE_GROUP?: string;
+  PLUGIN_HOOK_FANOUT_ENABLED?: string;
+  PLUGIN_HOOK_FANOUT_REQUIRED?: string;
+  PLUGIN_HOOK_FANOUT_CONCURRENCY?: string;
+  BULLMQ_REDIS_URL?: string;
+  REDIS_URL?: string;
+};
+
+export interface PluginHookFanoutPrismaSurface {
+  plugin: {
+    findMany(args: {
+      where: { tenantId: string; enabled: true };
+      select: {
+        pluginId: true;
+        version: true;
+        manifest: true;
+        capabilityGrants: { select: { capabilityName: true } };
+        userConfigs?: { where: { userId: string }; select: { enabled: true } };
+      };
+    }): Promise<
+      ReadonlyArray<{
+        pluginId: string;
+        version: string;
+        manifest: unknown;
+        capabilityGrants: ReadonlyArray<{ capabilityName: string }>;
+        userConfigs?: ReadonlyArray<{ enabled: boolean }>;
+      }>
+    >;
+  };
+}
+
+export interface PluginHookFanoutWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface PluginHookFanoutBootstrapDeps {
+  readonly logger?: PluginHookFanoutLogger;
+  readonly idempotency?: OutboxReceiverIdempotencyStore & { forget?(key: string): Promise<void> };
+  readonly prisma?: PluginHookFanoutPrismaSurface;
+  readonly pluginInvokeQueue?: TypedQueue<typeof queueNames.pluginInvoke>;
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number }
+  ) => PluginHookFanoutWorkerHandle;
+}
+
+export type PluginHookFanoutBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: PluginHookFanoutWorkerHandle };
+
+export class PluginHookFanoutRequiredBootstrapError extends Error {
+  constructor(readonly reason: string) {
+    super(`Plugin hook fanout is required but disabled: ${reason}`);
+    this.name = "PluginHookFanoutRequiredBootstrapError";
+  }
+}
+
+export function isPluginHookFanoutRequired(env: PluginHookFanoutBootstrapEnv, workerGroup = env.WORKER_QUEUE_GROUP): boolean {
+  if (workerGroup !== "general") {
+    return false;
+  }
+  if (env.PLUGIN_HOOK_FANOUT_REQUIRED === "true") {
+    return true;
+  }
+  const redisUrl = env.BULLMQ_REDIS_URL?.trim() || env.REDIS_URL?.trim();
+  if (!redisUrl) {
+    return false;
+  }
+  return describePluginRuntimeProductionGate(env) === null;
+}
+
+export function bootstrapPluginHookFanout(
+  env: PluginHookFanoutBootstrapEnv,
+  deps: PluginHookFanoutBootstrapDeps = {}
+): PluginHookFanoutBootstrapResult {
+  const logger = deps.logger;
+  const required = isPluginHookFanoutRequired(env);
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
+    return { status: "disabled", reason: "non-general worker group" };
+  }
+  if (env.PLUGIN_HOOK_FANOUT_ENABLED === "false") {
+    return disabledOrRequired("PLUGIN_HOOK_FANOUT_ENABLED=false", required);
+  }
+  const productionGateReason = describePluginRuntimeProductionGate(env);
+  if (productionGateReason) {
+    return disabledOrRequired(productionGateReason, required);
+  }
+  if (!deps.prisma) {
+    return disabledOrRequired("no prisma surface wired", required);
+  }
+  if (!deps.pluginInvokeQueue) {
+    return disabledOrRequired("no plugin invoke queue wired", required);
+  }
+
+  const store: PluginHookFanoutStore = {
+    listEnabledPlugins: ({ tenantId, userId }) =>
+      deps.prisma!.plugin.findMany({
+        where: { tenantId, enabled: true },
+        select: {
+          pluginId: true,
+          version: true,
+          manifest: true,
+          capabilityGrants: { select: { capabilityName: true } },
+          ...(userId ? { userConfigs: { where: { userId }, select: { enabled: true } } } : {}),
+        },
+      }),
+  };
+  const pluginInvokeQueue = deps.pluginInvokeQueue;
+  const concurrency = parsePositiveInt(env.PLUGIN_HOOK_FANOUT_CONCURRENCY, 5);
+  const processor = (job: { data: unknown }): Promise<unknown> =>
+    runPluginHookFanoutJob(job.data, {
+      store,
+      logger,
+      idempotency: deps.idempotency,
+      enqueuePluginInvoke: (
+        descriptor: ReturnType<typeof buildPluginInvokeJob>,
+        options: QueueEnqueueOptions
+      ) => enqueueJob(pluginInvokeQueue, descriptor, options),
+    });
+
+  const createWorkerImpl = deps.createWorkerImpl ?? defaultCreateWorker;
+  const handle = createWorkerImpl(PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE, processor, { concurrency });
+
+  logger?.info?.(
+    { component: "plugin-hook-fanout", queue: PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE, targetQueue: queueNames.pluginInvoke, concurrency },
+    "plugin hook fanout worker started"
+  );
+  return { status: "started", handle };
+}
+
+function disabledOrRequired(reason: string, required: boolean): { readonly status: "disabled"; readonly reason: string } {
+  if (required) {
+    throw new PluginHookFanoutRequiredBootstrapError(reason);
+  }
+  return { status: "disabled", reason };
+}
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+const defaultCreateWorker: NonNullable<PluginHookFanoutBootstrapDeps["createWorkerImpl"]> = () => {
+  throw new Error(
+    "bootstrapPluginHookFanout requires deps.createWorkerImpl in this build; the worker entry must wire the BullMQ Worker constructor."
+  );
+};

hivegrid-cloud/apps/worker/src/jobs/plugin-hook-fanout.test.ts

@@ -0,0 +1,795 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+import { queueNames } from "@toolctrl/queue";
+import type { OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
+import { registerHook } from "@toolctrl/plugin-runtime";
+import { z } from "zod";
+
+import {
+  PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY,
+  PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE,
+  type PluginHookFanoutResult,
+  parsePluginHookFanoutJob,
+  PluginHookFanoutJobError,
+  runPluginHookFanoutJob,
+  type PluginHookFanoutDeps,
+  type PluginHookFanoutStore,
+} from "./plugin-hook-fanout.js";
+
+const originalNodeEnv = process.env.NODE_ENV;
+
+afterEach(() => {
+  if (originalNodeEnv === undefined) {
+    delete process.env.NODE_ENV;
+  } else {
+    process.env.NODE_ENV = originalNodeEnv;
+  }
+});
+
+function envelope(overrides: Record<string, unknown> = {}) {
+  return {
+    outboxEventId: "evt_1",
+    tenantId: "tenant_1",
+    eventType: "plugin.hook.tenant.created",
+    payload: { tenantId: "tenant_1", slug: "acme", ownerEmail: "owner@example.test", createdAt: "2026-05-26T20:00:00.000Z" },
+    dispatchKey: "tenant:tenant_1:created",
+    ...overrides,
+  };
+}
+
+function validManifest(hooks: Array<{ name: string; export: string }>) {
+  return {
+    schemaVersion: 1 as const,
+    id: "com.acme.audit",
+    version: "1.0.0",
+    name: "Audit",
+    entry: "plugin.wasm",
+    wasmSha256: "a".repeat(64),
+    signature: "b".repeat(128),
+    signedBy: "c".repeat(64),
+    allowedHosts: [],
+    capabilities: [],
+    hooks,
+  };
+}
+
+function createMemoryIdempotencyStore(): OutboxReceiverIdempotencyStore & { forget(key: string): Promise<void> } {
+  const pending = new Set<string>();
+  const complete = new Map<string, unknown>();
+  return {
+    claim: vi.fn(async (key: string) => {
+      if (pending.has(key) || complete.has(key)) return false;
+      pending.add(key);
+      return true;
+    }),
+    complete: vi.fn(async (key: string, result: unknown) => {
+      pending.delete(key);
+      complete.set(key, result);
+    }),
+    get: vi.fn(async (key: string) => {
+      if (complete.has(key)) return { state: "complete" as const, result: complete.get(key) };
+      if (pending.has(key)) return { state: "pending" as const };
+      return null;
+    }),
+    forget: vi.fn(async (key: string) => {
+      pending.delete(key);
+      complete.delete(key);
+    }),
+  };
+}
+
+describe("plugin hook fanout constants", () => {
+  test("exports the stable outbox queue name", () => {
+    expect(PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE).toBe("outbox.plugin-hook");
+  });
+});
+
+describe("parsePluginHookFanoutJob", () => {
+  test("parses a plugin hook outbox envelope", () => {
+    const parsed = parsePluginHookFanoutJob(envelope());
+    expect(parsed.hookName).toBe("tenant.created");
+    expect(parsed.tenantId).toBe("tenant_1");
+  });
+
+  test("rejects non-plugin-hook event types", () => {
+    expect(() => parsePluginHookFanoutJob(envelope({ eventType: "mail.welcome.send" }))).toThrow(PluginHookFanoutJobError);
+  });
+
+  test("rejects malformed plugin hook names", () => {
+    expect(() => parsePluginHookFanoutJob(envelope({ eventType: "plugin.hook.singleword" }))).toThrow(
+      /invalid plugin hook name singleword/
+    );
+    expect(() => parsePluginHookFanoutJob(envelope({ eventType: "plugin.hook.Tenant.created" }))).toThrow(
+      /invalid plugin hook name Tenant\.created/
+    );
+  });
+
+  test("rejects syntactically valid but unregistered plugin hook names", () => {
+    expect(() => parsePluginHookFanoutJob(envelope({ eventType: "plugin.hook.billing.paid" }))).toThrow(
+      /unsupported plugin hook name billing\.paid/
+    );
+  });
+
+  test("rejects registered hooks outside the production allowlist in production", () => {
+    process.env.NODE_ENV = "production";
+    registerHook({
+      name: "billing.settled",
+      outboxEventType: "plugin.hook.billing.settled",
+      payloadSchema: z.object({ invoiceId: z.string() }),
+    });
+
+    expect(() => parsePluginHookFanoutJob(envelope({ eventType: "plugin.hook.billing.settled" }))).toThrow(
+      /not production-available/
+    );
+  });
+
+  test("requires tenant scope", () => {
+    expect(() => parsePluginHookFanoutJob(envelope({ tenantId: null }))).toThrow(/tenantId/);
+  });
+});
+
+describe("runPluginHookFanoutJob", () => {
+  test("enqueues one plugin.invoke job per enabled subscriber declaring the hook", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+        {
+          pluginId: "com.acme.other",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "entity.created", export: "onEntityCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:entity.created" }],
+        },
+      ]),
+    };
+    const enqueuedPlugins: string[] = [];
+    const enqueuePluginInvoke = vi.fn<PluginHookFanoutDeps["enqueuePluginInvoke"]>(async (descriptor) => {
+      enqueuedPlugins.push(descriptor.payload.pluginId);
+    });
+    const result = await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke });
+
+    expect(result).toMatchObject({ matched: 1, enqueued: 1, hookName: "tenant.created" });
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        queueName: queueNames.pluginInvoke,
+        jobName: "invoke-plugin",
+        payload: expect.objectContaining({
+          kind: "plugin.invoke",
+          tenantId: "tenant_1",
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          hookName: "onTenantCreated",
+          input: expect.objectContaining({ tenantId: "tenant_1", slug: "acme" }),
+        }),
+      }),
+      expect.objectContaining({
+        tenantId: "tenant_1",
+        idempotencyKey: "plugin-hook:evt_1:com.acme.audit:1.0.0",
+      })
+    );
+  });
+
+  test("invokes the declared WASM export while preserving the logical hook name", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn<PluginHookFanoutDeps["enqueuePluginInvoke"]>(async () => undefined);
+
+    await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke });
+
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        payload: expect.objectContaining({
+          hookName: "onTenantCreated",
+          metadata: expect.objectContaining({
+            logicalHookName: "tenant.created",
+            eventType: "plugin.hook.tenant.created",
+          }),
+        }),
+      }),
+      expect.any(Object)
+    );
+  });
+
+  test("skips enabled hook subscribers when hook consent grant is missing", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.granted",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+        {
+          pluginId: "com.acme.ungranted",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:entity.created" }],
+        },
+        {
+          pluginId: "com.acme.legacy",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn<PluginHookFanoutDeps["enqueuePluginInvoke"]>(async () => undefined);
+
+    const result = await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke });
+
+    expect(result).toMatchObject({ matched: 1, enqueued: 1 });
+    expect(enqueuePluginInvoke).toHaveBeenCalledTimes(1);
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        payload: expect.objectContaining({
+          pluginId: "com.acme.granted",
+        }),
+      }),
+      expect.any(Object)
+    );
+  });
+
+  test("skips subscribers with invalid stored manifests", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.invalid",
+          version: "1.0.0",
+          manifest: { hooks: [{ name: "tenant.created", export: "onTenantCreated" }] },
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+        {
+          pluginId: "com.acme.valid",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn<PluginHookFanoutDeps["enqueuePluginInvoke"]>(async () => undefined);
+
+    const result = await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke });
+
+    expect(result).toMatchObject({ matched: 1, enqueued: 1 });
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        payload: expect.objectContaining({
+          pluginId: "com.acme.valid",
+        }),
+      }),
+      expect.any(Object)
+    );
+  });
+
+  test("skips per-user disabled hook subscribers when payload has userId", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.user-enabled",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "user.created", export: "onUserCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:user.created" }],
+          userConfigs: [{ enabled: true }],
+        },
+        {
+          pluginId: "com.acme.user-disabled",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "user.created", export: "onUserCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:user.created" }],
+          userConfigs: [{ enabled: false }],
+        },
+        {
+          pluginId: "com.acme.tenant-default",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "user.created", export: "onUserCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:user.created" }],
+        },
+      ]),
+    };
+    const enqueuedPlugins: string[] = [];
+    const enqueuePluginInvoke = vi.fn<PluginHookFanoutDeps["enqueuePluginInvoke"]>(async (descriptor) => {
+      enqueuedPlugins.push(descriptor.payload.pluginId);
+    });
+
+    const result = await runPluginHookFanoutJob(
+      envelope({
+        eventType: "plugin.hook.user.created",
+        payload: {
+          userId: "user_1",
+          tenantId: "tenant_1",
+          email: "user@example.test",
+          locale: "en",
+          createdAt: "2026-05-26T20:00:00.000Z",
+        },
+      }),
+      { store, enqueuePluginInvoke }
+    );
+
+    expect(result).toMatchObject({ matched: 2, enqueued: 2, hookName: "user.created" });
+    expect(store.listEnabledPlugins).toHaveBeenCalledWith({ tenantId: "tenant_1", userId: "user_1" });
+    expect(enqueuedPlugins).toEqual([
+      "com.acme.user-enabled",
+      "com.acme.tenant-default",
+    ]);
+  });
+
+  test("uses stable per-subscriber idempotency keys across partial enqueue retries", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+        {
+          pluginId: "com.acme.crm",
+          version: "2.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "syncTenant" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async (_descriptor, options) => {
+      if (options.idempotencyKey === "plugin-hook:evt_1:com.acme.crm:2.0.0") {
+        throw new Error("plugin invoke queue unavailable");
+      }
+    });
+
+    await expect(runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke })).rejects.toThrow("plugin invoke queue unavailable");
+    expect(enqueuePluginInvoke.mock.calls.map(([, options]) => options.idempotencyKey)).toEqual([
+      "plugin-hook:evt_1:com.acme.audit:1.0.0",
+      "plugin-hook:evt_1:com.acme.crm:2.0.0",
+    ]);
+
+    enqueuePluginInvoke.mockReset();
+    enqueuePluginInvoke.mockResolvedValue(undefined);
+    await expect(runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke })).resolves.toMatchObject({ matched: 2, enqueued: 2 });
+    expect(enqueuePluginInvoke.mock.calls.map(([, options]) => options.idempotencyKey)).toEqual([
+      "plugin-hook:evt_1:com.acme.audit:1.0.0",
+      "plugin-hook:evt_1:com.acme.crm:2.0.0",
+    ]);
+  });
+
+  test("enqueues subscribers with bounded concurrency", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () =>
+        Array.from({ length: PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY + 3 }, (_, index) => ({
+          pluginId: `com.acme.audit-${index}`,
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        }))
+      ),
+    };
+    let active = 0;
+    let maxActive = 0;
+    const enqueuePluginInvoke = vi.fn(async () => {
+      active += 1;
+      maxActive = Math.max(maxActive, active);
+      await new Promise((resolve) => setTimeout(resolve, 0));
+      active -= 1;
+    });
+
+    const result = await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke });
+
+    expect(result).toMatchObject({
+      matched: PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY + 3,
+      enqueued: PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY + 3,
+    });
+    expect(maxActive).toBeGreaterThan(1);
+    expect(maxActive).toBeLessThanOrEqual(PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY);
+  });
+
+  test("uses dispatchKey for plugin invoke idempotency when outboxEventId is absent", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+    const result = await runPluginHookFanoutJob(envelope({ outboxEventId: undefined }), {
+      store,
+      enqueuePluginInvoke,
+    });
+
+    expect(result).toMatchObject({ matched: 1, enqueued: 1 });
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        payload: expect.objectContaining({
+          metadata: expect.objectContaining({
+            eventType: "plugin.hook.tenant.created",
+            dispatchKey: "tenant:tenant_1:created",
+          }),
+        }),
+      }),
+      expect.objectContaining({
+        tenantId: "tenant_1",
+        idempotencyKey: "plugin-hook:tenant:tenant_1:created:com.acme.audit:1.0.0",
+      })
+    );
+  });
+
+  test("wraps outbox deliveries in receiver-side idempotency when a store is provided", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+      ]),
+    };
+    const idempotency = createMemoryIdempotencyStore();
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+    const first = await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke, idempotency });
+    const second = await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke, idempotency });
+
+    expect(first).toMatchObject({ outcome: "fanned_out", matched: 1, enqueued: 1 } satisfies Partial<PluginHookFanoutResult>);
+    expect(second).toEqual(first);
+    expect(enqueuePluginInvoke).toHaveBeenCalledTimes(1);
+    expect(idempotency.claim).toHaveBeenCalledWith("outbox:evt_1");
+    expect(idempotency.complete).toHaveBeenCalledWith("outbox:evt_1", first);
+  });
+
+  test("releases the fanout idempotency claim when enqueue fails so redelivery can retry", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+      ]),
+    };
+    const idempotency = createMemoryIdempotencyStore();
+    const enqueuePluginInvoke = vi.fn(async (): Promise<void> => {
+      throw new Error("plugin invoke queue unavailable");
+    });
+
+    await expect(runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke, idempotency })).rejects.toThrow(
+      "plugin invoke queue unavailable"
+    );
+    expect(idempotency.forget).toHaveBeenCalledWith("outbox:evt_1");
+
+    enqueuePluginInvoke.mockImplementation(async () => undefined);
+    await expect(runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke, idempotency })).resolves.toMatchObject({
+      outcome: "fanned_out",
+      matched: 1,
+      enqueued: 1,
+    });
+  });
+
+  test("fans out workflow.executed without leaking workflow input, output, or raw errors", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.workflow-audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "workflow.executed", export: "onWorkflowExecuted" }]),
+          capabilityGrants: [{ capabilityName: "hook:workflow.executed" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+
+    const result = await runPluginHookFanoutJob(
+      envelope({
+        eventType: "plugin.hook.workflow.executed",
+        payload: {
+          tenantId: "tenant_1",
+          workflowId: "workflow_1",
+          workflowExecutionId: "execution_1",
+          userId: "user_1",
+          type: "trigger",
+          status: "success",
+          durationMs: 12,
+          errorCode: null,
+          appliesToAllTenants: false,
+          occurredAt: "2026-05-26T20:00:00.000Z",
+        },
+      }),
+      { store, enqueuePluginInvoke }
+    );
+
+    expect(result).toMatchObject({ matched: 1, enqueued: 1, hookName: "workflow.executed" });
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        queueName: queueNames.pluginInvoke,
+        jobName: "invoke-plugin",
+        payload: expect.objectContaining({
+          kind: "plugin.invoke",
+          tenantId: "tenant_1",
+          pluginId: "com.acme.workflow-audit",
+          version: "1.0.0",
+          hookName: "onWorkflowExecuted",
+          input: {
+            tenantId: "tenant_1",
+            workflowId: "workflow_1",
+            workflowExecutionId: "execution_1",
+            userId: "user_1",
+            type: "trigger",
+            status: "success",
+            durationMs: 12,
+            errorCode: null,
+            appliesToAllTenants: false,
+            occurredAt: "2026-05-26T20:00:00.000Z",
+          },
+          metadata: expect.objectContaining({
+            logicalHookName: "workflow.executed",
+            eventType: "plugin.hook.workflow.executed",
+          }),
+        }),
+      }),
+      expect.objectContaining({
+        tenantId: "tenant_1",
+        idempotencyKey: "plugin-hook:evt_1:com.acme.workflow-audit:1.0.0",
+      })
+    );
+    const firstCall = enqueuePluginInvoke.mock.calls.at(0) as Parameters<PluginHookFanoutDeps["enqueuePluginInvoke"]> | undefined;
+    expect(firstCall).toBeDefined();
+    const invokePayload = firstCall![0].payload;
+    expect(invokePayload.input).not.toHaveProperty("input");
+    expect(invokePayload.input).not.toHaveProperty("output");
+    expect(invokePayload.input).not.toHaveProperty("error");
+  });
+
+  test("fans out workflow.failed as failure-only metadata", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.workflow-failure",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "workflow.failed", export: "onWorkflowFailed" }]),
+          capabilityGrants: [{ capabilityName: "hook:workflow.failed" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+
+    const result = await runPluginHookFanoutJob(
+      envelope({
+        eventType: "plugin.hook.workflow.failed",
+        payload: {
+          tenantId: "tenant_1",
+          workflowId: "workflow_1",
+          workflowExecutionId: "execution_1",
+          userId: null,
+          type: "trigger",
+          status: "error",
+          durationMs: 12,
+          errorCode: "workflow_execution_failed",
+          appliesToAllTenants: false,
+          occurredAt: "2026-05-26T20:00:00.000Z",
+        },
+      }),
+      { store, enqueuePluginInvoke }
+    );
+
+    expect(result).toMatchObject({ matched: 1, enqueued: 1, hookName: "workflow.failed" });
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        payload: expect.objectContaining({
+          pluginId: "com.acme.workflow-failure",
+          hookName: "onWorkflowFailed",
+          input: expect.objectContaining({
+            status: "error",
+            userId: null,
+            errorCode: "workflow_execution_failed",
+          }),
+          metadata: expect.objectContaining({
+            logicalHookName: "workflow.failed",
+            eventType: "plugin.hook.workflow.failed",
+          }),
+        }),
+      }),
+      expect.objectContaining({
+        tenantId: "tenant_1",
+        idempotencyKey: "plugin-hook:evt_1:com.acme.workflow-failure:1.0.0",
+      })
+    );
+    const firstCall = enqueuePluginInvoke.mock.calls.at(0) as Parameters<PluginHookFanoutDeps["enqueuePluginInvoke"]> | undefined;
+    expect(firstCall).toBeDefined();
+    const invokePayload = firstCall![0].payload;
+    expect(invokePayload.input).not.toHaveProperty("input");
+    expect(invokePayload.input).not.toHaveProperty("output");
+    expect(invokePayload.input).not.toHaveProperty("error");
+  });
+
+  test.each([
+    {
+      logicalHookName: "workflow.created",
+      eventType: "plugin.hook.workflow.created",
+      exportName: "onWorkflowCreated",
+      payload: {
+        tenantId: "tenant_1",
+        workflowId: "workflow_1",
+        userId: "user_1",
+        status: "draft",
+        appliesToAllTenants: false,
+        occurredAt: "2026-05-26T20:00:00.000Z",
+      },
+    },
+    {
+      logicalHookName: "workflow.updated",
+      eventType: "plugin.hook.workflow.updated",
+      exportName: "onWorkflowUpdated",
+      payload: {
+        tenantId: "tenant_1",
+        workflowId: "workflow_1",
+        userId: "user_1",
+        status: "live",
+        appliesToAllTenants: false,
+        changedFields: ["name", "status"],
+        occurredAt: "2026-05-26T20:00:00.000Z",
+      },
+    },
+    {
+      logicalHookName: "workflow.deleted",
+      eventType: "plugin.hook.workflow.deleted",
+      exportName: "onWorkflowDeleted",
+      payload: {
+        tenantId: "tenant_1",
+        workflowId: "workflow_1",
+        userId: "user_1",
+        status: "archived",
+        appliesToAllTenants: false,
+        occurredAt: "2026-05-26T20:00:00.000Z",
+      },
+    },
+  ])("fans out $logicalHookName workflow definition hooks as metadata-only plugin invokes", async ({ logicalHookName, eventType, exportName, payload }) => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.workflow-definition-audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: logicalHookName, export: exportName }]),
+          capabilityGrants: [{ capabilityName: `hook:${logicalHookName}` }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+
+    const result = await runPluginHookFanoutJob(envelope({ eventType, payload }), { store, enqueuePluginInvoke });
+
+    expect(result).toMatchObject({ matched: 1, enqueued: 1, hookName: logicalHookName });
+    expect(enqueuePluginInvoke).toHaveBeenCalledWith(
+      expect.objectContaining({
+        queueName: queueNames.pluginInvoke,
+        jobName: "invoke-plugin",
+        payload: expect.objectContaining({
+          kind: "plugin.invoke",
+          tenantId: "tenant_1",
+          pluginId: "com.acme.workflow-definition-audit",
+          version: "1.0.0",
+          hookName: exportName,
+          input: payload,
+          metadata: expect.objectContaining({
+            logicalHookName,
+            eventType,
+          }),
+        }),
+      }),
+      expect.objectContaining({
+        tenantId: "tenant_1",
+        idempotencyKey: "plugin-hook:evt_1:com.acme.workflow-definition-audit:1.0.0",
+      })
+    );
+    const firstCall = enqueuePluginInvoke.mock.calls.at(0) as Parameters<PluginHookFanoutDeps["enqueuePluginInvoke"]> | undefined;
+    expect(firstCall).toBeDefined();
+    const invokePayload = firstCall![0].payload;
+    expect(invokePayload.input).not.toHaveProperty("blocks");
+    expect(invokePayload.input).not.toHaveProperty("credentials");
+    expect(invokePayload.input).not.toHaveProperty("input");
+    expect(invokePayload.input).not.toHaveProperty("output");
+    expect(invokePayload.input).not.toHaveProperty("error");
+  });
+
+  test("returns a zero fanout result when no plugin declares the hook", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.empty",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "entity.created", export: "onEntityCreated" }]),
+          capabilityGrants: [],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+    const result = await runPluginHookFanoutJob(envelope(), { store, enqueuePluginInvoke });
+    expect(result).toMatchObject({ matched: 0, enqueued: 0 });
+    expect(enqueuePluginInvoke).not.toHaveBeenCalled();
+  });
+
+  test("revalidates hook payloads before listing subscribers or enqueueing invokes", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "workflow.executed", export: "onWorkflowExecuted" }]),
+          capabilityGrants: [{ capabilityName: "hook:workflow.executed" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+    const logger = { warn: vi.fn() };
+
+    await expect(
+      runPluginHookFanoutJob(
+        envelope({
+          eventType: "plugin.hook.workflow.executed",
+          payload: {
+            tenantId: "tenant_1",
+            workflowId: "workflow_1",
+            workflowExecutionId: "execution_1",
+            type: "trigger",
+            status: "waitingBlock",
+            durationMs: 12,
+            errorCode: null,
+            appliesToAllTenants: false,
+            occurredAt: "2026-05-26T20:00:00.000Z",
+          },
+        }),
+        { store, enqueuePluginInvoke, logger }
+      )
+    ).rejects.toMatchObject({ code: "INVALID_PAYLOAD" });
+
+    expect(store.listEnabledPlugins).not.toHaveBeenCalled();
+    expect(enqueuePluginInvoke).not.toHaveBeenCalled();
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.objectContaining({ component: "plugin-hook-fanout", hookName: "workflow.executed", tenantId: "tenant_1" }),
+      "plugin hook fanout rejected invalid payload"
+    );
+  });
+
+  test("rejects payload tenant mismatches before listing subscribers or enqueueing invokes", async () => {
+    const store: PluginHookFanoutStore = {
+      listEnabledPlugins: vi.fn(async () => [
+        {
+          pluginId: "com.acme.audit",
+          version: "1.0.0",
+          manifest: validManifest([{ name: "tenant.created", export: "onTenantCreated" }]),
+          capabilityGrants: [{ capabilityName: "hook:tenant.created" }],
+        },
+      ]),
+    };
+    const enqueuePluginInvoke = vi.fn(async () => undefined);
+    const logger = { warn: vi.fn() };
+
+    await expect(
+      runPluginHookFanoutJob(
+        envelope({
+          tenantId: "tenant_1",
+          payload: { tenantId: "tenant_2", slug: "acme", ownerEmail: "owner@example.test", createdAt: "2026-05-26T20:00:00.000Z" },
+        }),
+        { store, enqueuePluginInvoke, logger }
+      )
+    ).rejects.toMatchObject({ code: "INVALID_PAYLOAD" });
+
+    expect(store.listEnabledPlugins).not.toHaveBeenCalled();
+    expect(enqueuePluginInvoke).not.toHaveBeenCalled();
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.objectContaining({ component: "plugin-hook-fanout", hookName: "tenant.created", tenantId: "tenant_1", payloadTenantId: "tenant_2" }),
+      "plugin hook fanout rejected tenant mismatch"
+    );
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-hook-fanout.ts

@@ -0,0 +1,224 @@
+import { buildPluginInvokeJob, type QueueEnqueueOptions } from "@toolctrl/queue";
+import { getPluginHook, isPluginHookProductionAvailable, parsePluginManifest, validatePluginHookPayload } from "@toolctrl/plugin-runtime";
+import { withOutboxIdempotency, type OutboxReceiverIdempotencyStore } from "@toolctrl/db/outbox-receiver";
+
+export const PLUGIN_HOOK_FANOUT_OUTBOX_QUEUE = "outbox.plugin-hook" as const;
+export const PLUGIN_HOOK_EVENT_PREFIX = "plugin.hook." as const;
+export const PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY = 5;
+
+export interface PluginHookFanoutLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export type PluginHookFanoutJobData = {
+  readonly outboxEventId?: string;
+  readonly tenantId: string;
+  readonly eventType: `plugin.hook.${string}`;
+  readonly hookName: string;
+  readonly payload: unknown;
+  readonly dispatchKey?: string | null;
+};
+
+export type PluginHookSubscriber = {
+  readonly pluginId: string;
+  readonly version: string;
+  readonly manifest: unknown;
+  readonly capabilityGrants?: ReadonlyArray<{ readonly capabilityName: string }>;
+  readonly userConfigs?: ReadonlyArray<{ readonly enabled: boolean }>;
+};
+
+type PluginHookFanoutSubscriber = PluginHookSubscriber & {
+  readonly hookExportName: string;
+};
+
+export interface PluginHookFanoutStore {
+  listEnabledPlugins(args: { tenantId: string; userId?: string }): Promise<ReadonlyArray<PluginHookSubscriber>>;
+}
+
+export interface PluginHookFanoutDeps {
+  readonly store: PluginHookFanoutStore;
+  readonly enqueuePluginInvoke: (
+    descriptor: ReturnType<typeof buildPluginInvokeJob>,
+    options: QueueEnqueueOptions
+  ) => Promise<unknown>;
+  readonly idempotency?: OutboxReceiverIdempotencyStore & { forget?(key: string): Promise<void> };
+  readonly logger?: PluginHookFanoutLogger;
+}
+
+export class PluginHookFanoutJobError extends Error {
+  override readonly name = "PluginHookFanoutJobError";
+  readonly code: "INVALID_PAYLOAD" | "UNSUPPORTED_EVENT";
+  constructor(message: string, code: PluginHookFanoutJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export type PluginHookFanoutResult = {
+  outcome: "fanned_out" | "skipped";
+  hookName: string;
+  tenantId: string;
+  matched: number;
+  enqueued: number;
+};
+
+export function parsePluginHookFanoutJob(data: unknown): PluginHookFanoutJobData {
+  if (!data || typeof data !== "object") {
+    throw new PluginHookFanoutJobError("plugin hook fanout job missing envelope", "INVALID_PAYLOAD");
+  }
+  const envelope = data as Record<string, unknown>;
+  const eventType = typeof envelope.eventType === "string" ? envelope.eventType : "";
+  if (!eventType.startsWith(PLUGIN_HOOK_EVENT_PREFIX)) {
+    throw new PluginHookFanoutJobError(`unsupported plugin hook eventType ${eventType}`, "UNSUPPORTED_EVENT");
+  }
+  const hookName = eventType.slice(PLUGIN_HOOK_EVENT_PREFIX.length);
+  if (!/^[a-z][a-z0-9]*(\.[a-z][a-z0-9]*)+$/u.test(hookName)) {
+    throw new PluginHookFanoutJobError(`invalid plugin hook name ${hookName}`, "UNSUPPORTED_EVENT");
+  }
+  if (!getPluginHook(hookName)) {
+    throw new PluginHookFanoutJobError(`unsupported plugin hook name ${hookName}`, "UNSUPPORTED_EVENT");
+  }
+  if (process.env.NODE_ENV === "production" && !isPluginHookProductionAvailable(hookName)) {
+    throw new PluginHookFanoutJobError(`plugin hook name ${hookName} is not production-available`, "UNSUPPORTED_EVENT");
+  }
+  const tenantId = typeof envelope.tenantId === "string" ? envelope.tenantId : "";
+  if (tenantId.length === 0) {
+    throw new PluginHookFanoutJobError("plugin hook fanout requires tenantId", "INVALID_PAYLOAD");
+  }
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId,
+    eventType: eventType as `plugin.hook.${string}`,
+    hookName,
+    payload: envelope.payload,
+    dispatchKey: typeof envelope.dispatchKey === "string" ? envelope.dispatchKey : null,
+  };
+}
+
+function getManifestHookExportName(manifest: unknown, hookName: string): string | null {
+  const parsed = parsePluginManifest(manifest);
+  if (!parsed.ok) return null;
+  return parsed.manifest.hooks.find((hook) => hook.name === hookName)?.export ?? null;
+}
+
+function hasHookConsentGrant(plugin: PluginHookSubscriber, hookName: string): boolean {
+  return plugin.capabilityGrants?.some((grant) => grant.capabilityName === `hook:${hookName}`) ?? false;
+}
+
+function isEnabledForUser(plugin: PluginHookSubscriber): boolean {
+  return plugin.userConfigs?.[0]?.enabled !== false;
+}
+
+function getPayloadUserId(payload: unknown): string | undefined {
+  if (!payload || typeof payload !== "object") return undefined;
+  const userId = (payload as { userId?: unknown }).userId;
+  return typeof userId === "string" && userId.length > 0 ? userId : undefined;
+}
+
+function getPayloadTenantId(payload: unknown): string | undefined {
+  if (!payload || typeof payload !== "object") return undefined;
+  const tenantId = (payload as { tenantId?: unknown }).tenantId;
+  return typeof tenantId === "string" && tenantId.length > 0 ? tenantId : undefined;
+}
+
+function invokeIdempotencyKey(job: PluginHookFanoutJobData, plugin: PluginHookSubscriber): string {
+  const source = job.outboxEventId ?? job.dispatchKey ?? `${job.tenantId}:${job.eventType}`;
+  return `plugin-hook:${source}:${plugin.pluginId}:${plugin.version}`;
+}
+
+export async function runPluginHookFanoutJob(
+  data: unknown,
+  deps: PluginHookFanoutDeps
+): Promise<PluginHookFanoutResult> {
+  const job = parsePluginHookFanoutJob(data);
+  if (deps.idempotency && job.outboxEventId) {
+    const outcome = await withOutboxIdempotency(deps.idempotency, job.outboxEventId, () => fanoutPluginHookJob(job, deps));
+    return isPluginHookFanoutResult(outcome.result)
+      ? outcome.result
+      : { outcome: "skipped", hookName: job.hookName, tenantId: job.tenantId, matched: 0, enqueued: 0 };
+  }
+
+  return fanoutPluginHookJob(job, deps);
+}
+
+async function fanoutPluginHookJob(job: PluginHookFanoutJobData, deps: PluginHookFanoutDeps): Promise<PluginHookFanoutResult> {
+  const hook = getPluginHook(job.hookName);
+  if (!hook) {
+    throw new PluginHookFanoutJobError(`unsupported plugin hook name ${job.hookName}`, "UNSUPPORTED_EVENT");
+  }
+  if (process.env.NODE_ENV === "production" && !isPluginHookProductionAvailable(job.hookName)) {
+    throw new PluginHookFanoutJobError(`plugin hook name ${job.hookName} is not production-available`, "UNSUPPORTED_EVENT");
+  }
+  const validation = validatePluginHookPayload(hook, job.payload);
+  if (!validation.ok) {
+    deps.logger?.warn?.(
+      { component: "plugin-hook-fanout", hookName: job.hookName, tenantId: job.tenantId, issues: validation.issues },
+      "plugin hook fanout rejected invalid payload"
+    );
+    throw new PluginHookFanoutJobError(`invalid plugin hook payload for ${job.hookName}`, "INVALID_PAYLOAD");
+  }
+  const payloadTenantId = getPayloadTenantId(validation.payload);
+  if (payloadTenantId !== job.tenantId) {
+    deps.logger?.warn?.(
+      { component: "plugin-hook-fanout", hookName: job.hookName, tenantId: job.tenantId, payloadTenantId },
+      "plugin hook fanout rejected tenant mismatch"
+    );
+    throw new PluginHookFanoutJobError(`plugin hook payload tenant mismatch for ${job.hookName}`, "INVALID_PAYLOAD");
+  }
+  const userId = getPayloadUserId(validation.payload);
+  const candidates = await deps.store.listEnabledPlugins({ tenantId: job.tenantId, userId });
+  const subscribers: PluginHookFanoutSubscriber[] = candidates.flatMap((plugin) => {
+    const hookExportName = getManifestHookExportName(plugin.manifest, job.hookName);
+    return hookExportName && hasHookConsentGrant(plugin, job.hookName) && isEnabledForUser(plugin) ? [{ ...plugin, hookExportName }] : [];
+  });
+
+  let enqueued = 0;
+  for (let index = 0; index < subscribers.length; index += PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY) {
+    const batch = subscribers.slice(index, index + PLUGIN_HOOK_FANOUT_ENQUEUE_CONCURRENCY);
+    const results = await Promise.allSettled(
+      batch.map((plugin) => {
+        const descriptor = buildPluginInvokeJob({
+          tenantId: job.tenantId,
+          pluginId: plugin.pluginId,
+          version: plugin.version,
+          hookName: plugin.hookExportName,
+          input: validation.payload,
+          metadata: {
+            outboxEventId: job.outboxEventId,
+            eventType: job.eventType,
+            dispatchKey: job.dispatchKey ?? undefined,
+            logicalHookName: job.hookName,
+          },
+        });
+        return deps.enqueuePluginInvoke(descriptor, {
+          tenantId: job.tenantId,
+          idempotencyKey: invokeIdempotencyKey(job, plugin),
+        });
+      })
+    );
+    const failed = results.find((result): result is PromiseRejectedResult => result.status === "rejected");
+    if (failed) {
+      throw failed.reason;
+    }
+    enqueued += results.length;
+  }
+
+  deps.logger?.info?.(
+    { component: "plugin-hook-fanout", hookName: job.hookName, tenantId: job.tenantId, matched: subscribers.length, enqueued },
+    "plugin hook fanout completed"
+  );
+  return { outcome: "fanned_out", hookName: job.hookName, tenantId: job.tenantId, matched: subscribers.length, enqueued };
+}
+
+function isPluginHookFanoutResult(value: unknown): value is PluginHookFanoutResult {
+  if (!value || typeof value !== "object") return false;
+  const result = value as Partial<PluginHookFanoutResult>;
+  return (
+    (result.outcome === "fanned_out" || result.outcome === "skipped") &&
+    typeof result.hookName === "string" &&
+    typeof result.tenantId === "string" &&
+    typeof result.matched === "number" &&
+    typeof result.enqueued === "number"
+  );
+}

hivegrid-cloud/apps/worker/src/jobs/plugin-invoke-bootstrap.test.ts

@@ -0,0 +1,899 @@
+/**
+ * Phase 06 #21 / F-015 — bootstrapPluginInvokeDeps wiring tests.
+ *
+ * Verifies that the production composition wires Prisma, SharedStore,
+ * env, and logger into a valid PluginInvokeDeps record. The deeper
+ * end-to-end behaviour of `handlePluginInvokeJob` is covered by
+ * `plugin-invoke.test.ts`; here we only assert the composition itself.
+ */
+
+import { describe, expect, test, vi } from "vitest";
+
+import { bootstrapPluginInvokeDeps, type PluginInvokeBootstrapPrismaSurface } from "./plugin-invoke-bootstrap.js";
+
+function makeFakeStore() {
+  const counters = new Map<string, number>();
+  const ttls = new Map<string, number>();
+  const values = new Map<string, string>();
+  return {
+    namespace: "test",
+    ttlDefault: undefined,
+    adapter: {} as never,
+    key: (s: string) => `test:${s}`,
+    incr: vi.fn(async (key: string, by = 1) => {
+      const next = (counters.get(key) ?? 0) + by;
+      counters.set(key, next);
+      return next;
+    }),
+    expire: vi.fn(async (key: string, ttlSeconds: number) => {
+      ttls.set(key, ttlSeconds);
+      return true;
+    }),
+    get: vi.fn(async (key: string) => values.get(key) ?? null),
+    set: vi.fn(async (key: string, value: string) => {
+      values.set(key, value);
+      return true;
+    }),
+    del: vi.fn(async (key: string) => (counters.delete(key) ? 1 : 0)),
+    mget: vi.fn(async () => []),
+    scan: vi.fn(async function* () {}),
+    publish: vi.fn(async () => 0),
+    subscribe: vi.fn(async () => async () => undefined),
+    zAdd: vi.fn(async () => 0),
+    zRemRangeByScore: vi.fn(async () => 0),
+    zCard: vi.fn(async () => 0),
+    eval: vi.fn(async () => undefined),
+    close: vi.fn(async () => undefined),
+    counters,
+    ttls,
+  };
+}
+
+type FakePluginRow = {
+  tenantId: string;
+  pluginId: string;
+  version: string;
+  manifest: unknown;
+  wasmSha256: string;
+  signaturePublicKey: string;
+  enabled: boolean;
+};
+
+function makeFakePrisma() {
+  return {
+    plugin: {
+      findUnique: vi.fn<(args: unknown) => Promise<FakePluginRow | null>>(async () => null),
+      update: vi.fn<(args: unknown) => Promise<unknown>>(async () => ({})),
+    },
+    outboxEvent: {
+      create: vi.fn<(args: unknown) => Promise<unknown>>(async () => ({})),
+      findUnique: vi.fn<(args: unknown) => Promise<unknown | null>>(async () => null),
+    },
+    pluginInvocation: {
+      create: vi.fn<(args: unknown) => Promise<unknown>>(async () => ({})),
+    },
+    tenantPluginCapabilityGrant: {
+      findMany: vi.fn<(args: unknown) => Promise<Array<{ capabilityName: string; scopeJson: unknown }>>>(async () => []),
+    },
+  };
+}
+
+const noopLogger = {
+  info: vi.fn(),
+  warn: vi.fn(),
+  error: vi.fn(),
+};
+
+const HOST_IMPORT_WASM = new Uint8Array([
+  0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
+  0x01, 0x0a, 0x02, 0x60, 0x01, 0x7f, 0x00, 0x60, 0x01, 0x7f, 0x01, 0x7f,
+  0x02, 0x30, 0x02,
+  0x03, 0x65, 0x6e, 0x76, 0x0c, 0x74, 0x6f, 0x6f, 0x6c, 0x63, 0x74, 0x72, 0x6c, 0x5f,
+  0x6c, 0x6f, 0x67, 0x00, 0x00,
+  0x03, 0x65, 0x6e, 0x76, 0x15, 0x74, 0x6f, 0x6f, 0x6c, 0x63, 0x74, 0x72, 0x6c, 0x5f,
+  0x61, 0x75, 0x64, 0x69, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x00, 0x00,
+  0x03, 0x02, 0x01, 0x01,
+  0x07, 0x07, 0x01, 0x03, 0x72, 0x75, 0x6e, 0x00, 0x02,
+  0x0a, 0x0e, 0x01, 0x0c, 0x00, 0x20, 0x00, 0x10, 0x00, 0x20, 0x00, 0x10, 0x01, 0x20,
+  0x00, 0x0b,
+]);
+
+const INFINITE_LOOP_WASM = new Uint8Array([
+  0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
+  0x01, 0x04, 0x01, 0x60, 0x00, 0x00,
+  0x03, 0x02, 0x01, 0x00,
+  0x07, 0x07, 0x01, 0x03, 0x72, 0x75, 0x6e, 0x00, 0x00,
+  0x0a, 0x09, 0x01, 0x07, 0x00, 0x03, 0x40, 0x0c, 0x00, 0x0b, 0x0b,
+]);
+
+function manifestWithCapabilities(capabilities: Array<{ name: "log" | "audit" }>) {
+  return {
+    schemaVersion: 1 as const,
+    id: "de.toolctrl.test",
+    version: "1.0.0",
+    name: "Test",
+    entry: "plugin.wasm",
+    wasmSha256: "0".repeat(64),
+    signature: "0".repeat(128),
+    signedBy: "0".repeat(64),
+    allowedHosts: [],
+    capabilities,
+    hooks: [{ name: "run", export: "run" }],
+  };
+}
+
+describe("bootstrapPluginInvokeDeps", () => {
+  test("fails closed in production until the plugin runtime rollout gate is enabled", () => {
+    expect(() =>
+      bootstrapPluginInvokeDeps({
+        env: { NODE_ENV: "production" },
+        prisma: makeFakePrisma(),
+        store: makeFakeStore() as never,
+        logger: noopLogger,
+      })
+    ).toThrow("PLUGIN_RUNTIME_PRODUCTION_ENABLED=true");
+  });
+
+  test("returns a record with every PluginInvokeDeps seam populated", () => {
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_TRUSTED_PUBLIC_KEYS: "aa".repeat(32) },
+      prisma: makeFakePrisma(),
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+    expect(typeof deps.loadPlugin).toBe("function");
+    expect(typeof deps.loadCapabilityGrants).toBe("function");
+    expect(typeof deps.loadBinary).toBe("function");
+    expect(typeof deps.verifySignature).toBe("function");
+    expect(typeof deps.consumeRateLimit).toBe("function");
+    expect(typeof deps.invokeInSandbox).toBe("function");
+    expect(typeof deps.recordAudit).toBe("function");
+    expect(typeof deps.recordOutcomeForCrashTracking).toBe("function");
+    expect(Array.isArray(deps.trustedPublicKeysHex)).toBe(true);
+    expect(deps.trustedPublicKeysHex).toEqual(["aa".repeat(32)]);
+    expect(deps.logger).toBe(noopLogger);
+  });
+
+  test("loadCapabilityGrants returns tenant plugin grants for runtime enforcement", async () => {
+    const prisma = makeFakePrisma();
+    prisma.tenantPluginCapabilityGrant.findMany.mockResolvedValue([
+      { capabilityName: "log", scopeJson: null },
+      { capabilityName: "http.outbound", scopeJson: { allowedHosts: ["api.example.com"] } },
+    ]);
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+
+    await expect(deps.loadCapabilityGrants({ tenantId: "tenant-1", pluginId: "de.toolctrl.test" })).resolves.toEqual([
+      { capabilityName: "log", scopeJson: null },
+      { capabilityName: "http.outbound", scopeJson: { allowedHosts: ["api.example.com"] } },
+    ]);
+    expect(prisma.tenantPluginCapabilityGrant.findMany).toHaveBeenCalledWith({
+      where: { tenantId: "tenant-1", pluginId: "de.toolctrl.test" },
+      select: { capabilityName: true, scopeJson: true },
+    });
+  });
+
+  test("loadPlugin returns null when the row is missing", async () => {
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+    const row = await deps.loadPlugin({ tenantId: "t", pluginId: "p", version: "1.0.0" });
+    expect(row).toBeNull();
+    expect(prisma.plugin.findUnique).toHaveBeenCalledWith({
+      where: { tenantId_pluginId: { tenantId: "t", pluginId: "p" } },
+    });
+  });
+
+  test("loadPlugin returns null when the stored row's version does not match", async () => {
+    const prisma = makeFakePrisma();
+    prisma.plugin.findUnique = vi.fn(async () => ({
+      tenantId: "t",
+      pluginId: "p",
+      version: "2.0.0",
+      manifest: {},
+      wasmSha256: "x",
+      signaturePublicKey: "y",
+      enabled: true,
+    }));
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+    expect(await deps.loadPlugin({ tenantId: "t", pluginId: "p", version: "1.0.0" })).toBeNull();
+  });
+
+  test("loadPlugin returns parsed rows only when the stored manifest is valid", async () => {
+    const prisma = makeFakePrisma();
+    const manifest = manifestWithCapabilities([{ name: "log" }]);
+    prisma.plugin.findUnique = vi.fn(async () => ({
+      tenantId: "t",
+      pluginId: "de.toolctrl.test",
+      version: "1.0.0",
+      manifest,
+      wasmSha256: "0".repeat(64),
+      signaturePublicKey: "0".repeat(64),
+      enabled: true,
+    }));
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+
+    await expect(deps.loadPlugin({ tenantId: "t", pluginId: "de.toolctrl.test", version: "1.0.0" })).resolves.toMatchObject({
+      tenantId: "t",
+      pluginId: "de.toolctrl.test",
+      manifest,
+      enabled: true,
+    });
+  });
+
+  test("loadPlugin returns null and warns when the stored manifest is invalid", async () => {
+    const prisma = makeFakePrisma();
+    prisma.plugin.findUnique = vi.fn(async () => ({
+      tenantId: "t",
+      pluginId: "de.toolctrl.test",
+      version: "1.0.0",
+      manifest: { hooks: [{ name: "run", export: "run" }] },
+      wasmSha256: "0".repeat(64),
+      signaturePublicKey: "0".repeat(64),
+      enabled: true,
+    }));
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    await expect(deps.loadPlugin({ tenantId: "t", pluginId: "de.toolctrl.test", version: "1.0.0" })).resolves.toBeNull();
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.objectContaining({
+        tenantId: "t",
+        pluginId: "de.toolctrl.test",
+        version: "1.0.0",
+      }),
+      "plugin.invoke.stored_manifest_invalid"
+    );
+  });
+
+  test("loadBinary returns null on ENOENT instead of throwing", async () => {
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_BINARY_BASE_PATH: "/tmp/does-not-exist-for-tests" },
+      prisma: makeFakePrisma(),
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+    const buf = await deps.loadBinary({ pluginId: "ghost", version: "1.0.0" });
+    expect(buf).toBeNull();
+  });
+
+  test("loadBinary refuses plugin paths that escape the configured base directory", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_BINARY_BASE_PATH: "/tmp/toolctrl-plugins" },
+      prisma: makeFakePrisma(),
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const buf = await deps.loadBinary({ pluginId: "../outside", version: "../../1.0.0" });
+
+    expect(buf).toBeNull();
+    expect(logger.warn).toHaveBeenCalledWith(
+      { pluginId: "../outside", version: "../../1.0.0" },
+      "plugin.invoke.binary_path_escape_blocked"
+    );
+  });
+
+  test("consumeRateLimit allows the first N calls then rejects", async () => {
+    const store = makeFakeStore();
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_RATE_LIMIT_PER_MINUTE: "3", PLUGIN_RATE_LIMIT_WINDOW_SECONDS: "60" },
+      prisma: makeFakePrisma(),
+      store: store as never,
+      logger: noopLogger,
+    });
+    const first = await deps.consumeRateLimit({ tenantId: "t", pluginId: "p" });
+    const second = await deps.consumeRateLimit({ tenantId: "t", pluginId: "p" });
+    const third = await deps.consumeRateLimit({ tenantId: "t", pluginId: "p" });
+    const fourth = await deps.consumeRateLimit({ tenantId: "t", pluginId: "p" });
+    expect([first.allowed, second.allowed, third.allowed, fourth.allowed]).toEqual([true, true, true, false]);
+    // First hit must seed the TTL so the window slides from the first hit.
+    expect(store.expire).toHaveBeenCalledTimes(1);
+  });
+
+  test("invokeInSandbox wires log and audit host imports through declared capabilities", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const capabilities = [{ name: "log" as const }, { name: "audit" as const }];
+    const result = await deps.invokeInSandbox({
+      pluginInvocationId: "inv-1",
+      tenantId: "tenant-1",
+      pluginId: "de.toolctrl.test",
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifestWithCapabilities(capabilities),
+      hookName: "run",
+      payload: 7,
+      capabilityGrants: capabilities.map((capability) => ({ capabilityName: capability.name, scopeJson: null })),
+    });
+
+    expect(result).toMatchObject({ ok: true, output: 7 });
+    expect(prisma.outboxEvent.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        tenantId: "tenant-1",
+        eventType: "audit.event.create",
+        dispatchKey: "plugin-host-audit:inv-1:0",
+        priority: 20,
+        payload: expect.objectContaining({
+          tenantId: "tenant-1",
+          actorId: "de.toolctrl.test",
+          actorType: "plugin",
+          action: "plugin.audit.recorded",
+          targetType: "plugin",
+          targetId: "de.toolctrl.test",
+          requestId: "inv-1",
+          sourceModule: "worker.plugin-runtime",
+          outcome: "success",
+          dataSensitivity: "internal",
+          regulatoryScope: ["soc2"],
+          metadata: expect.objectContaining({
+            pluginId: "de.toolctrl.test",
+            pluginManifestId: "de.toolctrl.test",
+            pluginInvocationId: "inv-1",
+            version: "1.0.0",
+            hookName: "run",
+            hostBinding: "audit",
+            sequence: 0,
+            value: { value: 7 },
+          }),
+        }),
+      }),
+    });
+    expect(prisma.outboxEvent.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        tenantId: "tenant-1",
+        eventType: "plugin.capability-use.batch",
+        dispatchKey: expect.stringMatching(/^plugin-capability-use:/),
+        priority: 10,
+      }),
+    });
+    const outboxArgs = prisma.outboxEvent.create.mock.calls
+      .map((call) => call[0] as { data: { eventType: string; payload: { entries?: Array<{ capability: string; outcome: string }> } } })
+      .find((call) => call.data.eventType === "plugin.capability-use.batch");
+    expect(outboxArgs).toBeDefined();
+    expect(outboxArgs?.data.payload.entries).toHaveLength(2);
+    expect(outboxArgs?.data.payload.entries?.map((entry) => [entry.capability, entry.outcome])).toEqual([
+      ["log", "success"],
+      ["audit", "success"],
+    ]);
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        pluginId: "de.toolctrl.test",
+        value: { value: 7 },
+        version: "1.0.0",
+      }),
+      "plugin.invoke.host.log"
+    );
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        pluginId: "de.toolctrl.test",
+        value: { value: 7 },
+        version: "1.0.0",
+      }),
+      "plugin.invoke.host.audit"
+    );
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        tenantId: "tenant-1",
+        pluginId: "de.toolctrl.test",
+        pluginInvocationId: "inv-1",
+        sequence: 0,
+      }),
+      "plugin.invoke.host.audit_enqueued"
+    );
+  });
+
+  test("invokeInSandbox fails closed when capability-use audit cannot be enqueued", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    prisma.outboxEvent.create.mockRejectedValueOnce(new Error("outbox down"));
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const capabilities = [{ name: "log" as const }];
+    await expect(
+      deps.invokeInSandbox({
+        pluginInvocationId: "inv-audit-down",
+        tenantId: "tenant-1",
+        pluginId: "de.toolctrl.test",
+        wasmBuffer: HOST_IMPORT_WASM,
+        manifest: manifestWithCapabilities(capabilities),
+        hookName: "run",
+        payload: 7,
+        capabilityGrants: capabilities.map((capability) => ({ capabilityName: capability.name, scopeJson: null })),
+      })
+    ).rejects.toThrow("outbox down");
+    expect(logger.error).toHaveBeenCalledWith(
+      expect.objectContaining({
+        pluginInvocationId: "inv-audit-down",
+        tenantId: "tenant-1",
+        pluginId: "de.toolctrl.test",
+        entryCount: expect.any(Number),
+      }),
+      "plugin capability-use audit flush failed"
+    );
+  });
+
+  test("invokeInSandbox fails closed when capability-use audit entries are truncated", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES: "1" },
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const capabilities = [{ name: "log" as const }, { name: "audit" as const }];
+    await expect(
+      deps.invokeInSandbox({
+        pluginInvocationId: "inv-audit-truncated",
+        tenantId: "tenant-1",
+        pluginId: "de.toolctrl.test",
+        wasmBuffer: HOST_IMPORT_WASM,
+        manifest: manifestWithCapabilities(capabilities),
+        hookName: "run",
+        payload: 7,
+        capabilityGrants: capabilities.map((capability) => ({ capabilityName: capability.name, scopeJson: null })),
+      })
+    ).rejects.toThrow("Plugin capability-use audit dropped 1 entries after maxEntries=1");
+    expect(prisma.outboxEvent.create).not.toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({
+          type: "plugin.capability-use.batch",
+        }),
+      })
+    );
+  });
+
+  test("invokeInSandbox treats duplicate capability-use dispatch keys as already enqueued", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    prisma.outboxEvent.create
+      .mockResolvedValueOnce({})
+      .mockRejectedValueOnce(Object.assign(new Error("Unique constraint failed"), { code: "P2002" }));
+    prisma.outboxEvent.findUnique.mockResolvedValueOnce({
+      id: "outbox-existing",
+      dispatchKey: "plugin-capability-use:inv-duplicate",
+    });
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const capabilities = [{ name: "log" as const }, { name: "audit" as const }];
+    const result = await deps.invokeInSandbox({
+      pluginInvocationId: "inv-duplicate",
+      tenantId: "tenant-1",
+      pluginId: "de.toolctrl.test",
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifestWithCapabilities(capabilities),
+      hookName: "run",
+      payload: 7,
+      capabilityGrants: capabilities.map((capability) => ({ capabilityName: capability.name, scopeJson: null })),
+    });
+
+    expect(result).toMatchObject({ ok: true, output: 7 });
+    expect(prisma.outboxEvent.findUnique).toHaveBeenCalledWith({
+      where: { dispatchKey: "plugin-capability-use:inv-duplicate" },
+    });
+    expect(logger.error).not.toHaveBeenCalledWith(
+      expect.objectContaining({
+        pluginInvocationId: "inv-duplicate",
+      }),
+      "plugin capability-use audit flush failed"
+    );
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        pluginInvocationId: "inv-duplicate",
+        entryCount: 2,
+      }),
+      "plugin.invoke.capability_use_enqueued"
+    );
+  });
+
+  test("invokeInSandbox treats duplicate host-audit dispatch keys as already enqueued", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    prisma.outboxEvent.create.mockRejectedValueOnce(Object.assign(new Error("Unique constraint failed"), { code: "P2002" }));
+    prisma.outboxEvent.findUnique.mockResolvedValueOnce({
+      id: "outbox-existing",
+      dispatchKey: "plugin-host-audit:inv-host-audit-duplicate:0",
+    });
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const capabilities = [{ name: "log" as const }, { name: "audit" as const }];
+    const result = await deps.invokeInSandbox({
+      pluginInvocationId: "inv-host-audit-duplicate",
+      tenantId: "tenant-1",
+      pluginId: "de.toolctrl.test",
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifestWithCapabilities(capabilities),
+      hookName: "run",
+      payload: 7,
+      capabilityGrants: capabilities.map((capability) => ({ capabilityName: capability.name, scopeJson: null })),
+    });
+
+    expect(result).toMatchObject({ ok: true, output: 7 });
+    expect(prisma.outboxEvent.findUnique).toHaveBeenCalledWith({
+      where: { dispatchKey: "plugin-host-audit:inv-host-audit-duplicate:0" },
+    });
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        pluginInvocationId: "inv-host-audit-duplicate",
+        sequence: 0,
+      }),
+      "plugin.invoke.host.audit_enqueued"
+    );
+    expect(prisma.outboxEvent.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        eventType: "plugin.capability-use.batch",
+        dispatchKey: "plugin-capability-use:inv-host-audit-duplicate",
+      }),
+    });
+  });
+
+  test("invokeInSandbox denies undeclared host imports through the gateway", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const capabilities = [{ name: "log" as const }];
+    const result = await deps.invokeInSandbox({
+      pluginInvocationId: "inv-2",
+      tenantId: "tenant-1",
+      pluginId: "de.toolctrl.test",
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifestWithCapabilities(capabilities),
+      hookName: "run",
+      payload: 7,
+      capabilityGrants: capabilities.map((capability) => ({ capabilityName: capability.name, scopeJson: null })),
+    });
+
+    expect(result.ok).toBe(false);
+    expect(prisma.outboxEvent.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        tenantId: "tenant-1",
+        eventType: "plugin.capability-use.batch",
+      }),
+    });
+    const outboxArgs = prisma.outboxEvent.create.mock.calls[0]?.[0] as { data: { payload: { entries: Array<{ capability: string; outcome: string; errorCode?: string }> } } };
+    expect(outboxArgs.data.payload.entries).toEqual([
+      expect.objectContaining({
+        capability: "log",
+        outcome: "success",
+      }),
+      expect.objectContaining({
+        capability: "audit",
+        outcome: "denied",
+        errorCode: "not_declared_in_manifest",
+      }),
+    ]);
+    if (!result.ok) {
+      expect(result.reason).toBe("trap");
+      expect(result.message).toContain("capability denied: audit");
+    }
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.objectContaining({
+        capability: "audit",
+        reason: "not_declared_in_manifest",
+      }),
+      "plugin.invoke.capability_denied"
+    );
+  });
+
+  test("invokeInSandbox denies manifest-declared host imports without tenant grants", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const capabilities = [{ name: "log" as const }, { name: "audit" as const }];
+    const result = await deps.invokeInSandbox({
+      pluginInvocationId: "inv-missing-grant",
+      tenantId: "tenant-1",
+      pluginId: "de.toolctrl.test",
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifestWithCapabilities(capabilities),
+      hookName: "run",
+      payload: 7,
+      capabilityGrants: [{ capabilityName: "log", scopeJson: null }],
+    });
+
+    expect(result.ok).toBe(false);
+    const outboxArgs = prisma.outboxEvent.create.mock.calls[0]?.[0] as { data: { payload: { entries: Array<{ capability: string; outcome: string; errorCode?: string }> } } };
+    expect(outboxArgs.data.payload.entries).toEqual([
+      expect.objectContaining({
+        capability: "log",
+        outcome: "success",
+      }),
+      expect.objectContaining({
+        capability: "audit",
+        outcome: "denied",
+        errorCode: "not_granted_for_tenant",
+      }),
+    ]);
+    if (!result.ok) {
+      expect(result.reason).toBe("trap");
+      expect(result.message).toContain("capability denied: audit");
+    }
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.objectContaining({
+        capability: "audit",
+        reason: "not_granted_for_tenant",
+      }),
+      "plugin.invoke.capability_denied"
+    );
+  });
+
+  test("invokeInSandbox enforces PLUGIN_MAX_WALL_CLOCK_MS", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_MAX_WALL_CLOCK_MS: "50" },
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    const result = await deps.invokeInSandbox({
+      pluginInvocationId: "inv-timeout",
+      tenantId: "tenant-1",
+      pluginId: "de.toolctrl.test",
+      wasmBuffer: INFINITE_LOOP_WASM,
+      manifest: manifestWithCapabilities([]),
+      hookName: "run",
+      payload: undefined,
+      capabilityGrants: [],
+    });
+
+    expect(result).toMatchObject({
+      ok: false,
+      reason: "timeout",
+      message: "plugin sandbox wall-clock timeout",
+    });
+    expect(prisma.outboxEvent.create).not.toHaveBeenCalled();
+  }, 5_000);
+
+  test("recordAudit success path logs at info level, failure at error", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+    await deps.recordAudit({
+      tenantId: "t",
+      pluginId: "p",
+      version: "1.0.0",
+      hookName: "tenant.created",
+      pluginInvocationId: "inv-success",
+      outcome: { ok: true, output: 1, fuelUsed: 0, durationMs: 5, pluginInvocationId: "inv-success" },
+    });
+    expect(prisma.pluginInvocation.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        id: "inv-success",
+        outcome: "success",
+        failureReason: null,
+        fuelUsed: 0,
+      }),
+    });
+    expect(logger.info).toHaveBeenCalledWith(expect.any(Object), "plugin.invoke.audit.success");
+    await deps.recordAudit({
+      tenantId: "t",
+      pluginId: "p",
+      version: "1.0.0",
+      hookName: "tenant.created",
+      pluginInvocationId: "inv-failure",
+      outcome: { ok: false, reason: "rate_limit_exceeded", durationMs: 1, pluginInvocationId: "inv-failure" },
+    });
+    expect(prisma.pluginInvocation.create).toHaveBeenLastCalledWith({
+      data: expect.objectContaining({
+        id: "inv-failure",
+        outcome: "failure",
+        failureReason: "rate_limit_exceeded",
+        fuelUsed: null,
+      }),
+    });
+    expect(logger.error).toHaveBeenCalledWith(expect.any(Object), "plugin.invoke.audit.failure");
+  });
+
+  test("recordOutcomeForCrashTracking records success vs failure on the tracker", async () => {
+    const store = makeFakeStore();
+    const deps = bootstrapPluginInvokeDeps({
+      env: {},
+      prisma: makeFakePrisma(),
+      store: store as never,
+      logger: noopLogger,
+    });
+    await deps.recordOutcomeForCrashTracking!({
+      tenantId: "t",
+      pluginId: "p",
+      outcome: { ok: false, reason: "sandbox_failed", durationMs: 0, pluginInvocationId: "inv-failure" },
+    });
+    expect(store.incr).toHaveBeenCalled();
+    await deps.recordOutcomeForCrashTracking!({
+      tenantId: "t",
+      pluginId: "p",
+      outcome: { ok: true, output: undefined, fuelUsed: 0, durationMs: 0, pluginInvocationId: "inv-success" },
+    });
+    // Success decrement reads + writes the counter.
+    expect(store.get).toHaveBeenCalled();
+  });
+
+  test("recordOutcomeForCrashTracking persists plugin auto-disable when the threshold is crossed", async () => {
+    const prisma = makeFakePrisma();
+    const transaction = vi.fn<NonNullable<PluginInvokeBootstrapPrismaSurface["$transaction"]>>(async (fn) => fn(prisma));
+    const prismaWithTransaction = { ...prisma, $transaction: transaction };
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_AUTO_DISABLE_THRESHOLD: "2" },
+      prisma: prismaWithTransaction,
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+
+    await deps.recordOutcomeForCrashTracking!({
+      tenantId: "tenant-1",
+      pluginId: "plugin-1",
+      outcome: { ok: false, reason: "sandbox_failed", durationMs: 0, pluginInvocationId: "inv-1" },
+    });
+    expect(prisma.plugin.update).not.toHaveBeenCalled();
+
+    await deps.recordOutcomeForCrashTracking!({
+      tenantId: "tenant-1",
+      pluginId: "plugin-1",
+      outcome: { ok: false, reason: "sandbox_failed", durationMs: 0, pluginInvocationId: "inv-2" },
+    });
+
+    expect(prisma.plugin.update).toHaveBeenCalledWith({
+      where: { tenantId_pluginId: { tenantId: "tenant-1", pluginId: "plugin-1" } },
+      data: { enabled: false },
+    });
+    expect(prisma.outboxEvent.create).toHaveBeenCalledWith({
+      data: expect.objectContaining({
+        tenantId: "tenant-1",
+        eventType: "audit.event.create",
+        dispatchKey: "plugin-auto-disable-audit:tenant-1:plugin-1:inv-2",
+        priority: 20,
+        payload: expect.objectContaining({
+          tenantId: "tenant-1",
+          actorType: "system",
+          action: "plugin.auto_disabled",
+          targetType: "plugin",
+          targetId: "plugin-1",
+          sourceModule: "worker.plugin-runtime",
+          outcome: "blocked",
+          metadata: expect.objectContaining({
+            pluginId: "plugin-1",
+            failureCount: 2,
+            reason: "sandbox_failed",
+            pluginInvocationId: "inv-2",
+            trigger: "crash-threshold",
+          }),
+        }),
+      }),
+    });
+    expect(transaction).toHaveBeenCalledOnce();
+  });
+
+  test("recordOutcomeForCrashTracking treats duplicate auto-disable audit dispatch keys as already enqueued", async () => {
+    const prisma = makeFakePrisma();
+    prisma.outboxEvent.create.mockRejectedValueOnce(Object.assign(new Error("Unique constraint failed"), { code: "P2002" }));
+    prisma.outboxEvent.findUnique.mockResolvedValueOnce({
+      id: "outbox-existing",
+      dispatchKey: "plugin-auto-disable-audit:tenant-1:plugin-1:inv-1",
+    });
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_AUTO_DISABLE_THRESHOLD: "1" },
+      prisma,
+      store: makeFakeStore() as never,
+      logger,
+    });
+
+    await deps.recordOutcomeForCrashTracking!({
+      tenantId: "tenant-1",
+      pluginId: "plugin-1",
+      outcome: { ok: false, reason: "sandbox_failed", durationMs: 0, pluginInvocationId: "inv-1" },
+    });
+
+    expect(prisma.plugin.update).toHaveBeenCalledWith({
+      where: { tenantId_pluginId: { tenantId: "tenant-1", pluginId: "plugin-1" } },
+      data: { enabled: false },
+    });
+    expect(prisma.outboxEvent.findUnique).toHaveBeenCalledWith({
+      where: { dispatchKey: "plugin-auto-disable-audit:tenant-1:plugin-1:inv-1" },
+    });
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        tenantId: "tenant-1",
+        pluginId: "plugin-1",
+        failureCount: 1,
+        reason: "sandbox_failed",
+      }),
+      "plugin.crash.auto_disable_audit_enqueued"
+    );
+  });
+
+  test("recordOutcomeForCrashTracking uses invocation-scoped auto-disable audit keys after re-enable", async () => {
+    const prisma = makeFakePrisma();
+    const deps = bootstrapPluginInvokeDeps({
+      env: { PLUGIN_AUTO_DISABLE_THRESHOLD: "1" },
+      prisma,
+      store: makeFakeStore() as never,
+      logger: noopLogger,
+    });
+
+    await deps.recordOutcomeForCrashTracking!({
+      tenantId: "tenant-1",
+      pluginId: "plugin-1",
+      outcome: { ok: false, reason: "sandbox_failed", durationMs: 0, pluginInvocationId: "inv-first-disable" },
+    });
+    await deps.recordOutcomeForCrashTracking!({
+      tenantId: "tenant-1",
+      pluginId: "plugin-1",
+      outcome: { ok: false, reason: "sandbox_failed", durationMs: 0, pluginInvocationId: "inv-second-disable" },
+    });
+
+    const dispatchKeys = prisma.outboxEvent.create.mock.calls.map((call) => {
+      const args = call[0] as { data?: { dispatchKey?: string } };
+      return args.data?.dispatchKey;
+    });
+    expect(dispatchKeys).toEqual([
+      "plugin-auto-disable-audit:tenant-1:plugin-1:inv-first-disable",
+      "plugin-auto-disable-audit:tenant-1:plugin-1:inv-second-disable",
+    ]);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-invoke-bootstrap.ts

@@ -0,0 +1,650 @@
+/**
+ * Phase 06 #21 — Production wiring for the plugin-invoke worker handler.
+ *
+ * `handlePluginInvokeJob` lives in `./plugin-invoke.ts` and is fully
+ * dependency-injected. This module composes the real production seams:
+ *
+ *   - `loadPlugin`         → Prisma `Plugin.findUnique`
+ *   - `loadBinary`         → filesystem read from `PLUGIN_BINARY_BASE_PATH`
+ *   - `verifySignature`    → `verifyPluginSignature` from plugin-runtime
+ *   - `consumeRateLimit`   → SharedStore sliding-window counter
+ *   - `invokeInSandbox`    → worker-thread sandbox + capability gateway
+ *   - `recordAudit`        → Prisma `PluginInvocation.create` + structured logger
+ *   - `trustedPublicKeysHex` → parsed once from `PLUGIN_TRUSTED_PUBLIC_KEYS`
+ *   - `recordOutcomeForCrashTracking` → `createCrashTracker` adapter
+ *
+ * Closes F-015 (Critical) — before this bootstrap, `handlePluginInvokeJob`
+ * was not wired into production; the `plugin-invoke` queue accepted jobs
+ * but the handler at `handlers.ts:752` was a silent no-op stub.
+ *
+ * Out of scope for this bootstrap (tracked in plugin-runtime deviations):
+ *   - host-fn implementations for higher-risk capabilities. The safe
+ *     `log` and `audit` imports are wired; network, storage, secret,
+ *     tenant.info and DB bindings remain fail-closed until reviewed.
+ *   - hot-reload of binaries / manifests
+ */
+
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+
+import {
+  PluginCapabilityUseAuditTruncatedError,
+  createCrashTracker,
+  createPluginCapabilityUseRecorder,
+  assertPluginRuntimeProductionEnabled,
+  parsePluginManifest,
+  parseTrustedPublicKeys,
+  verifyPluginSignature,
+  type CrashTrackerStore,
+  type PluginCapabilityGrant,
+  type PluginCapabilityUseBatch,
+  type PluginInvocationResult,
+} from "@toolctrl/plugin-runtime";
+import { AUDIT_EVENT_OUTBOX_TYPE, buildAuditEventCreateData } from "@toolctrl/db/audit-events.server";
+import type { SharedStore } from "@toolctrl/shared-store";
+
+import type {
+  PluginInvokeDeps,
+  PluginInvokeOutcome,
+  PluginInvokeRow,
+  RateLimitDecision,
+} from "./plugin-invoke.js";
+import { PLUGIN_CAPABILITY_USE_EVENT_TYPE } from "./plugin-capability-use.js";
+import { invokePluginSandboxInWorker } from "./plugin-sandbox-isolated.js";
+
+export type PluginInvokeBootstrapLogger = {
+  info: (fields: Record<string, unknown>, msg: string) => void;
+  warn?: (fields: Record<string, unknown>, msg?: string) => void;
+  error?: (fields: Record<string, unknown>, msg?: string) => void;
+};
+
+export type PluginInvokeBootstrapPrismaSurface = {
+  plugin: {
+    findUnique: (args: {
+      where: { tenantId_pluginId: { tenantId: string; pluginId: string } };
+    }) => Promise<{
+      tenantId: string;
+      pluginId: string;
+      version: string;
+      manifest: unknown;
+      wasmSha256: string;
+      signaturePublicKey: string;
+      enabled: boolean;
+    } | null>;
+    update: (args: {
+      where: { tenantId_pluginId: { tenantId: string; pluginId: string } };
+      data: { enabled: boolean };
+    }) => Promise<unknown>;
+  };
+  $transaction?: (fn: (tx: PluginInvokeBootstrapPrismaSurface) => Promise<unknown>) => Promise<unknown>;
+  outboxEvent: {
+    create: (args: {
+      data: {
+        tenantId: string | null;
+        eventType: string;
+        payload: unknown;
+        dispatchKey: string;
+        priority?: number;
+      };
+    }) => Promise<unknown>;
+    findUnique: (args: {
+      where: { dispatchKey: string };
+    }) => Promise<unknown | null>;
+  };
+  pluginInvocation: {
+    create: (args: {
+      data: {
+        id: string;
+        tenantId: string;
+        pluginId: string;
+        version: string;
+        hookName: string;
+        outcome: string;
+        failureReason?: string | null;
+        message?: string | null;
+        durationMs: number;
+        fuelUsed?: number | null;
+      };
+    }) => Promise<unknown>;
+  };
+  tenantPluginCapabilityGrant: {
+    findMany: (args: {
+      where: { tenantId: string; pluginId: string };
+      select: { capabilityName: true; scopeJson: true };
+    }) => Promise<Array<{ capabilityName: string; scopeJson: unknown }>>;
+  };
+};
+
+export type PluginInvokeBootstrapOptions = {
+  env: NodeJS.ProcessEnv;
+  prisma: PluginInvokeBootstrapPrismaSurface;
+  store: SharedStore;
+  logger: PluginInvokeBootstrapLogger;
+};
+
+/** Default base path for plugin binaries on disk. */
+const DEFAULT_PLUGIN_BINARY_BASE_PATH = "/app/var/plugins";
+const DEFAULT_RATE_LIMIT_PER_MINUTE = 100;
+const DEFAULT_RATE_LIMIT_WINDOW_SECONDS = 60;
+const DEFAULT_PLUGIN_MEMORY_LIMIT_MB = 64;
+const DEFAULT_PLUGIN_FUEL_LIMIT = 1_000_000;
+const DEFAULT_PLUGIN_WALL_CLOCK_MS = 5_000;
+const DEFAULT_PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES = 2_048;
+const DEFAULT_PLUGIN_AUTO_DISABLE_THRESHOLD = 10;
+const DEFAULT_PLUGIN_AUTO_DISABLE_WINDOW_MS = 600_000;
+
+function resolveIntEnv(env: NodeJS.ProcessEnv, key: string, fallback: number): number {
+  const raw = env[key];
+  if (typeof raw !== "string" || raw.length === 0) return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return parsed;
+}
+
+/**
+ * Thin pass-through from SharedStore (`incr`, `expire`, `get`, `set`,
+ * `del`) to the CrashTracker's expected store contract. The shapes are
+ * already aligned; the wrapper exists so callers can swap in a fake
+ * store in tests without dragging SharedStore in.
+ */
+function adaptCrashTrackerStore(store: SharedStore): CrashTrackerStore {
+  return {
+    incr: (key, by = 1) => store.incr(key, by),
+    expire: (key, ttlSeconds) => store.expire(key, ttlSeconds),
+    get: (key) => store.get(key),
+    set: async (key, value, opts) => store.set(key, value, opts?.ttlSeconds ? { ttlSeconds: opts.ttlSeconds } : undefined),
+    del: (key) => store.del(key),
+  };
+}
+
+/**
+ * Sliding-window rate limiter keyed by `(pluginId, tenantId)`. Uses the
+ * SharedStore counter primitive — atomic incr + ttl on first hit.
+ */
+function makeRateLimiter(args: {
+  store: SharedStore;
+  perMinute: number;
+  windowSeconds: number;
+}): PluginInvokeDeps["consumeRateLimit"] {
+  return async ({ pluginId, tenantId }): Promise<RateLimitDecision> => {
+    const key = `plugin-ratelimit-${pluginId}-${tenantId}`;
+    const count = await args.store.incr(key, 1);
+    if (count === 1) {
+      await args.store.expire(key, args.windowSeconds);
+    }
+    const allowed = count <= args.perMinute;
+    return { allowed, remaining: Math.max(0, args.perMinute - count) };
+  };
+}
+
+/**
+ * Filesystem loader. Reads `<base>/<pluginId>/<version>/plugin.wasm`.
+ * Returns null on ENOENT so the handler can route to `binary_load_failed`
+ * cleanly; any other error propagates (BullMQ retries).
+ */
+function makeBinaryLoader(args: {
+  basePath: string;
+  logger: PluginInvokeBootstrapLogger;
+}): PluginInvokeDeps["loadBinary"] {
+  const basePath = path.resolve(args.basePath);
+  return async ({ pluginId, version }) => {
+    const filePath = path.resolve(basePath, pluginId, version, "plugin.wasm");
+    const relativePath = path.relative(basePath, filePath);
+    if (relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
+      args.logger.warn?.({ pluginId, version }, "plugin.invoke.binary_path_escape_blocked");
+      return null;
+    }
+    try {
+      const buf = await fs.readFile(filePath);
+      return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+        args.logger.warn?.({ pluginId, version, filePath }, "plugin.invoke.binary_missing_on_disk");
+        return null;
+      }
+      throw err;
+    }
+  };
+}
+
+/**
+ * Prisma adapter. Reads the Plugin row + JSON manifest. Returns null on
+ * miss; the handler then short-circuits to `plugin_not_found`.
+ */
+function makePluginLoader(
+  prisma: PluginInvokeBootstrapPrismaSurface,
+  logger: PluginInvokeBootstrapLogger
+): PluginInvokeDeps["loadPlugin"] {
+  return async ({ tenantId, pluginId, version }) => {
+    const row = await prisma.plugin.findUnique({
+      where: { tenantId_pluginId: { tenantId, pluginId } },
+    });
+    if (!row) return null;
+    if (row.version !== version) return null;
+    const parsed = parsePluginManifest(row.manifest);
+    if (!parsed.ok) {
+      logger.warn?.({ tenantId, pluginId, version, issues: parsed.issues }, "plugin.invoke.stored_manifest_invalid");
+      return null;
+    }
+    return {
+      tenantId: row.tenantId,
+      pluginId: row.pluginId,
+      version: row.version,
+      manifest: parsed.manifest,
+      wasmSha256: row.wasmSha256,
+      signaturePublicKey: row.signaturePublicKey,
+      enabled: row.enabled,
+    } satisfies PluginInvokeRow;
+  };
+}
+
+function makeCapabilityGrantLoader(
+  prisma: PluginInvokeBootstrapPrismaSurface
+): PluginInvokeDeps["loadCapabilityGrants"] {
+  return async ({ tenantId, pluginId }): Promise<PluginCapabilityGrant[]> => {
+    const rows = await prisma.tenantPluginCapabilityGrant.findMany({
+      where: { tenantId, pluginId },
+      select: { capabilityName: true, scopeJson: true },
+    });
+    return rows.map((row) => ({
+      capabilityName: row.capabilityName,
+      scopeJson: row.scopeJson,
+    }));
+  };
+}
+
+function makeCapabilityUseSink(args: {
+  prisma: PluginInvokeBootstrapPrismaSurface;
+  logger: PluginInvokeBootstrapLogger;
+}) {
+  return {
+    emit: async (batch: PluginCapabilityUseBatch): Promise<void> => {
+      const dispatchKey = `plugin-capability-use:${batch.pluginInvocationId}`;
+      try {
+        await args.prisma.outboxEvent.create({
+          data: {
+            tenantId: batch.tenantId,
+            eventType: PLUGIN_CAPABILITY_USE_EVENT_TYPE,
+            payload: batch,
+            dispatchKey,
+            priority: 10,
+          },
+        });
+      } catch (error) {
+        if (!isUniqueConstraintViolation(error)) {
+          throw error;
+        }
+        const existing = await args.prisma.outboxEvent.findUnique({
+          where: { dispatchKey },
+        });
+        if (!existing) {
+          throw error;
+        }
+      }
+      args.logger.info(
+        {
+          tenantId: batch.tenantId,
+          pluginId: batch.pluginId,
+          pluginInvocationId: batch.pluginInvocationId,
+          entryCount: batch.entries.length,
+        },
+        "plugin.invoke.capability_use_enqueued"
+      );
+    },
+  };
+}
+
+type JsonPrimitive = string | number | boolean | null;
+type JsonSafeValue = JsonPrimitive | JsonSafeValue[] | { [key: string]: JsonSafeValue };
+
+const MAX_AUDIT_METADATA_DEPTH = 8;
+
+function toJsonSafeValue(value: unknown, depth = 0): JsonSafeValue {
+  if (depth > MAX_AUDIT_METADATA_DEPTH) {
+    return "[TRUNCATED]";
+  }
+  if (value === null || typeof value === "string" || typeof value === "boolean") {
+    return value;
+  }
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? value : String(value);
+  }
+  if (typeof value === "bigint") {
+    return value.toString();
+  }
+  if (value instanceof Date) {
+    return value.toISOString();
+  }
+  if (Array.isArray(value)) {
+    return value.map((entry) => toJsonSafeValue(entry, depth + 1));
+  }
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>)
+        .filter(([, entry]) => typeof entry !== "undefined" && typeof entry !== "function" && typeof entry !== "symbol")
+        .map(([key, entry]) => [key, toJsonSafeValue(entry, depth + 1)])
+    );
+  }
+  return String(value);
+}
+
+async function enqueuePluginHostAuditEvent(args: {
+  prisma: PluginInvokeBootstrapPrismaSurface;
+  logger: PluginInvokeBootstrapLogger;
+  tenantId: string;
+  pluginId: string;
+  version: string;
+  hookName: string;
+  pluginInvocationId: string;
+  sequence: number;
+  hostRecord: Record<string, unknown>;
+}): Promise<void> {
+  const payload = buildAuditEventCreateData({
+    tenantId: args.tenantId,
+    actorId: args.pluginId,
+    actorType: "plugin",
+    action: "plugin.audit.recorded",
+    targetType: "plugin",
+    targetId: args.pluginId,
+    requestId: args.pluginInvocationId,
+    sourceModule: "worker.plugin-runtime",
+    outcome: "success",
+    dataSensitivity: "internal",
+    regulatoryScope: ["soc2"],
+    metadata: {
+      pluginId: args.pluginId,
+      pluginManifestId: typeof args.hostRecord.pluginId === "string" ? args.hostRecord.pluginId : args.pluginId,
+      pluginInvocationId: args.pluginInvocationId,
+      version: args.version,
+      hookName: args.hookName,
+      hostBinding: "audit",
+      sequence: args.sequence,
+      value: toJsonSafeValue(args.hostRecord.value),
+    },
+  });
+  const dispatchKey = `plugin-host-audit:${args.pluginInvocationId}:${args.sequence}`;
+  try {
+    await args.prisma.outboxEvent.create({
+      data: {
+        tenantId: args.tenantId,
+        eventType: AUDIT_EVENT_OUTBOX_TYPE,
+        payload,
+        dispatchKey,
+        priority: 20,
+      },
+    });
+  } catch (error) {
+    if (!isUniqueConstraintViolation(error)) {
+      throw error;
+    }
+    const existing = await args.prisma.outboxEvent.findUnique({
+      where: { dispatchKey },
+    });
+    if (!existing) {
+      throw error;
+    }
+  }
+  args.logger.info(
+    {
+      tenantId: args.tenantId,
+      pluginId: args.pluginId,
+      version: args.version,
+      hookName: args.hookName,
+      pluginInvocationId: args.pluginInvocationId,
+      sequence: args.sequence,
+    },
+    "plugin.invoke.host.audit_enqueued"
+  );
+}
+
+async function enqueuePluginAutoDisableAudit(args: {
+  prisma: PluginInvokeBootstrapPrismaSurface;
+  logger: PluginInvokeBootstrapLogger;
+  tenantId: string;
+  pluginId: string;
+  failureCount: number;
+  reason: string;
+  correlationId: string;
+}): Promise<void> {
+  const payload = buildAuditEventCreateData({
+    tenantId: args.tenantId,
+    actorType: "system",
+    action: "plugin.auto_disabled",
+    targetType: "plugin",
+    targetId: args.pluginId,
+    sourceModule: "worker.plugin-runtime",
+    outcome: "blocked",
+    dataSensitivity: "internal",
+    regulatoryScope: ["soc2"],
+    metadata: {
+      pluginId: args.pluginId,
+      failureCount: args.failureCount,
+      reason: args.reason,
+      pluginInvocationId: args.correlationId,
+      trigger: "crash-threshold",
+    },
+  });
+  const dispatchKey = `plugin-auto-disable-audit:${args.tenantId}:${args.pluginId}:${args.correlationId}`;
+  try {
+    await args.prisma.outboxEvent.create({
+      data: {
+        tenantId: args.tenantId,
+        eventType: AUDIT_EVENT_OUTBOX_TYPE,
+        payload,
+        dispatchKey,
+        priority: 20,
+      },
+    });
+  } catch (error) {
+    if (!isUniqueConstraintViolation(error)) {
+      throw error;
+    }
+    const existing = await args.prisma.outboxEvent.findUnique({
+      where: { dispatchKey },
+    });
+    if (!existing) {
+      throw error;
+    }
+  }
+  args.logger.info(
+    {
+      tenantId: args.tenantId,
+      pluginId: args.pluginId,
+      failureCount: args.failureCount,
+      reason: args.reason,
+    },
+    "plugin.crash.auto_disable_audit_enqueued"
+  );
+}
+
+function isUniqueConstraintViolation(error: unknown): boolean {
+  return typeof error === "object" && error !== null && (error as { code?: unknown }).code === "P2002";
+}
+
+/**
+ * Sandbox adapter. Runs V8 WebAssembly in a killable worker thread and replays
+ * safe sync `log` and `audit` host imports back into the parent process for
+ * logging + capability-use auditing. Higher-risk host bindings remain absent
+ * and fail closed with `no_host_implementation`.
+ */
+function makeSandboxInvoker(args: {
+  env: NodeJS.ProcessEnv;
+  prisma: PluginInvokeBootstrapPrismaSurface;
+  logger: PluginInvokeBootstrapLogger;
+  capabilityUseSink: ReturnType<typeof makeCapabilityUseSink>;
+}): PluginInvokeDeps["invokeInSandbox"] {
+  const memoryLimitMb = resolveIntEnv(args.env, "PLUGIN_MAX_MEMORY_MB", DEFAULT_PLUGIN_MEMORY_LIMIT_MB);
+  const fuelLimit = resolveIntEnv(args.env, "PLUGIN_MAX_FUEL", DEFAULT_PLUGIN_FUEL_LIMIT);
+  const wallClockMs = resolveIntEnv(args.env, "PLUGIN_MAX_WALL_CLOCK_MS", DEFAULT_PLUGIN_WALL_CLOCK_MS);
+  const maxCapabilityCallEntries = resolveIntEnv(
+    args.env,
+    "PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES",
+    DEFAULT_PLUGIN_CAPABILITY_AUDIT_MAX_ENTRIES
+  );
+
+  return async ({ pluginInvocationId, tenantId, pluginId, wasmBuffer, manifest, capabilityGrants, hookName, payload }): Promise<PluginInvocationResult> => {
+    const capabilityRecorder = createPluginCapabilityUseRecorder({
+      pluginInvocationId,
+      tenantId,
+      pluginId,
+      sink: args.capabilityUseSink,
+      logger: args.logger,
+      failClosed: true,
+    });
+    try {
+      const result = await invokePluginSandboxInWorker({
+        wasmBuffer,
+        manifest,
+        capabilityGrants,
+        hookName,
+        payload,
+        memoryLimitMb,
+        fuelLimit,
+        maxCapabilityCallEntries,
+        wallClockMs,
+      });
+      if (result.droppedCapabilityCallCount > 0) {
+        throw new PluginCapabilityUseAuditTruncatedError(pluginInvocationId, result.droppedCapabilityCallCount, result.capabilityCalls.length);
+      }
+      for (const info of result.capabilityCalls) {
+        capabilityRecorder.record(info);
+      }
+      let auditSequence = 0;
+      for (const entry of result.hostLogs) {
+        if (entry.level === "warn") {
+          args.logger.warn?.(entry.record, entry.message);
+        } else {
+          args.logger.info(entry.record, entry.message);
+        }
+        if (entry.message === "plugin.invoke.host.audit") {
+          await enqueuePluginHostAuditEvent({
+            prisma: args.prisma,
+            logger: args.logger,
+            tenantId,
+            pluginId,
+            version: manifest.version,
+            hookName,
+            pluginInvocationId,
+            sequence: auditSequence,
+            hostRecord: entry.record,
+          });
+          auditSequence += 1;
+        }
+      }
+      return result.result;
+    } finally {
+      await capabilityRecorder.flush();
+    }
+  };
+}
+
+/**
+ * Compose the production PluginInvokeDeps from env + Prisma + SharedStore
+ * + logger. Caller (`apps/worker/src/index.ts`) injects the returned
+ * object into `extraHandlerDependencies.pluginInvokeDeps`.
+ */
+export function bootstrapPluginInvokeDeps(options: PluginInvokeBootstrapOptions): PluginInvokeDeps {
+  const { env, prisma, store, logger } = options;
+  assertPluginRuntimeProductionEnabled(env);
+  const basePath = env.PLUGIN_BINARY_BASE_PATH?.trim() || DEFAULT_PLUGIN_BINARY_BASE_PATH;
+  const perMinute = resolveIntEnv(env, "PLUGIN_RATE_LIMIT_PER_MINUTE", DEFAULT_RATE_LIMIT_PER_MINUTE);
+  const windowSeconds = resolveIntEnv(env, "PLUGIN_RATE_LIMIT_WINDOW_SECONDS", DEFAULT_RATE_LIMIT_WINDOW_SECONDS);
+  const autoDisableThreshold = resolveIntEnv(env, "PLUGIN_AUTO_DISABLE_THRESHOLD", DEFAULT_PLUGIN_AUTO_DISABLE_THRESHOLD);
+  const autoDisableWindowMs = resolveIntEnv(env, "PLUGIN_AUTO_DISABLE_WINDOW_MS", DEFAULT_PLUGIN_AUTO_DISABLE_WINDOW_MS);
+  const trustedPublicKeysHex = parseTrustedPublicKeys(env.PLUGIN_TRUSTED_PUBLIC_KEYS);
+  const capabilityUseSink = makeCapabilityUseSink({ prisma, logger });
+  const crashTracker = createCrashTracker({
+    store: adaptCrashTrackerStore(store),
+    disablePlugin: async ({ pluginId, tenantId, failureCount, reason, correlationId }) => {
+      logger.warn?.({ pluginId, tenantId, failureCount, reason }, "plugin.crash.auto_disable_requested");
+      const disableAndAudit = async (client: PluginInvokeBootstrapPrismaSurface) => {
+        await client.plugin.update({
+          where: { tenantId_pluginId: { tenantId, pluginId } },
+          data: { enabled: false },
+        });
+        await enqueuePluginAutoDisableAudit({
+          prisma: client,
+          logger,
+          tenantId,
+          pluginId,
+          failureCount,
+          reason,
+          correlationId,
+        });
+      };
+      if (typeof prisma.$transaction === "function") {
+        await prisma.$transaction(disableAndAudit);
+      } else {
+        await disableAndAudit(prisma);
+      }
+    },
+    notify: async ({ pluginId, tenantId, failureCount, reason }) => {
+      logger.error?.({ pluginId, tenantId, failureCount, reason }, "plugin.crash.threshold_exceeded");
+    },
+    options: {
+      logger,
+      threshold: autoDisableThreshold,
+      windowMs: autoDisableWindowMs,
+    },
+  });
+
+  return {
+    loadPlugin: makePluginLoader(prisma, logger),
+    loadCapabilityGrants: makeCapabilityGrantLoader(prisma),
+    loadBinary: makeBinaryLoader({ basePath, logger }),
+    verifySignature: ({ wasmBuffer, manifest, trustedPublicKeysHex: trusted }) =>
+      verifyPluginSignature({
+        wasmBuffer,
+        expectedWasmSha256: manifest.wasmSha256,
+        signatureHex: manifest.signature,
+        publicKeyHex: manifest.signedBy,
+        trustedPublicKeysHex: trusted,
+        manifest,
+      }),
+    consumeRateLimit: makeRateLimiter({ store, perMinute, windowSeconds }),
+    invokeInSandbox: makeSandboxInvoker({ env, prisma, logger, capabilityUseSink }),
+    recordAudit: async ({ tenantId, pluginId, version, hookName, pluginInvocationId, outcome }) => {
+      await prisma.pluginInvocation.create({
+        data: {
+          id: pluginInvocationId,
+          tenantId,
+          pluginId,
+          version,
+          hookName,
+          outcome: outcome.ok ? "success" : "failure",
+          failureReason: outcome.ok ? null : outcome.reason,
+          message: outcome.ok ? null : outcome.message ?? null,
+          durationMs: outcome.durationMs,
+          fuelUsed: outcome.ok ? outcome.fuelUsed : null,
+        },
+      });
+      if (outcome.ok) {
+        logger.info(
+          { tenantId, pluginId, version, hookName, pluginInvocationId, fuelUsed: outcome.fuelUsed, durationMs: outcome.durationMs },
+          "plugin.invoke.audit.success"
+        );
+      } else {
+        logger.error?.(
+          { tenantId, pluginId, version, hookName, pluginInvocationId, reason: outcome.reason, message: outcome.message, durationMs: outcome.durationMs },
+          "plugin.invoke.audit.failure"
+        );
+      }
+    },
+    trustedPublicKeysHex,
+    recordOutcomeForCrashTracking: async ({ tenantId, pluginId, outcome }) => {
+      if (outcome.ok) {
+        await crashTracker.recordSuccess({ pluginId, tenantId });
+      } else {
+        await crashTracker.recordFailure({
+          pluginId,
+          tenantId,
+          reason: outcome.reason,
+          correlationId: outcome.pluginInvocationId,
+        });
+      }
+    },
+    logger,
+  } satisfies PluginInvokeDeps;
+}
+
+export type { PluginInvokeOutcome };

hivegrid-cloud/apps/worker/src/jobs/plugin-invoke.test.ts

@@ -0,0 +1,425 @@
+import { describe, expect, test, vi } from "vitest";
+import type { PluginCapabilityGrant, PluginManifest } from "@toolctrl/plugin-runtime";
+import { handlePluginInvokeJob, type PluginInvokeDeps, type PluginInvokeJobPayload, type PluginInvokeRow } from "./plugin-invoke.js";
+
+function manifest(): PluginManifest {
+  return {
+    schemaVersion: 1,
+    id: "de.toolctrl.test",
+    version: "1.0.0",
+    name: "Test",
+    entry: "plugin.wasm",
+    wasmSha256: "0".repeat(64),
+    signature: "0".repeat(128),
+    signedBy: "0".repeat(64),
+    allowedHosts: [],
+    capabilities: [{ name: "log" }],
+    hooks: [{ name: "ping", export: "ping" }],
+  };
+}
+
+function hookOnlyManifest(): PluginManifest {
+  return {
+    ...manifest(),
+    capabilities: [],
+    hooks: [{ name: "tenant.created", export: "onTenantCreated" }],
+  };
+}
+
+function outboundManifest(): PluginManifest {
+  return {
+    ...manifest(),
+    capabilities: [{ name: "http.outbound", scope: { allowedHosts: ["api.example.com"] } }],
+  };
+}
+
+function row(): PluginInvokeRow {
+  return {
+    tenantId: "t-1",
+    pluginId: "de.toolctrl.test",
+    version: "1.0.0",
+    manifest: manifest(),
+    wasmSha256: "0".repeat(64),
+    signaturePublicKey: "0".repeat(64),
+    enabled: true,
+  };
+}
+
+function grants(): PluginCapabilityGrant[] {
+  return [
+    { capabilityName: "log", scopeJson: null },
+    { capabilityName: "hook:ping", scopeJson: null },
+  ];
+}
+
+function payload(): PluginInvokeJobPayload {
+  return {
+    tenantId: "t-1",
+    pluginId: "de.toolctrl.test",
+    version: "1.0.0",
+    hookName: "ping",
+    input: { x: 1 },
+  };
+}
+
+function defaultDeps(overrides: Partial<PluginInvokeDeps> = {}): PluginInvokeDeps {
+  return {
+    loadPlugin: vi.fn().mockResolvedValue(row()),
+    loadCapabilityGrants: vi.fn().mockResolvedValue(grants()),
+    loadBinary: vi.fn().mockResolvedValue(new Uint8Array([1, 2, 3])),
+    verifySignature: vi.fn().mockReturnValue({ ok: true }),
+    consumeRateLimit: vi.fn().mockResolvedValue({ allowed: true, remaining: 99 }),
+    invokeInSandbox: vi.fn().mockResolvedValue({ ok: true, output: "pong", fuelUsed: 5 }),
+    recordAudit: vi.fn().mockResolvedValue(undefined),
+    trustedPublicKeysHex: ["0".repeat(64)],
+    now: (() => {
+      let n = 1_000;
+      return () => (n += 10);
+    })(),
+    ...overrides,
+  };
+}
+
+describe("handlePluginInvokeJob", () => {
+  test("happy path returns the sandbox output and records audit", async () => {
+    const deps = defaultDeps();
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(true);
+    expect(result.pluginInvocationId).toMatch(/^[0-9a-f-]{36}$/);
+    if (result.ok) expect(result.output).toBe("pong");
+    expect(deps.recordAudit).toHaveBeenCalledTimes(1);
+    expect(deps.invokeInSandbox).toHaveBeenCalledTimes(1);
+    expect(deps.loadCapabilityGrants).toHaveBeenCalledWith({
+      tenantId: "t-1",
+      pluginId: "de.toolctrl.test",
+    });
+    const sandboxArgs = vi.mocked(deps.invokeInSandbox).mock.calls[0]?.[0];
+    const auditArgs = vi.mocked(deps.recordAudit).mock.calls[0]?.[0];
+    expect(sandboxArgs?.pluginInvocationId).toBe(result.pluginInvocationId);
+    expect(sandboxArgs?.capabilityGrants).toEqual([
+      { capabilityName: "log", scopeJson: null },
+      { capabilityName: "hook:ping", scopeJson: null },
+    ]);
+    expect(auditArgs?.pluginInvocationId).toBe(result.pluginInvocationId);
+    expect(auditArgs?.outcome.pluginInvocationId).toBe(result.pluginInvocationId);
+  });
+
+  test("returns plugin_not_found when the row is missing", async () => {
+    const deps = defaultDeps({ loadPlugin: vi.fn().mockResolvedValue(null) });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) expect(result.reason).toBe("plugin_not_found");
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+    expect(deps.recordAudit).toHaveBeenCalledTimes(1);
+  });
+
+  test("returns plugin_disabled when the row is disabled", async () => {
+    const deps = defaultDeps({ loadPlugin: vi.fn().mockResolvedValue({ ...row(), enabled: false }) });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) expect(result.reason).toBe("plugin_disabled");
+    expect(deps.loadCapabilityGrants).not.toHaveBeenCalled();
+  });
+
+  test("rejects queue hook names that are not declared as manifest exports", async () => {
+    const deps = defaultDeps();
+    const result = await handlePluginInvokeJob({ ...payload(), hookName: "rogueExport" }, deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("hook_not_declared");
+      expect(result.message).toBe('Hook export "rogueExport" is not declared by plugin manifest.');
+    }
+    expect(deps.loadCapabilityGrants).not.toHaveBeenCalled();
+    expect(deps.loadBinary).not.toHaveBeenCalled();
+    expect(deps.consumeRateLimit).not.toHaveBeenCalled();
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+    expect(deps.recordAudit).toHaveBeenCalledTimes(1);
+  });
+
+  test("allows manifest hook exports even when logical hook names differ", async () => {
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue({
+        ...row(),
+        manifest: { ...manifest(), hooks: [{ name: "tenant.created", export: "onTenantCreated" }] },
+      }),
+      loadCapabilityGrants: vi.fn().mockResolvedValue([
+        { capabilityName: "log", scopeJson: null },
+        { capabilityName: "hook:tenant.created", scopeJson: null },
+      ]),
+    });
+    const result = await handlePluginInvokeJob(
+      {
+        ...payload(),
+        hookName: "onTenantCreated",
+        metadata: { logicalHookName: "tenant.created" },
+        input: { tenantId: "t-1", slug: "acme", ownerEmail: "owner@example.com", createdAt: "2026-05-28T10:00:00.000Z" },
+      },
+      deps
+    );
+
+    expect(result.ok).toBe(true);
+    expect(deps.invokeInSandbox).toHaveBeenCalledWith(
+      expect.objectContaining({
+        hookName: "onTenantCreated",
+        payload: expect.objectContaining({ tenantId: "t-1", slug: "acme" }),
+      })
+    );
+  });
+
+  test("returns capability_grant_missing before loading the binary", async () => {
+    const deps = defaultDeps({ loadCapabilityGrants: vi.fn().mockResolvedValue([]) });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("capability_grant_missing");
+      expect(result.message).toContain("log");
+    }
+    expect(deps.loadBinary).not.toHaveBeenCalled();
+    expect(deps.consumeRateLimit).not.toHaveBeenCalled();
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+    expect(deps.recordAudit).toHaveBeenCalledTimes(1);
+  });
+
+  test("treats stale grant scopes as missing consent", async () => {
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue({ ...row(), manifest: outboundManifest() }),
+      loadCapabilityGrants: vi
+        .fn()
+        .mockResolvedValue([{ capabilityName: "http.outbound", scopeJson: { allowedHosts: ["api.example.com", "internal.example"] } }]),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("capability_grant_missing");
+      expect(result.message).toContain("http.outbound");
+    }
+    expect(deps.loadBinary).not.toHaveBeenCalled();
+  });
+
+  test("requires hook grants even when the manifest declares no capabilities", async () => {
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue({ ...row(), manifest: hookOnlyManifest() }),
+      loadCapabilityGrants: vi.fn().mockResolvedValue([]),
+    });
+    const result = await handlePluginInvokeJob({ ...payload(), hookName: "onTenantCreated", metadata: { logicalHookName: "tenant.created" } }, deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("capability_grant_missing");
+      expect(result.message).toContain("hook:tenant.created");
+    }
+    expect(deps.loadCapabilityGrants).toHaveBeenCalledWith({ tenantId: "t-1", pluginId: "de.toolctrl.test" });
+    expect(deps.loadBinary).not.toHaveBeenCalled();
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+  });
+
+  test("runs hook-only plugins when the logical hook grant is present", async () => {
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue({ ...row(), manifest: hookOnlyManifest() }),
+      loadCapabilityGrants: vi.fn().mockResolvedValue([{ capabilityName: "hook:tenant.created", scopeJson: null }]),
+    });
+    const result = await handlePluginInvokeJob(
+      {
+        ...payload(),
+        hookName: "onTenantCreated",
+        metadata: { logicalHookName: "tenant.created" },
+        input: { tenantId: "t-1", slug: "acme", plan: null, createdAt: "2026-05-28T10:00:00.000Z" },
+      },
+      deps
+    );
+
+    expect(result.ok).toBe(true);
+    expect(deps.invokeInSandbox).toHaveBeenCalledWith(
+      expect.objectContaining({
+        hookName: "onTenantCreated",
+        capabilityGrants: [{ capabilityName: "hook:tenant.created", scopeJson: null }],
+      })
+    );
+  });
+
+  test("rejects invalid registered hook payloads before loading the binary", async () => {
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue({ ...row(), manifest: hookOnlyManifest() }),
+      loadCapabilityGrants: vi.fn().mockResolvedValue([{ capabilityName: "hook:tenant.created", scopeJson: null }]),
+    });
+    const result = await handlePluginInvokeJob(
+      {
+        ...payload(),
+        hookName: "onTenantCreated",
+        metadata: { logicalHookName: "tenant.created" },
+        input: { tenantId: "t-1" },
+      },
+      deps
+    );
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("invalid_hook_payload");
+      expect(result.message).toBe('Invalid payload for plugin hook "tenant.created".');
+    }
+    expect(deps.loadBinary).not.toHaveBeenCalled();
+    expect(deps.verifySignature).not.toHaveBeenCalled();
+    expect(deps.consumeRateLimit).not.toHaveBeenCalled();
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+    expect(deps.recordAudit).toHaveBeenCalledTimes(1);
+  });
+
+  test("returns binary_load_failed when the .wasm bytes cannot be loaded", async () => {
+    const deps = defaultDeps({ loadBinary: vi.fn().mockResolvedValue(null) });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) expect(result.reason).toBe("binary_load_failed");
+  });
+
+  test("returns signature_invalid when verifySignature rejects", async () => {
+    const deps = defaultDeps({
+      verifySignature: vi.fn().mockReturnValue({ ok: false, reason: "public_key_not_trusted" }),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("signature_invalid");
+      expect(result.message).toBe("public_key_not_trusted");
+    }
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+  });
+
+  test("rejects rows whose stored wasm hash no longer matches the manifest", async () => {
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue({ ...row(), wasmSha256: "1".repeat(64) }),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("signature_invalid");
+      expect(result.message).toBe("stored_wasm_sha256_mismatch");
+    }
+    expect(deps.verifySignature).not.toHaveBeenCalled();
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+  });
+
+  test("rejects rows whose stored signing key no longer matches the manifest", async () => {
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue({ ...row(), signaturePublicKey: "1".repeat(64) }),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("signature_invalid");
+      expect(result.message).toBe("stored_signature_public_key_mismatch");
+    }
+    expect(deps.verifySignature).not.toHaveBeenCalled();
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+  });
+
+  test("returns rate_limit_exceeded when the limiter denies", async () => {
+    const deps = defaultDeps({
+      consumeRateLimit: vi.fn().mockResolvedValue({ allowed: false, remaining: 0 }),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) expect(result.reason).toBe("rate_limit_exceeded");
+    expect(deps.invokeInSandbox).not.toHaveBeenCalled();
+  });
+
+  test("returns sandbox_failed when the sandbox reports failure", async () => {
+    const deps = defaultDeps({
+      invokeInSandbox: vi.fn().mockResolvedValue({ ok: false, reason: "trap", fuelUsed: 12 }),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("sandbox_failed");
+      expect(result.message).toBe("trap");
+    }
+  });
+
+  test("preserves sandbox failure messages for operator diagnostics", async () => {
+    const deps = defaultDeps({
+      invokeInSandbox: vi.fn().mockResolvedValue({
+        ok: false,
+        reason: "timeout",
+        fuelUsed: 0,
+        message: "plugin sandbox wall-clock timeout",
+      }),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("sandbox_failed");
+      expect(result.message).toBe("plugin sandbox wall-clock timeout");
+    }
+  });
+
+  test("returns sandbox_failed when invokeInSandbox throws", async () => {
+    const deps = defaultDeps({
+      invokeInSandbox: vi.fn().mockRejectedValue(new Error("boom")),
+    });
+    const result = await handlePluginInvokeJob(payload(), deps);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) {
+      expect(result.reason).toBe("sandbox_failed");
+      expect(result.message).toBe("boom");
+    }
+  });
+
+  test("audit failure rejects so the queue can retry without losing forensics", async () => {
+    const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
+    const deps = defaultDeps({
+      recordAudit: vi.fn().mockRejectedValue(new Error("audit-down")),
+      logger,
+    });
+    await expect(handlePluginInvokeJob(payload(), deps)).rejects.toThrow("audit-down");
+    expect(logger.error).toHaveBeenCalledWith(
+      expect.objectContaining({
+        pluginId: "de.toolctrl.test",
+        pluginInvocationId: expect.any(String),
+        error: "audit-down",
+      }),
+      "plugin.invoke.audit_failed"
+    );
+  });
+
+  test("crash-tracker seam receives sandbox_failed outcomes", async () => {
+    const record = vi.fn().mockResolvedValue(undefined);
+    const deps = defaultDeps({
+      invokeInSandbox: vi.fn().mockResolvedValue({ ok: false, reason: "trap", fuelUsed: 1 }),
+      recordOutcomeForCrashTracking: record,
+    });
+    await handlePluginInvokeJob(payload(), deps);
+    expect(record).toHaveBeenCalledTimes(1);
+    const args = record.mock.calls[0]?.[0] as { outcome: { ok: boolean } };
+    expect(args.outcome.ok).toBe(false);
+  });
+
+  test("crash-tracker seam ignores plugin_not_found (config issue)", async () => {
+    const record = vi.fn().mockResolvedValue(undefined);
+    const deps = defaultDeps({
+      loadPlugin: vi.fn().mockResolvedValue(null),
+      recordOutcomeForCrashTracking: record,
+    });
+    await handlePluginInvokeJob(payload(), deps);
+    expect(record).not.toHaveBeenCalled();
+  });
+
+  test("crash-tracker seam failure does not break the handler", async () => {
+    const record = vi.fn().mockRejectedValue(new Error("redis-down"));
+    const deps = defaultDeps({ recordOutcomeForCrashTracking: record });
+    const result = await handlePluginInvokeJob(payload(), deps);
+    expect(result.ok).toBe(true);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-invoke.ts

@@ -0,0 +1,374 @@
+/**
+ * Phase 06 #21 — Plugin-invoke worker job.
+ *
+ * Handles one job from the `toolctrl.plugin-invoke` queue:
+ *   1. Loads the Plugin row + .wasm binary for the requested
+ *      (tenantId, pluginId, version) tuple.
+ *   2. Verifies the binary's SHA-256 matches the stored hash and the
+ *      manifest's Ed25519 signature is valid against the trusted-keys
+ *      allowlist.
+ *   3. Enforces a per-tenant sliding-window rate limit on plugin calls.
+ *   4. Builds a fresh WASM sandbox with memory + fuel limits, invokes
+ *      the requested hook, and writes an audit event with the outcome.
+ *
+ * The handler is host-agnostic — every dependency (DB lookup, binary
+ * loader, signature verify, rate limiter, audit writer, logger) is
+ * passed in as a seam. The worker bootstrap wires real implementations;
+ * tests inject fakes to assert behaviour without spinning a database.
+ *
+ * Deferred (see docs/improvement-plan/06-plugin-system/wasm-sandbox.md):
+ *  - hot-reload of plugin binaries (operator restarts pod)
+ *  - rich structured payload marshalling (covered by #22's hook lifecycle)
+ *  - Sentry / OTel span wiring (covered by #23's audit + docs)
+ */
+
+import { randomUUID } from "node:crypto";
+
+import {
+  getMissingCapabilityGrantNames,
+  getPluginHook,
+  sanitizePluginOutput,
+  type PluginCapabilityGrant,
+  type PluginInvocationResult,
+  type PluginManifest,
+  type SanitizationStrictness,
+  type VerifyPluginSignatureResult,
+  validatePluginHookPayload,
+} from "@toolctrl/plugin-runtime";
+
+/**
+ * Resolve the sanitization strictness from env. Default is `moderate`
+ * (redact PANs/JWTs/IBANs/API-keys but keep emails — emails are often
+ * legitimate plugin output). Unknown values fall back to `moderate`.
+ */
+function resolveSanitizationStrictness(): SanitizationStrictness {
+  const raw = process.env.PLUGIN_OUTPUT_SANITIZATION_STRICTNESS?.trim().toLowerCase();
+  if (raw === "strict" || raw === "moderate" || raw === "off") {
+    return raw;
+  }
+  return "moderate";
+}
+
+function validateStoredPluginVerificationMetadata(row: PluginInvokeRow): string | null {
+  if (row.wasmSha256 !== row.manifest.wasmSha256) {
+    return "stored_wasm_sha256_mismatch";
+  }
+  if (row.signaturePublicKey.toLowerCase() !== row.manifest.signedBy.toLowerCase()) {
+    return "stored_signature_public_key_mismatch";
+  }
+  return null;
+}
+
+export type PluginInvokeJobPayload = {
+  tenantId: string;
+  pluginId: string;
+  version: string;
+  hookName: string;
+  input?: unknown;
+  metadata?: Record<string, unknown>;
+};
+
+export type PluginInvokeRow = {
+  tenantId: string;
+  pluginId: string;
+  version: string;
+  manifest: PluginManifest;
+  wasmSha256: string;
+  signaturePublicKey: string;
+  enabled: boolean;
+};
+
+export type PluginInvokeOutcome =
+  | { ok: true; output: unknown; fuelUsed: number; durationMs: number; pluginInvocationId: string }
+  | { ok: false; reason: PluginInvokeFailure; message?: string; durationMs: number; pluginInvocationId: string };
+
+export type PluginInvokeFailure =
+  | "plugin_not_found"
+  | "plugin_disabled"
+  | "hook_not_declared"
+  | "capability_grant_missing"
+  | "binary_load_failed"
+  | "invalid_hook_payload"
+  | "signature_invalid"
+  | "rate_limit_exceeded"
+  | "sandbox_failed";
+
+export type RateLimitDecision = { allowed: boolean; remaining: number };
+
+export type PluginInvokeDeps = {
+  /** Resolve the Plugin row for a tenant+pluginId+version tuple. */
+  loadPlugin: (args: { tenantId: string; pluginId: string; version: string }) => Promise<PluginInvokeRow | null>;
+  /** Resolve operator-approved capability grants for this tenant+plugin pair. */
+  loadCapabilityGrants: (args: { tenantId: string; pluginId: string }) => Promise<ReadonlyArray<PluginCapabilityGrant>>;
+  /** Load the raw .wasm bytes from disk / object storage. */
+  loadBinary: (args: { pluginId: string; version: string }) => Promise<Uint8Array | null>;
+  /** Verify the binary signature against the trusted-keys allowlist. */
+  verifySignature: (args: { wasmBuffer: Uint8Array; manifest: PluginManifest; trustedPublicKeysHex: ReadonlyArray<string> }) => VerifyPluginSignatureResult;
+  /** Per-tenant sliding-window rate limit (returns allowed flag). */
+  consumeRateLimit: (args: { tenantId: string; pluginId: string }) => Promise<RateLimitDecision>;
+  /** Invoke the plugin inside the worker-owned sandbox boundary. */
+  invokeInSandbox: (args: {
+    pluginInvocationId: string;
+    tenantId: string;
+    pluginId: string;
+    wasmBuffer: Uint8Array;
+    manifest: PluginManifest;
+    hookName: string;
+    payload: unknown;
+    capabilityGrants: ReadonlyArray<PluginCapabilityGrant>;
+  }) => Promise<PluginInvocationResult>;
+  /** Write an audit row for this invocation (success or failure). */
+  recordAudit: (args: {
+    tenantId: string;
+    pluginId: string;
+    version: string;
+    hookName: string;
+    pluginInvocationId: string;
+    outcome: PluginInvokeOutcome;
+  }) => Promise<void>;
+  /** Trusted Ed25519 public keys (hex). Loaded once from env at boot. */
+  trustedPublicKeysHex: ReadonlyArray<string>;
+  /** Pino-compatible structured logger seam. */
+  logger?: {
+    info: (fields: Record<string, unknown>, msg: string) => void;
+    warn?: (fields: Record<string, unknown>, msg: string) => void;
+    error?: (fields: Record<string, unknown>, msg: string) => void;
+  };
+  /**
+   * Phase 06 #22 — crash-tracker seam. Receives every outcome so the
+   * sliding-window counter can auto-disable a misbehaving plugin. The
+   * worker bootstrap wires this to a SharedStore-backed tracker; tests
+   * leave it `undefined` and the handler short-circuits.
+   */
+  recordOutcomeForCrashTracking?: (args: { tenantId: string; pluginId: string; outcome: PluginInvokeOutcome }) => Promise<void>;
+  /** Injected clock; defaults to Date.now. */
+  now?: () => number;
+};
+
+/**
+ * Process exactly one plugin-invoke job. Returns the outcome (which
+ * is also persisted via `recordAudit`). Runtime failures are encoded as
+ * the discriminated `ok=false` shape. Durable audit failures deliberately
+ * throw so BullMQ can retry/dead-letter instead of acknowledging an
+ * invocation with no forensic trail.
+ */
+export async function handlePluginInvokeJob(payload: PluginInvokeJobPayload, deps: PluginInvokeDeps): Promise<PluginInvokeOutcome> {
+  const now = deps.now ?? Date.now;
+  const startedAt = now();
+  const pluginInvocationId = randomUUID();
+
+  const finalise = async (outcome: PluginInvokeOutcome): Promise<PluginInvokeOutcome> => {
+    await recordInvocationAudit(deps, payload, outcome);
+    // Phase 06 #22 — feed outcome to crash tracker. Some failure modes
+    // (plugin_not_found, plugin_disabled) are configuration issues, not
+    // plugin misbehaviour, so we deliberately skip them in the counter.
+    if (deps.recordOutcomeForCrashTracking) {
+      const trackable = outcome.ok || outcome.reason === "sandbox_failed" || outcome.reason === "signature_invalid" || outcome.reason === "binary_load_failed";
+      if (trackable) {
+        try {
+          await deps.recordOutcomeForCrashTracking({
+            tenantId: payload.tenantId,
+            pluginId: payload.pluginId,
+            outcome,
+          });
+        } catch (err) {
+          deps.logger?.warn?.({ ...payload, error: err instanceof Error ? err.message : String(err) }, "plugin.invoke.crash_tracker_failed");
+        }
+      }
+    }
+    return outcome;
+  };
+
+  const row = await deps.loadPlugin({
+    tenantId: payload.tenantId,
+    pluginId: payload.pluginId,
+    version: payload.version,
+  });
+  if (!row) {
+    deps.logger?.warn?.({ ...payload }, "plugin.invoke.not_found");
+    return finalise({ ok: false, reason: "plugin_not_found", durationMs: now() - startedAt, pluginInvocationId });
+  }
+  if (!row.enabled) {
+    deps.logger?.warn?.({ ...payload }, "plugin.invoke.disabled");
+    return finalise({ ok: false, reason: "plugin_disabled", durationMs: now() - startedAt, pluginInvocationId });
+  }
+  const declaredHook = getDeclaredHookForPayload(row.manifest, payload);
+  if (!declaredHook) {
+    deps.logger?.warn?.({ ...payload, declaredHooks: getDeclaredHookExports(row.manifest) }, "plugin.invoke.hook_not_declared");
+    return finalise({
+      ok: false,
+      reason: "hook_not_declared",
+      message: `Hook export "${payload.hookName}" is not declared by plugin manifest.`,
+      durationMs: now() - startedAt,
+      pluginInvocationId,
+    });
+  }
+
+  const capabilityGrants =
+    row.manifest.capabilities.length > 0 || row.manifest.hooks.length > 0
+      ? await deps.loadCapabilityGrants({ tenantId: payload.tenantId, pluginId: payload.pluginId })
+      : [];
+  const missingCapabilityGrants = [
+    ...getMissingCapabilityGrantNames({
+      manifest: row.manifest,
+      grants: capabilityGrants,
+    }),
+    ...getMissingHookGrantNames({ declaredHook, grants: capabilityGrants }),
+  ];
+  if (missingCapabilityGrants.length > 0) {
+    deps.logger?.warn?.({ ...payload, capabilities: missingCapabilityGrants }, "plugin.invoke.capability_grant_missing");
+    return finalise({
+      ok: false,
+      reason: "capability_grant_missing",
+      message: `Missing approved capability grants: ${missingCapabilityGrants.join(", ")}`,
+      durationMs: now() - startedAt,
+      pluginInvocationId,
+    });
+  }
+
+  let sandboxPayload = payload.input;
+  const registeredHook = getPluginHook(declaredHook.name);
+  if (registeredHook) {
+    const validation = validatePluginHookPayload(registeredHook, payload.input);
+    if (!validation.ok) {
+      deps.logger?.warn?.({ ...payload, hookName: declaredHook.name, issues: validation.issues }, "plugin.invoke.invalid_hook_payload");
+      return finalise({
+        ok: false,
+        reason: "invalid_hook_payload",
+        message: `Invalid payload for plugin hook "${declaredHook.name}".`,
+        durationMs: now() - startedAt,
+        pluginInvocationId,
+      });
+    }
+    sandboxPayload = validation.payload;
+  }
+
+  const wasmBuffer = await deps.loadBinary({ pluginId: payload.pluginId, version: payload.version });
+  if (!wasmBuffer) {
+    deps.logger?.error?.({ ...payload }, "plugin.invoke.binary_missing");
+    return finalise({ ok: false, reason: "binary_load_failed", durationMs: now() - startedAt, pluginInvocationId });
+  }
+
+  const storedMetadataError = validateStoredPluginVerificationMetadata(row);
+  if (storedMetadataError) {
+    deps.logger?.error?.({ ...payload, reason: storedMetadataError }, "plugin.invoke.stored_metadata_invalid");
+    return finalise({
+      ok: false,
+      reason: "signature_invalid",
+      message: storedMetadataError,
+      durationMs: now() - startedAt,
+      pluginInvocationId,
+    });
+  }
+
+  const verify = deps.verifySignature({
+    wasmBuffer,
+    manifest: row.manifest,
+    trustedPublicKeysHex: deps.trustedPublicKeysHex,
+  });
+  if (!verify.ok) {
+    deps.logger?.error?.({ ...payload, reason: verify.reason }, "plugin.invoke.signature_invalid");
+    return finalise({
+      ok: false,
+      reason: "signature_invalid",
+      message: verify.reason,
+      durationMs: now() - startedAt,
+      pluginInvocationId,
+    });
+  }
+
+  const rate = await deps.consumeRateLimit({ tenantId: payload.tenantId, pluginId: payload.pluginId });
+  if (!rate.allowed) {
+    deps.logger?.warn?.({ ...payload, remaining: rate.remaining }, "plugin.invoke.rate_limited");
+    return finalise({ ok: false, reason: "rate_limit_exceeded", durationMs: now() - startedAt, pluginInvocationId });
+  }
+
+  let result: PluginInvocationResult;
+  try {
+    result = await deps.invokeInSandbox({
+      pluginInvocationId,
+      tenantId: payload.tenantId,
+      pluginId: payload.pluginId,
+      wasmBuffer,
+      manifest: row.manifest,
+      hookName: payload.hookName,
+      payload: sandboxPayload,
+      capabilityGrants,
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    deps.logger?.error?.({ ...payload, error: message }, "plugin.invoke.sandbox_threw");
+    return finalise({
+      ok: false,
+      reason: "sandbox_failed",
+      message,
+      durationMs: now() - startedAt,
+      pluginInvocationId,
+    });
+  }
+
+  const durationMs = now() - startedAt;
+  if (!result.ok) {
+    deps.logger?.warn?.({ ...payload, reason: result.reason, message: result.message }, "plugin.invoke.sandbox_failed");
+    return finalise({
+      ok: false,
+      reason: "sandbox_failed",
+      message: result.message ?? result.reason,
+      durationMs,
+      pluginInvocationId,
+    });
+  }
+
+  deps.logger?.info({ ...payload, fuelUsed: result.fuelUsed, durationMs }, "plugin.invoke.completed");
+  // Phase 06 #23 — sanitize plugin output before it lands in the audit
+  // trail or worker logs. Strictness is env-driven (default moderate).
+  const sanitizedOutput = sanitizePluginOutput(result.output, resolveSanitizationStrictness());
+  return finalise({ ok: true, output: sanitizedOutput, fuelUsed: result.fuelUsed, durationMs, pluginInvocationId });
+}
+
+function getDeclaredHookForPayload(manifest: PluginManifest, payload: PluginInvokeJobPayload): PluginManifest["hooks"][number] | null {
+  if (!Array.isArray(manifest.hooks)) {
+    return null;
+  }
+  const candidates = manifest.hooks.filter((hook) => hook.export === payload.hookName);
+  if (candidates.length === 0) {
+    return null;
+  }
+  const logicalHookName = typeof payload.metadata?.logicalHookName === "string" ? payload.metadata.logicalHookName : null;
+  return candidates.find((hook) => hook.name === logicalHookName) ?? candidates[0] ?? null;
+}
+
+function getDeclaredHookExports(manifest: PluginManifest): string[] {
+  if (!Array.isArray(manifest.hooks)) {
+    return [];
+  }
+  return manifest.hooks.map((hook) => hook.export);
+}
+
+function getMissingHookGrantNames({
+  declaredHook,
+  grants,
+}: {
+  declaredHook: PluginManifest["hooks"][number];
+  grants: ReadonlyArray<PluginCapabilityGrant>;
+}): string[] {
+  const hookGrantName = `hook:${declaredHook.name}`;
+  return grants.some((grant) => grant.capabilityName === hookGrantName) ? [] : [hookGrantName];
+}
+
+async function recordInvocationAudit(deps: PluginInvokeDeps, payload: PluginInvokeJobPayload, outcome: PluginInvokeOutcome): Promise<void> {
+  try {
+    await deps.recordAudit({
+      tenantId: payload.tenantId,
+      pluginId: payload.pluginId,
+      version: payload.version,
+      hookName: payload.hookName,
+      pluginInvocationId: outcome.pluginInvocationId,
+      outcome,
+    });
+  } catch (err) {
+    const error = err instanceof Error ? err.message : String(err);
+    deps.logger?.error?.({ ...payload, pluginInvocationId: outcome.pluginInvocationId, error }, "plugin.invoke.audit_failed");
+    throw err;
+  }
+}

hivegrid-cloud/apps/worker/src/jobs/plugin-sandbox-isolated.test.ts

@@ -0,0 +1,142 @@
+import { describe, expect, test } from "vitest";
+
+import type { PluginManifest } from "@toolctrl/plugin-runtime";
+
+import { invokePluginSandboxInWorker, resolveSandboxWorkerLocationForTest } from "./plugin-sandbox-isolated.js";
+import { runSandboxForTest } from "./plugin-sandbox-worker.js";
+
+const INFINITE_LOOP_WASM = new Uint8Array([
+  0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
+  0x01, 0x04, 0x01, 0x60, 0x00, 0x00,
+  0x03, 0x02, 0x01, 0x00,
+  0x07, 0x07, 0x01, 0x03, 0x72, 0x75, 0x6e, 0x00, 0x00,
+  0x0a, 0x09, 0x01, 0x07, 0x00, 0x03, 0x40, 0x0c, 0x00, 0x0b, 0x0b,
+]);
+
+const HOST_IMPORT_WASM = new Uint8Array([
+  0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
+  0x01, 0x0a, 0x02, 0x60, 0x01, 0x7f, 0x00, 0x60, 0x01, 0x7f, 0x01, 0x7f,
+  0x02, 0x30, 0x02,
+  0x03, 0x65, 0x6e, 0x76, 0x0c, 0x74, 0x6f, 0x6f, 0x6c, 0x63, 0x74, 0x72, 0x6c, 0x5f,
+  0x6c, 0x6f, 0x67, 0x00, 0x00,
+  0x03, 0x65, 0x6e, 0x76, 0x15, 0x74, 0x6f, 0x6f, 0x6c, 0x63, 0x74, 0x72, 0x6c, 0x5f,
+  0x61, 0x75, 0x64, 0x69, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x00, 0x00,
+  0x03, 0x02, 0x01, 0x01,
+  0x07, 0x07, 0x01, 0x03, 0x72, 0x75, 0x6e, 0x00, 0x02,
+  0x0a, 0x0e, 0x01, 0x0c, 0x00, 0x20, 0x00, 0x10, 0x00, 0x20, 0x00, 0x10, 0x01, 0x20,
+  0x00, 0x0b,
+]);
+
+const manifest: PluginManifest = {
+  schemaVersion: 1,
+  id: "de.toolctrl.timeout-test",
+  version: "1.0.0",
+  name: "Timeout Test",
+  entry: "plugin.wasm",
+  wasmSha256: "0".repeat(64),
+  signature: "0".repeat(128),
+  signedBy: "0".repeat(64),
+  allowedHosts: [],
+  capabilities: [],
+  hooks: [{ name: "run", export: "run" }],
+};
+
+const manifestWithHostImports: PluginManifest = {
+  ...manifest,
+  capabilities: [{ name: "log" }, { name: "audit" }],
+};
+
+describe("invokePluginSandboxInWorker", () => {
+  test("fails closed in production when the compiled sandbox worker is missing", async () => {
+    let sourceBuildAttempted = false;
+    await expect(
+      resolveSandboxWorkerLocationForTest({
+        compiledCandidates: ["/missing/plugin-sandbox-worker.js"],
+        exists: () => false,
+        isProduction: true,
+        buildSource: async () => {
+          sourceBuildAttempted = true;
+          return new URL("data:text/javascript,");
+        },
+      })
+    ).rejects.toThrow("Compiled plugin sandbox worker is required in production.");
+    expect(sourceBuildAttempted).toBe(false);
+  });
+
+  test("terminates non-yielding WASM after the wall-clock budget", async () => {
+    const startedAt = Date.now();
+    const response = await invokePluginSandboxInWorker({
+      wasmBuffer: INFINITE_LOOP_WASM,
+      manifest,
+      capabilityGrants: [],
+      hookName: "run",
+      payload: undefined,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000_000,
+      wallClockMs: 50,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "timeout",
+      message: "plugin sandbox wall-clock timeout",
+    });
+    expect(Date.now() - startedAt).toBeLessThan(2_000);
+  }, 5_000);
+
+  test("enforces host import fuel before host logs can grow unbounded", async () => {
+    const response = await runSandboxForTest({
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifestWithHostImports,
+      capabilityGrants: [
+        { capabilityName: "log", scopeJson: null },
+        { capabilityName: "audit", scopeJson: null },
+      ],
+      hookName: "run",
+      payload: 7,
+      memoryLimitMb: 64,
+      fuelLimit: 100,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "fuel_exhausted",
+      fuelUsed: 100,
+      message: "plugin host import fuel exhausted",
+    });
+    expect(response.hostLogs).toHaveLength(1);
+    expect(response.capabilityCalls).toHaveLength(1);
+    expect(response.droppedCapabilityCallCount).toBe(0);
+  }, 5_000);
+
+  test("denies audit host binding when only log is granted", async () => {
+    const response = await invokePluginSandboxInWorker({
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifestWithHostImports,
+      capabilityGrants: [{ capabilityName: "log", scopeJson: null }],
+      hookName: "run",
+      payload: 7,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000,
+      wallClockMs: 1_000,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "trap",
+      message: expect.stringContaining("capability denied: audit (not_granted_for_tenant)"),
+    });
+    expect(response.capabilityCalls).toEqual([
+      expect.objectContaining({ capability: "log", outcome: "success" }),
+      expect.objectContaining({ capability: "audit", outcome: "denied", errorCode: "not_granted_for_tenant" }),
+    ]);
+    expect(response.hostLogs).toEqual([
+      expect.objectContaining({ level: "info", message: "plugin.invoke.host.log" }),
+      expect.objectContaining({
+        level: "warn",
+        message: "plugin.invoke.capability_denied",
+        record: expect.objectContaining({ capability: "audit", reason: "not_granted_for_tenant" }),
+      }),
+    ]);
+  }, 5_000);
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-sandbox-isolated.ts

@@ -0,0 +1,172 @@
+import { Worker } from "node:worker_threads";
+import { existsSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+import type { CapabilityCallInfo, PluginCapabilityGrant, PluginInvocationResult, PluginManifest } from "@toolctrl/plugin-runtime";
+
+import type { SandboxWorkerResponse } from "./plugin-sandbox-worker.js";
+
+export type PluginSandboxHostLogEntry = SandboxWorkerResponse["hostLogs"][number];
+
+export type PluginSandboxIsolatedResult = {
+  result: PluginInvocationResult;
+  capabilityCalls: CapabilityCallInfo[];
+  droppedCapabilityCallCount: number;
+  hostLogs: PluginSandboxHostLogEntry[];
+};
+
+type InvokePluginSandboxInWorkerArgs = {
+  wasmBuffer: Uint8Array;
+  manifest: PluginManifest;
+  capabilityGrants: ReadonlyArray<PluginCapabilityGrant>;
+  hookName: string;
+  payload: unknown;
+  memoryLimitMb: number;
+  fuelLimit: number;
+  maxCapabilityCallEntries?: number;
+  wallClockMs: number;
+  workerUrl?: URL;
+};
+
+const DEFAULT_TIMEOUT_MESSAGE = "plugin sandbox wall-clock timeout";
+
+type SandboxWorkerLocation = {
+  url: URL;
+  options?: ConstructorParameters<typeof Worker>[1];
+};
+
+type SandboxWorkerLocationDeps = {
+  compiledCandidates?: readonly string[];
+  sourcePath?: string;
+  exists?: (path: string) => boolean;
+  isProduction?: boolean;
+  buildSource?: (sourcePath: string) => Promise<URL>;
+};
+
+function getCurrentDir(): string {
+  return dirname(fileURLToPath(import.meta.url));
+}
+
+function getCompiledSandboxWorkerCandidates(): string[] {
+  const currentDir = getCurrentDir();
+  const candidates = [
+    join(currentDir, "plugin-sandbox-worker.js"),
+    join(currentDir, "jobs", "plugin-sandbox-worker.js"),
+  ];
+  if (currentDir.endsWith(`${join("src", "jobs")}`)) {
+    candidates.push(join(dirname(dirname(currentDir)), "dist", "jobs", "plugin-sandbox-worker.js"));
+  }
+  return candidates;
+}
+
+function getSourceSandboxWorkerPath(): string {
+  return join(getCurrentDir(), "plugin-sandbox-worker.ts");
+}
+
+async function buildSourceSandboxWorkerDataUrl(sourcePath: string): Promise<URL> {
+  const { build } = await import("esbuild");
+  const result = await build({
+    entryPoints: [sourcePath],
+    bundle: true,
+    platform: "node",
+    format: "esm",
+    target: "node20",
+    write: false,
+    sourcemap: "inline",
+    logLevel: "silent",
+  });
+  const code = result.outputFiles[0]?.text;
+  if (!code) {
+    throw new Error("plugin sandbox worker bundle emitted no output");
+  }
+  return new URL(`data:text/javascript;base64,${Buffer.from(code).toString("base64")}`);
+}
+
+async function resolveSandboxWorkerLocation(workerUrl?: URL, deps: SandboxWorkerLocationDeps = {}): Promise<SandboxWorkerLocation> {
+  if (workerUrl) {
+    return { url: workerUrl };
+  }
+  const exists = deps.exists ?? existsSync;
+  for (const compiled of deps.compiledCandidates ?? getCompiledSandboxWorkerCandidates()) {
+    if (exists(compiled)) {
+      return { url: pathToFileURL(compiled) };
+    }
+  }
+  if (deps.isProduction ?? process.env.NODE_ENV === "production") {
+    throw new Error("Compiled plugin sandbox worker is required in production.");
+  }
+  const buildSource = deps.buildSource ?? buildSourceSandboxWorkerDataUrl;
+  return { url: await buildSource(deps.sourcePath ?? getSourceSandboxWorkerPath()) };
+}
+
+export async function resolveSandboxWorkerLocationForTest(deps: SandboxWorkerLocationDeps): Promise<SandboxWorkerLocation> {
+  return resolveSandboxWorkerLocation(undefined, deps);
+}
+
+function timeoutResult(message = DEFAULT_TIMEOUT_MESSAGE): PluginSandboxIsolatedResult {
+  return {
+    result: { ok: false, reason: "timeout", fuelUsed: 0, message },
+    capabilityCalls: [],
+    droppedCapabilityCallCount: 0,
+    hostLogs: [],
+  };
+}
+
+export async function invokePluginSandboxInWorker(args: InvokePluginSandboxInWorkerArgs): Promise<PluginSandboxIsolatedResult> {
+  const wallClockMs = Math.max(1, Math.floor(args.wallClockMs));
+  const workerLocation = await resolveSandboxWorkerLocation(args.workerUrl);
+  const worker = new Worker(workerLocation.url, workerLocation.options);
+
+  return await new Promise<PluginSandboxIsolatedResult>((resolve) => {
+    let settled = false;
+    const settle = (result: PluginSandboxIsolatedResult) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      worker.removeAllListeners();
+      void worker.terminate().catch(() => undefined).finally(() => resolve(result));
+    };
+
+    const timeout = setTimeout(() => {
+      settle(timeoutResult());
+    }, wallClockMs);
+
+    worker.once("message", (response: SandboxWorkerResponse) => {
+      settle({
+        ...response,
+        droppedCapabilityCallCount: response.droppedCapabilityCallCount ?? 0,
+      });
+    });
+    worker.once("error", (error) => {
+      const message = error instanceof Error ? error.message : String(error);
+      settle({
+        result: { ok: false, reason: "trap", fuelUsed: 0, message },
+        capabilityCalls: [],
+        droppedCapabilityCallCount: 0,
+        hostLogs: [],
+      });
+    });
+    worker.once("exit", (code) => {
+      if (code !== 0) {
+        settle({
+          result: { ok: false, reason: "trap", fuelUsed: 0, message: `plugin sandbox worker exited with code ${code}` },
+          capabilityCalls: [],
+          droppedCapabilityCallCount: 0,
+          hostLogs: [],
+        });
+      }
+    });
+
+    worker.postMessage({
+      wasmBuffer: args.wasmBuffer,
+      manifest: args.manifest,
+      capabilityGrants: args.capabilityGrants,
+      hookName: args.hookName,
+      payload: args.payload,
+      memoryLimitMb: args.memoryLimitMb,
+      fuelLimit: args.fuelLimit,
+      maxCapabilityCallEntries: args.maxCapabilityCallEntries,
+    });
+  });
+}

hivegrid-cloud/apps/worker/src/jobs/plugin-sandbox-worker.test.ts

@@ -0,0 +1,248 @@
+import { describe, expect, test } from "vitest";
+
+import type { PluginManifest } from "@toolctrl/plugin-runtime";
+
+import { invokePluginSandboxInWorker } from "./plugin-sandbox-isolated.js";
+import { createPluginSandboxHostFns, runSandboxForTest } from "./plugin-sandbox-worker.js";
+
+const DEFERRED_HOST_BINDINGS = [
+  "http.outbound",
+  "secret.read",
+  "db.read",
+  "db.write",
+  "storage.read",
+  "storage.write",
+  "tenant.info",
+] as const;
+
+const UNREACHABLE_WASM = new Uint8Array([
+  0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
+  0x01, 0x04, 0x01, 0x60, 0x00, 0x00,
+  0x03, 0x02, 0x01, 0x00,
+  0x07, 0x07, 0x01, 0x03, 0x72, 0x75, 0x6e, 0x00, 0x00,
+  0x0a, 0x05, 0x01, 0x03, 0x00, 0x00, 0x0b,
+]);
+
+const HOST_IMPORT_WASM = new Uint8Array([
+  0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00,
+  0x01, 0x0a, 0x02, 0x60, 0x01, 0x7f, 0x00, 0x60, 0x01, 0x7f, 0x01, 0x7f,
+  0x02, 0x30, 0x02,
+  0x03, 0x65, 0x6e, 0x76, 0x0c, 0x74, 0x6f, 0x6f, 0x6c, 0x63, 0x74, 0x72, 0x6c, 0x5f,
+  0x6c, 0x6f, 0x67, 0x00, 0x00,
+  0x03, 0x65, 0x6e, 0x76, 0x15, 0x74, 0x6f, 0x6f, 0x6c, 0x63, 0x74, 0x72, 0x6c, 0x5f,
+  0x61, 0x75, 0x64, 0x69, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x00, 0x00,
+  0x03, 0x02, 0x01, 0x01,
+  0x07, 0x07, 0x01, 0x03, 0x72, 0x75, 0x6e, 0x00, 0x02,
+  0x0a, 0x0e, 0x01, 0x0c, 0x00, 0x20, 0x00, 0x10, 0x00, 0x20, 0x00, 0x10, 0x01, 0x20,
+  0x00, 0x0b,
+]);
+
+function manifest(capabilities: PluginManifest["capabilities"] = []): PluginManifest {
+  return {
+    schemaVersion: 1,
+    id: "de.toolctrl.worker-test",
+    version: "1.0.0",
+    name: "Worker Test",
+    entry: "plugin.wasm",
+    wasmSha256: "0".repeat(64),
+    signature: "0".repeat(128),
+    signedBy: "0".repeat(64),
+    allowedHosts: [],
+    capabilities,
+    hooks: [{ name: "run", export: "run" }],
+  };
+}
+
+function expectFailedResultMessage(result: { ok: boolean; message?: unknown }): string {
+  expect(result.ok).toBe(false);
+  expect(result.message).toEqual(expect.any(String));
+  return result.message as string;
+}
+
+describe("plugin sandbox worker", () => {
+  test("keeps deferred capabilities out of the production host-function map", () => {
+    const hostFns = createPluginSandboxHostFns({
+      manifest: manifest(),
+      pushHostLog: () => undefined,
+    });
+
+    expect(Object.keys(hostFns).sort()).toEqual(["audit", "log"]);
+    for (const capability of DEFERRED_HOST_BINDINGS) {
+      expect(hostFns).not.toHaveProperty(capability);
+    }
+  });
+
+  test("returns compile_failed for malformed wasm without host side effects", async () => {
+    const response = await runSandboxForTest({
+      wasmBuffer: new Uint8Array([0xde, 0xad, 0xbe, 0xef]),
+      manifest: manifest(),
+      capabilityGrants: [],
+      hookName: "run",
+      payload: 1,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "compile_failed",
+      fuelUsed: 0,
+    });
+    expectFailedResultMessage(response.result);
+    expect(response.capabilityCalls).toEqual([]);
+    expect(response.hostLogs).toEqual([]);
+  });
+
+  test("preserves wasm trap reason and diagnostic message shape", async () => {
+    const response = await runSandboxForTest({
+      wasmBuffer: UNREACHABLE_WASM,
+      manifest: manifest(),
+      capabilityGrants: [],
+      hookName: "run",
+      payload: undefined,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "trap",
+    });
+    expect(expectFailedResultMessage(response.result)).toMatch(/unreachable/i);
+    expect(response.capabilityCalls).toEqual([]);
+    expect(response.hostLogs).toEqual([]);
+  });
+
+  test("denies manifest-declared host imports that lack tenant grants", async () => {
+    const response = await runSandboxForTest({
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifest([{ name: "log" }, { name: "audit" }]),
+      capabilityGrants: [{ capabilityName: "log", scopeJson: null }],
+      hookName: "run",
+      payload: 7,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "trap",
+    });
+    expect(expectFailedResultMessage(response.result)).toContain("capability denied: audit (not_granted_for_tenant)");
+    expect(response.capabilityCalls).toEqual([
+      expect.objectContaining({ capability: "log", outcome: "success" }),
+      expect.objectContaining({ capability: "audit", outcome: "denied", errorCode: "not_granted_for_tenant" }),
+    ]);
+    expect(response.hostLogs).toEqual([
+      expect.objectContaining({ level: "info", message: "plugin.invoke.host.log" }),
+      expect.objectContaining({
+        level: "warn",
+        message: "plugin.invoke.capability_denied",
+        record: expect.objectContaining({ capability: "audit", reason: "not_granted_for_tenant" }),
+      }),
+    ]);
+  });
+
+  test("does not let tenant grants bypass manifest capability declarations", async () => {
+    const response = await runSandboxForTest({
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifest(),
+      capabilityGrants: [
+        { capabilityName: "log", scopeJson: null },
+        { capabilityName: "audit", scopeJson: null },
+      ],
+      hookName: "run",
+      payload: 7,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "trap",
+    });
+    expect(expectFailedResultMessage(response.result)).toContain("capability denied: log (not_declared_in_manifest)");
+    expect(response.capabilityCalls).toEqual([
+      expect.objectContaining({ capability: "log", outcome: "denied", errorCode: "not_declared_in_manifest" }),
+    ]);
+    expect(response.hostLogs).toEqual([
+      expect.objectContaining({
+        level: "warn",
+        message: "plugin.invoke.capability_denied",
+        record: expect.objectContaining({ capability: "log", reason: "not_declared_in_manifest" }),
+      }),
+    ]);
+  });
+
+  test("denies undeclared host imports even when another capability is granted", async () => {
+    const response = await runSandboxForTest({
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifest([{ name: "log" }]),
+      capabilityGrants: [{ capabilityName: "log", scopeJson: null }],
+      hookName: "run",
+      payload: 7,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000,
+    });
+
+    expect(response.result).toMatchObject({
+      ok: false,
+      reason: "trap",
+    });
+    expect(expectFailedResultMessage(response.result)).toContain("capability denied: audit (not_declared_in_manifest)");
+    expect(response.capabilityCalls).toEqual([
+      expect.objectContaining({ capability: "log", outcome: "success" }),
+      expect.objectContaining({ capability: "audit", outcome: "denied", errorCode: "not_declared_in_manifest" }),
+    ]);
+    expect(response.hostLogs.at(-1)).toEqual(
+      expect.objectContaining({
+        level: "warn",
+        message: "plugin.invoke.capability_denied",
+        record: expect.objectContaining({ capability: "audit", reason: "not_declared_in_manifest" }),
+      })
+    );
+  });
+
+  test("worker catch path posts trap responses for malformed sandbox limits", async () => {
+    const response = await invokePluginSandboxInWorker({
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifest([{ name: "log" }, { name: "audit" }]),
+      capabilityGrants: [
+        { capabilityName: "log", scopeJson: null },
+        { capabilityName: "audit", scopeJson: null },
+      ],
+      hookName: "run",
+      payload: 7,
+      memoryLimitMb: 64,
+      fuelLimit: 0,
+      wallClockMs: 1_000,
+    });
+
+    expect(response).toEqual({
+      result: { ok: false, reason: "trap", fuelUsed: 0, message: "fuelLimit must be positive" },
+      capabilityCalls: [],
+      droppedCapabilityCallCount: 0,
+      hostLogs: [],
+    });
+  }, 5_000);
+
+  test("reports dropped capability audit entries when host calls exceed the worker cap", async () => {
+    const response = await runSandboxForTest({
+      wasmBuffer: HOST_IMPORT_WASM,
+      manifest: manifest([{ name: "log" }, { name: "audit" }]),
+      capabilityGrants: [
+        { capabilityName: "log", scopeJson: null },
+        { capabilityName: "audit", scopeJson: null },
+      ],
+      hookName: "run",
+      payload: 7,
+      memoryLimitMb: 64,
+      fuelLimit: 1_000,
+      maxCapabilityCallEntries: 1,
+    });
+
+    expect(response.result).toMatchObject({ ok: true, output: 7 });
+    expect(response.capabilityCalls).toEqual([expect.objectContaining({ capability: "log", outcome: "success" })]);
+    expect(response.droppedCapabilityCallCount).toBe(1);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/plugin-sandbox-worker.ts

@@ -0,0 +1,158 @@
+import { parentPort } from "node:worker_threads";
+
+import {
+  createCapabilityGateway,
+  createPluginSandbox,
+  type CapabilityCallInfo,
+  type CapabilityHostFnMap,
+  type PluginCapabilityGrant,
+  type PluginInvocationResult,
+  type PluginManifest,
+} from "@toolctrl/plugin-runtime";
+
+type SandboxWorkerRequest = {
+  wasmBuffer: Uint8Array;
+  manifest: PluginManifest;
+  capabilityGrants: ReadonlyArray<PluginCapabilityGrant>;
+  hookName: string;
+  payload: unknown;
+  memoryLimitMb: number;
+  fuelLimit: number;
+  maxCapabilityCallEntries?: number;
+};
+
+type HostLogEntry = {
+  level: "info" | "warn";
+  record: Record<string, unknown>;
+  message: string;
+};
+
+const HOST_IMPORT_FUEL_COST = 100;
+const MAX_HOST_LOG_ENTRIES = 1_024;
+const MAX_CAPABILITY_CALL_ENTRIES = 2_048;
+
+export type SandboxWorkerResponse = {
+  result: PluginInvocationResult;
+  capabilityCalls: CapabilityCallInfo[];
+  droppedCapabilityCallCount: number;
+  hostLogs: HostLogEntry[];
+};
+
+function stringifyError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export function createPluginSandboxHostFns(args: {
+  manifest: PluginManifest;
+  pushHostLog: (entry: HostLogEntry) => void;
+}): CapabilityHostFnMap {
+  const { manifest, pushHostLog } = args;
+  return {
+    log: (value: unknown) => {
+      pushHostLog({ level: "info", record: { pluginId: manifest.id, version: manifest.version, value }, message: "plugin.invoke.host.log" });
+      return 0;
+    },
+    audit: (value: unknown) => {
+      pushHostLog({ level: "info", record: { pluginId: manifest.id, version: manifest.version, value }, message: "plugin.invoke.host.audit" });
+      return 0;
+    },
+  };
+}
+
+export async function runSandboxForTest({
+  wasmBuffer,
+  manifest,
+  capabilityGrants,
+  hookName,
+  payload,
+  memoryLimitMb,
+  fuelLimit,
+  maxCapabilityCallEntries = MAX_CAPABILITY_CALL_ENTRIES,
+}: SandboxWorkerRequest): Promise<SandboxWorkerResponse> {
+  const capabilityCalls: CapabilityCallInfo[] = [];
+  const hostLogs: HostLogEntry[] = [];
+  const capabilityCallEntryLimit = Number.isFinite(maxCapabilityCallEntries)
+    ? Math.max(0, Math.floor(maxCapabilityCallEntries))
+    : MAX_CAPABILITY_CALL_ENTRIES;
+  let droppedCapabilityCallCount = 0;
+  let hostFuelExhausted = false;
+  const pushHostLog = (entry: HostLogEntry) => {
+    if (hostLogs.length < MAX_HOST_LOG_ENTRIES) {
+      hostLogs.push(entry);
+    }
+  };
+  const hostFns = createPluginSandboxHostFns({ manifest, pushHostLog });
+  const gateway = createCapabilityGateway({
+    manifest,
+    approvedGrants: capabilityGrants,
+    hostFns,
+    fuelCostPerCall: HOST_IMPORT_FUEL_COST,
+    onDenied: (info) => {
+      pushHostLog({
+        level: "warn",
+        record: { pluginId: manifest.id, version: manifest.version, capability: info.capability, reason: info.reason },
+        message: "plugin.invoke.capability_denied",
+      });
+    },
+    onCall: (info) => {
+      if (capabilityCalls.length < capabilityCallEntryLimit) {
+        capabilityCalls.push(info);
+      } else {
+        droppedCapabilityCallCount += 1;
+      }
+    },
+  });
+  const invokeHostSync = (capability: "log" | "audit", value: number) => {
+    if (gateway.fuelCharged() + HOST_IMPORT_FUEL_COST > fuelLimit) {
+      hostFuelExhausted = true;
+      throw new Error("plugin host import fuel exhausted");
+    }
+    return gateway.invokeSync(capability, { value });
+  };
+  const sandbox = createPluginSandbox({
+    wasmBuffer,
+    manifest,
+    memoryLimitMb,
+    fuelLimit,
+    hostImports: {
+      env: {
+        toolctrl_log: (value: number) => invokeHostSync("log", value),
+        toolctrl_audit_record: (value: number) => invokeHostSync("audit", value),
+      },
+    },
+  });
+
+  try {
+    const result = await sandbox.invoke(hookName, payload);
+    const fuelUsed = Math.min(fuelLimit, result.fuelUsed + gateway.fuelCharged());
+    if (hostFuelExhausted) {
+      return {
+        result: { ok: false, reason: "fuel_exhausted", fuelUsed, message: "plugin host import fuel exhausted" },
+        capabilityCalls,
+        droppedCapabilityCallCount,
+        hostLogs,
+      };
+    }
+    return {
+      result: result.ok ? { ...result, fuelUsed } : { ...result, fuelUsed },
+      capabilityCalls,
+      droppedCapabilityCallCount,
+      hostLogs,
+    };
+  } finally {
+    sandbox.close();
+  }
+}
+
+parentPort?.on("message", (request: SandboxWorkerRequest) => {
+  void runSandboxForTest(request)
+    .then((response) => parentPort?.postMessage(response))
+    .catch((error: unknown) =>
+      parentPort?.postMessage({
+        result: { ok: false, reason: "trap", fuelUsed: 0, message: stringifyError(error) },
+        capabilityCalls: [],
+        droppedCapabilityCallCount: 0,
+        hostLogs: [],
+      } satisfies SandboxWorkerResponse)
+    );
+});

hivegrid-cloud/apps/worker/src/jobs/push-notification.test.ts

@@ -0,0 +1,212 @@
+import { describe, it, expect, vi } from "vitest";
+import {
+  dispatchPushNotification,
+  redactToken,
+  REVOCATION_ERROR_CODES,
+  type DeviceTokenRow,
+  type PushFanOutInput,
+  type PushPayload,
+  type PushProviderSeam,
+} from "./push-notification.js";
+
+function makeRow(overrides: Partial<DeviceTokenRow>): DeviceTokenRow {
+  return {
+    id: overrides.id ?? "dt-1",
+    token: overrides.token ?? "tokAAAAAAAAAAAAAA",
+    userId: overrides.userId ?? "u-1",
+    tenantId: overrides.tenantId ?? null,
+    platform: overrides.platform ?? "ios",
+    pushProvider: overrides.pushProvider ?? "apns",
+    revokedAt: overrides.revokedAt ?? null,
+  };
+}
+
+function makePayload(overrides: Partial<PushPayload> = {}): PushPayload {
+  return {
+    userId: overrides.userId ?? "u-1",
+    tenantId: overrides.tenantId ?? null,
+    title: overrides.title ?? "Hello",
+    body: overrides.body ?? "World",
+    data: overrides.data,
+  };
+}
+
+function makeProviders(overrides: Partial<PushProviderSeam> = {}): PushProviderSeam {
+  return {
+    sendApns: overrides.sendApns ?? vi.fn().mockResolvedValue({ success: true, messageId: "apns-1" }),
+    sendFcm: overrides.sendFcm ?? vi.fn().mockResolvedValue({ success: true, messageId: "fcm-1" }),
+    sendWebPush: overrides.sendWebPush ?? vi.fn().mockResolvedValue({ success: true, messageId: "web-1" }),
+  };
+}
+
+function deferred() {
+  let resolve!: () => void;
+  const promise = new Promise<void>((res) => {
+    resolve = res;
+  });
+  return { promise, resolve };
+}
+
+describe("dispatchPushNotification", () => {
+  it("returns zero attempts when no active tokens exist", async () => {
+    const input: PushFanOutInput = {
+      payload: makePayload(),
+      tokenLister: async () => [],
+      providers: makeProviders(),
+      tokenRevoker: vi.fn(),
+    };
+    const result = await dispatchPushNotification(input);
+    expect(result).toEqual({ attempted: 0, succeeded: 0, revoked: 0, failed: 0 });
+  });
+
+  it("filters out tokens with revokedAt set", async () => {
+    const providers = makeProviders();
+    const input: PushFanOutInput = {
+      payload: makePayload(),
+      tokenLister: async () => [
+        makeRow({ token: "t-active" }),
+        makeRow({ id: "dt-2", token: "t-revoked", revokedAt: new Date() }),
+      ],
+      providers,
+      tokenRevoker: vi.fn(),
+    };
+    const result = await dispatchPushNotification(input);
+    expect(result.attempted).toBe(1);
+    expect(result.succeeded).toBe(1);
+    expect(providers.sendApns).toHaveBeenCalledTimes(1);
+  });
+
+  it("routes ios tokens to APNs, android to FCM, web to WebPush", async () => {
+    const providers = makeProviders();
+    const input: PushFanOutInput = {
+      payload: makePayload(),
+      tokenLister: async () => [
+        makeRow({ token: "tok-ios", platform: "ios", pushProvider: "apns" }),
+        makeRow({ id: "dt-2", token: "tok-and", platform: "android", pushProvider: "fcm" }),
+        makeRow({ id: "dt-3", token: "tok-web", platform: "web", pushProvider: "webpush" }),
+      ],
+      providers,
+      tokenRevoker: vi.fn(),
+    };
+    const result = await dispatchPushNotification(input);
+    expect(result.succeeded).toBe(3);
+    expect(providers.sendApns).toHaveBeenCalledWith("tok-ios", expect.any(Object));
+    expect(providers.sendFcm).toHaveBeenCalledWith("tok-and", expect.any(Object));
+    expect(providers.sendWebPush).toHaveBeenCalledWith("tok-web", expect.any(Object));
+  });
+
+  it("starts multiple provider sends before the first delayed send resolves", async () => {
+    const releases = Array.from({ length: 3 }, () => deferred());
+    const started: string[] = [];
+    const completed: string[] = [];
+    let sendIndex = 0;
+    const providers = makeProviders({
+      sendApns: vi.fn(async (token) => {
+        const release = releases[sendIndex++]!;
+        started.push(token);
+        await release.promise;
+        completed.push(token);
+        return { success: true, messageId: `apns-${token}` };
+      }),
+    });
+    const input: PushFanOutInput = {
+      payload: makePayload(),
+      tokenLister: async () => [
+        makeRow({ id: "dt-1", token: "tok-one" }),
+        makeRow({ id: "dt-2", token: "tok-two" }),
+        makeRow({ id: "dt-3", token: "tok-three" }),
+      ],
+      providers,
+      tokenRevoker: vi.fn(),
+    };
+
+    const resultPromise = dispatchPushNotification(input);
+    try {
+      await vi.waitFor(
+        () => {
+          expect(started.length).toBeGreaterThanOrEqual(2);
+          expect(completed).toEqual([]);
+        },
+        { timeout: 100 }
+      );
+    } finally {
+      for (const release of releases) {
+        release.resolve();
+      }
+    }
+
+    await expect(resultPromise).resolves.toEqual({ attempted: 3, succeeded: 3, revoked: 0, failed: 0 });
+  });
+
+  it("revokes tokens when the provider reports an irrecoverable error code", async () => {
+    const revoker = vi.fn();
+    const providers = makeProviders({
+      sendApns: vi.fn().mockResolvedValue({ success: false, errorCode: "Unregistered" }),
+    });
+    const input: PushFanOutInput = {
+      payload: makePayload(),
+      tokenLister: async () => [makeRow({ token: "tok-dead" })],
+      providers,
+      tokenRevoker: revoker,
+    };
+    const result = await dispatchPushNotification(input);
+    expect(result.revoked).toBe(1);
+    expect(result.failed).toBe(0);
+    expect(revoker).toHaveBeenCalledWith("tok-dead");
+  });
+
+  it("counts other errors as failed without revoking", async () => {
+    const revoker = vi.fn();
+    const providers = makeProviders({
+      sendApns: vi.fn().mockResolvedValue({ success: false, errorCode: "InternalServerError" }),
+    });
+    const input: PushFanOutInput = {
+      payload: makePayload(),
+      tokenLister: async () => [makeRow({ token: "tok-flaky" })],
+      providers,
+      tokenRevoker: revoker,
+    };
+    const result = await dispatchPushNotification(input);
+    expect(result.failed).toBe(1);
+    expect(result.revoked).toBe(0);
+    expect(revoker).not.toHaveBeenCalled();
+  });
+
+  it("emits an info log on successful delivery and a warn log on revocation", async () => {
+    const info = vi.fn();
+    const warn = vi.fn();
+    const providers = makeProviders({
+      sendApns: vi.fn()
+        .mockResolvedValueOnce({ success: true, messageId: "id-1" })
+        .mockResolvedValueOnce({ success: false, errorCode: "BadDeviceToken" }),
+    });
+    const input: PushFanOutInput = {
+      payload: makePayload(),
+      tokenLister: async () => [
+        makeRow({ id: "a", token: "tok-good" }),
+        makeRow({ id: "b", token: "tok-bad" }),
+      ],
+      providers,
+      tokenRevoker: vi.fn(),
+      logger: { info, warn },
+    };
+    await dispatchPushNotification(input);
+    expect(info).toHaveBeenCalledWith(expect.objectContaining({ messageId: "id-1" }), "push delivered");
+    expect(warn).toHaveBeenCalledWith(expect.objectContaining({ errorCode: "BadDeviceToken" }), "push provider reported token revoked");
+  });
+});
+
+describe("redactToken", () => {
+  it("emits an ellipsis-truncated prefix", () => {
+    expect(redactToken("abcdefghij")).toBe("abcdefgh…");
+    expect(redactToken("short")).toBe("sh…");
+  });
+});
+
+describe("REVOCATION_ERROR_CODES", () => {
+  it("includes both APNs and FCM canonical errors", () => {
+    expect(REVOCATION_ERROR_CODES.has("Unregistered")).toBe(true);
+    expect(REVOCATION_ERROR_CODES.has("BadDeviceToken")).toBe(true);
+    expect(REVOCATION_ERROR_CODES.has("messaging/registration-token-not-registered")).toBe(true);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/push-notification.ts

@@ -0,0 +1,161 @@
+/**
+ * Phase 08 #65 — Push-notification fan-out worker.
+ *
+ * Consumes events emitted via the existing notifications pipeline and
+ * dispatches one APNs / FCM / WebPush call per active DeviceToken for
+ * the recipient user. The worker is intentionally provider-pluggable:
+ * the two SDK adapters (firebase-admin for FCM, node-apn for APNs) are
+ * injected through the `providers` seam so unit tests can mock them
+ * without dragging the actual libraries into Vitest's module graph.
+ *
+ * Provider response codes that signal a permanent invalidation
+ * (`Unregistered`, `InvalidRegistration`, `BadDeviceToken`) flip the
+ * matching DeviceToken's `revokedAt` so subsequent fan-out passes skip
+ * the dead row.
+ *
+ * Bootstrap-time config (apps/worker/src/index.ts) reads:
+ *   - FCM_SERVICE_ACCOUNT_JSON   (base64 or raw JSON for the FCM key)
+ *   - APNS_KEY                   (PEM contents of the .p8 sign key)
+ *   - APNS_KEY_ID                (10-char key id from Apple Developer)
+ *   - APNS_TEAM_ID               (10-char team id)
+ *   - APNS_BUNDLE_ID             (defaults to de.toolctrl.app)
+ *
+ * @see packages/db/src/device-tokens.server.ts
+ * @see apps/mobile/src/pushRegistration.ts
+ */
+
+import { mapWithConcurrency } from "@toolctrl/foundation";
+
+export type PushTargetPlatform = "ios" | "android" | "web";
+export type PushProviderResult = {
+  /** Provider-side message id, when available. */
+  messageId?: string;
+  /** Whether the provider accepted the push. */
+  success: boolean;
+  /** Provider error code (e.g. `Unregistered`, `InvalidRegistration`). */
+  errorCode?: string;
+};
+
+export type PushPayload = {
+  /** Bearer/user/tenant context — also forwarded to the provider as APS-data. */
+  userId: string;
+  tenantId: string | null;
+  title: string;
+  body: string;
+  data?: Record<string, string>;
+};
+
+export type DeviceTokenRow = {
+  id: string;
+  token: string;
+  userId: string;
+  tenantId: string | null;
+  platform: string;
+  pushProvider: string;
+  revokedAt: Date | null;
+};
+
+export type PushProviderSeam = {
+  sendFcm: (token: string, payload: PushPayload) => Promise<PushProviderResult>;
+  sendApns: (token: string, payload: PushPayload) => Promise<PushProviderResult>;
+  sendWebPush: (token: string, payload: PushPayload) => Promise<PushProviderResult>;
+};
+
+export type PushFanOutInput = {
+  payload: PushPayload;
+  /** Lister: active (revokedAt IS NULL) tokens for the recipient. */
+  tokenLister: (args: { userId: string; tenantId: string | null }) => Promise<readonly DeviceTokenRow[]>;
+  /** Provider seam — injected by apps/worker/src/index.ts. */
+  providers: PushProviderSeam;
+  /** Revoke seam — called when the provider returns a permanent error. */
+  tokenRevoker: (token: string) => Promise<void>;
+  /** Max active device tokens sent at once. Default is intentionally conservative. */
+  concurrency?: number;
+  /** Optional logger. */
+  logger?: {
+    info: (fields: Record<string, unknown>, msg: string) => void;
+    warn?: (fields: Record<string, unknown>, msg: string) => void;
+    error?: (fields: Record<string, unknown>, msg: string) => void;
+  };
+};
+
+export type PushFanOutResult = {
+  attempted: number;
+  succeeded: number;
+  revoked: number;
+  failed: number;
+};
+
+const REVOCATION_ERROR_CODES = new Set([
+  "Unregistered",
+  "InvalidRegistration",
+  "BadDeviceToken",
+  "DeviceTokenNotForTopic",
+  "messaging/registration-token-not-registered",
+  "messaging/invalid-registration-token",
+]);
+
+export const PUSH_NOTIFICATION_FAN_OUT_CONCURRENCY = 5;
+
+/**
+ * Fan out a single notification to every active device for the user.
+ *
+ * Pure orchestrator — no DB / SDK imports — so it can be unit-tested with
+ * Map-backed stubs. The worker entry point wires the seams against the
+ * real Prisma client and provider SDKs.
+ */
+export async function dispatchPushNotification(input: PushFanOutInput): Promise<PushFanOutResult> {
+  const tokens = await input.tokenLister({ userId: input.payload.userId, tenantId: input.payload.tenantId });
+  const active = tokens.filter((row) => row.revokedAt === null);
+  let succeeded = 0;
+  let revoked = 0;
+  let failed = 0;
+
+  await mapWithConcurrency(active, input.concurrency ?? PUSH_NOTIFICATION_FAN_OUT_CONCURRENCY, async (row) => {
+    const result = await sendForRow(row, input.payload, input.providers);
+    if (result.success) {
+      succeeded += 1;
+      input.logger?.info(
+        { token: redactToken(row.token), platform: row.platform, provider: row.pushProvider, messageId: result.messageId },
+        "push delivered",
+      );
+      return;
+    }
+
+    if (result.errorCode && REVOCATION_ERROR_CODES.has(result.errorCode)) {
+      await input.tokenRevoker(row.token);
+      revoked += 1;
+      input.logger?.warn?.(
+        { token: redactToken(row.token), platform: row.platform, errorCode: result.errorCode },
+        "push provider reported token revoked",
+      );
+      return;
+    }
+
+    failed += 1;
+    input.logger?.error?.(
+      { token: redactToken(row.token), platform: row.platform, errorCode: result.errorCode },
+      "push delivery failed",
+    );
+  });
+
+  return { attempted: active.length, succeeded, revoked, failed };
+}
+
+async function sendForRow(row: DeviceTokenRow, payload: PushPayload, providers: PushProviderSeam): Promise<PushProviderResult> {
+  if (row.platform === "ios" || row.pushProvider === "apns") {
+    return providers.sendApns(row.token, payload);
+  }
+  if (row.platform === "android" || row.pushProvider === "fcm") {
+    return providers.sendFcm(row.token, payload);
+  }
+  return providers.sendWebPush(row.token, payload);
+}
+
+/** Redacts everything but the leading 8 chars of a token for log lines. */
+export function redactToken(token: string): string {
+  if (token.length <= 8) return `${token.slice(0, 2)}…`;
+  return `${token.slice(0, 8)}…`;
+}
+
+export { REVOCATION_ERROR_CODES };

hivegrid-cloud/apps/worker/src/jobs/search-indexer-bootstrap.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, it, vi } from "vitest";
+import { createSearchClient } from "@toolctrl/search";
+import { createInMemoryMeili } from "@toolctrl/search/testing";
+import { bootstrapSearchIndexer } from "./search-indexer-bootstrap.js";
+import { SEARCH_INDEX_OUTBOX_QUEUE } from "./search-indexer.js";
+
+function fakeWorker() {
+  const close = vi.fn(async () => undefined);
+  const handle = { name: "fake-worker", close };
+  const createWorkerImpl = vi.fn(() => handle);
+  return { handle, createWorkerImpl };
+}
+
+describe("bootstrapSearchIndexer", () => {
+  it("returns disabled when MEILI_URL is unset", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapSearchIndexer({ WORKER_QUEUE_GROUP: "general" }, { createWorkerImpl });
+    expect(result.status).toBe("disabled");
+    if (result.status === "disabled") expect(result.reason).toMatch(/MEILI_URL/);
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  it("returns disabled when SEARCH_INDEXER_ENABLED is 'false'", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapSearchIndexer(
+      { WORKER_QUEUE_GROUP: "general", MEILI_URL: "http://x", SEARCH_INDEXER_ENABLED: "false" },
+      { createWorkerImpl }
+    );
+    expect(result.status).toBe("disabled");
+  });
+
+  it("returns disabled in non-general groups", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const result = bootstrapSearchIndexer({ WORKER_QUEUE_GROUP: "mail", MEILI_URL: "http://x" }, { createWorkerImpl });
+    expect(result.status).toBe("disabled");
+    expect(createWorkerImpl).not.toHaveBeenCalled();
+  });
+
+  it("subscribes the worker to the outbox.search-index queue when enabled", () => {
+    const { createWorkerImpl, handle } = fakeWorker();
+    const meili = createInMemoryMeili();
+    const searchClient = createSearchClient({ url: "http://memory", meiliFactory: meili.factory });
+    const result = bootstrapSearchIndexer(
+      { WORKER_QUEUE_GROUP: "general", MEILI_URL: "http://memory" },
+      { createWorkerImpl, searchClient }
+    );
+    expect(result.status).toBe("started");
+    expect(createWorkerImpl).toHaveBeenCalledWith(
+      SEARCH_INDEX_OUTBOX_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 5 })
+    );
+    if (result.status === "started") {
+      expect(result.handle).toBe(handle);
+    }
+  });
+
+  it("honours SEARCH_INDEXER_CONCURRENCY override", () => {
+    const { createWorkerImpl } = fakeWorker();
+    const meili = createInMemoryMeili();
+    const searchClient = createSearchClient({ url: "http://memory", meiliFactory: meili.factory });
+    bootstrapSearchIndexer(
+      { WORKER_QUEUE_GROUP: "general", MEILI_URL: "http://memory", SEARCH_INDEXER_CONCURRENCY: "12" },
+      { createWorkerImpl, searchClient }
+    );
+    expect(createWorkerImpl).toHaveBeenCalledWith(
+      SEARCH_INDEX_OUTBOX_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 12 })
+    );
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/search-indexer-bootstrap.ts

@@ -0,0 +1,106 @@
+/**
+ * Bootstrap helper that starts a dedicated BullMQ worker subscribed to
+ * the `outbox.search-index` queue inside the worker-general process.
+ *
+ * The worker is gated on:
+ *   - WORKER_QUEUE_GROUP === "general" (other groups never touch search)
+ *   - MEILI_URL set (otherwise the indexer cannot connect)
+ *   - SEARCH_INDEXER_ENABLED !== "false" (escape hatch)
+ *
+ * Returns `{ status: "disabled" }` when any precondition fails. Tests
+ * inject a fake `createWorkerImpl` to assert subscription semantics
+ * without spinning up a BullMQ Worker.
+ */
+
+import type { EnvRecord } from "@toolctrl/config";
+import { createSearchClient, type SearchClient } from "@toolctrl/search";
+import {
+  runSearchIndexerJob,
+  SEARCH_INDEX_OUTBOX_QUEUE,
+  type SearchIndexerDependencies,
+  type SearchIndexerLogger,
+} from "./search-indexer.js";
+
+export type SearchIndexerBootstrapEnv = EnvRecord & {
+  MEILI_URL?: string;
+  MEILI_API_KEY?: string;
+  SEARCH_INDEXER_ENABLED?: string;
+  WORKER_QUEUE_GROUP?: string;
+};
+
+export interface SearchIndexerWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface SearchIndexerBootstrapDeps {
+  readonly logger?: SearchIndexerLogger;
+  readonly bulkPagers?: SearchIndexerDependencies["bulkPagers"];
+  readonly searchClient?: SearchClient;
+  /** Override the BullMQ worker constructor so tests can inject a fake. */
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number }
+  ) => SearchIndexerWorkerHandle;
+}
+
+export type SearchIndexerBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: SearchIndexerWorkerHandle };
+
+export function bootstrapSearchIndexer(
+  env: SearchIndexerBootstrapEnv,
+  deps: SearchIndexerBootstrapDeps = {}
+): SearchIndexerBootstrapResult {
+  const logger = deps.logger;
+
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "general") {
+    return { status: "disabled", reason: "non-general worker group" };
+  }
+  if (env.SEARCH_INDEXER_ENABLED === "false") {
+    return { status: "disabled", reason: "SEARCH_INDEXER_ENABLED=false" };
+  }
+  if (!env.MEILI_URL) {
+    logger?.info?.({ component: "search-indexer" }, "search-indexer disabled: MEILI_URL unset");
+    return { status: "disabled", reason: "MEILI_URL unset" };
+  }
+
+  const client =
+    deps.searchClient ?? createSearchClient({ url: env.MEILI_URL, apiKey: env.MEILI_API_KEY ?? undefined });
+
+  const concurrency = parsePositiveInt(env.SEARCH_INDEXER_CONCURRENCY, 5);
+
+  const processor = async (job: { data: unknown }): Promise<unknown> => {
+    return runSearchIndexerJob(job.data, {
+      searchClient: client,
+      bulkPagers: deps.bulkPagers,
+      logger,
+    });
+  };
+
+  const createWorkerImpl = deps.createWorkerImpl ?? defaultCreateWorker;
+  const handle = createWorkerImpl(SEARCH_INDEX_OUTBOX_QUEUE, processor, { concurrency });
+
+  logger?.info?.(
+    { component: "search-indexer", queue: SEARCH_INDEX_OUTBOX_QUEUE, concurrency },
+    "search-indexer worker started"
+  );
+  return { status: "started", handle };
+}
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+// In production this is replaced by a real BullMQ Worker via createWorkerImpl
+// in the worker entry point. Calling it directly throws — tests must inject
+// the dependency or the bootstrap is short-circuited via SEARCH_INDEXER_ENABLED.
+const defaultCreateWorker: NonNullable<SearchIndexerBootstrapDeps["createWorkerImpl"]> = () => {
+  throw new Error(
+    "bootstrapSearchIndexer requires deps.createWorkerImpl in this build; the worker entry must wire the BullMQ Worker constructor."
+  );
+};
+

hivegrid-cloud/apps/worker/src/jobs/search-indexer-bulk-pagers.ts

@@ -0,0 +1,429 @@
+import { BULK_REINDEX_SUPPORTED_INDEX_NAMES, crmContactsIndex, kbArticlesIndex, rowsIndex, usersIndex } from "@toolctrl/search";
+import type { SearchIndexerBulkPager, SearchIndexerDependencies } from "./search-indexer.js";
+
+type KnowledgeBaseArticleRow = {
+  id: string;
+  knowledgeBaseId: string;
+  slug: string;
+  title: string;
+  description: string;
+  contentPublishedAsText: string;
+  language: string;
+  publishedAt: Date | string | null;
+  createdAt: Date | string;
+  updatedAt: Date | string | null;
+  knowledgeBase: { tenantId: string | null };
+};
+
+const SEARCHABLE_ROW_PROPERTY_TYPES = new Set([1, 8]); // TEXT, SELECT
+
+type RowSearchProperty = {
+  id: string;
+  name: string;
+  type: number;
+};
+
+type RowSearchValue = {
+  propertyId: string;
+  textValue: string | null;
+  numberValue: { toString(): string } | number | string | null;
+  dateValue: Date | string | null;
+  booleanValue: boolean | null;
+  property: RowSearchProperty;
+};
+
+type SearchRow = {
+  id: string;
+  tenantId: string | null;
+  entityId: string;
+  folio: number;
+  createdByUserId: string | null;
+  createdAt: Date | string;
+  updatedAt: Date | string;
+  deletedAt: Date | string | null;
+  entity: {
+    name: string;
+    properties: RowSearchProperty[];
+  };
+  values: RowSearchValue[];
+};
+
+const TENANT_USER_STATUS_ACTIVE = 2;
+
+type TenantUserSearchRow = {
+  id: string;
+  tenantId: string;
+  userId: string;
+  type: number;
+  status: number;
+  user: {
+    id: string;
+    createdAt: Date | string;
+    email: string;
+    firstName: string;
+    lastName: string;
+    active: boolean;
+    admin: { userId: string } | null;
+    logs: { createdAt: Date | string }[];
+  };
+};
+
+export type SearchIndexerBulkPagerPrisma = {
+  knowledgeBaseArticle: {
+    findMany(args: {
+      where: {
+        id?: { gt: string };
+        knowledgeBase: { tenantId: string };
+      };
+      orderBy: { id: "asc" };
+      take: number;
+      select: {
+        id: true;
+        knowledgeBaseId: true;
+        slug: true;
+        title: true;
+        description: true;
+        contentPublishedAsText: true;
+        language: true;
+        publishedAt: true;
+        createdAt: true;
+        updatedAt: true;
+        knowledgeBase: { select: { tenantId: true } };
+      };
+    }): Promise<KnowledgeBaseArticleRow[]>;
+  };
+  row: {
+    findMany(args: {
+      where: {
+        id?: { gt: string };
+        tenantId: string;
+        deletedAt: null;
+        entity?: { name: string };
+      };
+      orderBy: { id: "asc" };
+      take: number;
+      select: {
+        id: true;
+        tenantId: true;
+        entityId: true;
+        folio: true;
+        createdByUserId: true;
+        createdAt: true;
+        updatedAt: true;
+        deletedAt: true;
+        entity: {
+          select: {
+            name: true;
+            properties: {
+              select: {
+                id: true;
+                name: true;
+                type: true;
+              };
+            };
+          };
+        };
+        values: {
+          select: {
+            propertyId: true;
+            textValue: true;
+            numberValue: true;
+            dateValue: true;
+            booleanValue: true;
+            property: {
+              select: {
+                id: true;
+                name: true;
+                type: true;
+              };
+            };
+          };
+        };
+      };
+    }): Promise<SearchRow[]>;
+  };
+  tenantUser: {
+    findMany(args: {
+      where: {
+        id?: { gt: string };
+        tenantId: string;
+      };
+      orderBy: { id: "asc" };
+      take: number;
+      select: {
+        id: true;
+        tenantId: true;
+        userId: true;
+        type: true;
+        status: true;
+        user: {
+          select: {
+            id: true;
+            createdAt: true;
+            email: true;
+            firstName: true;
+            lastName: true;
+            active: true;
+            admin: { select: { userId: true } };
+            logs: {
+              where: { action: "Login" };
+              orderBy: { createdAt: "desc" };
+              take: 1;
+              select: { createdAt: true };
+            };
+          };
+        };
+      };
+    }): Promise<TenantUserSearchRow[]>;
+  };
+};
+
+export function createSearchIndexerBulkPagers(input: { prisma: SearchIndexerBulkPagerPrisma }): NonNullable<SearchIndexerDependencies["bulkPagers"]> {
+  return {
+    [kbArticlesIndex.name]: createKbArticlesPager(input.prisma),
+    [crmContactsIndex.name]: createCrmContactsPager(input.prisma),
+    [rowsIndex.name]: createRowsPager(input.prisma),
+    [usersIndex.name]: createUsersPager(input.prisma),
+  };
+}
+
+export const registeredSearchIndexerBulkPagerNames: readonly string[] = BULK_REINDEX_SUPPORTED_INDEX_NAMES;
+
+function createKbArticlesPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBulkPager {
+  return async ({ tenantId, cursor, pageSize }) => {
+    const rows = await prisma.knowledgeBaseArticle.findMany({
+      where: {
+        ...(cursor ? { id: { gt: cursor } } : {}),
+        knowledgeBase: { tenantId },
+      },
+      orderBy: { id: "asc" },
+      take: pageSize,
+      select: {
+        id: true,
+        knowledgeBaseId: true,
+        slug: true,
+        title: true,
+        description: true,
+        contentPublishedAsText: true,
+        language: true,
+        publishedAt: true,
+        createdAt: true,
+        updatedAt: true,
+        knowledgeBase: { select: { tenantId: true } },
+      },
+    });
+    return {
+      documents: rows
+        .filter((row) => row.knowledgeBase.tenantId === tenantId)
+        .map((row) => ({
+          id: row.id,
+          tenantId,
+          knowledgeBaseId: row.knowledgeBaseId,
+          slug: row.slug,
+          title: row.title,
+          summary: row.description,
+          content: row.contentPublishedAsText,
+          tags: [],
+          language: row.language,
+          isPublished: row.publishedAt !== null,
+          createdAt: serializeDate(row.createdAt),
+          updatedAt: serializeDate(row.updatedAt),
+          publishedAt: serializeDate(row.publishedAt),
+        })),
+      nextCursor: rows.length === pageSize ? (rows[rows.length - 1]?.id ?? null) : null,
+    };
+  };
+}
+
+function createCrmContactsPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBulkPager {
+  return async ({ tenantId, cursor, pageSize }) => {
+    const rows = await findSearchRows(prisma, {
+      tenantId,
+      cursor,
+      pageSize,
+      entityName: "contact",
+    });
+    return {
+      documents: rows.filter((row) => row.tenantId === tenantId && row.entity.name === "contact").map((row) => rowToContactDocument(row, tenantId)),
+      nextCursor: rows.length === pageSize ? (rows[rows.length - 1]?.id ?? null) : null,
+    };
+  };
+}
+
+function createRowsPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBulkPager {
+  return async ({ tenantId, cursor, pageSize }) => {
+    const rows = await findSearchRows(prisma, { tenantId, cursor, pageSize });
+    return {
+      documents: rows.filter((row) => row.tenantId === tenantId).map((row) => rowToRowsDocument(row, tenantId)),
+      nextCursor: rows.length === pageSize ? (rows[rows.length - 1]?.id ?? null) : null,
+    };
+  };
+}
+
+function createUsersPager(prisma: SearchIndexerBulkPagerPrisma): SearchIndexerBulkPager {
+  return async ({ tenantId, cursor, pageSize }) => {
+    const rows = await prisma.tenantUser.findMany({
+      where: {
+        ...(cursor ? { id: { gt: cursor } } : {}),
+        tenantId,
+      },
+      orderBy: { id: "asc" },
+      take: pageSize,
+      select: {
+        id: true,
+        tenantId: true,
+        userId: true,
+        type: true,
+        status: true,
+        user: {
+          select: {
+            id: true,
+            createdAt: true,
+            email: true,
+            firstName: true,
+            lastName: true,
+            active: true,
+            admin: { select: { userId: true } },
+            logs: {
+              where: { action: "Login" },
+              orderBy: { createdAt: "desc" },
+              take: 1,
+              select: { createdAt: true },
+            },
+          },
+        },
+      },
+    });
+    return {
+      documents: rows.filter((row) => row.tenantId === tenantId).map((row) => tenantUserToUserDocument(row, tenantId)),
+      nextCursor: rows.length === pageSize ? (rows[rows.length - 1]?.id ?? null) : null,
+    };
+  };
+}
+
+function findSearchRows(
+  prisma: SearchIndexerBulkPagerPrisma,
+  input: { tenantId: string; cursor: string | null; pageSize: number; entityName?: string }
+) {
+  return prisma.row.findMany({
+    where: {
+      ...(input.cursor ? { id: { gt: input.cursor } } : {}),
+      tenantId: input.tenantId,
+      deletedAt: null,
+      ...(input.entityName ? { entity: { name: input.entityName } } : {}),
+    },
+    orderBy: { id: "asc" },
+    take: input.pageSize,
+    select: {
+      id: true,
+      tenantId: true,
+      entityId: true,
+      folio: true,
+      createdByUserId: true,
+      createdAt: true,
+      updatedAt: true,
+      deletedAt: true,
+      entity: {
+        select: {
+          name: true,
+          properties: {
+            select: {
+              id: true,
+              name: true,
+              type: true,
+            },
+          },
+        },
+      },
+      values: {
+        select: {
+          propertyId: true,
+          textValue: true,
+          numberValue: true,
+          dateValue: true,
+          booleanValue: true,
+          property: {
+            select: {
+              id: true,
+              name: true,
+              type: true,
+            },
+          },
+        },
+      },
+    },
+  });
+}
+
+function tenantUserToUserDocument(row: TenantUserSearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
+  return {
+    id: row.user.id,
+    tenantId,
+    firstName: row.user.firstName,
+    lastName: row.user.lastName,
+    email: row.user.email,
+    type: row.type,
+    admin: row.user.admin !== null,
+    active: row.user.active && row.status === TENANT_USER_STATUS_ACTIVE,
+    createdAt: serializeDate(row.user.createdAt),
+    lastLoginAt: serializeDate(row.user.logs[0]?.createdAt ?? null),
+  };
+}
+
+function rowToContactDocument(row: SearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
+  return {
+    id: row.id,
+    tenantId,
+    firstName: getTextValue(row, "firstName"),
+    lastName: getTextValue(row, "lastName"),
+    email: getTextValue(row, "email"),
+    company: getTextValue(row, "company"),
+    jobTitle: getTextValue(row, "jobTitle"),
+    status: getTextValue(row, "status"),
+    ownerId: row.createdByUserId ?? "",
+    createdAt: serializeDate(row.createdAt),
+    updatedAt: serializeDate(row.updatedAt),
+  };
+}
+
+function rowToRowsDocument(row: SearchRow, tenantId: string): Record<string, unknown> & { id: string; tenantId: string } {
+  return {
+    id: row.id,
+    tenantId,
+    entityId: row.entityId,
+    entityName: row.entity.name,
+    folio: typeof row.folio === "number" ? row.folio : 0,
+    searchable: buildSearchableBlob(row),
+    createdById: row.createdByUserId ?? "",
+    createdAt: serializeDate(row.createdAt),
+    updatedAt: serializeDate(row.updatedAt),
+  };
+}
+
+function buildSearchableBlob(row: SearchRow): string {
+  return row.entity.properties
+    .filter((property) => SEARCHABLE_ROW_PROPERTY_TYPES.has(property.type))
+    .map((property) => getTextValue(row, property.name))
+    .filter((value) => value.length > 0)
+    .join(" \n ");
+}
+
+function getTextValue(row: SearchRow, propertyName: string): string {
+  const property = row.entity.properties.find((item) => item.name === propertyName);
+  const value = property ? row.values.find((item) => item.propertyId === property.id) : undefined;
+  return normalizeRowValue(value).trim();
+}
+
+function normalizeRowValue(value: RowSearchValue | undefined): string {
+  if (!value) return "";
+  if (value.textValue !== null) return value.textValue;
+  if (value.numberValue !== null) return value.numberValue.toString();
+  if (value.dateValue !== null) return serializeDate(value.dateValue) ?? "";
+  if (value.booleanValue !== null) return String(value.booleanValue);
+  return "";
+}
+
+function serializeDate(value: Date | string | null): string | null {
+  if (value === null) return null;
+  return value instanceof Date ? value.toISOString() : value;
+}

hivegrid-cloud/apps/worker/src/jobs/search-indexer.test.ts

@@ -0,0 +1,577 @@
+import { describe, expect, it, vi } from "vitest";
+import { createSearchClient, crmContactsIndex, kbArticlesIndex, rowsIndex, searchInIndex, usersIndex } from "@toolctrl/search";
+import { createInMemoryMeili } from "@toolctrl/search/testing";
+import { runSearchIndexerJob, SearchIndexerJobError, parseSearchIndexJob } from "./search-indexer.js";
+import { createSearchIndexerBulkPagers, registeredSearchIndexerBulkPagerNames } from "./search-indexer-bulk-pagers.js";
+
+function makeClient() {
+  const meili = createInMemoryMeili();
+  const client = createSearchClient({ url: "http://memory", meiliFactory: meili.factory });
+  return { meili, client };
+}
+
+describe("parseSearchIndexJob", () => {
+  it("rejects unknown actions", () => {
+    expect(() => parseSearchIndexJob({ payload: { action: "rename", indexName: "kb-articles", tenantId: "t" } })).toThrow(SearchIndexerJobError);
+  });
+
+  it("rejects missing tenantId in payload", () => {
+    expect(() => parseSearchIndexJob({ payload: { action: "upsert", indexName: "kb-articles" } })).toThrow(/tenantId/);
+  });
+
+  it("accepts a well-formed upsert envelope", () => {
+    const parsed = parseSearchIndexJob({
+      outboxEventId: "evt_1",
+      tenantId: "t1",
+      eventType: "search.index.kb.upsert",
+      payload: { action: "upsert", indexName: "kb-articles", tenantId: "t1", document: { id: "a", tenantId: "t1", title: "X" } },
+    });
+    expect(parsed.payload.action).toBe("upsert");
+    expect(parsed.outboxEventId).toBe("evt_1");
+  });
+
+  it("rejects envelope and payload tenant mismatches", () => {
+    for (const payload of [
+      { action: "upsert", indexName: "kb-articles", tenantId: "t2", document: { id: "a", tenantId: "t2", title: "X" } },
+      { action: "delete", indexName: "kb-articles", tenantId: "t2", documentId: "a" },
+      { action: "bulk-reindex", indexName: "kb-articles", tenantId: "t2" },
+    ]) {
+      expect(() =>
+        parseSearchIndexJob({
+          outboxEventId: "evt_1",
+          tenantId: "t1",
+          eventType: "search.index.kb.upsert",
+          payload,
+        })
+      ).toThrow(/tenantId mismatch/);
+    }
+  });
+});
+
+describe("runSearchIndexerJob", () => {
+  it("upserts a document and isolates it to the tenant", async () => {
+    const { client } = makeClient();
+    const data = {
+      eventType: "search.index.kb.upsert",
+      tenantId: "t1",
+      payload: {
+        action: "upsert",
+        indexName: kbArticlesIndex.name,
+        tenantId: "t1",
+        document: { id: "a", tenantId: "t1", title: "Hello" },
+      },
+    };
+    const result = await runSearchIndexerJob(data, { searchClient: client });
+    expect(result).toMatchObject({ action: "upsert", documentsAffected: 1, tenantId: "t1" });
+
+    const hits = await searchInIndex({ client, schema: kbArticlesIndex, tenantId: "t1", query: "Hello" });
+    expect(hits.hits.map((h) => h.document.id)).toEqual(["a"]);
+    const other = await searchInIndex({ client, schema: kbArticlesIndex, tenantId: "t2", query: "Hello" });
+    expect(other.hits).toHaveLength(0);
+  });
+
+  it("deletes a document", async () => {
+    const { client } = makeClient();
+    await runSearchIndexerJob(
+      {
+        eventType: "search.index.kb.upsert",
+        tenantId: "t1",
+        payload: { action: "upsert", indexName: kbArticlesIndex.name, tenantId: "t1", document: { id: "a", tenantId: "t1", title: "Hi" } },
+      },
+      { searchClient: client }
+    );
+    const result = await runSearchIndexerJob(
+      {
+        eventType: "search.index.kb.delete",
+        tenantId: "t1",
+        payload: { action: "delete", indexName: kbArticlesIndex.name, tenantId: "t1", documentId: "a" },
+      },
+      { searchClient: client }
+    );
+    expect(result.action).toBe("delete");
+    const hits = await searchInIndex({ client, schema: kbArticlesIndex, tenantId: "t1", query: "Hi" });
+    expect(hits.hits).toHaveLength(0);
+  });
+
+  it("runs bulk-reindex via the supplied pager", async () => {
+    const { client } = makeClient();
+    const pager = vi
+      .fn()
+      .mockResolvedValueOnce({
+        documents: [
+          { id: "1", tenantId: "t1", title: "Doc 1" },
+          { id: "2", tenantId: "t1", title: "Doc 2" },
+        ],
+        nextCursor: "cursor-1",
+      })
+      .mockResolvedValueOnce({ documents: [{ id: "3", tenantId: "t1", title: "Doc 3" }], nextCursor: null });
+    const result = await runSearchIndexerJob(
+      {
+        eventType: "search.index.kb.bulk-reindex",
+        tenantId: "t1",
+        payload: { action: "bulk-reindex", indexName: kbArticlesIndex.name, tenantId: "t1" },
+      },
+      {
+        searchClient: client,
+        bulkPagers: { [kbArticlesIndex.name]: pager },
+        bulkPageSize: 2,
+      }
+    );
+    expect(result).toMatchObject({ action: "bulk-reindex", documentsAffected: 3, tenantId: "t1" });
+    expect(pager).toHaveBeenCalledTimes(2);
+  });
+
+  it("fails bulk-reindex when the pager cursor does not advance", async () => {
+    const { client } = makeClient();
+    const pager = vi
+      .fn()
+      .mockResolvedValueOnce({
+        documents: [{ id: "1", tenantId: "t1", title: "Doc 1" }],
+        nextCursor: "cursor-1",
+      })
+      .mockResolvedValueOnce({
+        documents: [{ id: "2", tenantId: "t1", title: "Doc 2" }],
+        nextCursor: "cursor-1",
+      });
+
+    await expect(
+      runSearchIndexerJob(
+        {
+          eventType: "search.index.kb.bulk-reindex",
+          tenantId: "t1",
+          payload: { action: "bulk-reindex", indexName: kbArticlesIndex.name, tenantId: "t1" },
+        },
+        {
+          searchClient: client,
+          bulkPagers: { [kbArticlesIndex.name]: pager },
+          bulkPageSize: 1,
+        }
+      )
+    ).rejects.toMatchObject({ code: "BULK_PAGER_STALLED" });
+    expect(pager).toHaveBeenCalledTimes(2);
+  });
+
+  it("fails bulk-reindex when maxBulkPages is reached before completion", async () => {
+    const { client } = makeClient();
+    const pager = vi.fn().mockResolvedValueOnce({
+      documents: [{ id: "1", tenantId: "t1", title: "Doc 1" }],
+      nextCursor: "cursor-1",
+    });
+
+    await expect(
+      runSearchIndexerJob(
+        {
+          eventType: "search.index.kb.bulk-reindex",
+          tenantId: "t1",
+          payload: { action: "bulk-reindex", indexName: kbArticlesIndex.name, tenantId: "t1" },
+        },
+        {
+          searchClient: client,
+          bulkPagers: { [kbArticlesIndex.name]: pager },
+          bulkPageSize: 1,
+          maxBulkPages: 1,
+        }
+      )
+    ).rejects.toMatchObject({ code: "BULK_REINDEX_INCOMPLETE" });
+    expect(pager).toHaveBeenCalledTimes(1);
+  });
+
+  it("replaces the tenant slice during bulk-reindex", async () => {
+    const { client } = makeClient();
+    await runSearchIndexerJob(
+      {
+        eventType: "search.index.kb.upsert",
+        tenantId: "t1",
+        payload: { action: "upsert", indexName: kbArticlesIndex.name, tenantId: "t1", document: { id: "stale", tenantId: "t1", title: "Stale Doc" } },
+      },
+      { searchClient: client }
+    );
+    await runSearchIndexerJob(
+      {
+        eventType: "search.index.kb.upsert",
+        tenantId: "t2",
+        payload: { action: "upsert", indexName: kbArticlesIndex.name, tenantId: "t2", document: { id: "stale", tenantId: "t2", title: "Other Tenant" } },
+      },
+      { searchClient: client }
+    );
+
+    const result = await runSearchIndexerJob(
+      {
+        eventType: "search.index.kb.bulk-reindex",
+        tenantId: "t1",
+        payload: { action: "bulk-reindex", indexName: kbArticlesIndex.name, tenantId: "t1" },
+      },
+      {
+        searchClient: client,
+        bulkPagers: {
+          [kbArticlesIndex.name]: vi.fn().mockResolvedValueOnce({
+            documents: [{ id: "fresh", tenantId: "t1", title: "Fresh Doc" }],
+            nextCursor: null,
+          }),
+        },
+      }
+    );
+
+    expect(result).toMatchObject({ action: "bulk-reindex", documentsAffected: 1, tenantId: "t1" });
+    await expect(searchInIndex({ client, schema: kbArticlesIndex, tenantId: "t1", query: "Stale" })).resolves.toMatchObject({ hits: [] });
+    await expect(searchInIndex({ client, schema: kbArticlesIndex, tenantId: "t1", query: "Fresh" })).resolves.toMatchObject({ hits: [{ document: expect.objectContaining({ id: "fresh" }) }] });
+    await expect(searchInIndex({ client, schema: kbArticlesIndex, tenantId: "t2", query: "Other" })).resolves.toMatchObject({ hits: [{ document: expect.objectContaining({ id: "stale" }) }] });
+  });
+
+  it("throws when bulk-reindex has no pager registered for the index", async () => {
+    const { client } = makeClient();
+    await expect(
+      runSearchIndexerJob(
+        {
+          eventType: "search.index.kb.bulk-reindex",
+          tenantId: "t1",
+          payload: { action: "bulk-reindex", indexName: kbArticlesIndex.name, tenantId: "t1" },
+        },
+        { searchClient: client }
+      )
+    ).rejects.toThrow(SearchIndexerJobError);
+  });
+
+  it("registers a kb-articles bulk pager that maps tenant-scoped documents", async () => {
+    const findMany = vi.fn(async () => [
+      {
+        id: "article_1",
+        knowledgeBaseId: "kb_1",
+        slug: "hello",
+        title: "Hello",
+        description: "Summary",
+        contentPublishedAsText: "Published text",
+        language: "en",
+        publishedAt: new Date("2026-01-02T03:04:05.000Z"),
+        createdAt: new Date("2026-01-01T00:00:00.000Z"),
+        updatedAt: null,
+        knowledgeBase: { tenantId: "tenant_1" },
+      },
+    ]);
+    const pagers = createSearchIndexerBulkPagers({
+      prisma: fakeSearchPagerPrisma({ knowledgeBaseArticle: { findMany } }),
+    });
+    const result = await pagers[kbArticlesIndex.name]?.({
+      indexName: kbArticlesIndex.name,
+      tenantId: "tenant_1",
+      cursor: "article_0",
+      pageSize: 1,
+    });
+
+    expect(registeredSearchIndexerBulkPagerNames).toContain(kbArticlesIndex.name);
+    expect(findMany).toHaveBeenCalledWith(
+      expect.objectContaining({
+        where: { id: { gt: "article_0" }, knowledgeBase: { tenantId: "tenant_1" } },
+        take: 1,
+      })
+    );
+    expect(result?.documents).toEqual([
+      expect.objectContaining({
+        id: "article_1",
+        tenantId: "tenant_1",
+        knowledgeBaseId: "kb_1",
+        summary: "Summary",
+        content: "Published text",
+        isPublished: true,
+      }),
+    ]);
+    expect(result?.nextCursor).toBe("article_1");
+  });
+
+  it("defensively filters kb-article bulk pager rows by tenant and terminates short pages", async () => {
+    const findMany = vi.fn(async () => [
+      {
+        id: "article_1",
+        knowledgeBaseId: "kb_1",
+        slug: "hello",
+        title: "Hello",
+        description: "Summary",
+        contentPublishedAsText: "Published text",
+        language: "de",
+        publishedAt: "2026-01-02T03:04:05.000Z",
+        createdAt: new Date("2026-01-01T00:00:00.000Z"),
+        updatedAt: new Date("2026-01-03T00:00:00.000Z"),
+        knowledgeBase: { tenantId: "tenant_1" },
+      },
+      {
+        id: "article_2",
+        knowledgeBaseId: "kb_2",
+        slug: "other",
+        title: "Other",
+        description: "Other summary",
+        contentPublishedAsText: "Other text",
+        language: "en",
+        publishedAt: null,
+        createdAt: new Date("2026-01-01T00:00:00.000Z"),
+        updatedAt: null,
+        knowledgeBase: { tenantId: "tenant_2" },
+      },
+    ]);
+    const pagers = createSearchIndexerBulkPagers({
+      prisma: fakeSearchPagerPrisma({ knowledgeBaseArticle: { findMany } }),
+    });
+
+    const result = await pagers[kbArticlesIndex.name]?.({
+      indexName: kbArticlesIndex.name,
+      tenantId: "tenant_1",
+      cursor: null,
+      pageSize: 3,
+    });
+
+    expect(findMany).toHaveBeenCalledWith(
+      expect.objectContaining({
+        where: { knowledgeBase: { tenantId: "tenant_1" } },
+        orderBy: { id: "asc" },
+        take: 3,
+      })
+    );
+    expect(result?.documents).toEqual([
+      expect.objectContaining({
+        id: "article_1",
+        tenantId: "tenant_1",
+        language: "de",
+        createdAt: "2026-01-01T00:00:00.000Z",
+        updatedAt: "2026-01-03T00:00:00.000Z",
+        publishedAt: "2026-01-02T03:04:05.000Z",
+      }),
+    ]);
+    expect(result?.nextCursor).toBeNull();
+  });
+
+  it("registers a rows bulk pager that maps searchable tenant rows", async () => {
+    const findRows = vi.fn(async () => [
+      makeSearchRow({
+        id: "row_1",
+        tenantId: "tenant_1",
+        entityName: "account",
+        values: {
+          name: "Acme",
+          status: "active",
+          hiddenNumber: 42,
+        },
+      }),
+    ]);
+    const pagers = createSearchIndexerBulkPagers({
+      prisma: fakeSearchPagerPrisma({ row: { findMany: findRows } }),
+    });
+
+    const result = await pagers[rowsIndex.name]?.({
+      indexName: rowsIndex.name,
+      tenantId: "tenant_1",
+      cursor: "row_0",
+      pageSize: 1,
+    });
+
+    expect(registeredSearchIndexerBulkPagerNames).toContain(rowsIndex.name);
+    expect(findRows).toHaveBeenCalledWith(
+      expect.objectContaining({
+        where: { id: { gt: "row_0" }, tenantId: "tenant_1", deletedAt: null },
+        take: 1,
+      })
+    );
+    expect(result?.documents).toEqual([
+      expect.objectContaining({
+        id: "row_1",
+        tenantId: "tenant_1",
+        entityId: "entity_1",
+        entityName: "account",
+        folio: 7,
+        searchable: "Acme \n active",
+        createdById: "user_1",
+      }),
+    ]);
+    expect(result?.nextCursor).toBe("row_1");
+  });
+
+  it("registers a crm-contacts bulk pager from contact rows only", async () => {
+    const findRows = vi.fn(async () => [
+      makeSearchRow({
+        id: "contact_1",
+        tenantId: "tenant_1",
+        entityName: "contact",
+        values: {
+          firstName: "Ada",
+          lastName: "Lovelace",
+          email: "ada@example.test",
+          company: "Analytical Engines",
+          jobTitle: "Founder",
+          status: "customer",
+        },
+      }),
+      makeSearchRow({
+        id: "contact_2",
+        tenantId: "tenant_2",
+        entityName: "contact",
+        values: { firstName: "Other" },
+      }),
+    ]);
+    const pagers = createSearchIndexerBulkPagers({
+      prisma: fakeSearchPagerPrisma({ row: { findMany: findRows } }),
+    });
+
+    const result = await pagers[crmContactsIndex.name]?.({
+      indexName: crmContactsIndex.name,
+      tenantId: "tenant_1",
+      cursor: null,
+      pageSize: 3,
+    });
+
+    expect(registeredSearchIndexerBulkPagerNames).toContain(crmContactsIndex.name);
+    expect(findRows).toHaveBeenCalledWith(
+      expect.objectContaining({
+        where: { tenantId: "tenant_1", deletedAt: null, entity: { name: "contact" } },
+        orderBy: { id: "asc" },
+        take: 3,
+      })
+    );
+    expect(result?.documents).toEqual([
+      expect.objectContaining({
+        id: "contact_1",
+        tenantId: "tenant_1",
+        firstName: "Ada",
+        lastName: "Lovelace",
+        email: "ada@example.test",
+        company: "Analytical Engines",
+        jobTitle: "Founder",
+        status: "customer",
+        ownerId: "user_1",
+      }),
+    ]);
+    expect(result?.nextCursor).toBeNull();
+  });
+
+  it("registers a users bulk pager from tenant memberships", async () => {
+    const findTenantUsers = vi.fn(async () => [
+      {
+        id: "tenant_user_1",
+        tenantId: "tenant_1",
+        userId: "user_1",
+        type: 1,
+        status: 2,
+        user: {
+          id: "user_1",
+          createdAt: new Date("2026-01-01T00:00:00.000Z"),
+          email: "ada@example.test",
+          firstName: "Ada",
+          lastName: "Lovelace",
+          active: true,
+          admin: { userId: "user_1" },
+          logs: [{ createdAt: new Date("2026-01-03T00:00:00.000Z") }],
+        },
+      },
+      {
+        id: "tenant_user_2",
+        tenantId: "tenant_2",
+        userId: "user_2",
+        type: 2,
+        status: 2,
+        user: {
+          id: "user_2",
+          createdAt: new Date("2026-01-01T00:00:00.000Z"),
+          email: "other@example.test",
+          firstName: "Other",
+          lastName: "Tenant",
+          active: true,
+          admin: null,
+          logs: [],
+        },
+      },
+    ]);
+    const pagers = createSearchIndexerBulkPagers({
+      prisma: fakeSearchPagerPrisma({ tenantUser: { findMany: findTenantUsers } }),
+    });
+
+    const result = await pagers[usersIndex.name]?.({
+      indexName: usersIndex.name,
+      tenantId: "tenant_1",
+      cursor: "tenant_user_0",
+      pageSize: 2,
+    });
+
+    expect(registeredSearchIndexerBulkPagerNames).toContain(usersIndex.name);
+    expect(findTenantUsers).toHaveBeenCalledWith(
+      expect.objectContaining({
+        where: { id: { gt: "tenant_user_0" }, tenantId: "tenant_1" },
+        orderBy: { id: "asc" },
+        take: 2,
+      })
+    );
+    expect(result?.documents).toEqual([
+      expect.objectContaining({
+        id: "user_1",
+        tenantId: "tenant_1",
+        firstName: "Ada",
+        lastName: "Lovelace",
+        email: "ada@example.test",
+        type: 1,
+        admin: true,
+        active: true,
+        createdAt: "2026-01-01T00:00:00.000Z",
+        lastLoginAt: "2026-01-03T00:00:00.000Z",
+      }),
+    ]);
+    expect(result?.nextCursor).toBe("tenant_user_2");
+  });
+
+  it("rejects payload pointing at an unknown index", async () => {
+    const { client } = makeClient();
+    await expect(
+      runSearchIndexerJob(
+        {
+          eventType: "search.index.unknown.upsert",
+          tenantId: "t1",
+          payload: { action: "upsert", indexName: "does-not-exist", tenantId: "t1", document: { id: "a", tenantId: "t1" } },
+        },
+        { searchClient: client }
+      )
+    ).rejects.toThrow(/unknown search index/);
+  });
+});
+
+function fakeSearchPagerPrisma(overrides: Partial<Parameters<typeof createSearchIndexerBulkPagers>[0]["prisma"]>) {
+  return {
+    knowledgeBaseArticle: { findMany: vi.fn(async () => []) },
+    row: { findMany: vi.fn(async () => []) },
+    tenantUser: { findMany: vi.fn(async () => []) },
+    ...overrides,
+  } as Parameters<typeof createSearchIndexerBulkPagers>[0]["prisma"];
+}
+
+function makeSearchRow(input: {
+  id: string;
+  tenantId: string;
+  entityName: string;
+  values: Record<string, string | number | boolean>;
+}) {
+  const properties = Object.keys(input.values).map((name, index) => ({
+    id: `property_${name}`,
+    name,
+    type: typeof input.values[name] === "string" ? (name === "hiddenNumber" ? 0 : 1) : 0,
+    order: index,
+  }));
+  return {
+    id: input.id,
+    tenantId: input.tenantId,
+    entityId: "entity_1",
+    folio: 7,
+    createdByUserId: "user_1",
+    createdAt: new Date("2026-01-01T00:00:00.000Z"),
+    updatedAt: new Date("2026-01-02T00:00:00.000Z"),
+    deletedAt: null,
+    entity: {
+      name: input.entityName,
+      properties,
+    },
+    values: properties.map((property) => {
+      const value = input.values[property.name];
+      return {
+        propertyId: property.id,
+        textValue: typeof value === "string" ? value : null,
+        numberValue: typeof value === "number" ? value : null,
+        dateValue: null,
+        booleanValue: typeof value === "boolean" ? value : null,
+        property,
+      };
+    }),
+  };
+}

hivegrid-cloud/apps/worker/src/jobs/search-indexer.ts

@@ -0,0 +1,253 @@
+/**
+ * Search-indexer worker job.
+ *
+ * Subscribes to the `outbox.search-index` BullMQ queue (populated by the
+ * outbox dispatcher in apps/worker/src/outbox/dispatcher.ts) and projects
+ * domain events onto Meilisearch:
+ *
+ *  - `search.index.<entity>.upsert` → upsert into the named index
+ *  - `search.index.<entity>.delete` → delete document by primary key
+ *  - `search.index.<entity>.bulk-reindex` → page over Postgres in
+ *    1000-row batches and replace the tenant slice
+ *
+ * All writes go through the tenant-scoped wrapper in @toolctrl/search so
+ * the indexer cannot accidentally cross-pollinate tenants.
+ */
+
+import { ALL_SEARCH_INDEXES, bulkIndex, deleteFromIndex, getSearchIndexSchema, type SearchClient, type SearchIndexSchema } from "@toolctrl/search";
+
+export const SEARCH_INDEX_OUTBOX_QUEUE = "outbox.search-index" as const;
+
+export type SearchIndexerLogger = {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+};
+
+export type SearchIndexerBulkPager = (input: {
+  indexName: string;
+  tenantId: string;
+  cursor: string | null;
+  pageSize: number;
+}) => Promise<SearchIndexerBulkPage>;
+
+export type SearchIndexerBulkPage = {
+  documents: ReadonlyArray<Record<string, unknown> & { id: string | number }>;
+  nextCursor: string | null;
+};
+
+export interface SearchIndexerDependencies {
+  readonly searchClient: SearchClient;
+  /** Provide a pager so the worker can stream rows without coupling to a
+   *  specific Prisma model. The producer (CLI / route) decides which
+   *  table to scan. Returning `nextCursor=null` terminates the loop. */
+  readonly bulkPagers?: Partial<Record<string, SearchIndexerBulkPager>>;
+  readonly logger?: SearchIndexerLogger;
+  /** Cap how many pages a single bulk-reindex job will walk. Protects
+   *  against runaway scans; the job fails when the cap is hit so BullMQ
+   *  can retry instead of reporting a partial rebuild as complete. */
+  readonly maxBulkPages?: number;
+  /** Override the page size used by bulk-reindex. */
+  readonly bulkPageSize?: number;
+}
+
+export type SearchEventVerb = "upsert" | "delete" | "bulk-reindex";
+
+export interface SearchIndexJobData {
+  readonly outboxEventId?: string;
+  readonly tenantId: string | null;
+  readonly eventType: string;
+  readonly payload: SearchIndexJobPayload;
+  readonly dispatchKey?: string | null;
+  readonly retries?: number;
+}
+
+export type SearchIndexJobPayload =
+  | SearchUpsertPayload
+  | SearchDeletePayload
+  | SearchBulkReindexPayload;
+
+export interface SearchUpsertPayload {
+  readonly action: "upsert";
+  readonly indexName: string;
+  readonly tenantId: string;
+  readonly document: Record<string, unknown> & { id: string | number };
+}
+
+export interface SearchDeletePayload {
+  readonly action: "delete";
+  readonly indexName: string;
+  readonly tenantId: string;
+  readonly documentId: string | number;
+}
+
+export interface SearchBulkReindexPayload {
+  readonly action: "bulk-reindex";
+  readonly indexName: string;
+  /** "*" reindexes every tenant. Concrete IDs scope the job to a single
+   *  tenant slice. */
+  readonly tenantId: string;
+}
+
+export class SearchIndexerJobError extends Error {
+  override readonly name = "SearchIndexerJobError";
+  readonly code:
+    | "UNKNOWN_INDEX"
+    | "UNKNOWN_ACTION"
+    | "MISSING_BULK_PAGER"
+    | "INVALID_PAYLOAD"
+    | "BULK_PAGER_STALLED"
+    | "BULK_REINDEX_INCOMPLETE";
+  constructor(message: string, code: SearchIndexerJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export function parseSearchIndexJob(data: unknown): SearchIndexJobData {
+  if (!data || typeof data !== "object") {
+    throw new SearchIndexerJobError("search index job missing data envelope", "INVALID_PAYLOAD");
+  }
+  const envelope = data as Record<string, unknown>;
+  const payload = envelope.payload;
+  if (!payload || typeof payload !== "object") {
+    throw new SearchIndexerJobError("search index job missing payload", "INVALID_PAYLOAD");
+  }
+  const inner = payload as Record<string, unknown>;
+  const action = inner.action;
+  if (action !== "upsert" && action !== "delete" && action !== "bulk-reindex") {
+    throw new SearchIndexerJobError(`unknown search index action ${String(action)}`, "UNKNOWN_ACTION");
+  }
+  const indexName = typeof inner.indexName === "string" ? inner.indexName : "";
+  if (indexName.length === 0) {
+    throw new SearchIndexerJobError("search index job missing indexName", "INVALID_PAYLOAD");
+  }
+  const tenantIdInner = typeof inner.tenantId === "string" ? inner.tenantId : "";
+  if (tenantIdInner.length === 0) {
+    throw new SearchIndexerJobError("search index job missing payload.tenantId", "INVALID_PAYLOAD");
+  }
+  const tenantIdEnvelope = typeof envelope.tenantId === "string" ? envelope.tenantId : null;
+  if (tenantIdEnvelope !== null && tenantIdEnvelope !== tenantIdInner) {
+    throw new SearchIndexerJobError("search index job tenantId mismatch between envelope and payload", "INVALID_PAYLOAD");
+  }
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId: tenantIdEnvelope,
+    eventType: typeof envelope.eventType === "string" ? envelope.eventType : "",
+    payload: inner as unknown as SearchIndexJobPayload,
+    dispatchKey: typeof envelope.dispatchKey === "string" ? envelope.dispatchKey : null,
+    retries: typeof envelope.retries === "number" ? envelope.retries : undefined,
+  };
+}
+
+function tenantFilter(tenantId: string): string {
+  return `tenantId = "${tenantId.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"')}"`;
+}
+
+export function resolveSchema(indexName: string): SearchIndexSchema {
+  const schema = getSearchIndexSchema(indexName);
+  if (!schema) {
+    throw new SearchIndexerJobError(`unknown search index "${indexName}"`, "UNKNOWN_INDEX");
+  }
+  return schema;
+}
+
+export type SearchIndexerResult = {
+  action: SearchEventVerb;
+  indexName: string;
+  tenantId: string;
+  documentsAffected: number;
+};
+
+/**
+ * Process one BullMQ job from `outbox.search-index`. Returns a result
+ * record for logging; throws on retryable failures so BullMQ can
+ * exponentially back off via its own default policy.
+ */
+export async function runSearchIndexerJob(
+  data: unknown,
+  deps: SearchIndexerDependencies
+): Promise<SearchIndexerResult> {
+  const job = parseSearchIndexJob(data);
+  const schema = resolveSchema(job.payload.indexName);
+  const { searchClient, logger } = deps;
+
+  if (job.payload.action === "upsert") {
+    await bulkIndex({
+      client: searchClient,
+      schema,
+      tenantId: job.payload.tenantId,
+      documents: [job.payload.document],
+    });
+    logger?.info?.(
+      { component: "search-indexer", indexName: schema.name, tenantId: job.payload.tenantId, documentId: job.payload.document.id },
+      "search-indexer upserted document"
+    );
+    return { action: "upsert", indexName: schema.name, tenantId: job.payload.tenantId, documentsAffected: 1 };
+  }
+
+  if (job.payload.action === "delete") {
+    await deleteFromIndex({
+      client: searchClient,
+      schema,
+      tenantId: job.payload.tenantId,
+      documentIds: [job.payload.documentId],
+    });
+    logger?.info?.(
+      { component: "search-indexer", indexName: schema.name, tenantId: job.payload.tenantId, documentId: job.payload.documentId },
+      "search-indexer deleted document"
+    );
+    return { action: "delete", indexName: schema.name, tenantId: job.payload.tenantId, documentsAffected: 1 };
+  }
+
+  // bulk-reindex
+  const pager = deps.bulkPagers?.[schema.name];
+  if (!pager) {
+    throw new SearchIndexerJobError(
+      `search-indexer has no bulk pager registered for "${schema.name}"`,
+      "MISSING_BULK_PAGER"
+    );
+  }
+  const pageSize = deps.bulkPageSize ?? 1000;
+  const maxPages = deps.maxBulkPages ?? 1000;
+  let cursor: string | null = null;
+  let total = 0;
+  await searchClient.index(schema.name).deleteByFilter(tenantFilter(job.payload.tenantId));
+  for (let page = 0; page < maxPages; page += 1) {
+    const previousCursor: string | null = cursor;
+    const result = await pager({ indexName: schema.name, tenantId: job.payload.tenantId, cursor, pageSize });
+    if (result.documents.length > 0) {
+      await bulkIndex({
+        client: searchClient,
+        schema,
+        tenantId: job.payload.tenantId,
+        documents: result.documents as readonly (Record<string, unknown> & { id: string | number })[],
+      });
+      total += result.documents.length;
+    }
+    cursor = result.nextCursor;
+    if (cursor === null) break;
+    if (cursor === previousCursor) {
+      throw new SearchIndexerJobError(
+        `search-indexer bulk pager for "${schema.name}" did not advance cursor "${String(cursor)}"`,
+        "BULK_PAGER_STALLED"
+      );
+    }
+  }
+  if (cursor !== null) {
+    throw new SearchIndexerJobError(
+      `search-indexer bulk reindex for "${schema.name}" stopped after ${maxPages} pages before completion`,
+      "BULK_REINDEX_INCOMPLETE"
+    );
+  }
+  logger?.info?.(
+    { component: "search-indexer", indexName: schema.name, tenantId: job.payload.tenantId, documents: total },
+    "search-indexer bulk reindex complete"
+  );
+  return { action: "bulk-reindex", indexName: schema.name, tenantId: job.payload.tenantId, documentsAffected: total };
+}
+
+/** Convenience: enumerate every registered index schema name. */
+export function allSearchIndexNames(): readonly string[] {
+  return ALL_SEARCH_INDEXES.map((schema) => schema.name);
+}

hivegrid-cloud/apps/worker/src/jobs/task-archive-retention-cron.ts

@@ -0,0 +1,52 @@
+// P4 Tasks (AK-P4b) §7 — 30-day auto-archive cron (DoD 5).
+//
+// Daily maintenance job that archives completed tasks (`status = done`,
+// `completedAt <= now - 30d`, not yet archived) by setting `archivedAt`. The
+// execution logic lives in the maintenance handler (handlers.ts) which holds the
+// Prisma client; this file only carries the config + the idempotent scheduler
+// registration (BullMQ `upsertJobScheduler`, like the other retention crons).
+
+export const TASK_ARCHIVE_RETENTION_JOB_ID = "task-archive-retention" as const;
+export const TASK_ARCHIVE_RETENTION_EVERY_MS = 24 * 60 * 60 * 1000;
+/** Completed tasks older than this are auto-archived. 30 days = 720 hours. */
+export const TASK_ARCHIVE_RETENTION_DEFAULT_HOURS = 720;
+export const TASK_ARCHIVE_RETENTION_DEFAULT_BATCH_SIZE = 1000;
+
+export function resolveTaskArchiveRetentionCronConfig(env: NodeJS.ProcessEnv = process.env): {
+  olderThanHours: number;
+  take: number;
+} {
+  return {
+    olderThanHours: parsePositiveInteger(env.TASK_ARCHIVE_RETENTION_HOURS, TASK_ARCHIVE_RETENTION_DEFAULT_HOURS),
+    take: parsePositiveInteger(env.TASK_ARCHIVE_RETENTION_BATCH_SIZE, TASK_ARCHIVE_RETENTION_DEFAULT_BATCH_SIZE),
+  };
+}
+
+export async function registerTaskArchiveRetentionCron(
+  queue: {
+    upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
+  },
+  env: NodeJS.ProcessEnv = process.env
+): Promise<void> {
+  const config = resolveTaskArchiveRetentionCronConfig(env);
+  await queue.upsertJobScheduler(
+    TASK_ARCHIVE_RETENTION_JOB_ID,
+    { every: TASK_ARCHIVE_RETENTION_EVERY_MS },
+    {
+      name: TASK_ARCHIVE_RETENTION_JOB_ID,
+      data: {
+        kind: "cleanup.run",
+        task: TASK_ARCHIVE_RETENTION_JOB_ID,
+        olderThanHours: config.olderThanHours,
+        take: config.take,
+      },
+    }
+  );
+}
+
+function parsePositiveInteger(value: string | undefined, fallback: number): number {
+  if (!value) return fallback;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return Math.floor(parsed);
+}

hivegrid-cloud/apps/worker/src/jobs/task-reminder-cron.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "vitest";
+
+import { reminderChannel, reminderMessageKey, reminderScanBounds, reminderSourceKey, selectReminderWindows } from "./task-reminder-cron.js";
+
+const HOUR = 60 * 60 * 1000;
+const DAY = 24 * HOUR;
+
+describe("selectReminderWindows", () => {
+  const deadline = new Date("2026-06-25T12:00:00.000Z");
+
+  it("selects nothing before the 24h-before window opens", () => {
+    const now = new Date(deadline.getTime() - DAY - HOUR);
+    expect(selectReminderWindows(deadline, now)).toEqual([]);
+  });
+
+  it("selects only 'before' inside the 24h-before window", () => {
+    const now = new Date(deadline.getTime() - 2 * HOUR);
+    expect(selectReminderWindows(deadline, now)).toEqual(["before"]);
+  });
+
+  it("selects 'before' + 'due' at the deadline", () => {
+    expect(selectReminderWindows(deadline, deadline)).toEqual(["before", "due"]);
+  });
+
+  it("selects all three once the 24h-after window opens", () => {
+    const now = new Date(deadline.getTime() + DAY + HOUR);
+    expect(selectReminderWindows(deadline, now)).toEqual(["before", "due", "after"]);
+  });
+});
+
+describe("reminder helpers", () => {
+  it("escalates only the after-window to EMAIL", () => {
+    expect(reminderChannel("before")).toBe("IN_APP");
+    expect(reminderChannel("due")).toBe("IN_APP");
+    expect(reminderChannel("after")).toBe("EMAIL");
+  });
+
+  it("derives stable, per-window message + source keys (notify dedup)", () => {
+    expect(reminderMessageKey("before")).toBe("tasks.reminder.before");
+    expect(reminderSourceKey("t1", "after")).toBe("task-reminder-t1-after");
+    // Distinct windows -> distinct source keys so each fires exactly once.
+    expect(reminderSourceKey("t1", "before")).not.toBe(reminderSourceKey("t1", "due"));
+  });
+
+  it("bounds the scan to [-7d, +24h] around now", () => {
+    const now = new Date("2026-06-25T12:00:00.000Z");
+    const { deadlineGte, deadlineLte } = reminderScanBounds(now);
+    expect(deadlineGte.getTime()).toBe(now.getTime() - 7 * DAY);
+    expect(deadlineLte.getTime()).toBe(now.getTime() + DAY);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/task-reminder-cron.ts

@@ -0,0 +1,94 @@
+// P4 Tasks (AK-P4b) §7 — task reminder sweep (DoD 4).
+//
+// Hourly maintenance job. For every open task with a deadline it sends reminders
+// via notify() at three windows: 24h before, at the deadline, and 24h after
+// (the after-window escalates to EMAIL). Idempotency is carried by notify()'s
+// `sourceKey` upsert (one delivery row per task+window+user), so re-running the
+// sweep never double-sends and no per-job bookkeeping or delayed-job rescheduling
+// is needed on deadline changes / completion.
+//
+// The execution logic (querying tasks, resolving recipients, calling notify())
+// lives in the maintenance handler which holds the Prisma client; this file
+// carries the config + scheduler registration + the PURE window-selection core
+// (unit-tested without a DB).
+
+export const TASK_REMINDER_SWEEP_JOB_ID = "task-reminder-sweep" as const;
+/** Hourly cadence: fine-grained enough for "at deadline" within an hour. */
+export const TASK_REMINDER_SWEEP_EVERY_MS = 60 * 60 * 1000;
+export const TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE = 1000;
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+export type ReminderWindow = "before" | "due" | "after";
+
+/** Channel per window: the after-window escalates to EMAIL (DoD 4). */
+export function reminderChannel(window: ReminderWindow): "IN_APP" | "EMAIL" {
+  return window === "after" ? "EMAIL" : "IN_APP";
+}
+
+export function reminderMessageKey(window: ReminderWindow): string {
+  return `tasks.reminder.${window}`;
+}
+
+export function reminderSourceKey(taskId: string, window: ReminderWindow): string {
+  return `task-reminder-${taskId}-${window}`;
+}
+
+/**
+ * Pure: which reminder windows have started for a task with `deadline` at time
+ * `now`. Windows are cumulative — a task seen well past its deadline qualifies
+ * for all three (each fires once, deduped by notify() sourceKey).
+ *   before: now >= deadline - 24h
+ *   due:    now >= deadline
+ *   after:  now >= deadline + 24h
+ */
+export function selectReminderWindows(deadline: Date, now: Date): ReminderWindow[] {
+  const t = now.getTime();
+  const d = deadline.getTime();
+  const windows: ReminderWindow[] = [];
+  if (t >= d - DAY_MS) windows.push("before");
+  if (t >= d) windows.push("due");
+  if (t >= d + DAY_MS) windows.push("after");
+  return windows;
+}
+
+/**
+ * Pure: lower/upper deadline bounds to scan in one sweep. We only need tasks
+ * whose earliest window has started (deadline <= now + 24h) and whose latest
+ * window is still fresh enough to matter (deadline >= now - 7d), keeping the
+ * query bounded while notify() dedup guarantees correctness.
+ */
+export function reminderScanBounds(now: Date): { deadlineGte: Date; deadlineLte: Date } {
+  return {
+    deadlineGte: new Date(now.getTime() - 7 * DAY_MS),
+    deadlineLte: new Date(now.getTime() + DAY_MS),
+  };
+}
+
+export function resolveTaskReminderSweepConfig(env: NodeJS.ProcessEnv = process.env): { take: number } {
+  return { take: parsePositiveInteger(env.TASK_REMINDER_SWEEP_BATCH_SIZE, TASK_REMINDER_SWEEP_DEFAULT_BATCH_SIZE) };
+}
+
+export async function registerTaskReminderSweepCron(
+  queue: {
+    upsertJobScheduler: (schedulerId: string, options: { every: number }, template: { name: string; data?: Record<string, unknown> }) => Promise<unknown>;
+  },
+  env: NodeJS.ProcessEnv = process.env
+): Promise<void> {
+  const config = resolveTaskReminderSweepConfig(env);
+  await queue.upsertJobScheduler(
+    TASK_REMINDER_SWEEP_JOB_ID,
+    { every: TASK_REMINDER_SWEEP_EVERY_MS },
+    {
+      name: TASK_REMINDER_SWEEP_JOB_ID,
+      data: { kind: "cleanup.run", task: TASK_REMINDER_SWEEP_JOB_ID, take: config.take },
+    }
+  );
+}
+
+function parsePositiveInteger(value: string | undefined, fallback: number): number {
+  if (!value) return fallback;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return Math.floor(parsed);
+}

hivegrid-cloud/apps/worker/src/jobs/transactional-mail-outbox-bootstrap.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, it, vi } from "vitest";
+import { bootstrapTransactionalMailOutbox, isTransactionalMailOutboxRequired } from "./transactional-mail-outbox-bootstrap.js";
+import { TRANSACTIONAL_MAIL_OUTBOX_QUEUE } from "./transactional-mail-outbox.js";
+
+function buildHandlerDeps() {
+  return {
+    enqueueMail: vi.fn(async () => undefined),
+  };
+}
+
+describe("bootstrapTransactionalMailOutbox", () => {
+  it("disables itself outside the mail worker group", () => {
+    const res = bootstrapTransactionalMailOutbox({ WORKER_QUEUE_GROUP: "general" } as Record<string, string>);
+    expect(res.status).toBe("disabled");
+    if (res.status === "disabled") expect(res.reason).toMatch(/non-mail/);
+  });
+
+  it("disables itself when the env flag is false", () => {
+    const res = bootstrapTransactionalMailOutbox({
+      WORKER_QUEUE_GROUP: "mail",
+      TRANSACTIONAL_MAIL_OUTBOX_ENABLED: "false",
+    } as Record<string, string>);
+    expect(res.status).toBe("disabled");
+    if (res.status === "disabled") expect(res.reason).toMatch(/TRANSACTIONAL_MAIL_OUTBOX_ENABLED/);
+  });
+
+  it("subscribes to outbox.mail on the mail worker group", () => {
+    const create = vi.fn(() => ({ name: TRANSACTIONAL_MAIL_OUTBOX_QUEUE, close: async () => undefined }));
+    const res = bootstrapTransactionalMailOutbox(
+      { WORKER_QUEUE_GROUP: "mail" } as Record<string, string>,
+      { handlerDeps: buildHandlerDeps(), createWorkerImpl: create },
+    );
+    expect(res.status).toBe("started");
+    expect(create).toHaveBeenCalledWith(
+      TRANSACTIONAL_MAIL_OUTBOX_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 5 }),
+    );
+  });
+
+  it("honours TRANSACTIONAL_MAIL_OUTBOX_CONCURRENCY", () => {
+    const create = vi.fn(() => ({ name: TRANSACTIONAL_MAIL_OUTBOX_QUEUE, close: async () => undefined }));
+    bootstrapTransactionalMailOutbox(
+      {
+        WORKER_QUEUE_GROUP: "mail",
+        TRANSACTIONAL_MAIL_OUTBOX_CONCURRENCY: "9",
+      } as Record<string, string>,
+      { handlerDeps: buildHandlerDeps(), createWorkerImpl: create },
+    );
+    expect(create).toHaveBeenCalledWith(
+      TRANSACTIONAL_MAIL_OUTBOX_QUEUE,
+      expect.any(Function),
+      expect.objectContaining({ concurrency: 9 }),
+    );
+  });
+
+  it("treats required mode as mail-group only", () => {
+    expect(
+      isTransactionalMailOutboxRequired({
+        WORKER_QUEUE_GROUP: "mail",
+        TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true",
+      } as Record<string, string>),
+    ).toBe(true);
+    expect(
+      isTransactionalMailOutboxRequired({
+        WORKER_QUEUE_GROUP: "general",
+        TRANSACTIONAL_MAIL_OUTBOX_REQUIRED: "true",
+      } as Record<string, string>),
+    ).toBe(false);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/transactional-mail-outbox-bootstrap.ts

@@ -0,0 +1,78 @@
+import type { EnvRecord } from "@toolctrl/config";
+
+import {
+  runTransactionalMailOutboxJob,
+  TRANSACTIONAL_MAIL_OUTBOX_QUEUE,
+  type TransactionalMailOutboxDeps,
+} from "./transactional-mail-outbox.js";
+import type { FormSubmissionRouteLogger } from "./form-submission-route.js";
+
+export type TransactionalMailOutboxBootstrapEnv = EnvRecord & {
+  WORKER_QUEUE_GROUP?: string;
+  TRANSACTIONAL_MAIL_OUTBOX_ENABLED?: string;
+  TRANSACTIONAL_MAIL_OUTBOX_CONCURRENCY?: string;
+};
+
+export interface TransactionalMailOutboxWorkerHandle {
+  readonly name: string;
+  close(): Promise<void>;
+}
+
+export interface TransactionalMailOutboxBootstrapDeps {
+  readonly logger?: FormSubmissionRouteLogger;
+  readonly handlerDeps?: TransactionalMailOutboxDeps;
+  readonly createWorkerImpl?: (
+    queueName: string,
+    processor: (job: { data: unknown }) => Promise<unknown>,
+    options: { concurrency?: number },
+  ) => TransactionalMailOutboxWorkerHandle;
+}
+
+export type TransactionalMailOutboxBootstrapResult =
+  | { readonly status: "disabled"; readonly reason: string }
+  | { readonly status: "started"; readonly handle: TransactionalMailOutboxWorkerHandle };
+
+const DEFAULT_CONCURRENCY = 5;
+
+export function isTransactionalMailOutboxRequired(env: TransactionalMailOutboxBootstrapEnv, workerGroup = env.WORKER_QUEUE_GROUP): boolean {
+  return workerGroup === "mail" && env.TRANSACTIONAL_MAIL_OUTBOX_REQUIRED === "true";
+}
+
+export function bootstrapTransactionalMailOutbox(
+  env: TransactionalMailOutboxBootstrapEnv,
+  deps: TransactionalMailOutboxBootstrapDeps = {},
+): TransactionalMailOutboxBootstrapResult {
+  if (env.WORKER_QUEUE_GROUP && env.WORKER_QUEUE_GROUP !== "mail") {
+    return { status: "disabled", reason: "non-mail worker group" };
+  }
+  if (env.TRANSACTIONAL_MAIL_OUTBOX_ENABLED === "false") {
+    return { status: "disabled", reason: "TRANSACTIONAL_MAIL_OUTBOX_ENABLED=false" };
+  }
+  if (!deps.handlerDeps) {
+    return { status: "disabled", reason: "no handler deps wired (host must supply mail enqueue)" };
+  }
+  if (!deps.createWorkerImpl) {
+    return { status: "disabled", reason: "no worker factory wired" };
+  }
+
+  const concurrency = parsePositiveInt(env.TRANSACTIONAL_MAIL_OUTBOX_CONCURRENCY, DEFAULT_CONCURRENCY);
+  const handlerDeps = deps.handlerDeps;
+  const processor = (job: { data: unknown }) => runTransactionalMailOutboxJob(job.data, handlerDeps);
+  const handle = deps.createWorkerImpl(TRANSACTIONAL_MAIL_OUTBOX_QUEUE, processor, { concurrency });
+
+  deps.logger?.info?.(
+    {
+      component: "transactional-mail-outbox",
+      queue: TRANSACTIONAL_MAIL_OUTBOX_QUEUE,
+      concurrency,
+    },
+    "transactional mail outbox bridge started",
+  );
+  return { status: "started", handle };
+}
+
+function parsePositiveInt(value: unknown, fallback: number): number {
+  if (typeof value !== "string" || value.length === 0) return fallback;
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}

hivegrid-cloud/apps/worker/src/jobs/transactional-mail-outbox.test.ts

@@ -0,0 +1,134 @@
+import { describe, expect, it, vi } from "vitest";
+import { queueNames } from "@toolctrl/queue";
+import {
+  parseTransactionalMailOutboxJob,
+  runTransactionalMailOutboxJob,
+  TRANSACTIONAL_MAIL_OUTBOX_QUEUE,
+  TransactionalMailOutboxJobError,
+} from "./transactional-mail-outbox.js";
+
+function buildGenericJob(overrides: Record<string, unknown> = {}) {
+  return {
+    outboxEventId: "outbox_1",
+    tenantId: "tenant_1",
+    eventType: "mail.transactional.send",
+    dispatchKey: "mail:invite:inv_1",
+    payload: {
+      tenantId: "tenant_1",
+      to: ["ada@example.com"],
+      subject: "Welcome",
+      text: "Hello",
+      priorityOverride: "critical",
+      metadata: { inviteId: "inv_1" },
+      ...overrides,
+    },
+  };
+}
+
+function buildFormSubmissionJob(overrides: Record<string, unknown> = {}) {
+  return {
+    outboxEventId: "outbox_form_1",
+    tenantId: "tenant_1",
+    eventType: "mail.transactional.form-submission",
+    dispatchKey: "form-submission-mail:sub_1",
+    payload: {
+      submissionId: "sub_1",
+      formId: "form_1",
+      tenantId: "tenant_1",
+      recipients: ["ops@example.com", "alerts@example.com"],
+      values: { email: "ada@example.com", message: "<hello>" },
+      ...overrides,
+    },
+  };
+}
+
+describe("TRANSACTIONAL_MAIL_OUTBOX_QUEUE", () => {
+  it("matches the outbox mail queue", () => {
+    expect(TRANSACTIONAL_MAIL_OUTBOX_QUEUE).toBe("outbox.mail");
+  });
+});
+
+describe("parseTransactionalMailOutboxJob", () => {
+  it("accepts supported mail event types", () => {
+    expect(parseTransactionalMailOutboxJob(buildGenericJob()).eventType).toBe("mail.transactional.send");
+    expect(parseTransactionalMailOutboxJob(buildFormSubmissionJob()).eventType).toBe("mail.transactional.form-submission");
+  });
+
+  it("rejects unsupported mail event types", () => {
+    expect(() => parseTransactionalMailOutboxJob({ ...buildGenericJob(), eventType: "mail.marketing.send" })).toThrow(
+      TransactionalMailOutboxJobError,
+    );
+  });
+
+  it("rejects envelope and payload tenant mismatches", () => {
+    expect(() => parseTransactionalMailOutboxJob(buildGenericJob({ tenantId: "tenant_2" }))).toThrow(/tenantId mismatch/);
+  });
+});
+
+describe("runTransactionalMailOutboxJob", () => {
+  it("enqueues a generic transactional mail job with outbox idempotency and priority", async () => {
+    const enqueueMail = vi.fn(async () => undefined);
+
+    const result = await runTransactionalMailOutboxJob(buildGenericJob(), { enqueueMail });
+
+    expect(result).toEqual({ enqueued: true, queueName: queueNames.email, jobName: "send-transactional-email" });
+    expect(enqueueMail).toHaveBeenCalledWith(
+      expect.objectContaining({
+        queueName: queueNames.email,
+        jobName: "send-transactional-email",
+        payload: expect.objectContaining({
+          kind: "mail.transactional",
+          tenantId: "tenant_1",
+          to: ["ada@example.com"],
+          subject: "Welcome",
+          text: "Hello",
+          metadata: expect.objectContaining({
+            source: "outbox-mail",
+            outboxEventId: "outbox_1",
+            dispatchKey: "mail:invite:inv_1",
+            inviteId: "inv_1",
+          }),
+        }),
+      }),
+      expect.objectContaining({
+        tenantId: "tenant_1",
+        idempotencyKey: "outbox_1",
+        priorityOverride: "critical",
+      }),
+    );
+  });
+
+  it("renders form-submission follow-up notifications into transactional mail", async () => {
+    const enqueueMail = vi.fn(async () => undefined);
+
+    await runTransactionalMailOutboxJob(buildFormSubmissionJob(), { enqueueMail });
+
+    expect(enqueueMail).toHaveBeenCalledWith(
+      expect.objectContaining({
+        payload: expect.objectContaining({
+          kind: "mail.transactional",
+          subject: "New form submission",
+          tags: ["form-submission"],
+          to: ["ops@example.com", "alerts@example.com"],
+          text: expect.stringContaining("Submission: sub_1"),
+          html: expect.stringContaining("&lt;hello&gt;"),
+          metadata: expect.objectContaining({
+            source: "form-submission",
+            formId: "form_1",
+            submissionId: "sub_1",
+            outboxEventId: "outbox_form_1",
+          }),
+        }),
+      }),
+      expect.objectContaining({ tenantId: "tenant_1", idempotencyKey: "outbox_form_1" }),
+    );
+  });
+
+  it("rejects generic mail without body or template", async () => {
+    await expect(
+      runTransactionalMailOutboxJob(buildGenericJob({ text: undefined, html: undefined, templateId: undefined }), {
+        enqueueMail: vi.fn(),
+      }),
+    ).rejects.toThrow(/body\/template/);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/transactional-mail-outbox.ts

@@ -0,0 +1,237 @@
+/**
+ * Bridges transactional mail outbox events onto the existing
+ * `toolctrl.email` send-transactional-email processor.
+ */
+
+import {
+  buildTransactionalMailJob,
+  queueNames,
+  type QueueEnqueueOptions,
+  type TransactionalMailJobDescriptor,
+  type TransactionalMailJobInput,
+} from "@toolctrl/queue";
+
+export const TRANSACTIONAL_MAIL_OUTBOX_QUEUE = "outbox.mail" as const;
+
+type TransactionalMailOutboxEventType = "mail.transactional.send" | "mail.transactional.form-submission";
+
+export interface TransactionalMailOutboxJobData {
+  readonly outboxEventId?: string;
+  readonly tenantId: string;
+  readonly eventType: TransactionalMailOutboxEventType;
+  readonly dispatchKey?: string | null;
+  readonly payload: Record<string, unknown>;
+}
+
+export interface TransactionalMailOutboxDeps {
+  readonly enqueueMail: (
+    descriptor: TransactionalMailJobDescriptor,
+    options: QueueEnqueueOptions,
+  ) => Promise<unknown>;
+}
+
+export class TransactionalMailOutboxJobError extends Error {
+  override readonly name = "TransactionalMailOutboxJobError";
+  readonly code: "INVALID_PAYLOAD" | "INVALID_EVENT_TYPE" | "TENANT_MISMATCH";
+
+  constructor(message: string, code: TransactionalMailOutboxJobError["code"]) {
+    super(message);
+    this.code = code;
+  }
+}
+
+export function parseTransactionalMailOutboxJob(data: unknown): TransactionalMailOutboxJobData {
+  if (!data || typeof data !== "object") {
+    throw new TransactionalMailOutboxJobError("transactional mail job missing envelope", "INVALID_PAYLOAD");
+  }
+  const envelope = data as Record<string, unknown>;
+  if (envelope.eventType !== "mail.transactional.send" && envelope.eventType !== "mail.transactional.form-submission") {
+    throw new TransactionalMailOutboxJobError(
+      `transactional mail rejected eventType ${JSON.stringify(envelope.eventType)}`,
+      "INVALID_EVENT_TYPE",
+    );
+  }
+  const tenantId = typeof envelope.tenantId === "string" ? envelope.tenantId : "";
+  if (!tenantId) {
+    throw new TransactionalMailOutboxJobError("transactional mail job missing tenantId", "INVALID_PAYLOAD");
+  }
+  const payload = envelope.payload;
+  if (!payload || typeof payload !== "object") {
+    throw new TransactionalMailOutboxJobError("transactional mail job missing payload", "INVALID_PAYLOAD");
+  }
+  const payloadRecord = payload as Record<string, unknown>;
+  const payloadTenantId = typeof payloadRecord.tenantId === "string" ? payloadRecord.tenantId : tenantId;
+  if (payloadTenantId !== tenantId) {
+    throw new TransactionalMailOutboxJobError("transactional mail tenantId mismatch", "TENANT_MISMATCH");
+  }
+
+  return {
+    outboxEventId: typeof envelope.outboxEventId === "string" ? envelope.outboxEventId : undefined,
+    tenantId,
+    eventType: envelope.eventType,
+    dispatchKey: typeof envelope.dispatchKey === "string" ? envelope.dispatchKey : null,
+    payload: payloadRecord,
+  };
+}
+
+export async function runTransactionalMailOutboxJob(
+  data: unknown,
+  deps: TransactionalMailOutboxDeps,
+): Promise<{ enqueued: true; queueName: TransactionalMailJobDescriptor["queueName"]; jobName: TransactionalMailJobDescriptor["jobName"] }> {
+  const job = parseTransactionalMailOutboxJob(data);
+  const input =
+    job.eventType === "mail.transactional.form-submission"
+      ? buildFormSubmissionMailInput(job)
+      : buildGenericTransactionalMailInput(job);
+  const descriptor = buildTransactionalMailJob(input);
+
+  await deps.enqueueMail(descriptor, {
+    tenantId: job.tenantId,
+    idempotencyKey: job.outboxEventId ?? job.dispatchKey ?? `transactional-mail:${job.eventType}`,
+    priorityOverride: readPriorityOverride(job.payload),
+  });
+
+  return { enqueued: true, queueName: queueNames.email, jobName: "send-transactional-email" };
+}
+
+function buildGenericTransactionalMailInput(job: TransactionalMailOutboxJobData): TransactionalMailJobInput {
+  const payload = job.payload;
+  const to = readRecipients(payload.to);
+  if (to.length === 0) {
+    throw new TransactionalMailOutboxJobError("transactional mail payload missing recipients", "INVALID_PAYLOAD");
+  }
+  const subject = readOptionalString(payload.subject);
+  const text = readOptionalString(payload.text);
+  const html = readOptionalString(payload.html);
+  const templateId = readOptionalString(payload.templateId);
+  if (!text && !html && !templateId) {
+    throw new TransactionalMailOutboxJobError("transactional mail payload missing body/template", "INVALID_PAYLOAD");
+  }
+
+  return {
+    to,
+    tenantId: job.tenantId,
+    ...(subject ? { subject } : {}),
+    ...(text ? { text } : {}),
+    ...(html ? { html } : {}),
+    ...(templateId ? { templateId } : {}),
+    metadata: mergeMetadata(job, { source: "outbox-mail" }),
+  };
+}
+
+function buildFormSubmissionMailInput(job: TransactionalMailOutboxJobData): TransactionalMailJobInput {
+  const payload = job.payload;
+  const recipients = readRecipients(payload.recipients);
+  const submissionId = readRequiredString(payload.submissionId, "submissionId");
+  const formId = readRequiredString(payload.formId, "formId");
+  const values = payload.values && typeof payload.values === "object" ? (payload.values as Record<string, unknown>) : {};
+  if (recipients.length === 0) {
+    throw new TransactionalMailOutboxJobError("form-submission mail payload missing recipients", "INVALID_PAYLOAD");
+  }
+
+  const text = renderFormSubmissionText({ formId, submissionId, values });
+  const html = renderFormSubmissionHtml({ formId, submissionId, values });
+  return {
+    to: recipients,
+    tenantId: job.tenantId,
+    subject: "New form submission",
+    text,
+    html,
+    tags: ["form-submission"],
+    metadata: mergeMetadata(job, {
+      source: "form-submission",
+      formId,
+      submissionId,
+    }),
+  };
+}
+
+function readRecipients(value: unknown): string[] {
+  if (typeof value === "string") {
+    return value.includes("@") ? [value] : [];
+  }
+  if (!Array.isArray(value)) return [];
+  return value.filter((entry): entry is string => typeof entry === "string" && entry.includes("@"));
+}
+
+function readOptionalString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value : undefined;
+}
+
+function readRequiredString(value: unknown, field: string): string {
+  if (typeof value === "string" && value.trim()) return value;
+  throw new TransactionalMailOutboxJobError(`transactional mail payload missing ${field}`, "INVALID_PAYLOAD");
+}
+
+function readPriorityOverride(value: Record<string, unknown>): QueueEnqueueOptions["priorityOverride"] | undefined {
+  return value.priorityOverride === "critical" ||
+    value.priorityOverride === "high" ||
+    value.priorityOverride === "normal" ||
+    value.priorityOverride === "low"
+    ? value.priorityOverride
+    : undefined;
+}
+
+function mergeMetadata(job: TransactionalMailOutboxJobData, base: Record<string, unknown>): Record<string, unknown> {
+  const payloadMetadata = job.payload.metadata && typeof job.payload.metadata === "object" ? (job.payload.metadata as Record<string, unknown>) : {};
+  return {
+    ...base,
+    ...payloadMetadata,
+    ...(job.outboxEventId ? { outboxEventId: job.outboxEventId } : {}),
+    ...(job.dispatchKey ? { dispatchKey: job.dispatchKey } : {}),
+  };
+}
+
+function renderFormSubmissionText(input: {
+  formId: string;
+  submissionId: string;
+  values: Record<string, unknown>;
+}): string {
+  const lines = [
+    "A new form submission was received.",
+    "",
+    `Form: ${input.formId}`,
+    `Submission: ${input.submissionId}`,
+    "",
+    ...Object.entries(input.values).map(([key, value]) => `${key}: ${formatValue(value)}`),
+  ];
+  return lines.join("\n");
+}
+
+function renderFormSubmissionHtml(input: {
+  formId: string;
+  submissionId: string;
+  values: Record<string, unknown>;
+}): string {
+  const rows = Object.entries(input.values)
+    .map(([key, value]) => `<tr><th align="left">${escapeHtml(key)}</th><td>${escapeHtml(formatValue(value))}</td></tr>`)
+    .join("");
+  return [
+    "<p>A new form submission was received.</p>",
+    `<p><strong>Form:</strong> ${escapeHtml(input.formId)}<br/><strong>Submission:</strong> ${escapeHtml(input.submissionId)}</p>`,
+    rows ? `<table>${rows}</table>` : "",
+  ].join("");
+}
+
+function formatValue(value: unknown): string {
+  if (value === null || value === undefined) return "";
+  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return String(value);
+  return JSON.stringify(value);
+}
+
+function escapeHtml(value: string): string {
+  return value.replace(/[&<>"']/g, (char) => {
+    switch (char) {
+      case "&":
+        return "&amp;";
+      case "<":
+        return "&lt;";
+      case ">":
+        return "&gt;";
+      case '"':
+        return "&quot;";
+      default:
+        return "&#39;";
+    }
+  });
+}

hivegrid-cloud/apps/worker/src/jobs/trial-expiring-trigger.test.ts

@@ -0,0 +1,151 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { runTrialExpiringTriggerCron, type TrialExpiringCronDeps, type TrialExpiringRow } from "./trial-expiring-trigger.js";
+
+function buildDeps(rows: TrialExpiringRow[]): TrialExpiringCronDeps & {
+  published: Array<{ type: string; payload: Record<string, unknown>; dispatchKey: string; tenantId?: string | null }>;
+} {
+  const published: Array<{ type: string; payload: Record<string, unknown>; dispatchKey: string; tenantId?: string | null }> = [];
+  return {
+    store: {
+      findExpiringTrials: vi.fn(async () => rows),
+    },
+    publisher: {
+      publish: vi.fn(async (args) => {
+        published.push({
+          type: args.type,
+          payload: args.payload,
+          dispatchKey: args.dispatchKey,
+          tenantId: args.tenantId,
+        });
+        return { ok: true };
+      }),
+    },
+    logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn() },
+    now: () => new Date("2026-05-18T00:00:00Z"),
+    published,
+  };
+}
+
+const NOW = new Date("2026-05-18T00:00:00Z");
+const ROWS: TrialExpiringRow[] = [
+  {
+    tenantId: "tenant_1",
+    tenantEmail: "ops@tenant1.com",
+    tenantName: "Tenant One",
+    trialEndsAt: new Date("2026-05-20T00:00:00Z"), // 2 days remaining
+  },
+  {
+    tenantId: "tenant_2",
+    tenantEmail: "ops@tenant2.com",
+    tenantName: null,
+    trialEndsAt: new Date("2026-05-19T00:00:00Z"), // 1 day remaining
+  },
+];
+
+function deferred() {
+  let resolve!: () => void;
+  const promise = new Promise<void>((res) => {
+    resolve = res;
+  });
+  return { promise, resolve };
+}
+
+afterEach(() => {
+  delete process.env.MARKETING_SEQUENCES_ENABLED;
+});
+
+beforeEach(() => {
+  process.env.MARKETING_SEQUENCES_ENABLED = "true";
+});
+
+describe("runTrialExpiringTriggerCron", () => {
+  it("emits tenant.trial_expiring for every match", async () => {
+    const deps = buildDeps(ROWS);
+    const result = await runTrialExpiringTriggerCron(deps);
+    expect(result.emitted).toBe(2);
+    expect(deps.published.map((p) => p.type)).toEqual(["tenant.trial_expiring", "tenant.trial_expiring"]);
+  });
+
+  it("computes daysRemaining from trialEndsAt and now()", async () => {
+    const deps = buildDeps(ROWS);
+    await runTrialExpiringTriggerCron(deps);
+    expect(deps.published[0]?.payload["daysRemaining"]).toBe(2);
+    expect(deps.published[1]?.payload["daysRemaining"]).toBe(1);
+  });
+
+  it("uses date-keyed dispatchKey for idempotency", async () => {
+    const deps = buildDeps([ROWS[0]!]);
+    await runTrialExpiringTriggerCron(deps);
+    expect(deps.published[0]?.dispatchKey).toBe("tenant.trial_expiring.tenant_1.2026-05-18");
+  });
+
+  it("returns 0 when nothing matches", async () => {
+    const deps = buildDeps([]);
+    const result = await runTrialExpiringTriggerCron(deps);
+    expect(result.scanned).toBe(0);
+    expect(result.emitted).toBe(0);
+  });
+
+  it("short-circuits when killswitch active", async () => {
+    process.env.MARKETING_SEQUENCES_ENABLED = "false";
+    const deps = buildDeps(ROWS);
+    const result = await runTrialExpiringTriggerCron(deps);
+    expect(result.scanned).toBe(0);
+    expect(deps.store.findExpiringTrials).not.toHaveBeenCalled();
+  });
+
+  it("forwards horizonDays to the store", async () => {
+    const deps = buildDeps(ROWS);
+    await runTrialExpiringTriggerCron({ ...deps, horizonDays: 7 });
+    expect(deps.store.findExpiringTrials).toHaveBeenCalledWith(expect.objectContaining({ horizonDays: 7, now: NOW }));
+  });
+
+  it("counts publisher failures as skipped and keeps scanning", async () => {
+    const deps = buildDeps(ROWS);
+    deps.publisher.publish = vi.fn().mockRejectedValueOnce(new Error("simulated")).mockResolvedValueOnce({ ok: true });
+    const result = await runTrialExpiringTriggerCron(deps);
+    expect(result.scanned).toBe(2);
+    expect(result.emitted).toBe(1);
+    expect(result.skipped).toBe(1);
+    expect(deps.logger?.error).toHaveBeenCalled();
+  });
+
+  it("caps publish concurrency", async () => {
+    const rows = Array.from(
+      { length: 5 },
+      (_, index): TrialExpiringRow => ({
+        tenantId: `tenant_${index + 1}`,
+        tenantEmail: `ops-${index + 1}@example.com`,
+        tenantName: null,
+        trialEndsAt: new Date("2026-05-20T00:00:00Z"),
+      })
+    );
+    const deps = buildDeps(rows);
+    let active = 0;
+    let maxActive = 0;
+    let releaseIndex = 0;
+    const releases = Array.from({ length: 5 }, () => deferred());
+    deps.publisher.publish = vi.fn(async () => {
+      const release = releases[releaseIndex++]!;
+      active += 1;
+      maxActive = Math.max(maxActive, active);
+      await release.promise;
+      active -= 1;
+      return { ok: true };
+    });
+
+    const resultPromise = runTrialExpiringTriggerCron({ ...deps, concurrency: 2 });
+    await Promise.resolve();
+    releases[0]!.resolve();
+    releases[1]!.resolve();
+    await Promise.resolve();
+    releases[2]!.resolve();
+    releases[3]!.resolve();
+    await Promise.resolve();
+    releases[4]!.resolve();
+
+    const result = await resultPromise;
+    expect(result).toMatchObject({ scanned: 5, emitted: 5, skipped: 0 });
+    expect(maxActive).toBeLessThanOrEqual(2);
+  });
+});

hivegrid-cloud/apps/worker/src/jobs/trial-expiring-trigger.ts

@@ -0,0 +1,114 @@
+import { mapWithConcurrency } from "@toolctrl/foundation";
+
+/**
+ * Marketing-Funnel Phase 3 (Phase 05 #80) — trial-expiring trigger cron.
+ *
+ * Daily cron-style worker that scans active tenant subscriptions whose
+ * trial window ends within the configured horizon (default 3 days).
+ * For every match a `tenant.trial_expiring` outbox event is emitted;
+ * downstream the email-sequence trigger handler enrols
+ * trial-reminder sequences.
+ *
+ * Storage + clock + outbox publisher are injected so the handler is
+ * fully unit-testable. Intended production wiring is a daily BullMQ
+ * scheduled-job from `apps/worker/src/index.ts`; as of this slice the
+ * runner is not registered by the worker entrypoint.
+ *
+ * Idempotency: the dispatchKey is
+ * `tenant.trial_expiring.<tenantId>.<YYYY-MM-DD>` so re-running the
+ * cron on the same day is a no-op even on worker restart.
+ */
+
+export interface TrialExpiringRow {
+  readonly tenantId: string;
+  readonly tenantEmail: string | null;
+  readonly tenantName: string | null;
+  readonly trialEndsAt: Date;
+}
+
+export interface TrialExpiringStore {
+  findExpiringTrials(args: { now: Date; horizonDays: number; limit: number }): Promise<readonly TrialExpiringRow[]>;
+}
+
+export interface TrialExpiringPublisher {
+  publish(args: { type: string; payload: Record<string, unknown>; dispatchKey: string; tenantId?: string | null }): Promise<{ ok: boolean }>;
+}
+
+export interface TrialExpiringLogger {
+  info?: (record: Record<string, unknown>, message?: string) => void;
+  warn?: (record: Record<string, unknown>, message?: string) => void;
+  error: (record: Record<string, unknown>, message?: string) => void;
+}
+
+export interface TrialExpiringCronDeps {
+  readonly store: TrialExpiringStore;
+  readonly publisher: TrialExpiringPublisher;
+  readonly logger?: TrialExpiringLogger;
+  readonly now?: () => Date;
+  /** Trial-end horizon in days. Default 3 (matches "Trial Expiring" sequence). */
+  readonly horizonDays?: number;
+  /** Hard cap per cron tick. */
+  readonly limit?: number;
+  /** Max outbox publishes at once. Default 10. */
+  readonly concurrency?: number;
+}
+
+export interface TrialExpiringCronResult {
+  readonly scanned: number;
+  readonly emitted: number;
+  readonly skipped: number;
+}
+
+const DEFAULT_HORIZON_DAYS = 3;
+const DEFAULT_LIMIT = 500;
+
+function isoDate(d: Date): string {
+  return d.toISOString().slice(0, 10);
+}
+
+export async function runTrialExpiringTriggerCron(deps: TrialExpiringCronDeps): Promise<TrialExpiringCronResult> {
+  if (process.env.MARKETING_SEQUENCES_ENABLED !== "true") {
+    deps.logger?.info?.({ op: "trial.expiring.skipped" }, "marketing sequences disabled by env");
+    return { scanned: 0, emitted: 0, skipped: 0 };
+  }
+  const now = (deps.now ?? (() => new Date()))();
+  const horizonDays = deps.horizonDays ?? DEFAULT_HORIZON_DAYS;
+  const limit = deps.limit ?? DEFAULT_LIMIT;
+  const today = isoDate(now);
+
+  const rows = await deps.store.findExpiringTrials({ now, horizonDays, limit });
+  let emitted = 0;
+  let skipped = 0;
+  await mapWithConcurrency(rows, deps.concurrency ?? 10, async (row) => {
+    const daysRemaining = Math.max(0, Math.ceil((row.trialEndsAt.getTime() - now.getTime()) / (24 * 60 * 60 * 1000)));
+    const dispatchKey = `tenant.trial_expiring.${row.tenantId}.${today}`;
+    try {
+      await deps.publisher.publish({
+        type: "tenant.trial_expiring",
+        tenantId: row.tenantId,
+        payload: {
+          tenantId: row.tenantId,
+          email: row.tenantEmail,
+          recipientName: row.tenantName,
+          tenant: {
+            id: row.tenantId,
+            name: row.tenantName,
+            email: row.tenantEmail,
+            trialEndsAt: row.trialEndsAt.toISOString(),
+          },
+          daysRemaining,
+        },
+        dispatchKey,
+      });
+      emitted += 1;
+    } catch (error) {
+      skipped += 1;
+      deps.logger?.error(
+        { op: "trial.expiring.publish_failed", tenantId: row.tenantId, error: error instanceof Error ? error.message : String(error) },
+        "trial-expiring outbox publish failed"
+      );
+    }
+  });
+  deps.logger?.info?.({ op: "trial.expiring.done", scanned: rows.length, emitted, skipped, horizonDays }, "trial-expiring trigger cron complete");
+  return { scanned: rows.length, emitted, skipped };
+}

hivegrid-cloud/apps/worker/src/jobs/workflow-execution-stalled-reconciler-cron.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  WORKFLOW_EXECUTION_STALLED_RECONCILE_EVERY_MS,
+  registerWorkflowExecutionStalledReconcilerCron,
+  resolveWorkflowExecutionStalledReconcilerCronConfig,
+} from "./workflow-execution-stalled-reconciler-cron.js";
+import {
+  WORKFLOW_EXECUTION_STALLED_RECONCILE_DEFAULT_BATCH_SIZE,
+  WORKFLOW_EXECUTION_STALLED_RECONCILE_DEFAULT_OLDER_THAN_MINUTES,
+  WORKFLOW_EXECUTION_STALLED_RECONCILE_JOB_ID,
+} from "./workflow-execution-stalled-reconciler.js";
+
+describe("workflow-execution-stalled-reconciler-cron", () => {
+  it("resolves defaults and clamps invalid env", () => {
+    expect(resolveWorkflowExecutionStalledReconcilerCronConfig({})).toEqual({
+      olderThanMinutes: WORKFLOW_EXECUTION_STALLED_RECONCILE_DEFAULT_OLDER_THAN_MINUTES,
+      take: WORKFLOW_EXECUTION_STALLED_RECONCILE_DEFAULT_BATCH_SIZE,
+    });
+    expect(
+      resolveWorkflowExecutionStalledReconcilerCronConfig({
+        WORKFLOW_EXECUTION_STALLED_RECONCILE_OLDER_THAN_MINUTES: "45.9",
+        WORKFLOW_EXECUTION_STALLED_RECONCILE_BATCH_SIZE: "250.5",
+      } as NodeJS.ProcessEnv),
+    ).toEqual({ olderThanMinutes: 45, take: 250 });
+    expect(
+      resolveWorkflowExecutionStalledReconcilerCronConfig({
+        WORKFLOW_EXECUTION_STALLED_RECONCILE_OLDER_THAN_MINUTES: "0",
+        WORKFLOW_EXECUTION_STALLED_RECONCILE_BATCH_SIZE: "nope",
+      } as NodeJS.ProcessEnv),
+    ).toEqual({
+      olderThanMinutes: WORKFLOW_EXECUTION_STALLED_RECONCILE_DEFAULT_OLDER_THAN_MINUTES,
+      take: WORKFLOW_EXECUTION_STALLED_RECONCILE_DEFAULT_BATCH_SIZE,
+    });
+  });
+
+  it("registers a five-minute scheduler with cleanup.run payload", async () => {
+    const upsert = vi.fn().mockResolvedValue(undefined);
+    await registerWorkflowExecutionStalledReconcilerCron(
+      { upsertJobScheduler: upsert },
+      {
+        WORKFLOW_EXECUTION_STALLED_RECONCILE_OLDER_THAN_MINUTES: "60",
+        WORKFLOW_EXECUTION_STALLED_RECONCILE_BATCH_SIZE: "500",
+      } as NodeJS.ProcessEnv,
+    );
+
+    expect(upsert).toHaveBeenCalledTimes(1);
+    expect(upsert.mock.calls[0]?.[0]).toBe(WORKFLOW_EXECUTION_STALLED_RECONCILE_JOB_ID);
+    expect(upsert.mock.calls[0]?.[1]).toEqual({ every: WORKFLOW_EXECUTION_STALLED_RECONCILE_EVERY_MS });
+    expect(upsert.mock.calls[0]?.[2]).toEqual({
+      name: WORKFLOW_EXECUTION_STALLED_RECONCILE_JOB_ID,
+      data: {
+        kind: "cleanup.run",
+        task: "workflow-execution-stalled-reconcile",
+        olderThanMinutes: 60,
+        take: 500,
+      },
+    });
+  });
+});