Track HiveGrid Cloud planning reference

hivegrid-cloud/apps/api/README.md

@@ -0,0 +1,37 @@
+# @toolctrl/api
+
+Dedicated Hono API application for `api.<domain>/v1`.
+
+This workspace owns the canonical API host. External integrations should use the `/v1` routes documented by OpenAPI; API-owned usage and billing webhook handlers now run in this runtime rather than through web-runtime proxies.
+
+## Endpoints
+
+- `GET /v1/health` returns a lightweight JSON health payload.
+- `GET /healthz` returns liveness status and `GET /readyz` returns readiness/dependency status for infrastructure checks.
+- `GET /v1/openapi.json` and `GET /v1/docs` expose the API contract.
+- `GET /v1/session/current-user` requires `Authorization: Bearer <API_ACCESS_TOKEN>` and returns the authenticated service principal.
+- `POST /v1/api-keys/validate` validates `apiKey` from the JSON body, `x-api-key` header, or bearer token and returns `{ "active": true, "keyType": "service-token" }` for the configured service token.
+- `GET /v1/accounts`, `GET /v1/accounts/:id`, and `GET /v1/usage` expose service/API-key scoped account and usage metadata.
+- `/v1/entities/*`, `/v1/relationships/*`, `/v1/exports/*`, and `/v1/workspaces/*` expose the current tenant-scoped API surface.
+- `POST /v1/billing/stripe/webhooks` and `POST /v1/events/webhooks/attempts/:id` are signed webhook ingress routes.
+
+Use `/v1/openapi.json` and `apps/api/src/route-access-inventory.ts` for the detailed route inventory.
+
+## Auth and Request IDs
+
+Set `API_ACCESS_TOKEN` in the API environment to enable the service-token primitives. Missing or invalid credentials return JSON error envelopes with stable `error.code`, `error.message`, and `error.requestId` fields without echoing token values. Requests may provide `x-request-id`; the API echoes it in the response header and includes it in JSON error bodies for correlation. If no request ID is supplied, the API generates one.
+
+## Scripts
+
+- `pnpm --filter @toolctrl/api dev` starts the API with `tsx watch`.
+- `pnpm --filter @toolctrl/api build` compiles `src/` to `dist/`.
+- `pnpm --filter @toolctrl/api start` runs the compiled server.
+- `pnpm --filter @toolctrl/api typecheck` runs TypeScript without emitting files.
+
+The server listens on `API_PORT` when set, falls back to `PORT`, and otherwise defaults to `3001`.
+
+## Extraction Notes
+
+- Keep `src/app.ts` focused on Hono app composition so tests, adapters and future deployment targets can import the app without starting a listener.
+- Keep newly extracted API behavior in focused API modules instead of growing `src/app.ts` with service logic.
+- Keep shared environment validation in `@toolctrl/config` and update the catalog when API runtime keys change.

hivegrid-cloud/apps/api/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@toolctrl/api",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "sideEffects": [
+    "./src/otel.ts"
+  ],
+  "exports": {
+    "./openapi": "./src/openapi.ts"
+  },
+  "scripts": {
+    "dev": "tsx watch src/server.ts",
+    "start": "node dist/server.js",
+    "typecheck": "tsc --noEmit",
+    "build": "node scripts/build.mjs"
+  },
+  "dependencies": {
+    "@hono/node-server": "^1.19.6",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.76.0",
+    "@opentelemetry/core": "^2.7.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.218.0",
+    "@opentelemetry/resources": "^2.7.1",
+    "@opentelemetry/sdk-node": "^0.218.0",
+    "@opentelemetry/sdk-trace-base": "^2.7.1",
+    "@opentelemetry/semantic-conventions": "^1.40.0",
+    "@sentry/node": "^10.53.1",
+    "@toolctrl/config": "workspace:*",
+    "@toolctrl/db": "workspace:*",
+    "@toolctrl/error-reporter": "workspace:*",
+    "@toolctrl/foundation": "workspace:*",
+    "@toolctrl/logger": "workspace:*",
+    "@toolctrl/metrics": "workspace:*",
+    "@toolctrl/observability": "workspace:*",
+    "@toolctrl/plugin-runtime": "workspace:*",
+    "@toolctrl/queue": "workspace:*",
+    "@toolctrl/realtime": "workspace:*",
+    "@toolctrl/shared-store": "workspace:*",
+    "@toolctrl/storage": "workspace:*",
+    "bcryptjs": "^2.4.3",
+    "hono": "^4.12.17",
+    "jsonwebtoken": "^9.0.0",
+    "stripe": "^19.1.0",
+    "zod": "^3.25.72"
+  },
+  "devDependencies": {
+    "@types/bcryptjs": "^2.4.2",
+    "@types/jsonwebtoken": "^9.0.6",
+    "@types/node": "^25.6.0",
+    "tsx": "^4.6.2",
+    "typescript": "^5.3.2"
+  }
+}

hivegrid-cloud/apps/api/scripts/build.mjs

@@ -0,0 +1,66 @@
+import { mkdir } from "node:fs/promises";
+import { builtinModules } from "node:module";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { build } from "esbuild";
+
+const apiDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const entryPoint = path.join(apiDir, "src/server.ts");
+const outputFile = path.join(apiDir, "dist/server.js");
+const nodeBuiltins = new Set([...builtinModules, ...builtinModules.map((moduleName) => `node:${moduleName}`)]);
+const sourcemap = process.env.SOURCEMAP === "true" || process.env.BUILD_SOURCEMAPS === "true";
+
+const externalizeRuntimeDependencies = {
+  name: "externalize-runtime-dependencies",
+  setup(buildContext) {
+    buildContext.onResolve({ filter: /.*/ }, (args) => {
+      if (nodeBuiltins.has(args.path)) {
+        return { path: args.path, external: true };
+      }
+
+      if (isExternalRuntimePackage(args.path)) {
+        return { path: args.path, external: true };
+      }
+
+      return undefined;
+    });
+  },
+};
+
+await mkdir(path.dirname(outputFile), { recursive: true });
+
+await build({
+  entryPoints: [entryPoint],
+  outfile: outputFile,
+  bundle: true,
+  platform: "node",
+  format: "esm",
+  target: "node20",
+  sourcemap,
+  banner: {
+    js: 'import { createRequire as __toolctrlCreateRequire } from "node:module"; const require = __toolctrlCreateRequire(import.meta.url);',
+  },
+  plugins: [externalizeRuntimeDependencies],
+  logLevel: "warning",
+});
+
+function isBarePackageSpecifier(specifier) {
+  return !specifier.startsWith(".") && !specifier.startsWith("/") && !specifier.startsWith("#");
+}
+
+function isExternalRuntimePackage(specifier) {
+  if (!isBarePackageSpecifier(specifier)) {
+    return false;
+  }
+
+  return (
+    specifier === "@prisma/client" ||
+    specifier.startsWith("@prisma/client/") ||
+    specifier === "@prisma/adapter-pg" ||
+    specifier.startsWith("@prisma/adapter-pg/") ||
+    specifier === "jsonwebtoken" ||
+    specifier.startsWith("jsonwebtoken/") ||
+    specifier === "pino" ||
+    specifier.startsWith("pino/")
+  );
+}

hivegrid-cloud/apps/api/scripts/smoke-sentry.ts

@@ -0,0 +1,80 @@
+#!/usr/bin/env tsx
+/**
+ * Sentry smoke test for the API process.
+ *
+ * Triggers a fake captureException + a fake `Sentry.startSpan` transaction
+ * so an operator can verify end-to-end that:
+ *   - the API DSN is wired,
+ *   - `beforeSend` sanitises PII,
+ *   - the matching Sentry project receives both an Issue and a Transaction.
+ *
+ * Run from the repo root with one of:
+ *   pnpm --filter @toolctrl/api exec tsx scripts/smoke-sentry.ts
+ *   SENTRY_DSN_API=... pnpm --filter @toolctrl/api exec tsx scripts/smoke-sentry.ts
+ *
+ * Exits 0 on a clean run, 1 when the SDK fails to load. A no-DSN run is
+ * intentionally a successful no-op so CI can call it unconditionally.
+ *
+ * @see docs/improvement-plan/02-observability/sentry-wiring-api-worker.md
+ */
+import { initApiSentry, readApiSentryConfig } from "../src/sentry.js";
+
+async function main(): Promise<void> {
+  const config = readApiSentryConfig();
+  if (!config.dsn) {
+    // eslint-disable-next-line no-console
+    console.warn("[smoke-sentry] SENTRY_DSN_API is not set; nothing to do.");
+    return;
+  }
+
+  const reporter = initApiSentry(config);
+
+  // 1) Fake captureException — exercises beforeSend + sanitisation.
+  await reporter.report({
+    component: "api",
+    operation: "smoke.exception",
+    severity: "error",
+    error: new Error(`smoke-sentry: fake exception at ${new Date().toISOString()}`),
+    // Include obvious PII fields so the operator can confirm redaction in the
+    // Sentry UI (`token` / `password` should render as [Redacted]).
+    token: "fake-token-must-be-redacted",
+    password: "fake-password-must-be-redacted",
+  });
+
+  // 2) Fake transaction — exercises @sentry/node startSpan.
+  try {
+    const moduleName = "@sentry/node";
+    const sentry = (await import(/* @vite-ignore */ moduleName)) as {
+      startSpan?: (args: { op: string; name: string }, cb: () => Promise<void> | void) => Promise<void> | void;
+      default?: {
+        startSpan?: (args: { op: string; name: string }, cb: () => Promise<void> | void) => Promise<void> | void;
+      };
+    };
+    const startSpan = sentry.startSpan ?? sentry.default?.startSpan;
+    if (typeof startSpan === "function") {
+      await startSpan({ op: "test", name: "smoke-sentry/api" }, async () => {
+        await new Promise((resolve) => setTimeout(resolve, 25));
+      });
+    } else {
+      // eslint-disable-next-line no-console
+      console.warn("[smoke-sentry] @sentry/node loaded but startSpan is unavailable; transaction skipped.");
+    }
+  } catch (cause) {
+    // eslint-disable-next-line no-console
+    console.error("[smoke-sentry] failed to load @sentry/node for the transaction smoke:", cause);
+    process.exitCode = 1;
+  }
+
+  // Give the SDK enough time to flush. Sentry's `close()` would be more
+  // precise but pulling the type via dynamic import is more trouble than it
+  // is worth for a manual smoke script.
+  await new Promise((resolve) => setTimeout(resolve, 1500));
+  // eslint-disable-next-line no-console
+  console.log("[smoke-sentry] done — check the Sentry UI (Issues + Performance) for boilerplate-api.");
+}
+
+main().catch((cause) => {
+  // eslint-disable-next-line no-console
+  console.error("[smoke-sentry] unexpected failure:", cause);
+  process.exit(1);
+});

hivegrid-cloud/apps/api/src/api-types.ts

@@ -0,0 +1,7 @@
+import type { ApiRequestIdVariables } from "./middleware/request-id.js";
+
+export type ApiEnv = {
+  Variables: ApiRequestIdVariables;
+};
+
+export type ParseResult<T> = { ok: true; value: T } | { ok: false; error: string };

hivegrid-cloud/apps/api/src/app-dependencies.ts

@@ -0,0 +1,402 @@
+import {
+  createUniqueLegacyTenantIpAddress,
+  adminGetAllLegacyTenants,
+  findLegacyTenantById,
+  findLegacyTenantByIdOrSlug,
+  type LegacyTenantWithDetails,
+} from "@toolctrl/db/accounts.server";
+import {
+  createLegacyApiKeyLog,
+  findLegacyApiKeyByKey,
+  setLegacyApiKeyLogStatus,
+} from "@toolctrl/db/api-keys.server";
+import {
+  listWorkspaceAuthProviders,
+  revalidateWorkspaceContext,
+  type WorkspaceAuthProviderDto,
+} from "@toolctrl/db/auth-providers.server";
+import {
+  createAnalyticsEvent,
+  createAnalyticsPageView,
+  findFeatureFlagByName,
+  getAnalyticsOverview,
+  getAnalyticsSettings,
+  getOrCreateAnalyticsUniqueVisitor,
+  isAnalyticsPeriod,
+  defaultAnalyticsPeriod,
+  updateAnalyticsUniqueVisitorSource,
+  type AnalyticsOverview,
+  type AnalyticsInfo,
+  type AnalyticsPeriod,
+  type AnalyticsUniqueVisitor,
+} from "@toolctrl/db/analytics.server";
+import {
+  findLegacyAuthUserById,
+  findLegacyLoginUserByEmail,
+  recordLegacyLogin,
+  type LegacyAuthUser,
+  type LegacyLoginUser,
+} from "@toolctrl/db/users.server";
+import { findLegacyEntityByName, type LegacyEntity, type LegacyEntityWithDetails } from "@toolctrl/db/entities.server";
+import {
+  createLegacyEntityRow,
+  deleteLegacyEntityRow,
+  findLegacyRelationshipRowById,
+  getAllLegacyEntityRows,
+  getLegacyEntityRow,
+  updateLegacyEntityRow,
+  type LegacyEntityRow,
+  type LegacyEntityRowPage,
+  type LegacyRelationshipRow,
+} from "@toolctrl/db/rows.server";
+import {
+  createLegacyRowRelationship,
+  deleteLegacyRowRelationshipById,
+  findLegacyEntityRelationship,
+  findLegacyRowRelationshipById,
+  type LegacyEntityRelationshipDetails,
+  type LegacyRowRelationshipDetails,
+} from "@toolctrl/db/relationships.server";
+import {
+  findLegacyEventWebhookAttemptById,
+  updateLegacyEventWebhookAttempt,
+  type LegacyEventWebhookAttempt,
+  type LegacyEventWebhookAttemptUpdate,
+} from "@toolctrl/db/event-webhook-attempts.server";
+import { createLegacyEvent } from "@toolctrl/db/events.server";
+import { markEmailSequenceDeliveryBounced, type MarkEmailSequenceDeliveryBouncedInput, type EmailSequenceProviderWebhookResult } from "@toolctrl/db/email-sequences.server";
+import { findExportJobForDownload, type ExportJobDownloadRecord } from "@toolctrl/db/export-jobs.server";
+import { readExportProgressFromSharedStore } from "./export-progress.js";
+import {
+  findTenantSubscriptionProductByStripeSubscriptionId,
+  getTenantSubscription,
+  updateTenantSubscriptionProductFromStripe,
+  type StripeTenantSubscriptionProduct,
+  type TenantSubscriptionWithDetails,
+} from "@toolctrl/db/subscriptions.server";
+import { getTenantApiUsage } from "@toolctrl/db/usage.server";
+import { publishOutboxEvent } from "@toolctrl/db/outbox";
+import { createInboundSupportTicketFromEmail, type InboundSupportTicketInput, type InboundSupportTicketResult } from "@toolctrl/db/support-tickets.server";
+import {
+  acceptWorkspaceInvite,
+  createWorkspaceInvite,
+  listWorkspaceInvites,
+  revokeWorkspaceInvite,
+  type WorkspaceInviteCreateResult,
+  type WorkspaceInviteDto,
+} from "@toolctrl/db/workspace-invites.server";
+import { getDbClient } from "@toolctrl/db/client.server";
+import { getNotificationStreamSnapshot, type NotificationStreamCursor, type NotificationStreamSnapshot } from "@toolctrl/db/notifications.server";
+import bcrypt from "bcryptjs";
+import Stripe from "stripe";
+import {
+  checkLoginRateLimit,
+  recordFailedLoginAttempt,
+  recordSuccessfulLogin,
+  resetLoginRateLimiterForTests,
+  type LoginRateLimitRequest,
+} from "./auth/login-rate-limit.js";
+import { apiSessionStore } from "./auth/session/session-store.js";
+import { getApiRuntimeConfig } from "./config.js";
+import {
+  createMemoryIdempotencyStorageClient,
+  type IdempotencyStorageClient,
+} from "./idempotency/store.js";
+import { createSharedStoreIdempotencyStorageClient } from "./idempotency/shared-store-client.js";
+import { getSharedStore } from "./shared-store.js";
+import { createStorageFile, createStorageSignedDownloadUrl, type CreateStorageSignedDownloadUrlInput, type StorageSignedDownloadUrlResult } from "./storage.js";
+import {
+  queueLegacyEventWebhookAttemptDelivery,
+  type QueueLegacyEventWebhookAttemptDeliveryInput,
+  type QueueLegacyEventWebhookAttemptDeliveryResult,
+} from "./webhookDeliveryQueue.js";
+import {
+  queueExportArtifactRetention,
+  type QueueExportArtifactRetentionInput,
+  type QueueExportArtifactRetentionResult,
+} from "./exportRetentionQueue.js";
+
+export { defaultAnalyticsPeriod, isAnalyticsPeriod };
+export type {
+  AnalyticsInfo,
+  AnalyticsOverview,
+  AnalyticsPeriod,
+  AnalyticsUniqueVisitor,
+  ExportJobDownloadRecord,
+  LegacyEntity,
+  LegacyEntityRelationshipDetails,
+  LegacyEntityRow,
+  LegacyEntityRowPage,
+  LegacyEntityWithDetails,
+  LegacyEventWebhookAttempt,
+  LegacyRelationshipRow,
+  LegacyRowRelationshipDetails,
+  LegacyTenantWithDetails,
+  LoginRateLimitRequest,
+  NotificationStreamCursor,
+  NotificationStreamSnapshot,
+  StripeTenantSubscriptionProduct,
+  TenantSubscriptionWithDetails,
+  WorkspaceAuthProviderDto,
+  WorkspaceInviteCreateResult,
+  WorkspaceInviteDto,
+};
+
+type AnalyticsEventCreateData = {
+  uniqueVisitorId: string;
+  action: string;
+  category?: string;
+  label?: string;
+  value?: string;
+  url?: string;
+  route?: string;
+  featureFlagId?: string;
+  portalId?: string | null;
+  portalUserId?: string | null;
+};
+
+type AnalyticsPageViewCreateData = {
+  uniqueVisitorId: string;
+  url: string;
+  route?: string;
+  portalId?: string | null;
+  portalUserId?: string | null;
+};
+
+export type ApiAppDependencies = {
+  findLegacyApiKeyByKey: typeof findLegacyApiKeyByKey;
+  createLegacyApiKeyLog: typeof createLegacyApiKeyLog;
+  setLegacyApiKeyLogStatus: typeof setLegacyApiKeyLogStatus;
+  createUniqueLegacyTenantIpAddress: typeof createUniqueLegacyTenantIpAddress;
+  createAnalyticsEvent(data: AnalyticsEventCreateData): Promise<void>;
+  createAnalyticsPageView(data: AnalyticsPageViewCreateData): Promise<void>;
+  findFeatureFlagByName(name: string): Promise<{ id: string } | null>;
+  getAnalyticsOverview(args: { period: AnalyticsPeriod; portalId: string | null | undefined }): Promise<AnalyticsOverview>;
+  findLegacyAuthUserById(userId: string): Promise<LegacyAuthUser | null>;
+  findLegacyEntityByName(args: { tenantId: string | null | undefined; name: string; active?: boolean }): Promise<LegacyEntity>;
+  getAllLegacyEntityRows(args: { entityId: string; tenantId: string | null; page: number; pageSize: number }): Promise<LegacyEntityRowPage>;
+  getLegacyEntityRow(args: { entityId: string; tenantId: string | null; rowId: string }): Promise<LegacyEntityRow | null>;
+  findLegacyRelationshipRowById(args: { id: string; tenantId?: string | null }): Promise<LegacyRelationshipRow | null>;
+  findLegacyEntityRelationship(args: { parentId: string; childId: string }): Promise<LegacyEntityRelationshipDetails | null>;
+  findLegacyRowRelationshipById(id: string, tenantId?: string | null): Promise<LegacyRowRelationshipDetails | null>;
+  createLegacyRowRelationship(args: { relationshipId: string; parentId: string; childId: string; tenantId?: string | null; metadata?: string | null }): Promise<LegacyRowRelationshipDetails>;
+  deleteLegacyRowRelationshipById(args: { id: string; relationshipId: string; parentId: string; childId: string; tenantId: string | null }): Promise<unknown>;
+  createLegacyEntityRow(args: { entity: LegacyEntityWithDetails; tenantId: string | null; createdByUserId?: string | null; createdByApiKeyId?: string | null; data: Record<string, unknown> }): Promise<LegacyEntityRow>;
+  updateLegacyEntityRow(args: { entity: LegacyEntityWithDetails; tenantId: string | null; rowId: string; data: Record<string, unknown> }): Promise<LegacyEntityRow>;
+  deleteLegacyEntityRow(args: { entityId: string; tenantId: string | null; rowId: string }): Promise<string>;
+  findLegacyEventWebhookAttemptById(id: string): Promise<LegacyEventWebhookAttempt | null>;
+  findTenantSubscriptionProductByStripeSubscriptionId(stripeSubscriptionId: string): Promise<StripeTenantSubscriptionProduct | null>;
+  updateTenantSubscriptionProductFromStripe: typeof updateTenantSubscriptionProductFromStripe;
+  createLegacyEvent: typeof createLegacyEvent;
+  /**
+   * Phase 03 #42 — transaction runner used by the Stripe webhook so the
+   * subscription update + outbox publishes commit atomically. The
+   * callback receives a transaction client that doubles as both a
+   * Prisma-style subscription writer and an OutboxPublisherTransaction.
+   */
+  runStripeWebhookTransaction: import("./billing/stripe-webhook-route.js").StripeWebhookTransactionRunner;
+  /**
+   * Phase 03 #42 — outbox publisher bound to the surrounding
+   * transaction. Defaults to `publishOutboxEvent` from `@toolctrl/db/outbox`.
+   */
+  publishOutboxEvent: typeof import("@toolctrl/db/outbox").publishOutboxEvent;
+  findLegacyLoginUserByEmail(email: string): Promise<LegacyLoginUser | null>;
+  adminGetAllLegacyTenants(): Promise<LegacyTenantWithDetails[]>;
+  findLegacyTenantById(id: string): Promise<LegacyTenantWithDetails | null>;
+  findLegacyTenantByIdOrSlug(idOrSlug: string): Promise<{ id: string } | null>;
+  getTenantSubscription: typeof getTenantSubscription;
+  getTenantApiUsage: typeof getTenantApiUsage;
+  createInboundSupportTicketFromEmail(input: InboundSupportTicketInput): Promise<InboundSupportTicketResult>;
+  markEmailSequenceDeliveryBounced(input: MarkEmailSequenceDeliveryBouncedInput): Promise<EmailSequenceProviderWebhookResult>;
+  findExportJobForDownload(args: { id: string; tenantId: string }): Promise<ExportJobDownloadRecord | null>;
+  readExportProgress(args: { exportJobId: string; tenantId: string }): Promise<{ processedRows: number; totalEstimate: number | null; updatedAt: string | null } | null>;
+  getAnalyticsSettings(): Promise<{ public: boolean; ignorePages: string } | null>;
+  getOrCreateAnalyticsUniqueVisitor(args: { cookie: string; fromUrl?: string; fromRoute?: string; analytics: AnalyticsInfo }): Promise<AnalyticsUniqueVisitor>;
+  updateAnalyticsUniqueVisitorSource(visitor: AnalyticsUniqueVisitor, analytics: AnalyticsInfo): Promise<void>;
+  updateLegacyEventWebhookAttempt(args: { id: string; tenantId: string | null }, data: LegacyEventWebhookAttemptUpdate): Promise<void>;
+  queueLegacyEventWebhookAttemptDelivery(input: QueueLegacyEventWebhookAttemptDeliveryInput): Promise<QueueLegacyEventWebhookAttemptDeliveryResult>;
+  queueExportArtifactRetention(input: QueueExportArtifactRetentionInput): Promise<QueueExportArtifactRetentionResult>;
+  getNotificationStreamSnapshot(filters: { tenantId: string; userId: string; afterCursor?: NotificationStreamCursor | null }): Promise<NotificationStreamSnapshot>;
+  createWorkspaceInvite(input: Parameters<typeof createWorkspaceInvite>[0]): Promise<WorkspaceInviteCreateResult>;
+  listWorkspaceInvites(input: Parameters<typeof listWorkspaceInvites>[0]): Promise<WorkspaceInviteDto[]>;
+  acceptWorkspaceInvite(input: Parameters<typeof acceptWorkspaceInvite>[0]): Promise<WorkspaceInviteDto>;
+  revokeWorkspaceInvite(input: Parameters<typeof revokeWorkspaceInvite>[0]): Promise<WorkspaceInviteDto>;
+  listWorkspaceAuthProviders(input: Parameters<typeof listWorkspaceAuthProviders>[0]): Promise<WorkspaceAuthProviderDto[]>;
+  revalidateWorkspaceContext(input: Parameters<typeof revalidateWorkspaceContext>[0]): ReturnType<typeof revalidateWorkspaceContext>;
+  uploadFile(bucketId: string, path: string, file: UploadBody, contentType?: string): Promise<{ id: string; publicUrl: string | null }>;
+  createSignedDownloadUrl(bucketId: string, input: CreateStorageSignedDownloadUrlInput): Promise<StorageSignedDownloadUrlResult>;
+  constructStripeWebhookEvent(payload: string, signature: string): Stripe.Event;
+  retrieveStripeSubscription(id: string): Promise<Stripe.Subscription | null>;
+  recordLegacyLogin(userId: string): Promise<void>;
+  comparePassword(password: string, passwordHash: string): Promise<boolean>;
+  checkLoginRateLimit(request: LoginRateLimitRequest): ReturnType<typeof checkLoginRateLimit>;
+  recordFailedLoginAttempt(request: LoginRateLimitRequest): Promise<void>;
+  recordSuccessfulLogin(request: Pick<LoginRateLimitRequest, "account">): Promise<void>;
+  resetLoginRateLimiter(): Promise<void>;
+  resetApiSessionStore(): void;
+  createApiSession(userId: string): ReturnType<typeof apiSessionStore.createSession>;
+  getActiveApiSession(sessionId: string): ReturnType<typeof apiSessionStore.getActiveSession>;
+  rotateApiSession(sessionId: string): ReturnType<typeof apiSessionStore.rotateSession>;
+  revokeApiSession(sessionId: string): ReturnType<typeof apiSessionStore.revokeSession>;
+  getIdempotencyClient(): IdempotencyStorageClient;
+  checkReadiness(): Promise<void>;
+};
+
+type UploadBody = Parameters<typeof createStorageFile>[2];
+
+const STRIPE_API_VERSION = "2025-10-29.clover";
+let apiStripeClient: Stripe | null = null;
+let apiStripeClientSecretKey: string | undefined;
+
+function getApiStripeClient() {
+  const secretKey = getApiRuntimeConfig().stripe.secretKey;
+  if (!secretKey) {
+    throw new Error("STRIPE_SK is required before using Stripe API services.");
+  }
+  if (!apiStripeClient || apiStripeClientSecretKey !== secretKey) {
+    apiStripeClient = new Stripe(secretKey, { apiVersion: STRIPE_API_VERSION });
+    apiStripeClientSecretKey = secretKey;
+  }
+  return apiStripeClient;
+}
+
+/**
+ * Idempotency-storage backend selection (Phase 03 #38).
+ *
+ * Default is the Postgres-backed Prisma client, which is already
+ * multi-replica-safe. Setting `IDEMPOTENCY_BACKEND=shared-store` swaps in
+ * the SharedStore-backed client (Redis in production) so a deployment can
+ * keep idempotency state off the primary database. Any other value — or
+ * none — keeps the Prisma default.
+ */
+let sharedStoreIdempotencyClient: IdempotencyStorageClient | null = null;
+
+function resolveIdempotencyClient(): IdempotencyStorageClient {
+  if (process.env.IDEMPOTENCY_BACKEND === "shared-store") {
+    if (!sharedStoreIdempotencyClient) {
+      sharedStoreIdempotencyClient = createSharedStoreIdempotencyStorageClient({ store: getSharedStore() });
+    }
+    return sharedStoreIdempotencyClient;
+  }
+  return getDbClient() as unknown as IdempotencyStorageClient;
+}
+
+const defaultApiAppDependencies: ApiAppDependencies = {
+  findLegacyApiKeyByKey,
+  createLegacyApiKeyLog,
+  setLegacyApiKeyLogStatus,
+  createUniqueLegacyTenantIpAddress,
+  createAnalyticsEvent,
+  createAnalyticsPageView,
+  findFeatureFlagByName,
+  getAnalyticsOverview,
+  findLegacyAuthUserById,
+  findLegacyEntityByName,
+  getAllLegacyEntityRows,
+  getLegacyEntityRow,
+  findLegacyRelationshipRowById,
+  findLegacyEntityRelationship,
+  findLegacyRowRelationshipById,
+  createLegacyRowRelationship,
+  deleteLegacyRowRelationshipById,
+  createLegacyEntityRow,
+  updateLegacyEntityRow,
+  deleteLegacyEntityRow,
+  findLegacyEventWebhookAttemptById,
+  findTenantSubscriptionProductByStripeSubscriptionId,
+  updateTenantSubscriptionProductFromStripe,
+  createLegacyEvent,
+  // Phase 03 #42 — Stripe-webhook outbox migration. The default runs
+  // the callback inside `prisma.$transaction(...)`; the resulting tx
+  // client doubles as both StripeTenantSubscriptionProductClient and
+  // OutboxPublisherTransaction (Prisma transaction clients implement
+  // both surfaces structurally).
+  runStripeWebhookTransaction: (async <T,>(fn: (tx: import("./billing/stripe-webhook-route.js").StripeWebhookTransactionClient) => Promise<T>) => {
+    const prisma = getDbClient() as unknown as {
+      $transaction: <U>(runner: (tx: import("./billing/stripe-webhook-route.js").StripeWebhookTransactionClient) => Promise<U>) => Promise<U>;
+    };
+    return prisma.$transaction(fn);
+  }) as import("./billing/stripe-webhook-route.js").StripeWebhookTransactionRunner,
+  publishOutboxEvent,
+  findLegacyLoginUserByEmail,
+  adminGetAllLegacyTenants,
+  findLegacyTenantById,
+  findLegacyTenantByIdOrSlug,
+  getTenantSubscription,
+  getTenantApiUsage,
+  createInboundSupportTicketFromEmail,
+  markEmailSequenceDeliveryBounced,
+  findExportJobForDownload,
+  readExportProgress: readExportProgressFromSharedStore,
+  getAnalyticsSettings,
+  getOrCreateAnalyticsUniqueVisitor,
+  updateAnalyticsUniqueVisitorSource,
+  updateLegacyEventWebhookAttempt,
+  queueLegacyEventWebhookAttemptDelivery,
+  queueExportArtifactRetention,
+  getNotificationStreamSnapshot,
+  createWorkspaceInvite,
+  listWorkspaceInvites,
+  acceptWorkspaceInvite,
+  revokeWorkspaceInvite,
+  listWorkspaceAuthProviders,
+  revalidateWorkspaceContext,
+  uploadFile: createStorageFile,
+  createSignedDownloadUrl: createStorageSignedDownloadUrl,
+  constructStripeWebhookEvent(payload, signature) {
+    const config = getApiRuntimeConfig();
+    if (!config.stripe.webhookSigningSecret) {
+      throw new Error("STRIPE_WEBHOOK_SIGNING_SECRET is required before verifying Stripe webhooks.");
+    }
+    return getApiStripeClient().webhooks.constructEvent(payload, signature, config.stripe.webhookSigningSecret);
+  },
+  async retrieveStripeSubscription(id) {
+    try {
+      return await getApiStripeClient().subscriptions.retrieve(id);
+    } catch {
+      return null;
+    }
+  },
+  recordLegacyLogin,
+  comparePassword: bcrypt.compare,
+  checkLoginRateLimit,
+  recordFailedLoginAttempt,
+  recordSuccessfulLogin,
+  resetLoginRateLimiter: resetLoginRateLimiterForTests,
+  resetApiSessionStore: apiSessionStore.reset,
+  createApiSession: apiSessionStore.createSession,
+  getActiveApiSession: apiSessionStore.getActiveSession,
+  rotateApiSession: apiSessionStore.rotateSession,
+  revokeApiSession: apiSessionStore.revokeSession,
+  getIdempotencyClient() {
+    return resolveIdempotencyClient();
+  },
+  async checkReadiness() {
+    await getDbClient().$queryRaw`SELECT 1`;
+  },
+};
+
+let apiAppDependencies = defaultApiAppDependencies;
+
+export function getApiAppDependencies() {
+  return apiAppDependencies;
+}
+
+export function configureApiAppDependenciesForTests(dependencies: Partial<ApiAppDependencies> = {}) {
+  const idempotencyClient = createMemoryIdempotencyStorageClient();
+  // Fire-and-forget; the rate limiter and session store reset to a clean
+  // state on the next access regardless of when this promise settles.
+  void defaultApiAppDependencies.resetLoginRateLimiter();
+  defaultApiAppDependencies.resetApiSessionStore();
+  apiAppDependencies = {
+    ...defaultApiAppDependencies,
+    getIdempotencyClient: () => idempotencyClient,
+    ...dependencies,
+  };
+
+  return () => {
+    void apiAppDependencies.resetLoginRateLimiter();
+    apiAppDependencies.resetApiSessionStore();
+    apiAppDependencies = defaultApiAppDependencies;
+  };
+}

hivegrid-cloud/apps/api/src/app.ts

@@ -0,0 +1,136 @@
+import { Hono } from "hono";
+import { cors } from "hono/cors";
+import { isMetricsEnabled } from "@toolctrl/metrics";
+import { getApiAppDependencies, configureApiAppDependenciesForTests, type ApiAppDependencies } from "./app-dependencies.js";
+import type { ApiEnv } from "./api-types.js";
+import { postStripeBillingWebhookRequest } from "./billing/stripe-webhook-route.js";
+import { getAllowedCorsOrigin } from "./cors.js";
+import { apiLogger } from "./logger.js";
+import { getApiMetrics } from "./metrics.js";
+import { createBodyLimitMiddleware } from "./middleware/body-limit.js";
+import { createErrorHandler, createNotFoundHandler } from "./middleware/error-envelope.js";
+import { createMetricsMiddleware } from "./middleware/metrics.js";
+import { createRequestIdMiddleware } from "./middleware/request-id.js";
+import { createSentryTracingMiddleware } from "./middleware/sentryTracing.js";
+import { createAuditContextMiddleware } from "./middleware/audit-context.js";
+import { createApiMaintenanceMiddleware } from "./platform/maintenance/maintenance.js";
+import { apiErrorReporter } from "./sentry.js";
+import { registerAccountsRoutes } from "./routes/accounts-routes.js";
+import { registerAnalyticsRoutes } from "./routes/analytics-routes.js";
+import { registerAuthRoutes } from "./routes/auth-routes.js";
+import { registerDevicesRoutes } from "./routes/devices-routes.js";
+import { registerEntityRowRoutes } from "./routes/entity-row-routes.js";
+import { registerMaintenanceRoutes } from "./routes/maintenance-routes.js";
+import { handleApiMetricsRequest } from "./routes/metrics.js";
+import { registerNotificationRoutes } from "./routes/notifications-routes.js";
+import { registerRelationshipRoutes } from "./routes/relationship-routes.js";
+import { handleApiStatusRequest } from "./routes/status.js";
+import { registerSystemRoutes } from "./routes/system-routes.js";
+import { registerWorkspaceRoutes } from "./routes/workspace-routes.js";
+import { getUsageRequest } from "./usage/usage-route.js";
+
+export { configureApiAppDependenciesForTests };
+export type { ApiAppDependencies };
+
+export const app = new Hono<ApiEnv>();
+
+// Sentry tracing MUST be the first middleware so every request starts a span
+// (including 404s + auth-rejected requests). Self-disables when SENTRY_DSN_API
+// is empty so local dev keeps zero Sentry overhead.
+app.use("*", createSentryTracingMiddleware());
+
+app.use("*", createRequestIdMiddleware({ logger: apiLogger }));
+// Audit-capture context MUST come after requestId so deeper code paths
+// can attach the request-id to AuditEvent rows for cross-stack correlation.
+app.use("*", createAuditContextMiddleware());
+app.use("*", createBodyLimitMiddleware());
+app.use("*", createApiMaintenanceMiddleware());
+
+// Prometheus per-request instrumentation. Self-noop when METRICS_ENABLED is
+// false so dev/test runs incur zero overhead. The shared registry from
+// `getApiMetrics()` is also used by the /metrics endpoint below.
+const apiMetricsEnabled = isMetricsEnabled(process.env);
+const apiMetricsBundle = getApiMetrics();
+app.use(
+  "*",
+  createMetricsMiddleware({
+    enabled: apiMetricsEnabled,
+    metrics: apiMetricsBundle.metrics,
+  })
+);
+
+app.get("/status", (c) =>
+  handleApiStatusRequest(c, {
+    checkDb: () => getApiAppDependencies().checkReadiness(),
+    getRevision: () => process.env.APP_REVISION?.trim() ?? "",
+    getRegion: () => process.env.APP_REGION?.trim() ?? process.env.FLY_REGION?.trim() ?? "",
+    getUptimeSeconds: () => process.uptime(),
+  })
+);
+
+app.get("/metrics", (c) =>
+  handleApiMetricsRequest(c, {
+    enabled: apiMetricsEnabled,
+    configuredToken: process.env.METRICS_TOKEN,
+    registry: apiMetricsBundle.registry,
+  })
+);
+
+app.use("*", async (c, next) => {
+  await next();
+  c.header("x-content-type-options", "nosniff");
+  c.header("referrer-policy", "no-referrer");
+  c.header("x-frame-options", "DENY");
+  c.header("permissions-policy", "camera=(), microphone=(), geolocation=(), payment=()");
+  c.header("cross-origin-opener-policy", "same-origin");
+  c.header("cross-origin-resource-policy", "same-origin");
+});
+
+app.use(
+  "*",
+  cors({
+    origin: getAllowedCorsOrigin,
+    allowHeaders: [
+      "authorization",
+      "content-type",
+      "idempotency-key",
+      "last-event-id",
+      "x-account-id",
+      "x-analytics-token",
+      "x-api-key",
+      "x-filename",
+      "x-request-id",
+      "x-tenant-id",
+      "x-toolctrl-tenant-id",
+      "x-toolctrl-user-id",
+      "x-storage-object-status",
+      "webhook-id",
+      "webhook-timestamp",
+      "webhook-signature",
+      "x-webhook-id",
+      "x-webhook-timestamp",
+      "x-webhook-signature",
+    ],
+    allowMethods: ["GET", "POST", "PATCH", "DELETE", "OPTIONS"],
+    credentials: true,
+  })
+);
+
+registerSystemRoutes(app);
+registerAuthRoutes(app);
+registerAccountsRoutes(app);
+registerAnalyticsRoutes(app);
+registerNotificationRoutes(app);
+registerWorkspaceRoutes(app);
+registerMaintenanceRoutes(app);
+registerEntityRowRoutes(app);
+registerRelationshipRoutes(app);
+registerDevicesRoutes(app);
+
+app.get("/v1/usage", (c) => getUsageRequest(c, getApiAppDependencies()));
+app.post("/v1/billing/stripe/webhooks", (c) => postStripeBillingWebhookRequest(c, getApiAppDependencies()));
+
+app.notFound(createNotFoundHandler());
+app.onError(createErrorHandler({ reporter: apiErrorReporter }));
+
+export type ApiApp = typeof app;

hivegrid-cloud/apps/api/src/auth.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, test } from "vitest";
+import { authenticateServiceRequest, extractApiKeyFromRequest, extractBearerToken, isValidServiceToken } from "./auth.js";
+
+describe("api auth primitives", () => {
+  test("extracts bearer tokens with compatible casing and spacing", () => {
+    expect(extractBearerToken(new Request("https://api.test", { headers: { authorization: "Bearer service-token" } }))).toBe("service-token");
+    expect(extractBearerToken(new Request("https://api.test", { headers: { authorization: "bearer service-token" } }))).toBe("service-token");
+    expect(extractBearerToken(new Request("https://api.test", { headers: { authorization: "Basic service-token" } }))).toBeUndefined();
+  });
+
+  test("extracts API keys from JSON body or x-api-key header without using bearer tokens", () => {
+    expect(extractApiKeyFromRequest(new Request("https://api.test"), { apiKey: "json-key" })).toBe("json-key");
+    expect(extractApiKeyFromRequest(new Request("https://api.test", { headers: { "x-api-key": "header-key" } }), undefined)).toBe("header-key");
+    expect(extractApiKeyFromRequest(new Request("https://api.test", { headers: { authorization: "Bearer bearer-key" } }), undefined)).toBeNull();
+  });
+
+  test("validates configured service tokens and creates service principals", () => {
+    const config = { apiAccessToken: "secret" };
+
+    expect(isValidServiceToken("secret", config)).toBe(true);
+    expect(isValidServiceToken("wrong", config)).toBe(false);
+    expect(isValidServiceToken("secret", "secret")).toBe(true);
+    expect(authenticateServiceRequest(new Request("https://api.test", { headers: { authorization: "Bearer secret" } }), config)).toEqual({
+      type: "service-token",
+    });
+    expect(authenticateServiceRequest(new Request("https://api.test", { headers: { "x-api-key": "secret" } }), config)).toBeNull();
+    expect(authenticateServiceRequest(new Request("https://api.test", { headers: { authorization: "Bearer wrong" } }), config)).toBeNull();
+  });
+});

hivegrid-cloud/apps/api/src/auth.ts

@@ -0,0 +1,47 @@
+import { createHash, timingSafeEqual } from "node:crypto";
+import type { ApiConfig } from "@toolctrl/config";
+
+export type ServicePrincipal = {
+  type: "service-token";
+};
+
+export function extractBearerToken(request: Request) {
+  return request.headers.get("authorization")?.match(/^Bearer\s+(.+)$/i)?.[1];
+}
+
+export function extractApiKeyFromRequest(request: Request, body: Record<string, unknown> | undefined) {
+  if (typeof body?.apiKey === "string" && body.apiKey) {
+    return body.apiKey;
+  }
+
+  return request.headers.get("x-api-key");
+}
+
+export function isValidServiceToken(token: string, configOrToken: Pick<ApiConfig, "apiAccessToken"> | string | undefined) {
+  const configuredToken = typeof configOrToken === "string" ? configOrToken : configOrToken?.apiAccessToken;
+  if (!configuredToken || !token) {
+    return false;
+  }
+  // Constant-time comparison: hash both sides to a fixed length first so
+  // timingSafeEqual never throws on length mismatch and the comparison leaks
+  // neither the token contents nor its length.
+  const provided = createHash("sha256").update(token).digest();
+  const expected = createHash("sha256").update(configuredToken).digest();
+  return timingSafeEqual(provided, expected);
+}
+
+export function createServicePrincipal(): ServicePrincipal {
+  return {
+    type: "service-token",
+  };
+}
+
+export function authenticateServiceRequest(request: Request, configOrToken: Pick<ApiConfig, "apiAccessToken"> | string | undefined) {
+  const token = extractBearerToken(request);
+
+  if (!token || !isValidServiceToken(token, configOrToken)) {
+    return null;
+  }
+
+  return createServicePrincipal();
+}

hivegrid-cloud/apps/api/src/auth/cookie/session-cookie.test.ts

@@ -0,0 +1,102 @@
+import { describe, expect, test } from "vitest";
+import type { ApiConfig } from "@toolctrl/config";
+import { serializeApiAuthCookie, serializeExpiredApiAuthCookie } from "./session-cookie.js";
+
+const baseConfig: ApiConfig = {
+  appName: "Test API",
+  nodeEnv: "test",
+  revision: "unknown",
+  listen: { port: 3001 },
+  routing: {
+    appBaseDomain: "localhost:3000",
+    webAppHost: "app.localhost:3000",
+    adminAppHost: "admin.localhost:3000",
+    apiHost: "api.localhost:3000",
+    protocol: "http",
+  },
+  origins: {},
+  apiAccessToken: "test-api-access-token",
+  jwtSecret: "test-jwt-secret",
+  analytics: {},
+  mailjet: {},
+  stripe: {},
+  apiKeys: {
+    allowLegacyPlaintextFallback: true,
+  },
+  storage: {},
+};
+
+describe("serializeApiAuthCookie", () => {
+  test("serializes production API auth cookies with shared hardened attributes", () => {
+    const config = {
+      ...baseConfig,
+      nodeEnv: "production",
+      routing: {
+        appBaseDomain: "example.com",
+        webAppHost: "app.example.com",
+        adminAppHost: "admin.example.com",
+        apiHost: "api.example.com",
+        protocol: "https",
+      },
+    } satisfies ApiConfig;
+    const cookie = serializeApiAuthCookie("signed.jwt.value", config);
+
+    expect(cookie).toContain("RJ_api_session=signed.jwt.value");
+    expect(cookie).toContain("Domain=example.com");
+    expect(cookie).toContain("SameSite=Lax");
+    expect(cookie).toContain("Secure");
+    expect(cookie).toContain("HttpOnly");
+    expect(cookie).toContain("Path=/");
+    expect(cookie).not.toContain("RJ_session=");
+    expect(cookie).not.toContain(config.apiAccessToken);
+    expect(cookie).not.toContain(config.jwtSecret);
+  });
+
+  test("omits Domain and Secure for localhost HTTP development", () => {
+    const cookie = serializeApiAuthCookie("local-token", baseConfig);
+
+    expect(cookie).toContain("RJ_api_session=local-token");
+    expect(cookie).toContain("SameSite=Lax");
+    expect(cookie).toContain("HttpOnly");
+    expect(cookie).toContain("Path=/");
+    expect(cookie).not.toContain("Domain=");
+    expect(cookie).not.toContain("Secure");
+    expect(cookie).not.toContain(baseConfig.apiAccessToken);
+    expect(cookie).not.toContain(baseConfig.jwtSecret);
+  });
+
+  test("URL-encodes token values without serializing config secrets", () => {
+    const cookie = serializeApiAuthCookie("token with spaces;and=symbols", baseConfig);
+
+    expect(cookie).toContain("RJ_api_session=token%20with%20spaces%3Band%3Dsymbols");
+    expect(cookie).not.toContain("token with spaces;and=symbols");
+    expect(cookie).not.toContain(baseConfig.apiAccessToken);
+    expect(cookie).not.toContain(baseConfig.jwtSecret);
+  });
+
+  test("serializes expired API auth cookies with hardened shared attributes", () => {
+    const config = {
+      ...baseConfig,
+      nodeEnv: "production",
+      routing: {
+        appBaseDomain: "example.com",
+        webAppHost: "app.example.com",
+        adminAppHost: "admin.example.com",
+        apiHost: "api.example.com",
+        protocol: "https",
+      },
+    } satisfies ApiConfig;
+    const cookie = serializeExpiredApiAuthCookie(config);
+
+    expect(cookie).toContain("RJ_api_session=");
+    expect(cookie).toContain("Max-Age=0");
+    expect(cookie).toContain("Expires=Thu, 01 Jan 1970 00:00:00 GMT");
+    expect(cookie).toContain("Domain=example.com");
+    expect(cookie).toContain("SameSite=Lax");
+    expect(cookie).toContain("Secure");
+    expect(cookie).toContain("HttpOnly");
+    expect(cookie).toContain("Path=/");
+    expect(cookie).not.toContain(config.apiAccessToken);
+    expect(cookie).not.toContain(config.jwtSecret);
+  });
+});

hivegrid-cloud/apps/api/src/auth/cookie/session-cookie.ts

@@ -0,0 +1,27 @@
+import { resolveApiSessionCookiePolicy, type ApiConfig } from "@toolctrl/config";
+
+function serializeApiAuthCookieWithValue(value: string, config: ApiConfig, extraAttributes: string[] = []) {
+  const policy = resolveApiSessionCookiePolicy({
+    nodeEnv: config.nodeEnv,
+    routing: config.routing,
+  });
+  const attributes = [
+    `${policy.name}=${value}`,
+    `Path=${policy.path}`,
+    policy.domain ? `Domain=${policy.domain}` : undefined,
+    "SameSite=Lax",
+    policy.secure ? "Secure" : undefined,
+    policy.httpOnly ? "HttpOnly" : undefined,
+    ...extraAttributes,
+  ].filter((attribute): attribute is string => Boolean(attribute));
+
+  return attributes.join("; ");
+}
+
+export function serializeApiAuthCookie(token: string, config: ApiConfig) {
+  return serializeApiAuthCookieWithValue(encodeURIComponent(token), config);
+}
+
+export function serializeExpiredApiAuthCookie(config: ApiConfig) {
+  return serializeApiAuthCookieWithValue("", config, ["Max-Age=0", "Expires=Thu, 01 Jan 1970 00:00:00 GMT"]);
+}

hivegrid-cloud/apps/api/src/auth/login-rate-limit.test.ts

@@ -0,0 +1,158 @@
+import { afterEach, beforeEach, describe, expect, test } from "vitest";
+import { createSharedStore } from "@toolctrl/shared-store";
+
+import {
+  LOGIN_RATE_LIMIT_ACCOUNT_ATTEMPTS,
+  LOGIN_RATE_LIMIT_IP_ATTEMPTS,
+  checkLoginRateLimit,
+  createLoginRateLimitAccountFingerprint,
+  deriveLoginRateLimitClientIp,
+  recordFailedLoginAttempt,
+  recordSuccessfulLogin,
+  resetLoginRateLimiterForTests,
+  __resetLoginRateLimitStoreForTests,
+  __setLoginRateLimitStoreForTests,
+} from "./login-rate-limit.js";
+
+function request(account: string | undefined = "User@Example.com", ip: string | undefined = "203.0.113.10") {
+  return {
+    account,
+    headers: ip ? { "x-forwarded-for": ip } : {},
+  };
+}
+
+async function failAttempts(count: number, input = request()) {
+  for (let index = 0; index < count; index += 1) {
+    await recordFailedLoginAttempt(input);
+  }
+}
+
+beforeEach(() => {
+  // Fresh isolated memory store per test so keys never bleed across cases.
+  __setLoginRateLimitStoreForTests(createSharedStore({ namespace: "test-login-rate-limit" }));
+});
+
+afterEach(async () => {
+  await resetLoginRateLimiterForTests();
+  __resetLoginRateLimitStoreForTests();
+});
+
+describe("login rate limiter (shared-store)", () => {
+  test("allows through the last permitted attempt and blocks after the IP threshold", async () => {
+    for (let index = 0; index < LOGIN_RATE_LIMIT_IP_ATTEMPTS - 1; index += 1) {
+      await recordFailedLoginAttempt(request(`user-${index}@example.com`, "198.51.100.10"));
+    }
+    expect(await checkLoginRateLimit(request("next@example.com", "198.51.100.10"))).toMatchObject({ allowed: true });
+
+    await recordFailedLoginAttempt(request("last@example.com", "198.51.100.10"));
+
+    expect(await checkLoginRateLimit(request("blocked@example.com", "198.51.100.10"))).toMatchObject({
+      allowed: false,
+      scope: "ip",
+      clientIp: "198.51.100.10",
+      accountFingerprint: createLoginRateLimitAccountFingerprint("blocked@example.com"),
+    });
+  });
+
+  test("tracks account attempts across different IP buckets", async () => {
+    for (let index = 0; index < LOGIN_RATE_LIMIT_ACCOUNT_ATTEMPTS; index += 1) {
+      await recordFailedLoginAttempt(request("Shared@Example.com", `203.0.113.${index + 1}`));
+    }
+
+    expect(await checkLoginRateLimit(request(" shared@example.com ", "203.0.113.99"))).toMatchObject({
+      allowed: false,
+      scope: "account",
+      clientIp: "203.0.113.99",
+      accountFingerprint: createLoginRateLimitAccountFingerprint("shared@example.com"),
+    });
+
+    expect(await checkLoginRateLimit(request("other@example.com", "203.0.113.99"))).toMatchObject({ allowed: true });
+  });
+
+  test("keeps independent IP buckets for multiple clients", async () => {
+    await failAttempts(LOGIN_RATE_LIMIT_IP_ATTEMPTS, request("one@example.com", "203.0.113.1"));
+
+    expect(await checkLoginRateLimit(request("fresh@example.com", "203.0.113.1"))).toMatchObject({ allowed: false, scope: "ip" });
+    expect(await checkLoginRateLimit(request("fresh@example.com", "203.0.113.2"))).toMatchObject({ allowed: true });
+  });
+
+  test("uses the first x-forwarded-for segment before x-real-ip and remote fallback", () => {
+    expect(
+      deriveLoginRateLimitClientIp({
+        headers: new Headers({ "x-forwarded-for": " 198.51.100.7, 10.0.0.1 ", "x-real-ip": "198.51.100.8" }),
+        remoteAddress: "198.51.100.9",
+      }),
+    ).toBe("198.51.100.7");
+    expect(deriveLoginRateLimitClientIp({ headers: { "x-real-ip": "198.51.100.8" }, remoteAddress: "198.51.100.9" })).toBe("198.51.100.8");
+    expect(deriveLoginRateLimitClientIp({ remoteAddress: "198.51.100.9" })).toBe("198.51.100.9");
+    expect(deriveLoginRateLimitClientIp({})).toBe("unknown");
+  });
+
+  test("resets only the normalized account bucket after successful login", async () => {
+    await failAttempts(LOGIN_RATE_LIMIT_ACCOUNT_ATTEMPTS, request("Reset@Example.com", "203.0.113.11"));
+    await failAttempts(LOGIN_RATE_LIMIT_IP_ATTEMPTS, request("other@example.com", "203.0.113.12"));
+
+    expect(await checkLoginRateLimit(request("reset@example.com", "203.0.113.13"))).toMatchObject({ allowed: false, scope: "account" });
+    await recordSuccessfulLogin({ account: " reset@example.com " });
+
+    expect(await checkLoginRateLimit(request("reset@example.com", "203.0.113.13"))).toMatchObject({ allowed: true });
+    expect(await checkLoginRateLimit(request("fresh@example.com", "203.0.113.12"))).toMatchObject({ allowed: false, scope: "ip" });
+  });
+
+  test("normalizes malformed or missing accounts into one bounded account bucket", async () => {
+    for (const account of [undefined, null, "", "   ", 123]) {
+      await recordFailedLoginAttempt({ ...request(undefined, "203.0.113.30"), account });
+    }
+
+    expect(await checkLoginRateLimit({ ...request(undefined, "203.0.113.31"), account: undefined })).toMatchObject({
+      allowed: false,
+      scope: "account",
+      accountFingerprint: createLoginRateLimitAccountFingerprint(undefined),
+    });
+  });
+
+  test("public metadata exposes only stable fingerprints and no raw credentials", async () => {
+    const rawEmail = "Sensitive.User+token@example.com";
+    const rawPassword = "correct horse battery staple";
+    await failAttempts(LOGIN_RATE_LIMIT_ACCOUNT_ATTEMPTS, request(rawEmail, "203.0.113.40"));
+
+    const result = await checkLoginRateLimit({ ...request(rawEmail, "203.0.113.41"), password: rawPassword } as never);
+    const serialized = JSON.stringify(result);
+
+    expect(result).toMatchObject({ allowed: false, scope: "account" });
+    expect(serialized).toContain("acct_");
+    expect(serialized).not.toContain(rawEmail);
+    expect(serialized).not.toContain(rawEmail.toLowerCase());
+    expect(serialized).not.toContain(rawPassword);
+  });
+
+  test("uses the redis adapter when REDIS_URL is set", () => {
+    // Verify the store wiring picks the RedisAdapter when REDIS_URL is
+    // provided. We don't actually open a Redis connection — we inject a
+    // pre-built store with the redisUrl so it goes through buildAdapter.
+    const originalRedisUrl = process.env.REDIS_URL;
+    try {
+      process.env.REDIS_URL = "redis://does-not-exist:6379";
+      // Building the store should select the RedisAdapter without throwing.
+      const store = createSharedStore({ namespace: "redis-selection-test", redisUrl: process.env.REDIS_URL });
+      expect(store.adapter.constructor.name).toBe("RedisAdapter");
+      // Best-effort close — the adapter never actually connects until the
+      // first command, so this is cheap.
+      void store.close();
+    } finally {
+      if (originalRedisUrl === undefined) delete process.env.REDIS_URL;
+      else process.env.REDIS_URL = originalRedisUrl;
+    }
+  });
+
+  test("uses the memory adapter when REDIS_URL is unset", () => {
+    const originalRedisUrl = process.env.REDIS_URL;
+    delete process.env.REDIS_URL;
+    try {
+      const store = createSharedStore({ namespace: "memory-selection-test" });
+      expect(store.adapter.constructor.name).toBe("MemoryAdapter");
+    } finally {
+      if (originalRedisUrl !== undefined) process.env.REDIS_URL = originalRedisUrl;
+    }
+  });
+});

hivegrid-cloud/apps/api/src/auth/login-rate-limit.ts

@@ -0,0 +1,235 @@
+import { createFixedWindowRateLimiter, type FixedWindowRateLimiter, type SharedStore } from "@toolctrl/shared-store";
+import { getSharedStore } from "../shared-store.js";
+
+export const LOGIN_RATE_LIMIT_WINDOW_MS = 60_000;
+export const LOGIN_RATE_LIMIT_IP_ATTEMPTS = 10;
+export const LOGIN_RATE_LIMIT_ACCOUNT_ATTEMPTS = 5;
+
+const LOGIN_RATE_LIMIT_WINDOW_SECONDS = Math.ceil(LOGIN_RATE_LIMIT_WINDOW_MS / 1000);
+const UNKNOWN_CLIENT_IP = "unknown";
+const UNKNOWN_ACCOUNT_KEY = "unknown-account";
+const FINGERPRINT_PREFIX = "acct_";
+
+export type LoginRateLimitScope = "ip" | "account";
+
+export type LoginRateLimitRequest = {
+  headers?: Headers | Record<string, string | string[] | undefined>;
+  remoteAddress?: string | null;
+  account?: unknown;
+};
+
+export type LoginRateLimitAllowed = {
+  allowed: true;
+  clientIp: string;
+  accountFingerprint: string;
+};
+
+export type LoginRateLimitBlocked = {
+  allowed: false;
+  scope: LoginRateLimitScope;
+  retryAfterSeconds: number;
+  clientIp: string;
+  accountFingerprint: string;
+};
+
+export type LoginRateLimitResult = LoginRateLimitAllowed | LoginRateLimitBlocked;
+
+/**
+ * Lazy holder around the two fixed-window limiters. Built on first use
+ * so tests can swap the SharedStore via `__setLoginRateLimitStoreForTests`
+ * before any limiter is constructed.
+ */
+type Limiters = {
+  ip: FixedWindowRateLimiter;
+  account: FixedWindowRateLimiter;
+  store: SharedStore;
+};
+
+let limiters: Limiters | null = null;
+let storeFactory: () => SharedStore = () => getSharedStore();
+
+function getLimiters(): Limiters {
+  if (limiters !== null) {
+    return limiters;
+  }
+  const store = storeFactory();
+  limiters = {
+    store,
+    ip: createFixedWindowRateLimiter({
+      store,
+      prefix: "login-rate-limit:ip",
+      windowSeconds: LOGIN_RATE_LIMIT_WINDOW_SECONDS,
+      max: LOGIN_RATE_LIMIT_IP_ATTEMPTS,
+    }),
+    account: createFixedWindowRateLimiter({
+      store,
+      prefix: "login-rate-limit:account",
+      windowSeconds: LOGIN_RATE_LIMIT_WINDOW_SECONDS,
+      max: LOGIN_RATE_LIMIT_ACCOUNT_ATTEMPTS,
+    }),
+  };
+  return limiters;
+}
+
+export async function checkLoginRateLimit(request: LoginRateLimitRequest): Promise<LoginRateLimitResult> {
+  const clientIp = deriveLoginRateLimitClientIp(request);
+  const accountKey = normalizeLoginRateLimitAccount(request.account);
+  const accountFingerprint = createLoginRateLimitAccountFingerprint(accountKey);
+  const { ip, account, store } = getLimiters();
+
+  const ipState = await readCounter(store, "login-rate-limit:ip", clientIp);
+  if (ipState.count >= LOGIN_RATE_LIMIT_IP_ATTEMPTS) {
+    return {
+      allowed: false,
+      scope: "ip",
+      retryAfterSeconds: Math.max(1, ipState.ttlSeconds),
+      clientIp,
+      accountFingerprint,
+    };
+  }
+
+  const accountState = await readCounter(store, "login-rate-limit:account", accountKey);
+  if (accountState.count >= LOGIN_RATE_LIMIT_ACCOUNT_ATTEMPTS) {
+    return {
+      allowed: false,
+      scope: "account",
+      retryAfterSeconds: Math.max(1, accountState.ttlSeconds),
+      clientIp,
+      accountFingerprint,
+    };
+  }
+
+  // Silence unused-var: the limiter objects are kept on the cached `Limiters`
+  // because `recordFailedLoginAttempt` consumes them on the write path.
+  void ip;
+  void account;
+
+  return {
+    allowed: true,
+    clientIp,
+    accountFingerprint,
+  };
+}
+
+export async function recordFailedLoginAttempt(request: LoginRateLimitRequest): Promise<void> {
+  const { ip, account } = getLimiters();
+  const clientIp = deriveLoginRateLimitClientIp(request);
+  const accountKey = normalizeLoginRateLimitAccount(request.account);
+  await Promise.all([ip.consume(clientIp), account.consume(accountKey)]);
+}
+
+export async function recordSuccessfulLogin(request: Pick<LoginRateLimitRequest, "account">): Promise<void> {
+  const { account } = getLimiters();
+  await account.reset(normalizeLoginRateLimitAccount(request.account));
+}
+
+export async function resetLoginRateLimiterForTests(): Promise<void> {
+  // Drop any cached limiter pair and best-effort drain any keys the
+  // previous limiters wrote to the shared-store. Tests that need full
+  // isolation can also inject a fresh store via
+  // `__setLoginRateLimitStoreForTests`.
+  const previous = limiters;
+  limiters = null;
+  if (previous) {
+    try {
+      for await (const fullKey of previous.store.adapter.scan(`shared-store:${previous.store.namespace}:rl:fixed:login-rate-limit:*`)) {
+        // adapter keys are absolute (namespace already applied); strip the
+        // namespace prefix when calling store.del which re-applies it.
+        const prefix = `shared-store:${previous.store.namespace}:`;
+        if (fullKey.startsWith(prefix)) {
+          await previous.store.del(fullKey.slice(prefix.length));
+        }
+      }
+    } catch {
+      // Best-effort; isolated tests use the store seam instead.
+    }
+  }
+}
+
+/** Test seam: inject a SharedStore that backs the limiters. */
+export function __setLoginRateLimitStoreForTests(store: SharedStore): void {
+  limiters = null;
+  storeFactory = () => store;
+}
+
+/** Test seam: revert to the default factory (real shared-store singleton). */
+export function __resetLoginRateLimitStoreForTests(): void {
+  limiters = null;
+  storeFactory = () => getSharedStore();
+}
+
+export function deriveLoginRateLimitClientIp(request: Pick<LoginRateLimitRequest, "headers" | "remoteAddress">): string {
+  const forwardedFor = readHeader(request.headers, "x-forwarded-for");
+  const forwardedClient = forwardedFor?.split(",")[0]?.trim();
+  if (forwardedClient) {
+    return forwardedClient;
+  }
+
+  const realIp = readHeader(request.headers, "x-real-ip")?.trim();
+  if (realIp) {
+    return realIp;
+  }
+
+  const remoteAddress = request.remoteAddress?.trim();
+  return remoteAddress || UNKNOWN_CLIENT_IP;
+}
+
+export function createLoginRateLimitAccountFingerprint(account: unknown): string {
+  const normalized = normalizeLoginRateLimitAccount(account);
+  let hash = 0x811c9dc5;
+
+  for (let index = 0; index < normalized.length; index += 1) {
+    hash ^= normalized.charCodeAt(index);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+
+  return `${FINGERPRINT_PREFIX}${hash.toString(16).padStart(8, "0")}`;
+}
+
+function normalizeLoginRateLimitAccount(account: unknown): string {
+  if (typeof account !== "string") {
+    return UNKNOWN_ACCOUNT_KEY;
+  }
+
+  const normalized = account.trim().toLowerCase();
+  return normalized || UNKNOWN_ACCOUNT_KEY;
+}
+
+/**
+ * Peek at the current counter and remaining TTL without incrementing.
+ * The shared-store fixed-window helper only exposes `consume()` (which
+ * always increments) and `reset()`, so we read the underlying key
+ * directly to keep `checkLoginRateLimit` side-effect-free.
+ */
+async function readCounter(store: SharedStore, prefix: string, key: string): Promise<{ count: number; ttlSeconds: number }> {
+  const suffix = `rl:fixed:${prefix}:${key}`;
+  const raw = await store.get(suffix);
+  const count = raw === null ? 0 : Number.parseInt(raw, 10) || 0;
+  // The memory adapter does not expose a `ttl` helper, so derive a
+  // best-effort retry window from the configured window length. Redis
+  // counter TTL is enforced by the limiter on `consume`, so a fresh
+  // request entering the window gets the full window-seconds back.
+  const ttlSeconds = count > 0 ? LOGIN_RATE_LIMIT_WINDOW_SECONDS : 0;
+  return { count, ttlSeconds };
+}
+
+function readHeader(headers: LoginRateLimitRequest["headers"], name: string): string | undefined {
+  if (!headers) {
+    return undefined;
+  }
+
+  if (headers instanceof Headers) {
+    return headers.get(name) ?? undefined;
+  }
+
+  const lowerName = name.toLowerCase();
+  for (const [headerName, value] of Object.entries(headers)) {
+    if (headerName.toLowerCase() !== lowerName) {
+      continue;
+    }
+
+    return Array.isArray(value) ? value[0] : value;
+  }
+
+  return undefined;
+}

hivegrid-cloud/apps/api/src/auth/permission-wiring.test.ts

@@ -0,0 +1,43 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, resolve } from "node:path";
+
+import { describe, expect, test } from "vitest";
+
+const appSourcePath = resolve(dirname(fileURLToPath(import.meta.url)), "../app.ts");
+const entityRowRoutesSourcePath = resolve(dirname(fileURLToPath(import.meta.url)), "../routes/entity-row-routes.ts");
+const relationshipRoutesSourcePath = resolve(dirname(fileURLToPath(import.meta.url)), "../routes/relationship-routes.ts");
+
+function read(path: string) {
+  return readFileSync(path, "utf8");
+}
+
+describe("API permission helper wiring", () => {
+  // The legacy v1 row + relationship routes were extracted out of
+  // `app.ts` into dedicated route modules. The assertions here pin the
+  // shared `authorizeApiEntityRowScope` / `authorizeApiRelationshipScope`
+  // helpers to those extraction sites so a regression that re-inlines
+  // permission checks in `app.ts` (or a different file) fails CI.
+  test("entity-row routes call the shared API permission helper", () => {
+    const src = read(entityRowRoutesSourcePath);
+    expect(src).toContain('import { authorizeApiEntityRowScope } from "../auth/permission.js";');
+    expect(src.match(/authorizeApiEntityRowScope\(/g)).toHaveLength(5);
+  });
+
+  test("relationship routes call the shared API permission helper", () => {
+    const src = read(relationshipRoutesSourcePath);
+    expect(src).toContain('import { authorizeApiRelationshipScope } from "../auth/permission.js";');
+    expect(src.match(/authorizeApiRelationshipScope\(/g)).toHaveLength(3);
+  });
+
+  test("app.ts does not define or call stale local entity-row scope helpers", () => {
+    const appSource = read(appSourcePath);
+
+    expect(appSource).not.toMatch(/function\s+authorizeEntityRowScope\b/);
+    expect(appSource).not.toMatch(/const\s+authorizeEntityRowScope\b/);
+    expect(appSource).not.toMatch(/=>\s*authorizeEntityRowScope\(/);
+    expect(appSource).not.toMatch(/\bauthorizeEntityRowScope\(/);
+    expect(appSource).not.toMatch(/\bgetEntityPermission\b/);
+    expect(appSource).not.toMatch(/\bverifyUserHasPermission\b/);
+  });
+});

hivegrid-cloud/apps/api/src/auth/permission.test.ts

@@ -0,0 +1,187 @@
+import type { LegacyApiKeySimple } from "@toolctrl/db/api-keys.server";
+import type { LegacyEntity } from "@toolctrl/db/entities.server";
+import { describe, expect, test } from "vitest";
+
+import { createServicePrincipal } from "../auth.js";
+import { authorizeApiEntityRowScope, authorizeApiRelationshipScope } from "./permission.js";
+import { createStoredApiKeyPrincipal, type ApiScopeResult } from "./scope.js";
+
+const NOW = new Date("2026-01-15T12:00:00.000Z");
+
+function buildEntity(overrides: Partial<LegacyEntity> = {}): LegacyEntity {
+  return {
+    id: "entity-1",
+    name: "contacts",
+    description: null,
+    url: null,
+    multiTenant: true,
+    isPublic: false,
+    useAsUser: false,
+    createdById: null,
+    createdAt: new Date("2026-01-01T00:00:00.000Z"),
+    updatedAt: new Date("2026-01-01T00:00:00.000Z"),
+    ...overrides,
+  };
+}
+
+function buildApiKey(overrides: Partial<LegacyApiKeySimple> = {}): LegacyApiKeySimple {
+  return {
+    id: "api-key-1",
+    alias: "Production key",
+    expires: new Date("2026-02-01T00:00:00.000Z"),
+    tenantId: "tenant-1",
+    active: true,
+    entities: [
+      {
+        entityId: "entity-1",
+        create: true,
+        read: true,
+        update: true,
+        delete: false,
+      },
+      {
+        entityId: "child-entity",
+        create: true,
+        read: true,
+        update: false,
+        delete: false,
+      },
+    ],
+    tenant: {
+      id: "tenant-1",
+      name: "Tenant One",
+      slug: "tenant-one",
+      deactivatedReason: null,
+    },
+    ...overrides,
+  };
+}
+
+function expectRedactedDeny(result: ApiScopeResult, reason: string) {
+  expect(result).toEqual({
+    allowed: false,
+    principalType: "api-key",
+    reason,
+    code: "FORBIDDEN_API_KEY_SCOPE",
+  });
+  const serialized = JSON.stringify(result);
+  expect(serialized).not.toContain("plaintext-secret");
+  expect(serialized).not.toContain("hash-value");
+  expect(serialized).not.toContain("service-token-value");
+}
+
+describe("authorizeApiEntityRowScope", () => {
+  test("allows entity row operations when the stored API key has the entity operation scope", () => {
+    const result = authorizeApiEntityRowScope(createStoredApiKeyPrincipal(buildApiKey()), {
+      tenantId: "tenant-1",
+      entity: buildEntity(),
+      operation: "read",
+      now: NOW,
+    });
+
+    expect(result).toEqual({
+      allowed: true,
+      principalType: "api-key",
+    });
+  });
+
+  test("denies entity row operations when the stored API key lacks the entity operation scope", () => {
+    const result = authorizeApiEntityRowScope(createStoredApiKeyPrincipal(buildApiKey()), {
+      tenantId: "tenant-1",
+      entity: buildEntity(),
+      operation: "delete",
+      now: NOW,
+    });
+
+    expectRedactedDeny(result, "operation_permission_missing");
+  });
+
+  test("fails closed for malformed legacy entities without an id", () => {
+    expect(() =>
+      authorizeApiEntityRowScope(createStoredApiKeyPrincipal(buildApiKey()), {
+        tenantId: "tenant-1",
+        entity: buildEntity({ id: "" }),
+        operation: "read",
+        now: NOW,
+      }),
+    ).toThrow("Entity is missing an id.");
+  });
+
+  test("denies stored API keys when tenant scope cannot be resolved", () => {
+    const result = authorizeApiEntityRowScope(createStoredApiKeyPrincipal(buildApiKey()), {
+      tenantId: null,
+      entity: buildEntity(),
+      operation: "read",
+      now: NOW,
+    });
+
+    expectRedactedDeny(result, "tenant_mismatch");
+  });
+});
+
+describe("authorizeApiRelationshipScope", () => {
+  test("allows service-token principals for relationship operations without entity scopes", () => {
+    const result = authorizeApiRelationshipScope(createServicePrincipal(), {
+      tenantId: "tenant-1",
+      parentEntityId: "missing-parent",
+      childEntityId: "missing-child",
+      operation: "delete",
+    });
+
+    expect(result).toEqual({
+      allowed: true,
+      principalType: "service-token",
+    });
+  });
+
+  test("denies relationship operations when the parent entity lacks the requested operation", () => {
+    const result = authorizeApiRelationshipScope(createStoredApiKeyPrincipal(buildApiKey()), {
+      tenantId: "tenant-1",
+      parentEntityId: "missing-parent",
+      childEntityId: "child-entity",
+      operation: "read",
+      now: NOW,
+    });
+
+    expectRedactedDeny(result, "entity_permission_missing");
+  });
+
+  test("denies relationship operations when the child entity lacks the requested operation", () => {
+    const result = authorizeApiRelationshipScope(createStoredApiKeyPrincipal(buildApiKey()), {
+      tenantId: "tenant-1",
+      parentEntityId: "entity-1",
+      childEntityId: "child-entity",
+      operation: "update",
+      now: NOW,
+    });
+
+    expectRedactedDeny(result, "operation_permission_missing");
+  });
+
+  test("allows relationship operations when parent and child entities both include the requested operation", () => {
+    const result = authorizeApiRelationshipScope(createStoredApiKeyPrincipal(buildApiKey()), {
+      tenantId: "tenant-1",
+      parentEntityId: "entity-1",
+      childEntityId: "child-entity",
+      operation: "create",
+      now: NOW,
+    });
+
+    expect(result).toEqual({
+      allowed: true,
+      principalType: "api-key",
+    });
+  });
+
+  test("denies stored API keys for relationship operations when tenant scope cannot be resolved", () => {
+    const result = authorizeApiRelationshipScope(createStoredApiKeyPrincipal(buildApiKey()), {
+      tenantId: null,
+      parentEntityId: "entity-1",
+      childEntityId: "child-entity",
+      operation: "read",
+      now: NOW,
+    });
+
+    expectRedactedDeny(result, "tenant_mismatch");
+  });
+});

hivegrid-cloud/apps/api/src/auth/permission.ts

@@ -0,0 +1,54 @@
+import type { LegacyEntity } from "@toolctrl/db/entities.server";
+
+import { authorizeApiScope, type ApiKeyScopeOperation, type ApiScopedPrincipal, type ApiScopeResult } from "./scope.js";
+
+export type ApiEntityRowScopeRequest = {
+  tenantId: string | null;
+  entity: LegacyEntity;
+  operation: ApiKeyScopeOperation;
+  now?: Date;
+};
+
+export type ApiRelationshipScopeRequest = {
+  tenantId: string | null;
+  parentEntityId: string;
+  childEntityId: string;
+  operation: ApiKeyScopeOperation;
+  now?: Date;
+};
+
+export function authorizeApiEntityRowScope(principal: ApiScopedPrincipal, request: ApiEntityRowScopeRequest): ApiScopeResult {
+  return authorizeApiScope(principal, {
+    tenantId: request.tenantId ?? "",
+    entityId: getLegacyEntityId(request.entity),
+    operation: request.operation,
+    now: request.now,
+  });
+}
+
+export function authorizeApiRelationshipScope(principal: ApiScopedPrincipal, request: ApiRelationshipScopeRequest): ApiScopeResult {
+  const parentScope = authorizeApiScope(principal, {
+    tenantId: request.tenantId ?? "",
+    entityId: request.parentEntityId,
+    operation: request.operation,
+    now: request.now,
+  });
+
+  if (!parentScope.allowed) {
+    return parentScope;
+  }
+
+  return authorizeApiScope(principal, {
+    tenantId: request.tenantId ?? "",
+    entityId: request.childEntityId,
+    operation: request.operation,
+    now: request.now,
+  });
+}
+
+function getLegacyEntityId(entity: LegacyEntity) {
+  if (entity && typeof entity === "object" && "id" in entity && typeof entity.id === "string" && entity.id) {
+    return entity.id;
+  }
+  throw new Error("Entity is missing an id.");
+}

hivegrid-cloud/apps/api/src/auth/scope.test.ts

@@ -0,0 +1,130 @@
+import type { LegacyApiKeySimple } from "@toolctrl/db/api-keys.server";
+import { describe, expect, test } from "vitest";
+
+import { createServicePrincipal } from "../auth.js";
+import { authorizeApiScope, createStoredApiKeyPrincipal, type ApiScopeResult } from "./scope.js";
+
+const NOW = new Date("2026-01-15T12:00:00.000Z");
+
+function buildApiKey(overrides: Partial<LegacyApiKeySimple> = {}): LegacyApiKeySimple {
+  return {
+    id: "api-key-1",
+    alias: "Production key",
+    expires: new Date("2026-02-01T00:00:00.000Z"),
+    tenantId: "tenant-1",
+    active: true,
+    entities: [
+      {
+        entityId: "entity-1",
+        create: true,
+        read: true,
+        update: true,
+        delete: false,
+      },
+    ],
+    tenant: {
+      id: "tenant-1",
+      name: "Tenant One",
+      slug: "tenant-one",
+      deactivatedReason: null,
+    },
+    ...overrides,
+  };
+}
+
+function authorize(apiKey: LegacyApiKeySimple, overrides: Partial<Parameters<typeof authorizeApiScope>[1]> = {}): ApiScopeResult {
+  return authorizeApiScope(createStoredApiKeyPrincipal(apiKey), {
+    tenantId: "tenant-1",
+    entityId: "entity-1",
+    operation: "read",
+    now: NOW,
+    ...overrides,
+  });
+}
+
+function expectRedactedDeny(result: ApiScopeResult, reason: string) {
+  expect(result).toEqual({
+    allowed: false,
+    principalType: "api-key",
+    reason,
+    code: "FORBIDDEN_API_KEY_SCOPE",
+  });
+  const serialized = JSON.stringify(result);
+  expect(serialized).not.toContain("plaintext-secret");
+  expect(serialized).not.toContain("hash-value");
+  expect(serialized).not.toContain("service-token");
+}
+
+describe("authorizeApiScope", () => {
+  test("allows service-token principals for every scoped operation", () => {
+    const principal = createServicePrincipal();
+
+    expect(
+      authorizeApiScope(principal, {
+        tenantId: "other-tenant",
+        entityId: "unknown-entity",
+        operation: "delete",
+        now: NOW,
+      }),
+    ).toEqual({
+      allowed: true,
+      principalType: "service-token",
+    });
+  });
+
+  test("allows active stored API keys with matching tenant, entity, and CRUD permission", () => {
+    expect(authorize(buildApiKey(), { operation: "create" })).toEqual({
+      allowed: true,
+      principalType: "api-key",
+    });
+  });
+
+  test("denies inactive stored API keys", () => {
+    expectRedactedDeny(authorize(buildApiKey({ active: false })), "api_key_inactive");
+  });
+
+  test("denies expired stored API keys", () => {
+    expectRedactedDeny(authorize(buildApiKey({ expires: new Date("2026-01-01T00:00:00.000Z") })), "api_key_expired");
+  });
+
+  test("denies stored API keys for deactivated tenants", () => {
+    expectRedactedDeny(
+      authorize(
+        buildApiKey({
+          tenant: {
+            id: "tenant-1",
+            name: "Tenant One",
+            slug: "tenant-one",
+            deactivatedReason: "billing_issue",
+          },
+        }),
+      ),
+      "tenant_deactivated",
+    );
+  });
+
+  test("denies stored API keys whose tenant does not match the request tenant", () => {
+    expectRedactedDeny(authorize(buildApiKey(), { tenantId: "tenant-2" }), "tenant_mismatch");
+    expectRedactedDeny(
+      authorize(
+        buildApiKey({
+          tenant: {
+            id: "tenant-2",
+            name: "Tenant Two",
+            slug: "tenant-two",
+            deactivatedReason: null,
+          },
+        }),
+      ),
+      "tenant_mismatch",
+    );
+  });
+
+  test("denies stored API keys missing the requested entity permission", () => {
+    expectRedactedDeny(authorize(buildApiKey(), { entityId: "entity-2" }), "entity_permission_missing");
+  });
+
+  test("denies stored API keys missing the requested CRUD operation permission", () => {
+    expectRedactedDeny(authorize(buildApiKey(), { operation: "delete" }), "operation_permission_missing");
+  });
+});

hivegrid-cloud/apps/api/src/auth/scope.ts

@@ -0,0 +1,139 @@
+import type { LegacyApiKeySimple } from "@toolctrl/db/api-keys.server";
+
+import type { ServicePrincipal } from "../auth.js";
+
+export const API_KEY_SCOPE_OPERATIONS = ["read", "create", "update", "delete"] as const;
+
+export type ApiKeyScopeOperation = (typeof API_KEY_SCOPE_OPERATIONS)[number];
+
+export type StoredApiKeyPrincipal = {
+  type: "api-key";
+  apiKey: LegacyApiKeySimple;
+};
+
+export type ApiScopedPrincipal = ServicePrincipal | StoredApiKeyPrincipal;
+
+export type ApiScopeRequest = {
+  tenantId: string;
+  entityId: string;
+  operation: ApiKeyScopeOperation;
+  now?: Date;
+};
+
+export type ApiScopeDenyReason =
+  | "api_key_inactive"
+  | "api_key_expired"
+  | "tenant_deactivated"
+  | "tenant_mismatch"
+  | "entity_permission_missing"
+  | "operation_permission_missing";
+
+export type ApiScopeAllowed = {
+  allowed: true;
+  principalType: ApiScopedPrincipal["type"];
+};
+
+export type ApiTenantScopeAllowed = ApiScopeAllowed & {
+  tenantId: string | null;
+};
+
+export type ApiScopeDenied = {
+  allowed: false;
+  principalType: ApiScopedPrincipal["type"];
+  reason: ApiScopeDenyReason;
+  code: "FORBIDDEN_API_KEY_SCOPE";
+};
+
+export type ApiScopeResult = ApiScopeAllowed | ApiScopeDenied;
+
+function deny(principalType: ApiScopedPrincipal["type"], reason: ApiScopeDenyReason): ApiScopeDenied {
+  return {
+    allowed: false,
+    principalType,
+    reason,
+    code: "FORBIDDEN_API_KEY_SCOPE",
+  };
+}
+
+export function createStoredApiKeyPrincipal(apiKey: LegacyApiKeySimple): StoredApiKeyPrincipal {
+  return {
+    type: "api-key",
+    apiKey,
+  };
+}
+
+export function authorizeApiScope(principal: ApiScopedPrincipal, request: ApiScopeRequest): ApiScopeResult {
+  if (principal.type === "service-token") {
+    return {
+      allowed: true,
+      principalType: principal.type,
+    };
+  }
+
+  const { apiKey } = principal;
+
+  if (!apiKey.active) {
+    return deny(principal.type, "api_key_inactive");
+  }
+
+  if (apiKey.expires && apiKey.expires.getTime() <= (request.now ?? new Date()).getTime()) {
+    return deny(principal.type, "api_key_expired");
+  }
+
+  if (apiKey.tenant.deactivatedReason) {
+    return deny(principal.type, "tenant_deactivated");
+  }
+
+  if (apiKey.tenantId !== request.tenantId || apiKey.tenant.id !== request.tenantId) {
+    return deny(principal.type, "tenant_mismatch");
+  }
+
+  const entityPermission = apiKey.entities.find((entity) => entity.entityId === request.entityId);
+
+  if (!entityPermission) {
+    return deny(principal.type, "entity_permission_missing");
+  }
+
+  if (!entityPermission[request.operation]) {
+    return deny(principal.type, "operation_permission_missing");
+  }
+
+  return {
+    allowed: true,
+    principalType: principal.type,
+  };
+}
+
+export function resolveApiTenantScope(principal: ApiScopedPrincipal, requestedTenantId: string | null, now = new Date()): ApiTenantScopeAllowed | ApiScopeDenied {
+  if (principal.type === "service-token") {
+    return {
+      allowed: true,
+      principalType: principal.type,
+      tenantId: requestedTenantId,
+    };
+  }
+
+  const { apiKey } = principal;
+
+  if (!apiKey.active) {
+    return deny(principal.type, "api_key_inactive");
+  }
+
+  if (apiKey.expires && apiKey.expires.getTime() <= now.getTime()) {
+    return deny(principal.type, "api_key_expired");
+  }
+
+  if (apiKey.tenant.deactivatedReason) {
+    return deny(principal.type, "tenant_deactivated");
+  }
+
+  if (apiKey.tenantId !== apiKey.tenant.id || (requestedTenantId && requestedTenantId !== apiKey.tenantId)) {
+    return deny(principal.type, "tenant_mismatch");
+  }
+
+  return {
+    allowed: true,
+    principalType: principal.type,
+    tenantId: apiKey.tenantId,
+  };
+}

hivegrid-cloud/apps/api/src/auth/session/session-store.test.ts

@@ -0,0 +1,154 @@
+import { describe, expect, test } from "vitest";
+
+import { createApiSessionStore } from "./session-store.js";
+
+const START = new Date("2026-03-01T10:00:00.000Z");
+
+function createDeterministicStore(ids: string[] = ["session-1", "session-2", "session-3"]) {
+  let nowMs = START.getTime();
+  let idIndex = 0;
+
+  return {
+    store: createApiSessionStore({
+      now: () => new Date(nowMs),
+      generateSessionId: () => ids[idIndex++] ?? `session-${idIndex}`,
+    }),
+    advance(ms: number) {
+      nowMs += ms;
+    },
+  };
+}
+
+describe("createApiSessionStore", () => {
+  test("creates active sessions with non-secret metadata and supports active lookup", () => {
+    const { store } = createDeterministicStore();
+
+    const session = store.createSession("user-1");
+
+    expect(session).toEqual({
+      sessionId: "session-1",
+      userId: "user-1",
+      createdAt: START,
+      revokedAt: undefined,
+      revokeReason: undefined,
+    });
+    expect(store.getActiveSession("session-1")).toEqual(session);
+    expect(JSON.stringify(session)).not.toContain("jwt");
+    expect(JSON.stringify(session)).not.toContain("cookie");
+    expect(JSON.stringify(session)).not.toContain("token");
+  });
+
+  test("returns cloned records so callers cannot mutate stored state", () => {
+    const { store } = createDeterministicStore();
+    const session = store.createSession("user-1");
+
+    session.createdAt.setUTCFullYear(2000);
+    session.userId = "changed";
+
+    expect(store.getSession("session-1")).toMatchObject({
+      sessionId: "session-1",
+      userId: "user-1",
+      createdAt: START,
+    });
+  });
+
+  test("revoke is idempotent and unknown revocation is a safe no-op", () => {
+    const { store, advance } = createDeterministicStore();
+    store.createSession("user-1");
+
+    advance(1_000);
+    const revoked = store.revokeSession("session-1", "logout");
+    advance(1_000);
+    const revokedAgain = store.revokeSession("session-1", "test");
+
+    expect(store.revokeSession("missing", "logout")).toBeUndefined();
+    expect(revoked).toMatchObject({
+      sessionId: "session-1",
+      revokedAt: new Date(START.getTime() + 1_000),
+      revokeReason: "logout",
+    });
+    expect(revokedAgain).toEqual(revoked);
+    expect(store.getActiveSession("session-1")).toBeUndefined();
+  });
+
+  test("rotate revokes the old session and activates the replacement", () => {
+    const { store, advance } = createDeterministicStore();
+    store.createSession("user-1");
+
+    advance(5_000);
+    const result = store.rotateSession("session-1", "refresh");
+
+    expect(result).toEqual({
+      rotated: true,
+      previous: {
+        sessionId: "session-1",
+        userId: "user-1",
+        createdAt: START,
+        revokedAt: new Date(START.getTime() + 5_000),
+        revokeReason: "refresh",
+      },
+      session: {
+        sessionId: "session-2",
+        userId: "user-1",
+        createdAt: new Date(START.getTime() + 5_000),
+        revokedAt: undefined,
+        revokeReason: undefined,
+      },
+    });
+    expect(store.getActiveSession("session-1")).toBeUndefined();
+    expect(result.rotated).toBe(true);
+    if (!result.rotated) {
+      throw new Error("Expected session rotation to succeed");
+    }
+    expect(store.getActiveSession("session-2")).toEqual(result.session);
+  });
+
+  test("unknown and revoked sessions cannot rotate", () => {
+    const { store } = createDeterministicStore();
+    store.createSession("user-1");
+    store.revokeSession("session-1", "logout");
+
+    expect(store.rotateSession("missing")).toEqual({ rotated: false, reason: "unknown" });
+    expect(store.rotateSession("session-1")).toEqual({
+      rotated: false,
+      reason: "revoked",
+      previous: store.getSession("session-1"),
+    });
+  });
+
+  test("happy path: createSession inserts on a single try without a retry loop", () => {
+    const ids: string[] = [];
+    const { store } = createDeterministicStore(["session-1"]);
+    // Wrap generateSessionId via a fresh store-spy: easier to assert call count
+    // through the public surface by tracking ids generated above.
+    const session = store.createSession("user-1");
+    ids.push(session.sessionId);
+
+    expect(ids).toEqual(["session-1"]);
+    expect(store.getActiveSession("session-1")).toMatchObject({ userId: "user-1" });
+  });
+
+  test("uuid collision is treated as a systemic error and throws (no retry loop)", () => {
+    // Force the id generator to return the same value twice; the legacy
+    // implementation looped to a fresh id, masking the underlying entropy bug.
+    const { store } = createDeterministicStore(["session-1", "session-1"]);
+
+    store.createSession("user-1");
+
+    expect(() => store.createSession("user-2")).toThrow(/UUID collision/);
+    // Original session must remain untouched.
+    expect(store.getActiveSession("session-1")).toMatchObject({ userId: "user-1" });
+  });
+
+  test("reset clears all sessions and isolates store state", () => {
+    const { store } = createDeterministicStore();
+    store.createSession("user-1");
+    store.createSession("user-2");
+
+    store.reset();
+
+    expect(store.getSession("session-1")).toBeUndefined();
+    expect(store.getActiveSession("session-2")).toBeUndefined();
+    expect(store.rotateSession("session-1")).toEqual({ rotated: false, reason: "unknown" });
+  });
+});

hivegrid-cloud/apps/api/src/auth/session/session-store.ts

@@ -0,0 +1,121 @@
+import { randomUUID } from "node:crypto";
+
+export type ApiSessionRevokeReason = "logout" | "refresh" | "replaced" | "duplicate" | "test" | (string & {});
+
+export type ApiSessionRecord = {
+  sessionId: string;
+  userId: string;
+  createdAt: Date;
+  revokedAt?: Date;
+  revokeReason?: ApiSessionRevokeReason;
+};
+
+type Clock = () => Date;
+type IdGenerator = () => string;
+
+export type ApiSessionStoreOptions = {
+  now?: Clock;
+  generateSessionId?: IdGenerator;
+};
+
+export type RotateApiSessionResult =
+  | { rotated: true; previous: ApiSessionRecord; session: ApiSessionRecord }
+  | { rotated: false; reason: "unknown" | "revoked"; previous?: ApiSessionRecord };
+
+const defaultClock: Clock = () => new Date();
+const defaultIdGenerator: IdGenerator = () => randomUUID();
+
+function cloneSession(session: ApiSessionRecord): ApiSessionRecord {
+  return {
+    sessionId: session.sessionId,
+    userId: session.userId,
+    createdAt: new Date(session.createdAt),
+    revokedAt: session.revokedAt ? new Date(session.revokedAt) : undefined,
+    revokeReason: session.revokeReason,
+  };
+}
+
+function isActive(session: ApiSessionRecord | undefined): session is ApiSessionRecord {
+  return Boolean(session && !session.revokedAt);
+}
+
+export function createApiSessionStore(options: ApiSessionStoreOptions = {}) {
+  const sessions = new Map<string, ApiSessionRecord>();
+  const now = options.now ?? defaultClock;
+  const generateSessionId = options.generateSessionId ?? defaultIdGenerator;
+
+  function createSession(userId: string): ApiSessionRecord {
+    const sessionId = generateSessionId();
+    // UUID-v4 has 122 bits of entropy; the birthday-paradox collision rate
+    // is ~10^-18 per insert. A retry loop is cargo-cult: any collision here
+    // signals a broken entropy source, not normal operation. Treat it as a
+    // systemic error so we surface the bug instead of masking it.
+    if (sessions.has(sessionId)) {
+      throw new Error(`api-session-store: refusing to overwrite session ${sessionId} (UUID collision suggests broken entropy source).`);
+    }
+
+    const session: ApiSessionRecord = {
+      sessionId,
+      userId,
+      createdAt: now(),
+    };
+    sessions.set(sessionId, session);
+
+    return cloneSession(session);
+  }
+
+  function getSession(sessionId: string): ApiSessionRecord | undefined {
+    const session = sessions.get(sessionId);
+    return session ? cloneSession(session) : undefined;
+  }
+
+  function getActiveSession(sessionId: string): ApiSessionRecord | undefined {
+    const session = sessions.get(sessionId);
+    return isActive(session) ? cloneSession(session) : undefined;
+  }
+
+  function revokeSession(sessionId: string, reason: ApiSessionRevokeReason = "logout"): ApiSessionRecord | undefined {
+    const session = sessions.get(sessionId);
+    if (!session) {
+      return undefined;
+    }
+
+    if (!session.revokedAt) {
+      session.revokedAt = now();
+      session.revokeReason = reason;
+    }
+
+    return cloneSession(session);
+  }
+
+  function rotateSession(sessionId: string, reason: ApiSessionRevokeReason = "refresh"): RotateApiSessionResult {
+    const previous = sessions.get(sessionId);
+    if (!previous) {
+      return { rotated: false, reason: "unknown" };
+    }
+    if (!isActive(previous)) {
+      return { rotated: false, reason: "revoked", previous: cloneSession(previous) };
+    }
+
+    previous.revokedAt = now();
+    previous.revokeReason = reason;
+    const session = createSession(previous.userId);
+
+    return { rotated: true, previous: cloneSession(previous), session };
+  }
+
+  function reset() {
+    sessions.clear();
+  }
+
+  return {
+    createSession,
+    getSession,
+    getActiveSession,
+    revokeSession,
+    rotateSession,
+    reset,
+  };
+}
+
+export const apiSessionStore = createApiSessionStore();

hivegrid-cloud/apps/api/src/billing/stripe-webhook-route.test.ts

@@ -0,0 +1,566 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { Hono } from "hono";
+import type Stripe from "stripe";
+
+vi.mock("../cache/subscription-cache-publisher.js", () => {
+  const fn = vi.fn().mockResolvedValue(undefined);
+  (globalThis as typeof globalThis & { __publishMock?: ReturnType<typeof vi.fn> }).__publishMock = fn;
+  return {
+    publishSubscriptionInvalidation: fn,
+    __resetSubscriptionCachePublisherForTests: vi.fn(),
+  };
+});
+
+function getPublishMock(): ReturnType<typeof vi.fn> {
+  return (globalThis as typeof globalThis & { __publishMock: ReturnType<typeof vi.fn> }).__publishMock;
+}
+
+import { postStripeBillingWebhookRequest, type StripeWebhookTransactionClient } from "./stripe-webhook-route.js";
+import { getApiMetrics, resetApiMetricsForTests } from "../metrics.js";
+import { apiLogger } from "../logger.js";
+import type { ApiErrorEnv } from "../middleware/error-envelope.js";
+
+beforeEach(() => {
+  resetApiMetricsForTests();
+  getPublishMock().mockClear();
+  process.env.STRIPE_SK = "sk_test_abc";
+  process.env.STRIPE_WEBHOOK_SIGNING_SECRET = "whsec_test";
+  process.env.API_ACCESS_TOKEN = "test-access-token";
+  process.env.ANALYTICS_TOKEN = "test-analytics-token";
+  process.env.SESSION_SECRET = "test-session-secret-with-32-bytes-of-entropy-padding";
+  process.env.CRYPTO_SECRET = "test-crypto-secret-with-32-bytes-padding-yo";
+  process.env.APP_BASE_DOMAIN = "localhost";
+  process.env.DATABASE_URL = "postgres://test:test@localhost:5432/test";
+});
+
+function buildDeps(overrides: Partial<Parameters<typeof postStripeBillingWebhookRequest>[1]> = {}) {
+  // Default transaction runner just invokes the callback with a stub tx
+  // so callers can wire publishOutboxEvent. Tests overriding this must
+  // make sure the surrounding contract still holds.
+  return {
+    constructStripeWebhookEvent: ((payload: string, signature: string) => {
+      void payload;
+      void signature;
+      return { id: "evt_test_default", type: "invoice.paid", data: { object: { id: "in_test" } } } as unknown as Stripe.Event;
+    }) as never,
+    retrieveStripeSubscription: (async () => null) as never,
+    findTenantSubscriptionProductByStripeSubscriptionId: (async () => null) as never,
+    updateTenantSubscriptionProductFromStripe: (async () => undefined) as never,
+    runStripeWebhookTransaction: (async <T,>(fn: (tx: StripeWebhookTransactionClient) => Promise<T>) =>
+      fn({} as unknown as StripeWebhookTransactionClient)) as never,
+    publishOutboxEvent: (async () => ({ id: "outbox-1" })) as never,
+    ...overrides,
+  };
+}
+
+function buildApp(deps: ReturnType<typeof buildDeps>) {
+  const app = new Hono<ApiErrorEnv>();
+  app.post("/v1/billing/stripe/webhooks", (c) => postStripeBillingWebhookRequest(c, deps));
+  return app;
+}
+
+describe("postStripeBillingWebhookRequest", () => {
+  it("records status=received with the event_type label on success", async () => {
+    const app = buildApp(buildDeps());
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: JSON.stringify({}),
+    });
+    expect(res.status).toBe(200);
+    const text = await getApiMetrics().registry.metrics();
+    expect(text).toMatch(/stripe_webhook_events_total\{[^}]*event_type="invoice.paid"[^}]*status="received"[^}]*\} 1/);
+  });
+
+  it("records status=missing_signature when stripe-signature header is absent", async () => {
+    const app = buildApp(buildDeps());
+    const res = await app.request("/v1/billing/stripe/webhooks", { method: "POST", body: "{}" });
+    expect(res.status).toBe(400);
+    const text = await getApiMetrics().registry.metrics();
+    expect(text).toMatch(/stripe_webhook_events_total\{[^}]*event_type="unknown"[^}]*status="missing_signature"[^}]*\} 1/);
+  });
+
+  it("publishes a subscription invalidation after updating the tenant subscription product", async () => {
+    const stripeSubscription = {
+      id: "sub_test",
+      cancel_at: null,
+      canceled_at: null,
+      ended_at: null,
+      items: {
+        data: [
+          {
+            id: "si_test",
+            current_period_start: 1_700_000_000,
+            current_period_end: 1_702_592_000,
+          },
+        ],
+      },
+    } as unknown as Stripe.Subscription;
+    const tenantSubscriptionProduct = {
+      id: "tsp-1",
+      endsAt: null,
+      cancelledAt: null,
+      tenantSubscription: { tenant: { id: "tenant-abc", name: "Acme" } },
+      subscriptionProduct: { title: "Pro Plan" },
+    } as unknown as never;
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_test_sub_update", type: "customer.subscription.updated", data: { object: { id: "sub_test" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => stripeSubscription) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => tenantSubscriptionProduct as never) as never,
+        updateTenantSubscriptionProductFromStripe: (async () => undefined) as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(getPublishMock()).toHaveBeenCalledWith("tenant-abc");
+  });
+
+  it("wraps the subscription update and outbox publishes in a single transaction (Phase 03 #42)", async () => {
+    const stripeSubscription = {
+      id: "sub_test",
+      cancel_at: 1_700_100_000, // triggers both endsAt + cancelledAt branches
+      canceled_at: null,
+      ended_at: null,
+      items: {
+        data: [
+          {
+            id: "si_test",
+            current_period_start: 1_700_000_000,
+            current_period_end: 1_702_592_000,
+          },
+        ],
+      },
+    } as unknown as Stripe.Subscription;
+    const tenantSubscriptionProduct = {
+      id: "tsp-1",
+      endsAt: null,
+      cancelledAt: null,
+      tenantSubscription: { tenant: { id: "tenant-abc", name: "Acme" } },
+      subscriptionProduct: { title: "Pro Plan" },
+    } as unknown as never;
+
+    const updateCalls: Array<{ id: string; data: Record<string, unknown>; usedTxClient: boolean }> = [];
+    const publishCalls: Array<{ tenantId: string | null | undefined; type: string; payload: unknown; dispatchKey?: string | null }> = [];
+    const txClientSentinel = { __isStripeTx: true } as unknown as StripeWebhookTransactionClient;
+    let transactionsStarted = 0;
+
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_test_lifecycle", type: "customer.subscription.updated", data: { object: { id: "sub_test" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => stripeSubscription) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => tenantSubscriptionProduct as never) as never,
+        updateTenantSubscriptionProductFromStripe: (async (id: string, data: Record<string, unknown>, client?: unknown) => {
+          updateCalls.push({ id, data, usedTxClient: client === txClientSentinel });
+          return undefined;
+        }) as never,
+        runStripeWebhookTransaction: (async <T,>(fn: (tx: StripeWebhookTransactionClient) => Promise<T>) => {
+          transactionsStarted += 1;
+          return fn(txClientSentinel);
+        }) as never,
+        publishOutboxEvent: (async (tx: unknown, args: { tenantId?: string | null; type: string; payload: unknown; dispatchKey?: string | null }) => {
+          publishCalls.push({ tenantId: args.tenantId, type: args.type, payload: args.payload, dispatchKey: args.dispatchKey ?? null });
+          expect(tx).toBe(txClientSentinel);
+          return { id: `outbox-${publishCalls.length}` };
+        }) as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(transactionsStarted).toBe(1);
+    expect(updateCalls).toHaveLength(1);
+    expect(updateCalls[0]!.usedTxClient).toBe(true);
+    expect(publishCalls).toHaveLength(2);
+    const types = publishCalls.map((c) => c.type).sort();
+    expect(types).toEqual(["billing.subscription.cancelled", "billing.subscription.ended"]);
+    for (const c of publishCalls) {
+      expect(c.tenantId).toBe("tenant-abc");
+      expect(c.dispatchKey).toMatch(/^stripe:evt_test_lifecycle:subscription\.(ended|cancelled)$/);
+      const payload = c.payload as { name: string; tenantId: string; description: string };
+      expect(payload.tenantId).toBe("tenant-abc");
+      expect(payload.description).toMatch(/Acme subscription/);
+      expect(["subscription.ended", "subscription.cancelled"]).toContain(payload.name);
+    }
+  });
+
+  it("records status=invalid_signature when constructEvent throws", async () => {
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() => {
+          throw new Error("bad sig");
+        }) as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "bogus" },
+      body: "{}",
+    });
+    expect(res.status).toBe(400);
+    const text = await getApiMetrics().registry.metrics();
+    expect(text).toMatch(/stripe_webhook_events_total\{[^}]*status="invalid_signature"[^}]*\} 1/);
+  });
+
+  it("records status=config_unavailable when stripe secrets are missing", async () => {
+    const prevSk = process.env.STRIPE_SK;
+    const prevSecret = process.env.STRIPE_WEBHOOK_SIGNING_SECRET;
+    delete process.env.STRIPE_SK;
+    delete process.env.STRIPE_WEBHOOK_SIGNING_SECRET;
+    try {
+      const app = buildApp(buildDeps());
+      const res = await app.request("/v1/billing/stripe/webhooks", {
+        method: "POST",
+        headers: { "stripe-signature": "t=1,v1=abc" },
+        body: "{}",
+      });
+      expect(res.status).toBe(503);
+      const text = await getApiMetrics().registry.metrics();
+      expect(text).toMatch(/stripe_webhook_events_total\{[^}]*status="config_unavailable"[^}]*\} 1/);
+    } finally {
+      if (prevSk !== undefined) process.env.STRIPE_SK = prevSk;
+      if (prevSecret !== undefined) process.env.STRIPE_WEBHOOK_SIGNING_SECRET = prevSecret;
+    }
+  });
+
+  it("extracts the subscription id from a string-shaped subscription_schedule.canceled event", async () => {
+    const retrieveStripeSubscription = vi.fn(async () => null);
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({
+            id: "evt_schedule_string",
+            type: "subscription_schedule.canceled",
+            data: { object: { id: "schedule_1", subscription: "sub_via_string" } },
+          } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: retrieveStripeSubscription as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(retrieveStripeSubscription).toHaveBeenCalledWith("sub_via_string");
+  });
+
+  it("extracts the subscription id from an object-shaped subscription_schedule.canceled event", async () => {
+    const retrieveStripeSubscription = vi.fn(async () => null);
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({
+            id: "evt_schedule_object",
+            type: "subscription_schedule.canceled",
+            data: { object: { id: "schedule_2", subscription: { id: "sub_via_object" } } },
+          } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: retrieveStripeSubscription as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(retrieveStripeSubscription).toHaveBeenCalledWith("sub_via_object");
+  });
+
+  it("skips sync when subscription_schedule.canceled has no subscription reference", async () => {
+    const retrieveStripeSubscription = vi.fn(async () => null);
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({
+            id: "evt_schedule_none",
+            type: "subscription_schedule.canceled",
+            data: { object: { id: "schedule_3", subscription: null } },
+          } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: retrieveStripeSubscription as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(retrieveStripeSubscription).not.toHaveBeenCalled();
+  });
+
+  it("skips sync when a subscription lifecycle event has no object id", async () => {
+    const retrieveStripeSubscription = vi.fn(async () => null);
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({
+            id: "evt_subscription_without_id",
+            type: "customer.subscription.updated",
+            data: { object: {} },
+          } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: retrieveStripeSubscription as never,
+      }),
+    );
+
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+
+    expect(res.status).toBe(200);
+    expect(retrieveStripeSubscription).not.toHaveBeenCalled();
+  });
+
+  it("returns 200 and skips downstream work when retrieveStripeSubscription returns null", async () => {
+    const updateTenantSubscriptionProductFromStripe = vi.fn();
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_missing_sub", type: "customer.subscription.deleted", data: { object: { id: "sub_missing" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => null) as never,
+        updateTenantSubscriptionProductFromStripe: updateTenantSubscriptionProductFromStripe as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(updateTenantSubscriptionProductFromStripe).not.toHaveBeenCalled();
+  });
+
+  it("returns 200 and skips downstream work when no tenant subscription product matches the stripe subscription", async () => {
+    const updateTenantSubscriptionProductFromStripe = vi.fn();
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_no_tsp", type: "customer.subscription.deleted", data: { object: { id: "sub_known" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => ({ id: "sub_known", cancel_at: null, canceled_at: null, ended_at: null, items: { data: [] } } as unknown as Stripe.Subscription)) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => null) as never,
+        updateTenantSubscriptionProductFromStripe: updateTenantSubscriptionProductFromStripe as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(updateTenantSubscriptionProductFromStripe).not.toHaveBeenCalled();
+  });
+
+  it("derives endsAt from ended_at when canceled_at is set and the subscription has already ended", async () => {
+    const stripeSubscription = {
+      id: "sub_ended",
+      cancel_at: null,
+      canceled_at: 1_700_050_000,
+      ended_at: 1_700_060_000,
+      items: { data: [{ id: "si_test", current_period_start: 1_700_000_000, current_period_end: 1_702_592_000 }] },
+    } as unknown as Stripe.Subscription;
+    const tenantSubscriptionProduct = {
+      id: "tsp-2",
+      endsAt: null,
+      cancelledAt: null,
+      tenantSubscription: { tenant: { id: "tenant-end", name: "EndedCo" } },
+      subscriptionProduct: { title: "Pro" },
+    } as unknown as never;
+
+    const publishCalls: string[] = [];
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_ended", type: "customer.subscription.deleted", data: { object: { id: "sub_ended" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => stripeSubscription) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => tenantSubscriptionProduct as never) as never,
+        updateTenantSubscriptionProductFromStripe: (async (_id: string, data: Record<string, unknown>) => {
+          // endsAt should derive from ended_at, NOT from canceled_at.
+          expect((data.endsAt as Date).getTime()).toBe(1_700_060_000 * 1000);
+          expect((data.cancelledAt as Date).getTime()).toBe(1_700_050_000 * 1000);
+          return undefined;
+        }) as never,
+        publishOutboxEvent: (async (_tx: unknown, args: { type: string }) => {
+          publishCalls.push(args.type);
+          return { id: `outbox-${publishCalls.length}` };
+        }) as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(publishCalls.sort()).toEqual(["billing.subscription.cancelled", "billing.subscription.ended"]);
+  });
+
+  it("uses 'now' for endsAt when canceled_at is set but ended_at is null", async () => {
+    const stripeSubscription = {
+      id: "sub_cancel_no_end",
+      cancel_at: null,
+      canceled_at: 1_700_050_000,
+      ended_at: null,
+      items: { data: [{ id: "si_test", current_period_start: 1_700_000_000, current_period_end: 1_702_592_000 }] },
+    } as unknown as Stripe.Subscription;
+    const tenantSubscriptionProduct = {
+      id: "tsp-3",
+      endsAt: null,
+      cancelledAt: null,
+      tenantSubscription: { tenant: { id: "tenant-now", name: "NowCo" } },
+      subscriptionProduct: { title: "Pro" },
+    } as unknown as never;
+
+    let observedEndsAtMs = 0;
+    const before = Date.now();
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_cancel_now", type: "customer.subscription.updated", data: { object: { id: "sub_cancel_no_end" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => stripeSubscription) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => tenantSubscriptionProduct as never) as never,
+        updateTenantSubscriptionProductFromStripe: (async (_id: string, data: Record<string, unknown>) => {
+          observedEndsAtMs = (data.endsAt as Date).getTime();
+          return undefined;
+        }) as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    const after = Date.now();
+    expect(res.status).toBe(200);
+    expect(observedEndsAtMs).toBeGreaterThanOrEqual(before);
+    expect(observedEndsAtMs).toBeLessThanOrEqual(after);
+  });
+
+  it("falls back to epoch period dates when the stripe subscription has no item data", async () => {
+    const stripeSubscription = {
+      id: "sub_empty_items",
+      cancel_at: null,
+      canceled_at: null,
+      ended_at: null,
+      items: { data: [] },
+    } as unknown as Stripe.Subscription;
+    const tenantSubscriptionProduct = {
+      id: "tsp-empty-items",
+      endsAt: null,
+      cancelledAt: null,
+      tenantSubscription: { tenant: { id: "tenant-empty-items", name: "EmptyItemsCo" } },
+      subscriptionProduct: { title: "Pro" },
+    } as unknown as never;
+
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_empty_items", type: "customer.subscription.updated", data: { object: { id: "sub_empty_items" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => stripeSubscription) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => tenantSubscriptionProduct as never) as never,
+        updateTenantSubscriptionProductFromStripe: (async (_id: string, data: Record<string, unknown>) => {
+          expect((data.currentPeriodStart as Date).getTime()).toBe(0);
+          expect((data.currentPeriodEnd as Date).getTime()).toBe(0);
+          return undefined;
+        }) as never,
+      }),
+    );
+
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+
+    expect(res.status).toBe(200);
+  });
+
+  it("does not republish lifecycle events when endsAt/cancelledAt are unchanged from the previous record", async () => {
+    const stripeSubscription = {
+      id: "sub_noop",
+      cancel_at: 1_700_100_000,
+      canceled_at: null,
+      ended_at: null,
+      items: { data: [{ id: "si_test", current_period_start: 1_700_000_000, current_period_end: 1_702_592_000 }] },
+    } as unknown as Stripe.Subscription;
+    const tenantSubscriptionProduct = {
+      id: "tsp-4",
+      endsAt: new Date(1_700_100_000 * 1000),
+      cancelledAt: new Date(1_700_100_000 * 1000),
+      tenantSubscription: { tenant: { id: "tenant-noop", name: "NoopCo" } },
+      subscriptionProduct: { title: "Pro" },
+    } as unknown as never;
+
+    const publishCalls: string[] = [];
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_noop", type: "customer.subscription.updated", data: { object: { id: "sub_noop" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => stripeSubscription) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => tenantSubscriptionProduct as never) as never,
+        publishOutboxEvent: (async (_tx: unknown, args: { type: string }) => {
+          publishCalls.push(args.type);
+          return { id: `outbox-${publishCalls.length}` };
+        }) as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    expect(publishCalls).toEqual([]);
+  });
+
+  it("logs but does not surface publishSubscriptionInvalidation rejections", async () => {
+    const loggerWarn = vi.spyOn(apiLogger, "warn").mockImplementation(() => undefined);
+    const stripeSubscription = {
+      id: "sub_invalidate_fail",
+      cancel_at: null,
+      canceled_at: null,
+      ended_at: null,
+      items: { data: [{ id: "si_test", current_period_start: 1_700_000_000, current_period_end: 1_702_592_000 }] },
+    } as unknown as Stripe.Subscription;
+    const tenantSubscriptionProduct = {
+      id: "tsp-5",
+      endsAt: null,
+      cancelledAt: null,
+      tenantSubscription: { tenant: { id: "tenant-bad-pub", name: "PubCo" } },
+      subscriptionProduct: { title: "Pro" },
+    } as unknown as never;
+
+    getPublishMock().mockRejectedValueOnce(new Error("redis down"));
+    const app = buildApp(
+      buildDeps({
+        constructStripeWebhookEvent: (() =>
+          ({ id: "evt_pub_fail", type: "customer.subscription.updated", data: { object: { id: "sub_invalidate_fail" } } } as unknown as Stripe.Event)) as never,
+        retrieveStripeSubscription: (async () => stripeSubscription) as never,
+        findTenantSubscriptionProductByStripeSubscriptionId: (async () => tenantSubscriptionProduct as never) as never,
+      }),
+    );
+    const res = await app.request("/v1/billing/stripe/webhooks", {
+      method: "POST",
+      headers: { "stripe-signature": "t=1,v1=abc" },
+      body: "{}",
+    });
+    expect(res.status).toBe(200);
+    await Promise.resolve();
+    expect(getPublishMock()).toHaveBeenCalledWith("tenant-bad-pub");
+    expect(loggerWarn).toHaveBeenCalledWith(
+      expect.objectContaining({ tenantId: "tenant-bad-pub" }),
+      "stripe-webhook: failed to publish subscription invalidation",
+    );
+    loggerWarn.mockRestore();
+  });
+});

hivegrid-cloud/apps/api/src/billing/stripe-webhook-route.ts

@@ -0,0 +1,223 @@
+import type { StripeTenantSubscriptionProduct, StripeTenantSubscriptionProductClient } from "@toolctrl/db/subscriptions.server";
+import type { OutboxPublisherTransaction, OutboxPublishArgs } from "@toolctrl/db/outbox";
+import type { Context } from "hono";
+import type Stripe from "stripe";
+
+import type { ApiAppDependencies } from "../app.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { apiLogger } from "../logger.js";
+import { getApiMetrics } from "../metrics.js";
+import { apiError, type ApiErrorEnv } from "../middleware/error-envelope.js";
+import { publishSubscriptionInvalidation } from "../cache/subscription-cache-publisher.js";
+
+function recordStripeWebhookOutcome(eventType: string, status: "received" | "invalid_signature" | "missing_signature" | "config_unavailable") {
+  try {
+    getApiMetrics().metrics.stripeWebhookEventsTotal.labels({ event_type: eventType, status }).inc();
+  } catch {
+    // Metric writes must never break webhook handling.
+  }
+}
+
+/**
+ * Transaction runner contract. Production wires this to
+ * `prisma.$transaction(fn)` so the subscription update + outbox publish
+ * land atomically (Phase 03 #42). The transaction client must satisfy
+ * both `StripeTenantSubscriptionProductClient` (for the subscription
+ * update) and `OutboxPublisherTransaction` (for outbox publishes).
+ */
+export type StripeWebhookTransactionClient = StripeTenantSubscriptionProductClient & OutboxPublisherTransaction;
+
+export type StripeWebhookTransactionRunner = <T>(
+  fn: (tx: StripeWebhookTransactionClient) => Promise<T>,
+) => Promise<T>;
+
+type StripeBillingWebhookDependencies = Pick<
+  ApiAppDependencies,
+  | "constructStripeWebhookEvent"
+  | "retrieveStripeSubscription"
+  | "findTenantSubscriptionProductByStripeSubscriptionId"
+  | "updateTenantSubscriptionProductFromStripe"
+> & {
+  /**
+   * Runs the subscription update + outbox publish in a single Prisma
+   * transaction (Phase 03 #42). Production wiring lives in
+   * `app-dependencies.ts`.
+   */
+  runStripeWebhookTransaction: StripeWebhookTransactionRunner;
+  /**
+   * Outbox publisher bound against the transaction client. Defaults to
+   * `publishOutboxEvent` from `@toolctrl/db/outbox`; tests inject a
+   * recorder. Wrapped here as a dependency so we never reach for the
+   * top-level Prisma singleton inside the transaction.
+   */
+  publishOutboxEvent: (tx: OutboxPublisherTransaction, args: OutboxPublishArgs) => Promise<unknown>;
+};
+
+export async function postStripeBillingWebhookRequest(c: Context<ApiErrorEnv>, dependencies: StripeBillingWebhookDependencies) {
+  const config = getApiRuntimeConfig();
+  if (!config.stripe.secretKey || !config.stripe.webhookSigningSecret) {
+    recordStripeWebhookOutcome("unknown", "config_unavailable");
+    return apiError(c, 503, "dependency_unavailable", "Stripe billing webhook configuration is unavailable.");
+  }
+
+  const signature = c.req.header("stripe-signature");
+  if (!signature) {
+    recordStripeWebhookOutcome("unknown", "missing_signature");
+    return apiError(c, 400, "invalid_request", "Missing Stripe signature.");
+  }
+
+  const payload = await c.req.text();
+  let event: Stripe.Event;
+  try {
+    event = dependencies.constructStripeWebhookEvent(payload, signature);
+  } catch {
+    recordStripeWebhookOutcome("unknown", "invalid_signature");
+    return apiError(c, 400, "invalid_signature", "Invalid Stripe webhook signature.");
+  }
+
+  if (
+    event.type === "subscription_schedule.canceled" ||
+    event.type === "customer.subscription.deleted" ||
+    event.type === "customer.subscription.updated"
+  ) {
+    const stripeSubscriptionId = getStripeSubscriptionIdFromEvent(event);
+    if (stripeSubscriptionId) {
+      await syncStripeTenantSubscription(stripeSubscriptionId, event.id, dependencies);
+    }
+  }
+
+  recordStripeWebhookOutcome(event.type, "received");
+  return c.json({ received: true, event: event.type });
+}
+
+function getStripeSubscriptionIdFromEvent(event: Stripe.Event): string | null {
+  const object = event.data.object as { id?: string; subscription?: string | { id?: string } | null };
+  if (event.type === "subscription_schedule.canceled") {
+    if (typeof object.subscription === "string") return object.subscription;
+    return object.subscription?.id ?? null;
+  }
+  return object.id ?? null;
+}
+
+async function syncStripeTenantSubscription(
+  stripeSubscriptionId: string,
+  stripeEventId: string,
+  dependencies: StripeBillingWebhookDependencies,
+) {
+  const stripeSubscription = await dependencies.retrieveStripeSubscription(stripeSubscriptionId);
+  if (!stripeSubscription) {
+    return;
+  }
+  const tenantSubscriptionProduct = await dependencies.findTenantSubscriptionProductByStripeSubscriptionId(stripeSubscription.id);
+  if (!tenantSubscriptionProduct) {
+    return;
+  }
+
+  const previousEndsAt = tenantSubscriptionProduct.endsAt?.getTime() ?? null;
+  const previousCancelledAt = tenantSubscriptionProduct.cancelledAt?.getTime() ?? null;
+  let endsAt: Date | null = null;
+  let cancelledAt: Date | null = null;
+
+  if (stripeSubscription.cancel_at) {
+    endsAt = new Date(stripeSubscription.cancel_at * 1000);
+    cancelledAt = new Date(stripeSubscription.cancel_at * 1000);
+  } else if (stripeSubscription.canceled_at) {
+    cancelledAt = new Date(stripeSubscription.canceled_at * 1000);
+    endsAt = stripeSubscription.ended_at ? new Date(stripeSubscription.ended_at * 1000) : new Date();
+  }
+
+  // Phase 03 #42 — wrap the DB update + outbox publishes in a single
+  // transaction. The legacy in-process `createLegacyEvent` write moved
+  // into a receiver worker (apps/worker/src/jobs/legacy-event-receiver.ts)
+  // that consumes the published events from the `outbox.notification`
+  // BullMQ queue. A crash between commit and dispatcher pickup is now
+  // safe: the events stay PENDING and the dispatcher claims them on
+  // the next tick.
+  await dependencies.runStripeWebhookTransaction(async (tx) => {
+    await dependencies.updateTenantSubscriptionProductFromStripe(
+      tenantSubscriptionProduct.id,
+      {
+        cancelledAt,
+        endsAt,
+        currentPeriodStart: new Date((stripeSubscription.items.data[0]?.current_period_start ?? 0) * 1000),
+        currentPeriodEnd: new Date((stripeSubscription.items.data[0]?.current_period_end ?? 0) * 1000),
+      },
+      tx,
+    );
+
+    await publishSubscriptionLifecycleEvents(
+      {
+        tenantSubscriptionProduct,
+        endsAt,
+        cancelledAt,
+        previousEndsAt,
+        previousCancelledAt,
+        stripeEventId,
+      },
+      tx,
+      dependencies,
+    );
+  });
+
+  // Cross-replica cache invalidation: publish so every web replica drops
+  // its in-process subscription cache for this tenant. Fire-and-forget;
+  // failures must not block webhook delivery (5xx → Stripe retries).
+  const tenantIdToInvalidate = tenantSubscriptionProduct.tenantSubscription.tenant.id;
+  publishSubscriptionInvalidation(tenantIdToInvalidate).catch((err: unknown) => {
+    apiLogger.warn({ err, tenantId: tenantIdToInvalidate }, "stripe-webhook: failed to publish subscription invalidation");
+  });
+}
+
+async function publishSubscriptionLifecycleEvents(
+  {
+    tenantSubscriptionProduct,
+    endsAt,
+    cancelledAt,
+    previousEndsAt,
+    previousCancelledAt,
+    stripeEventId,
+  }: {
+    tenantSubscriptionProduct: StripeTenantSubscriptionProduct;
+    endsAt: Date | null;
+    cancelledAt: Date | null;
+    previousEndsAt: number | null;
+    previousCancelledAt: number | null;
+    stripeEventId: string;
+  },
+  tx: StripeWebhookTransactionClient,
+  dependencies: Pick<StripeBillingWebhookDependencies, "publishOutboxEvent">,
+) {
+  const tenant = tenantSubscriptionProduct.tenantSubscription.tenant;
+  const product = tenantSubscriptionProduct.subscriptionProduct;
+  if (endsAt && previousEndsAt !== endsAt.getTime()) {
+    await dependencies.publishOutboxEvent(tx, {
+      tenantId: tenant.id,
+      type: "billing.subscription.ended",
+      // Idempotent against duplicate Stripe deliveries: each Stripe event
+      // is identified by `evt_*`; combined with the lifecycle suffix this
+      // dedups re-deliveries of the same webhook event.
+      dispatchKey: `stripe:${stripeEventId}:subscription.ended`,
+      payload: {
+        name: "subscription.ended",
+        tenantId: tenant.id,
+        userId: null,
+        data: JSON.stringify({ tenant, product }),
+        description: `${tenant.name} subscription ended: ${product.title}`,
+      },
+    });
+  }
+  if (cancelledAt && previousCancelledAt !== cancelledAt.getTime()) {
+    await dependencies.publishOutboxEvent(tx, {
+      tenantId: tenant.id,
+      type: "billing.subscription.cancelled",
+      dispatchKey: `stripe:${stripeEventId}:subscription.cancelled`,
+      payload: {
+        name: "subscription.cancelled",
+        tenantId: tenant.id,
+        userId: null,
+        data: JSON.stringify({ tenant, product }),
+        description: `${tenant.name} subscription cancelled: ${product.title}`,
+      },
+    });
+  }
+}

hivegrid-cloud/apps/api/src/cache/subscription-cache-publisher.ts

@@ -0,0 +1,54 @@
+import { createTwoTierCache, type TwoTierCache } from "@toolctrl/shared-store";
+import { getSharedStore } from "../shared-store.js";
+import { apiLogger } from "../logger.js";
+
+/**
+ * Publisher-side handle to the subscription cache pub/sub channel.
+ *
+ * The API does NOT keep an L1 cache for subscriptions (every read goes
+ * to Postgres via the legacy DB layer), so the only thing we need on
+ * this side is the ability to PUBLISH an invalidation message that the
+ * web replicas listen for. We piggyback on the two-tier cache helper —
+ * its `invalidate(key)` lights up the same channel that the web
+ * subscribers consume.
+ *
+ * Sym-coupled with apps/web/app/utils/services/.server/subscriptionCache.server.ts.
+ * Namespace and channel MUST match: `subscription` / `invalidate`.
+ */
+const TTL_SECONDS = 60; // unused on this side; required by createTwoTierCache
+
+let publisherSingleton: TwoTierCache<unknown> | null = null;
+
+function pubsubEnabled(): boolean {
+  const raw = process.env.CACHE_INVALIDATION_ENABLED;
+  if (raw === undefined || raw === null || raw === "") return true;
+  return raw.toLowerCase() !== "false" && raw !== "0";
+}
+
+function getPublisher(): TwoTierCache<unknown> {
+  if (publisherSingleton) return publisherSingleton;
+  publisherSingleton = createTwoTierCache<unknown>({
+    store: getSharedStore(),
+    namespace: "subscription",
+    ttlSeconds: TTL_SECONDS,
+    pubsubEnabled: pubsubEnabled(),
+    logger: { warn: (obj, msg) => apiLogger.warn(obj, msg), error: (obj, msg) => apiLogger.error(obj, msg) },
+  });
+  return publisherSingleton;
+}
+
+/**
+ * Publish a subscription-invalidation for a single tenant. Resolves
+ * once the PUBLISH has been issued; remote subscribers see the event
+ * milliseconds later. Errors are returned (not thrown) so the caller
+ * can swallow them and never block webhook delivery.
+ */
+export async function publishSubscriptionInvalidation(tenantId: string): Promise<void> {
+  if (!tenantId) return;
+  await getPublisher().invalidate(tenantId);
+}
+
+/** Test seam: reset the singleton. */
+export function __resetSubscriptionCachePublisherForTests(): void {
+  publisherSingleton = null;
+}

hivegrid-cloud/apps/api/src/config.test.ts

@@ -0,0 +1,153 @@
+import { ApiConfigError, type ApiConfig } from "@toolctrl/config";
+import { afterEach, describe, expect, test } from "vitest";
+import { getApiRuntimeConfig, resetApiRuntimeConfigForTests, setApiRuntimeConfigForTests } from "./config.js";
+
+const originalEnv = { ...process.env };
+
+const REQUIRED_PRODUCTION_ENV = {
+  APP_NAME: "Production API",
+  DATABASE_URL: "postgres://example.invalid/app",
+  APP_BASE_DOMAIN: "example.com",
+  WEB_APP_HOST: "app.example.com",
+  ADMIN_APP_HOST: "admin.example.com",
+  API_HOST: "api.example.com",
+  SESSION_SECRET: "session-secret-value",
+  CRYPTO_SECRET: "crypto-secret-value",
+  API_ACCESS_TOKEN: "api-access-token-value",
+  JWT_SECRET: "jwt-secret-value",
+  ANALYTICS_TOKEN: "analytics-token-value",
+  API_PORT: "8081",
+  APP_PROTOCOL: "https",
+};
+
+const SECRET_VALUES = [
+  "api-access-token-value",
+  "jwt-secret-value",
+  "secret-like-invalid-port-value",
+  "session-secret-value",
+  "crypto-secret-value",
+  "analytics-token-value",
+];
+
+afterEach(() => {
+  resetApiRuntimeConfigForTests();
+  process.env = { ...originalEnv };
+});
+
+describe("getApiRuntimeConfig", () => {
+  test("loads required API runtime config from a complete production-like environment", () => {
+    process.env = {
+      ...originalEnv,
+      ...REQUIRED_PRODUCTION_ENV,
+      NODE_ENV: "production",
+    };
+
+    expect(getApiRuntimeConfig()).toMatchObject({
+      appName: "Production API",
+      nodeEnv: "production",
+      listen: { port: 8081 },
+      routing: {
+        appBaseDomain: "example.com",
+        webAppHost: "app.example.com",
+        adminAppHost: "admin.example.com",
+        apiHost: "api.example.com",
+        protocol: "https",
+      },
+      apiAccessToken: "api-access-token-value",
+      jwtSecret: "jwt-secret-value",
+      analytics: {
+        token: "analytics-token-value",
+      },
+    });
+  });
+
+  test("throws redacted ApiConfigError when API_ACCESS_TOKEN is missing", () => {
+    process.env = {
+      ...originalEnv,
+      ...REQUIRED_PRODUCTION_ENV,
+      NODE_ENV: "production",
+      API_ACCESS_TOKEN: undefined,
+    };
+
+    expect(() => getApiRuntimeConfig()).toThrow(ApiConfigError);
+
+    try {
+      getApiRuntimeConfig();
+    } catch (error) {
+      expect(error).toBeInstanceOf(ApiConfigError);
+      const apiConfigError = error as ApiConfigError;
+      expect(apiConfigError.message).toBe("Invalid API configuration (API_ACCESS_TOKEN:missing-required).");
+      expect(apiConfigError.details.diagnostics).toEqual([
+        expect.objectContaining({
+          code: "missing-required",
+          name: "API_ACCESS_TOKEN",
+          classification: "required-per-app",
+        }),
+      ]);
+      const errorText = JSON.stringify({ message: apiConfigError.message, details: apiConfigError.details });
+      for (const secretValue of SECRET_VALUES) {
+        expect(errorText).not.toContain(secretValue);
+      }
+    }
+  });
+
+  test("throws redacted ApiConfigError when API_PORT is invalid", () => {
+    process.env = {
+      ...originalEnv,
+      ...REQUIRED_PRODUCTION_ENV,
+      NODE_ENV: "production",
+      API_PORT: "secret-like-invalid-port-value",
+    };
+
+    try {
+      getApiRuntimeConfig();
+    } catch (error) {
+      expect(error).toBeInstanceOf(ApiConfigError);
+      const apiConfigError = error as ApiConfigError;
+      expect(apiConfigError.message).toBe("Invalid API configuration (API_PORT:invalid-port).");
+      expect(apiConfigError.details.diagnostics).toEqual([
+        expect.objectContaining({
+          code: "invalid-port",
+          name: "API_PORT",
+        }),
+      ]);
+      const errorText = JSON.stringify({ message: apiConfigError.message, details: apiConfigError.details });
+      for (const secretValue of SECRET_VALUES) {
+        expect(errorText).not.toContain(secretValue);
+      }
+    }
+  });
+
+  test("uses setApiRuntimeConfigForTests override without reading required environment", () => {
+    process.env = { NODE_ENV: "production" };
+    const overrideConfig: ApiConfig = {
+      appName: "Override API",
+      nodeEnv: "test",
+      revision: "unknown",
+      listen: { port: 9999 },
+      routing: {
+        appBaseDomain: "override.localhost",
+        webAppHost: "app.override.localhost",
+        adminAppHost: "admin.override.localhost",
+        apiHost: "api.override.localhost",
+        protocol: "http",
+      },
+      origins: {},
+      apiAccessToken: "override-token",
+      jwtSecret: "override-jwt-secret",
+      analytics: {},
+      mailjet: {},
+      stripe: {},
+      apiKeys: {
+        allowLegacyPlaintextFallback: true,
+      },
+      storage: {},
+    };
+
+    setApiRuntimeConfigForTests(overrideConfig);
+
+    expect(getApiRuntimeConfig()).toBe(overrideConfig);
+    resetApiRuntimeConfigForTests();
+    expect(() => getApiRuntimeConfig()).toThrow(ApiConfigError);
+  });
+});

hivegrid-cloud/apps/api/src/config.ts

@@ -0,0 +1,15 @@
+import { loadRequiredApiConfig, type ApiConfig } from "@toolctrl/config";
+
+let apiRuntimeConfigForTests: ApiConfig | undefined;
+
+export function getApiRuntimeConfig(): ApiConfig {
+  return apiRuntimeConfigForTests ?? loadRequiredApiConfig();
+}
+
+export function setApiRuntimeConfigForTests(config: ApiConfig) {
+  apiRuntimeConfigForTests = config;
+}
+
+export function resetApiRuntimeConfigForTests() {
+  apiRuntimeConfigForTests = undefined;
+}

hivegrid-cloud/apps/api/src/cors.test.ts

@@ -0,0 +1,66 @@
+import { afterEach, describe, expect, test } from "vitest";
+import type { ApiConfig } from "@toolctrl/config";
+import { resetApiRuntimeConfigForTests, setApiRuntimeConfigForTests } from "./config.js";
+import { getAllowedCorsOrigin } from "./cors.js";
+
+const TEST_API_CONFIG: ApiConfig = {
+  appName: "Test API",
+  nodeEnv: "test",
+  revision: "unknown",
+  listen: { port: 3001 },
+  routing: {
+    appBaseDomain: "localhost:3000",
+    webAppHost: "app.localhost:3000",
+    adminAppHost: "admin.localhost:3000",
+    apiHost: "api.localhost:3000",
+    protocol: "http",
+  },
+  origins: {
+    web: "https://web-origin.example",
+    admin: "https://admin-origin.example",
+  },
+  apiAccessToken: "test-api-access-token",
+  jwtSecret: "test-jwt-secret",
+  analytics: {},
+  mailjet: {},
+  stripe: {},
+  apiKeys: {
+    allowLegacyPlaintextFallback: true,
+  },
+  storage: {},
+};
+
+afterEach(() => {
+  resetApiRuntimeConfigForTests();
+});
+
+describe("getAllowedCorsOrigin", () => {
+  test("allows canonical app and admin origins from routing config", () => {
+    setApiRuntimeConfigForTests(TEST_API_CONFIG);
+
+    expect(getAllowedCorsOrigin("http://app.localhost:3000")).toBe("http://app.localhost:3000");
+    expect(getAllowedCorsOrigin("http://admin.localhost:3000")).toBe("http://admin.localhost:3000");
+  });
+
+  test("allows explicit WEB_ORIGIN and ADMIN_ORIGIN values", () => {
+    setApiRuntimeConfigForTests(TEST_API_CONFIG);
+
+    expect(getAllowedCorsOrigin("https://web-origin.example")).toBe("https://web-origin.example");
+    expect(getAllowedCorsOrigin("https://admin-origin.example")).toBe("https://admin-origin.example");
+  });
+
+  test("does not allow missing browser origin", () => {
+    setApiRuntimeConfigForTests(TEST_API_CONFIG);
+
+    expect(getAllowedCorsOrigin(undefined)).toBeUndefined();
+  });
+
+  test("rejects untrusted and malformed origin variants", () => {
+    setApiRuntimeConfigForTests(TEST_API_CONFIG);
+
+    expect(getAllowedCorsOrigin("https://attacker.example")).toBeUndefined();
+    expect(getAllowedCorsOrigin("http://app.localhost:3000.evil.example")).toBeUndefined();
+    expect(getAllowedCorsOrigin("https://app.localhost:3000")).toBeUndefined();
+    expect(getAllowedCorsOrigin("http://admin.localhost:3000/")).toBeUndefined();
+  });
+});

hivegrid-cloud/apps/api/src/cors.ts

@@ -0,0 +1,19 @@
+import { getApiRuntimeConfig } from "./config.js";
+
+export function getAllowedCorsOrigin(origin: string | undefined): string | undefined {
+  if (!origin) {
+    return undefined;
+  }
+
+  const config = getApiRuntimeConfig();
+  const allowedOrigins = new Set(
+    [
+      `${config.routing.protocol}://${config.routing.webAppHost}`,
+      `${config.routing.protocol}://${config.routing.adminAppHost}`,
+      config.origins.web,
+      config.origins.admin,
+    ].filter((allowedOrigin): allowedOrigin is string => Boolean(allowedOrigin))
+  );
+
+  return allowedOrigins.has(origin) ? origin : undefined;
+}

hivegrid-cloud/apps/api/src/csrf-inventory.test.ts

@@ -0,0 +1,216 @@
+import { beforeAll, describe, expect, test } from "vitest";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+import { scanApiRoutesFromFile, type ApiRoute } from "../../../tools/api-inventory/scan-api-routes.js";
+
+type MutatingMethod = "POST" | "PATCH" | "DELETE";
+type ProtectionClass =
+  | "jwt-auth"
+  | "service-token"
+  | "service-or-tenant-api-key"
+  | "optional-api-key-validation"
+  | "webhook-signature"
+  | "public-analytics-token"
+  | "public-widget"
+  | "auth-credential-exchange";
+
+type MutatingRoute = {
+  method: MutatingMethod;
+  path: string;
+};
+
+type InventoryEntry = MutatingRoute & {
+  protectionClass: ProtectionClass;
+  note: string;
+};
+
+const allowedProtectionClasses = new Set<ProtectionClass>([
+  "jwt-auth",
+  "service-token",
+  "service-or-tenant-api-key",
+  "optional-api-key-validation",
+  "webhook-signature",
+  "public-analytics-token",
+  "public-widget",
+  "auth-credential-exchange",
+]);
+
+const expectedInventory: InventoryEntry[] = [
+  {
+    method: "POST",
+    path: "/v1/auth/login",
+    protectionClass: "auth-credential-exchange",
+    note: "Exchanges email/password credentials for a JWT after password verification.",
+  },
+  {
+    method: "POST",
+    path: "/v1/auth/refresh-token",
+    protectionClass: "jwt-auth",
+    note: "Requires an existing bearer JWT before issuing a replacement JWT.",
+  },
+  {
+    method: "POST",
+    path: "/v1/auth/logout",
+    protectionClass: "jwt-auth",
+    note: "Requires an existing bearer JWT before revoking the active API session and clearing the API cookie.",
+  },
+  {
+    method: "POST",
+    path: "/v1/api-keys/validate",
+    protectionClass: "optional-api-key-validation",
+    note: "Public validation surface that classifies supplied API keys or service tokens without requiring browser CSRF state.",
+  },
+  {
+    method: "POST",
+    path: "/v1/ai/upload",
+    protectionClass: "service-token",
+    note: "Writes uploaded content using configured S3-compatible storage credentials for the API runtime.",
+  },
+  {
+    method: "POST",
+    path: "/v1/support/inbound-email",
+    protectionClass: "webhook-signature",
+    note: "Creates support tickets only after Mailjet webhook Basic-Auth credentials verify.",
+  },
+  {
+    method: "POST",
+    path: "/v1/analytics/events",
+    protectionClass: "public-analytics-token",
+    note: "Public analytics write endpoint scoped to analytics payload identifiers rather than browser CSRF state.",
+  },
+  {
+    method: "POST",
+    path: "/v1/analytics/page-views",
+    protectionClass: "public-analytics-token",
+    note: "Public analytics page-view write endpoint scoped to analytics payload identifiers rather than browser CSRF state.",
+  },
+  {
+    method: "POST",
+    path: "/v1/events/webhooks/attempts/:id",
+    protectionClass: "webhook-signature",
+    note: "Queues a known webhook attempt id only after Mailjet webhook Basic-Auth credentials verify.",
+  },
+  {
+    method: "POST",
+    path: "/v1/email-sequences/mailjet/webhooks",
+    protectionClass: "webhook-signature",
+    note: "Accepts sequence provider bounce events only after Mailjet webhook Basic-Auth credentials verify.",
+  },
+  {
+    method: "POST",
+    path: "/v1/billing/stripe/webhooks",
+    protectionClass: "webhook-signature",
+    note: "Forwards a raw billing lifecycle webhook payload only with Stripe signing headers preserved.",
+  },
+  {
+    method: "POST",
+    path: "/v1/maintenance/export-retention/enqueue",
+    protectionClass: "service-token",
+    note: "Queues export-artifact retention cleanup only after service-token authentication.",
+  },
+  {
+    method: "POST",
+    path: "/v1/entities/:entity/rows",
+    protectionClass: "service-or-tenant-api-key",
+    note: "Creates entity rows only after x-api-key or bearer service-token authentication.",
+  },
+  {
+    method: "PATCH",
+    path: "/v1/entities/:entity/rows/:rowId",
+    protectionClass: "service-or-tenant-api-key",
+    note: "Updates entity rows only after x-api-key or bearer service-token authentication.",
+  },
+  {
+    method: "DELETE",
+    path: "/v1/entities/:entity/rows/:rowId",
+    protectionClass: "service-or-tenant-api-key",
+    note: "Deletes entity rows only after x-api-key or bearer service-token authentication.",
+  },
+  {
+    method: "POST",
+    path: "/v1/relationships",
+    protectionClass: "service-or-tenant-api-key",
+    note: "Creates row relationships only after x-api-key or bearer service-token authentication.",
+  },
+  {
+    method: "DELETE",
+    path: "/v1/relationships/:id",
+    protectionClass: "service-or-tenant-api-key",
+    note: "Deletes row relationships only after x-api-key or bearer service-token authentication.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/context/revalidate",
+    protectionClass: "service-token",
+    note: "Workspace context revalidation requires a service token before tenant and user context is refreshed.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/invites",
+    protectionClass: "service-token",
+    note: "Workspace invite creation requires a service token before a tenant-scoped invitation is issued.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/invites/:inviteId/accept",
+    protectionClass: "service-token",
+    note: "Workspace invite acceptance requires a service token before the invite is consumed for membership state.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/invites/:inviteId/revoke",
+    protectionClass: "service-token",
+    note: "Workspace invite revocation requires a service token before tenant-scoped invite state is revoked.",
+  },
+  {
+    method: "POST",
+    path: "/v1/devices/register",
+    protectionClass: "service-token",
+    note: "Mobile push-token registration requires a service-token bearer credential carrying user identity; body userId + tenantId are reconciled against the authenticated principal downstream.",
+  },
+  {
+    method: "POST",
+    path: "/v1/devices/revoke",
+    protectionClass: "service-token",
+    note: "Mobile push-token revocation requires a service-token bearer credential; revokes by token hash without echoing the raw token.",
+  },
+];
+
+function routeKey(route: MutatingRoute) {
+  return `${route.method} ${route.path}`;
+}
+
+function discoverMutatingRoutes(routes: readonly ApiRoute[]): MutatingRoute[] {
+  return routes
+    .filter((route): route is ApiRoute & { method: MutatingMethod } => ["POST", "PATCH", "DELETE"].includes(route.method))
+    .filter((route) => route.path.startsWith("/v1"))
+    .map(({ method, path }) => ({ method, path }));
+}
+
+describe("API mutating route CSRF/auth inventory", () => {
+  let discoveredRoutes: MutatingRoute[];
+
+  beforeAll(async () => {
+    const scanResult = await scanApiRoutesFromFile(join(dirname(fileURLToPath(import.meta.url)), "app.ts"));
+    discoveredRoutes = discoverMutatingRoutes(scanResult.routes);
+  });
+
+  test("classifies every mutating v1 route and has no stale inventory entries", () => {
+    const discoveredKeys = new Set(discoveredRoutes.map(routeKey));
+    const inventoryKeys = new Set(expectedInventory.map(routeKey));
+
+    const missingFromInventory = discoveredRoutes.map(routeKey).filter((key) => !inventoryKeys.has(key));
+    const staleInventoryEntries = expectedInventory.map(routeKey).filter((key) => !discoveredKeys.has(key));
+
+    expect(missingFromInventory, `Unclassified mutating API routes:\n${missingFromInventory.join("\n")}`).toEqual([]);
+    expect(staleInventoryEntries, `Stale mutating API inventory entries:\n${staleInventoryEntries.join("\n")}`).toEqual([]);
+  });
+
+  test("uses an allowed protection class and note for every inventory entry", () => {
+    const malformedEntries = expectedInventory
+      .filter((entry) => !allowedProtectionClasses.has(entry.protectionClass) || entry.note.trim().length === 0)
+      .map((entry) => `${routeKey(entry)} class=${entry.protectionClass} note=${JSON.stringify(entry.note)}`);
+
+    expect(malformedEntries, `Invalid mutating API inventory entries:\n${malformedEntries.join("\n")}`).toEqual([]);
+  });
+});

hivegrid-cloud/apps/api/src/email-sequences/provider-webhook-route.ts

@@ -0,0 +1,199 @@
+import type { Context } from "hono";
+import type {
+  EmailSequenceProviderWebhookResult,
+  MarkEmailSequenceDeliveryBouncedInput,
+} from "@toolctrl/db/email-sequences.server";
+import type { ApiEnv } from "../api-types.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { verifyMailjetWebhookAuth } from "../webhooks/mailjet-webhook-auth.js";
+
+// Mailjet event-webhook `event` values that represent a non-delivery. Both hard
+// and soft bounces arrive as `event: "bounce"` (distinguished by `hard_bounce`);
+// `blocked` and `spam` are also treated as bounces for sequence suppression.
+const MAILJET_BOUNCE_EVENT_TYPES = new Set(["bounce", "blocked", "spam"]);
+
+export type EmailSequenceProviderWebhookDependencies = {
+  webhookSecret: string | null | undefined;
+  markEmailSequenceDeliveryBounced(input: MarkEmailSequenceDeliveryBouncedInput): Promise<EmailSequenceProviderWebhookResult>;
+};
+
+type ParsedMailjetSequenceWebhook =
+  | {
+      ok: true;
+      action: "ignore";
+      eventType: string;
+    }
+  | {
+      ok: true;
+      action: "bounce";
+      eventType: string;
+      providerMessageId: string;
+      bouncedAt: Date;
+      errorMessage: string;
+    }
+  | {
+      ok: false;
+      error: string;
+    };
+
+export async function postMailjetEmailSequenceWebhookRequest(
+  c: Context<ApiEnv>,
+  deps: EmailSequenceProviderWebhookDependencies
+) {
+  const authResult = verifyMailjetWebhookAuth({
+    secret: deps.webhookSecret,
+    headers: c.req.header(),
+  });
+
+  if (!authResult.ok) {
+    if (authResult.reason === "missing-secret") {
+      return apiError(c, 500, "webhook_signing_not_configured", "Mailjet webhook authentication is not configured.");
+    }
+    return apiError(c, 401, "invalid_webhook_signature", `Invalid Mailjet webhook credentials: ${authResult.reason}.`);
+  }
+
+  const rawBody = await c.req.raw.text();
+  const payloadResult = parseMailjetSequenceWebhookPayload(rawBody);
+  if (!payloadResult.ok) {
+    return apiError(c, 400, "invalid_request", payloadResult.error);
+  }
+  if (payloadResult.action === "ignore") {
+    return c.json({ accepted: true, status: "ignored" }, 202);
+  }
+
+  try {
+    const result = await deps.markEmailSequenceDeliveryBounced({
+      providerMessageId: payloadResult.providerMessageId,
+      bouncedAt: payloadResult.bouncedAt,
+      errorMessage: payloadResult.errorMessage,
+    });
+    if (result.status === "unknown_delivery") {
+      return c.json({ accepted: true, status: "ignored" }, 202);
+    }
+    return c.json({ accepted: true, status: result.status }, 202);
+  } catch {
+    return apiError(c, 500, "email_sequence_webhook_failed", "Email sequence provider webhook could not be processed.");
+  }
+}
+
+export function parseMailjetSequenceWebhookPayload(rawBody: string, now: Date = new Date()): ParsedMailjetSequenceWebhook {
+  let payload: unknown;
+  try {
+    payload = JSON.parse(rawBody) as unknown;
+  } catch {
+    return { ok: false, error: "Mailjet sequence webhook payload must be valid JSON." };
+  }
+  // Mailjet delivers one event object per POST (event grouping must stay off);
+  // if a batched array arrives, fold the first event so a single bounce still
+  // reconciles instead of being silently dropped.
+  if (Array.isArray(payload)) {
+    payload = payload[0];
+  }
+  if (!isRecord(payload)) {
+    return { ok: false, error: "Mailjet sequence webhook payload must be an object." };
+  }
+
+  const eventType = normalizeEventType(readFirstStringAt(payload, [["event"], ["type"], ["event_type"], ["data", "event"], ["data", "type"], ["data", "event_type"]]));
+  if (!eventType || !isBounceEventType(eventType)) {
+    return { ok: true, action: "ignore", eventType: eventType ?? "unknown" };
+  }
+
+  const providerMessageId = readFirstStringAt(payload, [
+    ["Message_GUID"],
+    ["MessageUUID"],
+    ["message_guid"],
+    ["messageUuid"],
+    ["MessageID"],
+    ["messageId"],
+    ["message_id"],
+    ["providerMessageId"],
+    ["provider_message_id"],
+    ["data", "Message_GUID"],
+    ["data", "MessageUUID"],
+    ["data", "MessageID"],
+    ["data", "messageId"],
+    ["data", "message_id"],
+  ]);
+  if (!providerMessageId) {
+    return { ok: false, error: "Mailjet sequence bounce payload is missing a provider message id." };
+  }
+
+  const timestamp = readFirstStringAt(payload, [
+    ["time"],
+    ["created_at"],
+    ["timestamp"],
+    ["occurred_at"],
+    ["data", "time"],
+    ["data", "created_at"],
+    ["data", "timestamp"],
+  ]);
+  const bouncedAt = parseProviderTimestamp(timestamp) ?? now;
+  const reason = readFirstStringAt(payload, [
+    ["error"],
+    ["error_related_to"],
+    ["comment"],
+    ["reason"],
+    ["description"],
+    ["data", "error"],
+    ["data", "error_related_to"],
+    ["data", "comment"],
+    ["data", "reason"],
+  ]);
+
+  return {
+    ok: true,
+    action: "bounce",
+    eventType,
+    providerMessageId,
+    bouncedAt,
+    errorMessage: reason ? `${eventType}: ${reason}` : eventType,
+  };
+}
+
+function isBounceEventType(eventType: string) {
+  return MAILJET_BOUNCE_EVENT_TYPES.has(eventType);
+}
+
+function normalizeEventType(value: string | undefined) {
+  return value?.trim().toLowerCase().replace(/[\s-]+/g, "_");
+}
+
+function parseProviderTimestamp(value: string | undefined) {
+  if (!value) {
+    return null;
+  }
+  const numeric = Number(value);
+  if (Number.isFinite(numeric) && numeric > 0) {
+    return new Date(numeric > 10_000_000_000 ? numeric : numeric * 1000);
+  }
+  const parsed = new Date(value);
+  return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+
+function readFirstStringAt(record: Record<string, unknown>, paths: string[][]) {
+  for (const path of paths) {
+    const value = readPath(record, path);
+    if (typeof value === "string" && value.trim()) {
+      return value.trim();
+    }
+    if (typeof value === "number" && Number.isFinite(value)) {
+      return String(value);
+    }
+  }
+  return undefined;
+}
+
+function readPath(record: Record<string, unknown>, path: string[]): unknown {
+  let current: unknown = record;
+  for (const key of path) {
+    if (!isRecord(current)) {
+      return undefined;
+    }
+    current = current[key];
+  }
+  return current;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}

hivegrid-cloud/apps/api/src/entity-row-plugin-hooks.test.ts

@@ -0,0 +1,227 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+import type { ApiConfig } from "@toolctrl/config";
+
+const emitPluginHookFireAndForget = vi.fn().mockResolvedValue(undefined);
+
+vi.mock("./plugin-hook-publisher.js", () => ({
+  emitPluginHookFireAndForget,
+}));
+
+const { app, configureApiAppDependenciesForTests } = await import("./app.js");
+const { resetApiRuntimeConfigForTests, setApiRuntimeConfigForTests } = await import("./config.js");
+const { PLUGIN_HOOK_ENTITY_CREATED, PLUGIN_HOOK_ENTITY_DELETED, PLUGIN_HOOK_ENTITY_UPDATED } = await import("@toolctrl/plugin-runtime");
+
+const testApp: Pick<typeof app, "request"> = {
+  request: ((input, init, ...rest) => app.request(input, addTestContentLength(init), ...rest)) as typeof app.request,
+};
+
+const TEST_API_CONFIG: ApiConfig = {
+  appName: "Test API",
+  nodeEnv: "test",
+  revision: "unknown",
+  listen: { port: 3001 },
+  routing: {
+    appBaseDomain: "localhost:3000",
+    webAppHost: "app.localhost:3000",
+    adminAppHost: "admin.localhost:3000",
+    apiHost: "api.localhost:3000",
+    protocol: "http",
+  },
+  origins: {},
+  apiAccessToken: "service-secret",
+  jwtSecret: "test-jwt-secret",
+  analytics: {},
+  mailjet: {},
+  stripe: {},
+  storage: {},
+  apiKeys: { allowLegacyPlaintextFallback: true },
+};
+
+let resetDependencies: (() => void) | undefined;
+
+afterEach(() => {
+  resetDependencies?.();
+  resetDependencies = undefined;
+  resetApiRuntimeConfigForTests();
+  emitPluginHookFireAndForget.mockClear();
+});
+
+describe("API entity row plugin hooks", () => {
+  test("emits minimal entity CRUD hooks after successful mutations without replay duplicates", async () => {
+    setApiRuntimeConfigForTests(TEST_API_CONFIG);
+    const entity = createLegacyEntityFixture();
+    const createLegacyEntityRow = vi.fn().mockResolvedValue(createLegacyRowFixture("row_created"));
+    const updateLegacyEntityRow = vi.fn().mockResolvedValue(createLegacyRowFixture("row_updated"));
+    const deleteLegacyEntityRow = vi.fn().mockResolvedValue("row_deleted");
+    resetDependencies = configureApiAppDependenciesForTests({
+      findLegacyEntityByName: vi.fn().mockResolvedValue(entity),
+      createLegacyEntityRow,
+      updateLegacyEntityRow,
+      deleteLegacyEntityRow,
+    });
+
+    const createResponse = await testApp.request("/v1/entities/contact/rows?tenantId=tenant_1", {
+      method: "POST",
+      headers: {
+        authorization: "Bearer service-secret",
+        "content-type": "application/json",
+        "idempotency-key": "hook-create-key",
+      },
+      body: JSON.stringify({ firstName: "Ada", secretNote: "do-not-emit" }),
+    });
+    const replayCreate = await testApp.request("/v1/entities/contact/rows?tenantId=tenant_1", {
+      method: "POST",
+      headers: {
+        authorization: "Bearer service-secret",
+        "content-type": "application/json",
+        "idempotency-key": "hook-create-key",
+      },
+      body: JSON.stringify({ firstName: "Ada", secretNote: "do-not-emit" }),
+    });
+
+    const updateResponse = await testApp.request("/v1/entities/contact/rows/row_updated?tenantId=tenant_1", {
+      method: "PATCH",
+      headers: {
+        authorization: "Bearer service-secret",
+        "content-type": "application/json",
+        "idempotency-key": "hook-update-key",
+      },
+      body: JSON.stringify({ firstName: "Grace", privateMemo: "do-not-emit" }),
+    });
+    const replayUpdate = await testApp.request("/v1/entities/contact/rows/row_updated?tenantId=tenant_1", {
+      method: "PATCH",
+      headers: {
+        authorization: "Bearer service-secret",
+        "content-type": "application/json",
+        "idempotency-key": "hook-update-key",
+      },
+      body: JSON.stringify({ firstName: "Grace", privateMemo: "do-not-emit" }),
+    });
+
+    const deleteResponse = await testApp.request("/v1/entities/contact/rows/row_deleted?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: {
+        authorization: "Bearer service-secret",
+        "idempotency-key": "hook-delete-key",
+      },
+    });
+    const replayDelete = await testApp.request("/v1/entities/contact/rows/row_deleted?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: {
+        authorization: "Bearer service-secret",
+        "idempotency-key": "hook-delete-key",
+      },
+    });
+
+    expect(createResponse.status).toBe(201);
+    expect(replayCreate.status).toBe(201);
+    expect(updateResponse.status).toBe(200);
+    expect(replayUpdate.status).toBe(200);
+    expect(deleteResponse.status).toBe(200);
+    expect(replayDelete.status).toBe(200);
+    expect(createLegacyEntityRow).toHaveBeenCalledTimes(1);
+    expect(updateLegacyEntityRow).toHaveBeenCalledTimes(1);
+    expect(deleteLegacyEntityRow).toHaveBeenCalledTimes(1);
+    expect(emitPluginHookFireAndForget).toHaveBeenCalledTimes(3);
+    expect(emitPluginHookFireAndForget).toHaveBeenNthCalledWith(
+      1,
+      PLUGIN_HOOK_ENTITY_CREATED,
+      expect.objectContaining({
+        entityType: "contact",
+        entityId: "row_created",
+        tenantId: "tenant_1",
+        userId: null,
+      }),
+      { tenantId: "tenant_1", dispatchKey: "entity:contact:row_created:created" },
+    );
+    expect(emitPluginHookFireAndForget).toHaveBeenNthCalledWith(
+      2,
+      PLUGIN_HOOK_ENTITY_UPDATED,
+      expect.objectContaining({
+        entityType: "contact",
+        entityId: "row_updated",
+        tenantId: "tenant_1",
+        userId: null,
+        changedFields: ["firstName", "privateMemo"],
+      }),
+      { tenantId: "tenant_1" },
+    );
+    expect(emitPluginHookFireAndForget).toHaveBeenNthCalledWith(
+      3,
+      PLUGIN_HOOK_ENTITY_DELETED,
+      expect.objectContaining({
+        entityType: "contact",
+        entityId: "row_deleted",
+        tenantId: "tenant_1",
+        userId: null,
+      }),
+      { tenantId: "tenant_1", dispatchKey: "entity:contact:row_deleted:deleted" },
+    );
+
+    for (const [, payload] of emitPluginHookFireAndForget.mock.calls) {
+      expect(payload).not.toHaveProperty("firstName");
+      expect(payload).not.toHaveProperty("secretNote");
+      expect(payload).not.toHaveProperty("privateMemo");
+      expect(payload).toHaveProperty("occurredAt", expect.any(String));
+    }
+  });
+});
+
+function addTestContentLength<T extends RequestInit | undefined>(init: T): T {
+  if (!init?.body) {
+    return init;
+  }
+
+  const headers = new Headers(init.headers);
+  if (headers.has("content-length")) {
+    return init;
+  }
+
+  const length = getTestBodyLength(init.body);
+  if (length === null) {
+    return init;
+  }
+
+  headers.set("content-length", String(length));
+  return { ...init, headers } as T;
+}
+
+function getTestBodyLength(body: BodyInit): number | null {
+  if (typeof body === "string") {
+    return Buffer.byteLength(body);
+  }
+  if (body instanceof URLSearchParams) {
+    return Buffer.byteLength(body.toString());
+  }
+  if (body instanceof Blob) {
+    return body.size;
+  }
+  return null;
+}
+
+function createLegacyEntityFixture() {
+  return {
+    id: "entity_1",
+    name: "contact",
+    title: "Contact",
+    properties: [],
+    views: [],
+    tags: [],
+    parentEntities: [],
+    childEntities: [],
+    templates: [],
+  };
+}
+
+function createLegacyRowFixture(id: string) {
+  return {
+    id,
+    entityId: "entity_1",
+    tenantId: "tenant_1",
+    createdAt: "2026-01-01T00:00:00.000Z",
+    updatedAt: "2026-01-01T00:00:00.000Z",
+    values: [],
+    parentRows: [],
+    childRows: [],
+  };
+}

hivegrid-cloud/apps/api/src/export-progress.test.ts

@@ -0,0 +1,53 @@
+import { createSharedStore } from "@toolctrl/shared-store";
+import { beforeEach, describe, expect, it } from "vitest";
+
+import { readExportProgressFromSharedStore } from "./export-progress.js";
+import { __resetSharedStoreForTests, __setSharedStoreFactoryForTests } from "./shared-store.js";
+
+describe("readExportProgressFromSharedStore", () => {
+  beforeEach(() => {
+    const store = createSharedStore({ namespace: "test-export-progress" });
+    __setSharedStoreFactoryForTests(() => store);
+    __resetSharedStoreForTests();
+  });
+
+  it("returns null when no progress entry has been published", async () => {
+    const snapshot = await readExportProgressFromSharedStore({ exportJobId: "missing", tenantId: "t" });
+    expect(snapshot).toBeNull();
+  });
+
+  it("decodes the worker payload schema", async () => {
+    const { getSharedStore } = await import("./shared-store.js");
+    const store = getSharedStore();
+    await store.set(
+      "export:progress:job-1",
+      JSON.stringify({ processedRows: 4500, totalEstimate: 12000, updatedAt: "2026-05-17T10:00:00.000Z" }),
+      { ttlSeconds: 60 },
+    );
+
+    const snapshot = await readExportProgressFromSharedStore({ exportJobId: "job-1", tenantId: "t" });
+    expect(snapshot).toEqual({
+      processedRows: 4500,
+      totalEstimate: 12000,
+      updatedAt: "2026-05-17T10:00:00.000Z",
+    });
+  });
+
+  it("returns null when stored payload is malformed", async () => {
+    const { getSharedStore } = await import("./shared-store.js");
+    await getSharedStore().set("export:progress:job-bad", "{not-json", { ttlSeconds: 60 });
+    const snapshot = await readExportProgressFromSharedStore({ exportJobId: "job-bad", tenantId: "t" });
+    expect(snapshot).toBeNull();
+  });
+
+  it("normalises missing optional fields", async () => {
+    const { getSharedStore } = await import("./shared-store.js");
+    await getSharedStore().set(
+      "export:progress:job-partial",
+      JSON.stringify({ processedRows: 10 }),
+      { ttlSeconds: 60 },
+    );
+    const snapshot = await readExportProgressFromSharedStore({ exportJobId: "job-partial", tenantId: "t" });
+    expect(snapshot).toEqual({ processedRows: 10, totalEstimate: null, updatedAt: null });
+  });
+});

hivegrid-cloud/apps/api/src/export-progress.ts

@@ -0,0 +1,35 @@
+import { getSharedStore } from "./shared-store.js";
+
+export interface ExportProgressSnapshot {
+  processedRows: number;
+  totalEstimate: number | null;
+  updatedAt: string | null;
+}
+
+/**
+ * Reads the latest export progress published by the worker job into the
+ * shared store. The worker writes `export:progress:<jobId>` every N rows
+ * (default 5_000) with a 1h TTL; the API surfaces that snapshot to the
+ * front-end so users can poll for a row-count counter without going to
+ * Postgres.
+ *
+ * Returns `null` when no progress entry exists (job not started, expired,
+ * or never wrote any rows). Returns malformed payloads as `null` to keep
+ * polling resilient against future schema changes.
+ */
+export async function readExportProgressFromSharedStore(args: { exportJobId: string; tenantId: string }): Promise<ExportProgressSnapshot | null> {
+  void args.tenantId; // tenant scoping is enforced by the caller; key is global per jobId.
+  const raw = await getSharedStore().get(`export:progress:${args.exportJobId}`);
+  if (!raw) return null;
+  try {
+    const parsed = JSON.parse(raw) as Partial<ExportProgressSnapshot>;
+    if (typeof parsed.processedRows !== "number") return null;
+    return {
+      processedRows: parsed.processedRows,
+      totalEstimate: typeof parsed.totalEstimate === "number" ? parsed.totalEstimate : null,
+      updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : null,
+    };
+  } catch {
+    return null;
+  }
+}

hivegrid-cloud/apps/api/src/exportRetentionQueue.test.ts

@@ -0,0 +1,138 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+import { queueExportArtifactRetention } from "./exportRetentionQueue.js";
+import { queueNames } from "@toolctrl/queue";
+
+const globalMockStore = globalThis as typeof globalThis & {
+  __exportRetentionQueueTestMocks__?: {
+    createQueue: ReturnType<typeof vi.fn>;
+  };
+};
+
+globalMockStore.__exportRetentionQueueTestMocks__ ??= {
+  createQueue: vi.fn(),
+};
+
+const mocks = globalMockStore.__exportRetentionQueueTestMocks__;
+
+vi.mock("@toolctrl/queue/runtime", () => ({
+  createQueue: mocks.createQueue,
+}));
+
+describe("queueExportArtifactRetention", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  test("enqueues expired export artifact cleanup on the maintenance queue and closes owned queues", async () => {
+    const queue = {
+      add: vi.fn().mockResolvedValue({ id: "retention_job_1" }),
+      close: vi.fn().mockResolvedValue(undefined),
+    };
+    mocks.createQueue.mockReturnValueOnce(queue);
+
+    await expect(
+      queueExportArtifactRetention({
+        requestId: "req_retention_1",
+        idempotencyKey: "retention-key-1",
+        take: 25,
+        env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0", EXPORT_RETENTION_QUEUE_ENABLED: "true" },
+      }),
+    ).resolves.toEqual({ enqueued: true, job: { id: "retention_job_1" } });
+
+    expect(mocks.createQueue).toHaveBeenCalledWith(queueNames.maintenance, {
+      connection: expect.objectContaining({ host: "localhost", port: 6379 }),
+    });
+    expect(queue.add).toHaveBeenCalledWith(
+      "run-cleanup",
+      {
+        kind: "cleanup.run",
+        task: "expired-export-artifacts",
+        take: 25,
+        metadata: { requestId: "req_retention_1", idempotencyKey: "retention-key-1" },
+      },
+      // Maintenance cleanup is pinned to CRITICAL (priority=1) via
+      // tenantId: null so it never trails user work.
+      expect.objectContaining({ jobId: "retention-key-1", priority: 1 }),
+    );
+    expect(queue.close).toHaveBeenCalled();
+  });
+
+  test("keeps request IDs as metadata when no idempotency key is provided", async () => {
+    const queue = {
+      add: vi.fn().mockResolvedValue({ id: "retention_job_2" }),
+      close: vi.fn().mockResolvedValue(undefined),
+    };
+
+    await expect(
+      queueExportArtifactRetention({
+        requestId: "req_retention_2",
+        take: 10,
+        queue,
+      }),
+    ).resolves.toEqual({ enqueued: true, job: { id: "retention_job_2" } });
+
+    expect(queue.add).toHaveBeenCalledWith(
+      "run-cleanup",
+      {
+        kind: "cleanup.run",
+        task: "expired-export-artifacts",
+        take: 10,
+        metadata: { requestId: "req_retention_2" },
+      },
+      expect.not.objectContaining({ jobId: expect.any(String) }),
+    );
+    expect(queue.close).not.toHaveBeenCalled();
+  });
+
+  test("returns missing connection without creating a queue", async () => {
+    await expect(queueExportArtifactRetention({ env: { EXPORT_RETENTION_QUEUE_ENABLED: "true" } })).resolves.toEqual({
+      enqueued: false,
+      reason: "missing-redis-connection",
+    });
+
+    expect(mocks.createQueue).not.toHaveBeenCalled();
+  });
+
+  test("does not enqueue unless the export retention worker is explicitly enabled", async () => {
+    await expect(queueExportArtifactRetention({ env: { BULLMQ_REDIS_URL: "redis://localhost:6379/0" } })).resolves.toEqual({
+      enqueued: false,
+      reason: "maintenance-worker-disabled",
+    });
+
+    expect(mocks.createQueue).not.toHaveBeenCalled();
+  });
+
+  test("falls back to process Redis env when no explicit env is provided", async () => {
+    const previousUrl = process.env.BULLMQ_REDIS_URL;
+    const previousEnabled = process.env.EXPORT_RETENTION_QUEUE_ENABLED;
+    process.env.BULLMQ_REDIS_URL = "redis://localhost:6380/2";
+    process.env.EXPORT_RETENTION_QUEUE_ENABLED = "true";
+    const queue = {
+      add: vi.fn().mockResolvedValue({ id: "retention_job_process_env" }),
+      close: vi.fn().mockResolvedValue(undefined),
+    };
+    mocks.createQueue.mockReturnValueOnce(queue);
+
+    try {
+      await expect(queueExportArtifactRetention({ requestId: "req_process_env" })).resolves.toEqual({
+        enqueued: true,
+        job: { id: "retention_job_process_env" },
+      });
+    } finally {
+      if (previousUrl === undefined) {
+        delete process.env.BULLMQ_REDIS_URL;
+      } else {
+        process.env.BULLMQ_REDIS_URL = previousUrl;
+      }
+      if (previousEnabled === undefined) {
+        delete process.env.EXPORT_RETENTION_QUEUE_ENABLED;
+      } else {
+        process.env.EXPORT_RETENTION_QUEUE_ENABLED = previousEnabled;
+      }
+    }
+
+    expect(mocks.createQueue).toHaveBeenCalledWith(queueNames.maintenance, {
+      connection: expect.objectContaining({ host: "localhost", port: 6380, db: 2 }),
+    });
+  });
+});

hivegrid-cloud/apps/api/src/exportRetentionQueue.ts

@@ -0,0 +1,102 @@
+import {
+  buildCleanupJob,
+  enqueueJob,
+  queueNames,
+  resolveRedisConnectionOptions,
+  type RedisConnectionEnv,
+  type TypedQueue,
+} from "@toolctrl/queue";
+
+type MaintenanceQueue = TypedQueue<typeof queueNames.maintenance>;
+type ExportRetentionQueueEnv = RedisConnectionEnv & {
+  EXPORT_RETENTION_QUEUE_ENABLED?: string;
+};
+
+export type QueueExportArtifactRetentionInput = {
+  requestId?: string;
+  idempotencyKey?: string;
+  take?: number;
+  env?: ExportRetentionQueueEnv;
+  queue?: MaintenanceQueue;
+};
+
+export type QueueExportArtifactRetentionResult =
+  | {
+      enqueued: true;
+      job: Awaited<ReturnType<MaintenanceQueue["add"]>>;
+    }
+  | {
+      enqueued: false;
+      reason: "maintenance-worker-disabled" | "missing-redis-connection";
+    };
+
+export async function queueExportArtifactRetention({
+  requestId,
+  idempotencyKey,
+  take,
+  env,
+  queue,
+}: QueueExportArtifactRetentionInput): Promise<QueueExportArtifactRetentionResult> {
+  const effectiveEnv = env ?? (process.env as ExportRetentionQueueEnv);
+  if (!queue && effectiveEnv.EXPORT_RETENTION_QUEUE_ENABLED !== "true") {
+    return {
+      enqueued: false,
+      reason: "maintenance-worker-disabled",
+    };
+  }
+
+  const ownedQueue = queue ? undefined : await createInternalQueue(effectiveEnv);
+
+  if (!queue && !ownedQueue) {
+    return {
+      enqueued: false,
+      reason: "missing-redis-connection",
+    };
+  }
+
+  const targetQueue = queue ?? ownedQueue;
+
+  if (!targetQueue) {
+    return {
+      enqueued: false,
+      reason: "missing-redis-connection",
+    };
+  }
+
+  try {
+    const metadata = {
+      ...(requestId ? { requestId } : {}),
+      ...(idempotencyKey ? { idempotencyKey } : {}),
+    };
+    const descriptor = buildCleanupJob({
+      task: "expired-export-artifacts",
+      take,
+      metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+    });
+    const job = await enqueueJob(targetQueue, descriptor, {
+      ...(idempotencyKey ? { idempotencyKey } : {}),
+      // Maintenance cleanup is internal — never demote behind user work.
+      tenantId: null,
+    });
+
+    return {
+      enqueued: true,
+      job,
+    };
+  } finally {
+    if (ownedQueue) {
+      await ownedQueue.close();
+    }
+  }
+}
+
+async function createInternalQueue(env?: ExportRetentionQueueEnv): Promise<MaintenanceQueue | undefined> {
+  const connection = resolveRedisConnectionOptions({ env: env ?? (process.env as RedisConnectionEnv) });
+
+  if (!connection) {
+    return undefined;
+  }
+
+  const { createQueue } = await import("@toolctrl/queue/runtime");
+  return createQueue(queueNames.maintenance, { connection });
+}

hivegrid-cloud/apps/api/src/idempotency/shared-store-client.test.ts

@@ -0,0 +1,259 @@
+import { describe, expect, it } from "vitest";
+import { MemoryAdapter, createSharedStore } from "@toolctrl/shared-store";
+import { createSharedStoreIdempotencyStorageClient } from "./shared-store-client.js";
+import { createIdempotencyStore } from "./store.js";
+
+function makeClient(now?: () => number) {
+  const store = createSharedStore({ namespace: "api", adapter: new MemoryAdapter() });
+  return { client: createSharedStoreIdempotencyStorageClient({ store, now }), store };
+}
+
+const SCOPE = "api:tenant:t1";
+
+/**
+ * `IdempotencyStorageClient.findUnique` is typed via an intersection of
+ * the base Prisma contract and an extended delegate; TS resolves the
+ * call to the base overload, which omits `createdAt`. The SharedStore
+ * client always populates `createdAt`, so narrow the result here to read
+ * it in the assertions below.
+ */
+type WithCreatedAt = { createdAt?: Date | null; expiresAt?: Date | null };
+
+describe("createSharedStoreIdempotencyStorageClient — Prisma-shaped contract", () => {
+  it("round-trips a created entry through findUnique with Date fields", async () => {
+    const { client } = makeClient(() => 1_000);
+    await client.idempotencyKey.create({
+      data: {
+        scope: SCOPE,
+        key: "k1",
+        requestHash: "hash-1",
+        responseStatus: 201,
+        responseBody: { ok: true },
+        responseHeaders: { "x-test": "1" },
+        expiresAt: new Date(1_000 + 60_000),
+      },
+    });
+
+    const found = await client.idempotencyKey.findUnique({
+      where: { idempotency_key_scope_key: { scope: SCOPE, key: "k1" } },
+    });
+
+    expect(found).not.toBeNull();
+    expect(found?.requestHash).toBe("hash-1");
+    expect(found?.responseStatus).toBe(201);
+    expect(found?.responseBody).toEqual({ ok: true });
+    expect(found?.responseHeaders).toEqual({ "x-test": "1" });
+    const withDates = found as WithCreatedAt | null;
+    expect(withDates?.createdAt).toBeInstanceOf(Date);
+    expect(withDates?.expiresAt?.getTime()).toBe(61_000);
+  });
+
+  it("returns null for an unknown key", async () => {
+    const { client } = makeClient();
+    const found = await client.idempotencyKey.findUnique({
+      where: { idempotency_key_scope_key: { scope: SCOPE, key: "missing" } },
+    });
+    expect(found).toBeNull();
+  });
+
+  it("throws a Prisma-style P2002 when the same scope+key is claimed twice", async () => {
+    const { client } = makeClient();
+    const data = {
+      scope: SCOPE,
+      key: "dup",
+      requestHash: "h",
+      responseStatus: 0,
+      responseBody: null,
+      responseHeaders: null,
+      expiresAt: new Date(Date.now() + 60_000),
+    };
+    await client.idempotencyKey.create({ data });
+
+    await expect(client.idempotencyKey.create({ data })).rejects.toMatchObject({ code: "P2002" });
+  });
+
+  it("update overwrites the response and preserves createdAt", async () => {
+    const { client } = makeClient(() => 5_000);
+    await client.idempotencyKey.create({
+      data: {
+        scope: SCOPE,
+        key: "k2",
+        requestHash: "h",
+        responseStatus: 0,
+        responseBody: null,
+        responseHeaders: null,
+        expiresAt: new Date(5_000 + 60_000),
+      },
+    });
+    const before = await client.idempotencyKey.findUnique({
+      where: { idempotency_key_scope_key: { scope: SCOPE, key: "k2" } },
+    });
+
+    await client.idempotencyKey.update({
+      where: { idempotency_key_scope_key: { scope: SCOPE, key: "k2" } },
+      data: { responseStatus: 200, responseBody: { done: true }, responseHeaders: { h: "v" } },
+    });
+    const after = await client.idempotencyKey.findUnique({
+      where: { idempotency_key_scope_key: { scope: SCOPE, key: "k2" } },
+    });
+
+    expect(after?.responseStatus).toBe(200);
+    expect(after?.responseBody).toEqual({ done: true });
+    expect((after as WithCreatedAt | null)?.createdAt?.getTime()).toBe(
+      (before as WithCreatedAt | null)?.createdAt?.getTime(),
+    );
+  });
+
+  it("update throws when the reservation does not exist", async () => {
+    const { client } = makeClient();
+    await expect(
+      client.idempotencyKey.update({
+        where: { idempotency_key_scope_key: { scope: SCOPE, key: "ghost" } },
+        data: { responseStatus: 200 },
+      }),
+    ).rejects.toThrow(/not found/i);
+  });
+
+  it("deleteMany with scope+key removes exactly that entry", async () => {
+    const { client } = makeClient();
+    const base = {
+      scope: SCOPE,
+      requestHash: "h",
+      responseStatus: 200,
+      responseBody: null,
+      responseHeaders: null,
+      expiresAt: new Date(Date.now() + 60_000),
+    };
+    await client.idempotencyKey.create({ data: { ...base, key: "a" } });
+    await client.idempotencyKey.create({ data: { ...base, key: "b" } });
+
+    await client.idempotencyKey.deleteMany?.({ where: { scope: SCOPE, key: "a" } });
+
+    expect(
+      await client.idempotencyKey.findUnique({ where: { idempotency_key_scope_key: { scope: SCOPE, key: "a" } } }),
+    ).toBeNull();
+    expect(
+      await client.idempotencyKey.findUnique({ where: { idempotency_key_scope_key: { scope: SCOPE, key: "b" } } }),
+    ).not.toBeNull();
+  });
+
+  it("deleteMany with expiresAt.lte only removes entries already past their expiry", async () => {
+    const { client } = makeClient(() => 10_000);
+    await client.idempotencyKey.create({
+      data: {
+        scope: SCOPE,
+        key: "fresh",
+        requestHash: "h",
+        responseStatus: 200,
+        responseBody: null,
+        responseHeaders: null,
+        expiresAt: new Date(10_000 + 60_000),
+      },
+    });
+
+    // Cut-off earlier than the entry's expiry → entry is kept.
+    await client.idempotencyKey.deleteMany?.({
+      where: { scope: SCOPE, key: "fresh", expiresAt: { lte: new Date(20_000) } },
+    });
+
+    expect(
+      await client.idempotencyKey.findUnique({ where: { idempotency_key_scope_key: { scope: SCOPE, key: "fresh" } } }),
+    ).not.toBeNull();
+  });
+
+  it("deleteMany scoped to one scope leaves other scopes intact", async () => {
+    const { client } = makeClient();
+    const base = {
+      requestHash: "h",
+      responseStatus: 200,
+      responseBody: null,
+      responseHeaders: null,
+      expiresAt: new Date(Date.now() + 60_000),
+    };
+    await client.idempotencyKey.create({ data: { ...base, scope: "api:system", key: "x" } });
+    await client.idempotencyKey.create({ data: { ...base, scope: "api:tenant:other", key: "y" } });
+
+    await client.idempotencyKey.deleteMany?.({ where: { scope: "api:system" } });
+
+    expect(await client.idempotencyKey.count?.({ where: { scope: "api:system" } })).toBe(0);
+    expect(await client.idempotencyKey.count?.({ where: { scope: "api:tenant:other" } })).toBe(1);
+  });
+
+  it("count tallies only entries in the requested scope", async () => {
+    const { client } = makeClient();
+    const base = {
+      requestHash: "h",
+      responseStatus: 200,
+      responseBody: null,
+      responseHeaders: null,
+      expiresAt: new Date(Date.now() + 60_000),
+    };
+    await client.idempotencyKey.create({ data: { ...base, scope: SCOPE, key: "1" } });
+    await client.idempotencyKey.create({ data: { ...base, scope: SCOPE, key: "2" } });
+    await client.idempotencyKey.create({ data: { ...base, scope: "api:system", key: "3" } });
+
+    expect(await client.idempotencyKey.count?.({ where: { scope: SCOPE } })).toBe(2);
+  });
+});
+
+describe("createSharedStoreIdempotencyStorageClient — driven through createIdempotencyStore", () => {
+  it("stores then replays a successful operation", async () => {
+    const { client } = makeClient();
+    const idempotency = createIdempotencyStore({ client, scope: SCOPE });
+    const response = { status: 200, headers: {}, body: { value: 1 } };
+
+    const first = await idempotency.run("key-A", "fp-A", async () => response);
+    const second = await idempotency.run("key-A", "fp-A", async () => ({
+      status: 500,
+      headers: {},
+      body: { value: 999 },
+    }));
+
+    expect(first.type).toBe("stored");
+    expect(second.type).toBe("replay");
+    if (second.type === "replay") {
+      expect(second.response.body).toEqual({ value: 1 });
+    }
+  });
+
+  it("reports a conflict when the same key arrives with a different fingerprint", async () => {
+    const { client } = makeClient();
+    const idempotency = createIdempotencyStore({ client, scope: SCOPE });
+
+    await idempotency.run("key-B", "fp-1", async () => ({ status: 200, headers: {}, body: null }));
+    const conflict = await idempotency.run("key-B", "fp-2", async () => ({ status: 200, headers: {}, body: null }));
+
+    expect(conflict.type).toBe("conflict");
+  });
+
+  it("fans an invalidation out across two replicas sharing one adapter", async () => {
+    // Two SharedStore facades over one MemoryAdapter == two API replicas
+    // pointed at the same Redis. A claim on replica A must be visible to B.
+    const adapter = new MemoryAdapter();
+    const replicaA = createSharedStoreIdempotencyStorageClient({
+      store: createSharedStore({ namespace: "api", adapter }),
+    });
+    const replicaB = createSharedStoreIdempotencyStorageClient({
+      store: createSharedStore({ namespace: "api", adapter }),
+    });
+    const storeA = createIdempotencyStore({ client: replicaA, scope: SCOPE });
+    const storeB = createIdempotencyStore({ client: replicaB, scope: SCOPE });
+
+    const stored = await storeA.run("shared-key", "fp", async () => ({
+      status: 201,
+      headers: {},
+      body: { replica: "A" },
+    }));
+    const replayed = await storeB.run("shared-key", "fp", async () => ({
+      status: 500,
+      headers: {},
+      body: { replica: "B" },
+    }));
+
+    expect(stored.type).toBe("stored");
+    expect(replayed.type).toBe("replay");
+    if (replayed.type === "replay") {
+      expect(replayed.response.body).toEqual({ replica: "A" });
+    }
+  });
+});

hivegrid-cloud/apps/api/src/idempotency/shared-store-client.ts

@@ -0,0 +1,200 @@
+/**
+ * SharedStore-backed implementation of the Prisma-shaped
+ * `IdempotencyStorageClient` contract (Phase 03 #38, step 3).
+ *
+ * The idempotency layer (`createIdempotencyStore` + `withIdempotency`)
+ * speaks a narrow Prisma-delegate interface: `idempotencyKey.{findUnique,
+ * create, update, deleteMany, count}`. Production wires the real Prisma
+ * client; tests wire `createMemoryIdempotencyStorageClient` (an
+ * in-process Map). Neither is cross-replica-safe without Postgres.
+ *
+ * This module provides a third backend: the same contract on top of a
+ * `SharedStore` (Redis in production, MemoryAdapter in dev/test). It lets
+ * a deployment that already runs Redis get multi-replica idempotency
+ * without a Postgres round-trip, and without the in-process Map's
+ * single-process blind spot.
+ *
+ * It is opt-in — `app-dependencies.ts` selects it only when
+ * `IDEMPOTENCY_BACKEND=shared-store`; the default stays Prisma.
+ *
+ * Atomicity: `create` maps to `set(..., { nx: true })`. A `false` return
+ * (key already present) is surfaced as a Prisma-style `P2002` error so
+ * `withIdempotency` detects the concurrent claim exactly as it does with
+ * the real Prisma unique-constraint violation.
+ *
+ * Expiry: every entry is written with a Redis TTL derived from
+ * `expiresAt`, so expired reservations self-evict. The `expiresAt` field
+ * is also stored in the value so the contract's explicit expiry checks
+ * (`deleteMany({ expiresAt: { lte } })`) keep working against the
+ * MemoryAdapter, which honors TTL lazily on read.
+ */
+import type { SharedStore } from "@toolctrl/shared-store";
+import type { IdempotencyStorageClient } from "./store.js";
+
+type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue };
+
+type PersistedEntry = {
+  scope: string;
+  key: string;
+  requestHash: string;
+  responseStatus: number;
+  responseBody: JsonValue | null;
+  responseHeaders: Record<string, string> | null;
+  createdAt: number;
+  expiresAt: number | null;
+};
+
+const KEY_PREFIX = "idem";
+
+/** Hex-encode so scope/key cannot contain the `:` delimiter or SharedStore-forbidden chars. */
+function hex(value: string): string {
+  return Buffer.from(value, "utf8").toString("hex");
+}
+
+function suffixFor(scope: string, key: string): string {
+  return `${KEY_PREFIX}:${hex(scope)}:${hex(key)}`;
+}
+
+function ttlSecondsUntil(expiresAt: number | null, now: () => number): number | undefined {
+  if (expiresAt === null) return undefined;
+  const seconds = Math.ceil((expiresAt - now()) / 1000);
+  return seconds >= 1 ? seconds : 1;
+}
+
+export type SharedStoreIdempotencyClientOptions = {
+  store: SharedStore;
+  now?: () => number;
+};
+
+export function createSharedStoreIdempotencyStorageClient(
+  options: SharedStoreIdempotencyClientOptions,
+): IdempotencyStorageClient {
+  const { store } = options;
+  const now = options.now ?? Date.now;
+
+  // The factory prepends a namespace prefix to every suffix. `scan`
+  // yields fully-prefixed keys, so derive the prefix once to strip it
+  // back to a suffix `get`/`del` will accept.
+  const prefix = store.key("x").slice(0, -1);
+  const stripPrefix = (fullKey: string): string =>
+    fullKey.startsWith(prefix) ? fullKey.slice(prefix.length) : fullKey;
+
+  const readEntry = async (suffix: string): Promise<PersistedEntry | null> => {
+    const raw = await store.get(suffix);
+    if (raw === null) return null;
+    try {
+      return JSON.parse(raw) as PersistedEntry;
+    } catch {
+      // A corrupt value is treated as absent — the next claim re-creates it.
+      return null;
+    }
+  };
+
+  async function* iterateEntries(): AsyncGenerator<{ suffix: string; entry: PersistedEntry }> {
+    for await (const fullKey of store.scan(`${KEY_PREFIX}:*`)) {
+      const suffix = stripPrefix(fullKey);
+      const entry = await readEntry(suffix);
+      if (entry) yield { suffix, entry };
+    }
+  }
+
+  return {
+    idempotencyKey: {
+      async findUnique({ where }) {
+        const { scope, key } = where.idempotency_key_scope_key;
+        const entry = await readEntry(suffixFor(scope, key));
+        if (!entry) return null;
+        return {
+          requestHash: entry.requestHash,
+          responseStatus: entry.responseStatus,
+          responseBody: entry.responseBody ?? null,
+          responseHeaders: entry.responseHeaders ?? null,
+          createdAt: entry.createdAt != null ? new Date(entry.createdAt) : null,
+          expiresAt: entry.expiresAt != null ? new Date(entry.expiresAt) : null,
+        };
+      },
+
+      async create({ data }) {
+        const expiresAt = data.expiresAt ? data.expiresAt.getTime() : null;
+        const entry: PersistedEntry = {
+          scope: data.scope,
+          key: data.key,
+          requestHash: data.requestHash,
+          responseStatus: data.responseStatus,
+          responseBody: (data.responseBody ?? null) as JsonValue | null,
+          responseHeaders: data.responseHeaders ?? null,
+          createdAt: now(),
+          expiresAt,
+        };
+        const ttlSeconds = ttlSecondsUntil(expiresAt, now);
+        const written = await store.set(suffixFor(data.scope, data.key), JSON.stringify(entry), {
+          nx: true,
+          ...(ttlSeconds !== undefined ? { ttlSeconds } : {}),
+        });
+        if (!written) {
+          // Mirror Prisma's unique-constraint violation so withIdempotency
+          // recognises the concurrent claim and replays instead of erroring.
+          throw Object.assign(new Error("Unique constraint failed on idempotency key"), { code: "P2002" });
+        }
+      },
+
+      async update({ where, data }) {
+        const { scope, key } = where.idempotency_key_scope_key;
+        const suffix = suffixFor(scope, key);
+        const existing = await readEntry(suffix);
+        if (!existing) {
+          throw new Error("Idempotency key reservation not found.");
+        }
+        const expiresAt = data.expiresAt ? data.expiresAt.getTime() : existing.expiresAt;
+        const next: PersistedEntry = {
+          ...existing,
+          responseStatus: data.responseStatus,
+          responseBody: (data.responseBody ?? null) as JsonValue | null,
+          responseHeaders: data.responseHeaders ?? null,
+          expiresAt,
+        };
+        const ttlSeconds = ttlSecondsUntil(expiresAt, now);
+        await store.set(suffix, JSON.stringify(next), ttlSeconds !== undefined ? { ttlSeconds } : undefined);
+      },
+
+      async deleteMany(args) {
+        const where = args?.where;
+
+        // Targeted single-key delete — no scan needed.
+        if (where?.scope && where?.key) {
+          const suffix = suffixFor(where.scope, where.key);
+          if (where.expiresAt?.lte) {
+            const entry = await readEntry(suffix);
+            const cutoff = where.expiresAt.lte.getTime();
+            if (entry && entry.expiresAt !== null && entry.expiresAt <= cutoff) {
+              await store.del(suffix);
+            }
+          } else {
+            await store.del(suffix);
+          }
+          return;
+        }
+
+        // Scope-wide (clear / prune) — enumerate. The entry set is bounded
+        // by TTL so this stays cheap; it is never on the request hot path.
+        const cutoff = where?.expiresAt?.lte ? where.expiresAt.lte.getTime() : null;
+        for await (const { suffix, entry } of iterateEntries()) {
+          if (where?.scope && entry.scope !== where.scope) continue;
+          if (where?.key && entry.key !== where.key) continue;
+          if (cutoff !== null && !(entry.expiresAt !== null && entry.expiresAt <= cutoff)) continue;
+          await store.del(suffix);
+        }
+      },
+
+      async count(args) {
+        const scope = args?.where?.scope;
+        let total = 0;
+        for await (const { entry } of iterateEntries()) {
+          if (scope && entry.scope !== scope) continue;
+          total += 1;
+        }
+        return total;
+      },
+    },
+  };
+}

hivegrid-cloud/apps/api/src/idempotency/store.test.ts

@@ -0,0 +1,238 @@
+import { describe, expect, test, vi } from "vitest";
+import { createApiIdempotencyScope, createIdempotencyStore, createMemoryIdempotencyStorageClient, createRequestFingerprint, normalizeIdempotencyKey, type StoredIdempotencyResponse } from "./store.js";
+
+const response: StoredIdempotencyResponse = {
+    status: 201,
+    headers: { "content-type": "application/json", "x-request-id": "req_123" },
+    body: { ok: true, nested: { count: 1 } },
+};
+
+function fingerprint(parsedBody: unknown = { name: "Ada" }) {
+    return createRequestFingerprint({
+        method: "post",
+        route: "/v1/:tenant/entities/:entityId/rows/:rowId",
+        tenantId: "tenant_1",
+        entityId: "entity_1",
+        rowId: "row_1",
+        parsedBody: parsedBody as never,
+    });
+}
+
+describe("idempotency store", () => {
+    test("stores the first successful response for a valid key", async () => {
+        const store = createIdempotencyStore({ now: () => 1000 });
+
+        await expect(store.recordSuccess(" key_1 ", fingerprint(), response)).resolves.toEqual({
+            type: "stored",
+            response,
+            createdAt: 1000,
+        });
+        await expect(store.size()).resolves.toBe(1);
+    });
+
+    test("replays the stored response for the same key and fingerprint", async () => {
+        const store = createIdempotencyStore({ now: () => 1000 });
+        const requestFingerprint = fingerprint();
+
+        await store.recordSuccess("key_1", requestFingerprint, response);
+
+        await expect(store.recordSuccess("key_1", requestFingerprint, { status: 500, headers: {}, body: { ok: false } })).resolves.toEqual({
+            type: "replay",
+            response,
+            createdAt: 1000,
+        });
+    });
+
+    test("rejects a repeated key with a changed body fingerprint", async () => {
+        const store = createIdempotencyStore({ now: () => 1000 });
+        const originalFingerprint = fingerprint({ name: "Ada" });
+        const changedFingerprint = fingerprint({ name: "Grace" });
+
+        await store.recordSuccess("key_1", originalFingerprint, response);
+
+        await expect(store.recordSuccess("key_1", changedFingerprint, response)).resolves.toEqual({
+            type: "conflict",
+            existingFingerprint: originalFingerprint,
+            receivedFingerprint: changedFingerprint,
+            createdAt: 1000,
+        });
+    });
+
+    test("isolates records by idempotency key", async () => {
+        const store = createIdempotencyStore();
+        const requestFingerprint = fingerprint();
+
+        await store.recordSuccess("key_1", requestFingerprint, response);
+
+        await expect(store.peek("key_2", requestFingerprint)).resolves.toEqual({ type: "miss" });
+    });
+
+    test("isolates API idempotency keys by tenant scope", async () => {
+        const client = createMemoryIdempotencyStorageClient();
+        const tenantOneStore = createIdempotencyStore({ client, scope: createApiIdempotencyScope({ tenantId: "tenant_1" }) });
+        const tenantTwoStore = createIdempotencyStore({ client, scope: createApiIdempotencyScope({ tenantId: "tenant_2" }) });
+        const requestFingerprint = createRequestFingerprint({
+            method: "POST",
+            route: "/v1/entities/:entity/rows",
+            tenantId: "same-shape",
+            parsedBody: { name: "Ada" },
+        });
+
+        const first = await tenantOneStore.run("shared-key", requestFingerprint, async () => ({
+            ...response,
+            body: { tenantId: "tenant_1" },
+        }));
+        const second = await tenantTwoStore.run("shared-key", requestFingerprint, async () => ({
+            ...response,
+            body: { tenantId: "tenant_2" },
+        }));
+
+        expect(first.type).toBe("stored");
+        expect(second.type).toBe("stored");
+        expect(first.type === "stored" ? first.response.body : null).toEqual({ tenantId: "tenant_1" });
+        expect(second.type === "stored" ? second.response.body : null).toEqual({ tenantId: "tenant_2" });
+    });
+
+    test("bypasses malformed or missing idempotency keys", async () => {
+        const store = createIdempotencyStore();
+        const requestFingerprint = fingerprint();
+
+        expect(normalizeIdempotencyKey(undefined)).toBeNull();
+        expect(normalizeIdempotencyKey("   ")).toBeNull();
+        expect(normalizeIdempotencyKey("bad\nkey")).toBeNull();
+        await expect(store.recordSuccess(undefined, requestFingerprint, response)).resolves.toEqual({ type: "bypass" });
+        await expect(store.recordSuccess("   ", requestFingerprint, response)).resolves.toEqual({ type: "bypass" });
+        await expect(store.size()).resolves.toBe(0);
+    });
+
+    test("prunes old records by max age", async () => {
+        let now = 1000;
+        const store = createIdempotencyStore({ maxAgeMs: 10, now: () => now });
+        const requestFingerprint = fingerprint();
+
+        await store.recordSuccess("key_1", requestFingerprint, response);
+        now = 1011;
+
+        await expect(store.peek("key_1", requestFingerprint)).resolves.toEqual({ type: "miss" });
+        await expect(store.size()).resolves.toBe(0);
+    });
+
+    test("caps growth by evicting the oldest entries when max entries is exceeded", async () => {
+        const store = createIdempotencyStore({ maxEntries: 1 });
+        const requestFingerprint = fingerprint();
+
+        await store.recordSuccess("key_1", requestFingerprint, response);
+        await store.recordSuccess("key_2", requestFingerprint, response);
+
+        await expect(store.peek("key_1", requestFingerprint)).resolves.toEqual({ type: "miss" });
+        await expect(store.peek("key_2", requestFingerprint)).resolves.toMatchObject({ type: "replay" });
+    });
+
+    test("does not fingerprint credential headers or raw request bodies", () => {
+        const baseInput = {
+            method: "post",
+            route: "/v1/:tenant/entities/:entityId/rows",
+            tenantId: "tenant_1",
+            entityId: "entity_1",
+            parsedBody: { safe: true },
+        };
+
+        const withoutSecrets = createRequestFingerprint(baseInput);
+        const withSecrets = createRequestFingerprint({
+            ...baseInput,
+            authorization: "Bearer secret-token",
+            "x-api-key": "secret-api-key",
+            rawBody: "plaintext secret body",
+        } as never);
+
+        expect(withSecrets).toBe(withoutSecrets);
+        expect(withSecrets).not.toContain("secret");
+    });
+
+    test("returns cloned response bodies instead of sharing mutable references", async () => {
+        const store = createIdempotencyStore();
+        const requestFingerprint = fingerprint();
+        const originalResponse: StoredIdempotencyResponse = {
+            status: 200,
+            headers: {},
+            body: { nested: { count: 1 } },
+        };
+
+        const stored = await store.recordSuccess("key_1", requestFingerprint, originalResponse);
+        originalResponse.body = { nested: { count: 99 } };
+        if (stored.type === "stored" && typeof stored.response.body === "object" && stored.response.body !== null && !Array.isArray(stored.response.body)) {
+            stored.response.body.nested = { count: 42 };
+        }
+
+        await expect(store.peek("key_1", requestFingerprint)).resolves.toMatchObject({
+            type: "replay",
+            response: { body: { nested: { count: 1 } } },
+        });
+    });
+
+    test("clear resets all stored idempotency records", async () => {
+        const store = createIdempotencyStore();
+        const requestFingerprint = fingerprint();
+
+        await store.recordSuccess("key_1", requestFingerprint, response);
+        await store.clear();
+
+        await expect(store.size()).resolves.toBe(0);
+        await expect(store.peek("key_1", requestFingerprint)).resolves.toEqual({ type: "miss" });
+    });
+
+    test("runs operations through a structural persisted idempotency client", async () => {
+        const client = createMemoryIdempotencyStorageClient({ now: () => 1000 });
+        const findUnique = vi.spyOn(client.idempotencyKey, "findUnique");
+        const create = vi.spyOn(client.idempotencyKey, "create");
+        const update = vi.spyOn(client.idempotencyKey, "update");
+        const store = createIdempotencyStore({ client, now: () => 1000 });
+        const operation = vi.fn(async () => response);
+        const requestFingerprint = fingerprint();
+
+        await expect(store.run("key_1", requestFingerprint, operation)).resolves.toMatchObject({ type: "stored", response });
+        await expect(store.run("key_1", requestFingerprint, operation)).resolves.toMatchObject({ type: "replay", response });
+
+        expect(operation).toHaveBeenCalledTimes(1);
+        expect(findUnique).toHaveBeenCalled();
+        expect(create).toHaveBeenCalledWith({
+            data: expect.objectContaining({
+                scope: "api",
+                key: "key_1",
+                responseStatus: 0,
+                responseBody: null,
+                responseHeaders: null,
+            }),
+        });
+        expect(update).toHaveBeenCalledWith({
+            where: { idempotency_key_scope_key: { scope: "api", key: "key_1" } },
+            data: expect.objectContaining({
+                responseStatus: response.status,
+                responseBody: response.body,
+                responseHeaders: response.headers,
+            }),
+        });
+    });
+
+    test("reports an in-flight database reservation without running the operation", async () => {
+        const client = createMemoryIdempotencyStorageClient({ now: () => 1000 });
+        const store = createIdempotencyStore({ client, now: () => 1000 });
+        const requestFingerprint = fingerprint();
+        const requestHash = JSON.stringify(requestFingerprint);
+        await client.idempotencyKey.create({
+            data: {
+                scope: "api",
+                key: "key_reserved",
+                requestHash,
+                responseStatus: 0,
+                responseBody: null,
+                responseHeaders: null,
+                expiresAt: new Date(2000),
+            },
+        });
+        const operation = vi.fn(async () => response);
+
+        await expect(store.run("key_reserved", requestFingerprint, operation)).resolves.toMatchObject({ type: "in_progress" });
+        expect(operation).not.toHaveBeenCalled();
+    });
+});

hivegrid-cloud/apps/api/src/idempotency/store.ts

@@ -0,0 +1,416 @@
+import { createHash } from "node:crypto";
+import { CatalogError, foundationErrors, withIdempotency, type IdempotencyClient } from "@toolctrl/foundation";
+
+export type IdempotencyKey = string;
+export type RequestFingerprint = string;
+
+export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue };
+
+export type StoredResponseStatus = number;
+export type StoredResponseHeaders = Record<string, string>;
+export type StoredResponseBody = JsonValue;
+
+export type StoredIdempotencyResponse = {
+  status: StoredResponseStatus;
+  headers: StoredResponseHeaders;
+  body: StoredResponseBody;
+};
+
+export type IdempotencyFingerprintInput = {
+  method: string;
+  route: string;
+  tenantId?: string | null;
+  entityId?: string | null;
+  rowId?: string | null;
+  relationshipId?: string | null;
+  parsedBody?: JsonValue;
+};
+
+export type IdempotencyStoreOptions = {
+  client?: IdempotencyStorageClient;
+  maxAgeMs?: number;
+  maxEntries?: number;
+  now?: () => number;
+  scope?: string;
+};
+
+type StoredEntry = {
+  requestHash: string;
+  responseStatus: number;
+  responseBody: JsonValue | null;
+  responseHeaders: Record<string, string> | null;
+  createdAt?: Date | null;
+  expiresAt?: Date | null;
+};
+
+type IdempotencyKeyDelegate = IdempotencyClient["idempotencyKey"] & {
+  findUnique(args: { where: { idempotency_key_scope_key: { scope: string; key: string } } }): Promise<StoredEntry | null>;
+  deleteMany?(args?: { where?: { scope?: string; key?: string; expiresAt?: { lte?: Date } } }): Promise<unknown>;
+  count?(args?: { where?: { scope?: string } }): Promise<number>;
+};
+
+export type IdempotencyStorageClient = Omit<IdempotencyClient, "idempotencyKey"> & {
+  idempotencyKey: IdempotencyKeyDelegate;
+};
+
+export type IdempotencyStoreResult =
+  | { type: "stored"; response: StoredIdempotencyResponse; createdAt: number }
+  | { type: "replay"; response: StoredIdempotencyResponse; createdAt: number }
+  | { type: "conflict"; existingFingerprint: RequestFingerprint; receivedFingerprint: RequestFingerprint; createdAt: number }
+  | { type: "in_progress"; createdAt: number };
+
+export type IdempotencyStore = {
+  recordSuccess: (
+    key: IdempotencyKey | null | undefined,
+    fingerprint: RequestFingerprint,
+    response: StoredIdempotencyResponse,
+  ) => Promise<IdempotencyStoreResult | { type: "bypass" }>;
+  run: (
+    key: IdempotencyKey | null | undefined,
+    fingerprint: RequestFingerprint,
+    operation: () => Promise<StoredIdempotencyResponse>,
+  ) => Promise<IdempotencyStoreResult | { type: "bypass"; response: StoredIdempotencyResponse }>;
+  peek: (
+    key: IdempotencyKey | null | undefined,
+    fingerprint: RequestFingerprint,
+  ) => Promise<Exclude<IdempotencyStoreResult, { type: "stored" }> | { type: "miss" } | { type: "bypass" }>;
+  clear: () => Promise<void>;
+  size: () => Promise<number>;
+  prune: () => Promise<void>;
+};
+
+const DEFAULT_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_MAX_ENTRIES = 1000;
+const DEFAULT_SCOPE = "api";
+
+export function normalizeIdempotencyKey(value: string | null | undefined): IdempotencyKey | null {
+  const trimmed = value?.trim();
+
+  if (!trimmed || trimmed.length > 255 || /[\u0000-\u001f\u007f]/u.test(trimmed)) {
+    return null;
+  }
+
+  return trimmed;
+}
+
+export function createRequestFingerprint(input: IdempotencyFingerprintInput): RequestFingerprint {
+  const safeInput = {
+    method: input.method.toUpperCase(),
+    route: input.route,
+    tenantId: input.tenantId ?? null,
+    entityId: input.entityId ?? null,
+    rowId: input.rowId ?? null,
+    relationshipId: input.relationshipId ?? null,
+    parsedBody: input.parsedBody ?? null,
+  };
+
+  return createHash("sha256").update(stableStringify(safeInput)).digest("hex");
+}
+
+export function createApiIdempotencyScope(input: Pick<IdempotencyFingerprintInput, "tenantId">): string {
+  return input.tenantId ? `${DEFAULT_SCOPE}:tenant:${input.tenantId}` : `${DEFAULT_SCOPE}:system`;
+}
+
+export function createIdempotencyStore(options: IdempotencyStoreOptions = {}): IdempotencyStore {
+  const maxAgeMs = options.maxAgeMs ?? DEFAULT_MAX_AGE_MS;
+  const now = options.now ?? Date.now;
+  const scope = options.scope ?? DEFAULT_SCOPE;
+  const client = options.client ?? createMemoryIdempotencyStorageClient({ maxEntries: options.maxEntries ?? DEFAULT_MAX_ENTRIES, now });
+
+  const expiresAt = () => new Date(now() + maxAgeMs);
+
+  const findEntry = async (key: IdempotencyKey) => findStoredEntry(client, scope, key, now);
+
+  return {
+    async recordSuccess(key, fingerprint, response) {
+      const result = await this.run(key, fingerprint, async () => response);
+
+      if (result.type === "bypass") {
+        return { type: "bypass" };
+      }
+
+      return result;
+    },
+    async run(key, fingerprint, operation) {
+      const normalizedKey = normalizeIdempotencyKey(key);
+
+      if (normalizedKey === null) {
+        return { type: "bypass", response: cloneResponse(await operation()) };
+      }
+
+      try {
+        const createdAt = now();
+        const response = await withIdempotency(
+          { scope, key: normalizedKey, fingerprint, expiresAt: expiresAt() },
+          createExpiringIdempotencyClient(client, now),
+          async () => {
+            const operationResponse = await operation();
+
+            return {
+              status: operationResponse.status,
+              body: cloneJson(operationResponse.body),
+              headers: { ...operationResponse.headers },
+            };
+          },
+        );
+
+        const storedResponse = normalizeStoredResponse({
+          status: response.status,
+          body: response.body ?? null,
+          headers: response.headers ?? {},
+        });
+
+        if (response.replayed) {
+          const existing = await findEntry(normalizedKey);
+          return { type: "replay", response: storedResponse, createdAt: entryCreatedAt(existing, createdAt) };
+        }
+
+        return { type: "stored", response: storedResponse, createdAt };
+      } catch (error) {
+        if (isIdempotencyConflictError(error)) {
+          const existing = await findEntry(normalizedKey);
+
+          return {
+            type: "conflict",
+            existingFingerprint: parseStoredFingerprint(existing?.requestHash) ?? "",
+            receivedFingerprint: fingerprint,
+            createdAt: entryCreatedAt(existing, now()),
+          };
+        }
+
+        if (isIdempotencyInProgressError(error)) {
+          const existing = await findEntry(normalizedKey);
+          return { type: "in_progress", createdAt: entryCreatedAt(existing, now()) };
+        }
+
+        throw error;
+      }
+    },
+    async peek(key, fingerprint) {
+      const normalizedKey = normalizeIdempotencyKey(key);
+
+      if (normalizedKey === null) {
+        return { type: "bypass" };
+      }
+
+      const existing = await findEntry(normalizedKey);
+
+      if (!existing) {
+        return { type: "miss" };
+      }
+
+      const requestHash = stableStringify(fingerprint);
+      if (existing.requestHash !== requestHash) {
+        return {
+          type: "conflict",
+          existingFingerprint: parseStoredFingerprint(existing.requestHash) ?? "",
+          receivedFingerprint: fingerprint,
+          createdAt: entryCreatedAt(existing, now()),
+        };
+      }
+
+      if (existing.responseStatus <= 0) {
+        return { type: "in_progress", createdAt: entryCreatedAt(existing, now()) };
+      }
+
+      return { type: "replay", response: normalizeStoredResponse(existing), createdAt: entryCreatedAt(existing, now()) };
+    },
+    async clear() {
+      await client.idempotencyKey.deleteMany?.({ where: { scope } });
+    },
+    async size() {
+      await this.prune();
+      return client.idempotencyKey.count?.({ where: { scope } }) ?? 0;
+    },
+    async prune() {
+      await client.idempotencyKey.deleteMany?.({ where: { scope, expiresAt: { lte: new Date(now()) } } });
+    },
+  };
+}
+
+export function createMemoryIdempotencyStorageClient(options: { maxEntries?: number; now?: () => number } = {}): IdempotencyStorageClient {
+  const maxEntries = options.maxEntries ?? DEFAULT_MAX_ENTRIES;
+  const now = options.now ?? Date.now;
+  const rows = new Map<string, StoredEntry & { scope: string; key: string }>();
+  const rowKey = (scope: string, key: string) => `${scope}:${key}`;
+
+  const pruneOverflow = () => {
+    while (rows.size > maxEntries) {
+      const oldestKey = rows.keys().next().value as string | undefined;
+
+      if (!oldestKey) {
+        break;
+      }
+
+      rows.delete(oldestKey);
+    }
+  };
+
+  return {
+    idempotencyKey: {
+      async findUnique({ where }) {
+        return rows.get(rowKey(where.idempotency_key_scope_key.scope, where.idempotency_key_scope_key.key)) ?? null;
+      },
+      async create({ data }) {
+        const key = rowKey(data.scope, data.key);
+        if (rows.has(key)) {
+          throw Object.assign(new Error("Unique constraint failed on idempotency key"), { code: "P2002" });
+        }
+        rows.set(rowKey(data.scope, data.key), {
+          scope: data.scope,
+          key: data.key,
+          requestHash: data.requestHash,
+          responseStatus: data.responseStatus,
+          responseBody: data.responseBody ?? null,
+          responseHeaders: data.responseHeaders ?? null,
+          createdAt: new Date(now()),
+          expiresAt: data.expiresAt ?? null,
+        });
+        pruneOverflow();
+      },
+      async update({ where, data }) {
+        const key = rowKey(where.idempotency_key_scope_key.scope, where.idempotency_key_scope_key.key);
+        const existing = rows.get(key);
+        if (!existing) {
+          throw new Error("Idempotency key reservation not found.");
+        }
+        rows.set(key, {
+          ...existing,
+          responseStatus: data.responseStatus,
+          responseBody: data.responseBody ?? null,
+          responseHeaders: data.responseHeaders ?? null,
+          expiresAt: data.expiresAt ?? existing.expiresAt ?? null,
+        });
+      },
+      async deleteMany(args) {
+        const where = args?.where;
+        for (const [key, row] of rows) {
+          if (where?.scope && row.scope !== where.scope) {
+            continue;
+          }
+          if (where?.key && row.key !== where.key) {
+            continue;
+          }
+          if (where?.expiresAt?.lte && (!row.expiresAt || row.expiresAt > where.expiresAt.lte)) {
+            continue;
+          }
+          rows.delete(key);
+        }
+      },
+      async count(args) {
+        const scope = args?.where?.scope;
+        return scope ? [...rows.values()].filter((row) => row.scope === scope).length : rows.size;
+      },
+    },
+  };
+}
+
+function createExpiringIdempotencyClient(client: IdempotencyStorageClient, now: () => number): IdempotencyClient {
+  return {
+    idempotencyKey: {
+      async findUnique(args) {
+        const existing = await client.idempotencyKey.findUnique(args);
+
+        if (existing?.expiresAt && existing.expiresAt.getTime() <= now()) {
+          await client.idempotencyKey.deleteMany?.({
+            where: {
+              scope: args.where.idempotency_key_scope_key.scope,
+              key: args.where.idempotency_key_scope_key.key,
+              expiresAt: { lte: new Date(now()) },
+            },
+          });
+          return null;
+        }
+
+        return existing;
+      },
+      create(args) {
+        return client.idempotencyKey.create(args);
+      },
+      update(args) {
+        return client.idempotencyKey.update(args);
+      },
+      async deleteMany(args) {
+        await client.idempotencyKey.deleteMany?.(args);
+      },
+    },
+  };
+}
+
+async function findStoredEntry(client: IdempotencyStorageClient, scope: string, key: IdempotencyKey, now: () => number) {
+  const existing = await client.idempotencyKey.findUnique({ where: { idempotency_key_scope_key: { scope, key } } });
+
+  if (existing?.expiresAt && existing.expiresAt.getTime() <= now()) {
+    await client.idempotencyKey.deleteMany?.({ where: { scope, key, expiresAt: { lte: new Date(now()) } } });
+    return null;
+  }
+
+  return existing;
+}
+
+function isIdempotencyConflictError(error: unknown) {
+  return error instanceof CatalogError && error.code === foundationErrors.idempotencyConflict.code;
+}
+
+function isIdempotencyInProgressError(error: unknown) {
+  return error instanceof CatalogError && error.code === foundationErrors.idempotencyInProgress.code;
+}
+
+function entryCreatedAt(entry: StoredEntry | null | undefined, fallback: number) {
+  return entry?.createdAt instanceof Date ? entry.createdAt.getTime() : fallback;
+}
+
+function normalizeStoredResponse(response: {
+  status?: number;
+  headers?: Record<string, string> | null;
+  body?: JsonValue | null;
+  responseStatus?: number;
+  responseHeaders?: Record<string, string> | null;
+  responseBody?: JsonValue | null;
+}): StoredIdempotencyResponse {
+  return {
+    status: response.responseStatus ?? response.status ?? 200,
+    headers: { ...(response.responseHeaders ?? response.headers ?? {}) },
+    body: cloneJson(response.responseBody ?? response.body ?? null),
+  };
+}
+
+function parseStoredFingerprint(requestHash: string | undefined) {
+  if (!requestHash) {
+    return null;
+  }
+
+  try {
+    const parsed = JSON.parse(requestHash);
+    return typeof parsed === "string" ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+function cloneResponse(response: StoredIdempotencyResponse): StoredIdempotencyResponse {
+  return {
+    status: response.status,
+    headers: { ...response.headers },
+    body: cloneJson(response.body),
+  };
+}
+
+function cloneJson<T extends JsonValue>(value: T): T {
+  return JSON.parse(JSON.stringify(value)) as T;
+}
+
+function stableStringify(value: JsonValue | Record<string, unknown>): string {
+  if (value === null || typeof value !== "object") {
+    return JSON.stringify(value);
+  }
+
+  if (Array.isArray(value)) {
+    return `[${value.map((item) => stableStringify(item)).join(",")}]`;
+  }
+
+  return `{${Object.keys(value)
+    .sort()
+    .map((key) => `${JSON.stringify(key)}:${stableStringify((value as Record<string, JsonValue>)[key] ?? null)}`)
+    .join(",")}}`;
+}

hivegrid-cloud/apps/api/src/isolation/api-key-entity-rows.test.ts

@@ -0,0 +1,291 @@
+import { describe, expect, test, vi } from "vitest";
+
+import { app } from "../app.js";
+import {
+  TEST_API_CONFIG,
+  configureApiIsolationDependenciesForTests,
+  createLegacyEntityFixture,
+  createLegacyRowFixture,
+  createStoredApiKeyFixture,
+  installApiIsolationTestHooks,
+} from "./fixtures/api-key.js";
+
+installApiIsolationTestHooks();
+
+const STORED_API_KEY_SECRET = "stored-row-key-secret";
+const INVALID_API_KEY_SECRET = "invalid-row-key-secret";
+const REQUEST_ID = "req_entity_rows_scope";
+
+function createScopedEntityApiKey(patch: Parameters<typeof createStoredApiKeyFixture>[0] = {}) {
+  return createStoredApiKeyFixture({
+    entities: [{ entityId: "entity_1", read: true, create: true, update: true, delete: true }],
+    ...patch,
+  });
+}
+
+function createEntityRowDependencies(apiKey = createScopedEntityApiKey()) {
+  const entity = createLegacyEntityFixture();
+  const listRow = createLegacyRowFixture("row_list_1");
+  const itemRow = createLegacyRowFixture("row_item_1");
+  const createdRow = createLegacyRowFixture("row_created", { createdByApiKeyId: apiKey.id });
+  const updatedRow = createLegacyRowFixture("row_item_1", { updatedAt: new Date("2026-01-02T00:00:00.000Z") });
+
+  const dependencies = {
+    findLegacyApiKeyByKey: vi.fn(async (key: string) => (key === STORED_API_KEY_SECRET ? apiKey : null)),
+    findLegacyEntityByName: vi.fn(async () => entity),
+    getAllLegacyEntityRows: vi.fn(async () => ({ page: 1, pageSize: 25, totalItems: 1, totalPages: 1, items: [listRow] })),
+    getLegacyEntityRow: vi.fn(async () => itemRow),
+    createLegacyEntityRow: vi.fn(async () => createdRow),
+    updateLegacyEntityRow: vi.fn(async () => updatedRow),
+    deleteLegacyEntityRow: vi.fn(async () => "row_item_1"),
+  };
+
+  configureApiIsolationDependenciesForTests(dependencies);
+
+  return { apiKey, entity, listRow, itemRow, createdRow, updatedRow, dependencies };
+}
+
+function apiKeyHeaders(secret = STORED_API_KEY_SECRET, requestId = REQUEST_ID) {
+  return { "x-api-key": secret, "x-request-id": requestId, "idempotency-key": `${requestId}-idempotency` };
+}
+
+function bearerHeaders(secret = STORED_API_KEY_SECRET, requestId = REQUEST_ID) {
+  return { authorization: `Bearer ${secret}`, "x-request-id": requestId, "idempotency-key": `${requestId}-idempotency` };
+}
+
+function jsonHeaders(headers: Record<string, string>) {
+  return { ...headers, "content-type": "application/json" };
+}
+
+function jsonRequest(headers: Record<string, string>, body: unknown) {
+  const serialized = JSON.stringify(body);
+  return {
+    body: serialized,
+    headers: { ...jsonHeaders(headers), "content-length": String(Buffer.byteLength(serialized)) },
+  };
+}
+
+async function expectRedactedError(response: Response, expected: { status: number; code: string; requestId: string }) {
+  expect(response.status).toBe(expected.status);
+  expect(response.headers.get("x-request-id")).toBe(expected.requestId);
+
+  const text = await response.text();
+  expect(text).toContain(expected.requestId);
+  expect(text).not.toContain(STORED_API_KEY_SECRET);
+  expect(text).not.toContain(INVALID_API_KEY_SECRET);
+  expect(text).not.toContain(TEST_API_CONFIG.apiAccessToken);
+  expect(text).not.toContain("scope-secret");
+
+  expect(JSON.parse(text)).toEqual({
+    error: expect.objectContaining({
+      code: expected.code,
+      requestId: expected.requestId,
+    }),
+  });
+}
+
+describe("stored API-key entity row isolation", () => {
+  test("allows header API key read on GET /v1/entities/:entity/rows for tenant_1 entity_1", async () => {
+    const { dependencies, listRow } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_1", { headers: apiKeyHeaders() });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toMatchObject({
+      success: true,
+      results: [expect.objectContaining({ id: listRow.id, entityId: "entity_1", tenantId: "tenant_1" })],
+      data: [expect.objectContaining({ id: listRow.id, entityId: "entity_1", tenantId: "tenant_1" })],
+    });
+    expect(dependencies.findLegacyApiKeyByKey).toHaveBeenCalledWith(STORED_API_KEY_SECRET, undefined, "scope-secret", { allowPlaintextFallback: true });
+    expect(dependencies.findLegacyEntityByName).toHaveBeenCalledWith({ tenantId: "tenant_1", name: "contact", active: true });
+    expect(dependencies.getAllLegacyEntityRows).toHaveBeenCalledWith({ entityId: "entity_1", tenantId: "tenant_1", page: 1, pageSize: 100 });
+  });
+
+  test("allows header API key read on GET /v1/entities/:entity/rows/:rowId for tenant_1 entity_1", async () => {
+    const { dependencies, itemRow } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows/row_item_1?tenantId=tenant_1", { headers: apiKeyHeaders() });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toMatchObject({ id: itemRow.id, entityId: "entity_1", tenantId: "tenant_1" });
+    expect(dependencies.getLegacyEntityRow).toHaveBeenCalledWith({ entityId: "entity_1", tenantId: "tenant_1", rowId: "row_item_1" });
+  });
+
+  test("allows header API key create on POST /v1/entities/:entity/rows and stores createdByApiKeyId", async () => {
+    const { apiKey, dependencies } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_1", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(), { firstName: "Ada" }),
+    });
+
+    expect(response.status).toBe(201);
+    await expect(response.json()).resolves.toMatchObject({ id: "row_created", createdByApiKeyId: apiKey.id });
+    expect(dependencies.findLegacyEntityByName).toHaveBeenCalledWith({ tenantId: "tenant_1", name: "contact", active: true });
+    expect(dependencies.createLegacyEntityRow).toHaveBeenCalledWith({
+      entity: expect.objectContaining({ id: "entity_1", name: "contact" }),
+      tenantId: "tenant_1",
+      createdByApiKeyId: apiKey.id,
+      data: { firstName: "Ada" },
+    });
+  });
+
+  test("allows header API key update on PATCH /v1/entities/:entity/rows/:rowId for tenant_1 entity_1", async () => {
+    const { dependencies } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows/row_item_1?tenantId=tenant_1", {
+      method: "PATCH",
+      ...jsonRequest(apiKeyHeaders(), { firstName: "Grace" }),
+    });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toMatchObject({ id: "row_item_1" });
+    expect(dependencies.findLegacyEntityByName).toHaveBeenCalledWith({ tenantId: "tenant_1", name: "contact", active: true });
+    expect(dependencies.updateLegacyEntityRow).toHaveBeenCalledWith({
+      entity: expect.objectContaining({ id: "entity_1", name: "contact" }),
+      tenantId: "tenant_1",
+      rowId: "row_item_1",
+      data: { firstName: "Grace" },
+    });
+  });
+
+  test("allows header API key delete on DELETE /v1/entities/:entity/rows/:rowId for tenant_1 entity_1", async () => {
+    const { dependencies } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows/row_item_1?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: apiKeyHeaders(),
+    });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({ success: true, id: "row_item_1" });
+    expect(dependencies.findLegacyEntityByName).toHaveBeenCalledWith({ tenantId: "tenant_1", name: "contact", active: true });
+    expect(dependencies.deleteLegacyEntityRow).toHaveBeenCalledWith({ entityId: "entity_1", tenantId: "tenant_1", rowId: "row_item_1" });
+  });
+
+  test("denies read on GET /v1/entities/:entity/rows when tenantId=tenant_2 tampers tenant", async () => {
+    const { dependencies } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_2", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_wrong_tenant") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_wrong_tenant" });
+    expect(dependencies.findLegacyEntityByName).not.toHaveBeenCalled();
+    expect(dependencies.getAllLegacyEntityRows).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.updateLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyEntityRow).not.toHaveBeenCalled();
+  });
+
+  test("denies create on POST /v1/entities/:entity/rows when API key lacks create scope", async () => {
+    const apiKey = createScopedEntityApiKey({ entities: [{ entityId: "entity_1", read: true, create: false, update: true, delete: true }] });
+    const { dependencies } = createEntityRowDependencies(apiKey);
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_1", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_missing_create"), { firstName: "Ada" }),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_missing_create" });
+    expect(dependencies.createLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.updateLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyEntityRow).not.toHaveBeenCalled();
+  });
+
+  test("returns the same forbidden response for missing entity names with stored API keys", async () => {
+    const { dependencies } = createEntityRowDependencies(createScopedEntityApiKey({ entities: [] }));
+    dependencies.findLegacyEntityByName.mockRejectedValueOnce(new Error("Entity with name missingContact not found"));
+
+    const response = await app.request("/v1/entities/missingContact/rows?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_missing_entity") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_missing_entity" });
+    expect(dependencies.getAllLegacyEntityRows).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.updateLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyEntityRow).not.toHaveBeenCalled();
+  });
+
+  test("denies update on PATCH /v1/entities/:entity/rows/:rowId when API key lacks update scope", async () => {
+    const apiKey = createScopedEntityApiKey({ entities: [{ entityId: "entity_1", read: true, create: true, update: false, delete: true }] });
+    const { dependencies } = createEntityRowDependencies(apiKey);
+
+    const response = await app.request("/v1/entities/contact/rows/row_item_1?tenantId=tenant_1", {
+      method: "PATCH",
+      ...jsonRequest(apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_missing_update"), { firstName: "Grace" }),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_missing_update" });
+    expect(dependencies.updateLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyEntityRow).not.toHaveBeenCalled();
+  });
+
+  test("denies delete on DELETE /v1/entities/:entity/rows/:rowId when API key lacks delete scope", async () => {
+    const apiKey = createScopedEntityApiKey({ entities: [{ entityId: "entity_1", read: true, create: true, update: true, delete: false }] });
+    const { dependencies } = createEntityRowDependencies(apiKey);
+
+    const response = await app.request("/v1/entities/contact/rows/row_item_1?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_missing_delete"),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_missing_delete" });
+    expect(dependencies.deleteLegacyEntityRow).not.toHaveBeenCalled();
+  });
+
+  test("denies read on GET /v1/entities/:entity/rows for inactive API key", async () => {
+    const { dependencies } = createEntityRowDependencies(createScopedEntityApiKey({ active: false }));
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_inactive") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_inactive" });
+    expect(dependencies.getAllLegacyEntityRows).not.toHaveBeenCalled();
+  });
+
+  test("denies read on GET /v1/entities/:entity/rows for expired API key", async () => {
+    const { dependencies } = createEntityRowDependencies(createScopedEntityApiKey({ expires: new Date("2020-01-01T00:00:00.000Z") }));
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_expired") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_expired" });
+    expect(dependencies.getAllLegacyEntityRows).not.toHaveBeenCalled();
+  });
+
+  test("denies read on GET /v1/entities/:entity/rows for deactivated tenant API key", async () => {
+    const apiKey = createScopedEntityApiKey({ tenant: { id: "tenant_1", name: "Acme", slug: "acme", deactivatedReason: "billing_failed" } });
+    const { dependencies } = createEntityRowDependencies(apiKey);
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_rows_deactivated") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_rows_deactivated" });
+    expect(dependencies.getAllLegacyEntityRows).not.toHaveBeenCalled();
+  });
+
+  test("denies read on GET /v1/entities/:entity/rows with invalid x-api-key without service-token fallback", async () => {
+    const { dependencies } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows?tenantId=tenant_1", { headers: apiKeyHeaders(INVALID_API_KEY_SECRET, "req_rows_invalid_key") });
+
+    await expectRedactedError(response, { status: 401, code: "unauthorized", requestId: "req_rows_invalid_key" });
+    expect(dependencies.findLegacyApiKeyByKey).toHaveBeenCalledWith(INVALID_API_KEY_SECRET, undefined, "scope-secret", { allowPlaintextFallback: true });
+    expect(dependencies.findLegacyEntityByName).not.toHaveBeenCalled();
+    expect(dependencies.getAllLegacyEntityRows).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.updateLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyEntityRow).not.toHaveBeenCalled();
+  });
+
+  test("denies read on GET /v1/entities/:entity/rows/:rowId with invalid bearer token without stored-key fallback", async () => {
+    const { dependencies } = createEntityRowDependencies();
+
+    const response = await app.request("/v1/entities/contact/rows/row_item_1?tenantId=tenant_1", {
+      headers: bearerHeaders(INVALID_API_KEY_SECRET, "req_rows_invalid_bearer"),
+    });
+
+    await expectRedactedError(response, { status: 401, code: "unauthorized", requestId: "req_rows_invalid_bearer" });
+    expect(dependencies.findLegacyApiKeyByKey).not.toHaveBeenCalled();
+    expect(dependencies.findLegacyEntityByName).not.toHaveBeenCalled();
+    expect(dependencies.getAllLegacyEntityRows).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.updateLegacyEntityRow).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyEntityRow).not.toHaveBeenCalled();
+  });
+});

hivegrid-cloud/apps/api/src/isolation/api-key-fixtures.test.ts

@@ -0,0 +1,133 @@
+import { describe, expect, test, vi } from "vitest";
+
+import { app } from "../app.js";
+import { getApiRuntimeConfig } from "../config.js";
+import {
+  TEST_API_CONFIG,
+  TEST_JWT_SECRET,
+  configureApiIsolationDependenciesForTests,
+  createLegacyEntityFixture,
+  createLegacyRelationshipRowFixture,
+  createLegacyRowFixture,
+  createLegacyRowRelationshipFixture,
+  createStoredApiKeyFixture,
+  installApiIsolationTestHooks,
+  patchApiIsolationConfig,
+  resetApiIsolationFixturesForTests,
+} from "./fixtures/api-key.js";
+
+installApiIsolationTestHooks();
+
+describe("api-key isolation fixtures", () => {
+  test("exposes stable default runtime and row fixtures", () => {
+    const apiKey = createStoredApiKeyFixture();
+    const entity = createLegacyEntityFixture();
+    const row = createLegacyRowFixture();
+    const relationshipRow = createLegacyRelationshipRowFixture();
+    const rowRelationship = createLegacyRowRelationshipFixture();
+
+    expect(TEST_API_CONFIG.apiAccessToken).toBe("service-secret");
+    expect(TEST_API_CONFIG.jwtSecret).toBe(TEST_JWT_SECRET);
+    expect(TEST_JWT_SECRET).toBe("scope-secret");
+    expect(getApiRuntimeConfig()).toMatchObject({ apiAccessToken: "service-secret", jwtSecret: "scope-secret" });
+    expect(apiKey).toMatchObject({ tenantId: "tenant_1", tenant: { id: "tenant_1" }, active: true, entities: [] });
+    expect(entity).toMatchObject({ id: "entity_1", name: "contact" });
+    expect(row).toMatchObject({ id: "row_1", entityId: "entity_1", tenantId: "tenant_1", parentRows: [], childRows: [] });
+    expect(relationshipRow).toEqual({ id: "row_1", entityId: "entity_1", tenantId: "tenant_1" });
+    expect(rowRelationship).toMatchObject({
+      id: "row_relationship_1",
+      parentId: "parent_row_1",
+      childId: "child_row_1",
+      relationship: { parentId: "parent_entity_1", childId: "child_entity_1" },
+    });
+  });
+
+  test("patches tenant-related fixture values without hiding mismatches", () => {
+    patchApiIsolationConfig({ apiAccessToken: "tenant-2-service", jwtSecret: "tenant-2-secret" });
+
+    const apiKey = createStoredApiKeyFixture({ tenantId: "tenant_2" });
+    const mismatchedApiKey = createStoredApiKeyFixture({ tenantId: "tenant_2", tenant: { id: "tenant_3", name: "Other", slug: "other", deactivatedReason: null } });
+    const row = createLegacyRowFixture("row_tenant_2", { tenantId: "tenant_2" });
+
+    expect(getApiRuntimeConfig()).toMatchObject({ apiAccessToken: "tenant-2-service", jwtSecret: "tenant-2-secret" });
+    expect(apiKey).toMatchObject({ tenantId: "tenant_2", tenant: { id: "tenant_2" } });
+    expect(mismatchedApiKey).toMatchObject({ tenantId: "tenant_2", tenant: { id: "tenant_3" } });
+    expect(row).toMatchObject({ id: "row_tenant_2", tenantId: "tenant_2" });
+  });
+
+  test("resets runtime config patches after each test via installed hooks", () => {
+    patchApiIsolationConfig({ apiAccessToken: "temporary-service" });
+    resetApiIsolationFixturesForTests();
+    patchApiIsolationConfig();
+
+    expect(getApiRuntimeConfig()).toMatchObject({ apiAccessToken: "service-secret", jwtSecret: "scope-secret" });
+  });
+
+  test("reset callback clears dependency overrides and runtime config", async () => {
+    patchApiIsolationConfig({ apiAccessToken: "patched-service" });
+    const checkReadiness = vi.fn().mockResolvedValue(undefined);
+    const resetDependencies = configureApiIsolationDependenciesForTests({ checkReadiness });
+
+    const patchedResponse = await app.request("/readyz");
+    expect(patchedResponse.status).toBe(200);
+    expect(checkReadiness).toHaveBeenCalledTimes(1);
+    expect(getApiRuntimeConfig().apiAccessToken).toBe("patched-service");
+
+    resetApiIsolationFixturesForTests();
+
+    expect(getApiRuntimeConfig).toThrow();
+    expect(checkReadiness).toHaveBeenCalledTimes(1);
+    resetDependencies();
+  });
+
+  test("merges nested config patches without losing other defaults", () => {
+    patchApiIsolationConfig({
+      listen: { port: 4242 },
+      routing: { appBaseDomain: "tenant-2.test", webAppHost: "app.tenant-2.test", adminAppHost: "admin.tenant-2.test", apiHost: "api.tenant-2.test", protocol: "https" },
+      origins: { web: "https://web.tenant-2.test" },
+      analytics: { token: "analytics_tenant2" },
+      mailjet: { webhookSecret: "mailer_tenant2_whsec" },
+      stripe: { secretKey: "sk_test_tenant2", webhookSigningSecret: "whsec_tenant2" },
+      storage: { bucket: "tenant-2-bucket", region: "eu-central-1" },
+    });
+
+    expect(getApiRuntimeConfig()).toMatchObject({
+      apiAccessToken: "service-secret",
+      jwtSecret: "scope-secret",
+      listen: { port: 4242 },
+      routing: { apiHost: "api.tenant-2.test", appBaseDomain: "tenant-2.test", protocol: "https" },
+      origins: { web: "https://web.tenant-2.test" },
+      analytics: { token: "analytics_tenant2" },
+      mailjet: { webhookSecret: "mailer_tenant2_whsec" },
+      stripe: { secretKey: "sk_test_tenant2", webhookSigningSecret: "whsec_tenant2" },
+      storage: { bucket: "tenant-2-bucket", region: "eu-central-1" },
+    });
+  });
+
+  test("reset closure released by configureApiIsolationDependenciesForTests clears its own override", async () => {
+    const checkReadiness = vi.fn().mockResolvedValue(undefined);
+    const release = configureApiIsolationDependenciesForTests({ checkReadiness });
+
+    const firstResponse = await app.request("/readyz");
+    expect(firstResponse.status).toBe(200);
+    expect(checkReadiness).toHaveBeenCalledTimes(1);
+
+    release();
+
+    await app.request("/readyz");
+    expect(checkReadiness).toHaveBeenCalledTimes(1);
+  });
+
+  test("configureApiIsolationDependenciesForTests called twice releases the prior override first", async () => {
+    const firstReadiness = vi.fn().mockResolvedValue(undefined);
+    const secondReadiness = vi.fn().mockResolvedValue(undefined);
+
+    configureApiIsolationDependenciesForTests({ checkReadiness: firstReadiness });
+    configureApiIsolationDependenciesForTests({ checkReadiness: secondReadiness });
+
+    const response = await app.request("/readyz");
+    expect(response.status).toBe(200);
+    expect(firstReadiness).not.toHaveBeenCalled();
+    expect(secondReadiness).toHaveBeenCalledTimes(1);
+  });
+});

hivegrid-cloud/apps/api/src/isolation/api-key-relationships.test.ts

@@ -0,0 +1,300 @@
+import { describe, expect, test, vi } from "vitest";
+
+import { app } from "../app.js";
+import {
+  TEST_API_CONFIG,
+  configureApiIsolationDependenciesForTests,
+  createLegacyRelationshipRowFixture,
+  createLegacyRowFixture,
+  createLegacyRowRelationshipFixture,
+  createStoredApiKeyFixture,
+  installApiIsolationTestHooks,
+} from "./fixtures/api-key.js";
+
+installApiIsolationTestHooks();
+
+const STORED_API_KEY_SECRET = "stored-relationship-key-secret";
+const INVALID_API_KEY_SECRET = "invalid-relationship-key-secret";
+const REQUEST_ID = "req_relationship_scope";
+
+function createScopedRelationshipApiKey(patch: Parameters<typeof createStoredApiKeyFixture>[0] = {}) {
+  return createStoredApiKeyFixture({
+    entities: [
+      { entityId: "parent_entity_1", read: true, create: true, update: false, delete: true },
+      { entityId: "child_entity_1", read: true, create: true, update: false, delete: true },
+    ],
+    ...patch,
+  });
+}
+
+function createRelationshipDependencies(apiKey = createScopedRelationshipApiKey()) {
+  const parentRelationshipRow = createLegacyRelationshipRowFixture("parent_row_1", "parent_entity_1");
+  const childRelationshipRow = createLegacyRelationshipRowFixture("child_row_1", "child_entity_1");
+  const parentEntityRow = createLegacyRowFixture("parent_row_1", { entityId: "parent_entity_1" });
+  const childEntityRow = createLegacyRowFixture("child_row_1", { entityId: "child_entity_1" });
+  const rowRelationship = createLegacyRowRelationshipFixture("row_relationship_1", "parent_row_1", "child_row_1", "parent_entity_1", "child_entity_1");
+  const entityRelationship = { id: "entity_relationship_1", parentId: "parent_entity_1", childId: "child_entity_1" };
+  const createdRelationship = createLegacyRowRelationshipFixture("row_relationship_created", "parent_row_1", "child_row_1", "parent_entity_1", "child_entity_1");
+
+  const dependencies = {
+    findLegacyApiKeyByKey: vi.fn(async (key: string) => (key === STORED_API_KEY_SECRET ? apiKey : null)),
+    findLegacyRelationshipRowById: vi.fn(async ({ id, tenantId }: { id: string; tenantId?: string | null }) => {
+      if (tenantId !== "tenant_1") return null;
+      if (id === parentRelationshipRow.id) return parentRelationshipRow;
+      if (id === childRelationshipRow.id) return childRelationshipRow;
+      return null;
+    }),
+    findLegacyEntityRelationship: vi.fn(async () => entityRelationship),
+    findLegacyRowRelationshipById: vi.fn(async (id: string, tenantId?: string | null) => (id === rowRelationship.id && tenantId === "tenant_1" ? rowRelationship : null)),
+    getLegacyEntityRow: vi.fn(async ({ rowId }: { rowId: string }) => {
+      if (rowId === parentEntityRow.id) return parentEntityRow;
+      if (rowId === childEntityRow.id) return childEntityRow;
+      return null;
+    }),
+    createLegacyRowRelationship: vi.fn(async () => createdRelationship),
+    deleteLegacyRowRelationshipById: vi.fn(async () => undefined),
+  };
+
+  configureApiIsolationDependenciesForTests(dependencies);
+
+  return { apiKey, parentRelationshipRow, childRelationshipRow, parentEntityRow, childEntityRow, rowRelationship, entityRelationship, createdRelationship, dependencies };
+}
+
+function apiKeyHeaders(secret = STORED_API_KEY_SECRET, requestId = REQUEST_ID) {
+  return { "x-api-key": secret, "x-request-id": requestId, "idempotency-key": `${requestId}-idempotency` };
+}
+
+function bearerHeaders(secret = STORED_API_KEY_SECRET, requestId = REQUEST_ID) {
+  return { authorization: `Bearer ${secret}`, "x-request-id": requestId, "idempotency-key": `${requestId}-idempotency` };
+}
+
+function jsonHeaders(headers: Record<string, string>) {
+  return { ...headers, "content-type": "application/json" };
+}
+
+function jsonRequest(headers: Record<string, string>, body: unknown) {
+  const serialized = JSON.stringify(body);
+  return {
+    body: serialized,
+    headers: { ...jsonHeaders(headers), "content-length": String(Buffer.byteLength(serialized)) },
+  };
+}
+
+async function expectRedactedError(response: Response, expected: { status: number; code: string; requestId: string }) {
+  expect(response.status).toBe(expected.status);
+  expect(response.headers.get("x-request-id")).toBe(expected.requestId);
+
+  const text = await response.text();
+  expect(text).toContain(expected.requestId);
+  expect(text).not.toContain(STORED_API_KEY_SECRET);
+  expect(text).not.toContain(INVALID_API_KEY_SECRET);
+  expect(text).not.toContain(TEST_API_CONFIG.apiAccessToken);
+  expect(text).not.toContain("scope-secret");
+
+  expect(JSON.parse(text)).toEqual({
+    error: expect.objectContaining({
+      code: expected.code,
+      requestId: expected.requestId,
+    }),
+  });
+}
+
+function expectNoMutationAfterDeniedScope(dependencies: ReturnType<typeof createRelationshipDependencies>["dependencies"]) {
+  expect(dependencies.findLegacyEntityRelationship).not.toHaveBeenCalled();
+  expect(dependencies.createLegacyRowRelationship).not.toHaveBeenCalled();
+  expect(dependencies.deleteLegacyRowRelationshipById).not.toHaveBeenCalled();
+}
+
+describe("stored API-key relationship isolation", () => {
+  test("allows create on POST /v1/relationships when parent and child have create scope", async () => {
+    const { dependencies, createdRelationship } = createRelationshipDependencies();
+
+    const response = await app.request("/v1/relationships?tenantId=tenant_1", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(), { parent: "parent_row_1", child: "child_row_1" }),
+    });
+
+    expect(response.status).toBe(201);
+    await expect(response.json()).resolves.toMatchObject({
+      relationship: { id: createdRelationship.id, parent: { id: "parent_row_1", entityId: "parent_entity_1" }, child: { id: "child_row_1", entityId: "child_entity_1" } },
+    });
+    expect(dependencies.findLegacyApiKeyByKey).toHaveBeenCalledWith(STORED_API_KEY_SECRET, undefined, "scope-secret", { allowPlaintextFallback: true });
+    expect(dependencies.findLegacyEntityRelationship).toHaveBeenCalledWith({ parentId: "parent_entity_1", childId: "child_entity_1" });
+    expect(dependencies.createLegacyRowRelationship).toHaveBeenCalledWith({
+      relationshipId: "entity_relationship_1",
+      parentId: "parent_row_1",
+      childId: "child_row_1",
+      tenantId: "tenant_1",
+    });
+  });
+
+  test("allows header API key read on GET /v1/relationships/:id when parent and child have read scope", async () => {
+    const { dependencies, rowRelationship } = createRelationshipDependencies();
+
+    const response = await app.request("/v1/relationships/row_relationship_1?tenantId=tenant_1", { headers: apiKeyHeaders() });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toMatchObject({ relationship: { id: rowRelationship.id, parent: { entityId: "parent_entity_1" }, child: { entityId: "child_entity_1" } } });
+    expect(dependencies.findLegacyRowRelationshipById).toHaveBeenCalledWith("row_relationship_1", "tenant_1");
+    expect(dependencies.getLegacyEntityRow).toHaveBeenCalledWith({ entityId: "parent_entity_1", tenantId: "tenant_1", rowId: "parent_row_1" });
+    expect(dependencies.getLegacyEntityRow).toHaveBeenCalledWith({ entityId: "child_entity_1", tenantId: "tenant_1", rowId: "child_row_1" });
+  });
+
+  test("allows delete on DELETE /v1/relationships/:id when parent and child have delete scope", async () => {
+    const { dependencies, rowRelationship } = createRelationshipDependencies();
+
+    const response = await app.request("/v1/relationships/row_relationship_1?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: apiKeyHeaders(),
+    });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toMatchObject({ deleted: { id: rowRelationship.id, parent: { entityId: "parent_entity_1" }, child: { entityId: "child_entity_1" } } });
+    expect(dependencies.deleteLegacyRowRelationshipById).toHaveBeenCalledWith({
+      id: "row_relationship_1",
+      relationshipId: "entity_relationship_1",
+      parentId: "parent_row_1",
+      childId: "child_row_1",
+      tenantId: "tenant_1",
+    });
+  });
+
+  test("denies create on POST /v1/relationships when parent create scope is missing", async () => {
+    const apiKey = createScopedRelationshipApiKey({
+      entities: [
+        { entityId: "parent_entity_1", read: true, create: false, update: false, delete: true },
+        { entityId: "child_entity_1", read: true, create: true, update: false, delete: true },
+      ],
+    });
+    const { dependencies } = createRelationshipDependencies(apiKey);
+
+    const response = await app.request("/v1/relationships?tenantId=tenant_1", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(STORED_API_KEY_SECRET, "req_relationship_missing_parent_create"), { parent: "parent_row_1", child: "child_row_1" }),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_relationship_missing_parent_create" });
+    expectNoMutationAfterDeniedScope(dependencies);
+  });
+
+  test("denies create on POST /v1/relationships when child create scope is missing", async () => {
+    const apiKey = createScopedRelationshipApiKey({
+      entities: [
+        { entityId: "parent_entity_1", read: true, create: true, update: false, delete: true },
+        { entityId: "child_entity_1", read: true, create: false, update: false, delete: true },
+      ],
+    });
+    const { dependencies } = createRelationshipDependencies(apiKey);
+
+    const response = await app.request("/v1/relationships?tenantId=tenant_1", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(STORED_API_KEY_SECRET, "req_relationship_missing_child_create"), { parent: "parent_row_1", child: "child_row_1" }),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_relationship_missing_child_create" });
+    expectNoMutationAfterDeniedScope(dependencies);
+  });
+
+  test("returns the same forbidden response for missing relationship rows with stored API keys", async () => {
+    const apiKey = createScopedRelationshipApiKey({ entities: [] });
+    const { dependencies } = createRelationshipDependencies(apiKey);
+
+    const response = await app.request("/v1/relationships?tenantId=tenant_1", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(STORED_API_KEY_SECRET, "req_relationship_missing_row"), { parent: "missing_parent_row", child: "child_row_1" }),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_relationship_missing_row" });
+    expectNoMutationAfterDeniedScope(dependencies);
+  });
+
+  test("denies read on GET /v1/relationships/:id when tenantId=tenant_2 tampers tenant", async () => {
+    const { dependencies } = createRelationshipDependencies();
+
+    const response = await app.request("/v1/relationships/row_relationship_1?tenantId=tenant_2", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_relationship_wrong_tenant") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_relationship_wrong_tenant" });
+    expect(dependencies.findLegacyRowRelationshipById).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyRowRelationshipById).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyRowRelationship).not.toHaveBeenCalled();
+  });
+
+  test("returns the same forbidden response for missing relationship ids with stored API keys", async () => {
+    const apiKey = createScopedRelationshipApiKey({ entities: [] });
+    const { dependencies } = createRelationshipDependencies(apiKey);
+
+    const response = await app.request("/v1/relationships/missing_relationship?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_relationship_missing_id") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_relationship_missing_id" });
+    expect(dependencies.deleteLegacyRowRelationshipById).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyRowRelationship).not.toHaveBeenCalled();
+  });
+
+  test("denies delete on DELETE /v1/relationships/:id when parent delete scope is missing", async () => {
+    const apiKey = createScopedRelationshipApiKey({
+      entities: [
+        { entityId: "parent_entity_1", read: true, create: true, update: false, delete: false },
+        { entityId: "child_entity_1", read: true, create: true, update: false, delete: true },
+      ],
+    });
+    const { dependencies } = createRelationshipDependencies(apiKey);
+
+    const response = await app.request("/v1/relationships/row_relationship_1?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_relationship_missing_parent_delete"),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_relationship_missing_parent_delete" });
+    expect(dependencies.deleteLegacyRowRelationshipById).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyRowRelationship).not.toHaveBeenCalled();
+    expect(dependencies.findLegacyEntityRelationship).not.toHaveBeenCalled();
+  });
+
+  test("denies delete on DELETE /v1/relationships/:id when child delete scope is missing", async () => {
+    const apiKey = createScopedRelationshipApiKey({
+      entities: [
+        { entityId: "parent_entity_1", read: true, create: true, update: false, delete: true },
+        { entityId: "child_entity_1", read: true, create: true, update: false, delete: false },
+      ],
+    });
+    const { dependencies } = createRelationshipDependencies(apiKey);
+
+    const response = await app.request("/v1/relationships/row_relationship_1?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_relationship_missing_child_delete"),
+    });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_relationship_missing_child_delete" });
+    expect(dependencies.deleteLegacyRowRelationshipById).not.toHaveBeenCalled();
+    expect(dependencies.createLegacyRowRelationship).not.toHaveBeenCalled();
+    expect(dependencies.findLegacyEntityRelationship).not.toHaveBeenCalled();
+  });
+
+  test("denies create on POST /v1/relationships with invalid x-api-key without service-token fallback", async () => {
+    const { dependencies } = createRelationshipDependencies();
+
+    const response = await app.request("/v1/relationships?tenantId=tenant_1", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(INVALID_API_KEY_SECRET, "req_relationship_invalid_key"), { parent: "parent_row_1", child: "child_row_1" }),
+    });
+
+    await expectRedactedError(response, { status: 401, code: "unauthorized", requestId: "req_relationship_invalid_key" });
+    expect(dependencies.findLegacyApiKeyByKey).toHaveBeenCalledWith(INVALID_API_KEY_SECRET, undefined, "scope-secret", { allowPlaintextFallback: true });
+    expect(dependencies.findLegacyRelationshipRowById).not.toHaveBeenCalled();
+    expectNoMutationAfterDeniedScope(dependencies);
+  });
+
+  test("denies delete on DELETE /v1/relationships/:id with invalid bearer token without stored-key fallback", async () => {
+    const { dependencies } = createRelationshipDependencies();
+
+    const response = await app.request("/v1/relationships/row_relationship_1?tenantId=tenant_1", {
+      method: "DELETE",
+      headers: bearerHeaders(INVALID_API_KEY_SECRET, "req_relationship_invalid_bearer"),
+    });
+
+    await expectRedactedError(response, { status: 401, code: "unauthorized", requestId: "req_relationship_invalid_bearer" });
+    expect(dependencies.findLegacyApiKeyByKey).not.toHaveBeenCalled();
+    expect(dependencies.findLegacyRowRelationshipById).not.toHaveBeenCalled();
+    expect(dependencies.deleteLegacyRowRelationshipById).not.toHaveBeenCalled();
+  });
+});

hivegrid-cloud/apps/api/src/isolation/api-key-service-boundaries.test.ts

@@ -0,0 +1,301 @@
+import { describe, expect, test, vi } from "vitest";
+
+import { app } from "../app.js";
+import {
+  TEST_API_CONFIG,
+  configureApiIsolationDependenciesForTests,
+  createLegacyEntityFixture,
+  createStoredApiKeyFixture,
+  installApiIsolationTestHooks,
+} from "./fixtures/api-key.js";
+
+installApiIsolationTestHooks();
+
+const STORED_API_KEY_SECRET = "stored-service-boundary-key-secret";
+const SECOND_TENANT_API_KEY_SECRET = "stored-service-boundary-tenant-2-key-secret";
+const INVALID_API_KEY_SECRET = "invalid-service-boundary-key-secret";
+const REQUEST_ID = "req_service_boundary";
+const SIGNED_DOWNLOAD_URL = "https://storage.local/signed/export_job_1.csv?signature=redacted";
+
+function createBoundaryApiKey(patch: Parameters<typeof createStoredApiKeyFixture>[0] = {}) {
+  return createStoredApiKeyFixture({
+    id: "api_key_boundary_1",
+    tenantId: "tenant_1",
+    tenant: { id: "tenant_1", name: "Acme", slug: "acme", deactivatedReason: null },
+    entities: [{ entityId: "entity_1", read: true, create: true, update: true, delete: true }],
+    ...patch,
+  });
+}
+
+function createExportJob(patch: Record<string, unknown> = {}) {
+  return {
+    id: "export_job_1",
+    tenantId: "tenant_1",
+    requestedByUserId: null,
+    format: "csv",
+    status: "COMPLETED" as const,
+    artifactBucket: "exports",
+    artifactKey: "tenant_1/export_job_1.csv",
+    rowCount: 1,
+    expiresAt: new Date("2999-01-02T00:00:00.000Z"),
+    errorName: null,
+    requestId: "req_export_job_1",
+    ...patch,
+  };
+}
+
+function createBoundaryDependencies() {
+  const primaryApiKey = createBoundaryApiKey();
+  const secondTenantApiKey = createBoundaryApiKey({
+    id: "api_key_boundary_2",
+    tenantId: "tenant_2",
+    tenant: { id: "tenant_2", name: "Other", slug: "other", deactivatedReason: null },
+  });
+  const exportJob = createExportJob();
+  const entity = createLegacyEntityFixture();
+
+  const dependencies = {
+    findLegacyApiKeyByKey: vi.fn(async (key: string) => {
+      if (key === STORED_API_KEY_SECRET) return primaryApiKey;
+      if (key === SECOND_TENANT_API_KEY_SECRET) return secondTenantApiKey;
+      return null;
+    }),
+    findExportJobForDownload: vi.fn(async ({ id, tenantId }: { id: string; tenantId?: string | null }) => {
+      if (id === exportJob.id && tenantId === exportJob.tenantId) return exportJob;
+      return null;
+    }),
+    createSignedDownloadUrl: vi.fn(async () => ({
+      method: "GET" as const,
+      url: SIGNED_DOWNLOAD_URL,
+      expiresAt: new Date("2026-01-01T00:05:00.000Z"),
+      expiresInSeconds: 300,
+      headers: { "x-download": "allowed" },
+    })),
+    findLegacyEntityByName: vi.fn(async () => entity),
+  };
+
+  configureApiIsolationDependenciesForTests(dependencies);
+
+  return { primaryApiKey, secondTenantApiKey, exportJob, entity, dependencies };
+}
+
+function apiKeyHeaders(secret = STORED_API_KEY_SECRET, requestId = REQUEST_ID) {
+  return { "x-api-key": secret, "x-request-id": requestId };
+}
+
+function bearerHeaders(secret: string, requestId = REQUEST_ID) {
+  return { authorization: `Bearer ${secret}`, "x-request-id": requestId };
+}
+
+function jsonHeaders(headers: Record<string, string>) {
+  return { ...headers, "content-type": "application/json" };
+}
+
+function jsonRequest(headers: Record<string, string>, body: unknown) {
+  const serialized = JSON.stringify(body);
+  return {
+    headers: { ...jsonHeaders(headers), "content-length": String(Buffer.byteLength(serialized)) },
+    body: serialized,
+  };
+}
+
+async function expectSerializedBodyRedacts(response: Response, secrets: string[]) {
+  const text = await response.text();
+  const configuredServiceToken = TEST_API_CONFIG.apiAccessToken;
+  for (const secret of secrets) {
+    expect(text).not.toContain(secret);
+  }
+  expect(text).not.toContain(configuredServiceToken);
+  expect(text).not.toContain("scope-secret");
+  return text;
+}
+
+async function expectRedactedError(response: Response, expected: { status: number; code: string; requestId: string }) {
+  expect(response.status).toBe(expected.status);
+  expect(response.headers.get("x-request-id")).toBe(expected.requestId);
+
+  const text = await expectSerializedBodyRedacts(response, [STORED_API_KEY_SECRET, SECOND_TENANT_API_KEY_SECRET, INVALID_API_KEY_SECRET]);
+  expect(text).toContain(expected.requestId);
+  expect(JSON.parse(text)).toEqual({
+    error: expect.objectContaining({
+      code: expected.code,
+      requestId: expected.requestId,
+    }),
+  });
+}
+
+describe("API-key validation service boundary", () => {
+  test("returns active stored-key metadata for the stored key tenant without raw key material", async () => {
+    const { primaryApiKey, dependencies } = createBoundaryDependencies();
+
+    const response = await app.request("/v1/api-keys/validate", {
+      method: "POST",
+      ...jsonRequest(apiKeyHeaders(STORED_API_KEY_SECRET, "req_validate_active"), { apiKey: STORED_API_KEY_SECRET }),
+    });
+
+    expect(response.status).toBe(200);
+    const text = await expectSerializedBodyRedacts(response, [STORED_API_KEY_SECRET]);
+    expect(JSON.parse(text)).toEqual({
+      active: true,
+      keyType: "stored-api-key",
+      apiKey: {
+        id: primaryApiKey.id,
+        alias: primaryApiKey.alias,
+        tenantId: "tenant_1",
+        tenant: { id: "tenant_1", name: "Acme", slug: "acme" },
+        expires: null,
+      },
+    });
+    expect(dependencies.findLegacyApiKeyByKey).toHaveBeenCalledWith(STORED_API_KEY_SECRET, undefined, "scope-secret", { allowPlaintextFallback: true });
+  });
+
+  test.each([
+    { label: "expired", patch: { expires: new Date("2000-01-01T00:00:00.000Z") } },
+    { label: "inactive", patch: { active: false } },
+    { label: "deactivated", patch: { tenant: { id: "tenant_1", name: "Acme", slug: "acme", deactivatedReason: "closed" } } },
+  ])("returns inactive stored-key metadata for $label keys without raw key or tenant material", async ({ patch }) => {
+    const inactiveKey = createBoundaryApiKey(patch);
+    configureApiIsolationDependenciesForTests({
+      findLegacyApiKeyByKey: vi.fn(async (key: string) => (key === STORED_API_KEY_SECRET ? inactiveKey : null)),
+    });
+
+    const response = await app.request("/v1/api-keys/validate", {
+      method: "POST",
+      ...jsonRequest({ "x-request-id": "req_validate_inactive" }, { apiKey: STORED_API_KEY_SECRET }),
+    });
+
+    expect(response.status).toBe(200);
+    const text = await expectSerializedBodyRedacts(response, [STORED_API_KEY_SECRET]);
+    expect(JSON.parse(text)).toMatchObject({
+      active: false,
+      keyType: "stored-api-key",
+      apiKey: { id: inactiveKey.id, alias: inactiveKey.alias },
+    });
+    expect(text).not.toContain("tenant_1");
+    expect(text).not.toContain("Acme");
+  });
+
+  test("returns service-token validation result without exposing the submitted token", async () => {
+    createBoundaryDependencies();
+    const serviceToken = TEST_API_CONFIG.apiAccessToken ?? "";
+
+    const response = await app.request("/v1/api-keys/validate", {
+      method: "POST",
+      ...jsonRequest({ "x-request-id": "req_validate_service" }, { apiKey: serviceToken }),
+    });
+
+    expect(response.status).toBe(200);
+    const text = await expectSerializedBodyRedacts(response, [serviceToken]);
+    expect(JSON.parse(text)).toEqual({ active: true, keyType: "service-token" });
+  });
+
+  test("returns inactive service-token validation for invalid keys without service-token fallback", async () => {
+    const { dependencies } = createBoundaryDependencies();
+
+    const response = await app.request("/v1/api-keys/validate", {
+      method: "POST",
+      ...jsonRequest({ "x-request-id": "req_validate_invalid" }, { apiKey: INVALID_API_KEY_SECRET }),
+    });
+
+    expect(response.status).toBe(200);
+    const text = await expectSerializedBodyRedacts(response, [INVALID_API_KEY_SECRET]);
+    expect(JSON.parse(text)).toEqual({ active: false, keyType: "service-token" });
+    expect(dependencies.findLegacyApiKeyByKey).toHaveBeenCalledWith(INVALID_API_KEY_SECRET, undefined, "scope-secret", { allowPlaintextFallback: true });
+  });
+});
+
+describe("export download service boundary", () => {
+  test("rejects stored key export lookup when tenantId query is tampered", async () => {
+    const { dependencies } = createBoundaryDependencies();
+
+    const response = await app.request("/v1/exports/export_job_1/download-url?tenantId=tenant_2", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_export_tamper") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_export_tamper" });
+    expect(dependencies.findExportJobForDownload).not.toHaveBeenCalled();
+    expect(dependencies.createSignedDownloadUrl).not.toHaveBeenCalled();
+  });
+
+  test("rejects stored keys whose tenant does not match the export job", async () => {
+    const { dependencies } = createBoundaryDependencies();
+
+    const response = await app.request("/v1/exports/export_job_1/download-url?tenantId=tenant_1", { headers: apiKeyHeaders(SECOND_TENANT_API_KEY_SECRET, "req_export_tenant_mismatch") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_export_tenant_mismatch" });
+    expect(dependencies.findExportJobForDownload).not.toHaveBeenCalled();
+    expect(dependencies.createSignedDownloadUrl).not.toHaveBeenCalled();
+  });
+
+  test("rejects inactive stored keys before export lookup", async () => {
+    const { primaryApiKey, dependencies } = createBoundaryDependencies();
+    primaryApiKey.active = false;
+
+    const response = await app.request("/v1/exports/export_job_1/download-url?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_export_inactive_key") });
+
+    await expectRedactedError(response, { status: 403, code: "forbidden", requestId: "req_export_inactive_key" });
+    expect(dependencies.findExportJobForDownload).not.toHaveBeenCalled();
+    expect(dependencies.createSignedDownloadUrl).not.toHaveBeenCalled();
+  });
+
+  test("rejects paused exports before signing artifact URLs without leaking pause reason", async () => {
+    const pauseReason = "retention pause secret_token=super-secret-value";
+    const { dependencies } = createBoundaryDependencies();
+    dependencies.findExportJobForDownload.mockResolvedValueOnce(
+      createExportJob({
+        errorName: "ExportPaused",
+        errorMessage: pauseReason,
+        artifactBucket: "exports",
+        artifactKey: "tenant_1/export_job_1.csv",
+      })
+    );
+
+    const response = await app.request("/v1/exports/export_job_1/download-url?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_export_paused") });
+
+    const body = await response.text();
+    expect(response.status).toBe(409);
+    expect(response.headers.get("x-request-id")).toBe("req_export_paused");
+    expect(JSON.parse(body)).toEqual({
+      error: expect.objectContaining({
+        code: "export_job_paused",
+        requestId: "req_export_paused",
+      }),
+    });
+    expect(body).not.toContain(pauseReason);
+    expect(body).not.toContain("super-secret-value");
+    expect(dependencies.createSignedDownloadUrl).not.toHaveBeenCalled();
+  });
+
+  test("preserves service-token lookup by requested tenant", async () => {
+    const { dependencies } = createBoundaryDependencies();
+
+    const serviceToken = TEST_API_CONFIG.apiAccessToken ?? "";
+    const response = await app.request("/v1/exports/export_job_1/download-url?tenantId=tenant_1", { headers: bearerHeaders(serviceToken, "req_export_service") });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toMatchObject({ exportJobId: "export_job_1", requestId: "req_export_service" });
+    expect(dependencies.findLegacyApiKeyByKey).not.toHaveBeenCalledWith(serviceToken, undefined, expect.any(String));
+    expect(dependencies.findExportJobForDownload).toHaveBeenCalledWith({ id: "export_job_1", tenantId: "tenant_1" });
+  });
+
+  test("denies invalid API keys without falling through to service-token lookup", async () => {
+    const { dependencies } = createBoundaryDependencies();
+
+    const response = await app.request("/v1/exports/export_job_1/download-url?tenantId=tenant_1", { headers: apiKeyHeaders(INVALID_API_KEY_SECRET, "req_export_invalid") });
+
+    await expectRedactedError(response, { status: 401, code: "unauthorized", requestId: "req_export_invalid" });
+    expect(dependencies.findLegacyApiKeyByKey).toHaveBeenCalledWith(INVALID_API_KEY_SECRET, undefined, "scope-secret", { allowPlaintextFallback: true });
+    expect(dependencies.findExportJobForDownload).not.toHaveBeenCalled();
+    expect(dependencies.createSignedDownloadUrl).not.toHaveBeenCalled();
+  });
+});
+
+describe("service-credential-only route boundaries", () => {
+  test("keeps non-CRUD entity metadata route outside stored API-key CRUD scope claims", async () => {
+    const { dependencies } = createBoundaryDependencies();
+
+    const response = await app.request("/v1/entities/contact?tenantId=tenant_1", { headers: apiKeyHeaders(STORED_API_KEY_SECRET, "req_service_only_entity") });
+
+    await expectRedactedError(response, { status: 401, code: "unauthorized", requestId: "req_service_only_entity" });
+    expect(dependencies.findLegacyApiKeyByKey).not.toHaveBeenCalled();
+    expect(dependencies.findLegacyEntityByName).not.toHaveBeenCalled();
+  });
+});

hivegrid-cloud/apps/api/src/isolation/fixtures/api-key.ts

@@ -0,0 +1,248 @@
+import { afterEach, beforeEach } from "vitest";
+
+import type { ApiConfig } from "@toolctrl/config";
+import type { LegacyApiKeySimple } from "@toolctrl/db/api-keys.server";
+import type { LegacyEntityWithDetails } from "@toolctrl/db/entities.server";
+import type { LegacyRowRelationshipDetails } from "@toolctrl/db/relationships.server";
+import type { LegacyEntityRow, LegacyRelationshipRow } from "@toolctrl/db/rows.server";
+
+import { configureApiAppDependenciesForTests } from "../../app.js";
+import { resetApiRuntimeConfigForTests, setApiRuntimeConfigForTests } from "../../config.js";
+
+export const TEST_JWT_SECRET = "scope-secret";
+
+export const TEST_API_CONFIG: ApiConfig = {
+  appName: "API Key Isolation Test API",
+  nodeEnv: "test",
+  revision: "unknown",
+  listen: { port: 3001 },
+  routing: {
+    appBaseDomain: "localhost:3000",
+    webAppHost: "app.localhost:3000",
+    adminAppHost: "admin.localhost:3000",
+    apiHost: "api.localhost:3000",
+    protocol: "http",
+  },
+  origins: {},
+  apiAccessToken: "service-secret",
+  jwtSecret: TEST_JWT_SECRET,
+  analytics: {},
+  mailjet: {},
+  stripe: {},
+  apiKeys: {
+    allowLegacyPlaintextFallback: true,
+  },
+  storage: {},
+};
+
+type DependencyReset = () => void;
+
+let testApiRuntimeConfig: ApiConfig = TEST_API_CONFIG;
+let resetApiAppDependencies: DependencyReset | undefined;
+
+function cloneApiConfig(config: ApiConfig): ApiConfig {
+  return {
+    ...config,
+    listen: { ...config.listen },
+    routing: { ...config.routing },
+    origins: { ...config.origins },
+    analytics: { ...config.analytics },
+    apiKeys: { ...config.apiKeys },
+    mailjet: { ...config.mailjet },
+    stripe: { ...config.stripe },
+    storage: { ...config.storage },
+  };
+}
+
+function mergeApiConfigPatch(config: ApiConfig, patch: Partial<ApiConfig>): ApiConfig {
+  return {
+    ...config,
+    ...patch,
+    listen: patch.listen ? { ...config.listen, ...patch.listen } : { ...config.listen },
+    routing: patch.routing ? { ...config.routing, ...patch.routing } : { ...config.routing },
+    origins: patch.origins ? { ...config.origins, ...patch.origins } : { ...config.origins },
+    analytics: patch.analytics ? { ...config.analytics, ...patch.analytics } : { ...config.analytics },
+    mailjet: patch.mailjet ? { ...config.mailjet, ...patch.mailjet } : { ...config.mailjet },
+    stripe: patch.stripe ? { ...config.stripe, ...patch.stripe } : { ...config.stripe },
+    storage: patch.storage ? { ...config.storage, ...patch.storage } : { ...config.storage },
+  };
+}
+
+export function patchApiIsolationConfig(patch: Partial<ApiConfig> = {}): ApiConfig {
+  testApiRuntimeConfig = mergeApiConfigPatch(testApiRuntimeConfig, patch);
+  setApiRuntimeConfigForTests(testApiRuntimeConfig);
+  return testApiRuntimeConfig;
+}
+
+export function resetApiIsolationFixturesForTests() {
+  resetApiAppDependencies?.();
+  resetApiAppDependencies = undefined;
+  resetApiRuntimeConfigForTests();
+  testApiRuntimeConfig = cloneApiConfig(TEST_API_CONFIG);
+}
+
+export function installApiIsolationTestHooks() {
+  beforeEach(() => {
+    resetApiIsolationFixturesForTests();
+    patchApiIsolationConfig();
+  });
+
+  afterEach(() => {
+    resetApiIsolationFixturesForTests();
+  });
+}
+
+export function configureApiIsolationDependenciesForTests(
+  dependencies: Parameters<typeof configureApiAppDependenciesForTests>[0] = {},
+): DependencyReset {
+  resetApiAppDependencies?.();
+  resetApiAppDependencies = configureApiAppDependenciesForTests(dependencies);
+
+  return () => {
+    resetApiAppDependencies?.();
+    resetApiAppDependencies = undefined;
+  };
+}
+
+export function createStoredApiKeyFixture(patch: Partial<LegacyApiKeySimple> = {}): LegacyApiKeySimple {
+  const tenantId = patch.tenantId ?? patch.tenant?.id ?? "tenant_1";
+
+  return {
+    id: "api_key_1",
+    alias: "Primary integration key",
+    expires: null,
+    tenantId,
+    active: true,
+    entities: [],
+    tenant: { id: tenantId, name: "Acme", slug: "acme", deactivatedReason: null },
+    ...patch,
+  };
+}
+
+export function createLegacyEntityFixture(patch: Partial<LegacyEntityWithDetails> = {}): LegacyEntityWithDetails {
+  return {
+    id: "entity_1",
+    moduleId: null,
+    name: "contact",
+    title: "Contact",
+    titlePlural: "Contacts",
+    active: true,
+    slug: "contacts",
+    prefix: "CON",
+    type: "default",
+    hasApi: true,
+    showInSidebar: true,
+    hasTags: true,
+    hasComments: true,
+    hasTasks: true,
+    hasActivity: true,
+    hasBulkDelete: false,
+    hasViews: true,
+    defaultVisibility: "private",
+    onCreated: "redirectToOverview",
+    icon: "User",
+    onEdit: null,
+    promptFlowGroupId: null,
+    isAutogenerated: false,
+    order: 1,
+    createdAt: new Date("2026-01-01T00:00:00.000Z"),
+    updatedAt: new Date("2026-01-01T00:00:00.000Z"),
+    properties: [
+      {
+        id: "property_1",
+        tenantId: null,
+        entityId: "entity_1",
+        name: "firstName",
+        title: "First name",
+        type: 0,
+        subtype: null,
+        isHidden: false,
+        isDisplay: false,
+        isRequired: false,
+        isDefault: false,
+        isUnique: false,
+        canUpdate: true,
+        isReadOnly: false,
+        showInCreate: true,
+        order: 1,
+        formulaId: null,
+        attributes: [],
+        options: [],
+        formula: null,
+      },
+      {
+        id: "property_2",
+        tenantId: null,
+        entityId: "entity_1",
+        name: "hiddenName",
+        title: "Hidden name",
+        type: 0,
+        subtype: null,
+        isHidden: true,
+        isDisplay: false,
+        isRequired: false,
+        isDefault: false,
+        isUnique: false,
+        canUpdate: true,
+        isReadOnly: false,
+        showInCreate: false,
+        order: 2,
+        formulaId: null,
+        attributes: [],
+        options: [],
+        formula: null,
+      },
+    ],
+    views: [],
+    tags: [],
+    parentEntities: [],
+    childEntities: [],
+    templates: [],
+    ...patch,
+  };
+}
+
+export function createLegacyRowFixture(id = "row_1", patch: Partial<LegacyEntityRow> = {}): LegacyEntityRow {
+  return {
+    id,
+    entityId: "entity_1",
+    tenantId: "tenant_1",
+    createdAt: new Date("2026-01-01T00:00:00.000Z"),
+    updatedAt: new Date("2026-01-01T00:00:00.000Z"),
+    deletedAt: null,
+    createdByUserId: null,
+    createdByApiKeyId: null,
+    folio: 1,
+    order: 1,
+    createdByUser: null,
+    createdByApiKey: null,
+    values: [],
+    parentRows: [] ,
+    childRows: [],
+    ...patch,
+  };
+}
+
+export function createLegacyRelationshipRowFixture(id = "row_1", entityId = "entity_1", patch: Partial<LegacyRelationshipRow> = {}): LegacyRelationshipRow {
+  return { id, entityId, tenantId: "tenant_1", ...patch };
+}
+
+export function createLegacyRowRelationshipFixture(
+  id = "row_relationship_1",
+  parentId = "parent_row_1",
+  childId = "child_row_1",
+  parentEntityId = "parent_entity_1",
+  childEntityId = "child_entity_1",
+  patch: Partial<LegacyRowRelationshipDetails> = {},
+): LegacyRowRelationshipDetails {
+  return {
+    id,
+    relationshipId: "entity_relationship_1",
+    parentId,
+    childId,
+    metadata: null,
+    createdAt: new Date("2026-01-01T00:00:00.000Z"),
+    relationship: { id: "entity_relationship_1", parentId: parentEntityId, childId: childEntityId },
+    ...patch,
+  };
+}

hivegrid-cloud/apps/api/src/logger.ts

@@ -0,0 +1,24 @@
+import { createSentryErrorReporter, initializeTracing, type ErrorReporter } from "@toolctrl/error-reporter";
+import { createChildLogger, defaultLogger } from "@toolctrl/logger";
+
+export const apiLogger = createChildLogger(defaultLogger, { component: "api" });
+
+const sentryDsn = process.env.SENTRY_DSN?.trim();
+const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim();
+const otelServiceName = process.env.OTEL_SERVICE_NAME?.trim() || "api";
+
+export const apiErrorReporter: ErrorReporter | undefined = sentryDsn
+  ? createSentryErrorReporter({
+      dsn: sentryDsn,
+      environment: process.env.SENTRY_ENVIRONMENT?.trim() || process.env.NODE_ENV,
+      release: process.env.APP_REVISION?.trim(),
+    })
+  : undefined;
+
+if (otlpEndpoint) {
+  void initializeTracing({
+    serviceName: otelServiceName,
+    otlpEndpoint,
+    enabled: true,
+  });
+}

hivegrid-cloud/apps/api/src/metrics.ts

@@ -0,0 +1,50 @@
+/**
+ * API service-scoped Prometheus registry singleton.
+ *
+ * Lazily constructed so test files that mock env vars can swap the registry
+ * via {@link resetApiMetricsForTests}. The singleton is cheap (~30 KB of
+ * prom-client default metrics) and is intentionally NOT gated behind
+ * METRICS_ENABLED — the gate lives on the endpoints, not the registry. This
+ * lets business code call `apiMetrics.aiTokensConsumedTotal.inc(...)` without
+ * branching on an env flag everywhere.
+ */
+import { createDefaultMetrics, createMetricsRegistry, type DefaultMetrics, type MetricsRegistry } from "@toolctrl/metrics";
+import { setEnqueueObserver } from "@toolctrl/queue";
+
+type ApiMetricsBundle = {
+  registry: MetricsRegistry;
+  metrics: DefaultMetrics;
+};
+
+let cached: ApiMetricsBundle | undefined;
+
+export function getApiMetrics(): ApiMetricsBundle {
+  if (!cached) {
+    cached = buildApiMetrics();
+  }
+  return cached;
+}
+
+/**
+ * Reset the cached bundle. Test-only — production code never calls this.
+ */
+export function resetApiMetricsForTests(): void {
+  cached = undefined;
+}
+
+function buildApiMetrics(): ApiMetricsBundle {
+  const revision = process.env.APP_REVISION?.trim() ?? "";
+  const registry = createMetricsRegistry({
+    serviceName: "api",
+    collectDefaultProcessMetrics: true,
+    ...(revision ? { defaultLabels: { revision } } : {}),
+  });
+  const metrics = createDefaultMetrics(registry);
+  // Wire BullMQ enqueue events into queue_jobs_enqueued_total. The queue
+  // package keeps a module-level observer slot so we register exactly once
+  // per process; subsequent registrations from re-built bundles overwrite.
+  setEnqueueObserver((event) => {
+    metrics.queueJobsEnqueuedTotal.labels({ queue: event.queueName, priority: event.priorityLabel }).inc();
+  });
+  return { registry, metrics };
+}

hivegrid-cloud/apps/api/src/middleware/audit-context.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it } from "vitest";
+
+import { createAuditContextMiddleware, extractActorIp, extractActorUserAgent, getAuditContext, runWithAuditContext } from "./audit-context.js";
+
+describe("extractActorIp", () => {
+  it("prefers cf-connecting-ip, then x-real-ip, then x-forwarded-for first hop", () => {
+    expect(extractActorIp((n) => (n === "cf-connecting-ip" ? "1.1.1.1" : "9.9.9.9"))).toBe("1.1.1.1");
+    expect(extractActorIp((n) => (n === "x-real-ip" ? "2.2.2.2" : null))).toBe("2.2.2.2");
+    expect(extractActorIp((n) => (n === "x-forwarded-for" ? "3.3.3.3, 8.8.8.8" : null))).toBe("3.3.3.3");
+  });
+
+  it("returns null for missing or oversized values", () => {
+    expect(extractActorIp(() => null)).toBeNull();
+    expect(extractActorIp(() => "x".repeat(200))).toBeNull();
+  });
+});
+
+describe("extractActorUserAgent", () => {
+  it("returns trimmed value capped at 512 chars", () => {
+    expect(extractActorUserAgent(() => "  ua  ")).toBe("ua");
+    expect(extractActorUserAgent(() => "x".repeat(600))?.length).toBe(512);
+    expect(extractActorUserAgent(() => "")).toBeNull();
+    expect(extractActorUserAgent(() => null)).toBeNull();
+  });
+});
+
+describe("audit context async local storage", () => {
+  it("getAuditContext returns null outside any run scope", () => {
+    expect(getAuditContext()).toBeNull();
+  });
+
+  it("runWithAuditContext binds and unbinds the store", async () => {
+    const seen = await runWithAuditContext({ actorIp: "10.0.0.1", actorUserAgent: "vitest", requestId: "req_1" }, async () => {
+      return getAuditContext();
+    });
+    expect(seen).toEqual({ actorIp: "10.0.0.1", actorUserAgent: "vitest", requestId: "req_1" });
+    expect(getAuditContext()).toBeNull();
+  });
+
+  it("middleware captures headers and binds context for downstream handlers", async () => {
+    const middleware = createAuditContextMiddleware();
+    let captured: ReturnType<typeof getAuditContext> = null;
+    const c = {
+      req: {
+        header(name: string): string | undefined {
+          if (name === "x-forwarded-for") return "172.16.0.5, 8.8.8.8";
+          if (name === "user-agent") return "Test/1.0";
+          return undefined;
+        },
+      },
+      get(name: string): unknown {
+        return name === "requestId" ? "req_1" : undefined;
+      },
+    } as unknown as Parameters<typeof middleware>[0];
+    await middleware(c, async () => {
+      captured = getAuditContext();
+    });
+    expect(captured).toEqual({ actorIp: "172.16.0.5", actorUserAgent: "Test/1.0", requestId: "req_1" });
+  });
+});

hivegrid-cloud/apps/api/src/middleware/audit-context.ts

@@ -0,0 +1,76 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { MiddlewareHandler } from "hono";
+
+/**
+ * Phase 10 Task #67 — Audit auto-capture middleware (API).
+ *
+ * Captures the request-level audit context (actor IP, user agent, request
+ * id) once per Hono request and stows it in AsyncLocalStorage so deeper
+ * call sites — controllers, services, repository helpers — can read the
+ * context via {@link getAuditContext} without having to thread the
+ * Request object through their signatures.
+ *
+ * The middleware is read-only and never throws: a missing header is just
+ * recorded as `null`. The store is scoped to the request lifetime; once
+ * `next()` resolves the binding is dropped automatically by ALS.
+ */
+
+export type AuditContext = {
+  actorIp: string | null;
+  actorUserAgent: string | null;
+  requestId: string | null;
+};
+
+const auditContextStore = new AsyncLocalStorage<AuditContext>();
+
+const FORWARDED_HEADERS = ["cf-connecting-ip", "x-real-ip", "x-forwarded-for"] as const;
+
+export function extractActorIp(headerLookup: (name: string) => string | null | undefined): string | null {
+  for (const name of FORWARDED_HEADERS) {
+    const raw = headerLookup(name);
+    if (typeof raw !== "string") continue;
+    const first = raw.split(",")[0]?.trim();
+    if (first && first.length > 0 && first.length <= 64) return first;
+  }
+  return null;
+}
+
+export function extractActorUserAgent(headerLookup: (name: string) => string | null | undefined): string | null {
+  const raw = headerLookup("user-agent");
+  if (typeof raw !== "string") return null;
+  const trimmed = raw.trim();
+  if (trimmed.length === 0) return null;
+  // Cap to 512 chars to bound storage cost and avoid pathological UA logs.
+  return trimmed.length > 512 ? trimmed.slice(0, 512) : trimmed;
+}
+
+export function createAuditContextMiddleware(): MiddlewareHandler {
+  return async (c, next) => {
+    const ctx: AuditContext = {
+      actorIp: extractActorIp((n) => c.req.header(n)),
+      actorUserAgent: extractActorUserAgent((n) => c.req.header(n)),
+      requestId: (c.get("requestId") as string | undefined) ?? null,
+    };
+    await auditContextStore.run(ctx, async () => {
+      await next();
+    });
+  };
+}
+
+/**
+ * Read the current request's audit context. Returns null when invoked
+ * outside the middleware (e.g. in a worker or test harness without an
+ * explicit run scope).
+ */
+export function getAuditContext(): AuditContext | null {
+  return auditContextStore.getStore() ?? null;
+}
+
+/**
+ * Test/imperative helper: run a function with an explicit audit context.
+ * Useful for unit tests and worker jobs that want to attribute writes to
+ * a specific actor without using a full HTTP request.
+ */
+export async function runWithAuditContext<T>(ctx: AuditContext, fn: () => Promise<T>): Promise<T> {
+  return auditContextStore.run(ctx, fn);
+}

hivegrid-cloud/apps/api/src/middleware/body-limit.test.ts

@@ -0,0 +1,176 @@
+import { Hono } from "hono";
+import { describe, expect, test, vi } from "vitest";
+import { BODY_LIMIT_HEADER, createBodyLimitMiddleware } from "./body-limit.js";
+import { createRequestIdMiddleware, type ApiRequestIdVariables } from "./request-id.js";
+
+type TestEnv = {
+  Variables: ApiRequestIdVariables;
+};
+
+function createTestApp() {
+  const app = new Hono<TestEnv>();
+  const writeHandler = vi.fn((c) => c.json({ ok: true, requestId: c.get("requestId") }));
+  const readHandler = vi.fn((c) => c.json({ ok: true, requestId: c.get("requestId") }));
+
+  app.use("*", createRequestIdMiddleware());
+  app.use("*", createBodyLimitMiddleware({ jsonLimitBytes: 100, uploadLimitBytes: 1_000 }));
+  app.post("/v1/analytics/events", writeHandler);
+  app.post("/v1/ai/upload", writeHandler);
+  app.post("/v1/widgets/:id", writeHandler);
+  app.get("/v1/entities/widgets/rows", readHandler);
+
+  return { app, readHandler, writeHandler };
+}
+
+describe("createBodyLimitMiddleware", () => {
+  test("rejects oversized JSON write requests by content-length without echoing body or secrets", async () => {
+    const { app, writeHandler } = createTestApp();
+
+    const response = await app.request("/v1/analytics/events", {
+      method: "POST",
+      headers: {
+        "content-length": "101",
+        "content-type": "application/json",
+        "x-request-id": "req_body_limit_json",
+      },
+      body: JSON.stringify({ apiKey: "secret-api-key", token: "secret-token", arbitrary: "body" }),
+    });
+    const responseText = await response.text();
+
+    expect(response.status).toBe(413);
+    expect(response.headers.get("x-request-id")).toBe("req_body_limit_json");
+    expect(response.headers.get(BODY_LIMIT_HEADER)).toBe("100");
+    expect(responseText).not.toContain("secret-api-key");
+    expect(responseText).not.toContain("secret-token");
+    expect(responseText).not.toContain("arbitrary");
+    expect(JSON.parse(responseText)).toEqual({
+      error: {
+        code: "body_too_large",
+        message: "Request body is too large.",
+        requestId: "req_body_limit_json",
+      },
+    });
+    expect(writeHandler).not.toHaveBeenCalled();
+  });
+
+  test("applies the JSON write limit to widget actions", async () => {
+    const { app, writeHandler } = createTestApp();
+
+    const response = await app.request("/v1/widgets/widget_1", {
+      method: "POST",
+      headers: {
+        "content-length": "101",
+        "content-type": "application/x-www-form-urlencoded",
+        "x-request-id": "req_body_limit_widget",
+      },
+      body: "action=test-widget&content=secret-widget-content",
+    });
+
+    expect(response.status).toBe(413);
+    expect(response.headers.get(BODY_LIMIT_HEADER)).toBe("100");
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "body_too_large",
+        message: "Request body is too large.",
+        requestId: "req_body_limit_widget",
+      },
+    });
+    expect(writeHandler).not.toHaveBeenCalled();
+  });
+
+  test.each(["/v1/ai/upload", "/v1/ai/upload/tenant_1/asset.png"])(
+    "uses the explicit upload limit for AI upload requests at %s before handlers run",
+    async (path) => {
+      const { app, writeHandler } = createTestApp();
+
+      const response = await app.request(path, {
+        method: "POST",
+        headers: {
+          "content-length": "1001",
+          "content-type": "application/octet-stream",
+          "x-request-id": "req_body_limit_upload",
+        },
+        body: "upload-secret-content",
+      });
+
+      expect(response.status).toBe(413);
+      expect(response.headers.get("x-request-id")).toBe("req_body_limit_upload");
+      expect(response.headers.get(BODY_LIMIT_HEADER)).toBe("1000");
+      await expect(response.json()).resolves.toEqual({
+        error: {
+          code: "body_too_large",
+          message: "Request body is too large.",
+          requestId: "req_body_limit_upload",
+        },
+      });
+      expect(writeHandler).not.toHaveBeenCalled();
+    }
+  );
+
+  test("allows GET requests even when content-length exceeds write limits", async () => {
+    const { app, readHandler } = createTestApp();
+
+    const response = await app.request("/v1/entities/widgets/rows", {
+      method: "GET",
+      headers: {
+        "content-length": "999999",
+        "x-request-id": "req_body_limit_get",
+      },
+    });
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get("x-request-id")).toBe("req_body_limit_get");
+    expect(response.headers.get(BODY_LIMIT_HEADER)).toBeNull();
+    await expect(response.json()).resolves.toEqual({ ok: true, requestId: "req_body_limit_get" });
+    expect(readHandler).toHaveBeenCalledOnce();
+  });
+
+  test("rejects write requests with bodies when content-length is missing or malformed", async () => {
+    const { app, writeHandler } = createTestApp();
+
+    const missingLengthResponse = await app.request("/v1/analytics/events", {
+      method: "POST",
+      headers: { "x-request-id": "req_body_limit_missing" },
+      body: "{}",
+    });
+    const malformedLengthResponse = await app.request("/v1/analytics/events", {
+      method: "POST",
+      headers: { "content-length": "not-a-number", "x-request-id": "req_body_limit_malformed" },
+      body: "{}",
+    });
+
+    expect(missingLengthResponse.status).toBe(411);
+    expect(malformedLengthResponse.status).toBe(411);
+    expect(missingLengthResponse.headers.get(BODY_LIMIT_HEADER)).toBe("100");
+    expect(malformedLengthResponse.headers.get(BODY_LIMIT_HEADER)).toBe("100");
+    await expect(missingLengthResponse.json()).resolves.toEqual({
+      error: {
+        code: "content_length_required",
+        message: "Write requests with a body require a valid Content-Length header.",
+        requestId: "req_body_limit_missing",
+      },
+    });
+    await expect(malformedLengthResponse.json()).resolves.toEqual({
+      error: {
+        code: "content_length_required",
+        message: "Write requests with a body require a valid Content-Length header.",
+        requestId: "req_body_limit_malformed",
+      },
+    });
+    expect(writeHandler).not.toHaveBeenCalled();
+  });
+
+  test("allows write requests without bodies when content-length is absent", async () => {
+    const { app, writeHandler } = createTestApp();
+
+    const response = await app.request("/v1/analytics/events", {
+      method: "POST",
+      headers: { "x-request-id": "req_body_limit_no_body" },
+    });
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get(BODY_LIMIT_HEADER)).toBe("100");
+    await expect(response.json()).resolves.toEqual({ ok: true, requestId: "req_body_limit_no_body" });
+    expect(writeHandler).toHaveBeenCalledOnce();
+  });
+});

hivegrid-cloud/apps/api/src/middleware/body-limit.ts

@@ -0,0 +1,73 @@
+import type { MiddlewareHandler } from "hono";
+import { apiError, type ApiErrorEnv } from "./error-envelope.js";
+
+export const DEFAULT_JSON_BODY_LIMIT_BYTES = 1_048_576;
+export const DEFAULT_UPLOAD_BODY_LIMIT_BYTES = 10_485_760;
+export const BODY_LIMIT_HEADER = "x-api-body-limit";
+
+const WRITE_METHODS = new Set(["POST", "PUT", "PATCH"]);
+
+type BodyLimitRule = {
+  limitBytes: number;
+  pathPrefix?: string;
+};
+
+type BodyLimitMiddlewareOptions = {
+  jsonLimitBytes?: number;
+  uploadLimitBytes?: number;
+  uploadPathPrefix?: string;
+};
+
+export function createBodyLimitMiddleware(options: BodyLimitMiddlewareOptions = {}): MiddlewareHandler<ApiErrorEnv> {
+  const jsonLimitBytes = options.jsonLimitBytes ?? DEFAULT_JSON_BODY_LIMIT_BYTES;
+  const uploadLimitBytes = options.uploadLimitBytes ?? DEFAULT_UPLOAD_BODY_LIMIT_BYTES;
+  const uploadPathPrefix = options.uploadPathPrefix ?? "/v1/ai/upload";
+
+  return async (c, next) => {
+    const method = c.req.method.toUpperCase();
+    if (!WRITE_METHODS.has(method)) {
+      await next();
+      return;
+    }
+
+    const rule = getBodyLimitRule(c.req.path, { jsonLimitBytes, uploadLimitBytes, uploadPathPrefix });
+    const contentLength = parseContentLength(c.req.header("content-length"));
+    c.header(BODY_LIMIT_HEADER, String(rule.limitBytes));
+
+    if (contentLength === null && requestHasBody(c.req.raw)) {
+      return apiError(c, 411, "content_length_required", "Write requests with a body require a valid Content-Length header.");
+    }
+
+    if (contentLength !== null && contentLength > rule.limitBytes) {
+      return apiError(c, 413, "body_too_large", "Request body is too large.");
+    }
+
+    await next();
+  };
+}
+
+function getBodyLimitRule(path: string, options: Required<BodyLimitMiddlewareOptions>): BodyLimitRule {
+  if (path === options.uploadPathPrefix || path.startsWith(`${options.uploadPathPrefix}/`)) {
+    return { limitBytes: options.uploadLimitBytes, pathPrefix: options.uploadPathPrefix };
+  }
+
+  return { limitBytes: options.jsonLimitBytes };
+}
+
+function parseContentLength(value: string | undefined): number | null {
+  if (!value) {
+    return null;
+  }
+
+  const normalized = value.trim();
+  if (!/^\d+$/.test(normalized)) {
+    return null;
+  }
+
+  const parsed = Number(normalized);
+  return Number.isSafeInteger(parsed) ? parsed : null;
+}
+
+function requestHasBody(request: Request) {
+  return request.body !== null;
+}

hivegrid-cloud/apps/api/src/middleware/error-envelope.test.ts

@@ -0,0 +1,182 @@
+import { Hono } from "hono";
+import { HTTPException } from "hono/http-exception";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import type { ErrorReporter } from "@toolctrl/error-reporter";
+import { apiLogger } from "../logger.js";
+import { apiError, createErrorHandler, createNotFoundHandler, normalizeErrorStatus, type ApiErrorEnv } from "./error-envelope.js";
+
+type TestEnv = ApiErrorEnv;
+
+function createTestApp(requestId = "req_error_test", reporter?: ErrorReporter) {
+  const app = new Hono<TestEnv>();
+
+  app.use("*", async (c, next) => {
+    c.set("requestId", requestId);
+    await next();
+  });
+  app.notFound(createNotFoundHandler());
+  app.onError(createErrorHandler({ reporter }));
+
+  return app;
+}
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe("error envelope middleware", () => {
+  test("returns a request-ID-correlated not-found envelope", async () => {
+    const app = createTestApp("req_missing_route");
+
+    const response = await app.request("/missing");
+
+    expect(response.status).toBe(404);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "not_found",
+        message: "Endpoint not found.",
+        requestId: "req_missing_route",
+      },
+    });
+  });
+
+  test("normalizes HTTPException statuses into public http_error envelopes", async () => {
+    const app = createTestApp("req_http_exception");
+    app.get("/teapot", () => {
+      throw new HTTPException(418, { message: "Short and stout." });
+    });
+
+    const response = await app.request("/teapot");
+
+    expect(response.status).toBe(400);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "http_error",
+        message: "Request failed.",
+        params: { status: 418 },
+        requestId: "req_http_exception",
+      },
+    });
+  });
+
+  test("redacts unhandled error details while logging request ID and reporting sanitized context once", async () => {
+    const reporter: ErrorReporter = { report: vi.fn() };
+    const app = createTestApp("req_unhandled_secret", reporter);
+    const error = new Error("postgres://user:secret@db.local/app token=fake-secret stack trace");
+    Object.assign(error, {
+      connectionString: "postgres://user:secret@db.local/app",
+      token: "fake-secret",
+    });
+    const loggerError = vi.spyOn(apiLogger, "error").mockImplementation(() => undefined);
+    app.get("/boom", () => {
+      throw error;
+    });
+
+    const response = await app.request("/boom?token=fake-secret", {
+      headers: {
+        authorization: "Bearer fake-secret",
+        cookie: "session=fake-secret",
+      },
+    });
+    const responseText = await response.text();
+
+    expect(response.status).toBe(500);
+    expect(responseText).not.toContain("fake-secret");
+    expect(responseText).not.toContain("postgres://");
+    expect(responseText).not.toContain("stack trace");
+    expect(JSON.parse(responseText)).toEqual({
+      error: {
+        code: "internal_error",
+        message: "Internal server error.",
+        requestId: "req_unhandled_secret",
+      },
+    });
+    expect(loggerError).toHaveBeenCalledWith(
+      {
+        requestId: "req_unhandled_secret",
+        error,
+      },
+      "Unhandled API error",
+    );
+    expect(loggerError.mock.calls[0]?.[0]).not.toEqual(
+      expect.objectContaining({
+        body: expect.anything(),
+        headers: expect.anything(),
+        token: expect.anything(),
+      }),
+    );
+    expect(reporter.report).toHaveBeenCalledTimes(1);
+    expect(reporter.report).toHaveBeenCalledWith(
+      expect.objectContaining({
+        component: "api",
+        operation: "http.unhandled_error",
+        severity: "error",
+        requestId: "req_unhandled_secret",
+        method: "GET",
+        path: "/boom",
+        status: 500,
+        error: expect.objectContaining({
+          errorName: "Error",
+          token: "[Redacted]",
+          connectionString: "[Redacted]",
+        }),
+      }),
+    );
+
+    const serializedReport = JSON.stringify(vi.mocked(reporter.report).mock.calls);
+    expect(serializedReport).not.toContain("fake-secret");
+    expect(serializedReport).not.toContain("postgres://");
+    expect(serializedReport).not.toContain("authorization");
+    expect(serializedReport).not.toContain("cookie");
+    expect(serializedReport).not.toContain("body");
+  });
+
+  test("swallows reporter failures without changing the 500 envelope", async () => {
+    const reporter: ErrorReporter = {
+      async report() {
+        throw new Error("reporter transport failed");
+      },
+    };
+    const app = createTestApp("req_reporter_failure", reporter);
+    app.get("/reporter-failure", () => {
+      throw new Error("database token=fake-secret failed");
+    });
+
+    const response = await app.request("/reporter-failure");
+
+    expect(response.status).toBe(500);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "internal_error",
+        message: "Internal server error.",
+        requestId: "req_reporter_failure",
+      },
+    });
+  });
+
+  test("includes the current context request ID in direct error envelopes", async () => {
+    const app = createTestApp("req_direct_envelope");
+    app.get("/direct", (c) => apiError(c, 403, "forbidden", "Forbidden."));
+
+    const response = await app.request("/direct");
+
+    expect(response.status).toBe(403);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "forbidden",
+        message: "Forbidden.",
+        requestId: "req_direct_envelope",
+      },
+    });
+  });
+});
+
+describe("normalizeErrorStatus", () => {
+  test("keeps supported public statuses and buckets unsupported statuses", () => {
+    expect(normalizeErrorStatus(401)).toBe(401);
+    expect(normalizeErrorStatus(403)).toBe(403);
+    expect(normalizeErrorStatus(404)).toBe(404);
+    expect(normalizeErrorStatus(418)).toBe(400);
+    expect(normalizeErrorStatus(503)).toBe(500);
+  });
+});

hivegrid-cloud/apps/api/src/middleware/error-envelope.ts

@@ -0,0 +1,77 @@
+import { reportError, type ErrorReporter } from "@toolctrl/error-reporter";
+import type { Context } from "hono";
+import { HTTPException } from "hono/http-exception";
+import { apiLogger } from "../logger.js";
+import type { ApiRequestIdVariables } from "./request-id.js";
+
+type ApiErrorStatus = 400 | 401 | 402 | 403 | 404 | 409 | 410 | 411 | 413 | 415 | 429 | 500 | 503;
+export type ApiErrorParams = Record<string, string | number | boolean | null>;
+
+export type ApiErrorEnv = {
+  Variables: ApiRequestIdVariables;
+};
+
+export type ApiErrorOptions = {
+  params?: ApiErrorParams;
+};
+
+export function apiError(c: Context<ApiErrorEnv>, status: ApiErrorStatus, code: string, message: string, options: ApiErrorOptions = {}) {
+  return c.json(
+    {
+      error: {
+        code,
+        message,
+        ...(options.params && Object.keys(options.params).length > 0 ? { params: options.params } : {}),
+        requestId: c.get("requestId"),
+      },
+    },
+    status
+  );
+}
+
+export function normalizeErrorStatus(status: number): ApiErrorStatus {
+  if (status === 401 || status === 403 || status === 404 || status === 409) {
+    return status;
+  }
+
+  return status >= 500 ? 500 : 400;
+}
+
+export function createNotFoundHandler() {
+  return (c: Context<ApiErrorEnv>) => apiError(c, 404, "not_found", "Endpoint not found.");
+}
+
+export interface CreateErrorHandlerOptions {
+  reporter?: ErrorReporter;
+}
+
+export function createErrorHandler(options: CreateErrorHandlerOptions = {}) {
+  return (error: Error, c: Context<ApiErrorEnv>) => {
+    if (error instanceof HTTPException) {
+      return apiError(c, normalizeErrorStatus(error.status), "http_error", "Request failed.", { params: { status: error.status } });
+    }
+
+    const requestId = c.get("requestId");
+
+    apiLogger.error(
+      {
+        requestId,
+        error,
+      },
+      "Unhandled API error",
+    );
+
+    reportError(options.reporter, {
+      component: "api",
+      operation: "http.unhandled_error",
+      severity: "error",
+      requestId,
+      method: c.req.method,
+      path: c.req.path,
+      status: 500,
+      error,
+    });
+
+    return apiError(c, 500, "internal_error", "Internal server error.");
+  };
+}

hivegrid-cloud/apps/api/src/middleware/metrics.test.ts

@@ -0,0 +1,82 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { Hono } from "hono";
+import { createDefaultMetrics, createMetricsRegistry, type DefaultMetrics, type MetricsRegistry } from "@toolctrl/metrics";
+import { createMetricsMiddleware } from "./metrics.js";
+
+let registry: MetricsRegistry;
+let metrics: DefaultMetrics;
+
+beforeEach(() => {
+  registry = createMetricsRegistry({ serviceName: "api-test", collectDefaultProcessMetrics: false });
+  metrics = createDefaultMetrics(registry);
+});
+
+describe("createMetricsMiddleware", () => {
+  it("is a no-op when disabled", async () => {
+    const app = new Hono();
+    app.use("*", createMetricsMiddleware({ enabled: false, metrics }));
+    app.get("/v1/foo", (c) => c.json({ ok: true }));
+    const res = await app.request("/v1/foo");
+    expect(res.status).toBe(200);
+    const text = await registry.metrics();
+    expect(text).not.toContain("http_requests_total{");
+  });
+
+  it("records counter + histogram with route pattern (not raw URL)", async () => {
+    const app = new Hono();
+    app.use("*", createMetricsMiddleware({ enabled: true, metrics }));
+    app.get("/v1/widgets/:id", (c) => c.json({ id: c.req.param("id") }));
+    const res = await app.request("/v1/widgets/abc-123");
+    expect(res.status).toBe(200);
+    const text = await registry.metrics();
+    expect(text).toMatch(/http_requests_total\{[^}]*method="GET"[^}]*route="\/v1\/widgets\/:id"[^}]*status="200"[^}]*\} 1/);
+    expect(text).not.toContain("abc-123");
+    expect(text).toMatch(/http_request_duration_seconds_count\{[^}]*route="\/v1\/widgets\/:id"[^}]*\} 1/);
+  });
+
+  it("labels unmatched routes as <unmatched>", async () => {
+    const app = new Hono();
+    app.use("*", createMetricsMiddleware({ enabled: true, metrics }));
+    const res = await app.request("/does-not-exist");
+    expect(res.status).toBe(404);
+    const text = await registry.metrics();
+    expect(text).toContain('route="<unmatched>"');
+  });
+
+  it("records 500 when the handler throws", async () => {
+    const app = new Hono();
+    app.use("*", createMetricsMiddleware({ enabled: true, metrics }));
+    app.get("/boom", () => {
+      throw new Error("kaboom");
+    });
+    app.onError((_err, c) => c.json({ error: "internal" }, 500));
+    const res = await app.request("/boom");
+    expect(res.status).toBe(500);
+    const text = await registry.metrics();
+    expect(text).toContain('route="/boom"');
+    expect(text).toContain('status="500"');
+  });
+
+  it("uses an injectable clock for deterministic durations", async () => {
+    let t = 0;
+    const app = new Hono();
+    app.use(
+      "*",
+      createMetricsMiddleware({
+        enabled: true,
+        metrics,
+        now: () => {
+          const next = t;
+          t += 250; // 250ms between start and end
+          return next;
+        },
+      }),
+    );
+    app.get("/v1/slow", (c) => c.json({ ok: true }));
+    await app.request("/v1/slow");
+    const text = await registry.metrics();
+    // 250ms => 0.25s — should bump the 0.5 bucket but not the 0.1 bucket
+    expect(text).toMatch(/http_request_duration_seconds_bucket\{le="0.1",[^}]*route="\/v1\/slow"[^}]*\} 0/);
+    expect(text).toMatch(/http_request_duration_seconds_bucket\{le="0.5",[^}]*route="\/v1\/slow"[^}]*\} 1/);
+  });
+});

hivegrid-cloud/apps/api/src/middleware/metrics.ts

@@ -0,0 +1,81 @@
+/**
+ * Per-request Prometheus instrumentation for the Hono API.
+ *
+ * Increments `http_requests_total` and observes `http_request_duration_seconds`
+ * with labels `{ method, route, status }`. The `route` label is the matched
+ * route PATTERN (`/v1/tenants/:id`), never the raw URL — that keeps the metric
+ * cardinality bounded by the route table, not by tenant/user input.
+ *
+ * Self-disables when `METRICS_ENABLED` is false: the middleware simply calls
+ * `next()` without touching the registry. This keeps dev / Sentry-less envs
+ * zero-overhead.
+ *
+ * @see docs/improvement-plan/02-observability/metrics-grafana.md
+ */
+import type { Context, MiddlewareHandler } from "hono";
+import type { DefaultMetrics } from "@toolctrl/metrics";
+
+export type ApiMetricsMiddlewareOptions = {
+  enabled: boolean;
+  metrics: Pick<DefaultMetrics, "httpRequestsTotal" | "httpRequestDurationSeconds">;
+  /**
+   * Optional time source — tests inject a deterministic clock. Defaults to
+   * `performance.now()` for sub-millisecond precision.
+   */
+  now?: () => number;
+};
+
+const UNKNOWN_ROUTE = "<unmatched>";
+const UNKNOWN_METHOD = "UNKNOWN";
+
+export function createMetricsMiddleware(options: ApiMetricsMiddlewareOptions): MiddlewareHandler {
+  const { enabled, metrics } = options;
+  const now = options.now ?? (() => performance.now());
+
+  if (!enabled) {
+    return async (_c, next) => {
+      await next();
+    };
+  }
+
+  return async (c, next) => {
+    const started = now();
+    let nextError: unknown;
+    try {
+      await next();
+    } catch (error) {
+      nextError = error;
+      throw error;
+    } finally {
+      const durationSeconds = (now() - started) / 1000;
+      const method = (c.req.method ?? UNKNOWN_METHOD).toUpperCase();
+      const route = pickRouteLabel(c);
+      const status = pickStatusLabel(c, nextError);
+      try {
+        metrics.httpRequestsTotal.labels({ method, route, status }).inc();
+        metrics.httpRequestDurationSeconds.labels({ method, route, status }).observe(durationSeconds);
+      } catch {
+        // Metric write must never break a real request.
+      }
+    }
+  };
+}
+
+function pickRouteLabel(c: Context): string {
+  const matched = c.req.routePath;
+  if (typeof matched === "string" && matched.length > 0 && matched !== "/*") {
+    return matched;
+  }
+  return UNKNOWN_ROUTE;
+}
+
+function pickStatusLabel(c: Context, error: unknown): string {
+  if (error) {
+    return "500";
+  }
+  const status = c.res?.status;
+  if (typeof status === "number" && Number.isFinite(status)) {
+    return String(status);
+  }
+  return "0";
+}

hivegrid-cloud/apps/api/src/middleware/request-id.test.ts

@@ -0,0 +1,128 @@
+import { Hono } from "hono";
+import { describe, expect, test, vi } from "vitest";
+import { createRequestIdMiddleware, type ApiRequestIdVariables, type RequestIdLogger } from "./request-id.js";
+
+type TestEnv = {
+  Variables: ApiRequestIdVariables;
+};
+
+function createLogger(): Required<RequestIdLogger> {
+  return {
+    error: vi.fn(),
+    info: vi.fn(),
+  };
+}
+
+function createTestApp(logger?: RequestIdLogger) {
+  const app = new Hono<TestEnv>();
+
+  app.use("*", createRequestIdMiddleware({ logger }));
+  app.get("/request-id", (c) => c.json({ requestId: c.get("requestId") }));
+  app.get("/boom", () => {
+    throw new Error("boom");
+  });
+  app.onError((error, c) => c.json({ requestId: c.get("requestId"), message: error.message }, 500));
+
+  return app;
+}
+
+describe("createRequestIdMiddleware", () => {
+  test("echoes a trusted incoming request ID in the response body and header", async () => {
+    const logger = createLogger();
+    const app = createTestApp(logger);
+
+    const response = await app.request("/request-id", {
+      headers: { "x-request-id": "req_existing_123" },
+    });
+
+    await expect(response.json()).resolves.toEqual({ requestId: "req_existing_123" });
+    expect(response.headers.get("x-request-id")).toBe("req_existing_123");
+    expect(logger.info).toHaveBeenCalledWith(
+      expect.objectContaining({
+        durationMs: expect.any(Number),
+        method: "GET",
+        path: "/request-id",
+        requestId: "req_existing_123",
+        status: 200,
+      }),
+      "API request completed"
+    );
+  });
+
+  test("generates a non-empty request ID when the request omits the header", async () => {
+    const app = createTestApp();
+
+    const response = await app.request("/request-id");
+    const body = (await response.json()) as { requestId: string };
+
+    expect(body.requestId).toEqual(expect.any(String));
+    expect(body.requestId).not.toHaveLength(0);
+    expect(response.headers.get("x-request-id")).toBe(body.requestId);
+  });
+
+  test.each([
+    ["blank", "   "],
+    ["oversized", "r".repeat(129)],
+    ["DEL control character", "req\u007Fpoison"],
+  ])("generates a safe request ID for a %s incoming header", async (_name, incomingRequestId) => {
+    const app = createTestApp();
+    const request = new Request("http://localhost/request-id");
+    vi.spyOn(request.headers, "get").mockReturnValue(incomingRequestId);
+
+    const response = await app.request(request);
+    const body = (await response.json()) as { requestId: string };
+
+    expect(body.requestId).toEqual(expect.any(String));
+    expect(body.requestId).not.toBe(incomingRequestId.trim());
+    expect(response.headers.get("x-request-id")).toBe(body.requestId);
+  });
+
+  test("makes the request ID available to downstream handlers", async () => {
+    const app = createTestApp();
+
+    const response = await app.request("/request-id", {
+      headers: { "x-request-id": "req_context_456" },
+    });
+
+    await expect(response.json()).resolves.toEqual({ requestId: "req_context_456" });
+  });
+
+  test("keeps generated response header parity when downstream returns JSON", async () => {
+    const app = createTestApp();
+
+    const response = await app.request("/request-id");
+    const body = (await response.json()) as { requestId: string };
+
+    expect(response.headers.get("content-type")).toContain("application/json");
+    expect(response.headers.get("x-request-id")).toBe(body.requestId);
+  });
+
+  test("emits a correlated error log and preserves response header parity when downstream throws", async () => {
+    const logger = createLogger();
+    const app = new Hono<TestEnv>();
+
+    app.use("*", createRequestIdMiddleware({ logger }));
+    app.get("/boom", () => {
+      throw new Error("boom");
+    });
+    app.onError((error, c) => c.json({ message: error.message }, 500));
+
+    const response = await app.request("/boom", {
+      headers: { "x-request-id": "req_error_789" },
+    });
+
+    expect(response.status).toBe(500);
+    expect(response.headers.get("x-request-id")).toBe("req_error_789");
+    expect(logger.error).toHaveBeenCalledWith(
+      expect.objectContaining({
+        durationMs: expect.any(Number),
+        error: expect.any(Error),
+        method: "GET",
+        path: "/boom",
+        requestId: "req_error_789",
+        status: 500,
+      }),
+      "API request failed"
+    );
+  });
+});

hivegrid-cloud/apps/api/src/middleware/request-id.ts

@@ -0,0 +1,59 @@
+import type { MiddlewareHandler } from "hono";
+
+const REQUEST_ID_HEADER = "x-request-id";
+const MAX_REQUEST_ID_LENGTH = 128;
+const PRINTABLE_ASCII = /^[\x20-\x7E]+$/;
+
+export type ApiRequestIdVariables = {
+  requestId: string;
+};
+
+export type RequestIdLogger = {
+  error?: (fields: Record<string, unknown>, message: string) => void;
+  info?: (fields: Record<string, unknown>, message: string) => void;
+};
+
+export type RequestIdMiddlewareOptions = {
+  logger?: RequestIdLogger;
+};
+
+export function createRequestIdMiddleware(
+  options: RequestIdMiddlewareOptions = {}
+): MiddlewareHandler<{ Variables: ApiRequestIdVariables }> {
+  return async (c, next) => {
+    const requestId = normalizeRequestId(c.req.header(REQUEST_ID_HEADER));
+    const startedAt = performance.now();
+
+    c.set("requestId", requestId);
+
+    await next();
+    c.header(REQUEST_ID_HEADER, requestId);
+
+    if (c.error) {
+      options.logger?.error?.({ ...createLogFields(c, requestId, startedAt), error: c.error }, "API request failed");
+      return;
+    }
+
+    options.logger?.info?.(createLogFields(c, requestId, startedAt), "API request completed");
+  };
+}
+
+function normalizeRequestId(value: string | undefined): string {
+  const requestId = value?.trim();
+
+  if (!requestId || requestId.length > MAX_REQUEST_ID_LENGTH || !PRINTABLE_ASCII.test(requestId)) {
+    return crypto.randomUUID();
+  }
+
+  return requestId;
+}
+
+function createLogFields(c: Parameters<MiddlewareHandler>[0], requestId: string, startedAt: number) {
+  return {
+    requestId,
+    method: c.req.method,
+    path: c.req.path,
+    status: c.res.status,
+    durationMs: Math.max(0, Math.round(performance.now() - startedAt)),
+  };
+}

hivegrid-cloud/apps/api/src/middleware/sentryTracing.test.ts

@@ -0,0 +1,126 @@
+import { Hono } from "hono";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import {
+  __resetSentryTracingForTests,
+  createSentryTracingMiddleware,
+  type SentryLikeForTracing,
+  type SentryStartSpanArgs,
+} from "./sentryTracing.js";
+
+afterEach(() => {
+  __resetSentryTracingForTests();
+});
+
+function buildFakeSentry(): {
+  module: SentryLikeForTracing;
+  captureException: ReturnType<typeof vi.fn>;
+  spans: Array<{ args: SentryStartSpanArgs; attributes: Record<string, unknown>; status?: { code: number; message?: string } }>;
+} {
+  const captureException = vi.fn();
+  const spans: Array<{ args: SentryStartSpanArgs; attributes: Record<string, unknown>; status?: { code: number; message?: string } }> = [];
+
+  const module: SentryLikeForTracing = {
+    captureException,
+    async startSpan(args, callback) {
+      const record = { args, attributes: { ...(args.attributes ?? {}) }, status: undefined as undefined | { code: number; message?: string } };
+      spans.push(record);
+      return callback({
+        setAttribute(key, value) {
+          record.attributes[key] = value;
+        },
+        setStatus(status) {
+          record.status = status;
+        },
+      });
+    },
+  };
+
+  return { module, captureException, spans };
+}
+
+describe("sentryTracingMiddleware", () => {
+  it("self-disables when SENTRY_DSN_API is empty and no override is provided", async () => {
+    const { module } = buildFakeSentry();
+    const middleware = createSentryTracingMiddleware({ enabled: false, sentry: module });
+
+    const app = new Hono();
+    app.use("*", middleware);
+    app.get("/", (c) => c.text("ok"));
+
+    const res = await app.request("/");
+    expect(res.status).toBe(200);
+    // No spans because enabled=false short-circuits before sentry.startSpan.
+    expect((module as unknown as { startSpan: unknown }).startSpan).toBeDefined();
+  });
+
+  it("starts a span per request and tags method + status_code", async () => {
+    const { module, spans } = buildFakeSentry();
+    const middleware = createSentryTracingMiddleware({ enabled: true, sentry: module });
+
+    const app = new Hono();
+    app.use("*", middleware);
+    app.get("/widgets", (c) => c.json({ ok: true }));
+
+    const res = await app.request("/widgets");
+    expect(res.status).toBe(200);
+
+    expect(spans).toHaveLength(1);
+    const span = spans[0];
+    expect(span.args.op).toBe("http.server");
+    expect(span.args.name).toContain("GET");
+    expect(span.attributes["http.method"]).toBe("GET");
+    expect(span.attributes["http.status_code"]).toBe(200);
+  });
+
+  it("sets error status and captures exception when a route throws (via c.error)", async () => {
+    const { module, spans, captureException } = buildFakeSentry();
+    const middleware = createSentryTracingMiddleware({ enabled: true, sentry: module });
+
+    const app = new Hono();
+    app.use("*", middleware);
+    app.get("/boom", () => {
+      throw new Error("kaboom");
+    });
+    // Standard production-style error envelope: Hono attaches the error to
+    // `c.error` before invoking the registered `onError` handler.
+    app.onError((err, c) => c.text(err.message, 500));
+
+    const res = await app.request("/boom");
+    expect(res.status).toBe(500);
+    expect(spans).toHaveLength(1);
+    expect(spans[0].status?.code).toBe(2);
+    expect(captureException).toHaveBeenCalledTimes(1);
+    const [thrown, captureContext] = captureException.mock.calls[0] ?? [];
+    expect((thrown as Error).message).toBe("kaboom");
+    expect((captureContext as { tags?: Record<string, unknown> } | undefined)?.tags?.component).toBe("api");
+  });
+
+  it("flags 5xx status as error and 4xx as client error", async () => {
+    const { module, spans } = buildFakeSentry();
+    const middleware = createSentryTracingMiddleware({ enabled: true, sentry: module });
+
+    const app = new Hono();
+    app.use("*", middleware);
+    app.get("/not-found", (c) => c.text("nope", 404));
+    app.get("/broken", (c) => c.text("broken", 503));
+
+    await app.request("/not-found");
+    await app.request("/broken");
+
+    expect(spans).toHaveLength(2);
+    expect(spans[0].status?.code).toBe(1);
+    expect(spans[1].status?.code).toBe(2);
+  });
+
+  it("falls back to the literal path when the Sentry SDK is missing", async () => {
+    const middleware = createSentryTracingMiddleware({ enabled: true, sentry: undefined, loadSentry: async () => undefined });
+
+    const app = new Hono();
+    app.use("*", middleware);
+    app.get("/ping", (c) => c.text("pong"));
+
+    const res = await app.request("/ping");
+    expect(res.status).toBe(200);
+  });
+});

hivegrid-cloud/apps/api/src/middleware/sentryTracing.ts

@@ -0,0 +1,163 @@
+/**
+ * Per-request Sentry tracing middleware for the Hono API.
+ *
+ * Hono has no first-party Sentry integration, so we wrap each request in a
+ * manual `Sentry.startSpan({ op: 'http.server', ... })`. The span name uses
+ * the matched route pattern (`c.req.routePath`) when available — falling
+ * back to the literal path — so high-cardinality URLs (`/v1/widgets/:id`)
+ * group correctly in the Sentry Performance UI.
+ *
+ * The middleware self-disables when `SENTRY_DSN_API` is empty: it simply
+ * calls `next()` without touching the Sentry SDK. This keeps local dev
+ * and Sentry-less staging environments noise-free.
+ *
+ * @see docs/improvement-plan/02-observability/sentry-wiring-api-worker.md
+ */
+import type { Context, MiddlewareHandler } from "hono";
+
+export type SentryStartSpanArgs = {
+  op: string;
+  name: string;
+  attributes?: Record<string, string | number | boolean | undefined>;
+};
+
+export type SentrySpan = {
+  setAttribute(key: string, value: string | number | boolean | undefined): void;
+  setStatus?(status: { code: number; message?: string }): void;
+};
+
+export type SentryLikeForTracing = {
+  startSpan: <T>(args: SentryStartSpanArgs, callback: (span: SentrySpan) => Promise<T> | T) => Promise<T> | T;
+  captureException: (error: unknown, captureContext?: Record<string, unknown>) => void;
+};
+
+export interface SentryTracingMiddlewareOptions {
+  /**
+   * Set to false to short-circuit the middleware (used when `SENTRY_DSN_API`
+   * is empty). Defaults to true.
+   */
+  enabled?: boolean;
+  /**
+   * Optional Sentry-like module. Production code injects `@sentry/node`;
+   * tests pass a fake.
+   */
+  sentry?: SentryLikeForTracing | undefined;
+  /**
+   * Optional override for the loader used when `sentry` is not provided.
+   * Defaults to a dynamic `import('@sentry/node')`.
+   */
+  loadSentry?: () => Promise<SentryLikeForTracing | undefined>;
+}
+
+let cachedSentry: SentryLikeForTracing | undefined;
+let cachedLoad: Promise<SentryLikeForTracing | undefined> | undefined;
+
+async function defaultLoadSentry(): Promise<SentryLikeForTracing | undefined> {
+  if (cachedSentry) return cachedSentry;
+  if (!cachedLoad) {
+    cachedLoad = (async () => {
+      try {
+        const moduleName = "@sentry/node";
+        const loaded = (await import(/* @vite-ignore */ moduleName)) as SentryLikeForTracing & {
+          default?: SentryLikeForTracing;
+        };
+        const resolved =
+          typeof loaded?.startSpan === "function"
+            ? loaded
+            : typeof loaded?.default?.startSpan === "function"
+              ? loaded.default
+              : undefined;
+        cachedSentry = resolved;
+        return resolved;
+      } catch {
+        return undefined;
+      }
+    })();
+  }
+  return cachedLoad;
+}
+
+export function createSentryTracingMiddleware(options: SentryTracingMiddlewareOptions = {}): MiddlewareHandler {
+  const enabled = options.enabled ?? Boolean(process.env.SENTRY_DSN_API);
+
+  return async function sentryTracingMiddleware(c: Context, next) {
+    if (!enabled) {
+      await next();
+      return;
+    }
+
+    const sentry = options.sentry ?? (await (options.loadSentry ?? defaultLoadSentry)());
+    if (!sentry) {
+      await next();
+      return;
+    }
+
+    const method = c.req.method;
+    const routePattern = getRoutePattern(c);
+    const requestId = c.get("requestId");
+
+    await sentry.startSpan(
+      {
+        op: "http.server",
+        name: `${method} ${routePattern}`,
+        attributes: {
+          "http.method": method,
+          "http.target": c.req.path,
+          "http.route": routePattern,
+          ...(typeof requestId === "string" && requestId ? { "toolctrl.request_id": requestId } : {}),
+        },
+      },
+      async (span) => {
+        let thrownError: unknown;
+        try {
+          await next();
+        } catch (error) {
+          thrownError = error;
+        }
+
+        // Hono routes throws through `app.onError`, leaving the dispatched
+        // error attached to `c.error` rather than propagating it through
+        // the middleware chain. We treat either signal as an unhandled
+        // exception and forward it to Sentry with span context.
+        const honoError = (c as unknown as { error?: unknown }).error;
+        const errorForCapture = thrownError ?? honoError;
+        const status = c.res?.status ?? (errorForCapture ? 500 : 200);
+
+        span.setAttribute("http.status_code", status);
+        if (status >= 500 || errorForCapture) {
+          span.setStatus?.({ code: 2, message: "internal_error" });
+        } else if (status >= 400) {
+          span.setStatus?.({ code: 1, message: "client_error" });
+        }
+
+        if (errorForCapture) {
+          sentry.captureException(errorForCapture, {
+            tags: {
+              component: "api",
+              "http.method": method,
+              "http.route": routePattern,
+            },
+          });
+        }
+
+        if (thrownError) {
+          throw thrownError;
+        }
+      }
+    );
+  };
+}
+
+function getRoutePattern(c: Context): string {
+  // Hono exposes `routePath` via the underlying request shape when a route
+  // matched; otherwise we degrade to the literal path. Both branches keep
+  // cardinality bounded for the Sentry transaction-name field.
+  const candidate = (c as unknown as { req: { routePath?: string; matchedPath?: string; path: string } }).req;
+  return candidate.routePath ?? candidate.matchedPath ?? candidate.path;
+}
+
+/** Test-only: reset the lazy `@sentry/node` cache. */
+export function __resetSentryTracingForTests(): void {
+  cachedSentry = undefined;
+  cachedLoad = undefined;
+}

hivegrid-cloud/apps/api/src/otel.test.ts

@@ -0,0 +1,113 @@
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  __getApiOtelConfigForTests,
+  __resetApiOtelForTests,
+  initApiOtel,
+  parseOtlpHeaders,
+  parseSamplerArg,
+  readApiOtelConfig,
+} from "./otel.js";
+
+afterEach(() => {
+  __resetApiOtelForTests();
+});
+
+describe("parseSamplerArg", () => {
+  it("returns the fallback for undefined/empty values", () => {
+    expect(parseSamplerArg(undefined, 0.5)).toBe(0.5);
+    expect(parseSamplerArg("", 0.25)).toBe(0.25);
+  });
+
+  it("returns the fallback for non-numeric input", () => {
+    expect(parseSamplerArg("abc", 0.3)).toBe(0.3);
+  });
+
+  it("clamps to [0, 1]", () => {
+    expect(parseSamplerArg("-0.5", 0)).toBe(0);
+    expect(parseSamplerArg("1.5", 0)).toBe(1);
+  });
+
+  it("returns parsed numbers in range", () => {
+    expect(parseSamplerArg("0.42", 0)).toBeCloseTo(0.42);
+  });
+});
+
+describe("parseOtlpHeaders", () => {
+  it("returns undefined for missing input", () => {
+    expect(parseOtlpHeaders(undefined)).toBeUndefined();
+    expect(parseOtlpHeaders("")).toBeUndefined();
+  });
+
+  it("parses a single key=value pair", () => {
+    expect(parseOtlpHeaders("x-sentry-auth=Sentry sentry_key=abc")).toEqual({
+      "x-sentry-auth": "Sentry sentry_key=abc",
+    });
+  });
+
+  it("parses comma-separated pairs and trims whitespace", () => {
+    expect(parseOtlpHeaders(" foo=bar, baz=qux ")).toEqual({ foo: "bar", baz: "qux" });
+  });
+
+  it("skips empty pairs and pairs without `=`", () => {
+    expect(parseOtlpHeaders("foo=bar,,no-equals")).toEqual({ foo: "bar" });
+  });
+});
+
+describe("readApiOtelConfig", () => {
+  it("defaults service.name to boilerplate-api", () => {
+    expect(readApiOtelConfig({}).serviceName).toBe("boilerplate-api");
+  });
+
+  it("respects OTEL_SERVICE_NAME override", () => {
+    expect(readApiOtelConfig({ OTEL_SERVICE_NAME: "api-canary" }).serviceName).toBe("api-canary");
+  });
+
+  it("defaults the traces-sampler to 0.1", () => {
+    expect(readApiOtelConfig({}).tracesSamplerArg).toBe(0.1);
+  });
+
+  it("respects OTEL_TRACES_SAMPLER_ARG override", () => {
+    expect(readApiOtelConfig({ OTEL_TRACES_SAMPLER_ARG: "1.0" }).tracesSamplerArg).toBe(1);
+    expect(readApiOtelConfig({ OTEL_TRACES_SAMPLER_ARG: "0" }).tracesSamplerArg).toBe(0);
+  });
+
+  it("treats an empty endpoint as undefined", () => {
+    expect(readApiOtelConfig({ OTEL_EXPORTER_OTLP_ENDPOINT: "" }).endpoint).toBeUndefined();
+  });
+
+  it("prefers SENTRY_ENVIRONMENT over NODE_ENV for deployment.environment", () => {
+    expect(readApiOtelConfig({ NODE_ENV: "production", SENTRY_ENVIRONMENT: "canary" }).deploymentEnvironment).toBe("canary");
+  });
+
+  it("derives a service.instance.id with hostname + pid", () => {
+    const { serviceInstanceId } = readApiOtelConfig({});
+    expect(serviceInstanceId).toMatch(/^.+-\d+$/);
+    expect(serviceInstanceId.endsWith(`-${process.pid}`)).toBe(true);
+  });
+
+  it("prefers SENTRY_RELEASE, falling back to APP_REVISION, for service.version", () => {
+    expect(readApiOtelConfig({ SENTRY_RELEASE: "sha-1" }).serviceVersion).toBe("sha-1");
+    expect(readApiOtelConfig({ APP_REVISION: "sha-2" }).serviceVersion).toBe("sha-2");
+    expect(readApiOtelConfig({ SENTRY_RELEASE: "sha-1", APP_REVISION: "sha-2" }).serviceVersion).toBe("sha-1");
+  });
+});
+
+describe("initApiOtel", () => {
+  it("is a no-op when OTEL_EXPORTER_OTLP_ENDPOINT is unset", () => {
+    const sdk = initApiOtel({
+      endpoint: undefined,
+      headers: undefined,
+      serviceName: "test",
+      serviceVersion: undefined,
+      serviceInstanceId: "test-1",
+      deploymentEnvironment: undefined,
+      serviceNamespace: "toolctrl",
+      tracesSamplerArg: 0,
+      debug: false,
+    });
+    expect(sdk).toBeUndefined();
+    // Cached config is still recorded so the test contract can be asserted.
+    expect(__getApiOtelConfigForTests()?.serviceName).toBe("test");
+  });
+});

hivegrid-cloud/apps/api/src/otel.ts

@@ -0,0 +1,172 @@
+/**
+ * OpenTelemetry bootstrap for the API process.
+ *
+ * MUST be imported as the very first statement in `apps/api/src/server.ts`
+ * so the auto-instrumentations can hook Node modules (`http`, `hono`,
+ * `prisma`, `bullmq`, `redis`, ...) before they are required for the first
+ * time. The hook order is verified by `apps/api/src/server.ts` keeping
+ * `import "./otel.js"` above every other import.
+ *
+ * No-op behaviour:
+ *   - When `OTEL_EXPORTER_OTLP_ENDPOINT` is empty/unset the SDK is never
+ *     constructed and the auto-instrumentations are never registered, so
+ *     local development and Sentry-only staging keep working unchanged.
+ *
+ * Sentry-coexistence:
+ *   - `@sentry/node` v10 builds on top of OpenTelemetry's TracerProvider.
+ *     Initialising the OTel `NodeSDK` first installs the global provider;
+ *     the Sentry SDK then plugs into that provider on its own init call
+ *     (see `apps/api/src/sentry.ts`). Both stacks share the same tracer
+ *     and therefore the same trace_id.
+ *
+ * @see docs/improvement-plan/02-observability/opentelemetry.md (Step 3 + 4)
+ */
+import { hostname } from "node:os";
+
+import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from "@opentelemetry/sdk-trace-base";
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
+import { PIISpanProcessor } from "@toolctrl/observability";
+
+export interface ApiOtelConfig {
+  endpoint: string | undefined;
+  headers: Record<string, string> | undefined;
+  serviceName: string;
+  serviceVersion: string | undefined;
+  serviceInstanceId: string;
+  deploymentEnvironment: string | undefined;
+  serviceNamespace: string;
+  tracesSamplerArg: number;
+  debug: boolean;
+}
+
+const DEFAULT_SERVICE_NAME = "boilerplate-api";
+const DEFAULT_SERVICE_NAMESPACE = "toolctrl";
+const DEFAULT_TRACES_SAMPLER_ARG = 0.1;
+
+/**
+ * Parses the OTel sampler ratio (0..1). Falls back to `fallback` on missing
+ * or malformed input. Symmetric clamp matches `parseSampleRate` in `sentry.ts`.
+ */
+export function parseSamplerArg(raw: string | undefined, fallback: number): number {
+  if (raw === undefined || raw === null || raw === "") return fallback;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed)) return fallback;
+  if (parsed < 0) return 0;
+  if (parsed > 1) return 1;
+  return parsed;
+}
+
+/**
+ * Parses an `OTEL_EXPORTER_OTLP_HEADERS`-style comma-separated `key=value`
+ * list into a plain header bag. Whitespace around keys/values is trimmed;
+ * empty pairs are skipped; later assignments override earlier ones (matches
+ * OTel SDK behaviour).
+ */
+export function parseOtlpHeaders(raw: string | undefined): Record<string, string> | undefined {
+  if (!raw) return undefined;
+  const out: Record<string, string> = {};
+  for (const pair of raw.split(",")) {
+    const eq = pair.indexOf("=");
+    if (eq < 0) continue;
+    const key = pair.slice(0, eq).trim();
+    const value = pair.slice(eq + 1).trim();
+    if (!key) continue;
+    out[key] = value;
+  }
+  return Object.keys(out).length > 0 ? out : undefined;
+}
+
+export function readApiOtelConfig(env: NodeJS.ProcessEnv = process.env): ApiOtelConfig {
+  return {
+    endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT || undefined,
+    headers: parseOtlpHeaders(env.OTEL_EXPORTER_OTLP_HEADERS),
+    serviceName: env.OTEL_SERVICE_NAME || DEFAULT_SERVICE_NAME,
+    serviceVersion: env.SENTRY_RELEASE || env.APP_REVISION || undefined,
+    serviceInstanceId: `${hostname()}-${process.pid}`,
+    deploymentEnvironment: env.SENTRY_ENVIRONMENT || env.NODE_ENV || undefined,
+    serviceNamespace: DEFAULT_SERVICE_NAMESPACE,
+    tracesSamplerArg: parseSamplerArg(env.OTEL_TRACES_SAMPLER_ARG, DEFAULT_TRACES_SAMPLER_ARG),
+    debug: env.OTEL_LOG_LEVEL?.toLowerCase() === "debug",
+  };
+}
+
+let cachedSdk: NodeSDK | undefined;
+let cachedConfig: ApiOtelConfig | undefined;
+
+/**
+ * Builds the OTel `NodeSDK` for the API process and starts it.
+ *
+ * Idempotent: subsequent calls return the cached instance. Returns
+ * `undefined` (no-op) when `OTEL_EXPORTER_OTLP_ENDPOINT` is unset, so the
+ * caller can chain `initApiOtel()?.shutdown()` in graceful-shutdown paths.
+ */
+export function initApiOtel(config: ApiOtelConfig = readApiOtelConfig()): NodeSDK | undefined {
+  if (cachedSdk) return cachedSdk;
+  cachedConfig = config;
+
+  if (!config.endpoint) return undefined;
+
+  if (config.debug) {
+    diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);
+  }
+
+  const exporter = new OTLPTraceExporter({
+    url: config.endpoint,
+    headers: config.headers,
+  });
+
+  const resource = resourceFromAttributes({
+    [ATTR_SERVICE_NAME]: config.serviceName,
+    ...(config.serviceVersion ? { [ATTR_SERVICE_VERSION]: config.serviceVersion } : {}),
+    "service.instance.id": config.serviceInstanceId,
+    ...(config.deploymentEnvironment ? { "deployment.environment": config.deploymentEnvironment } : {}),
+    "service.namespace": config.serviceNamespace,
+  });
+
+  const sampler = new ParentBasedSampler({
+    root: new TraceIdRatioBasedSampler(config.tracesSamplerArg),
+  });
+
+  const sdk = new NodeSDK({
+    resource,
+    sampler,
+    spanProcessors: [new PIISpanProcessor(new BatchSpanProcessor(exporter))],
+    instrumentations: [
+      getNodeAutoInstrumentations({
+        "@opentelemetry/instrumentation-fs": { enabled: false },
+      }),
+    ],
+  });
+
+  sdk.start();
+  cachedSdk = sdk;
+
+  // Best-effort flush on process exit. NodeSDK.shutdown() is async and
+  // process.on("beforeExit") allows it to run; SIGTERM is handled by the
+  // hono server, not here, so we don't double-wire that signal.
+  process.on("beforeExit", () => {
+    void sdk.shutdown().catch(() => undefined);
+  });
+
+  return sdk;
+}
+
+/** Test-only: reset the cached SDK so tests can re-invoke `initApiOtel()`. */
+export function __resetApiOtelForTests(): void {
+  cachedSdk = undefined;
+  cachedConfig = undefined;
+}
+
+/** Test-only: read back the resolved config (or `undefined` when not initialised). */
+export function __getApiOtelConfigForTests(): ApiOtelConfig | undefined {
+  return cachedConfig;
+}
+
+// Auto-init at module-load time so the very-first-import contract in
+// `apps/api/src/server.ts` is enough: importing this file boots OTel.
+initApiOtel();

hivegrid-cloud/apps/api/src/pagination/rows.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, test } from "vitest";
+import { ROWS_PAGINATION_ERROR_MESSAGE, parseRowsPaginationFromQuery } from "./rows.js";
+
+function parseQuery(values: Record<string, string | undefined>) {
+  return parseRowsPaginationFromQuery((name) => values[name]);
+}
+
+const invalidResult = { ok: false, error: ROWS_PAGINATION_ERROR_MESSAGE };
+
+describe("row pagination parser", () => {
+  test("uses default pagination when query parameters are absent", () => {
+    expect(parseQuery({})).toEqual({ ok: true, value: { page: 1, pageSize: 100 } });
+  });
+
+  test("accepts explicit positive integer pagination values", () => {
+    expect(parseQuery({ page: "2", pageSize: "25" })).toEqual({ ok: true, value: { page: 2, pageSize: 25 } });
+    expect(parseQuery({ page: "1000000", pageSize: "100" })).toEqual({ ok: true, value: { page: 1000000, pageSize: 100 } });
+  });
+
+  test("rejects non-digit and otherwise untrusted query values without coercion", () => {
+    for (const value of ["abc", "1.5", "+1", "-1", " 1", "", "Infinity"]) {
+      expect(parseQuery({ page: value })).toEqual(invalidResult);
+      expect(parseQuery({ pageSize: value })).toEqual(invalidResult);
+    }
+  });
+
+  test("rejects zero and negative values", () => {
+    expect(parseQuery({ page: "0" })).toEqual(invalidResult);
+    expect(parseQuery({ pageSize: "0" })).toEqual(invalidResult);
+    expect(parseQuery({ page: "-1" })).toEqual(invalidResult);
+    expect(parseQuery({ pageSize: "-1" })).toEqual(invalidResult);
+  });
+
+  test("rejects unsafe and out-of-range page values", () => {
+    expect(parseQuery({ page: "9007199254740992" })).toEqual(invalidResult);
+    expect(parseQuery({ page: "1000001" })).toEqual(invalidResult);
+  });
+
+  test("rejects oversized pageSize values", () => {
+    expect(parseQuery({ pageSize: "101" })).toEqual(invalidResult);
+    expect(parseQuery({ pageSize: "9007199254740992" })).toEqual(invalidResult);
+  });
+});

hivegrid-cloud/apps/api/src/pagination/rows.ts

@@ -0,0 +1,38 @@
+export type RowsPagination = {
+  page: number;
+  pageSize: number;
+};
+
+export type RowsPaginationResult = { ok: true; value: RowsPagination } | { ok: false; error: string };
+
+export type RowsPaginationQueryReader = (name: string) => string | undefined;
+
+export const ROWS_PAGINATION_DEFAULT_PAGE = 1;
+export const ROWS_PAGINATION_DEFAULT_PAGE_SIZE = 100;
+export const ROWS_PAGINATION_MAX_PAGE = 1000000;
+export const ROWS_PAGINATION_MAX_PAGE_SIZE = 100;
+export const ROWS_PAGINATION_ERROR_MESSAGE = "page and pageSize must be positive integers; pageSize must be at most 100.";
+
+export function parseRowsPaginationFromQuery(query: RowsPaginationQueryReader): RowsPaginationResult {
+  const page = parseBoundedInteger(query("page"), ROWS_PAGINATION_DEFAULT_PAGE, 1, ROWS_PAGINATION_MAX_PAGE);
+  const pageSize = parseBoundedInteger(query("pageSize"), ROWS_PAGINATION_DEFAULT_PAGE_SIZE, 1, ROWS_PAGINATION_MAX_PAGE_SIZE);
+
+  if (page === null || pageSize === null) {
+    return { ok: false, error: ROWS_PAGINATION_ERROR_MESSAGE };
+  }
+
+  return { ok: true, value: { page, pageSize } };
+}
+
+function parseBoundedInteger(value: string | undefined, defaultValue: number, min: number, max: number) {
+  if (value === undefined) {
+    return defaultValue;
+  }
+
+  if (!/^\d+$/.test(value)) {
+    return null;
+  }
+
+  const parsed = Number(value);
+  return Number.isSafeInteger(parsed) && parsed >= min && parsed <= max ? parsed : null;
+}

hivegrid-cloud/apps/api/src/permission-matrix.test.ts

@@ -0,0 +1,142 @@
+import { describe, expect, test } from "vitest";
+import { API_KEY_SCOPE_OPERATIONS } from "./auth/scope.js";
+import { openApiDocument } from "./openapi.js";
+
+const scopedPermissionMatrix = [
+  { path: "/v1/entities/{entity}/rows", method: "get", operation: "read" },
+  { path: "/v1/entities/{entity}/rows", method: "post", operation: "create" },
+  { path: "/v1/entities/{entity}/rows/{rowId}", method: "get", operation: "read" },
+  { path: "/v1/entities/{entity}/rows/{rowId}", method: "patch", operation: "update" },
+  { path: "/v1/entities/{entity}/rows/{rowId}", method: "delete", operation: "delete" },
+  { path: "/v1/relationships", method: "post", operation: "create" },
+  { path: "/v1/relationships/{id}", method: "get", operation: "read" },
+  { path: "/v1/relationships/{id}", method: "delete", operation: "delete" },
+] as const;
+
+const unscopedPublicOrLegacyPaths = [
+  "/healthz",
+  "/readyz",
+  "/v1/api-keys/validate",
+  "/v1/analytics/public",
+] as const;
+
+const serviceCredentialMatrix = [
+  { path: "/v1/exports/{exportJobId}/download-url", method: "get" },
+] as const;
+
+const serviceTokenOnlyMatrix = [
+  { path: "/v1/entities/{entity}", method: "get" },
+] as const;
+
+const expectedSecurity = [{ bearerAuth: [] }, { apiKeyHeader: [] }];
+const expectedServiceTokenOnlySecurity = [{ bearerAuth: [] }];
+const expectedRuntimeCrudOperations = ["read", "create", "update", "delete"] as const;
+
+type OpenApiPath = keyof typeof openApiDocument.paths;
+type ScopedMatrixEntry = (typeof scopedPermissionMatrix)[number];
+type ServiceCredentialMatrixEntry = (typeof serviceCredentialMatrix)[number] | (typeof serviceTokenOnlyMatrix)[number];
+
+type DocumentedOperation = {
+  security?: unknown;
+  responses?: Record<string, unknown>;
+  "x-api-key-permissions"?: {
+    operation?: unknown;
+    entityPermissions?: Array<{ operation?: unknown }>;
+  };
+};
+
+function getDocumentedOperation({ path, method }: ScopedMatrixEntry | ServiceCredentialMatrixEntry): DocumentedOperation {
+  const pathItem = openApiDocument.paths[path];
+  expect(pathItem, `${path} must exist in openApiDocument.paths`).toBeDefined();
+
+  const operation = (pathItem as Record<typeof method, unknown>)[method];
+  expect(operation, `${method.toUpperCase()} ${path} must exist in openApiDocument.paths`).toBeDefined();
+
+  return operation as DocumentedOperation;
+}
+
+function collectPermissionOperations(permission: NonNullable<DocumentedOperation["x-api-key-permissions"]>) {
+  const operations = new Set<unknown>();
+  if (permission.operation !== undefined) {
+    operations.add(permission.operation);
+  }
+  for (const entityPermission of permission.entityPermissions ?? []) {
+    operations.add(entityPermission.operation);
+  }
+  return [...operations];
+}
+
+describe("scoped API-key permission matrix", () => {
+  test.each(scopedPermissionMatrix)("documents $method $path as requiring $operation scope", (entry) => {
+    const operation = getDocumentedOperation(entry);
+
+    expect(operation.security).toEqual(expectedSecurity);
+    expect(operation.responses).toHaveProperty("403");
+    expect(operation["x-api-key-permissions"], `${entry.method.toUpperCase()} ${entry.path} must document API-key permissions`).toBeDefined();
+
+    const permission = operation["x-api-key-permissions"];
+    expect(permission?.operation ?? permission?.entityPermissions?.[0]?.operation).toBe(entry.operation);
+    expect(collectPermissionOperations(permission ?? {})).toEqual([entry.operation]);
+  });
+
+  test("uses only supported scope operations and represents every runtime CRUD operation", () => {
+    const supportedOperations = new Set<string>(API_KEY_SCOPE_OPERATIONS);
+    const documentedOperations = new Set<string>();
+
+    for (const entry of scopedPermissionMatrix) {
+      const permission = getDocumentedOperation(entry)["x-api-key-permissions"];
+      expect(permission).toBeDefined();
+
+      for (const operation of collectPermissionOperations(permission ?? {})) {
+        expect(typeof operation, `${entry.method.toUpperCase()} ${entry.path} operation must be a string`).toBe("string");
+        expect(supportedOperations.has(operation as string), `${entry.method.toUpperCase()} ${entry.path} uses unsupported operation ${String(operation)}`).toBe(true);
+        documentedOperations.add(operation as string);
+      }
+    }
+
+    expect([...documentedOperations].sort()).toEqual([...expectedRuntimeCrudOperations].sort());
+    expect([...documentedOperations].sort()).toEqual([...API_KEY_SCOPE_OPERATIONS].sort());
+  });
+
+  test("covers exactly the scoped OpenAPI operations that publish API-key permissions", () => {
+    const expectedEntries = scopedPermissionMatrix.map(({ method, path }) => `${method.toUpperCase()} ${path}`).sort();
+    const documentedEntries = Object.entries(openApiDocument.paths)
+      .flatMap(([path, pathItem]) =>
+        Object.entries(pathItem)
+          .filter(([, operation]) => typeof operation === "object" && operation !== null && "x-api-key-permissions" in operation)
+          .map(([method]) => `${method.toUpperCase()} ${path}`)
+      )
+      .sort();
+
+    expect(documentedEntries).toEqual(expectedEntries);
+  });
+
+  test.each(unscopedPublicOrLegacyPaths)("keeps %s out of the scoped permission matrix", (path) => {
+    const scopedMatrixPaths = new Set<OpenApiPath>(scopedPermissionMatrix.map((entry) => entry.path));
+    expect(scopedMatrixPaths.has(path)).toBe(false);
+
+    const pathItem = openApiDocument.paths[path];
+    expect(pathItem, `${path} should remain documented as an unscoped public endpoint`).toBeDefined();
+    expect(Object.values(pathItem).some((operation) => typeof operation === "object" && operation !== null && "x-api-key-permissions" in operation)).toBe(false);
+  });
+
+  test.each(serviceCredentialMatrix)("documents $method $path as service-credential protected but not CRUD scoped", (entry) => {
+    const scopedMatrixPaths = new Set<OpenApiPath>(scopedPermissionMatrix.map(({ path }) => path));
+    expect(scopedMatrixPaths.has(entry.path)).toBe(false);
+
+    const operation = getDocumentedOperation(entry);
+    expect(operation.security).toEqual(expectedSecurity);
+    expect(operation.responses).toHaveProperty("401");
+    expect(operation["x-api-key-permissions"], `${entry.method.toUpperCase()} ${entry.path} must not publish CRUD scope claims`).toBeUndefined();
+  });
+
+  test.each(serviceTokenOnlyMatrix)("documents $method $path as bearer service-token protected but not CRUD scoped", (entry) => {
+    const scopedMatrixPaths = new Set<OpenApiPath>(scopedPermissionMatrix.map(({ path }) => path));
+    expect(scopedMatrixPaths.has(entry.path)).toBe(false);
+
+    const operation = getDocumentedOperation(entry);
+    expect(operation.security).toEqual(expectedServiceTokenOnlySecurity);
+    expect(operation.responses).toHaveProperty("401");
+    expect(operation["x-api-key-permissions"], `${entry.method.toUpperCase()} ${entry.path} must not publish CRUD scope claims`).toBeUndefined();
+  });
+});

hivegrid-cloud/apps/api/src/platform/maintenance/maintenance.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, test } from "vitest";
+import { Hono } from "hono";
+import { createRequestIdMiddleware, type ApiRequestIdVariables } from "../../middleware/request-id.js";
+import {
+  createApiMaintenanceMiddleware,
+  getApiMaintenanceConfig,
+  isApiMaintenanceExemptPath,
+} from "./maintenance.js";
+
+type TestEnv = {
+  Variables: ApiRequestIdVariables;
+};
+
+function createTestApp(config: { enabled: boolean; retryAfterSeconds?: number }) {
+  const app = new Hono<TestEnv>();
+
+  app.use("*", createRequestIdMiddleware());
+  app.use("*", createApiMaintenanceMiddleware(() => config));
+  app.get("/v1/widgets", (c) => c.json({ status: "ok", requestId: c.get("requestId") }));
+  app.get("/healthz", (c) => c.json({ status: "ok", requestId: c.get("requestId") }));
+
+  return app;
+}
+
+describe("API maintenance middleware", () => {
+  test("parses enabled and retry-after config from safe env values", () => {
+    expect(
+      getApiMaintenanceConfig({
+        TOOLCTRL_API_MAINTENANCE_MODE: "1",
+        TOOLCTRL_API_MAINTENANCE_RETRY_AFTER_SECONDS: "120",
+      })
+    ).toEqual({ enabled: true, retryAfterSeconds: 120 });
+  });
+
+  test("ignores disabled flags and invalid retry-after values", () => {
+    expect(
+      getApiMaintenanceConfig({
+        TOOLCTRL_API_MAINTENANCE_MODE: "true",
+        TOOLCTRL_API_MAINTENANCE_RETRY_AFTER_SECONDS: "postgres://user:secret@db.local/app",
+      })
+    ).toEqual({ enabled: false, retryAfterSeconds: undefined });
+
+    expect(
+      getApiMaintenanceConfig({
+        TOOLCTRL_API_MAINTENANCE_MODE: "1",
+        TOOLCTRL_API_MAINTENANCE_RETRY_AFTER_SECONDS: "1.5",
+      })
+    ).toEqual({ enabled: true, retryAfterSeconds: undefined });
+  });
+
+  test.each(["/healthz", "/readyz", "/v1/openapi.json", "/v1/docs", "/v1/docs/"])("exempts operator path %s", (path) => {
+    expect(isApiMaintenanceExemptPath(path)).toBe(true);
+  });
+
+  test("passes through requests when maintenance mode is disabled", async () => {
+    const app = createTestApp({ enabled: false });
+
+    const response = await app.request("/v1/widgets", {
+      headers: { "x-request-id": "req_disabled" },
+    });
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get("x-request-id")).toBe("req_disabled");
+    await expect(response.json()).resolves.toEqual({ status: "ok", requestId: "req_disabled" });
+  });
+
+  test("returns request-id-correlated maintenance response with retry-after", async () => {
+    const app = createTestApp({ enabled: true, retryAfterSeconds: 60 });
+
+    const response = await app.request("/v1/widgets", {
+      headers: { "x-request-id": "req_maintenance" },
+    });
+
+    expect(response.status).toBe(503);
+    expect(response.headers.get("x-request-id")).toBe("req_maintenance");
+    expect(response.headers.get("retry-after")).toBe("60");
+    await expect(response.json()).resolves.toEqual({
+      status: "maintenance",
+      service: "api",
+      requestId: "req_maintenance",
+      error: {
+        code: "maintenance_mode",
+        message: "The API is temporarily unavailable for maintenance. Please retry later.",
+      },
+    });
+  });
+
+  test("does not include secret-like env values in maintenance response", async () => {
+    const app = createTestApp({ enabled: true });
+
+    const response = await app.request("/v1/widgets", {
+      headers: { "x-request-id": "req_redacted" },
+    });
+    const bodyText = await response.text();
+
+    expect(response.status).toBe(503);
+    expect(response.headers.get("retry-after")).toBeNull();
+    expect(bodyText).not.toContain("postgres://user:secret@db.local/app");
+    expect(bodyText).not.toContain("super-secret");
+    expect(bodyText).not.toContain("stack trace");
+    expect(JSON.parse(bodyText)).toEqual({
+      status: "maintenance",
+      service: "api",
+      requestId: "req_redacted",
+      error: {
+        code: "maintenance_mode",
+        message: "The API is temporarily unavailable for maintenance. Please retry later.",
+      },
+    });
+  });
+
+  test("allows exempt health path while maintenance mode is enabled", async () => {
+    const app = createTestApp({ enabled: true });
+
+    const response = await app.request("/healthz", {
+      headers: { "x-request-id": "req_health" },
+    });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({ status: "ok", requestId: "req_health" });
+  });
+});

hivegrid-cloud/apps/api/src/platform/maintenance/maintenance.ts

@@ -0,0 +1,81 @@
+import type { MiddlewareHandler } from "hono";
+import type { ApiRequestIdVariables } from "../../middleware/request-id.js";
+
+const MAINTENANCE_MODE_ENV = "TOOLCTRL_API_MAINTENANCE_MODE";
+const RETRY_AFTER_ENV = "TOOLCTRL_API_MAINTENANCE_RETRY_AFTER_SECONDS";
+const MAINTENANCE_MESSAGE = "The API is temporarily unavailable for maintenance. Please retry later.";
+const EXEMPT_PATHS = new Set(["/healthz", "/readyz", "/v1/openapi.json", "/v1/docs"]);
+
+type MaintenanceEnv = {
+  Variables: ApiRequestIdVariables;
+};
+
+export type ApiMaintenanceConfig = {
+  enabled: boolean;
+  retryAfterSeconds?: number;
+};
+
+export function getApiMaintenanceConfig(env: NodeJS.ProcessEnv = process.env): ApiMaintenanceConfig {
+  return {
+    enabled: env[MAINTENANCE_MODE_ENV] === "1",
+    retryAfterSeconds: parseRetryAfterSeconds(env[RETRY_AFTER_ENV]),
+  };
+}
+
+export function isApiMaintenanceExemptPath(path: string): boolean {
+  return EXEMPT_PATHS.has(normalizePath(path));
+}
+
+export function createApiMaintenanceMiddleware(
+  readConfig: () => ApiMaintenanceConfig = () => getApiMaintenanceConfig()
+): MiddlewareHandler<MaintenanceEnv> {
+  return async (c, next) => {
+    const config = readConfig();
+
+    if (!config.enabled || isApiMaintenanceExemptPath(c.req.path)) {
+      await next();
+      return;
+    }
+
+    const requestId = c.get("requestId");
+
+    if (config.retryAfterSeconds !== undefined) {
+      c.header("retry-after", String(config.retryAfterSeconds));
+    }
+
+    return c.json(
+      {
+        status: "maintenance",
+        service: "api",
+        requestId,
+        error: {
+          code: "maintenance_mode",
+          message: MAINTENANCE_MESSAGE,
+        },
+      },
+      503
+    );
+  };
+}
+
+function parseRetryAfterSeconds(value: string | undefined): number | undefined {
+  if (!value?.trim()) {
+    return undefined;
+  }
+
+  const parsed = Number(value);
+
+  if (!Number.isInteger(parsed) || parsed < 0) {
+    return undefined;
+  }
+
+  return parsed;
+}
+
+function normalizePath(path: string): string {
+  if (path.length > 1 && path.endsWith("/")) {
+    return path.slice(0, -1);
+  }
+
+  return path;
+}

hivegrid-cloud/apps/api/src/plugin-hook-publisher.ts

@@ -0,0 +1,57 @@
+import { getDbClient } from "@toolctrl/db/client.server";
+import { publishOutboxEvent, type OutboxPublisherTransaction } from "@toolctrl/db/outbox";
+import { emitPluginHook, PluginHookValidationError, type PluginHookDefinition } from "@toolctrl/plugin-runtime";
+import { apiLogger } from "./logger.js";
+
+const pluginHookLogger = apiLogger.child?.({ component: "plugin-hook-publisher" }) ?? apiLogger;
+
+export async function emitPluginHookFireAndForget<TPayload>(
+  def: PluginHookDefinition<TPayload>,
+  payload: TPayload,
+  options: {
+    tenantId?: string | null;
+    dispatchKey?: string | null;
+  } = {},
+): Promise<void> {
+  try {
+    const db = getDbClient() as unknown as {
+      $transaction: <T>(runner: (tx: OutboxPublisherTransaction) => Promise<T>) => Promise<T>;
+    };
+    await db.$transaction(async (tx) => {
+      await emitPluginHook(
+        tx,
+        def,
+        payload,
+        async (innerTx, args) => {
+          await publishOutboxEvent(innerTx as OutboxPublisherTransaction, {
+            tenantId: args.tenantId,
+            type: args.type,
+            payload: args.payload as unknown as Parameters<typeof publishOutboxEvent>[1]["payload"],
+            dispatchKey: args.dispatchKey,
+          });
+        },
+        options,
+      );
+    });
+  } catch (error) {
+    if (error instanceof PluginHookValidationError) {
+      pluginHookLogger.warn(
+        {
+          hookName: def.name,
+          tenantId: options.tenantId ?? null,
+          issues: error.issues,
+        },
+        "plugin.hook.payload_invalid",
+      );
+      return;
+    }
+    pluginHookLogger.error(
+      {
+        hookName: def.name,
+        tenantId: options.tenantId ?? null,
+        error: error instanceof Error ? error.message : String(error),
+      },
+      "plugin.hook.emit_failed",
+    );
+  }
+}

hivegrid-cloud/apps/api/src/route-access-inventory-doc.test.ts

@@ -0,0 +1,102 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "vitest";
+import { routeAccessInventory, routeAccessKey, type RouteAccessKey, type RouteMethod } from "./route-access-inventory.js";
+
+const routeAccessInventoryDoc = readFileSync(join(process.cwd(), "docs/security/route-access-inventory.md"), "utf8");
+
+const requiredSectionHeadings = [
+  "# Route Access Inventory",
+  "## Scope and Source of Truth",
+  "## Redaction Rules",
+  "## Access Class Legend",
+  "## Full Route Access Table",
+  "## Verification Commands",
+];
+
+const forbiddenSecretMarkers = [
+  /\bBearer\s+[-._~+/A-Za-z0-9]{20,}=*/,
+  /\b(?:sk|pk|rk)_(?:live|test)_[A-Za-z0-9_]+/,
+  /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/,
+  /\b(?:password|secret|token|api[-_ ]?key|jwt|credential)\s*[:=]\s*[^\s,;`|]+/i,
+  /\b(?:TENANT|CUSTOMER|FIXTURE)[-_][A-Za-z0-9]+\b/,
+  /\b(?:DATABASE_URL|SUPABASE_SERVICE_ROLE_KEY|JWT_SECRET|API_KEY|AUTH_TOKEN)\b/,
+];
+
+function missingItems(items: readonly string[], text: string) {
+  return items.filter((item) => !text.includes(item)).sort();
+}
+
+function escapeRegExp(text: string) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+const routeMethods: readonly RouteMethod[] = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "ALL"];
+
+function isRouteMethod(value: string): value is RouteMethod {
+  return routeMethods.includes(value as RouteMethod);
+}
+
+function documentedRouteKeys(text: string) {
+  return Array.from(text.matchAll(/^\| `([A-Z]+) (\/[^`]+)` \|/gm), (match) => {
+    const method = match[1] ?? "";
+    const path = match[2] ?? "";
+    if (!isRouteMethod(method)) {
+      throw new Error(`Unsupported route method in route access inventory documentation: ${method}`);
+    }
+    return `${method} ${path}` satisfies RouteAccessKey;
+  }).sort();
+}
+
+describe("route access inventory documentation", () => {
+  test("contains the required scope, redaction, legend, table, and verification sections", () => {
+    const missingHeadings = missingItems(requiredSectionHeadings, routeAccessInventoryDoc);
+
+    expect(missingHeadings, `Missing route access inventory document sections:\n${missingHeadings.join("\n")}`).toEqual([]);
+    expect(routeAccessInventoryDoc).toContain("docs/security/csrf-inventory.md");
+    expect(routeAccessInventoryDoc).toContain("apps/api/src/route-access-inventory.ts");
+    expect(routeAccessInventoryDoc).toContain("apps/api/src/route-access-inventory.test.ts");
+    expect(routeAccessInventoryDoc).toContain("apps/api/src/route-access-inventory-doc.test.ts");
+  });
+
+  test("documents every classified route key from the typed inventory exactly once", () => {
+    const routeKeys = routeAccessInventory.map(routeAccessKey);
+    const missingRouteKeys = missingItems(routeKeys, routeAccessInventoryDoc);
+    const extraRouteKeys = documentedRouteKeys(routeAccessInventoryDoc).filter((key) => !routeKeys.includes(key)).sort();
+    const duplicateRouteKeys = routeKeys
+      .filter((key) => routeAccessInventoryDoc.match(new RegExp(String.raw`\| \`${escapeRegExp(key)}\` \|`, "g"))?.length !== 1)
+      .sort();
+
+    expect(missingRouteKeys, `Missing route access inventory document route keys:\n${missingRouteKeys.join("\n")}`).toEqual([]);
+    expect(extraRouteKeys, `Stale route access inventory document route keys:\n${extraRouteKeys.join("\n")}`).toEqual([]);
+    expect(duplicateRouteKeys, `Route access inventory document keys not present exactly once in the table:\n${duplicateRouteKeys.join("\n")}`).toEqual([]);
+  });
+
+  test("keeps access classes and verification commands visible for future route changes", () => {
+    const missingAccessClasses = missingItems(
+      Array.from(new Set(routeAccessInventory.map((entry) => `\`${entry.accessClass}\``))),
+      routeAccessInventoryDoc,
+    );
+    const missingVerificationCommands = missingItems(
+      [
+        "pnpm exec vitest run apps/api/src/route-access-inventory.test.ts apps/api/src/route-access-inventory-doc.test.ts",
+        "pnpm exec eslint apps/api/src/route-access-inventory.ts apps/api/src/route-access-inventory.test.ts apps/api/src/route-access-inventory-doc.test.ts --max-warnings=0",
+      ],
+      routeAccessInventoryDoc,
+    );
+
+    expect(missingAccessClasses, `Missing route access class legend entries:\n${missingAccessClasses.join("\n")}`).toEqual([]);
+    expect(missingVerificationCommands, `Missing route access inventory verification commands:\n${missingVerificationCommands.join("\n")}`).toEqual([]);
+  });
+
+  test("does not expose secret-like markers, fixture identifiers, customer data, or environment variable names", () => {
+    const leakedMarkers = forbiddenSecretMarkers
+      .map((pattern) => ({ pattern: pattern.toString(), match: routeAccessInventoryDoc.match(pattern)?.[0] }))
+      .filter((result): result is { pattern: string; match: string } => result.match !== undefined);
+
+    expect(
+      leakedMarkers.map((result) => `${result.pattern} matched ${JSON.stringify(result.match)}`),
+      `Forbidden secret-like markers in route access inventory document:\n${leakedMarkers.map((result) => `${result.pattern} matched ${JSON.stringify(result.match)}`).join("\n")}`,
+    ).toEqual([]);
+  });
+});

hivegrid-cloud/apps/api/src/route-access-inventory.test.ts

@@ -0,0 +1,190 @@
+import { beforeAll, describe, expect, test } from "vitest";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { scanApiRoutesFromFile, type ApiRoute } from "../../../tools/api-inventory/scan-api-routes.js";
+import { openApiDocument } from "./openapi.js";
+import { routeAccessClasses, routeAccessInventory, routeAccessKey, type RouteAccessInventoryEntry } from "./route-access-inventory.js";
+
+type OpenApiOperation = {
+  security?: unknown;
+  "x-api-key-permissions"?: unknown;
+};
+
+const appSourcePath = join(dirname(fileURLToPath(import.meta.url)), "app.ts");
+const allowedAccessClasses = new Set<string>(routeAccessClasses);
+const documentedOpenApiPaths = new Set(["/healthz", "/readyz", "/status", "/metrics", ...Object.keys(openApiDocument.paths).filter((path) => path.startsWith("/v1/"))]);
+const sensitiveTextPattern = /\b(secret|password|token|api[-_ ]?key|bearer|credential|jwt|email|user id|userid)\s*[:=]\s*[^\s,;]+/i;
+
+function toOpenApiPath(path: string) {
+  return path.replace(/:([A-Za-z0-9_]+)/g, "{$1}");
+}
+
+function getOpenApiOperation(route: Pick<RouteAccessInventoryEntry, "method" | "path">): OpenApiOperation | undefined {
+  const pathItem = openApiDocument.paths[toOpenApiPath(route.path) as keyof typeof openApiDocument.paths];
+  if (!pathItem) {
+    return undefined;
+  }
+  return (pathItem as Record<string, OpenApiOperation | undefined>)[route.method.toLowerCase()];
+}
+
+function expectedSecurityFor(entry: RouteAccessInventoryEntry) {
+  if (
+    [
+      "/v1/session/current-user",
+      "/v1/notifications/stream",
+      "/v1/workspaces/:tenantId/auth/providers",
+      "/v1/workspaces/:tenantId/context/revalidate",
+      "/v1/workspaces/:tenantId/invites",
+      "/v1/workspaces/:tenantId/invites/:inviteId/accept",
+      "/v1/workspaces/:tenantId/invites/:inviteId/revoke",
+    ].includes(entry.path)
+  ) {
+    return [{ bearerAuth: [] }];
+  }
+  switch (entry.accessClass) {
+    case "public-health":
+      return [];
+    case "public-docs":
+    case "auth-credential-exchange":
+      return undefined;
+    case "public-analytics-token":
+      return [{ analyticsTokenHeader: [] }];
+    case "canonical-redirect":
+      return undefined;
+    case "webhook-signature":
+      if (entry.path === "/v1/billing/stripe/webhooks") {
+        return [{ stripeWebhookSignature: [] }];
+      }
+      return [{ mailjetWebhookAuth: [] }];
+    case "jwt-auth":
+      return [{ apiSessionBearer: [] }];
+    case "service-token":
+      return [{ bearerAuth: [] }];
+    case "service-or-tenant-api-key":
+      return [{ bearerAuth: [] }, { apiKeyHeader: [] }];
+    case "optional-api-key-validation":
+      return [{ bearerAuth: [] }, { apiKeyHeader: [] }, {}];
+    case "tenant-api-key":
+      return [{ apiKeyHeader: [] }];
+  }
+}
+
+function routeDiagnostics(routes: Array<Pick<RouteAccessInventoryEntry, "method" | "path">>) {
+  return routes.map(routeAccessKey).sort();
+}
+
+function discoveredRouteDiagnostics(routes: readonly ApiRoute[]) {
+  return routes.map((route) => route.key).sort();
+}
+
+describe("route access inventory", () => {
+  let scanResult: Awaited<ReturnType<typeof scanApiRoutesFromFile>>;
+  let discoveredRoutes: ApiRoute[];
+
+  beforeAll(async () => {
+    scanResult = await scanApiRoutesFromFile(appSourcePath);
+    discoveredRoutes = scanResult.routes;
+  });
+
+  test("classifies every scanner-discovered Hono route exactly once and has no stale route entries", () => {
+    const discoveredKeys = new Set(discoveredRoutes.map((route) => route.key));
+    const inventoryKeys = new Set(routeAccessInventory.map(routeAccessKey));
+
+    const duplicateEntries = routeAccessInventory
+      .map(routeAccessKey)
+      .filter((key, index, keys) => keys.indexOf(key) !== index)
+      .sort();
+    const duplicateDiscoveredRoutes = scanResult.duplicateRoutes.map((duplicate) => duplicate.key).sort();
+    const unresolvedMountedRouters = scanResult.unresolvedMountedRouters.map(
+      (router) => `${router.prefix} -> ${router.receiver} (${router.location.line}:${router.location.column})`
+    );
+    const missingFromInventory = discoveredRouteDiagnostics(discoveredRoutes.filter((route) => !inventoryKeys.has(route.key)));
+    const staleInventoryEntries = routeDiagnostics(routeAccessInventory.filter((entry) => !discoveredKeys.has(routeAccessKey(entry))));
+
+    expect(unresolvedMountedRouters, `Unresolved mounted API routers:\n${unresolvedMountedRouters.join("\n")}`).toEqual([]);
+    expect(duplicateDiscoveredRoutes, `Duplicate scanner-discovered API routes:\n${duplicateDiscoveredRoutes.join("\n")}`).toEqual([]);
+    expect(duplicateEntries, `Duplicate route access inventory entries:\n${duplicateEntries.join("\n")}`).toEqual([]);
+    expect(missingFromInventory, `Missing route access inventory entries:\n${missingFromInventory.join("\n")}`).toEqual([]);
+    expect(staleInventoryEntries, `Stale route access inventory entries:\n${staleInventoryEntries.join("\n")}`).toEqual([]);
+  });
+
+  test("uses supported classes, non-empty rationales, and redacted sensitive-data descriptions", () => {
+    const malformedEntries = routeAccessInventory
+      .filter((entry) => !allowedAccessClasses.has(entry.accessClass) || entry.rationale.trim().length === 0 || (typeof entry.sensitiveData === "string" && entry.sensitiveData.trim().length === 0) || sensitiveTextPattern.test(entry.rationale) || (typeof entry.sensitiveData === "string" && sensitiveTextPattern.test(entry.sensitiveData)))
+      .map((entry) => `${routeAccessKey(entry)} class=${entry.accessClass} rationale=${JSON.stringify(entry.rationale)} sensitiveData=${JSON.stringify(entry.sensitiveData)}`);
+
+    expect(malformedEntries, `Invalid route access inventory fields:\n${malformedEntries.join("\n")}`).toEqual([]);
+  });
+
+  test("inventories workspace auth boundary endpoints with redacted service-token contracts", () => {
+    const authBoundaryEntries = routeAccessInventory.filter((entry) =>
+      ["/v1/workspaces/:tenantId/auth/providers", "/v1/workspaces/:tenantId/context/revalidate"].includes(entry.path)
+    );
+
+    expect(authBoundaryEntries.map(routeAccessKey).sort()).toEqual([
+      "GET /v1/workspaces/:tenantId/auth/providers",
+      "POST /v1/workspaces/:tenantId/context/revalidate",
+    ]);
+    for (const entry of authBoundaryEntries) {
+      expect(entry.accessClass).toBe("service-token");
+      expect(entry.rationale).toContain("service token");
+      expect(entry.sensitiveData).toEqual(expect.stringMatching(/redacted|omitted/i));
+      expect(`${entry.rationale} ${entry.sensitiveData}`).toMatch(/provider secrets|client secrets|SSO secret|SAML certificate/i);
+      expect(getOpenApiOperation(entry)?.security).toEqual([{ bearerAuth: [] }]);
+    }
+  });
+
+  test("documents matching OpenAPI security metadata for represented OpenAPI routes", () => {
+    const missingOpenApiOperations = routeAccessInventory
+      .filter((entry) => documentedOpenApiPaths.has(toOpenApiPath(entry.path)) && !getOpenApiOperation(entry))
+      .map(routeAccessKey);
+
+    const securityDrift = routeAccessInventory
+      .filter((entry) => documentedOpenApiPaths.has(toOpenApiPath(entry.path)))
+      .flatMap((entry) => {
+        const operation = getOpenApiOperation(entry);
+        const expectedSecurity = expectedSecurityFor(entry);
+        return JSON.stringify(operation?.security) === JSON.stringify(expectedSecurity)
+          ? []
+          : [`${routeAccessKey(entry)} expected security ${JSON.stringify(expectedSecurity)} but found ${JSON.stringify(operation?.security)}`];
+      });
+
+    expect(missingOpenApiOperations, `Missing OpenAPI operations for inventory routes:\n${missingOpenApiOperations.join("\n")}`).toEqual([]);
+    expect(securityDrift, `OpenAPI security drift:\n${securityDrift.join("\n")}`).toEqual([]);
+  });
+
+  test("keeps OpenAPI path coverage aligned with route inventory", () => {
+    const inventoryOpenApiKeys = new Set(routeAccessInventory.map((entry) => `${entry.method} ${toOpenApiPath(entry.path)}`));
+    const documentedRouteKeys = Object.entries(openApiDocument.paths)
+      .flatMap(([path, pathItem]) =>
+        Object.keys(pathItem)
+          .filter((method) => ["get", "post", "put", "patch", "delete", "options"].includes(method))
+          .map((method) => `${method.toUpperCase()} ${path}`)
+      )
+      .filter((key) => key !== "GET /v1/openapi.json")
+      .sort();
+
+    const undocumentedInventoryKeys = routeAccessInventory
+      .filter((entry) => !documentedOpenApiPaths.has(toOpenApiPath(entry.path)))
+      .map((entry) => `${routeAccessKey(entry)} is intentionally runtime-only`)
+      .sort();
+    const staleDocumentedKeys = documentedRouteKeys.filter((key) => !inventoryOpenApiKeys.has(key));
+
+    expect(undocumentedInventoryKeys, `Runtime routes without OpenAPI docs:\n${undocumentedInventoryKeys.join("\n")}`).toEqual([]);
+    expect(staleDocumentedKeys, `OpenAPI operations missing from route access inventory:\n${staleDocumentedKeys.join("\n")}`).toEqual([]);
+  });
+
+  test("negative diagnostics remain empty for scanner missing/stale entries and invalid classifications", () => {
+    const discoveredKeys = new Set(discoveredRoutes.map((route) => route.key));
+    const inventoryKeys = new Set(routeAccessInventory.map(routeAccessKey));
+    const missingEntries = discoveredRoutes.filter((route) => !inventoryKeys.has(route.key));
+    const staleEntries = routeAccessInventory.filter((entry) => !discoveredKeys.has(routeAccessKey(entry)));
+    const invalidClassEntries = routeAccessInventory.filter((entry) => !allowedAccessClasses.has(entry.accessClass));
+    const emptyRationaleEntries = routeAccessInventory.filter((entry) => entry.rationale.trim().length === 0);
+
+    expect(discoveredRouteDiagnostics(missingEntries), `Missing entries:\n${discoveredRouteDiagnostics(missingEntries).join("\n")}`).toEqual([]);
+    expect(routeDiagnostics(staleEntries), `Stale entries:\n${routeDiagnostics(staleEntries).join("\n")}`).toEqual([]);
+    expect(routeDiagnostics(invalidClassEntries), `Unsupported classes:\n${routeDiagnostics(invalidClassEntries).join("\n")}`).toEqual([]);
+    expect(routeDiagnostics(emptyRationaleEntries), `Empty rationales:\n${routeDiagnostics(emptyRationaleEntries).join("\n")}`).toEqual([]);
+  });
+});

hivegrid-cloud/apps/api/src/route-access-inventory.ts

@@ -0,0 +1,349 @@
+export const routeAccessClasses = [
+  "public-health",
+  "public-docs",
+  "service-token",
+  "service-or-tenant-api-key",
+  "optional-api-key-validation",
+  "tenant-api-key",
+  "webhook-signature",
+  "jwt-auth",
+  "auth-credential-exchange",
+  "public-analytics-token",
+  "canonical-redirect",
+] as const;
+
+export type RouteAccessClass = (typeof routeAccessClasses)[number];
+
+export type RouteMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" | "ALL";
+
+export type RouteAccessKey = `${RouteMethod} ${string}`;
+
+export type RouteAccessInventoryEntry = {
+  method: RouteMethod;
+  path: string;
+  accessClass: RouteAccessClass;
+  rationale: string;
+  sensitiveData: boolean | string;
+};
+
+export function routeAccessKey(route: Pick<RouteAccessInventoryEntry, "method" | "path">): RouteAccessKey {
+  return `${route.method} ${route.path}`;
+}
+
+export const routeAccessInventory = [
+	  {
+	    method: "GET",
+	    path: "/healthz",
+	    accessClass: "public-health",
+	    rationale: "Unauthenticated liveness probe returns only service status and request correlation metadata.",
+	    sensitiveData: false,
+	  },
+	  {
+	    method: "GET",
+	    path: "/v1/health",
+	    accessClass: "public-health",
+	    rationale: "Unauthenticated versioned liveness probe returns only service status and request correlation metadata.",
+	    sensitiveData: false,
+	  },
+	  {
+	    method: "GET",
+	    path: "/readyz",
+	    accessClass: "public-health",
+	    rationale: "Unauthenticated readiness probe returns redacted dependency status for infrastructure checks.",
+	    sensitiveData: "Dependency health state is exposed without implementation details.",
+	  },
+	  {
+	    method: "GET",
+	    path: "/status",
+	    accessClass: "public-health",
+	    rationale: "Unauthenticated status probe for external uptime monitors. Returns boolean-only health, revision, region, uptime.",
+	    sensitiveData: "Boolean health state and deployment region tag only — no connection strings, hostnames or stack traces.",
+	  },
+	  {
+	    method: "GET",
+	    path: "/metrics",
+	    accessClass: "service-token",
+	    rationale: "Prometheus scrape endpoint requires bearer METRICS_TOKEN. Returns 404 when METRICS_ENABLED is false.",
+	    sensitiveData: "Prom text-format counters and gauges — includes business KPIs (AI tokens, mail, webhook, queue depth). Token-protected.",
+	  },
+  {
+    method: "GET",
+    path: "/v1/openapi.json",
+    accessClass: "public-docs",
+    rationale: "OpenAPI discovery is intentionally public for client integration and contains schema metadata only.",
+    sensitiveData: false,
+  },
+  {
+    method: "GET",
+    path: "/v1/docs",
+    accessClass: "public-docs",
+    rationale: "Documentation shell is intentionally public and renders the OpenAPI document without runtime data.",
+    sensitiveData: false,
+  },
+  {
+    method: "GET",
+    path: "/v1/session/current-user",
+    accessClass: "service-token",
+    rationale: "Current service principal endpoint requires a bearer service token before returning authentication state.",
+    sensitiveData: "Returns authenticated principal type metadata only.",
+  },
+  {
+    method: "POST",
+    path: "/v1/auth/login",
+    accessClass: "auth-credential-exchange",
+    rationale: "Login exchanges submitted email and password credentials for a short-lived JWT after password verification.",
+    sensitiveData: "Consumes credentials and returns a JWT plus basic user identity.",
+  },
+  {
+    method: "POST",
+    path: "/v1/auth/refresh-token",
+    accessClass: "jwt-auth",
+    rationale: "Refresh requires an existing bearer JWT before issuing a replacement token for the same user.",
+    sensitiveData: "Returns a JWT plus basic user identity.",
+  },
+  {
+    method: "POST",
+    path: "/v1/auth/logout",
+    accessClass: "jwt-auth",
+    rationale: "Logout requires an existing bearer JWT before revoking the active API session and clearing the API cookie.",
+    sensitiveData: "Consumes JWT credential material but returns only a success envelope and expired cookie attributes.",
+  },
+  {
+    method: "POST",
+    path: "/v1/api-keys/validate",
+    accessClass: "optional-api-key-validation",
+    rationale: "API-key validation optionally accepts a presented stored API key or service token and reports redacted validation status without requiring a prior credential.",
+    sensitiveData: "Returns redacted stored API-key metadata but never plaintext key material.",
+  },
+  {
+    method: "POST",
+    path: "/v1/ai/upload",
+    accessClass: "service-token",
+    rationale: "AI upload writes request content through configured S3-compatible service storage credentials and depends on server-side upload configuration.",
+    sensitiveData: "Consumes uploaded file content and may return a public storage URL.",
+  },
+  {
+    method: "POST",
+    path: "/v1/support/inbound-email",
+    accessClass: "webhook-signature",
+    rationale: "Inbound support email verifies Mailjet webhook Basic-Auth credentials before resolving tenant inbound addresses and creating tickets.",
+    sensitiveData: "Consumes signed email metadata and message body while omitting provider payloads and signing material from responses.",
+  },
+  {
+    method: "POST",
+    path: "/v1/analytics/events",
+    accessClass: "public-analytics-token",
+    rationale: "Analytics event ingestion is public-facing and scoped by analytics payload identifiers rather than browser session state.",
+    sensitiveData: "Consumes analytics identifiers, URLs, routes, and event labels.",
+  },
+  {
+    method: "POST",
+    path: "/v1/analytics/page-views",
+    accessClass: "public-analytics-token",
+    rationale: "Analytics page-view ingestion is public-facing and scoped by analytics payload identifiers plus ignore-page settings.",
+    sensitiveData: "Consumes analytics identifiers, URLs, and route names.",
+  },
+  {
+    method: "GET",
+    path: "/v1/analytics/public",
+    accessClass: "public-analytics-token",
+    rationale: "Public analytics overview requires the analytics token header before returning aggregate metrics.",
+    sensitiveData: "Returns aggregate analytics counts and top-level chart data.",
+  },
+  {
+    method: "GET",
+    path: "/v1/analytics/admin",
+    accessClass: "service-token",
+    rationale: "Admin analytics overview requires service credentials before returning tenant or portal scoped metrics.",
+    sensitiveData: "Returns aggregate analytics metrics that may be scoped to a portal identifier.",
+  },
+  {
+    method: "GET",
+    path: "/v1/notifications/stream",
+    accessClass: "service-token",
+    rationale: "Notification stream requires service credentials before returning tenant and user scoped in-app delivery diagnostics over SSE.",
+    sensitiveData: "Returns bounded, sanitized notification delivery snapshots and cursors without provider payloads, metadata, or raw delivery secrets.",
+  },
+  {
+    method: "GET",
+    path: "/v1/accounts",
+    accessClass: "service-token",
+    rationale: "Account listing requires service credentials before returning tenant account metadata and subscription summaries.",
+    sensitiveData: "Returns tenant identifiers, names, slugs, icons, and subscription summary fields without plaintext secrets.",
+  },
+  {
+    method: "GET",
+    path: "/v1/accounts/:id",
+    accessClass: "service-token",
+    rationale: "Account lookup requires service credentials before returning one tenant account metadata and subscription summary.",
+    sensitiveData: "Returns tenant identifiers, names, slugs, icons, and subscription summary fields without plaintext secrets.",
+  },
+  {
+    method: "GET",
+    path: "/v1/usage",
+    accessClass: "tenant-api-key",
+    rationale: "Usage lookups enter through the canonical API host and validate tenant API-key credentials in the API runtime.",
+    sensitiveData: "Returns plan and remaining API usage metadata for the credential-scoped tenant.",
+  },
+  {
+    method: "POST",
+    path: "/v1/billing/stripe/webhooks",
+    accessClass: "webhook-signature",
+    rationale: "Stripe billing webhook ingress enters through the canonical API host and verifies the raw signed payload in the API runtime.",
+    sensitiveData: "Consumes signed billing lifecycle webhook payloads without returning signing material or plaintext credentials.",
+  },
+  {
+    method: "POST",
+    path: "/v1/email-sequences/mailjet/webhooks",
+    accessClass: "webhook-signature",
+    rationale: "Email-sequence provider webhooks verify Mailjet Basic-Auth credentials before correlating bounce events to provider message ids.",
+    sensitiveData: "Consumes delivery event metadata while responses omit provider payloads, recipient data, and credentials.",
+  },
+  {
+    method: "GET",
+    path: "/v1/workspaces/:tenantId/auth/providers",
+    accessClass: "service-token",
+    rationale: "Workspace auth-provider discovery requires a service token before returning tenant-scoped SSO metadata.",
+    sensitiveData: "Returns redacted OIDC/SAML provider metadata only; client secrets, SAML certificate bodies, private keys, token hashes, invite tokens, and raw metadata documents are omitted.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/context/revalidate",
+    accessClass: "service-token",
+    rationale: "Workspace context revalidation requires a service token before confirming tenant/user membership for a context switch.",
+    sensitiveData: "Consumes tenant and user identifiers and returns a redacted validity envelope without provider secrets, invite tokens, token hashes, or SSO secret material.",
+  },
+  {
+    method: "GET",
+    path: "/v1/workspaces/:tenantId/invites",
+    accessClass: "service-token",
+    rationale: "Workspace invite listing requires a service token before returning tenant-scoped invite lifecycle state.",
+    sensitiveData: "Returns invite email, lifecycle timestamps, audit user ids, and tokenPrefix only; plaintext tokens and token hashes are omitted.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/invites",
+    accessClass: "service-token",
+    rationale: "Workspace invite creation requires a service token before issuing a tenant-scoped invite token.",
+    sensitiveData: "Consumes invite recipient details and returns the one-time plaintext invite token only in the create response.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/invites/:inviteId/accept",
+    accessClass: "service-token",
+    rationale: "Workspace invite acceptance requires a service token and a one-time invite token before finalizing tenant membership invitation state.",
+    sensitiveData: "Consumes plaintext invite token and returns redacted invite lifecycle metadata without echoing the token.",
+  },
+  {
+    method: "POST",
+    path: "/v1/workspaces/:tenantId/invites/:inviteId/revoke",
+    accessClass: "service-token",
+    rationale: "Workspace invite revocation requires a service token before finalizing the tenant-scoped invite as revoked.",
+    sensitiveData: "Consumes revoke audit metadata and returns redacted invite lifecycle metadata without token material.",
+  },
+  {
+    method: "POST",
+    path: "/v1/events/webhooks/attempts/:id",
+    accessClass: "webhook-signature",
+    rationale: "Webhook retry verifies Mailjet webhook Basic-Auth credentials before looking up or queueing a known delivery attempt.",
+    sensitiveData: "Consumes signed webhook delivery metadata and an attempt id while omitting provider payloads and signing material from responses.",
+  },
+  {
+    method: "POST",
+    path: "/v1/maintenance/export-retention/enqueue",
+    accessClass: "service-token",
+    rationale: "Export artifact retention cleanup is an operator maintenance action and requires service credentials before enqueueing work.",
+    sensitiveData: "Consumes an optional bounded batch size and returns only queue task metadata plus request correlation.",
+  },
+  {
+    method: "GET",
+    path: "/v1/entities/:entity/rows",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Entity row listing requires a service token or scoped stored API key with read permission for the resolved entity.",
+    sensitiveData: "Returns tenant-scoped legacy entity row data.",
+  },
+  {
+    method: "POST",
+    path: "/v1/entities/:entity/rows",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Entity row creation requires a service token or scoped stored API key with create permission for the resolved entity.",
+    sensitiveData: "Consumes and stores tenant-scoped legacy entity row data.",
+  },
+  {
+    method: "GET",
+    path: "/v1/entities/:entity/rows/:rowId",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Entity row lookup requires a service token or scoped stored API key with read permission for the resolved entity.",
+    sensitiveData: "Returns tenant-scoped legacy entity row data.",
+  },
+  {
+    method: "PATCH",
+    path: "/v1/entities/:entity/rows/:rowId",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Entity row update requires a service token or scoped stored API key with update permission for the resolved entity.",
+    sensitiveData: "Consumes and returns tenant-scoped legacy entity row data.",
+  },
+  {
+    method: "DELETE",
+    path: "/v1/entities/:entity/rows/:rowId",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Entity row deletion requires a service token or scoped stored API key with delete permission for the resolved entity.",
+    sensitiveData: "Deletes tenant-scoped legacy entity row data and returns the deleted row id.",
+  },
+  {
+    method: "POST",
+    path: "/v1/relationships",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Relationship creation requires a service token or scoped stored API key with create permission for both related entities.",
+    sensitiveData: "Consumes row identifiers and returns relationship metadata.",
+  },
+  {
+    method: "GET",
+    path: "/v1/relationships/:id",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Relationship lookup requires a service token or scoped stored API key with read permission for both related entities.",
+    sensitiveData: "Returns relationship metadata with parent and child row envelopes.",
+  },
+  {
+    method: "DELETE",
+    path: "/v1/relationships/:id",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Relationship deletion requires a service token or scoped stored API key with delete permission for both related entities.",
+    sensitiveData: "Deletes and returns relationship metadata with parent and child row envelopes.",
+  },
+  {
+    method: "GET",
+    path: "/v1/exports/:exportJobId/download-url",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Export download URL creation requires service credentials or a tenant-bound stored API key and blocks compliance-paused exports before returning a short-lived signed artifact URL envelope.",
+    sensitiveData: "Returns a signed artifact URL envelope without artifact bucket/key values, storage credentials, pause reason internals, filters, or object bodies.",
+  },
+  {
+    method: "GET",
+    path: "/v1/entities/:entity",
+    accessClass: "service-token",
+    rationale: "Legacy entity lookup requires service credentials before returning the named entity detail graph.",
+    sensitiveData: "Returns legacy entity schema and relationship details.",
+  },
+  {
+    method: "GET",
+    path: "/v1/exports/:exportJobId/progress",
+    accessClass: "service-or-tenant-api-key",
+    rationale: "Export progress polling requires service credentials or a tenant-bound stored API key; the handler scopes lookups to the caller's tenant and never leaks artifact bucket/key details.",
+    sensitiveData: "Returns export progress envelope (status, percent, rowCount) — no artifact URLs, no compliance-pause internals, no filters.",
+  },
+  {
+    method: "POST",
+    path: "/v1/devices/register",
+    accessClass: "service-token",
+    rationale: "Mobile push-token registration requires a service credential (bearer/API-key) carrying user identity; the body's userId + tenantId are reconciled against the authenticated principal downstream.",
+    sensitiveData: "Persists device-bound push token. Stored token is hashed at rest; response contains only the registration id + masked token preview.",
+  },
+  {
+    method: "POST",
+    path: "/v1/devices/revoke",
+    accessClass: "service-token",
+    rationale: "Mobile push-token revocation requires a service credential; the handler revokes by token hash and never echoes the raw token.",
+    sensitiveData: "Returns revocation confirmation without echoing the device token or push provider internals.",
+  },
+] as const satisfies readonly RouteAccessInventoryEntry[];

hivegrid-cloud/apps/api/src/routes/accounts-routes.ts

@@ -0,0 +1,113 @@
+import { Hono } from "hono";
+import type { LegacyTenantWithDetails, TenantSubscriptionWithDetails } from "../app-dependencies.js";
+import { getApiAppDependencies } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { requireServiceCredential } from "./shared.js";
+
+export function registerAccountsRoutes(app: Hono<ApiEnv>) {
+  app.get("/v1/accounts", async (c) => {
+    const unauthorized = requireServiceCredential(c);
+    if (unauthorized) {
+      return unauthorized;
+    }
+
+    const dependencies = getApiAppDependencies();
+    const tenants = await dependencies.adminGetAllLegacyTenants();
+    const data = tenants.map((tenant) => serializeAccountTenant(tenant, tenant.subscription));
+
+    return c.json({
+      success: true,
+      total_results: data.length,
+      data,
+    });
+  });
+
+  app.get("/v1/accounts/:id", async (c) => {
+    const unauthorized = requireServiceCredential(c);
+    if (unauthorized) {
+      return unauthorized;
+    }
+
+    const id = c.req.param("id")?.trim();
+    if (!id) {
+      return apiError(c, 400, "invalid_request", "Account id is required.");
+    }
+
+    const dependencies = getApiAppDependencies();
+    const tenantSimple = await dependencies.findLegacyTenantByIdOrSlug(id);
+    if (!tenantSimple) {
+      return apiError(c, 404, "account_not_found", "Account was not found.");
+    }
+
+    const tenant = await dependencies.findLegacyTenantById(tenantSimple.id);
+    if (!tenant) {
+      return apiError(c, 404, "account_not_found", "Account was not found.");
+    }
+
+    return c.json({
+      success: true,
+      data: serializeAccountTenant(tenant, await dependencies.getTenantSubscription(tenant.id)),
+    });
+  });
+}
+
+function serializeAccountTenant(tenant: LegacyTenantWithDetails, subscription: TenantSubscriptionWithDetails | null) {
+  return {
+    id: tenant.id,
+    createdAt: tenant.createdAt,
+    updatedAt: tenant.updatedAt,
+    slug: tenant.slug,
+    name: tenant.name,
+    icon: tenant.icon,
+    subscription: subscription
+      ? {
+          id: subscription.id,
+          stripeCustomerId: subscription.stripeCustomerId,
+          products: subscription.products.map((product) => ({
+            id: product.id,
+            product: product.subscriptionProduct.title,
+            quantity: product.quantity,
+            currentPeriodStart: product.currentPeriodStart,
+            currentPeriodEnd: product.currentPeriodEnd,
+            model: product.subscriptionProduct.model,
+            cancelledAt: product.cancelledAt ?? undefined,
+            endsAt: product.endsAt ?? undefined,
+            stripeId: product.stripeSubscriptionId,
+            prices: product.prices.map((price) => ({
+              id: price.id,
+              ...(price.subscriptionPrice
+                ? {
+                    stripeId: price.subscriptionPrice.stripeId,
+                    type: price.subscriptionPrice.type,
+                    billingPeriod: price.subscriptionPrice.billingPeriod,
+                    price: Number(price.subscriptionPrice.price),
+                    currency: price.subscriptionPrice.currency,
+                  }
+                : price.subscriptionUsageBasedPrice
+                  ? {
+                      stripeId: price.subscriptionUsageBasedPrice.stripeId,
+                      billingPeriod: price.subscriptionUsageBasedPrice.billingPeriod,
+                      currency: price.subscriptionUsageBasedPrice.currency,
+                      unit: price.subscriptionUsageBasedPrice.unit,
+                      unitTitle: price.subscriptionUsageBasedPrice.unitTitle,
+                      unitTitlePlural: price.subscriptionUsageBasedPrice.unitTitlePlural,
+                      usageType: price.subscriptionUsageBasedPrice.usageType,
+                      aggregateUsage: price.subscriptionUsageBasedPrice.aggregateUsage,
+                      tiersMode: price.subscriptionUsageBasedPrice.tiersMode,
+                      billingScheme: price.subscriptionUsageBasedPrice.billingScheme,
+                      tiers: price.subscriptionUsageBasedPrice.tiers.map((tier) => ({
+                        id: tier.id,
+                        from: Number(tier.from),
+                        to: tier.to ? Number(tier.to) : null,
+                        perUnitPrice: tier.perUnitPrice ? Number(tier.perUnitPrice) : undefined,
+                        flatFeePrice: tier.flatFeePrice ? Number(tier.flatFeePrice) : undefined,
+                      })),
+                    }
+                  : {}),
+            })),
+          })),
+        }
+      : null,
+  };
+}

hivegrid-cloud/apps/api/src/routes/analytics-routes.ts

@@ -0,0 +1,181 @@
+import { Hono, type Context } from "hono";
+import { defaultAnalyticsPeriod, getApiAppDependencies, isAnalyticsPeriod } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { authenticateServiceRequest } from "../auth.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { parseAnalyticsEventPayload, parseAnalyticsPageViewPayload } from "../validation/request-schemas.js";
+import { readJsonBody } from "./shared.js";
+
+const analyticsIgnorePagesCacheTtlMs = 5_000;
+
+type AnalyticsDependencies = ReturnType<typeof getApiAppDependencies>;
+
+let analyticsIgnorePagesCache:
+  | {
+      dependencies: AnalyticsDependencies;
+      expiresAt: number;
+      ignoredPages: string[];
+    }
+  | undefined;
+
+export function registerAnalyticsRoutes(app: Hono<ApiEnv>) {
+  app.post("/v1/analytics/events", postAnalyticsEventRequest);
+  app.post("/v1/analytics/page-views", postAnalyticsPageViewRequest);
+  app.get("/v1/analytics/public", getPublicAnalyticsRequest);
+  app.get("/v1/analytics/admin", getAdminAnalyticsRequest);
+}
+
+async function getCachedAnalyticsIgnoredPages(dependencies: AnalyticsDependencies) {
+  const now = Date.now();
+  if (analyticsIgnorePagesCache?.dependencies === dependencies && analyticsIgnorePagesCache.expiresAt > now) {
+    return analyticsIgnorePagesCache.ignoredPages;
+  }
+
+  const settings = await dependencies.getAnalyticsSettings();
+  const ignoredPages = settings?.ignorePages?.split(",").map((item) => item.trim()).filter(Boolean) ?? [];
+  analyticsIgnorePagesCache = {
+    dependencies,
+    expiresAt: now + analyticsIgnorePagesCacheTtlMs,
+    ignoredPages,
+  };
+  return ignoredPages;
+}
+
+function validateAnalyticsTokenRequest(c: Context<ApiEnv>) {
+  const analyticsToken = getApiRuntimeConfig().analytics.token;
+  if (!analyticsToken) {
+    return apiError(c, 500, "analytics_token_not_configured", "ANALYTICS_TOKEN is not set.");
+  }
+  if (analyticsToken === "1234567890") {
+    return apiError(c, 500, "analytics_token_not_configured", "ANALYTICS_TOKEN cannot be the default value.");
+  }
+
+  const token = c.req.header("x-analytics-token");
+  if (token !== analyticsToken) {
+    return apiError(c, 401, "unauthorized", "Invalid analytics token.");
+  }
+  return null;
+}
+
+async function postAnalyticsEventRequest(c: Context<ApiEnv>) {
+  const unauthorized = validateAnalyticsTokenRequest(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseAnalyticsEventPayload(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const analyticsId = payload.value.analytics.userAnalyticsId;
+    const visitor = await dependencies.getOrCreateAnalyticsUniqueVisitor({
+      cookie: analyticsId,
+      fromUrl: payload.value.url,
+      fromRoute: payload.value.route,
+      analytics: payload.value.analytics,
+    });
+
+    const featureFlag = payload.value.category === "featureFlag" ? await dependencies.findFeatureFlagByName(payload.value.action) : null;
+
+    await dependencies.createAnalyticsEvent({
+      uniqueVisitorId: visitor.id,
+      action: payload.value.action,
+      category: payload.value.category,
+      label: payload.value.label,
+      value: payload.value.value,
+      url: payload.value.url,
+      route: payload.value.route,
+      featureFlagId: featureFlag?.id,
+      portalId: payload.value.analytics.portalId ?? null,
+      portalUserId: payload.value.analytics.portalUserId ?? null,
+    });
+
+    return c.json({ success: true }, 201);
+  } catch {
+    return apiError(c, 500, "analytics_event_failed", "Analytics event could not be recorded.");
+  }
+}
+
+async function postAnalyticsPageViewRequest(c: Context<ApiEnv>) {
+  const unauthorized = validateAnalyticsTokenRequest(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseAnalyticsPageViewPayload(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const ignoredPages = await getCachedAnalyticsIgnoredPages(dependencies);
+    if (ignoredPages.some((ignoredPage) => payload.value.url.includes(ignoredPage))) {
+      return c.body(null, 204);
+    }
+
+    const analyticsId = payload.value.analytics.userAnalyticsId;
+    const visitor = await dependencies.getOrCreateAnalyticsUniqueVisitor({
+      cookie: analyticsId,
+      fromUrl: payload.value.url,
+      fromRoute: payload.value.route,
+      analytics: payload.value.analytics,
+    });
+    await dependencies.updateAnalyticsUniqueVisitorSource(visitor, payload.value.analytics);
+    await dependencies.createAnalyticsPageView({
+      uniqueVisitorId: visitor.id,
+      url: payload.value.url,
+      route: payload.value.route,
+      portalId: payload.value.analytics.portalId ?? null,
+      portalUserId: payload.value.analytics.portalUserId ?? null,
+    });
+
+    return c.json({ success: true }, 201);
+  } catch {
+    return apiError(c, 500, "analytics_page_view_failed", "Analytics page view could not be recorded.");
+  }
+}
+
+async function getPublicAnalyticsRequest(c: Context<ApiEnv>) {
+  const unauthorized = validateAnalyticsTokenRequest(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const dependencies = getApiAppDependencies();
+  const settings = await dependencies.getAnalyticsSettings();
+  if (!settings?.public) {
+    return apiError(c, 403, "forbidden", "Public analytics overview is disabled.");
+  }
+
+  const periodParam = c.req.query("period") ?? defaultAnalyticsPeriod;
+  if (!isAnalyticsPeriod(periodParam)) {
+    return apiError(c, 400, "invalid_period", "Invalid period.", { params: { period: periodParam } });
+  }
+
+  try {
+    return c.json(await dependencies.getAnalyticsOverview({ period: periodParam, portalId: null }));
+  } catch {
+    return apiError(c, 500, "analytics_overview_failed", "Analytics overview could not be loaded.");
+  }
+}
+
+async function getAdminAnalyticsRequest(c: Context<ApiEnv>) {
+  const principal = authenticateServiceRequest(c.req.raw, getApiRuntimeConfig());
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const periodParam = c.req.query("period") ?? defaultAnalyticsPeriod;
+  if (!isAnalyticsPeriod(periodParam)) {
+    return apiError(c, 400, "invalid_period", "Invalid period.", { params: { period: periodParam } });
+  }
+
+  return c.json(await getApiAppDependencies().getAnalyticsOverview({ period: periodParam, portalId: c.req.query("portalId") }));
+}

hivegrid-cloud/apps/api/src/routes/auth-routes.ts

@@ -0,0 +1,215 @@
+import { Hono } from "hono";
+import jwt from "jsonwebtoken";
+import { getApiAppDependencies } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { authenticateServiceRequest, extractApiKeyFromRequest, extractBearerToken, isValidServiceToken } from "../auth.js";
+import { serializeApiAuthCookie, serializeExpiredApiAuthCookie } from "../auth/cookie/session-cookie.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { parseApiKeyValidationPayload, parseLoginCredentialsPayload } from "../validation/request-schemas.js";
+import {
+  createLoginRateLimitRequest,
+  createLoginRateLimitResponse,
+  getApiSessionIdFromJwt,
+  getBearerToken,
+  getLegacyUserIdFromJwt,
+  readJsonBody,
+} from "./shared.js";
+
+// A valid bcrypt hash compared against when the account does not exist or has
+// no password, so the response time of a failed login is independent of whether
+// the email maps to a real password account (prevents account enumeration via
+// timing). Mirrors the web AuthService DUMMY_PASSWORD_HASH.
+const DUMMY_PASSWORD_HASH = "$2a$10$7EqJtq98hPqEX7fNZaFWoOhi8VfIf6z9SjG8/aYx8vvjE5LiE3D0C";
+
+const AUTH_INVALID_CREDENTIALS_MESSAGE = "Authentication failed.";
+const AUTH_INVALID_TOKEN_MESSAGE = "Authentication token is invalid or expired.";
+const AUTH_MISSING_TOKEN_MESSAGE = "Authentication token is required.";
+const API_KEY_REQUIRED_MESSAGE = "API key is required.";
+
+export function registerAuthRoutes(app: Hono<ApiEnv>) {
+  app.get("/v1/session/current-user", (c) => {
+    const principal = authenticateServiceRequest(c.req.raw, getApiRuntimeConfig());
+    if (!principal) {
+      return apiError(c, 401, "unauthorized", "A valid bearer token is required.");
+    }
+    return c.json({ authenticated: true, principal });
+  });
+
+  app.post("/v1/auth/login", async (c) => {
+    try {
+      const body = await readJsonBody(c.req.raw);
+      const payload = parseLoginCredentialsPayload(body);
+      if (!payload.ok) {
+        if (body === undefined) {
+          return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+        }
+        return apiError(c, 400, "invalid_credentials", AUTH_INVALID_CREDENTIALS_MESSAGE);
+      }
+
+      const dependencies = getApiAppDependencies();
+      const { email, password } = payload.value;
+      const rateLimitRequest = createLoginRateLimitRequest(c, email);
+      const rateLimit = await dependencies.checkLoginRateLimit(rateLimitRequest);
+      if (!rateLimit.allowed) {
+        return createLoginRateLimitResponse(c, rateLimit);
+      }
+
+      const user = await dependencies.findLegacyLoginUserByEmail(email);
+      if (!user) {
+        // Run a comparison against a dummy hash so the response time matches the
+        // real-account path and does not leak account existence via timing.
+        await dependencies.comparePassword(password, DUMMY_PASSWORD_HASH);
+        await dependencies.recordFailedLoginAttempt(rateLimitRequest);
+        return apiError(c, 400, "invalid_credentials", AUTH_INVALID_CREDENTIALS_MESSAGE);
+      }
+
+      // Phase 01 / #20 — magic-link-only accounts have `passwordHash = null`
+      // (invitation flow). Refuse the password-login attempt without leaking
+      // account existence; the web `/forgot-password` flow is the correct
+      // entrypoint to authenticate these users.
+      if (user.passwordHash === null) {
+        // Same dummy-hash comparison as the no-user branch so password accounts
+        // and passwordless accounts are indistinguishable by response time.
+        await dependencies.comparePassword(password, DUMMY_PASSWORD_HASH);
+        await dependencies.recordFailedLoginAttempt(rateLimitRequest);
+        return apiError(c, 400, "invalid_credentials", AUTH_INVALID_CREDENTIALS_MESSAGE);
+      }
+
+      const isCorrectPassword = await dependencies.comparePassword(password, user.passwordHash);
+      if (!isCorrectPassword) {
+        await dependencies.recordFailedLoginAttempt(rateLimitRequest);
+        return apiError(c, 400, "invalid_credentials", AUTH_INVALID_CREDENTIALS_MESSAGE);
+      }
+
+      await dependencies.recordSuccessfulLogin({ account: email });
+      await dependencies.recordLegacyLogin(user.id);
+      const config = getApiRuntimeConfig();
+      const session = dependencies.createApiSession(user.id);
+      const token = jwt.sign({ userId: user.id, sessionId: session.sessionId }, config.jwtSecret!, { expiresIn: "1h" });
+      const response = c.json({ token, user: { id: user.id, email: user.email } });
+      response.headers.append("Set-Cookie", serializeApiAuthCookie(token, config));
+      return response;
+    } catch {
+      return apiError(c, 401, "invalid_credentials", AUTH_INVALID_CREDENTIALS_MESSAGE);
+    }
+  });
+
+  app.post("/v1/auth/refresh-token", async (c) => {
+    try {
+      const token = getBearerToken(c.req.raw);
+      if (!token) {
+        return apiError(c, 401, "missing_token", AUTH_MISSING_TOKEN_MESSAGE);
+      }
+
+      const dependencies = getApiAppDependencies();
+      const decoded = jwt.verify(token, getApiRuntimeConfig().jwtSecret!);
+      const userId = getLegacyUserIdFromJwt(decoded);
+      const sessionId = getApiSessionIdFromJwt(decoded);
+      if (!userId || !sessionId) {
+        return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+      }
+
+      const session = dependencies.getActiveApiSession(sessionId);
+      if (!session || session.userId !== userId) {
+        return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+      }
+
+      const user = await dependencies.findLegacyAuthUserById(userId);
+      if (!user) {
+        return apiError(c, 400, "invalid_credentials", AUTH_INVALID_CREDENTIALS_MESSAGE);
+      }
+
+      const rotation = dependencies.rotateApiSession(sessionId);
+      if (!rotation.rotated || rotation.session.userId !== userId) {
+        return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+      }
+
+      const config = getApiRuntimeConfig();
+      const newToken = jwt.sign({ userId, sessionId: rotation.session.sessionId }, config.jwtSecret!, { expiresIn: "1h" });
+      const response = c.json({ token: newToken, user: { id: user.id, email: user.email } });
+      response.headers.append("Set-Cookie", serializeApiAuthCookie(newToken, config));
+      return response;
+    } catch {
+      return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+    }
+  });
+
+  app.post("/v1/auth/logout", (c) => {
+    const config = getApiRuntimeConfig();
+    try {
+      const token = getBearerToken(c.req.raw);
+      if (!token) {
+        return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+      }
+
+      const dependencies = getApiAppDependencies();
+      const decoded = jwt.verify(token, config.jwtSecret!);
+      const userId = getLegacyUserIdFromJwt(decoded);
+      const sessionId = getApiSessionIdFromJwt(decoded);
+      if (!userId || !sessionId) {
+        return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+      }
+
+      const session = dependencies.getActiveApiSession(sessionId);
+      if (!session || session.userId !== userId) {
+        return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+      }
+
+      dependencies.revokeApiSession(sessionId);
+      const response = c.json({ success: true });
+      response.headers.append("Set-Cookie", serializeExpiredApiAuthCookie(config));
+      return response;
+    } catch {
+      return apiError(c, 401, "invalid_token", AUTH_INVALID_TOKEN_MESSAGE);
+    }
+  });
+
+  app.post("/v1/api-keys/validate", async (c) => {
+    const body = await readJsonBody(c.req.raw);
+    const payload = parseApiKeyValidationPayload(body);
+    if (body !== undefined && !payload.ok) {
+      return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+    }
+
+    const config = getApiRuntimeConfig();
+    const bearerToken = extractBearerToken(c.req.raw);
+    if (bearerToken && isValidServiceToken(bearerToken, config)) {
+      return c.json({ active: true, keyType: "service-token" });
+    }
+
+    const apiKey = extractApiKeyFromRequest(c.req.raw, payload.ok ? payload.value : undefined);
+    if (!apiKey) {
+      return apiError(c, 400, "api_key_required", API_KEY_REQUIRED_MESSAGE, { params: { credential: "api_key", sources: "body.apiKey,x-api-key,service-bearer" } });
+    }
+
+    if (isValidServiceToken(apiKey, config)) {
+      return c.json({ active: true, keyType: "service-token" });
+    }
+
+    const storedApiKey = await getApiAppDependencies().findLegacyApiKeyByKey(apiKey, undefined as never, config.jwtSecret, {
+      allowPlaintextFallback: config.apiKeys.allowLegacyPlaintextFallback,
+    });
+    if (storedApiKey) {
+      const expiresAt = storedApiKey.expires?.getTime() ?? null;
+      const active = storedApiKey.active && !storedApiKey.tenant.deactivatedReason && (expiresAt === null || expiresAt > Date.now());
+      return c.json({
+        active,
+        keyType: "stored-api-key",
+        apiKey: {
+          id: storedApiKey.id,
+          alias: storedApiKey.alias,
+          ...(active
+            ? {
+                tenantId: storedApiKey.tenantId,
+                tenant: { id: storedApiKey.tenant.id, name: storedApiKey.tenant.name, slug: storedApiKey.tenant.slug },
+              }
+            : {}),
+          expires: storedApiKey.expires ? storedApiKey.expires.toISOString() : null,
+        },
+      });
+    }
+
+    return c.json({ active: false, keyType: "service-token" });
+  });
+}

hivegrid-cloud/apps/api/src/routes/devices-routes.test.ts

@@ -0,0 +1,175 @@
+import { describe, expect, it, beforeEach, afterEach, vi } from "vitest";
+import { Hono } from "hono";
+import type { ApiEnv } from "../api-types.js";
+import { registerDevicesRoutes } from "./devices-routes.js";
+import { setApiRuntimeConfigForTests, resetApiRuntimeConfigForTests } from "../config.js";
+
+const TEST_TOKEN = "test-service-token";
+
+let mockStore: Map<string, { id: string; userId: string; tenantId: string | null; platform: string; token: string; pushProvider: string; appVersion: string | null; createdAt: Date; lastSeenAt: Date; revokedAt: Date | null }>;
+
+vi.mock("@toolctrl/db/device-tokens.server", () => ({
+  parseDevicePlatform: (value: unknown) => {
+    return value === "ios" || value === "android" || value === "web" ? value : null;
+  },
+  parsePushProvider: (value: unknown) => {
+    return value === "fcm" || value === "apns" || value === "webpush" ? value : null;
+  },
+  async registerDeviceToken(input: { userId: string; tenantId: string | null; platform: "ios" | "android" | "web"; token: string; pushProvider?: "fcm" | "apns" | "webpush"; appVersion?: string | null }) {
+    const provider = input.pushProvider ?? (input.platform === "ios" ? "apns" : input.platform === "android" ? "fcm" : "webpush");
+    const now = new Date();
+    const existing = mockStore.get(input.token);
+    if (existing) {
+      existing.userId = input.userId;
+      existing.tenantId = input.tenantId;
+      existing.platform = input.platform;
+      existing.pushProvider = provider;
+      existing.appVersion = input.appVersion ?? null;
+      existing.lastSeenAt = now;
+      existing.revokedAt = null;
+      return existing;
+    }
+    const row = {
+      id: `dt-${mockStore.size + 1}`,
+      userId: input.userId,
+      tenantId: input.tenantId,
+      platform: input.platform,
+      token: input.token,
+      pushProvider: provider,
+      appVersion: input.appVersion ?? null,
+      createdAt: now,
+      lastSeenAt: now,
+      revokedAt: null,
+    };
+    mockStore.set(input.token, row);
+    return row;
+  },
+  async revokeDeviceToken(token: string) {
+    const row = mockStore.get(token);
+    if (!row) return null;
+    row.revokedAt = new Date();
+    return row;
+  },
+}));
+
+function buildApp() {
+  const app = new Hono<ApiEnv>();
+  registerDevicesRoutes(app);
+  return app;
+}
+
+beforeEach(() => {
+  mockStore = new Map();
+  setApiRuntimeConfigForTests({
+    apiAccessToken: TEST_TOKEN,
+    jwtSecret: "jwt",
+    routing: { appBaseDomain: "example.com", webAppHost: "app.example.com", adminAppHost: "admin.example.com", apiHost: "api.example.com", protocol: "https" },
+    apiKeys: { allowLegacyPlaintextFallback: false },
+  } as never);
+});
+
+afterEach(() => {
+  resetApiRuntimeConfigForTests();
+});
+
+function postRegister(app: ReturnType<typeof buildApp>, body: unknown, auth: string | null = `Bearer ${TEST_TOKEN}`) {
+  return app.request("/v1/devices/register", {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      ...(auth ? { authorization: auth } : {}),
+    },
+    body: JSON.stringify(body),
+  });
+}
+
+describe("POST /v1/devices/register", () => {
+  it("rejects requests without service credentials", async () => {
+    const app = buildApp();
+    const response = await postRegister(app, {}, null);
+    expect(response.status).toBe(401);
+  });
+
+  it("rejects non-object bodies", async () => {
+    const app = buildApp();
+    const response = await app.request("/v1/devices/register", {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: `Bearer ${TEST_TOKEN}` },
+      body: "not-json",
+    });
+    expect(response.status).toBe(400);
+  });
+
+  it("rejects missing userId", async () => {
+    const app = buildApp();
+    const response = await postRegister(app, { platform: "ios", token: "t" });
+    expect(response.status).toBe(400);
+    const json = (await response.json()) as { error: { params: { field: string } } };
+    expect(json.error.params.field).toBe("userId");
+  });
+
+  it("rejects unknown platform", async () => {
+    const app = buildApp();
+    const response = await postRegister(app, { userId: "u1", platform: "wear", token: "t" });
+    expect(response.status).toBe(400);
+  });
+
+  it("rejects missing token", async () => {
+    const app = buildApp();
+    const response = await postRegister(app, { userId: "u1", platform: "ios" });
+    expect(response.status).toBe(400);
+    const json = (await response.json()) as { error: { params: { field: string } } };
+    expect(json.error.params.field).toBe("token");
+  });
+
+  it("upserts a valid registration and defaults the push provider", async () => {
+    const app = buildApp();
+    const response = await postRegister(app, {
+      userId: "u-1",
+      tenantId: "t-1",
+      platform: "ios",
+      token: "tok-1",
+      appVersion: "1.2.3",
+    });
+    expect(response.status).toBe(200);
+    const json = (await response.json()) as { success: boolean; data: { pushProvider: string; platform: string; appVersion: string | null } };
+    expect(json.success).toBe(true);
+    expect(json.data.platform).toBe("ios");
+    expect(json.data.pushProvider).toBe("apns");
+    expect(json.data.appVersion).toBe("1.2.3");
+    expect(mockStore.size).toBe(1);
+  });
+
+  it("is idempotent: re-registering the same token bumps lastSeenAt without duplicating", async () => {
+    const app = buildApp();
+    await postRegister(app, { userId: "u-1", tenantId: null, platform: "android", token: "tok-x" });
+    const second = await postRegister(app, { userId: "u-1", tenantId: null, platform: "android", token: "tok-x" });
+    expect(second.status).toBe(200);
+    expect(mockStore.size).toBe(1);
+  });
+});
+
+describe("POST /v1/devices/revoke", () => {
+  it("revokes a known token", async () => {
+    const app = buildApp();
+    await postRegister(app, { userId: "u-1", platform: "ios", token: "tok-a" });
+    const response = await app.request("/v1/devices/revoke", {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: `Bearer ${TEST_TOKEN}` },
+      body: JSON.stringify({ token: "tok-a" }),
+    });
+    expect(response.status).toBe(200);
+    const json = (await response.json()) as { data: { revokedAt: string | null } };
+    expect(json.data.revokedAt).not.toBeNull();
+  });
+
+  it("returns 404 for unknown tokens", async () => {
+    const app = buildApp();
+    const response = await app.request("/v1/devices/revoke", {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: `Bearer ${TEST_TOKEN}` },
+      body: JSON.stringify({ token: "missing" }),
+    });
+    expect(response.status).toBe(404);
+  });
+});

hivegrid-cloud/apps/api/src/routes/devices-routes.ts

@@ -0,0 +1,145 @@
+/**
+ * Mobile device-token registration routes (Phase 08 #65).
+ *
+ * The Capacitor mobile shell calls `POST /v1/devices/register` once it
+ * has obtained an APNs / FCM / WebPush token from the native plugin.
+ * The endpoint upserts by `token` (globally unique) so the same payload
+ * can be replayed on every cold start without producing duplicate rows.
+ *
+ * Authentication: service-credential (bearer or API-key) — the mobile
+ * client forwards its session bearer through the same auth pipeline as
+ * every other API call. `userId` and `tenantId` come from the request
+ * body because the wrapping service-token has no inherent identity in
+ * the boilerplate's auth model; downstream calls verify these against
+ * the authenticated principal.
+ *
+ * @see packages/db/src/device-tokens.server.ts
+ * @see apps/mobile/src/pushRegistration.ts
+ */
+import {
+  parseDevicePlatform,
+  parsePushProvider,
+  registerDeviceToken,
+  revokeDeviceToken,
+  type DeviceTokenRecord,
+} from "@toolctrl/db/device-tokens.server";
+import { Hono, type Context } from "hono";
+import type { ApiEnv } from "../api-types.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { requireServiceCredential, readJsonBody, isPlainObject } from "./shared.js";
+
+const MAX_TOKEN_LENGTH = 4_096;
+const MAX_USER_ID_LENGTH = 128;
+const MAX_TENANT_ID_LENGTH = 128;
+const MAX_APP_VERSION_LENGTH = 64;
+
+export function registerDevicesRoutes(app: Hono<ApiEnv>) {
+  app.post("/v1/devices/register", postDeviceRegisterRequest);
+  app.post("/v1/devices/revoke", postDeviceRevokeRequest);
+}
+
+async function postDeviceRegisterRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  if (!isPlainObject(body)) {
+    return apiError(c, 400, "invalid_request", "Request body must be a JSON object.", { params: { reason: "body_invalid", bodyType: "device_register" } });
+  }
+
+  const userId = typeof body.userId === "string" ? body.userId.trim() : "";
+  const tenantIdRaw = typeof body.tenantId === "string" ? body.tenantId.trim() : "";
+  const platform = parseDevicePlatform(body.platform);
+  const token = typeof body.token === "string" ? body.token.trim() : "";
+  const pushProvider = parsePushProvider(body.pushProvider) ?? undefined;
+  const appVersion = typeof body.appVersion === "string" ? body.appVersion.trim() : "";
+
+  if (!userId) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_required", field: "userId" } });
+  }
+  if (userId.length > MAX_USER_ID_LENGTH) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_too_long", field: "userId", max: MAX_USER_ID_LENGTH } });
+  }
+  if (tenantIdRaw.length > MAX_TENANT_ID_LENGTH) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_too_long", field: "tenantId", max: MAX_TENANT_ID_LENGTH } });
+  }
+  if (!platform) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_value_invalid", field: "platform", allowed: "ios,android,web" } });
+  }
+  if (!token) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_required", field: "token" } });
+  }
+  if (token.length > MAX_TOKEN_LENGTH) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_too_long", field: "token", max: MAX_TOKEN_LENGTH } });
+  }
+  if (appVersion.length > MAX_APP_VERSION_LENGTH) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_too_long", field: "appVersion", max: MAX_APP_VERSION_LENGTH } });
+  }
+
+  let record: DeviceTokenRecord;
+  try {
+    record = await registerDeviceToken({
+      userId,
+      tenantId: tenantIdRaw || null,
+      platform,
+      token,
+      pushProvider,
+      appVersion: appVersion || null,
+    });
+  } catch {
+    return apiError(c, 500, "device_token_register_failed", "Device token could not be registered.");
+  }
+
+  return c.json(
+    {
+      success: true,
+      data: {
+        id: record.id,
+        userId: record.userId,
+        tenantId: record.tenantId,
+        platform: record.platform,
+        pushProvider: record.pushProvider,
+        appVersion: record.appVersion,
+        createdAt: record.createdAt.toISOString(),
+        lastSeenAt: record.lastSeenAt.toISOString(),
+      },
+    },
+    200,
+  );
+}
+
+async function postDeviceRevokeRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  if (!isPlainObject(body)) {
+    return apiError(c, 400, "invalid_request", "Request body must be a JSON object.", { params: { reason: "body_invalid", bodyType: "device_revoke" } });
+  }
+
+  const token = typeof body.token === "string" ? body.token.trim() : "";
+  if (!token) {
+    return apiError(c, 400, "invalid_request", "Request validation failed.", { params: { reason: "field_required", field: "token" } });
+  }
+
+  const record = await revokeDeviceToken(token);
+  if (!record) {
+    return apiError(c, 404, "device_token_not_found", "Device token was not found.");
+  }
+
+  return c.json(
+    {
+      success: true,
+      data: {
+        id: record.id,
+        token: record.token,
+        revokedAt: record.revokedAt?.toISOString() ?? null,
+      },
+    },
+    200,
+  );
+}

hivegrid-cloud/apps/api/src/routes/entity-row-routes.ts

@@ -0,0 +1,343 @@
+import { Hono, type Context } from "hono";
+import { PLUGIN_HOOK_ENTITY_CREATED, PLUGIN_HOOK_ENTITY_DELETED, PLUGIN_HOOK_ENTITY_UPDATED } from "@toolctrl/plugin-runtime";
+import { getApiAppDependencies, type LegacyEntityWithDetails } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { authenticateServiceRequest } from "../auth.js";
+import { authorizeApiEntityRowScope } from "../auth/permission.js";
+import { resolveApiTenantScope } from "../auth/scope.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { parseRowsPaginationFromQuery } from "../pagination/rows.js";
+import { emitPluginHookFireAndForget } from "../plugin-hook-publisher.js";
+import { parseEntityRowObjectBody } from "../validation/request-schemas.js";
+import {
+  apiKeyScopedMissingResourceError,
+  authenticateServiceCredentialRequest,
+  forbiddenApiKeyScopeError,
+  getLegacyEntityId,
+  isMissingLegacyEntityError,
+  isMissingLegacyRowError,
+  missingApiTenantScopeError,
+  normalizeUnknownError,
+  parseEntityRowRouteParams,
+  parseOptionalTenantId,
+  readJsonBody,
+  runIdempotentJson,
+  toJsonValue,
+} from "./shared.js";
+
+export function registerEntityRowRoutes(app: Hono<ApiEnv>) {
+  app.get("/v1/entities/:entity/rows", getEntityRowsRequest);
+  app.post("/v1/entities/:entity/rows", postEntityRowRequest);
+  app.get("/v1/entities/:entity/rows/:rowId", getEntityRowRequest);
+  app.patch("/v1/entities/:entity/rows/:rowId", patchEntityRowRequest);
+  app.delete("/v1/entities/:entity/rows/:rowId", deleteEntityRowRequest);
+  app.get("/v1/entities/:entity", getEntityRequest);
+}
+
+async function getEntityRowsRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const entityName = c.req.param("entity");
+  if (!entityName) {
+    return apiError(c, 400, "invalid_request", "Expected entity.");
+  }
+
+  const pagination = parseRowsPaginationFromQuery((name) => c.req.query(name));
+  if (!pagination.ok) {
+    return apiError(c, 400, "invalid_pagination", pagination.error);
+  }
+
+  const tenantResult = resolveTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const entity = await dependencies.findLegacyEntityByName({ tenantId: tenantResult.tenantId, name: entityName, active: true });
+    const scope = authorizeApiEntityRowScope(principal, { tenantId: tenantResult.tenantId, entity, operation: "read" });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+    const page = await dependencies.getAllLegacyEntityRows({ entityId: getLegacyEntityId(entity), tenantId: tenantResult.tenantId, ...pagination.value });
+    return c.json({ success: true, page: page.page, total_pages: page.totalPages, total_results: page.totalItems, results: page.items, data: page.items });
+  } catch (error) {
+    const normalizedError = normalizeUnknownError(error);
+    const apiKeyMissing = isMissingLegacyEntityError(normalizedError) ? apiKeyScopedMissingResourceError(c, principal) : null;
+    if (apiKeyMissing) {
+      return apiKeyMissing;
+    }
+    return apiError(c, isMissingLegacyEntityError(normalizedError) ? 404 : 500, isMissingLegacyEntityError(normalizedError) ? "entity_not_found" : "entity_rows_failed", isMissingLegacyEntityError(normalizedError) ? "Entity not found." : "Entity rows could not be loaded.");
+  }
+}
+
+async function postEntityRowRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const entityName = c.req.param("entity");
+  if (!entityName) {
+    return apiError(c, 400, "invalid_request", "Expected entity.");
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseEntityRowObjectBody(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_json", payload.error.message, { params: payload.error.params });
+  }
+
+  const tenantResult = resolveTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const entity = await dependencies.findLegacyEntityByName({ tenantId: tenantResult.tenantId, name: entityName, active: true });
+    const scope = authorizeApiEntityRowScope(principal, { tenantId: tenantResult.tenantId, entity, operation: "create" });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+    return await runIdempotentJson(c, {
+      method: "POST",
+      route: "/v1/entities/:entity/rows",
+      tenantId: tenantResult.tenantId,
+      entityId: getLegacyEntityId(entity),
+      parsedBody: toJsonValue(payload.value),
+    }, async () => {
+      const row = await dependencies.createLegacyEntityRow({
+        entity: entity as LegacyEntityWithDetails,
+        tenantId: tenantResult.tenantId,
+        createdByApiKeyId: principal.type === "api-key" ? principal.apiKey.id : null,
+        data: payload.value,
+      });
+      void emitPluginHookFireAndForget(
+        PLUGIN_HOOK_ENTITY_CREATED,
+        {
+          entityType: entityName,
+          entityId: row.id,
+          tenantId: tenantResult.tenantId,
+          userId: null,
+          occurredAt: new Date().toISOString(),
+        },
+        { tenantId: tenantResult.tenantId, dispatchKey: `entity:${entityName}:${row.id}:created` },
+      );
+      return { status: 201, body: row as never };
+    });
+  } catch (error) {
+    const normalizedError = normalizeUnknownError(error);
+    const apiKeyMissing = isMissingLegacyEntityError(normalizedError) ? apiKeyScopedMissingResourceError(c, principal) : null;
+    if (apiKeyMissing) {
+      return apiKeyMissing;
+    }
+    return apiError(c, isMissingLegacyEntityError(normalizedError) ? 404 : 400, isMissingLegacyEntityError(normalizedError) ? "entity_not_found" : "entity_row_create_failed", isMissingLegacyEntityError(normalizedError) ? "Entity not found." : "Entity row could not be created.");
+  }
+}
+
+async function getEntityRowRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const routeParams = parseEntityRowRouteParams(c);
+  if (!routeParams.ok) {
+    return apiError(c, 400, "invalid_request", routeParams.error);
+  }
+
+  const tenantResult = resolveTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const entity = await dependencies.findLegacyEntityByName({ tenantId: tenantResult.tenantId, name: routeParams.value.entityName, active: true });
+    const scope = authorizeApiEntityRowScope(principal, { tenantId: tenantResult.tenantId, entity, operation: "read" });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+    const row = await dependencies.getLegacyEntityRow({ entityId: getLegacyEntityId(entity), tenantId: tenantResult.tenantId, rowId: routeParams.value.rowId });
+    if (!row) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "row_not_found", "Row not found.", { params: { rowId: routeParams.value.rowId } });
+    }
+    return c.json(row as never);
+  } catch (error) {
+    const normalizedError = normalizeUnknownError(error);
+    const apiKeyMissing = isMissingLegacyEntityError(normalizedError) ? apiKeyScopedMissingResourceError(c, principal) : null;
+    if (apiKeyMissing) {
+      return apiKeyMissing;
+    }
+    return apiError(c, isMissingLegacyEntityError(normalizedError) ? 404 : 500, isMissingLegacyEntityError(normalizedError) ? "entity_not_found" : "entity_row_failed", isMissingLegacyEntityError(normalizedError) ? "Entity not found." : "Entity row could not be loaded.");
+  }
+}
+
+async function patchEntityRowRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const routeParams = parseEntityRowRouteParams(c);
+  if (!routeParams.ok) {
+    return apiError(c, 400, "invalid_request", routeParams.error);
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseEntityRowObjectBody(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_json", payload.error.message, { params: payload.error.params });
+  }
+
+  const tenantResult = resolveTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const entity = await dependencies.findLegacyEntityByName({ tenantId: tenantResult.tenantId, name: routeParams.value.entityName, active: true });
+    const scope = authorizeApiEntityRowScope(principal, { tenantId: tenantResult.tenantId, entity, operation: "update" });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+    return await runIdempotentJson(c, {
+      method: "PATCH",
+      route: "/v1/entities/:entity/rows/:rowId",
+      tenantId: tenantResult.tenantId,
+      entityId: getLegacyEntityId(entity),
+      rowId: routeParams.value.rowId,
+      parsedBody: toJsonValue(payload.value),
+    }, async () => {
+      const row = await dependencies.updateLegacyEntityRow({ entity: entity as LegacyEntityWithDetails, tenantId: tenantResult.tenantId, rowId: routeParams.value.rowId, data: payload.value });
+      void emitPluginHookFireAndForget(
+        PLUGIN_HOOK_ENTITY_UPDATED,
+        {
+          entityType: routeParams.value.entityName,
+          entityId: routeParams.value.rowId,
+          tenantId: tenantResult.tenantId,
+          userId: null,
+          changedFields: Object.keys(payload.value),
+          occurredAt: new Date().toISOString(),
+        },
+        { tenantId: tenantResult.tenantId },
+      );
+      return { status: 200, body: row as never };
+    });
+  } catch (error) {
+    const normalizedError = normalizeUnknownError(error);
+    const missing = isMissingLegacyEntityError(normalizedError) || isMissingLegacyRowError(normalizedError);
+    const apiKeyMissing = missing ? apiKeyScopedMissingResourceError(c, principal) : null;
+    if (apiKeyMissing) {
+      return apiKeyMissing;
+    }
+    return apiError(c, missing ? 404 : 400, isMissingLegacyEntityError(normalizedError) ? "entity_not_found" : isMissingLegacyRowError(normalizedError) ? "row_not_found" : "entity_row_update_failed", isMissingLegacyEntityError(normalizedError) ? "Entity not found." : isMissingLegacyRowError(normalizedError) ? "Row not found." : "Entity row could not be updated.");
+  }
+}
+
+async function deleteEntityRowRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const routeParams = parseEntityRowRouteParams(c);
+  if (!routeParams.ok) {
+    return apiError(c, 400, "invalid_request", routeParams.error);
+  }
+
+  const tenantResult = resolveTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const entity = await dependencies.findLegacyEntityByName({ tenantId: tenantResult.tenantId, name: routeParams.value.entityName, active: true });
+    const scope = authorizeApiEntityRowScope(principal, { tenantId: tenantResult.tenantId, entity, operation: "delete" });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+    const entityId = getLegacyEntityId(entity);
+    return await runIdempotentJson(c, {
+      method: "DELETE",
+      route: "/v1/entities/:entity/rows/:rowId",
+      tenantId: tenantResult.tenantId,
+      entityId,
+      rowId: routeParams.value.rowId,
+    }, async () => {
+      const id = await dependencies.deleteLegacyEntityRow({ entityId, tenantId: tenantResult.tenantId, rowId: routeParams.value.rowId });
+      void emitPluginHookFireAndForget(
+        PLUGIN_HOOK_ENTITY_DELETED,
+        {
+          entityType: routeParams.value.entityName,
+          entityId: id,
+          tenantId: tenantResult.tenantId,
+          userId: null,
+          occurredAt: new Date().toISOString(),
+        },
+        { tenantId: tenantResult.tenantId, dispatchKey: `entity:${routeParams.value.entityName}:${id}:deleted` },
+      );
+      return { status: 200, body: { success: true, id } };
+    });
+  } catch (error) {
+    const normalizedError = normalizeUnknownError(error);
+    const missing = isMissingLegacyEntityError(normalizedError) || isMissingLegacyRowError(normalizedError);
+    const apiKeyMissing = missing ? apiKeyScopedMissingResourceError(c, principal) : null;
+    if (apiKeyMissing) {
+      return apiKeyMissing;
+    }
+    return apiError(c, missing ? 404 : 500, isMissingLegacyEntityError(normalizedError) ? "entity_not_found" : isMissingLegacyRowError(normalizedError) ? "row_not_found" : "entity_row_delete_failed", isMissingLegacyEntityError(normalizedError) ? "Entity not found." : isMissingLegacyRowError(normalizedError) ? "Row not found." : "Entity row could not be deleted.");
+  }
+}
+
+async function getEntityRequest(c: Context<ApiEnv>) {
+  const principal = authenticateServiceRequest(c.req.raw, getApiRuntimeConfig());
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const entityName = c.req.param("entity");
+  if (!entityName) {
+    return apiError(c, 400, "invalid_request", "Expected entity.");
+  }
+
+  const tenantScope = parseOptionalTenantId(c);
+  if (!tenantScope.ok) {
+    return apiError(c, 400, "invalid_request", tenantScope.error);
+  }
+  if (!tenantScope.value) {
+    return missingApiTenantScopeError(c);
+  }
+
+  try {
+    const entity = await getApiAppDependencies().findLegacyEntityByName({ tenantId: tenantScope.value, name: entityName, active: true });
+    return c.json(entity as never);
+  } catch (error) {
+    const normalizedError = normalizeUnknownError(error);
+    return apiError(c, isMissingLegacyEntityError(normalizedError) ? 404 : 500, isMissingLegacyEntityError(normalizedError) ? "entity_not_found" : "entity_failed", isMissingLegacyEntityError(normalizedError) ? "Entity not found." : "Entity lookup failed.");
+  }
+}
+
+function resolveTenantScope(c: Context<ApiEnv>, principal: Awaited<ReturnType<typeof authenticateServiceCredentialRequest>>) {
+  const tenantScope = parseOptionalTenantId(c);
+  if (!tenantScope.ok) {
+    return { ok: false as const, response: apiError(c, 400, "invalid_request", tenantScope.error) };
+  }
+  const resolvedTenantScope = resolveApiTenantScope(principal!, tenantScope.value);
+  if (!resolvedTenantScope.allowed) {
+    return { ok: false as const, response: forbiddenApiKeyScopeError(c) };
+  }
+  const tenantId = resolvedTenantScope.tenantId;
+  if (!tenantId) {
+    return { ok: false as const, response: missingApiTenantScopeError(c) };
+  }
+  return { ok: true as const, tenantId };
+}

hivegrid-cloud/apps/api/src/routes/maintenance-routes.ts

@@ -0,0 +1,432 @@
+import { authorizeStorageObjectAccess, authorizeStorageUploadConstraints } from "@toolctrl/storage";
+import { Hono, type Context } from "hono";
+import { getApiAppDependencies, type ExportJobDownloadRecord, type LegacyEventWebhookAttempt } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { authenticateServiceRequest } from "../auth.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { normalizeIdempotencyKey } from "../idempotency/store.js";
+import { postMailjetEmailSequenceWebhookRequest } from "../email-sequences/provider-webhook-route.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { verifyMailjetWebhookAuth } from "../webhooks/mailjet-webhook-auth.js";
+import {
+  missingApiTenantScopeError,
+  parseMaintenanceTake,
+  parseOptionalTenantId,
+  parseStorageObjectStatus,
+  readOptionalJsonObjectBody,
+  requireWorkspaceServiceCredential,
+  authenticateServiceCredentialRequest,
+  apiKeyScopedMissingResourceError,
+  forbiddenApiKeyScopeError,
+} from "./shared.js";
+import { resolveApiTenantScope } from "../auth/scope.js";
+
+const AI_UPLOAD_MAX_BYTES = 10 * 1024 * 1024;
+const AI_UPLOAD_ALLOWED_CONTENT_TYPES = ["image/png", "image/jpeg", "image/webp", "application/pdf", "text/plain", "text/csv", "application/json"];
+
+export function registerMaintenanceRoutes(app: Hono<ApiEnv>) {
+  app.post("/v1/ai/upload", uploadAiFileRequest);
+  app.post("/v1/support/inbound-email", postSupportInboundEmailRequest);
+  app.post("/v1/email-sequences/mailjet/webhooks", (c) =>
+    postMailjetEmailSequenceWebhookRequest(c, {
+      webhookSecret: getApiRuntimeConfig().mailjet.webhookSecret,
+      markEmailSequenceDeliveryBounced: getApiAppDependencies().markEmailSequenceDeliveryBounced,
+    })
+  );
+  app.post("/v1/events/webhooks/attempts/:id", postEventWebhookAttemptRequest);
+  app.post("/v1/maintenance/export-retention/enqueue", postExportRetentionEnqueueRequest);
+  app.get("/v1/exports/:exportJobId/download-url", getExportDownloadUrlRequest);
+  app.get("/v1/exports/:exportJobId/progress", getExportProgressRequest);
+}
+
+async function postSupportInboundEmailRequest(c: Context<ApiEnv>) {
+  const authResult = verifyMailjetWebhookAuth({
+    secret: getApiRuntimeConfig().mailjet.webhookSecret,
+    headers: c.req.header(),
+  });
+
+  if (!authResult.ok) {
+    if (authResult.reason === "missing-secret") {
+      return apiError(c, 500, "webhook_signing_not_configured", "Mailjet webhook authentication is not configured.");
+    }
+    return apiError(c, 401, "invalid_webhook_signature", `Invalid Mailjet webhook credentials: ${authResult.reason}.`);
+  }
+
+  const rawBody = await c.req.raw.text();
+  const payloadResult = parseInboundEmailPayload(rawBody);
+  if (!payloadResult.ok) {
+    return apiError(c, 400, "invalid_request", payloadResult.error);
+  }
+
+  const result = await getApiAppDependencies().createInboundSupportTicketFromEmail(payloadResult.value);
+  if (result.status === "unknown-recipient") {
+    return c.json({ accepted: true, status: "ignored" }, 202);
+  }
+
+  return c.json({ accepted: true, status: result.status, ticketId: result.ticketId }, result.status === "created" ? 201 : 202);
+}
+
+async function uploadAiFileRequest(c: Context<ApiEnv>) {
+  const principal = authenticateServiceRequest(c.req.raw, getApiRuntimeConfig());
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const { storage } = getApiRuntimeConfig();
+  if (!storage.endpoint || !storage.region || !storage.bucket || !storage.accessKeyId || !storage.secretAccessKey) {
+    return apiError(c, 401, "upload_config_missing", "AI upload storage is not configured.");
+  }
+
+  try {
+    const file = c.req.raw.body || "";
+    const tenantScope = parseOptionalTenantId(c);
+    if (!tenantScope.ok) {
+      return apiError(c, 400, "invalid_request", tenantScope.error);
+    }
+    const tenantId = tenantScope.value ?? "";
+    if (!tenantId) {
+      return missingApiTenantScopeError(c);
+    }
+    const filename = c.req.header("x-filename") || "file.txt";
+    const contentType = c.req.header("content-type") || "text/plain";
+    const contentLength = parseContentLength(c.req.header("content-length"));
+    if (contentLength === null) {
+      return apiError(c, 411, "content_length_required", "AI upload requires a valid Content-Length header.");
+    }
+
+    const constraintDecision = authorizeStorageUploadConstraints({
+      operation: "upload",
+      contentType,
+      byteSize: contentLength,
+      allowedContentTypes: AI_UPLOAD_ALLOWED_CONTENT_TYPES,
+      maxBytes: AI_UPLOAD_MAX_BYTES,
+      metadata: { requestId: c.get("requestId") },
+    });
+    if (constraintDecision) {
+      const status = constraintDecision.reason === "object_too_large" ? 413 : 415;
+      return apiError(c, status, constraintDecision.reason, "AI upload storage access denied.");
+    }
+    const fileType = `.${contentType.split("/")[1]}`;
+    const finalName = filename.includes(fileType) ? filename : `${filename}${fileType}`;
+    const path = `tenants/${tenantId}/${new Date().getTime()}-${finalName}`;
+    if (finalName.includes("/") || finalName.includes("\\")) {
+      return apiError(c, 400, "invalid_key", "AI upload storage access denied.");
+    }
+    const storageStatus = parseStorageObjectStatus(c.req.header("x-storage-object-status"));
+    const accessDecision = authorizeStorageObjectAccess({ tenantId, key: path, operation: "upload", status: storageStatus });
+
+    if (!accessDecision.allowed) {
+      const status = accessDecision.reason === "invalid_tenant" || accessDecision.reason === "invalid_key" ? 400 : 403;
+      return apiError(c, status, accessDecision.reason, "AI upload storage access denied.");
+    }
+
+    const dependencies = getApiAppDependencies();
+    const tenant = await dependencies.findLegacyTenantById(tenantId);
+    if (!tenant) {
+      return apiError(c, 404, "tenant_not_found", "AI upload tenant was not found.");
+    }
+    if (!tenant.active || tenant.deactivatedReason) {
+      return apiError(c, 403, "tenant_inactive", "AI upload tenant is not active.");
+    }
+
+    const response = await dependencies.uploadFile(storage.bucket, accessDecision.key, file, contentType);
+    return c.json({ success: true, url: response.publicUrl });
+  } catch {
+    return apiError(c, 500, "ai_upload_failed", "AI file upload failed.");
+  }
+}
+
+function parseContentLength(contentLength: string | undefined): number | null {
+  if (contentLength === undefined) {
+    return null;
+  }
+  const parsed = Number(contentLength);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
+}
+
+type InboundEmailPayload = {
+  messageId: string;
+  toEmail: string;
+  fromEmail: string;
+  subject: string;
+  message: string;
+};
+
+function parseInboundEmailPayload(rawBody: string): { ok: true; value: InboundEmailPayload } | { ok: false; error: string } {
+  let payload: unknown;
+  try {
+    payload = JSON.parse(rawBody) as unknown;
+  } catch {
+    return { ok: false, error: "Inbound email payload must be valid JSON." };
+  }
+  if (!isRecord(payload)) {
+    return { ok: false, error: "Inbound email payload must be an object." };
+  }
+
+  const messageId = readString(payload, "messageId") ?? readString(payload, "message_id") ?? readString(payload, "id");
+  const subject = readString(payload, "subject") ?? "";
+  const message = readString(payload, "text") ?? readString(payload, "textBody") ?? readString(payload, "text_body") ?? readString(payload, "html") ?? readString(payload, "htmlBody") ?? readString(payload, "html_body");
+  const fromEmail = readEmail(payload.from) ?? readString(payload, "fromEmail") ?? readString(payload, "from_email") ?? readString(payload, "from");
+  const toEmail = readEmail(payload.to) ?? readFirstEmail(payload.to) ?? readString(payload, "toEmail") ?? readString(payload, "to_email") ?? readString(payload, "recipient");
+
+  if (!messageId || !fromEmail || !toEmail || !message) {
+    return { ok: false, error: "Inbound email payload is missing messageId, fromEmail, toEmail, or message." };
+  }
+
+  return {
+    ok: true,
+    value: {
+      messageId,
+      fromEmail,
+      toEmail,
+      subject,
+      message,
+    },
+  };
+}
+
+function readString(record: Record<string, unknown>, key: string) {
+  const value = record[key];
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function readEmail(value: unknown): string | undefined {
+  if (typeof value === "string") {
+    return value.trim() || undefined;
+  }
+  if (isRecord(value)) {
+    return readString(value, "email") ?? readString(value, "address");
+  }
+  return undefined;
+}
+
+function readFirstEmail(value: unknown): string | undefined {
+  if (!Array.isArray(value)) {
+    return undefined;
+  }
+  for (const item of value) {
+    const email = readEmail(item);
+    if (email) {
+      return email;
+    }
+  }
+  return undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+async function postEventWebhookAttemptRequest(c: Context<ApiEnv>) {
+  let attempt: LegacyEventWebhookAttempt | null = null;
+  const authResult = verifyMailjetWebhookAuth({
+    secret: getApiRuntimeConfig().mailjet.webhookSecret,
+    headers: c.req.header(),
+  });
+
+  if (!authResult.ok) {
+    if (authResult.reason === "missing-secret") {
+      return apiError(c, 500, "webhook_signing_not_configured", "Mailjet webhook authentication is not configured.");
+    }
+    return apiError(c, 401, "invalid_webhook_signature", `Invalid Mailjet webhook credentials: ${authResult.reason}.`);
+  }
+
+  const rawBody = await c.req.raw.text();
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const attemptId = c.req.param("id") || "";
+    attempt = await dependencies.findLegacyEventWebhookAttemptById(attemptId);
+    if (!attempt) {
+      throw new Error("Invalid event webhook attempt");
+    }
+
+    await dependencies.updateLegacyEventWebhookAttempt({ id: attempt.id, tenantId: attempt.tenantId }, { startedAt: new Date() });
+    let queueResult;
+    try {
+      queueResult = await dependencies.queueLegacyEventWebhookAttemptDelivery({ attempt, body: JSON.parse(rawBody) as unknown, requestId: c.get("requestId") });
+    } catch {
+      await markWebhookAttemptUnavailable(attempt);
+      return apiError(c, 503, "maintenance_queue_unavailable", "Webhook delivery queue is unavailable.");
+    }
+
+    if (!queueResult.enqueued) {
+      await markWebhookAttemptUnavailable(attempt);
+      return apiError(c, 503, "maintenance_queue_unavailable", "Webhook delivery queue is unavailable.");
+    }
+    return c.json({}, 200);
+  } catch {
+    if (attempt) {
+      await getApiAppDependencies().updateLegacyEventWebhookAttempt(
+        { id: attempt.id, tenantId: attempt.tenantId },
+        { finishedAt: new Date(), success: false, status: 400, message: "Webhook delivery request could not be accepted." }
+      );
+    }
+    return apiError(c, 400, "webhook_delivery_failed", "Webhook delivery request could not be accepted.");
+  }
+}
+
+async function markWebhookAttemptUnavailable(attempt: LegacyEventWebhookAttempt) {
+  await getApiAppDependencies().updateLegacyEventWebhookAttempt(
+    { id: attempt.id, tenantId: attempt.tenantId },
+    { finishedAt: new Date(), success: false, status: 503, message: "Webhook delivery queue is unavailable." }
+  );
+}
+
+async function postExportRetentionEnqueueRequest(c: Context<ApiEnv>) {
+  const credentialError = requireWorkspaceServiceCredential(c);
+  if (credentialError) {
+    return credentialError;
+  }
+
+  const bodyResult = await readOptionalJsonObjectBody(c.req.raw);
+  if (!bodyResult.ok) {
+    return apiError(c, 400, "invalid_request", bodyResult.error);
+  }
+
+  const body = bodyResult.value;
+  const unexpectedKeys = Object.keys(body ?? {}).filter((key) => key !== "take");
+  if (unexpectedKeys.length > 0) {
+    return apiError(c, 400, "invalid_request", "Unexpected export retention enqueue fields.");
+  }
+
+  const rawIdempotencyKey = c.req.header("idempotency-key");
+  const idempotencyKey = normalizeIdempotencyKey(rawIdempotencyKey) ?? undefined;
+  if (rawIdempotencyKey !== undefined && !idempotencyKey) {
+    return apiError(c, 400, "invalid_request", "Idempotency-Key header must be a non-empty string when provided.");
+  }
+
+  const takeResult = parseMaintenanceTake(body);
+  if (!takeResult.ok) {
+    return apiError(c, 400, "invalid_request", takeResult.error);
+  }
+
+  const requestId = c.get("requestId");
+  let queueResult;
+  try {
+    queueResult = await getApiAppDependencies().queueExportArtifactRetention({ requestId, idempotencyKey, take: takeResult.value });
+  } catch {
+    return apiError(c, 503, "maintenance_queue_unavailable", "Maintenance queue is unavailable: queue enqueue failed.");
+  }
+
+  if (!queueResult.enqueued) {
+    if (queueResult.reason === "maintenance-worker-disabled") {
+      return apiError(c, 503, "maintenance_queue_unavailable", "Maintenance queue is unavailable: export retention worker is disabled.");
+    }
+    return apiError(c, 503, "maintenance_queue_unavailable", "Maintenance queue is unavailable: missing Redis connection.");
+  }
+
+  return c.json({ enqueued: true, task: "expired-export-artifacts", jobId: String(queueResult.job.id ?? ""), requestId }, 202);
+}
+
+async function getExportDownloadUrlRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential or API key is required.");
+  }
+
+  const exportJobId = c.req.param("exportJobId");
+  if (!exportJobId) {
+    return apiError(c, 400, "invalid_request", "Expected exportJobId.");
+  }
+
+  const requestedTenantScope = parseOptionalTenantId(c);
+  if (!requestedTenantScope.ok) {
+    return apiError(c, 400, "invalid_request", requestedTenantScope.error);
+  }
+  const resolvedTenantScope = resolveApiTenantScope(principal, requestedTenantScope.value);
+  if (!resolvedTenantScope.allowed) {
+    return forbiddenApiKeyScopeError(c);
+  }
+  const lookupTenantId = resolvedTenantScope.tenantId;
+  if (!lookupTenantId) {
+    return apiError(c, 400, "invalid_request", "tenantId is required.");
+  }
+
+  try {
+    const exportJob = await getApiAppDependencies().findExportJobForDownload({ id: exportJobId, tenantId: lookupTenantId });
+    if (!exportJob) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      if (apiKeyMissing) {
+        return apiKeyMissing;
+      }
+      return apiError(c, 404, "export_job_not_found", `Export job ${exportJobId} not found.`);
+    }
+
+    if (exportJob.errorName === "ExportPaused") {
+      return apiError(c, 409, "export_job_paused", "Export job is paused and cannot be downloaded.");
+    }
+    if (exportJob.status !== "COMPLETED") {
+      return apiError(c, 409, "export_job_not_completed", "Export job is not completed.");
+    }
+    if (!exportJob.artifactBucket || !exportJob.artifactKey) {
+      return apiError(c, 409, "export_artifact_missing", "Export artifact is not available.");
+    }
+    if (exportJob.expiresAt && exportJob.expiresAt.getTime() <= Date.now()) {
+      return apiError(c, 409, "export_job_expired", "Export job has expired.");
+    }
+
+    const signedUrl = await getApiAppDependencies().createSignedDownloadUrl(exportJob.artifactBucket, {
+      key: exportJob.artifactKey,
+      expiresInSeconds: 300,
+      responseContentDisposition: `attachment; filename="${exportJob.id}.${getExportArtifactExtension(exportJob)}"`,
+    });
+
+    return c.json({
+      exportJobId: exportJob.id,
+      requestId: c.get("requestId"),
+      method: signedUrl.method,
+      url: signedUrl.url,
+      expiresAt: signedUrl.expiresAt.toISOString(),
+      expiresInSeconds: signedUrl.expiresInSeconds,
+      headers: signedUrl.headers,
+    });
+  } catch {
+    return apiError(c, 500, "export_download_url_failed", "Export download URL could not be created.");
+  }
+}
+
+function getExportArtifactExtension(exportJob: Pick<ExportJobDownloadRecord, "format" | "artifactKey">) {
+  const artifactExtension = exportJob.artifactKey?.match(/\.([A-Za-z0-9]+)$/)?.[1]?.toLowerCase();
+  if (artifactExtension && ["csv", "json", "xlsx"].includes(artifactExtension)) {
+    return artifactExtension;
+  }
+  return exportJob.format;
+}
+
+async function getExportProgressRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential or API key is required.");
+  }
+
+  const exportJobId = c.req.param("exportJobId");
+  if (!exportJobId) {
+    return apiError(c, 400, "invalid_request", "Expected exportJobId.");
+  }
+
+  const requestedTenantScope = parseOptionalTenantId(c);
+  if (!requestedTenantScope.ok) {
+    return apiError(c, 400, "invalid_request", requestedTenantScope.error);
+  }
+  const resolvedTenantScope = resolveApiTenantScope(principal, requestedTenantScope.value);
+  if (!resolvedTenantScope.allowed) {
+    return forbiddenApiKeyScopeError(c);
+  }
+  const lookupTenantId = resolvedTenantScope.tenantId;
+  if (!lookupTenantId) {
+    return apiError(c, 400, "invalid_request", "tenantId is required.");
+  }
+
+  const progress = await getApiAppDependencies().readExportProgress({ exportJobId, tenantId: lookupTenantId });
+  if (!progress) {
+    return c.json({ exportJobId, processedRows: 0, totalEstimate: null, updatedAt: null, requestId: c.get("requestId") });
+  }
+  return c.json({
+    exportJobId,
+    processedRows: progress.processedRows,
+    totalEstimate: progress.totalEstimate,
+    updatedAt: progress.updatedAt,
+    requestId: c.get("requestId"),
+  });
+}

hivegrid-cloud/apps/api/src/routes/metrics.test.ts

@@ -0,0 +1,56 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { Hono } from "hono";
+import { createDefaultMetrics, createMetricsRegistry, type DefaultMetrics, type MetricsRegistry } from "@toolctrl/metrics";
+import { handleApiMetricsRequest } from "./metrics.js";
+
+const TOKEN = "x".repeat(64);
+
+let registry: MetricsRegistry;
+let metrics: DefaultMetrics;
+
+beforeEach(() => {
+  registry = createMetricsRegistry({ serviceName: "api-test", collectDefaultProcessMetrics: false });
+  metrics = createDefaultMetrics(registry);
+  metrics.httpRequestsTotal.labels({ method: "GET", route: "/status", status: "200" }).inc();
+});
+
+function app(enabled: boolean, token: string | undefined): Hono {
+  const a = new Hono();
+  a.get("/metrics", (c) => handleApiMetricsRequest(c, { enabled, configuredToken: token, registry }));
+  return a;
+}
+
+describe("handleApiMetricsRequest", () => {
+  it("returns 404 when metrics are disabled", async () => {
+    const res = await app(false, TOKEN).request("/metrics");
+    expect(res.status).toBe(404);
+  });
+
+  it("returns 500 when enabled but no token configured", async () => {
+    const res = await app(true, undefined).request("/metrics");
+    expect(res.status).toBe(500);
+  });
+
+  it("returns 401 without bearer header", async () => {
+    const res = await app(true, TOKEN).request("/metrics");
+    expect(res.status).toBe(401);
+  });
+
+  it("returns 401 with wrong token", async () => {
+    const res = await app(true, TOKEN).request("/metrics", {
+      headers: { authorization: `Bearer ${"y".repeat(TOKEN.length)}` },
+    });
+    expect(res.status).toBe(401);
+  });
+
+  it("returns 200 + prom text with valid token", async () => {
+    const res = await app(true, TOKEN).request("/metrics", {
+      headers: { authorization: `Bearer ${TOKEN}` },
+    });
+    expect(res.status).toBe(200);
+    expect(res.headers.get("content-type")).toContain("text/plain");
+    expect(res.headers.get("cache-control")).toBe("no-store");
+    const body = await res.text();
+    expect(body).toContain("http_requests_total");
+  });
+});

hivegrid-cloud/apps/api/src/routes/metrics.ts

@@ -0,0 +1,51 @@
+/**
+ * Token-protected `/metrics` endpoint. Returns the Prometheus text exposition
+ * format when authenticated, 401 otherwise. Returns 404 when
+ * `METRICS_ENABLED=false` so the surface is invisible to opportunistic
+ * scanners.
+ *
+ * Auth: `Authorization: Bearer <METRICS_TOKEN>`. Constant-time compare via
+ * `@toolctrl/metrics`.
+ *
+ * @see docs/improvement-plan/02-observability/metrics-grafana.md
+ */
+import type { Context } from "hono";
+import { validateMetricsTokenAuth } from "@toolctrl/metrics";
+import type { MetricsRegistry } from "@toolctrl/metrics";
+
+export type ApiMetricsRouteDependencies = {
+  enabled: boolean;
+  configuredToken: string | undefined;
+  registry: MetricsRegistry;
+};
+
+export async function handleApiMetricsRequest(c: Context, deps: ApiMetricsRouteDependencies): Promise<Response> {
+  if (!deps.enabled) {
+    return c.json({ error: { code: "not_found", message: "Endpoint not enabled." } }, 404);
+  }
+  const auth = validateMetricsTokenAuth({
+    metricsEnabled: true,
+    configuredToken: deps.configuredToken,
+    authorizationHeader: c.req.header("authorization"),
+  });
+  if (!auth.ok) {
+    if (auth.status === 500) {
+      return c.json({ error: { code: "token_misconfigured", message: "Metrics token not configured." } }, 500);
+    }
+    if (auth.status === 403) {
+      // metrics_disabled — caller passed enabled=false; the early return above
+      // means we shouldn't reach this branch, but exhaust the union safely.
+      return c.json({ error: { code: "not_found", message: "Endpoint not enabled." } }, 404);
+    }
+    return c.json({ error: { code: auth.code, message: "Authentication required." } }, 401);
+  }
+
+  const body = await deps.registry.metrics();
+  return new Response(body, {
+    status: 200,
+    headers: {
+      "content-type": deps.registry.contentType,
+      "cache-control": "no-store",
+    },
+  });
+}

hivegrid-cloud/apps/api/src/routes/notifications-routes.ts

@@ -0,0 +1,113 @@
+import {
+  REALTIME_EVENT_NAMES,
+  assertNotificationEventInAppScope,
+  normalizeNotificationStreamScope,
+  redactNotificationGuardDiagnostic,
+  serializeRealtimeComment,
+  serializeRealtimeEvent,
+} from "@toolctrl/realtime";
+import { Hono, type Context } from "hono";
+import { getApiAppDependencies } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { authenticateServiceRequest } from "../auth.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { handleWorkspaceContextRevalidationError } from "./shared.js";
+
+export function registerNotificationRoutes(app: Hono<ApiEnv>) {
+  app.get("/v1/notifications/stream", getNotificationStreamRequest);
+}
+
+async function getNotificationStreamRequest(c: Context<ApiEnv>) {
+  const principal = authenticateServiceRequest(c.req.raw, getApiRuntimeConfig());
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid bearer token is required.");
+  }
+
+  const requestId: string = c.get("requestId");
+  const scopeResult:
+    | { ok: true; scope: ReturnType<typeof normalizeNotificationStreamScope> }
+    | { ok: false; error: string } = (() => {
+    try {
+      return {
+        ok: true,
+        scope: normalizeNotificationStreamScope({
+          tenantId: c.req.header("x-toolctrl-tenant-id") ?? c.req.query("tenantId"),
+          userId: c.req.header("x-toolctrl-user-id") ?? c.req.query("userId"),
+          afterCursor: c.req.header("last-event-id") ?? c.req.query("afterCursor"),
+        }),
+      };
+    } catch (error) {
+      return { ok: false, error: redactNotificationGuardDiagnostic(error) };
+    }
+  })();
+
+  if (!scopeResult.ok) {
+    return apiError(c, 400, "invalid_request", scopeResult.error);
+  }
+
+  const { scope } = scopeResult;
+  const dependencies = getApiAppDependencies();
+  try {
+    await dependencies.revalidateWorkspaceContext({ tenantId: scope.tenantId, userId: scope.userId });
+  } catch (error) {
+    return handleWorkspaceContextRevalidationError(c, error);
+  }
+  const afterCursor = scope.afterCursor ?? null;
+  const stream = new ReadableStream<Uint8Array>({
+    async start(controller) {
+      const encoder = new TextEncoder();
+      const write = (chunk: string) => controller.enqueue(encoder.encode(chunk));
+      const writeRealtimeEvent = (event: Parameters<typeof assertNotificationEventInAppScope>[0]) => {
+        assertNotificationEventInAppScope(event, scope);
+        write(serializeRealtimeEvent(event));
+      };
+
+      try {
+        const snapshot = await dependencies.getNotificationStreamSnapshot({ tenantId: scope.tenantId, userId: scope.userId, afterCursor });
+        const cursor = snapshot.cursor ?? afterCursor ?? "initial";
+        writeRealtimeEvent({
+          event: REALTIME_EVENT_NAMES.notificationSnapshot,
+          requestId,
+          tenantId: scope.tenantId,
+          userId: scope.userId,
+          cursor,
+          items: snapshot.items,
+        });
+        write(serializeRealtimeComment(`keepalive ${JSON.stringify({ requestId, cursor })}`));
+        writeRealtimeEvent({ event: REALTIME_EVENT_NAMES.keepalive, requestId, tenantId: scope.tenantId, userId: scope.userId, cursor });
+      } catch {
+        const cursor = afterCursor ?? "error";
+        write(
+          serializeRealtimeEvent({
+            event: REALTIME_EVENT_NAMES.error,
+            requestId,
+            tenantId: scope.tenantId,
+            userId: scope.userId,
+            cursor,
+            code: "notification_stream_failed",
+            message: "Notification stream snapshot failed.",
+          })
+        );
+      } finally {
+        controller.close();
+      }
+    },
+  });
+
+  return new Response(stream, {
+    status: 200,
+    headers: {
+      "content-type": "text/event-stream; charset=utf-8",
+      "cache-control": "no-cache",
+      connection: "keep-alive",
+      "x-request-id": requestId,
+      "x-content-type-options": "nosniff",
+      "referrer-policy": "no-referrer",
+      "x-frame-options": "DENY",
+      "permissions-policy": "camera=(), microphone=(), geolocation=(), payment=()",
+      "cross-origin-opener-policy": "same-origin",
+      "cross-origin-resource-policy": "same-origin",
+    },
+  });
+}

hivegrid-cloud/apps/api/src/routes/relationship-routes.ts

@@ -0,0 +1,233 @@
+import { Hono, type Context } from "hono";
+import { getApiAppDependencies, type LegacyRelationshipRow } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { authorizeApiRelationshipScope } from "../auth/permission.js";
+import { resolveApiTenantScope } from "../auth/scope.js";
+import { apiError } from "../middleware/error-envelope.js";
+import { parseRelationshipCreatePayload } from "../validation/request-schemas.js";
+import {
+  ApiRouteError,
+  apiKeyScopedMissingResourceError,
+  authenticateServiceCredentialRequest,
+  forbiddenApiKeyScopeError,
+  parseOptionalTenantId,
+  readJsonBody,
+  runIdempotentJson,
+  toRelationshipEnvelope,
+} from "./shared.js";
+
+export function registerRelationshipRoutes(app: Hono<ApiEnv>) {
+  app.post("/v1/relationships", postRelationshipRequest);
+  app.get("/v1/relationships/:id", getRelationshipRequest);
+  app.delete("/v1/relationships/:id", deleteRelationshipRequest);
+}
+
+async function postRelationshipRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseRelationshipCreatePayload(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+  }
+
+  const tenantResult = resolveRelationshipTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const parent = await dependencies.findLegacyRelationshipRowById({ id: payload.value.parent, tenantId: tenantResult.tenantId });
+    if (!parent) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "parent_row_not_found", "Parent row not found.", { params: { rowId: payload.value.parent } });
+    }
+
+    const child = await dependencies.findLegacyRelationshipRowById({ id: payload.value.child, tenantId: tenantResult.tenantId });
+    if (!child) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "child_row_not_found", "Child row not found.", { params: { rowId: payload.value.child } });
+    }
+    if (!relationshipRowsBelongToTenant(tenantResult.tenantId, parent, child)) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "relationship_rows_not_found", "Relationship rows not found.");
+    }
+
+    const scope = authorizeApiRelationshipScope(principal, { tenantId: tenantResult.tenantId, parentEntityId: parent.entityId, childEntityId: child.entityId, operation: "create" });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+
+    return await runIdempotentJson(c, {
+      method: "POST",
+      route: "/v1/relationships",
+      tenantId: tenantResult.tenantId,
+      entityId: parent.entityId,
+      relationshipId: `${parent.id}:${child.id}`,
+      parsedBody: payload.value,
+    }, async () => {
+      const entityRelationship = await dependencies.findLegacyEntityRelationship({ parentId: parent.entityId, childId: child.entityId });
+      if (!entityRelationship) {
+        throw new ApiRouteError(404, "entity_relationship_not_found", "Entity relationship not found.");
+      }
+
+      const relationship = await dependencies.createLegacyRowRelationship({
+        relationshipId: entityRelationship.id,
+        parentId: parent.id,
+        childId: child.id,
+        tenantId: tenantResult.tenantId,
+      });
+      return { status: 201, body: { relationship: toRelationshipEnvelope(relationship, parent, child) } };
+    });
+  } catch (error) {
+    if (error instanceof ApiRouteError) {
+      return apiError(c, error.status, error.code, error.message, { params: error.params });
+    }
+    return apiError(c, 500, "relationship_create_failed", "Relationship could not be created.");
+  }
+}
+
+function relationshipRowsBelongToTenant(tenantId: string | null, ...rows: LegacyRelationshipRow[]) {
+  return Boolean(tenantId) && rows.every((row) => row.tenantId === tenantId);
+}
+
+async function getRelationshipRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const id = c.req.param("id");
+  if (!id) {
+    return apiError(c, 400, "invalid_request", "Expected relationship id.");
+  }
+
+  const tenantResult = resolveRelationshipTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const relationship = await dependencies.findLegacyRowRelationshipById(id, tenantResult.tenantId);
+    if (!relationship) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "relationship_not_found", "Relationship not found.", { params: { relationshipId: id } });
+    }
+
+    const scope = authorizeApiRelationshipScope(principal, {
+      tenantId: tenantResult.tenantId,
+      parentEntityId: relationship.relationship.parentId,
+      childEntityId: relationship.relationship.childId,
+      operation: "read",
+    });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+
+    const [parent, child] = await Promise.all([
+      dependencies.getLegacyEntityRow({ entityId: relationship.relationship.parentId, tenantId: tenantResult.tenantId, rowId: relationship.parentId }),
+      dependencies.getLegacyEntityRow({ entityId: relationship.relationship.childId, tenantId: tenantResult.tenantId, rowId: relationship.childId }),
+    ]);
+    if (!parent) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "parent_row_not_found", "Parent row not found.", { params: { rowId: relationship.parentId } });
+    }
+    if (!child) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "child_row_not_found", "Child row not found.", { params: { rowId: relationship.childId } });
+    }
+
+    return c.json({ relationship: toRelationshipEnvelope(relationship, parent, child) });
+  } catch {
+    return apiError(c, 500, "relationship_get_failed", "Relationship could not be loaded.");
+  }
+}
+
+async function deleteRelationshipRequest(c: Context<ApiEnv>) {
+  const principal = await authenticateServiceCredentialRequest(c);
+  if (!principal) {
+    return apiError(c, 401, "unauthorized", "A valid service credential is required.");
+  }
+
+  const id = c.req.param("id");
+  if (!id) {
+    return apiError(c, 400, "invalid_request", "Expected relationship id.");
+  }
+
+  const tenantResult = resolveRelationshipTenantScope(c, principal);
+  if (!tenantResult.ok) {
+    return tenantResult.response;
+  }
+
+  try {
+    const dependencies = getApiAppDependencies();
+    const relationship = await dependencies.findLegacyRowRelationshipById(id, tenantResult.tenantId);
+    if (!relationship) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "relationship_not_found", "Relationship not found.", { params: { relationshipId: id } });
+    }
+
+    const scope = authorizeApiRelationshipScope(principal, {
+      tenantId: tenantResult.tenantId,
+      parentEntityId: relationship.relationship.parentId,
+      childEntityId: relationship.relationship.childId,
+      operation: "delete",
+    });
+    if (!scope.allowed) {
+      return forbiddenApiKeyScopeError(c);
+    }
+
+    const [parent, child] = await Promise.all([
+      dependencies.getLegacyEntityRow({ entityId: relationship.relationship.parentId, tenantId: tenantResult.tenantId, rowId: relationship.parentId }),
+      dependencies.getLegacyEntityRow({ entityId: relationship.relationship.childId, tenantId: tenantResult.tenantId, rowId: relationship.childId }),
+    ]);
+    if (!parent) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "parent_row_not_found", "Parent row not found.", { params: { rowId: relationship.parentId } });
+    }
+    if (!child) {
+      const apiKeyMissing = apiKeyScopedMissingResourceError(c, principal);
+      return apiKeyMissing ?? apiError(c, 404, "child_row_not_found", "Child row not found.", { params: { rowId: relationship.childId } });
+    }
+
+    return await runIdempotentJson(c, {
+      method: "DELETE",
+      route: "/v1/relationships/:id",
+      tenantId: tenantResult.tenantId,
+      entityId: parent.entityId,
+      relationshipId: id,
+    }, async () => {
+      await dependencies.deleteLegacyRowRelationshipById({
+        id,
+        relationshipId: relationship.relationshipId,
+        parentId: relationship.parentId,
+        childId: relationship.childId,
+        tenantId: tenantResult.tenantId,
+      });
+      return { status: 200, body: { deleted: toRelationshipEnvelope(relationship, parent, child) } };
+    });
+  } catch {
+    return apiError(c, 500, "relationship_delete_failed", "Relationship could not be deleted.");
+  }
+}
+
+function resolveRelationshipTenantScope(c: Context<ApiEnv>, principal: Awaited<ReturnType<typeof authenticateServiceCredentialRequest>>) {
+  const tenantScope = parseOptionalTenantId(c);
+  if (!tenantScope.ok) {
+    return { ok: false as const, response: apiError(c, 400, "invalid_request", tenantScope.error) };
+  }
+  const resolvedTenantScope = resolveApiTenantScope(principal!, tenantScope.value);
+  if (!resolvedTenantScope.allowed) {
+    return { ok: false as const, response: forbiddenApiKeyScopeError(c) };
+  }
+  const tenantId = resolvedTenantScope.tenantId;
+  if (!tenantId) {
+    return { ok: false as const, response: apiError(c, 400, "invalid_request", "tenantId is required.") };
+  }
+  return { ok: true as const, tenantId };
+}

hivegrid-cloud/apps/api/src/routes/shared.ts

@@ -0,0 +1,399 @@
+import type { Context } from "hono";
+import jwt from "jsonwebtoken";
+import { WorkspaceContextRevalidationError } from "@toolctrl/db/auth-providers.server";
+import { WorkspaceInviteError, type WorkspaceInviteStatus } from "@toolctrl/db/workspace-invites.server";
+import type { StorageObjectStatus } from "@toolctrl/storage";
+import type { LegacyEntity, LegacyEntityRow, LegacyRelationshipRow, LegacyRowRelationshipDetails, WorkspaceAuthProviderDto, WorkspaceInviteDto } from "../app-dependencies.js";
+import { getApiAppDependencies } from "../app-dependencies.js";
+import type { ApiEnv, ParseResult } from "../api-types.js";
+import { authenticateServiceRequest, createServicePrincipal, extractBearerToken, isValidServiceToken } from "../auth.js";
+import { type LoginRateLimitBlocked, type LoginRateLimitRequest } from "../auth/login-rate-limit.js";
+import { createStoredApiKeyPrincipal, type ApiScopedPrincipal } from "../auth/scope.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { apiLogger } from "../logger.js";
+import { apiError } from "../middleware/error-envelope.js";
+import {
+  createApiIdempotencyScope,
+  createIdempotencyStore,
+  createRequestFingerprint,
+  normalizeIdempotencyKey,
+  type JsonValue,
+  type StoredIdempotencyResponse,
+} from "../idempotency/store.js";
+
+export function getBearerToken(request: Request) {
+  return extractBearerToken(request);
+}
+
+export function getLegacyUserIdFromJwt(decoded: string | jwt.JwtPayload) {
+  if (!decoded || typeof decoded !== "object") {
+    return null;
+  }
+  return "userId" in decoded && typeof decoded.userId === "string" && decoded.userId ? decoded.userId : null;
+}
+
+export function getApiSessionIdFromJwt(decoded: string | jwt.JwtPayload) {
+  if (!decoded || typeof decoded !== "object") {
+    return null;
+  }
+  return "sessionId" in decoded && typeof decoded.sessionId === "string" && decoded.sessionId ? decoded.sessionId : null;
+}
+
+export function requireWorkspaceServiceCredential(c: Context<ApiEnv>) {
+  return authenticateServiceRequest(c.req.raw, getApiRuntimeConfig()) ? null : apiError(c, 401, "unauthorized", "A valid service credential is required.");
+}
+
+export function requireServiceCredential(c: Context<ApiEnv>) {
+  return authenticateServiceRequest(c.req.raw, getApiRuntimeConfig()) ? null : apiError(c, 401, "unauthorized", "A valid service credential is required.");
+}
+
+export function requireWorkspaceInviteServiceCredential(c: Context<ApiEnv>) {
+  return requireWorkspaceServiceCredential(c);
+}
+
+export function parseWorkspaceTenantId(c: Context<ApiEnv>) {
+  const tenantId = c.req.param("tenantId")?.trim();
+  return tenantId || null;
+}
+
+export function parseWorkspaceInviteStatusQuery(status: string | undefined): WorkspaceInviteStatus | undefined {
+  if (status === "pending" || status === "accepted" || status === "revoked" || status === "expired") {
+    return status;
+  }
+  return undefined;
+}
+
+export function parseWorkspaceAuthProviderKindQuery(kind: string | undefined) {
+  return kind === "oidc" || kind === "saml" ? kind : undefined;
+}
+
+export function parseOptionalBooleanQuery(value: string | undefined): boolean | undefined {
+  if (value === "true") {
+    return true;
+  }
+  if (value === "false") {
+    return false;
+  }
+  return undefined;
+}
+
+export function handleWorkspaceContextRevalidationError(c: Context<ApiEnv>, error: unknown) {
+  if (error instanceof WorkspaceContextRevalidationError) {
+    const status = error.code === "context_missing_tenant_id" || error.code === "context_missing_user_id" ? 400 : 403;
+    return apiError(c, status, error.code, "Workspace context is not valid for this tenant and user.");
+  }
+  return apiError(c, 500, "workspace_context_revalidation_failed", "Workspace context revalidation failed.");
+}
+
+export function handleWorkspaceInviteError(c: Context<ApiEnv>, error: unknown) {
+  if (error instanceof WorkspaceInviteError) {
+    if (error.code === "duplicate_active_invite") {
+      return apiError(c, 409, error.code, "An active workspace invite already exists.");
+    }
+    if (error.code === "invite_not_found") {
+      return apiError(c, 404, error.code, "Workspace invite was not found.");
+    }
+    if (error.code === "invite_expired") {
+      return apiError(c, 410, error.code, "Workspace invite has expired.");
+    }
+    if (error.code === "invite_already_finalized") {
+      return apiError(c, 400, error.code, "Workspace invite is already finalized.");
+    }
+  }
+  return apiError(c, 500, "workspace_invite_failed", "Workspace invite request failed.");
+}
+
+export function toWorkspaceAuthProviderApiResponse(provider: WorkspaceAuthProviderDto) {
+  return {
+    id: provider.id,
+    tenantId: provider.tenantId,
+    kind: provider.kind,
+    name: provider.name,
+    slug: provider.slug,
+    enabled: provider.enabled,
+    status: provider.status,
+    issuer: provider.issuer,
+    authorizationUrl: provider.authorizationUrl,
+    tokenUrl: provider.tokenUrl,
+    jwksUrl: provider.jwksUrl,
+    clientId: provider.clientId,
+    entityId: provider.entityId,
+    ssoUrl: provider.ssoUrl,
+    certificateFingerprint: provider.certificateFingerprint,
+    metadataUpdatedAt: provider.metadataUpdatedAt ? provider.metadataUpdatedAt.toISOString() : null,
+    createdAt: provider.createdAt.toISOString(),
+    updatedAt: provider.updatedAt.toISOString(),
+  };
+}
+
+export function toWorkspaceInviteApiResponse(invite: WorkspaceInviteDto) {
+  return {
+    id: invite.id,
+    tenantId: invite.tenantId,
+    email: invite.email,
+    firstName: invite.firstName,
+    lastName: invite.lastName,
+    type: invite.type,
+    status: invite.status,
+    pending: invite.pending,
+    tokenPrefix: invite.tokenPrefix,
+    createdUserId: invite.createdUserId,
+    fromUserId: invite.fromUserId,
+    acceptedByUserId: invite.acceptedByUserId,
+    revokedByUserId: invite.revokedByUserId,
+    createdAt: null,
+    expiresAt: invite.expiresAt ? invite.expiresAt.toISOString() : null,
+    acceptedAt: invite.acceptedAt ? invite.acceptedAt.toISOString() : null,
+    revokedAt: invite.revokedAt ? invite.revokedAt.toISOString() : null,
+  };
+}
+
+export function parseStorageObjectStatus(value: string | undefined): StorageObjectStatus | undefined {
+  if (value === undefined) {
+    return "clean";
+  }
+  if (value === "clean" || value === "available" || value === "blocked" || value === "quarantined") {
+    return value;
+  }
+  return "blocked";
+}
+
+export function normalizeUnknownError(error: unknown) {
+  if (error instanceof Error) {
+    return error;
+  }
+  return new Error(String(error));
+}
+
+export async function readJsonBody(request: Request): Promise<unknown> {
+  try {
+    return await request.json();
+  } catch {
+    return undefined;
+  }
+}
+
+export async function readOptionalJsonObjectBody(request: Request): Promise<ParseResult<Record<string, unknown> | undefined>> {
+  const rawBody = await request.text();
+  if (rawBody.trim() === "") {
+    return { ok: true, value: undefined };
+  }
+
+  let body: unknown;
+  try {
+    body = JSON.parse(rawBody);
+  } catch {
+    return { ok: false, error: "JSON body must be valid JSON when provided." };
+  }
+
+  if (!isPlainObject(body)) {
+    return { ok: false, error: "JSON body must be an object when provided." };
+  }
+  return { ok: true, value: body };
+}
+
+export function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function parseMaintenanceTake(body: unknown): ParseResult<number | undefined> {
+  if (!isPlainObject(body) || body.take === undefined) {
+    return { ok: true, value: undefined };
+  }
+  const take = body.take;
+  if (typeof take !== "number" || !Number.isInteger(take) || take < 1 || take > 500) {
+    return { ok: false, error: "take must be an integer between 1 and 500 when provided." };
+  }
+  return { ok: true, value: take };
+}
+
+export function createLoginRateLimitRequest(c: Context<ApiEnv>, account: string): LoginRateLimitRequest {
+  return { headers: c.req.raw.headers, remoteAddress: null, account };
+}
+
+export function createLoginRateLimitResponse(c: Context<ApiEnv>, block: LoginRateLimitBlocked) {
+  c.header("Retry-After", String(block.retryAfterSeconds));
+  apiLogger.warn(
+    {
+      requestId: c.get("requestId"),
+      scope: block.scope,
+      retryAfterSeconds: block.retryAfterSeconds,
+      accountFingerprint: block.accountFingerprint,
+    },
+    "Login rate limit blocked request"
+  );
+
+  return c.json(
+    {
+      error: {
+        code: "rate_limited",
+        message: "Too many login attempts. Try again later.",
+        params: { retryAfterSeconds: block.retryAfterSeconds },
+        requestId: c.get("requestId"),
+      },
+    },
+    429
+  );
+}
+
+export async function authenticateServiceCredentialRequest(c: Context<ApiEnv>): Promise<ApiScopedPrincipal | null> {
+  const apiKey = c.req.header("x-api-key");
+  const bearerToken = extractBearerToken(c.req.raw);
+  if (apiKey && bearerToken) {
+    return null;
+  }
+
+  const config = getApiRuntimeConfig();
+  if (bearerToken) {
+    if (!isValidServiceToken(bearerToken, config)) {
+      return null;
+    }
+    return createServicePrincipal();
+  }
+
+  if (!apiKey) {
+    return null;
+  }
+
+  const storedApiKey = await getApiAppDependencies().findLegacyApiKeyByKey(apiKey, undefined as never, config.jwtSecret, {
+    allowPlaintextFallback: config.apiKeys.allowLegacyPlaintextFallback,
+  });
+  if (storedApiKey) {
+    return createStoredApiKeyPrincipal(storedApiKey);
+  }
+  return null;
+}
+
+export function forbiddenApiKeyScopeError(c: Context<ApiEnv>) {
+  return apiError(c, 403, "forbidden", "API key is not authorized for this entity operation.");
+}
+
+export class ApiRouteError extends Error {
+  constructor(
+    readonly status: 404,
+    readonly code: string,
+    message: string,
+    readonly params?: Record<string, string | number | boolean | null>,
+  ) {
+    super(message);
+    this.name = "ApiRouteError";
+  }
+}
+
+export async function runIdempotentJson(
+  c: Context<ApiEnv>,
+  input: Parameters<typeof createRequestFingerprint>[0],
+  operation: () => Promise<{ status: number; body: JsonValue }>,
+) {
+  const idempotencyKey = c.req.header("idempotency-key");
+  if (!normalizeIdempotencyKey(idempotencyKey)) {
+    return apiError(c, 400, "idempotency_key_required", "Idempotency-Key header is required for this operation.");
+  }
+
+  const fingerprint = createRequestFingerprint(input);
+  const idempotencyStore = createIdempotencyStore({ client: getApiAppDependencies().getIdempotencyClient(), scope: createApiIdempotencyScope(input) });
+  const result = await idempotencyStore.run(idempotencyKey, fingerprint, async () => {
+    const response = await operation();
+    return {
+      status: response.status,
+      headers: { "content-type": "application/json" },
+      body: response.body,
+    };
+  });
+
+  if (result.type === "replay") {
+    return replayIdempotentJson(c, result.response);
+  }
+  if (result.type === "conflict") {
+    return apiError(c, 409, "idempotency_key_conflict", "Idempotency-Key was already used for a different request.");
+  }
+  if (result.type === "in_progress") {
+    return apiError(c, 409, "idempotency_key_in_progress", "Idempotency-Key is already being processed.");
+  }
+  return c.json(result.response.body as never, result.response.status as never);
+}
+
+function replayIdempotentJson(c: Context<ApiEnv>, response: StoredIdempotencyResponse) {
+  for (const [name, value] of Object.entries(response.headers)) {
+    c.header(name, value);
+  }
+  return c.json(response.body as never, response.status as never);
+}
+
+export function apiKeyScopedMissingResourceError(c: Context<ApiEnv>, principal: ApiScopedPrincipal) {
+  return principal.type === "api-key" ? forbiddenApiKeyScopeError(c) : null;
+}
+
+export function missingApiTenantScopeError(c: Context<ApiEnv>) {
+  return apiError(c, 400, "invalid_request", "tenantId is required.");
+}
+
+export function parseOptionalTenantId(c: Context<ApiEnv>): ParseResult<string | null> {
+  const tenantScopes = [
+    { source: "tenantId", value: c.req.query("tenantId") },
+    { source: "x-tenant-id", value: c.req.header("x-tenant-id") },
+    { source: "x-account-id", value: c.req.header("x-account-id") },
+  ]
+    .map((scope) => ({ ...scope, value: typeof scope.value === "string" ? scope.value.trim() : "" }))
+    .filter((scope) => scope.value.length > 0);
+
+  const distinctValues = new Set(tenantScopes.map((scope) => scope.value));
+  if (distinctValues.size > 1) {
+    return { ok: false, error: "Conflicting tenant scope values." };
+  }
+  return { ok: true, value: tenantScopes[0]?.value ?? null };
+}
+
+export function parseEntityRowRouteParams(c: Context<ApiEnv>): ParseResult<{ entityName: string; rowId: string }> {
+  const entityName = c.req.param("entity");
+  if (!entityName) {
+    return { ok: false, error: "Expected entity." };
+  }
+  const rowId = c.req.param("rowId");
+  if (!rowId) {
+    return { ok: false, error: "Expected rowId." };
+  }
+  return { ok: true, value: { entityName, rowId } };
+}
+
+export function getLegacyEntityId(entity: LegacyEntity) {
+  if (entity && typeof entity === "object" && "id" in entity && typeof entity.id === "string" && entity.id) {
+    return entity.id;
+  }
+  throw new Error("Entity is missing an id.");
+}
+
+export function isMissingLegacyEntityError(error: Error) {
+  return /entity with name .+ not found/i.test(error.message);
+}
+
+export function isMissingLegacyRowError(error: Error) {
+  return /^Row .+ not found$/.test(error.message);
+}
+
+export function toRelationshipEnvelope(relationship: LegacyRowRelationshipDetails, parent: LegacyRelationshipRow | LegacyEntityRow, child: LegacyRelationshipRow | LegacyEntityRow): JsonValue {
+  return toJsonValue({ id: relationship.id, parent, child });
+}
+
+export function toJsonValue(value: unknown): JsonValue {
+  return JSON.parse(JSON.stringify(value)) as JsonValue;
+}
+
+export function escapeHtml(value: string) {
+  return value.replace(/[&<>"']/g, (char) => {
+    switch (char) {
+      case "&":
+        return "&amp;";
+      case "<":
+        return "&lt;";
+      case ">":
+        return "&gt;";
+      case '"':
+        return "&quot;";
+      case "'":
+        return "&#39;";
+      default:
+        return char;
+    }
+  });
+}

hivegrid-cloud/apps/api/src/routes/status.test.ts

@@ -0,0 +1,77 @@
+import { describe, it, expect } from "vitest";
+import { Hono } from "hono";
+import { buildApiStatusReport, handleApiStatusRequest } from "./status.js";
+
+function fixedDeps(overrides: Partial<Parameters<typeof handleApiStatusRequest>[1]> = {}) {
+  return {
+    checkDb: async () => undefined,
+    getRevision: () => "abc1234",
+    getRegion: () => "eu-central-1",
+    getUptimeSeconds: () => 12.4,
+    ...overrides,
+  };
+}
+
+describe("buildApiStatusReport", () => {
+  it("returns ok when all checks pass", async () => {
+    const report = await buildApiStatusReport(fixedDeps());
+    expect(report).toEqual({
+      status: "ok",
+      service: "api",
+      checks: { db: "ok" },
+      version: "abc1234",
+      uptime_seconds: 12,
+      region: "eu-central-1",
+    });
+  });
+
+  it("returns down when db check rejects", async () => {
+    const report = await buildApiStatusReport(
+      fixedDeps({ checkDb: () => Promise.reject(new Error("connection refused")) }),
+    );
+    expect(report.status).toBe("down");
+    expect(report.checks.db).toBe("down");
+  });
+
+  it("defaults missing revision and region", async () => {
+    const report = await buildApiStatusReport(
+      fixedDeps({ getRevision: () => "", getRegion: () => "" }),
+    );
+    expect(report.version).toBe("unknown");
+    expect(report.region).toBe("unknown");
+  });
+
+  it("never leaks connection-strings on db failure", async () => {
+    const report = await buildApiStatusReport(
+      fixedDeps({
+        checkDb: () => {
+          throw new Error("postgres://user:secret@db.internal/app failed");
+        },
+      }),
+    );
+    const serialised = JSON.stringify(report);
+    expect(serialised).not.toContain("secret");
+    expect(serialised).not.toContain("db.internal");
+  });
+});
+
+describe("handleApiStatusRequest", () => {
+  it("returns 200 when status is ok", async () => {
+    const app = new Hono();
+    app.get("/status", (c) => handleApiStatusRequest(c, fixedDeps()));
+    const res = await app.request("/status");
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { status: string; checks: { db: string } };
+    expect(body.status).toBe("ok");
+    expect(body.checks.db).toBe("ok");
+  });
+
+  it("returns 503 when status is down", async () => {
+    const app = new Hono();
+    app.get("/status", (c) =>
+      handleApiStatusRequest(c, fixedDeps({ checkDb: () => Promise.reject(new Error("x")) })),
+    );
+    const res = await app.request("/status");
+    expect(res.status).toBe(503);
+  });
+});

hivegrid-cloud/apps/api/src/routes/status.ts

@@ -0,0 +1,77 @@
+/**
+ * Public `/status` endpoint for uptime checks (Uptime Kuma, Statuspage, etc.).
+ *
+ * Returns booleans only — never connection strings, hostnames or
+ * stack-traces. Empty bearer-token auth so external monitors can poll it.
+ *
+ * Returns 200 when all critical checks (`db`) are ok. 503 when any critical
+ * check is down. The schema is stable across services so a single Statuspage
+ * component can scrape all three apps.
+ *
+ * @see docs/improvement-plan/02-observability/metrics-grafana.md
+ */
+import type { Context } from "hono";
+
+export type ApiStatusCheckOutcome = "ok" | "degraded" | "down" | "skipped";
+
+export type ApiStatusReport = {
+  status: "ok" | "degraded" | "down";
+  service: "api";
+  checks: {
+    db: ApiStatusCheckOutcome;
+  };
+  version: string;
+  uptime_seconds: number;
+  region: string;
+};
+
+export type ApiStatusDependencies = {
+  /**
+   * Cheap DB liveness probe. Should resolve when the DB is reachable; reject
+   * with any error otherwise.
+   */
+  checkDb: () => Promise<void> | void;
+  /**
+   * Returns the build revision (typically the git SHA) to embed in the
+   * response. Empty string when unknown — never `undefined`.
+   */
+  getRevision: () => string;
+  /**
+   * Returns the deployment region tag. Defaults to `"unknown"` when not set
+   * via env.
+   */
+  getRegion: () => string;
+  /**
+   * Returns the process uptime in seconds. Tests inject a deterministic
+   * counter; production uses `process.uptime()`.
+   */
+  getUptimeSeconds: () => number;
+};
+
+export async function buildApiStatusReport(deps: ApiStatusDependencies): Promise<ApiStatusReport> {
+  const dbStatus = await runCheck(deps.checkDb);
+
+  return {
+    status: dbStatus === "ok" ? "ok" : "down",
+    service: "api",
+    checks: { db: dbStatus },
+    version: deps.getRevision() || "unknown",
+    uptime_seconds: Math.round(deps.getUptimeSeconds()),
+    region: deps.getRegion() || "unknown",
+  };
+}
+
+export async function handleApiStatusRequest(c: Context, deps: ApiStatusDependencies): Promise<Response> {
+  const report = await buildApiStatusReport(deps);
+  const status = report.status === "ok" ? 200 : 503;
+  return c.json(report, status);
+}
+
+async function runCheck(check: () => Promise<void> | void): Promise<ApiStatusCheckOutcome> {
+  try {
+    await check();
+    return "ok";
+  } catch {
+    return "down";
+  }
+}

hivegrid-cloud/apps/api/src/routes/system-routes.ts

@@ -0,0 +1,89 @@
+import { Hono, type Context } from "hono";
+import { getApiAppDependencies } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { createOpenApiDocument } from "../openapi.js";
+import { escapeHtml } from "./shared.js";
+
+function apiHealthResponse(c: Context<ApiEnv>) {
+  return c.json({
+    status: "ok",
+    service: "api",
+    revision: getApiRuntimeConfig().revision,
+    requestId: c.get("requestId"),
+  });
+}
+
+export function registerSystemRoutes(app: Hono<ApiEnv>) {
+  app.get("/healthz", apiHealthResponse);
+  app.get("/v1/health", apiHealthResponse);
+
+  app.get("/readyz", async (c) => {
+    try {
+      await getApiAppDependencies().checkReadiness();
+      return c.json({
+        status: "ready",
+        service: "api",
+        revision: getApiRuntimeConfig().revision,
+        requestId: c.get("requestId"),
+      });
+    } catch {
+      return c.json(
+        {
+          status: "not_ready",
+          service: "api",
+          requestId: c.get("requestId"),
+          error: {
+            code: "dependency_unavailable",
+            message: "A required dependency is unavailable.",
+          },
+        },
+        503
+      );
+    }
+  });
+
+  app.get("/v1/openapi.json", (c) => {
+    return c.json(createOpenApiDocument(getApiRuntimeConfig()));
+  });
+
+  app.get("/v1/docs", (c) => {
+    const appName = getApiRuntimeConfig().appName;
+
+    return c.html(`<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>API Documentation | ${escapeHtml(appName)}</title>
+    <meta property="og:title" content="API Documentation | ${escapeHtml(appName)}">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style>
+      body { margin: 0; background: #0f172a; color: #e2e8f0; font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
+      header { border-bottom: 1px solid #334155; padding: 20px 24px; }
+      main { padding: 24px; }
+      a { color: #93c5fd; }
+      pre { margin: 0; overflow: auto; border: 1px solid #334155; border-radius: 8px; background: #020617; padding: 16px; line-height: 1.5; }
+    </style>
+  </head>
+  <body>
+    <header>
+      <h1>API Documentation</h1>
+      <a href="/v1/openapi.json">OpenAPI JSON</a>
+    </header>
+    <main>
+      <pre id="openapi">Loading /v1/openapi.json...</pre>
+    </main>
+    <script>
+      fetch("/v1/openapi.json")
+        .then((response) => response.json())
+        .then((spec) => {
+          document.getElementById("openapi").textContent = JSON.stringify(spec, null, 2);
+        })
+        .catch((error) => {
+          document.getElementById("openapi").textContent = error.message;
+        });
+    </script>
+  </body>
+</html>`);
+  });
+}

hivegrid-cloud/apps/api/src/routes/workspace-routes.ts

@@ -0,0 +1,245 @@
+import { Hono, type Context } from "hono";
+import { getApiAppDependencies } from "../app-dependencies.js";
+import type { ApiEnv } from "../api-types.js";
+import { apiError } from "../middleware/error-envelope.js";
+import {
+  parseWorkspaceContextRevalidationPayload,
+  parseWorkspaceInviteAcceptPayload,
+  parseWorkspaceInviteCreatePayload,
+  parseWorkspaceInviteRevokePayload,
+} from "../validation/request-schemas.js";
+import {
+  handleWorkspaceContextRevalidationError,
+  handleWorkspaceInviteError,
+  parseOptionalBooleanQuery,
+  parseWorkspaceAuthProviderKindQuery,
+  parseWorkspaceInviteStatusQuery,
+  parseWorkspaceTenantId,
+  readJsonBody,
+  requireWorkspaceInviteServiceCredential,
+  requireWorkspaceServiceCredential,
+  toWorkspaceAuthProviderApiResponse,
+  toWorkspaceInviteApiResponse,
+} from "./shared.js";
+
+export function registerWorkspaceRoutes(app: Hono<ApiEnv>) {
+  app.get("/v1/workspaces/:tenantId/auth/providers", getWorkspaceAuthProvidersRequest);
+  app.post("/v1/workspaces/:tenantId/context/revalidate", postWorkspaceContextRevalidateRequest);
+  app.get("/v1/workspaces/:tenantId/invites", getWorkspaceInvitesRequest);
+  app.post("/v1/workspaces/:tenantId/invites", postWorkspaceInviteRequest);
+  app.post("/v1/workspaces/:tenantId/invites/:inviteId/accept", postWorkspaceInviteAcceptRequest);
+  app.post("/v1/workspaces/:tenantId/invites/:inviteId/revoke", postWorkspaceInviteRevokeRequest);
+}
+
+async function getWorkspaceAuthProvidersRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireWorkspaceServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const tenantId = parseWorkspaceTenantId(c);
+  if (!tenantId) {
+    return apiError(c, 400, "invalid_request", "tenantId is required.");
+  }
+  const tenantGuard = await requireActiveWorkspaceTenant(c, tenantId);
+  if (tenantGuard) {
+    return tenantGuard;
+  }
+
+  try {
+    const providers = await getApiAppDependencies().listWorkspaceAuthProviders({
+      tenantId,
+      kind: parseWorkspaceAuthProviderKindQuery(c.req.query("kind")),
+      enabled: parseOptionalBooleanQuery(c.req.query("enabled")),
+    });
+
+    return c.json({ providers: providers.map(toWorkspaceAuthProviderApiResponse) });
+  } catch {
+    return apiError(c, 500, "workspace_auth_provider_failed", "Workspace auth provider request failed.");
+  }
+}
+
+async function requireActiveWorkspaceTenant(c: Context<ApiEnv>, tenantId: string) {
+  try {
+    const tenant = await getApiAppDependencies().findLegacyTenantById(tenantId);
+    if (!tenant) {
+      return apiError(c, 404, "tenant_not_found", "Workspace tenant was not found.");
+    }
+    if (!tenant.active) {
+      return apiError(c, 403, "tenant_inactive", "Workspace tenant is inactive.");
+    }
+    return null;
+  } catch {
+    return apiError(c, 500, "workspace_tenant_failed", "Workspace tenant validation failed.");
+  }
+}
+
+async function postWorkspaceContextRevalidateRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireWorkspaceServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const tenantId = parseWorkspaceTenantId(c);
+  if (!tenantId) {
+    return apiError(c, 400, "invalid_request", "tenantId is required.");
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseWorkspaceContextRevalidationPayload(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+  }
+
+  try {
+    const context = await getApiAppDependencies().revalidateWorkspaceContext({ tenantId, userId: payload.value.userId });
+    return c.json({
+      context: {
+        valid: context.valid,
+        tenantId: context.tenantId,
+        userId: context.userId,
+        ...(payload.value.providerSlug ? { providerSlug: payload.value.providerSlug } : {}),
+      },
+    });
+  } catch (error) {
+    return handleWorkspaceContextRevalidationError(c, error);
+  }
+}
+
+async function getWorkspaceInvitesRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireWorkspaceInviteServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const tenantId = parseWorkspaceTenantId(c);
+  if (!tenantId) {
+    return apiError(c, 400, "invalid_request", "tenantId is required.");
+  }
+  const tenantGuard = await requireActiveWorkspaceTenant(c, tenantId);
+  if (tenantGuard) {
+    return tenantGuard;
+  }
+
+  try {
+    const invites = await getApiAppDependencies().listWorkspaceInvites({
+      tenantId,
+      status: parseWorkspaceInviteStatusQuery(c.req.query("status")),
+      email: c.req.query("email"),
+    });
+
+    return c.json({ invites: invites.map(toWorkspaceInviteApiResponse) });
+  } catch {
+    return apiError(c, 500, "workspace_invite_failed", "Workspace invite request failed.");
+  }
+}
+
+async function postWorkspaceInviteRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireWorkspaceInviteServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const tenantId = parseWorkspaceTenantId(c);
+  if (!tenantId) {
+    return apiError(c, 400, "invalid_request", "tenantId is required.");
+  }
+  const tenantGuard = await requireActiveWorkspaceTenant(c, tenantId);
+  if (tenantGuard) {
+    return tenantGuard;
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseWorkspaceInviteCreatePayload(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+  }
+
+  try {
+    const invite = await getApiAppDependencies().createWorkspaceInvite({
+      tenantId,
+      email: payload.value.email,
+      firstName: payload.value.firstName,
+      lastName: payload.value.lastName,
+      type: payload.value.type,
+      expiresAt: payload.value.expiresAt ? new Date(payload.value.expiresAt) : null,
+    });
+
+    return c.json({ invite: toWorkspaceInviteApiResponse(invite), token: invite.token }, 201);
+  } catch (error) {
+    return handleWorkspaceInviteError(c, error);
+  }
+}
+
+async function postWorkspaceInviteAcceptRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireWorkspaceInviteServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const tenantId = parseWorkspaceTenantId(c);
+  if (!tenantId) {
+    return apiError(c, 400, "invalid_request", "tenantId is required.");
+  }
+  const tenantGuard = await requireActiveWorkspaceTenant(c, tenantId);
+  if (tenantGuard) {
+    return tenantGuard;
+  }
+  const inviteId = c.req.param("inviteId")?.trim();
+  if (!inviteId) {
+    return apiError(c, 400, "invalid_request", "inviteId is required.");
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseWorkspaceInviteAcceptPayload(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+  }
+
+  try {
+    const invite = await getApiAppDependencies().acceptWorkspaceInvite({
+      tenantId,
+      inviteId,
+      token: payload.value.token,
+      acceptedByUserId: payload.value.userId ?? "",
+    });
+
+    return c.json({ invite: toWorkspaceInviteApiResponse(invite) });
+  } catch (error) {
+    return handleWorkspaceInviteError(c, error);
+  }
+}
+
+async function postWorkspaceInviteRevokeRequest(c: Context<ApiEnv>) {
+  const unauthorized = requireWorkspaceInviteServiceCredential(c);
+  if (unauthorized) {
+    return unauthorized;
+  }
+
+  const tenantId = parseWorkspaceTenantId(c);
+  const inviteId = c.req.param("inviteId")?.trim();
+  if (!tenantId || !inviteId) {
+    return apiError(c, 400, "invalid_request", "tenantId and inviteId are required.");
+  }
+  const tenantGuard = await requireActiveWorkspaceTenant(c, tenantId);
+  if (tenantGuard) {
+    return tenantGuard;
+  }
+
+  const body = await readJsonBody(c.req.raw);
+  const payload = parseWorkspaceInviteRevokePayload(body);
+  if (!payload.ok) {
+    return apiError(c, 400, "invalid_request", payload.error.message, { params: payload.error.params });
+  }
+
+  try {
+    const invite = await getApiAppDependencies().revokeWorkspaceInvite({
+      tenantId,
+      inviteId,
+      revokedByUserId: payload.value.revokedByUserId ?? "",
+    });
+    return c.json({ invite: toWorkspaceInviteApiResponse(invite) });
+  } catch (error) {
+    return handleWorkspaceInviteError(c, error);
+  }
+}

hivegrid-cloud/apps/api/src/sentry.test.ts

@@ -0,0 +1,123 @@
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+  __getApiSentryConfigForTests,
+  __resetApiSentryForTests,
+  apiErrorReporter,
+  initApiSentry,
+  parseSampleRate,
+  readApiSentryConfig,
+} from "./sentry.js";
+
+afterEach(() => {
+  __resetApiSentryForTests();
+});
+
+describe("parseSampleRate", () => {
+  it("returns the fallback for undefined/null/empty values", () => {
+    expect(parseSampleRate(undefined, 0.5)).toBe(0.5);
+    expect(parseSampleRate("", 0.25)).toBe(0.25);
+  });
+
+  it("returns the fallback for non-numeric input", () => {
+    expect(parseSampleRate("abc", 0.3)).toBe(0.3);
+  });
+
+  it("clamps to [0, 1]", () => {
+    expect(parseSampleRate("-0.5", 0)).toBe(0);
+    expect(parseSampleRate("1.5", 0)).toBe(1);
+  });
+
+  it("returns parsed numbers in range", () => {
+    expect(parseSampleRate("0.42", 0)).toBeCloseTo(0.42);
+  });
+});
+
+describe("readApiSentryConfig", () => {
+  it("defaults to dev sample rate 1.0 when NODE_ENV is not production", () => {
+    const config = readApiSentryConfig({ NODE_ENV: "development" });
+    expect(config.tracesSampleRate).toBe(1);
+    expect(config.profilesSampleRate).toBe(0);
+  });
+
+  it("defaults to prod sample rate 0.1 when NODE_ENV=production", () => {
+    const config = readApiSentryConfig({ NODE_ENV: "production" });
+    expect(config.tracesSampleRate).toBe(0.1);
+  });
+
+  it("respects SENTRY_TRACES_SAMPLE_RATE override", () => {
+    const config = readApiSentryConfig({ NODE_ENV: "production", SENTRY_TRACES_SAMPLE_RATE: "0.5" });
+    expect(config.tracesSampleRate).toBe(0.5);
+  });
+
+  it("falls back environment to NODE_ENV when SENTRY_ENVIRONMENT is unset", () => {
+    expect(readApiSentryConfig({ NODE_ENV: "staging" }).environment).toBe("staging");
+  });
+
+  it("prefers SENTRY_ENVIRONMENT over NODE_ENV", () => {
+    expect(readApiSentryConfig({ NODE_ENV: "production", SENTRY_ENVIRONMENT: "canary" }).environment).toBe("canary");
+  });
+
+  it("treats empty DSN as undefined", () => {
+    expect(readApiSentryConfig({ SENTRY_DSN_API: "" }).dsn).toBeUndefined();
+  });
+
+  it("forwards the release tag", () => {
+    expect(readApiSentryConfig({ SENTRY_RELEASE: "abc123" }).release).toBe("abc123");
+  });
+});
+
+describe("initApiSentry", () => {
+  it("returns a no-op reporter when DSN is missing", () => {
+    const reporter = initApiSentry({
+      dsn: undefined,
+      environment: "test",
+      release: undefined,
+      tracesSampleRate: 0,
+      profilesSampleRate: 0,
+    });
+    expect(typeof reporter.report).toBe("function");
+    // A no-op reporter must not throw on any event:
+    expect(() => reporter.report({ component: "api", operation: "test" })).not.toThrow();
+    expect(__getApiSentryConfigForTests()?.dsn).toBeUndefined();
+  });
+
+  it("returns a sentry-backed reporter when DSN is set, with proper sample rates", () => {
+    const reporter = initApiSentry({
+      dsn: "https://example@sentry.invalid/1",
+      environment: "test",
+      release: "rev-x",
+      tracesSampleRate: 0.25,
+      profilesSampleRate: 0.05,
+    });
+    expect(typeof reporter.report).toBe("function");
+    expect(__getApiSentryConfigForTests()?.tracesSampleRate).toBe(0.25);
+    expect(__getApiSentryConfigForTests()?.profilesSampleRate).toBe(0.05);
+    expect(__getApiSentryConfigForTests()?.release).toBe("rev-x");
+  });
+
+  it("is idempotent - subsequent calls reuse the cached reporter", () => {
+    const first = initApiSentry({
+      dsn: "https://example@sentry.invalid/1",
+      environment: "test",
+      release: undefined,
+      tracesSampleRate: 0,
+      profilesSampleRate: 0,
+    });
+    const second = initApiSentry({
+      dsn: "https://different@sentry.invalid/2",
+      environment: "test",
+      release: undefined,
+      tracesSampleRate: 0,
+      profilesSampleRate: 0,
+    });
+    expect(second).toBe(first);
+  });
+});
+
+describe("apiErrorReporter facade", () => {
+  it("lazily initialises with default config when not initialised", () => {
+    expect(typeof apiErrorReporter.report).toBe("function");
+    expect(() => apiErrorReporter.report({ component: "api", operation: "test" })).not.toThrow();
+  });
+});

hivegrid-cloud/apps/api/src/sentry.ts

@@ -0,0 +1,105 @@
+/**
+ * Sentry bootstrap for the API process.
+ *
+ * Exposes {@link initApiSentry} (called from `server.ts` as the very first
+ * statement) and {@link apiErrorReporter} (an `ErrorReporter` facade that
+ * `createErrorHandler` and any other code-path can use to forward unhandled
+ * errors to Sentry).
+ *
+ * When `SENTRY_DSN_API` is empty or unset every export becomes a no-op so
+ * local development and staging environments without a Sentry instance keep
+ * working unchanged.
+ *
+ * @see docs/improvement-plan/02-observability/sentry-wiring-api-worker.md
+ */
+import { createNoopErrorReporter, createSentryErrorReporter, type ErrorReporter } from "@toolctrl/error-reporter";
+
+export interface ApiSentryConfig {
+  dsn: string | undefined;
+  environment: string | undefined;
+  release: string | undefined;
+  tracesSampleRate: number;
+  profilesSampleRate: number;
+}
+
+const DEFAULT_TRACES_SAMPLE_RATE_PROD = 0.1;
+const DEFAULT_TRACES_SAMPLE_RATE_DEV = 1.0;
+const DEFAULT_PROFILES_SAMPLE_RATE = 0.0;
+
+export function parseSampleRate(raw: string | undefined, fallback: number): number {
+  if (raw === undefined || raw === null || raw === "") return fallback;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed)) return fallback;
+  if (parsed < 0) return 0;
+  if (parsed > 1) return 1;
+  return parsed;
+}
+
+export function readApiSentryConfig(env: NodeJS.ProcessEnv = process.env): ApiSentryConfig {
+  const nodeEnv = env.NODE_ENV || "development";
+  const tracesDefault = nodeEnv === "production" ? DEFAULT_TRACES_SAMPLE_RATE_PROD : DEFAULT_TRACES_SAMPLE_RATE_DEV;
+  return {
+    dsn: env.SENTRY_DSN_API || undefined,
+    environment: env.SENTRY_ENVIRONMENT || nodeEnv || undefined,
+    release: env.SENTRY_RELEASE || undefined,
+    tracesSampleRate: parseSampleRate(env.SENTRY_TRACES_SAMPLE_RATE, tracesDefault),
+    profilesSampleRate: parseSampleRate(env.SENTRY_PROFILES_SAMPLE_RATE, DEFAULT_PROFILES_SAMPLE_RATE),
+  };
+}
+
+let cachedReporter: ErrorReporter | undefined;
+let cachedConfig: ApiSentryConfig | undefined;
+
+/**
+ * Idempotent. The first call builds the Sentry-backed reporter (or the
+ * no-op reporter when `SENTRY_DSN_API` is empty); subsequent calls return
+ * the cached instance. The Sentry SDK is loaded lazily through the adapter
+ * in `@toolctrl/error-reporter`, so unit tests that do not import this
+ * module never touch `@sentry/node` at all.
+ */
+export function initApiSentry(config: ApiSentryConfig = readApiSentryConfig()): ErrorReporter {
+  if (cachedReporter) {
+    return cachedReporter;
+  }
+
+  cachedConfig = config;
+
+  if (!config.dsn) {
+    cachedReporter = createNoopErrorReporter();
+    return cachedReporter;
+  }
+
+  cachedReporter = createSentryErrorReporter({
+    dsn: config.dsn,
+    environment: config.environment,
+    release: config.release,
+    tracesSampleRate: config.tracesSampleRate,
+    profilesSampleRate: config.profilesSampleRate,
+    maxBreadcrumbs: 50,
+  });
+
+  return cachedReporter;
+}
+
+/**
+ * Reporter facade. Lazy on first access so `apps/api/src/server.ts` controls
+ * when initialisation happens (and so unit tests that import `app.ts` without
+ * setting up env variables still get a no-op reporter).
+ */
+export const apiErrorReporter: ErrorReporter = {
+  report(event) {
+    const reporter = cachedReporter ?? initApiSentry();
+    return reporter.report(event);
+  },
+};
+
+/** Test-only: reset the cached reporter/config so multiple test cases can call init. */
+export function __resetApiSentryForTests(): void {
+  cachedReporter = undefined;
+  cachedConfig = undefined;
+}
+
+/** Test-only: read the cached config (or undefined if not initialised). */
+export function __getApiSentryConfigForTests(): ApiSentryConfig | undefined {
+  return cachedConfig;
+}

hivegrid-cloud/apps/api/src/server.ts

@@ -0,0 +1,22 @@
+// IMPORTANT: `./otel.js` MUST stay the very first import in this file.
+// The OpenTelemetry NodeSDK can only patch (http, hono, prisma, bullmq,
+// redis, ...) when it loads before those modules are required for the
+// first time. The module is a no-op when OTEL_EXPORTER_OTLP_ENDPOINT is
+// unset, so this contract is safe in every environment.
+import "./otel.js";
+
+import { serve } from "@hono/node-server";
+
+import { app } from "./app.js";
+import { getApiRuntimeConfig } from "./config.js";
+import { apiLogger } from "./logger.js";
+
+serve(
+  {
+    fetch: app.fetch,
+    port: getApiRuntimeConfig().listen.port,
+  },
+  (info) => {
+    apiLogger.info({ port: info.port }, "API listening");
+  },
+);

hivegrid-cloud/apps/api/src/shared-store.test.ts

@@ -0,0 +1,53 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+
+import { __resetSharedStoreForTests, __setSharedStoreFactoryForTests, getSharedStore } from "./shared-store.js";
+
+afterEach(() => {
+  __resetSharedStoreForTests();
+});
+
+describe("api shared-store singleton", () => {
+  test("memoizes the factory result", () => {
+    const factory = vi.fn(() => ({
+      namespace: "api",
+      ttlDefault: undefined,
+      adapter: {} as never,
+      key: (s: string) => s,
+      get: async () => null,
+      set: async () => true,
+      del: async () => 0,
+      incr: async () => 0,
+      expire: async () => true,
+      mget: async () => [],
+      scan: async function* () {},
+      publish: async () => 0,
+      subscribe: async () => async () => undefined,
+      zAdd: async () => 0,
+      zRemRangeByScore: async () => 0,
+      zCard: async () => 0,
+      eval: async () => null,
+      onReconnect: () => () => undefined,
+      close: async () => undefined,
+    }));
+    __setSharedStoreFactoryForTests(factory);
+
+    const first = getSharedStore();
+    const second = getSharedStore();
+    expect(first).toBe(second);
+    expect(factory).toHaveBeenCalledTimes(1);
+  });
+
+  test("returns a working memory-backed store when REDIS_URL is unset", async () => {
+    const originalRedisUrl = process.env.REDIS_URL;
+    delete process.env.REDIS_URL;
+    __resetSharedStoreForTests();
+    try {
+      const store = getSharedStore();
+      expect(store.namespace).toBe("api");
+      await store.set("ping", "pong");
+      expect(await store.get("ping")).toBe("pong");
+    } finally {
+      if (originalRedisUrl !== undefined) process.env.REDIS_URL = originalRedisUrl;
+    }
+  });
+});

hivegrid-cloud/apps/api/src/shared-store.ts

@@ -0,0 +1,33 @@
+import { createSharedStore, type SharedStore } from "@toolctrl/shared-store";
+
+/**
+ * Singleton SharedStore for the API process. All in-process Map-backed
+ * stores migrated as part of task #38 receive their `SharedStore` from
+ * this module so the underlying Redis connection (or memory fallback)
+ * is reused across the API lifetime.
+ */
+let cachedStore: SharedStore | null = null;
+let factory: () => SharedStore = () => createSharedStore({ namespace: "api", redisUrl: process.env.REDIS_URL });
+
+export function getSharedStore(): SharedStore {
+  if (cachedStore === null) {
+    cachedStore = factory();
+  }
+  return cachedStore;
+}
+
+/**
+ * Test seam: override the factory and clear the cached instance. Tests
+ * call this to inject a SharedStore that uses an in-memory adapter or a
+ * specific Redis test container.
+ */
+export function __setSharedStoreFactoryForTests(next: () => SharedStore): void {
+  factory = next;
+  cachedStore = null;
+}
+
+/** Test seam: reset to the default factory and drop the cached instance. */
+export function __resetSharedStoreForTests(): void {
+  factory = () => createSharedStore({ namespace: "api", redisUrl: process.env.REDIS_URL });
+  cachedStore = null;
+}

hivegrid-cloud/apps/api/src/storage.test.ts

@@ -0,0 +1,135 @@
+import { describe, expect, test, beforeEach } from "vitest";
+
+import { resetApiRuntimeConfigForTests, setApiRuntimeConfigForTests } from "./config.js";
+import { createStorageSignedDownloadUrl, resetStorageCachesForTests, storageCacheStatsForTests } from "./storage.js";
+
+const baseConfig = {
+  appName: "Test API",
+  nodeEnv: "test",
+  revision: "unknown",
+  listen: { port: 3001 },
+  routing: {
+    appBaseDomain: "localhost:3000",
+    webAppHost: "app.localhost:3000",
+    adminAppHost: "admin.localhost:3000",
+    apiHost: "api.localhost:3000",
+    protocol: "http",
+  },
+  origins: {},
+  apiAccessToken: "service-token",
+  jwtSecret: "jwt-secret",
+  analytics: {},
+  mailjet: {},
+  stripe: {},
+  apiKeys: {
+    allowLegacyPlaintextFallback: true,
+  },
+  storage: {
+    endpoint: "https://storage.example.com",
+    region: "us-east-1",
+    accessKeyId: "test-access-key",
+    secretAccessKey: "test-secret-key",
+    forcePathStyle: "true",
+  },
+} as const;
+
+describe("createStorageSignedDownloadUrl", () => {
+  test("creates a signed GET URL without returning bucket or object key fields", async () => {
+    setApiRuntimeConfigForTests(baseConfig);
+
+    try {
+      const result = await createStorageSignedDownloadUrl("exports", {
+        key: "tenants/tenant_1/exports/job_1.csv",
+        expiresInSeconds: 300,
+        responseContentType: "text/csv",
+        responseContentDisposition: 'attachment; filename="export.csv"',
+      });
+      const serialized = JSON.stringify(result);
+
+      expect(result.method).toBe("GET");
+      expect(result.expiresInSeconds).toBe(300);
+      expect(result.expiresAt).toBeInstanceOf(Date);
+      expect(result.headers).toEqual({});
+      expect(result).not.toHaveProperty("bucket");
+      expect(result).not.toHaveProperty("key");
+      expect(serialized).not.toContain("test-secret-key");
+      expect(result.url).toContain("https://storage.example.com/exports/tenants/tenant_1/exports/job_1.csv");
+      expect(result.url).toContain("X-Amz-Algorithm=AWS4-HMAC-SHA256");
+      expect(result.url).toContain("X-Amz-Signature=");
+      expect(result.url).toContain("response-content-type=text%2Fcsv");
+    } finally {
+      resetApiRuntimeConfigForTests();
+    }
+  });
+});
+
+describe("storage adapter + signer caching", () => {
+  beforeEach(() => {
+    resetStorageCachesForTests();
+  });
+
+  test("cache-hit: repeated calls with the same config reuse the signed-url adapter and signing key", async () => {
+    setApiRuntimeConfigForTests(baseConfig);
+
+    try {
+      await createStorageSignedDownloadUrl("exports", {
+        key: "a/b/1.csv",
+        expiresInSeconds: 300,
+      });
+      await createStorageSignedDownloadUrl("exports", {
+        key: "a/b/2.csv",
+        expiresInSeconds: 300,
+      });
+      await createStorageSignedDownloadUrl("exports", {
+        key: "a/b/3.csv",
+        expiresInSeconds: 300,
+      });
+
+      const stats = storageCacheStatsForTests();
+      expect(stats.signedUrlAdapters).toBe(1);
+      // All three calls happen within the same UTC date so the signing key is
+      // cached once.
+      expect(stats.signingKeys).toBe(1);
+    } finally {
+      resetApiRuntimeConfigForTests();
+    }
+  });
+
+  test("cache-miss: a different bucket produces a new signed-url adapter entry but reuses the per-credential signing key", async () => {
+    setApiRuntimeConfigForTests(baseConfig);
+
+    try {
+      await createStorageSignedDownloadUrl("exports", { key: "a.csv", expiresInSeconds: 300 });
+      await createStorageSignedDownloadUrl("imports", { key: "b.csv", expiresInSeconds: 300 });
+
+      const stats = storageCacheStatsForTests();
+      expect(stats.signedUrlAdapters).toBe(2);
+      expect(stats.signingKeys).toBe(1);
+    } finally {
+      resetApiRuntimeConfigForTests();
+    }
+  });
+
+  test("cache-miss: rotating the secret access key invalidates the cached adapter", async () => {
+    setApiRuntimeConfigForTests(baseConfig);
+    try {
+      await createStorageSignedDownloadUrl("exports", { key: "a.csv", expiresInSeconds: 300 });
+    } finally {
+      resetApiRuntimeConfigForTests();
+    }
+
+    setApiRuntimeConfigForTests({
+      ...baseConfig,
+      storage: { ...baseConfig.storage, secretAccessKey: "rotated-secret-key" },
+    });
+    try {
+      await createStorageSignedDownloadUrl("exports", { key: "a.csv", expiresInSeconds: 300 });
+
+      const stats = storageCacheStatsForTests();
+      expect(stats.signedUrlAdapters).toBe(2);
+      expect(stats.signingKeys).toBe(2);
+    } finally {
+      resetApiRuntimeConfigForTests();
+    }
+  });
+});

hivegrid-cloud/apps/api/src/storage.ts

@@ -0,0 +1,252 @@
+import { createHash, createHmac } from "node:crypto";
+
+import {
+  StorageAdapterError,
+  createS3SignedUrlAdapter,
+  createS3StorageAdapter,
+  type CreateSignedDownloadUrlInput,
+  type S3StorageAdapterConfig,
+  type SignedDownloadSignerInput,
+  type SignedDownloadUrlResult,
+  type StorageAdapter,
+  type UploadBody,
+} from "@toolctrl/storage";
+
+import { getApiRuntimeConfig } from "./config.js";
+
+/**
+ * Caches the per-config S3 storage adapter, signed-URL adapter and the per-day
+ * SigV4 signing key to avoid reconstructing them on every hot-path call.
+ *
+ * Cache key intentionally hashes the secret access key (never stores it raw)
+ * and includes endpoint/region/accessKeyId/forcePathStyle so a credential
+ * rotation transparently misses the cache and rebuilds the adapter.
+ */
+const storageAdapterCache = new Map<string, StorageAdapter>();
+const signedUrlAdapterCache = new Map<string, ReturnType<typeof createS3SignedUrlAdapter>>();
+const signingKeyCache = new Map<string, Buffer>();
+
+function storageAdapterCacheKey(config: S3StorageAdapterConfig): string {
+  const secretFingerprint = createHash("sha256").update(config.secretAccessKey ?? "").digest("hex");
+  return createHash("sha256")
+    .update(
+      JSON.stringify({
+        endpoint: config.endpoint,
+        region: config.region,
+        bucket: config.bucket,
+        accessKeyId: config.accessKeyId,
+        sessionToken: config.sessionToken ?? null,
+        publicBaseUrl: config.publicBaseUrl ?? null,
+        forcePathStyle: config.forcePathStyle ?? null,
+        secretFingerprint,
+      }),
+    )
+    .digest("hex");
+}
+
+function getCachedStorageAdapter(config: S3StorageAdapterConfig): StorageAdapter {
+  const key = storageAdapterCacheKey(config);
+  let adapter = storageAdapterCache.get(key);
+  if (!adapter) {
+    adapter = createS3StorageAdapter(config);
+    storageAdapterCache.set(key, adapter);
+  }
+  return adapter;
+}
+
+function getCachedSignedUrlAdapter(config: S3StorageAdapterConfig) {
+  const key = storageAdapterCacheKey(config);
+  let adapter = signedUrlAdapterCache.get(key);
+  if (!adapter) {
+    adapter = createS3SignedUrlAdapter({
+      bucket: config.bucket,
+      downloadSigner: (signerInput) => createS3SignedGetUrl(config, signerInput),
+      uploadSigner: () => {
+        throw new StorageAdapterError("signed_upload_failed", "Signed upload URLs are not supported by the API storage helper.", {
+          operation: "createSignedUploadUrl",
+          bucket: config.bucket,
+        });
+      },
+    });
+    signedUrlAdapterCache.set(key, adapter);
+  }
+  return adapter;
+}
+
+/** Test-only: drop cached adapters and signing keys. */
+export function resetStorageCachesForTests() {
+  storageAdapterCache.clear();
+  signedUrlAdapterCache.clear();
+  signingKeyCache.clear();
+}
+
+/** Test-only: inspect cache sizes for assertions. */
+export function storageCacheStatsForTests() {
+  return {
+    adapters: storageAdapterCache.size,
+    signedUrlAdapters: signedUrlAdapterCache.size,
+    signingKeys: signingKeyCache.size,
+  };
+}
+
+function createStorageAdapterConfig(bucket: string): S3StorageAdapterConfig {
+  const { storage } = getApiRuntimeConfig();
+
+  return {
+    endpoint: storage.endpoint ?? "",
+    region: storage.region ?? "",
+    bucket,
+    accessKeyId: storage.accessKeyId ?? "",
+    secretAccessKey: storage.secretAccessKey ?? "",
+    sessionToken: storage.sessionToken,
+    publicBaseUrl: storage.publicBaseUrl,
+    forcePathStyle: storage.forcePathStyle === undefined ? undefined : storage.forcePathStyle.toLowerCase() === "true",
+  };
+}
+
+export async function createStorageFile(
+  bucketId: string,
+  path: string,
+  file: UploadBody,
+  contentType?: string,
+): Promise<{
+  id: string;
+  publicUrl: string | null;
+}> {
+  const adapter = getCachedStorageAdapter(createStorageAdapterConfig(bucketId));
+  const createdFile = await adapter.putObject({
+    key: path,
+    body: file,
+    contentType,
+  });
+
+  return {
+    id: createdFile.key,
+    publicUrl: createdFile.publicUrl,
+  };
+}
+
+export type CreateStorageSignedDownloadUrlInput = CreateSignedDownloadUrlInput;
+export type StorageSignedDownloadUrlResult = Pick<SignedDownloadUrlResult, "url" | "method" | "expiresAt" | "expiresInSeconds" | "headers">;
+
+export async function createStorageSignedDownloadUrl(
+  bucketId: string,
+  input: CreateStorageSignedDownloadUrlInput,
+): Promise<StorageSignedDownloadUrlResult> {
+  const adapterConfig = createStorageAdapterConfig(bucketId);
+  const adapter = getCachedSignedUrlAdapter(adapterConfig);
+  const result = await adapter.createSignedDownloadUrl(input);
+
+  return {
+    url: result.url,
+    method: result.method,
+    expiresAt: result.expiresAt,
+    expiresInSeconds: result.expiresInSeconds,
+    headers: result.headers,
+  };
+}
+
+function createS3SignedGetUrl(config: S3StorageAdapterConfig, input: SignedDownloadSignerInput): string {
+  const missingFields = ["endpoint", "region", "bucket", "accessKeyId", "secretAccessKey"].filter((field) => !isPresent(config[field as keyof S3StorageAdapterConfig]));
+
+  if (missingFields.length > 0) {
+    throw new StorageAdapterError("config_missing", `Missing signed URL storage configuration: ${missingFields.join(", ")}`, {
+      operation: "createStorageSignedDownloadUrl",
+      bucket: isPresent(config.bucket) ? config.bucket : undefined,
+      key: input.key,
+      missingFields,
+    });
+  }
+
+  const endpoint = new URL(config.endpoint);
+  const bucketSegment = encodePathSegment(config.bucket);
+  const keyPath = input.key.split("/").map(encodePathSegment).join("/");
+  const pathname = config.forcePathStyle ? `${trimTrailingSlash(endpoint.pathname)}/${bucketSegment}/${keyPath}` : `${trimTrailingSlash(endpoint.pathname)}/${keyPath}`;
+  const host = config.forcePathStyle ? endpoint.host : `${config.bucket}.${endpoint.host}`;
+  const url = new URL(endpoint.toString());
+  url.host = host;
+  url.pathname = pathname;
+  url.search = "";
+  if (input.responseContentType) {
+    url.searchParams.set("response-content-type", input.responseContentType);
+  }
+  if (input.responseContentDisposition) {
+    url.searchParams.set("response-content-disposition", input.responseContentDisposition);
+  }
+
+  const now = new Date();
+  const amzDate = toAmzDate(now);
+  const dateStamp = amzDate.slice(0, 8);
+  const credentialScope = `${dateStamp}/${config.region}/s3/aws4_request`;
+  const signedHeaders = "host";
+  url.searchParams.set("X-Amz-Algorithm", "AWS4-HMAC-SHA256");
+  url.searchParams.set("X-Amz-Credential", `${config.accessKeyId}/${credentialScope}`);
+  url.searchParams.set("X-Amz-Date", amzDate);
+  url.searchParams.set("X-Amz-Expires", String(input.expiresInSeconds));
+  url.searchParams.set("X-Amz-SignedHeaders", signedHeaders);
+  if (config.sessionToken) {
+    url.searchParams.set("X-Amz-Security-Token", config.sessionToken);
+  }
+
+  const canonicalQuery = canonicalizeSearchParams(url.searchParams);
+  const canonicalRequest = ["GET", url.pathname, canonicalQuery, `host:${host}\n`, signedHeaders, "UNSIGNED-PAYLOAD"].join("\n");
+  const stringToSign = ["AWS4-HMAC-SHA256", amzDate, credentialScope, sha256Hex(canonicalRequest)].join("\n");
+  const signature = hmacHex(getSigningKey(config.secretAccessKey, dateStamp, config.region), stringToSign);
+  url.searchParams.set("X-Amz-Signature", signature);
+
+  return url.toString();
+}
+
+function canonicalizeSearchParams(params: URLSearchParams): string {
+  return [...params.entries()]
+    .sort(([leftKey, leftValue], [rightKey, rightValue]) => leftKey.localeCompare(rightKey) || leftValue.localeCompare(rightValue))
+    .map(([key, value]) => `${encodeRfc3986(key)}=${encodeRfc3986(value)}`)
+    .join("&");
+}
+
+function getSigningKey(secretAccessKey: string, dateStamp: string, region: string): Buffer {
+  const secretFingerprint = createHash("sha256").update(secretAccessKey).digest("hex");
+  const cacheKey = `${secretFingerprint}:${region}:${dateStamp}`;
+  const cached = signingKeyCache.get(cacheKey);
+  if (cached) {
+    return cached;
+  }
+  const dateKey = hmacBuffer(`AWS4${secretAccessKey}`, dateStamp);
+  const regionKey = hmacBuffer(dateKey, region);
+  const serviceKey = hmacBuffer(regionKey, "s3");
+  const signingKey = hmacBuffer(serviceKey, "aws4_request");
+  signingKeyCache.set(cacheKey, signingKey);
+  return signingKey;
+}
+
+function hmacBuffer(key: string | Buffer, value: string): Buffer {
+  return createHmac("sha256", key).update(value).digest();
+}
+
+function hmacHex(key: string | Buffer, value: string): string {
+  return createHmac("sha256", key).update(value).digest("hex");
+}
+
+function sha256Hex(value: string): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function toAmzDate(date: Date): string {
+  return date.toISOString().replace(/[:-]|\.\d{3}/g, "");
+}
+
+function encodePathSegment(segment: string): string {
+  return encodeRfc3986(segment);
+}
+
+function encodeRfc3986(value: string): string {
+  return encodeURIComponent(value).replace(/[!'()*]/g, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`);
+}
+
+function trimTrailingSlash(pathname: string): string {
+  return pathname.replace(/\/+$/, "");
+}
+
+function isPresent(value: unknown): value is string {
+  return typeof value === "string" && value.trim().length > 0;
+}

hivegrid-cloud/apps/api/src/usage/usage-route.test.ts

@@ -0,0 +1,296 @@
+import { Hono } from "hono";
+import type { ApiConfig } from "@toolctrl/config";
+import type { LegacyApiKeySimple } from "@toolctrl/db/api-keys.server";
+import { SUBSCRIPTION_FEATURE_LIMIT, type TenantApiUsageFeature } from "@toolctrl/db/usage.server";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+import { resetApiRuntimeConfigForTests, setApiRuntimeConfigForTests } from "../config.js";
+import type { ApiErrorEnv } from "../middleware/error-envelope.js";
+import { getUsageRequest } from "./usage-route.js";
+
+type UsageRouteDependencies = Parameters<typeof getUsageRequest>[1];
+
+const TEST_API_CONFIG: ApiConfig = {
+  appName: "Usage Route Test API",
+  nodeEnv: "test",
+  revision: "unknown",
+  listen: { port: 3001 },
+  routing: {
+    appBaseDomain: "localhost:3000",
+    webAppHost: "app.localhost:3000",
+    adminAppHost: "admin.localhost:3000",
+    apiHost: "api.localhost:3000",
+    protocol: "http",
+  },
+  origins: {},
+  apiAccessToken: "service-secret",
+  jwtSecret: "usage-route-secret",
+  analytics: {},
+  mailjet: {},
+  stripe: {},
+  apiKeys: {
+    allowLegacyPlaintextFallback: true,
+  },
+  storage: {},
+};
+
+beforeEach(() => {
+  setApiRuntimeConfigForTests(TEST_API_CONFIG);
+});
+
+afterEach(() => {
+  resetApiRuntimeConfigForTests();
+  vi.restoreAllMocks();
+});
+
+function createApiKeyFixture(patch: Partial<LegacyApiKeySimple> = {}): LegacyApiKeySimple {
+  const tenantId = patch.tenantId ?? patch.tenant?.id ?? "tenant_usage_1";
+
+  return {
+    id: "api_key_usage_1",
+    alias: "Primary usage key",
+    expires: null,
+    tenantId,
+    active: true,
+    entities: [],
+    tenant: { id: tenantId, name: "Acme Usage", slug: "acme-usage", deactivatedReason: null },
+    ...patch,
+  };
+}
+
+function createUsageFixture(patch: Partial<TenantApiUsageFeature> = {}): TenantApiUsageFeature {
+  return {
+    title: "Plan with {0} calls",
+    type: SUBSCRIPTION_FEATURE_LIMIT.monthly,
+    value: 100,
+    used: 10,
+    remaining: 90,
+    enabled: true,
+    message: "",
+    ...patch,
+  };
+}
+
+function buildDeps(options: { apiKey?: LegacyApiKeySimple | null; usage?: TenantApiUsageFeature; ipLoggingError?: Error } = {}) {
+  const apiKey = options.apiKey === undefined ? createApiKeyFixture() : options.apiKey;
+  const mocks = {
+    findLegacyApiKeyByKey: vi.fn(async () => apiKey),
+    createLegacyApiKeyLog: vi.fn(async (data: { tenantId?: string | null }) => ({ id: "api_key_log_usage_1", tenantId: data.tenantId ?? null })),
+    setLegacyApiKeyLogStatus: vi.fn(async () => ({ count: 1 })),
+    createUniqueLegacyTenantIpAddress: vi.fn(async () => {
+      if (options.ipLoggingError) {
+        throw options.ipLoggingError;
+      }
+      return { id: "tenant_ip_usage_1" };
+    }),
+    getTenantSubscription: vi.fn(async () => null),
+    getTenantApiUsage: vi.fn(async () => options.usage),
+  };
+  const dependencies: UsageRouteDependencies = {
+    findLegacyApiKeyByKey: mocks.findLegacyApiKeyByKey as never,
+    createLegacyApiKeyLog: mocks.createLegacyApiKeyLog as never,
+    setLegacyApiKeyLogStatus: mocks.setLegacyApiKeyLogStatus as never,
+    createUniqueLegacyTenantIpAddress: mocks.createUniqueLegacyTenantIpAddress as never,
+    getTenantSubscription: mocks.getTenantSubscription as never,
+    getTenantApiUsage: mocks.getTenantApiUsage as never,
+  };
+
+  return { dependencies, mocks };
+}
+
+function buildApp(dependencies: UsageRouteDependencies) {
+  const app = new Hono<ApiErrorEnv>();
+  app.use("*", async (c, next) => {
+    c.set("requestId", "req_usage_route");
+    await next();
+  });
+  app.get("/v1/usage", (c) => getUsageRequest(c, dependencies));
+  return app;
+}
+
+async function requestUsage(dependencies: UsageRouteDependencies, headers: Record<string, string> = {}) {
+  return buildApp(dependencies).request("/v1/usage?source=test", { headers });
+}
+
+describe("getUsageRequest", () => {
+  test("rejects missing API keys and records an unauthenticated usage log", async () => {
+    const { dependencies, mocks } = buildDeps();
+
+    const response = await requestUsage(dependencies);
+
+    expect(response.status).toBe(401);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "unauthorized",
+        message: "Valid API key credentials are required.",
+        requestId: "req_usage_route",
+      },
+    });
+    expect(mocks.findLegacyApiKeyByKey).not.toHaveBeenCalled();
+    expect(mocks.createLegacyApiKeyLog).toHaveBeenCalledWith({
+      tenantId: null,
+      apiKeyId: null,
+      endpoint: "/v1/usage",
+      ip: "",
+      method: "GET",
+      params: "source=test",
+      error: "Invalid API Key",
+      status: 401,
+    });
+    expect(mocks.setLegacyApiKeyLogStatus).not.toHaveBeenCalled();
+  });
+
+  test("rejects invalid API keys after lookup and records an unauthenticated usage log", async () => {
+    const { dependencies, mocks } = buildDeps({ apiKey: null });
+
+    const response = await requestUsage(dependencies, { "x-api-key": "invalid-secret" });
+
+    expect(response.status).toBe(401);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "unauthorized",
+        message: "Valid API key credentials are required.",
+        requestId: "req_usage_route",
+      },
+    });
+    expect(mocks.findLegacyApiKeyByKey).toHaveBeenCalledWith("invalid-secret", undefined, "usage-route-secret", { allowPlaintextFallback: true });
+    expect(mocks.createLegacyApiKeyLog).toHaveBeenCalledWith(
+      expect.objectContaining({
+        tenantId: null,
+        apiKeyId: null,
+        endpoint: "/v1/usage",
+        error: "Invalid API Key",
+        status: 401,
+      }),
+    );
+    expect(mocks.setLegacyApiKeyLogStatus).not.toHaveBeenCalled();
+  });
+
+  test("blocks inactive tenants before usage checks", async () => {
+    const apiKey = createApiKeyFixture({
+      tenant: { id: "tenant_usage_1", name: "Acme Usage", slug: "acme-usage", deactivatedReason: "billing_past_due" },
+    });
+    const { dependencies, mocks } = buildDeps({ apiKey });
+
+    const response = await requestUsage(dependencies, { "x-api-key": "tenant-secret" });
+
+    expect(response.status).toBe(403);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "tenant_inactive",
+        message: "Account is inactive.",
+        requestId: "req_usage_route",
+      },
+    });
+    expect(mocks.setLegacyApiKeyLogStatus).toHaveBeenCalledWith(
+      { id: "api_key_log_usage_1", tenantId: "tenant_usage_1" },
+      expect.objectContaining({ status: 403, error: "Account is inactive." }),
+    );
+    expect(mocks.createUniqueLegacyTenantIpAddress).not.toHaveBeenCalled();
+    expect(mocks.getTenantApiUsage).not.toHaveBeenCalled();
+  });
+
+  test("blocks inactive API keys before usage checks", async () => {
+    const { dependencies, mocks } = buildDeps({ apiKey: createApiKeyFixture({ active: false }) });
+
+    const response = await requestUsage(dependencies, { "x-api-key": "inactive-secret" });
+
+    expect(response.status).toBe(400);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "api_key_inactive",
+        message: "API key is inactive.",
+        requestId: "req_usage_route",
+      },
+    });
+    expect(mocks.setLegacyApiKeyLogStatus).toHaveBeenCalledWith(
+      { id: "api_key_log_usage_1", tenantId: "tenant_usage_1" },
+      expect.objectContaining({ status: 400, error: "API key is inactive." }),
+    );
+    expect(mocks.createUniqueLegacyTenantIpAddress).not.toHaveBeenCalled();
+    expect(mocks.getTenantApiUsage).not.toHaveBeenCalled();
+  });
+
+  test("blocks expired API keys before usage checks", async () => {
+    const { dependencies, mocks } = buildDeps({ apiKey: createApiKeyFixture({ expires: new Date("2020-01-01T00:00:00.000Z") }) });
+
+    const response = await requestUsage(dependencies, { "x-api-key": "expired-secret" });
+
+    expect(response.status).toBe(400);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "api_key_expired",
+        message: "API key is expired.",
+        requestId: "req_usage_route",
+      },
+    });
+    expect(mocks.setLegacyApiKeyLogStatus).toHaveBeenCalledWith(
+      { id: "api_key_log_usage_1", tenantId: "tenant_usage_1" },
+      expect.objectContaining({ status: 400, error: "API key is expired." }),
+    );
+    expect(mocks.createUniqueLegacyTenantIpAddress).not.toHaveBeenCalled();
+    expect(mocks.getTenantApiUsage).not.toHaveBeenCalled();
+  });
+
+  test("swallows IP logging failures and still returns usage", async () => {
+    const { dependencies, mocks } = buildDeps({ ipLoggingError: new Error("ip write failed") });
+
+    const response = await requestUsage(dependencies, { "x-api-key": "valid-secret", "x-forwarded-for": "203.0.113.10, 10.0.0.1" });
+
+    expect(response.status).toBe(200);
+    await expect(response.json()).resolves.toEqual({ plan: "" });
+    expect(mocks.createUniqueLegacyTenantIpAddress).toHaveBeenCalledWith({
+      ip: "203.0.113.10",
+      fromUrl: "/v1/usage",
+      tenantId: "tenant_usage_1",
+      apiKeyId: "api_key_usage_1",
+    });
+    expect(mocks.getTenantSubscription).toHaveBeenCalledWith("tenant_usage_1");
+    expect(mocks.getTenantApiUsage).toHaveBeenCalledWith("tenant_usage_1", null);
+    expect(mocks.setLegacyApiKeyLogStatus).toHaveBeenCalledWith(
+      { id: "api_key_log_usage_1", tenantId: "tenant_usage_1" },
+      expect.objectContaining({ status: 200 }),
+    );
+  });
+
+  test("returns payment required when API usage is disabled or exhausted", async () => {
+    const usage = createUsageFixture({ enabled: false, remaining: 0, message: "Usage exhausted." });
+    const { dependencies, mocks } = buildDeps({ usage });
+
+    const response = await requestUsage(dependencies, { "x-api-key": "valid-secret" });
+
+    expect(response.status).toBe(402);
+    await expect(response.json()).resolves.toEqual({
+      error: {
+        code: "payment_required",
+        message: "API usage is disabled or exhausted.",
+        requestId: "req_usage_route",
+      },
+    });
+    expect(mocks.setLegacyApiKeyLogStatus).toHaveBeenCalledWith(
+      { id: "api_key_log_usage_1", tenantId: "tenant_usage_1" },
+      expect.objectContaining({ status: 402, error: "API usage is disabled or exhausted." }),
+    );
+  });
+
+  test("omits remaining for included-plan usage", async () => {
+    const usage = createUsageFixture({
+      title: "Included plan with {0} calls",
+      type: SUBSCRIPTION_FEATURE_LIMIT.included,
+      value: 25,
+      remaining: 25,
+    });
+    const { dependencies, mocks } = buildDeps({ usage });
+
+    const response = await requestUsage(dependencies, { "x-api-key": "valid-secret" });
+    const body = await response.json();
+
+    expect(response.status).toBe(200);
+    expect(body).toEqual({ plan: "Included plan with 25 calls" });
+    expect(body).not.toHaveProperty("remaining");
+    expect(mocks.setLegacyApiKeyLogStatus).toHaveBeenCalledWith(
+      { id: "api_key_log_usage_1", tenantId: "tenant_usage_1" },
+      expect.objectContaining({ status: 200 }),
+    );
+  });
+});

hivegrid-cloud/apps/api/src/usage/usage-route.ts

@@ -0,0 +1,110 @@
+import { formatTenantApiUsagePlanTitle, SUBSCRIPTION_FEATURE_LIMIT } from "@toolctrl/db/usage.server";
+import type { Context } from "hono";
+
+import { extractApiKeyFromRequest } from "../auth.js";
+import { getApiRuntimeConfig } from "../config.js";
+import { apiError, type ApiErrorEnv } from "../middleware/error-envelope.js";
+import type { ApiAppDependencies } from "../app.js";
+
+type UsageRouteDependencies = Pick<
+  ApiAppDependencies,
+  | "findLegacyApiKeyByKey"
+  | "createLegacyApiKeyLog"
+  | "setLegacyApiKeyLogStatus"
+  | "createUniqueLegacyTenantIpAddress"
+  | "getTenantSubscription"
+  | "getTenantApiUsage"
+>;
+
+export async function getUsageRequest(c: Context<ApiErrorEnv>, dependencies: UsageRouteDependencies) {
+  const startTime = performance.now();
+  const endpoint = new URL(c.req.url).pathname;
+  const config = getApiRuntimeConfig();
+  const apiKeyValue = extractApiKeyFromRequest(c.req.raw, undefined);
+  if (!apiKeyValue) {
+    await createApiUsageErrorLog(c, dependencies, endpoint, "Invalid API Key", 401);
+    return apiError(c, 401, "unauthorized", "Valid API key credentials are required.");
+  }
+
+  const apiKey = await dependencies.findLegacyApiKeyByKey(apiKeyValue, undefined as never, config.jwtSecret, {
+    allowPlaintextFallback: config.apiKeys.allowLegacyPlaintextFallback,
+  });
+  if (!apiKey) {
+    await createApiUsageErrorLog(c, dependencies, endpoint, "Invalid API Key", 401);
+    return apiError(c, 401, "unauthorized", "Valid API key credentials are required.");
+  }
+
+  const apiKeyLog = await dependencies.createLegacyApiKeyLog({
+    tenantId: apiKey.tenantId,
+    apiKeyId: apiKey.id,
+    endpoint,
+    ip: getClientIpAddress(c.req.raw) ?? "",
+    method: c.req.method,
+    params: serializeRequestSearchParams(c.req.raw),
+    alias: apiKey.alias,
+    tenantName: apiKey.tenant.name,
+  });
+
+  const fail = async (status: 400 | 402 | 403, code: string, message: string) => {
+    await dependencies.setLegacyApiKeyLogStatus({ id: apiKeyLog.id, tenantId: apiKey.tenantId }, { status, error: message, startTime });
+    return apiError(c, status, code, message);
+  };
+
+  if (apiKey.tenant.deactivatedReason) {
+    return fail(403, "tenant_inactive", "Account is inactive.");
+  }
+  if (!apiKey.active) {
+    return fail(400, "api_key_inactive", "API key is inactive.");
+  }
+  if (apiKey.expires && apiKey.expires < new Date()) {
+    return fail(400, "api_key_expired", "API key is expired.");
+  }
+
+  await dependencies
+    .createUniqueLegacyTenantIpAddress({
+      ip: getClientIpAddress(c.req.raw) ?? "Unknown",
+      fromUrl: endpoint,
+      tenantId: apiKey.tenantId,
+      apiKeyId: apiKey.id,
+    })
+    .catch(() => undefined);
+
+  const tenantSubscription = await dependencies.getTenantSubscription(apiKey.tenantId);
+  const usage = await dependencies.getTenantApiUsage(apiKey.tenantId, tenantSubscription);
+  if (usage && !usage.enabled) {
+    return fail(402, "payment_required", "API usage is disabled or exhausted.");
+  }
+
+  await dependencies.setLegacyApiKeyLogStatus({ id: apiKeyLog.id, tenantId: apiKey.tenantId }, { status: 200, startTime });
+  return c.json({
+    plan: usage ? formatTenantApiUsagePlanTitle(usage) : "",
+    remaining: usage?.type === SUBSCRIPTION_FEATURE_LIMIT.included ? undefined : usage?.remaining,
+  });
+}
+
+async function createApiUsageErrorLog(
+  c: Context<ApiErrorEnv>,
+  dependencies: Pick<UsageRouteDependencies, "createLegacyApiKeyLog">,
+  endpoint: string,
+  error: string,
+  status: number,
+) {
+  await dependencies.createLegacyApiKeyLog({
+    tenantId: null,
+    apiKeyId: null,
+    endpoint,
+    ip: getClientIpAddress(c.req.raw) ?? "",
+    method: c.req.method,
+    params: serializeRequestSearchParams(c.req.raw),
+    error,
+    status,
+  });
+}
+
+function getClientIpAddress(request: Request) {
+  return request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() || request.headers.get("x-real-ip") || "";
+}
+
+function serializeRequestSearchParams(request: Request) {
+  return new URL(request.url).searchParams.toString();
+}

hivegrid-cloud/apps/api/src/validation/request-schemas.test.ts

@@ -0,0 +1,188 @@
+import { describe, expect, test } from "vitest";
+import {
+  parseAnalyticsEventPayload,
+  parseAnalyticsPageViewPayload,
+  parseApiKeyValidationPayload,
+  parseEntityRowObjectBody,
+  parseLoginCredentialsPayload,
+  parseRelationshipCreatePayload,
+  parseWorkspaceContextRevalidationPayload,
+  parseWorkspaceInviteAcceptPayload,
+  parseWorkspaceInviteCreatePayload,
+  parseWorkspaceInviteRevokePayload,
+} from "./request-schemas.js";
+
+const secretValue = "sk_live_secret_api_key_should_not_leak_value";
+
+function expectValidationError(result: { ok: boolean; error?: { message: string; params: Record<string, unknown> } }, params: Record<string, unknown>) {
+  expect(result.ok).toBe(false);
+  expect(result.error).toMatchObject({
+    message: expect.any(String),
+    params,
+  });
+  expect(JSON.stringify(result.error)).not.toContain(secretValue);
+}
+
+function expectErrorWithoutSecret(result: { ok: boolean; error?: { message: string; params: Record<string, unknown> } }) {
+  expect(result.ok).toBe(false);
+  expect(result.error?.message).toBeTypeOf("string");
+  expect(JSON.stringify(result.error)).not.toContain(secretValue);
+}
+
+describe("request schema validators", () => {
+  test("accepts valid login credentials", () => {
+    expect(parseLoginCredentialsPayload({ email: "user@example.com", password: "correct horse" })).toEqual({
+      ok: true,
+      value: { email: "user@example.com", password: "correct horse" },
+    });
+  });
+
+  test("rejects malformed login credentials with field-level redacted messages", () => {
+    expectValidationError(parseLoginCredentialsPayload(null), { reason: "body_invalid", bodyType: "login_credentials" });
+    expectValidationError(parseLoginCredentialsPayload([]), { reason: "body_invalid", bodyType: "login_credentials" });
+    expectValidationError(parseLoginCredentialsPayload({ password: "present" }), { reason: "field_required", field: "email" });
+    expectValidationError(parseLoginCredentialsPayload({ email: 123, password: secretValue }), { reason: "field_type_invalid", field: "email", expected: "string" });
+  });
+
+  test("accepts API-key validation bodies without leaking invalid secret-like values", () => {
+    expect(parseApiKeyValidationPayload({ apiKey: "valid-key" })).toEqual({ ok: true, value: { apiKey: "valid-key" } });
+    expect(parseApiKeyValidationPayload({})).toEqual({ ok: true, value: {} });
+
+    expectErrorWithoutSecret(parseApiKeyValidationPayload({ apiKey: { plaintext: secretValue } }));
+    expectValidationError(parseApiKeyValidationPayload(null), { reason: "body_invalid", bodyType: "api_key_validation" });
+  });
+
+  test("accepts valid analytics events with optional string fields", () => {
+    expect(
+      parseAnalyticsEventPayload({
+        action: "clicked",
+        category: "featureFlag",
+        label: "cta",
+        value: "1",
+        url: "https://example.test/page",
+        route: "/page",
+        analytics: {
+          userAnalyticsId: "visitor-1",
+          portalId: "portal-1",
+          portalUserId: "portal-user-1",
+          referrer: "https://referrer.test",
+        },
+      }),
+    ).toEqual({
+      ok: true,
+      value: {
+        action: "clicked",
+        category: "featureFlag",
+        label: "cta",
+        value: "1",
+        url: "https://example.test/page",
+        route: "/page",
+        analytics: {
+          userAnalyticsId: "visitor-1",
+          portalId: "portal-1",
+          portalUserId: "portal-user-1",
+          referrer: "https://referrer.test",
+        },
+      },
+    });
+  });
+
+  test("rejects malformed analytics event bodies and analytics fields without echoing values", () => {
+    expectValidationError(parseAnalyticsEventPayload([]), { reason: "body_invalid", bodyType: "analytics_event" });
+    expectValidationError(parseAnalyticsEventPayload({ category: "featureFlag", analytics: { userAnalyticsId: "visitor-1" } }), { reason: "field_required", field: "action" });
+    expectValidationError(parseAnalyticsEventPayload({ action: "clicked", category: 42, analytics: { userAnalyticsId: "visitor-1" } }), { reason: "field_type_invalid", field: "category", expected: "string" });
+    expectValidationError(parseAnalyticsEventPayload({ action: "clicked", category: "featureFlag", analytics: null }), { reason: "field_type_invalid", field: "analytics", expected: "object" });
+    expectErrorWithoutSecret(parseAnalyticsEventPayload({ action: "clicked", category: "featureFlag", analytics: { userAnalyticsId: { token: secretValue } } }));
+  });
+
+  test("accepts and rejects analytics page-view payloads", () => {
+    expect(parseAnalyticsPageViewPayload({ url: "https://example.test", analytics: { userAnalyticsId: "visitor-1" } })).toEqual({
+      ok: true,
+      value: { url: "https://example.test", analytics: { userAnalyticsId: "visitor-1" } },
+    });
+    expect(parseAnalyticsPageViewPayload({ url: "https://example.test", route: "/home", analytics: { userAnalyticsId: "visitor-1", portalId: null } })).toEqual({
+      ok: true,
+      value: { url: "https://example.test", route: "/home", analytics: { userAnalyticsId: "visitor-1", portalId: null } },
+    });
+    expectValidationError(parseAnalyticsPageViewPayload(null), { reason: "body_invalid", bodyType: "analytics_page_view" });
+    expectValidationError(parseAnalyticsPageViewPayload({ url: 123, analytics: { userAnalyticsId: "visitor-1" } }), { reason: "field_type_invalid", field: "url", expected: "string" });
+    expectErrorWithoutSecret(parseAnalyticsPageViewPayload({ url: "https://example.test", analytics: { userAnalyticsId: secretValue, portalId: 123 } }));
+  });
+
+  test("accepts relationship create payloads and rejects missing or non-string row ids", () => {
+    expect(parseRelationshipCreatePayload({ parent: "row-parent", child: "row-child" })).toEqual({
+      ok: true,
+      value: { parent: "row-parent", child: "row-child" },
+    });
+    expectValidationError(parseRelationshipCreatePayload(null), { reason: "body_invalid", bodyType: "relationship" });
+    expectValidationError(parseRelationshipCreatePayload({ child: "row-child" }), { reason: "field_required", field: "parent" });
+    expectValidationError(parseRelationshipCreatePayload({ parent: secretValue, child: 7 }), { reason: "field_type_invalid", field: "child", expected: "string" });
+  });
+
+  test("entity row bodies only require non-array JSON objects and preserve arbitrary data", () => {
+    const row = { name: "Widget", count: 3, nested: { enabled: true }, tags: ["a", "b"], secret: secretValue };
+
+    expect(parseEntityRowObjectBody(row)).toEqual({ ok: true, value: row });
+    expectValidationError(parseEntityRowObjectBody(null), { reason: "body_invalid", bodyType: "entity_row" });
+    expectValidationError(parseEntityRowObjectBody([]), { reason: "body_invalid", bodyType: "entity_row" });
+    expectValidationError(parseEntityRowObjectBody("not object"), { reason: "body_invalid", bodyType: "entity_row" });
+  });
+
+  test("accepts valid workspace invite create payloads", () => {
+    expect(
+      parseWorkspaceInviteCreatePayload({
+        email: "MEMBER@example.com",
+        firstName: "Ada",
+        lastName: "Lovelace",
+        type: 2,
+        expiresAt: "2026-06-01T12:00:00.000Z",
+      }),
+    ).toEqual({
+      ok: true,
+      value: {
+        email: "MEMBER@example.com",
+        firstName: "Ada",
+        lastName: "Lovelace",
+        type: 2,
+        expiresAt: "2026-06-01T12:00:00.000Z",
+      },
+    });
+  });
+
+  test("rejects malformed workspace invite create payloads without echoing values", () => {
+    expectValidationError(parseWorkspaceInviteCreatePayload(null), { reason: "body_invalid", bodyType: "workspace_invite_create" });
+    expectValidationError(parseWorkspaceInviteCreatePayload({ firstName: "Ada", lastName: "Lovelace", type: 2 }), { reason: "field_required", field: "email" });
+    expectValidationError(parseWorkspaceInviteCreatePayload({ email: 7, firstName: "Ada", lastName: "Lovelace", type: 2 }), { reason: "field_type_invalid", field: "email", expected: "string" });
+    expectValidationError(parseWorkspaceInviteCreatePayload({ email: "member@example.com", firstName: secretValue, lastName: "Lovelace", type: "2" }), { reason: "field_type_invalid", field: "type", expected: "number" });
+    expectValidationError(parseWorkspaceInviteCreatePayload({ email: "member@example.com", firstName: "Ada", lastName: "Lovelace", type: 2, expiresAt: "tomorrow" }), { reason: "field_value_invalid", field: "expiresAt", expected: "datetime" });
+    expectErrorWithoutSecret(parseWorkspaceInviteCreatePayload({ email: "member@example.com", firstName: secretValue, lastName: 99, type: 2 }));
+  });
+
+  test("accepts and rejects workspace invite accept payloads with redacted token errors", () => {
+    expect(parseWorkspaceInviteAcceptPayload({ token: secretValue, userId: "user_1" })).toEqual({ ok: true, value: { token: secretValue, userId: "user_1" } });
+    expectValidationError(parseWorkspaceInviteAcceptPayload(null), { reason: "body_invalid", bodyType: "workspace_invite_accept" });
+    expectValidationError(parseWorkspaceInviteAcceptPayload({ userId: "user_1" }), { reason: "field_required", field: "token" });
+    expectValidationError(parseWorkspaceInviteAcceptPayload({ token: { plaintext: secretValue }, userId: "user_1" }), { reason: "field_type_invalid", field: "token", expected: "string" });
+    expectErrorWithoutSecret(parseWorkspaceInviteAcceptPayload({ token: secretValue, userId: 123 }));
+  });
+
+  test("accepts and rejects workspace invite revoke payloads", () => {
+    expect(parseWorkspaceInviteRevokePayload({ reason: "duplicate", revokedByUserId: "admin_1" })).toEqual({
+      ok: true,
+      value: { reason: "duplicate", revokedByUserId: "admin_1" },
+    });
+    expect(parseWorkspaceInviteRevokePayload({})).toEqual({ ok: true, value: {} });
+    expectValidationError(parseWorkspaceInviteRevokePayload([]), { reason: "body_invalid", bodyType: "workspace_invite_revoke" });
+    expectValidationError(parseWorkspaceInviteRevokePayload({ reason: 404, revokedByUserId: secretValue }), { reason: "field_type_invalid", field: "reason", expected: "string" });
+    expectErrorWithoutSecret(parseWorkspaceInviteRevokePayload({ reason: "duplicate", revokedByUserId: { token: secretValue } }));
+  });
+
+  test("accepts and rejects workspace context revalidation payloads without echoing secret-like values", () => {
+    expect(parseWorkspaceContextRevalidationPayload({ userId: "user_1" })).toEqual({ ok: true, value: { userId: "user_1" } });
+    expect(parseWorkspaceContextRevalidationPayload({ userId: "user_1", providerSlug: "okta" })).toEqual({ ok: true, value: { userId: "user_1", providerSlug: "okta" } });
+    expectValidationError(parseWorkspaceContextRevalidationPayload(null), { reason: "body_invalid", bodyType: "workspace_context_revalidation" });
+    expectValidationError(parseWorkspaceContextRevalidationPayload({ providerSlug: "okta" }), { reason: "field_required", field: "userId" });
+    expectValidationError(parseWorkspaceContextRevalidationPayload({ userId: 123, providerSlug: secretValue }), { reason: "field_type_invalid", field: "userId", expected: "string" });
+    expectErrorWithoutSecret(parseWorkspaceContextRevalidationPayload({ userId: "user_1", providerSlug: { secret: secretValue } }));
+  });
+});

hivegrid-cloud/apps/api/src/validation/request-schemas.ts

@@ -0,0 +1,265 @@
+import { z } from "zod";
+import type { ApiErrorParams } from "../middleware/error-envelope.js";
+
+export type ValidationErrorReason = "body_invalid" | "field_required" | "field_type_invalid" | "field_value_invalid";
+
+export type RequestValidationError = {
+  message: string;
+  params: ApiErrorParams;
+};
+
+export type ParseResult<T> = { ok: true; value: T } | { ok: false; error: RequestValidationError };
+
+export type LoginCredentialsPayload = {
+  email: string;
+  password: string;
+};
+
+export type ApiKeyValidationPayload = {
+  apiKey?: string;
+};
+
+export type AnalyticsInfoPayload = {
+  userAnalyticsId: string;
+  portalId?: string | null;
+  portalUserId?: string | null;
+  [key: string]: unknown;
+};
+
+export type AnalyticsEventPayload = {
+  action: string;
+  category: string;
+  label?: string;
+  value?: string;
+  url?: string;
+  route?: string;
+  analytics: AnalyticsInfoPayload;
+};
+
+export type AnalyticsPageViewPayload = {
+  url: string;
+  route?: string;
+  analytics: AnalyticsInfoPayload;
+};
+
+export type RelationshipCreatePayload = {
+  parent: string;
+  child: string;
+};
+
+export type WorkspaceInviteCreatePayload = {
+  email: string;
+  firstName: string;
+  lastName: string;
+  type: number;
+  expiresAt?: string;
+};
+
+export type WorkspaceInviteAcceptPayload = {
+  token: string;
+  userId?: string;
+};
+
+export type WorkspaceInviteRevokePayload = {
+  reason?: string;
+  revokedByUserId?: string;
+};
+
+export type WorkspaceContextRevalidationPayload = {
+  userId: string;
+  providerSlug?: string;
+};
+
+const jsonObjectSchema = z.object({}).passthrough();
+const requiredString = z.string().min(1);
+const optionalString = z.string().optional();
+const optionalNullableString = z.string().nullable().optional();
+const optionalDateTimeString = z.string().datetime().optional();
+
+const analyticsInfoSchema = z
+  .object({
+    userAnalyticsId: requiredString,
+    portalId: optionalNullableString,
+    portalUserId: optionalNullableString,
+  })
+  .passthrough();
+
+const loginCredentialsSchema = z
+  .object({
+    email: requiredString,
+    password: requiredString,
+  })
+  .passthrough();
+
+const apiKeyValidationSchema = z
+  .object({
+    apiKey: optionalString,
+  })
+  .passthrough();
+
+const analyticsEventSchema = z
+  .object({
+    action: requiredString,
+    category: requiredString,
+    label: optionalString,
+    value: optionalString,
+    url: optionalString,
+    route: optionalString,
+    analytics: analyticsInfoSchema,
+  })
+  .passthrough();
+
+const analyticsPageViewSchema = z
+  .object({
+    url: requiredString,
+    route: optionalString,
+    analytics: analyticsInfoSchema,
+  })
+  .passthrough();
+
+const relationshipCreateSchema = z
+  .object({
+    parent: requiredString,
+    child: requiredString,
+  })
+  .passthrough();
+
+const workspaceInviteCreateSchema = z
+  .object({
+    email: requiredString,
+    firstName: requiredString,
+    lastName: requiredString,
+    type: z.number().int(),
+    expiresAt: optionalDateTimeString,
+  })
+  .passthrough();
+
+const workspaceInviteAcceptSchema = z
+  .object({
+    token: requiredString,
+    userId: optionalString,
+  })
+  .passthrough();
+
+const workspaceInviteRevokeSchema = z
+  .object({
+    reason: optionalString,
+    revokedByUserId: optionalString,
+  })
+  .passthrough();
+
+const workspaceContextRevalidationSchema = z
+  .object({
+    userId: requiredString,
+    providerSlug: optionalString,
+  })
+  .passthrough();
+
+export function parseLoginCredentialsPayload(body: unknown): ParseResult<LoginCredentialsPayload> {
+  return parseSchema(body, loginCredentialsSchema, "login_credentials");
+}
+
+export function parseApiKeyValidationPayload(body: unknown): ParseResult<ApiKeyValidationPayload> {
+  return parseSchema(body, apiKeyValidationSchema, "api_key_validation");
+}
+
+export function parseAnalyticsEventPayload(body: unknown): ParseResult<AnalyticsEventPayload> {
+  return parseSchema(body, analyticsEventSchema, "analytics_event");
+}
+
+export function parseAnalyticsPageViewPayload(body: unknown): ParseResult<AnalyticsPageViewPayload> {
+  return parseSchema(body, analyticsPageViewSchema, "analytics_page_view");
+}
+
+export function parseRelationshipCreatePayload(body: unknown): ParseResult<RelationshipCreatePayload> {
+  return parseSchema(body, relationshipCreateSchema, "relationship");
+}
+
+export function parseWorkspaceInviteCreatePayload(body: unknown): ParseResult<WorkspaceInviteCreatePayload> {
+  return parseSchema(body, workspaceInviteCreateSchema, "workspace_invite_create");
+}
+
+export function parseWorkspaceInviteAcceptPayload(body: unknown): ParseResult<WorkspaceInviteAcceptPayload> {
+  return parseSchema(body, workspaceInviteAcceptSchema, "workspace_invite_accept");
+}
+
+export function parseWorkspaceInviteRevokePayload(body: unknown): ParseResult<WorkspaceInviteRevokePayload> {
+  return parseSchema(body, workspaceInviteRevokeSchema, "workspace_invite_revoke");
+}
+
+export function parseWorkspaceContextRevalidationPayload(body: unknown): ParseResult<WorkspaceContextRevalidationPayload> {
+  return parseSchema(body, workspaceContextRevalidationSchema, "workspace_context_revalidation");
+}
+
+export function parseEntityRowObjectBody(body: unknown): ParseResult<Record<string, unknown>> {
+  return parseSchema(body, jsonObjectSchema, "entity_row");
+}
+
+function parseSchema<T>(body: unknown, schema: z.ZodType<T>, bodyType: string): ParseResult<T> {
+  if (!isJsonObject(body)) {
+    return { ok: false, error: createValidationError("body_invalid", { bodyType }) };
+  }
+
+  const result = schema.safeParse(body);
+  if (!result.success) {
+    return { ok: false, error: formatValidationError(result.error) };
+  }
+
+  return { ok: true, value: result.data };
+}
+
+function isJsonObject(body: unknown): body is Record<string, unknown> {
+  return Boolean(body && typeof body === "object" && !Array.isArray(body));
+}
+
+function formatValidationError(error: z.ZodError): RequestValidationError {
+  const issue = error.issues[0];
+  const field = issue?.path.length ? issue.path.join(".") : "body";
+  return createValidationError(reasonForIssue(issue), {
+    field,
+    ...paramsForIssue(issue),
+  });
+}
+
+function reasonForIssue(issue: z.ZodIssue | undefined): ValidationErrorReason {
+  if (!issue) {
+    return "field_value_invalid";
+  }
+
+  if (issue.code === z.ZodIssueCode.invalid_type) {
+    return issue.received === "undefined" ? "field_required" : "field_type_invalid";
+  }
+
+  if (issue.code === z.ZodIssueCode.too_small && issue.type === "string") {
+    return "field_required";
+  }
+
+  return "field_value_invalid";
+}
+
+function paramsForIssue(issue: z.ZodIssue | undefined): ApiErrorParams {
+  if (!issue) {
+    return {};
+  }
+
+  if (issue.code === z.ZodIssueCode.invalid_type && issue.received !== "undefined") {
+    return { expected: issue.expected };
+  }
+
+  if (issue.code === z.ZodIssueCode.too_small && issue.type === "string") {
+    return { min: Number(issue.minimum) };
+  }
+
+  if (issue.code === z.ZodIssueCode.invalid_string) {
+    return { expected: String(issue.validation) };
+  }
+
+  return {};
+}
+
+function createValidationError(reason: ValidationErrorReason, params: ApiErrorParams = {}): RequestValidationError {
+  return {
+    message: reason === "body_invalid" ? "Request body must be a JSON object." : "Request validation failed.",
+    params: { reason, ...params },
+  };
+}

hivegrid-cloud/apps/api/src/webhookDeliveryQueue.test.ts

@@ -0,0 +1,123 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+import { buildRetryJobOptions } from "@toolctrl/queue";
+import { buildLegacyEventWebhookAttemptDeliveryJob, queueLegacyEventWebhookAttemptDelivery } from "./webhookDeliveryQueue.js";
+import type { LegacyEventWebhookAttempt } from "@toolctrl/db/event-webhook-attempts.server";
+
+
+const attempt = {
+  id: "attempt_123",
+  endpoint: "https://example.test/webhooks/events",
+  tenantId: "tenant_123",
+} satisfies LegacyEventWebhookAttempt;
+
+const body = {
+  id: "evt_123",
+  type: "member.created",
+};
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe("buildLegacyEventWebhookAttemptDeliveryJob", () => {
+  test("builds a deliver-webhook descriptor for legacy attempts", () => {
+    expect(buildLegacyEventWebhookAttemptDeliveryJob(attempt, body, "req_webhook_123")).toEqual({
+      queueName: "toolctrl.email",
+      jobName: "deliver-webhook",
+      payload: {
+        kind: "webhook.deliver",
+        url: attempt.endpoint,
+        method: "POST",
+        eventName: "legacy.event_webhook_attempt",
+        body,
+        metadata: {
+          legacyAttemptId: attempt.id,
+          legacyAttemptTenantId: attempt.tenantId,
+          requestId: "req_webhook_123",
+        },
+      },
+    });
+  });
+
+  test("omits requestId metadata when requestId is absent", () => {
+    expect(buildLegacyEventWebhookAttemptDeliveryJob(attempt, body).payload.metadata).toEqual({
+      legacyAttemptId: attempt.id,
+      legacyAttemptTenantId: attempt.tenantId,
+    });
+  });
+});
+
+describe("queueLegacyEventWebhookAttemptDelivery", () => {
+  test("returns typed non-enqueued result when Redis env is absent", async () => {
+    const result = await queueLegacyEventWebhookAttemptDelivery({
+      attempt,
+      body,
+      env: {},
+    });
+
+    expect(result).toEqual({
+      enqueued: false,
+      reason: "missing-redis-connection",
+    });
+  });
+
+  test("enqueues on an injected queue without touching process environment", async () => {
+    const queue = buildQueue();
+    const job = { id: "job_123" };
+    queue.add.mockResolvedValue(job);
+
+    const result = await queueLegacyEventWebhookAttemptDelivery({
+      attempt,
+      body,
+      requestId: "req_enqueue_123",
+      queue,
+    });
+
+    const descriptor = buildLegacyEventWebhookAttemptDeliveryJob(attempt, body, "req_enqueue_123");
+    expect(queue.add).toHaveBeenCalledWith(
+      "deliver-webhook",
+      descriptor.payload,
+      // priority is resolved from attempt.tenantId; without a registered
+      // plan lookup it falls back to NORMAL (5).
+      { ...buildRetryJobOptions(descriptor), removeOnComplete: false, jobId: attempt.id, priority: 5 }
+    );
+    expect(result).toEqual({
+      enqueued: true,
+      job,
+    });
+    expect(queue.close).not.toHaveBeenCalled();
+  });
+
+  test("propagates enqueue failures for injected queues", async () => {
+    const error = new Error("enqueue failed");
+    const queue = buildQueue();
+    queue.add.mockRejectedValue(error);
+
+    await expect(
+      queueLegacyEventWebhookAttemptDelivery({
+        attempt,
+        body,
+        queue,
+      })
+    ).rejects.toThrow(error);
+    const descriptor = buildLegacyEventWebhookAttemptDeliveryJob(attempt, body);
+    expect(queue.add).toHaveBeenCalledWith(
+      "deliver-webhook",
+      expect.objectContaining({
+        kind: "webhook.deliver",
+      }),
+      { ...buildRetryJobOptions(descriptor), removeOnComplete: false, jobId: attempt.id, priority: 5 }
+    );
+    expect(queue.close).not.toHaveBeenCalled();
+  });
+});
+
+function buildQueue() {
+  return {
+    add: vi.fn(),
+    close: vi.fn().mockResolvedValue(undefined),
+  } as unknown as Parameters<typeof queueLegacyEventWebhookAttemptDelivery>[0]["queue"] & {
+    add: ReturnType<typeof vi.fn>;
+    close: ReturnType<typeof vi.fn>;
+  };
+}

hivegrid-cloud/apps/api/src/webhookDeliveryQueue.ts

@@ -0,0 +1,102 @@
+import {
+  buildWebhookDeliveryJob,
+  enqueueJob,
+  queueNames,
+  resolveRedisConnectionOptions,
+  type RedisConnectionEnv,
+  type TypedQueue,
+  type WebhookDeliveryJobDescriptor,
+} from "@toolctrl/queue";
+import type { LegacyEventWebhookAttempt } from "@toolctrl/db/event-webhook-attempts.server";
+
+type EmailQueue = TypedQueue<typeof queueNames.email>;
+
+export type QueueLegacyEventWebhookAttemptDeliveryInput = {
+  attempt: LegacyEventWebhookAttempt;
+  body: unknown;
+  requestId?: string;
+  env?: RedisConnectionEnv;
+  queue?: EmailQueue;
+};
+
+export type QueueLegacyEventWebhookAttemptDeliveryResult =
+  | {
+      enqueued: true;
+      job: Awaited<ReturnType<EmailQueue["add"]>>;
+    }
+  | {
+      enqueued: false;
+      reason: "missing-redis-connection";
+    };
+
+export function buildLegacyEventWebhookAttemptDeliveryJob(
+  attempt: LegacyEventWebhookAttempt,
+  body: unknown,
+  requestId?: string,
+): WebhookDeliveryJobDescriptor {
+  return buildWebhookDeliveryJob({
+    url: attempt.endpoint,
+    method: "POST",
+    eventName: "legacy.event_webhook_attempt",
+    body,
+    metadata: {
+      legacyAttemptId: attempt.id,
+      legacyAttemptTenantId: attempt.tenantId,
+      ...(requestId ? { requestId } : {}),
+    },
+  });
+}
+
+export async function queueLegacyEventWebhookAttemptDelivery({
+  attempt,
+  body,
+  requestId,
+  env,
+  queue,
+}: QueueLegacyEventWebhookAttemptDeliveryInput): Promise<QueueLegacyEventWebhookAttemptDeliveryResult> {
+  const ownedQueue = queue ? undefined : await createInternalQueue(env);
+
+  if (!queue && !ownedQueue) {
+    return {
+      enqueued: false,
+      reason: "missing-redis-connection",
+    };
+  }
+
+  const targetQueue = queue ?? ownedQueue;
+
+  if (!targetQueue) {
+    return {
+      enqueued: false,
+      reason: "missing-redis-connection",
+    };
+  }
+
+  try {
+    const descriptor = buildLegacyEventWebhookAttemptDeliveryJob(attempt, body, requestId);
+    const job = await enqueueJob(targetQueue, descriptor, {
+      idempotencyKey: attempt.id,
+      tenantId: attempt.tenantId,
+    });
+
+    return {
+      enqueued: true,
+      job,
+    };
+  } finally {
+    if (ownedQueue) {
+      await ownedQueue.close();
+    }
+  }
+}
+
+async function createInternalQueue(env?: RedisConnectionEnv): Promise<EmailQueue | undefined> {
+  const connection = resolveRedisConnectionOptions({ env: env ?? (process.env as RedisConnectionEnv) });
+
+  if (!connection) {
+    return undefined;
+  }
+
+  const { createQueue } = await import("@toolctrl/queue/runtime");
+  return createQueue(queueNames.email, { connection });
+}