HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/scripts/coolify-deploy.test.mjs

import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import http from "node:http";
import path from "node:path";
import { describe, it } from "vitest";

const scriptPath = path.resolve("scripts/coolify-deploy.mjs");
const bearerToken = "coolify-test-token-sentinel";
const deployRevision = "a".repeat(40);
const resourceUuid = "r".repeat(24);
const otherResourceUuid = "s".repeat(24);
const deploymentUuid = "d".repeat(24);
const otherDeploymentUuid = "e".repeat(24);
const mcpResourceUuid = "m".repeat(24);
const apiResourceUuid = "a".repeat(24);
const hocuspocusResourceUuid = "h".repeat(24);
const portalDomainControllerResourceUuid = "c".repeat(24);
const workerGroupResourceUuids = Object.freeze({
  general: "g".repeat(24),
  mail: "l".repeat(24),
  import: "i".repeat(24),
  webhook: "v".repeat(24),
  plugin: "q".repeat(24),
});
const workerResourceUuid = "w".repeat(24);
const legacyMcpResourceUuid = "c".repeat(24);
const webResourceUuid = "b".repeat(24);
const adminResourceUuid = "n".repeat(24);
const portalResourceUuid = "p".repeat(24);
const adminDeploymentUuid = "f".repeat(24);
const mcpImage = "registry.example.test/toolctrl-labs/hivegrid/mcp";
const workerImage = "registry.example.test/toolctrl-labs/hivegrid/worker";
const webImage = "registry.example.test/toolctrl-labs/hivegrid/web";
const apiImage = "registry.example.test/toolctrl-labs/hivegrid/api";
const hocuspocusImage = "registry.example.test/toolctrl-labs/hivegrid/hocuspocus";
const portalDomainControllerImage = "registry.example.test/toolctrl-labs/hivegrid/portal-domain-controller";
const webDigest = `sha256:${"f".repeat(64)}`;
const sharedCryptoSecret = "shared-tenant-envelope-secret-sentinel";
const sharedAdminConnectionMasterKey = "shared-admin-connection-master-key-sentinel";
const gdprExportSseCKey = Buffer.alloc(32, 7).toString("base64");
const alternateGdprExportSseCKey = Buffer.alloc(32, 8).toString("base64");
const workerRedisSecrets = Object.freeze(
  Object.fromEntries(Object.keys(workerGroupResourceUuids).map((group) => [group, `worker-${group}-redis-secret-sentinel`]))
);
const gdprExportStorageEnv = Object.freeze({
  STORAGE_ENDPOINT: "https://storage.example.test",
  STORAGE_REGION: "eu-central-1",
  STORAGE_BUCKET: "private-exports",
  STORAGE_ACCESS_KEY_ID: "storage-access-key-sentinel",
  STORAGE_SECRET_ACCESS_KEY: "storage-secret-key-sentinel",
  STORAGE_FORCE_PATH_STYLE: "false",
  GDPR_EXPORT_SSE_C_KEY_BASE64: gdprExportSseCKey,
});
const portalGdprExportStorageEnv = Object.freeze({
  ...gdprExportStorageEnv,
  STORAGE_ACCESS_KEY_ID: "portal-storage-access-key-sentinel",
  STORAGE_SECRET_ACCESS_KEY: "portal-storage-secret-key-sentinel",
});
const importGdprExportStorageEnv = Object.freeze({
  ...gdprExportStorageEnv,
  STORAGE_ACCESS_KEY_ID: "import-storage-access-key-sentinel",
  STORAGE_SECRET_ACCESS_KEY: "import-storage-secret-key-sentinel",
});
const GDPR_EXPORT_STORAGE_TEST_KEYS = new Set(Object.keys(gdprExportStorageEnv));
const gdprExportRetentionStorageEnv = Object.freeze(
  Object.fromEntries(
    Object.entries({
      ...gdprExportStorageEnv,
      STORAGE_ACCESS_KEY_ID: "general-storage-access-key-sentinel",
      STORAGE_SECRET_ACCESS_KEY: "general-storage-secret-key-sentinel",
    }).filter(([key]) => key !== "GDPR_EXPORT_SSE_C_KEY_BASE64")
  )
);
const validRuntimeResourceLimits = Object.freeze({ limits_cpus: ".5", limits_memory: "512m" });
const validWebSurfaceResourceLimits = validRuntimeResourceLimits;
const vapidSubject = "mailto:ops@hivegrid.de";
const vapidPublicKey = (() => {
  const bytes = Buffer.alloc(65, 1);
  bytes[0] = 0x04;
  return bytes.toString("base64url");
})();
const alternateVapidPublicKey = (() => {
  const bytes = Buffer.alloc(65, 3);
  bytes[0] = 0x04;
  return bytes.toString("base64url");
})();
const vapidPrivateKey = Buffer.alloc(32, 2).toString("base64url");

function portalRuntimeEnv(overrides = {}) {
  const values = {
    DATABASE_URL: "postgres://hivegrid_portal:portal-secret-sentinel@db.example.test:6432/hivegrid",
    DATABASE_READ_URL: "postgres://hivegrid_portal:portal-read-secret-sentinel@db.example.test/hivegrid",
    DATABASE_RUNTIME_ROLE: "hivegrid_portal",
    DATABASE_POOL_MAX: "10",
    RLS_RUNTIME_MODE: "required",
    WEB_RUNTIME_SURFACE: "portal",
    PORTAL_SESSION_SECRET: "dedicated-portal-session-secret-sentinel",
    CAPTCHA_PROVIDER: "mcaptcha",
    MCAPTCHA_INSTANCE_URL: "https://captcha.example.test",
    MCAPTCHA_SITEKEY_LOGIN: "portal-login-sitekey",
    MCAPTCHA_ACCOUNT_SECRET: "portal-mcaptcha-secret-sentinel",
    REDIS_URL: "rediss://portal-redis-secret-sentinel@redis.example.test:6380/3",
    ...overrides,
  };
  return Object.entries(values)
    .filter(([, value]) => value !== undefined)
    .map(([key, value]) => ({ key, value }));
}

function runtimeDatabaseEnv(role, overrides = {}) {
  const values = {
    DATABASE_URL: `postgres://${role}:runtime-secret-sentinel@db.example.test:6432/hivegrid`,
    DATABASE_READ_URL: `postgres://${role}:read-secret-sentinel@db.example.test/hivegrid`,
    DATABASE_RUNTIME_ROLE: role,
    DATABASE_POOL_MAX: "10",
    RLS_RUNTIME_MODE: "required",
    ...overrides,
  };
  return Object.entries(values)
    .filter(([, value]) => value !== undefined)
    .map(([key, value]) => ({ key, value }));
}

function runtimePreflightFixture() {
  const apps = [
    {
      uuid: webResourceUuid,
      name: "hivegrid-web",
      build_pack: "dockerimage",
      docker_registry_image_name: webImage,
      ...validRuntimeResourceLimits,
    },
    {
      uuid: portalResourceUuid,
      name: "hivegrid-portal",
      build_pack: "dockerimage",
      docker_registry_image_name: webImage,
      ...validRuntimeResourceLimits,
    },
    {
      uuid: adminResourceUuid,
      name: "hivegrid-admin",
      build_pack: "dockerimage",
      docker_registry_image_name: webImage,
      ...validRuntimeResourceLimits,
    },
    { uuid: apiResourceUuid, name: "hivegrid-api", docker_registry_image_name: apiImage, ...validRuntimeResourceLimits },
    { uuid: mcpResourceUuid, name: "hivegrid-mcp", docker_registry_image_name: mcpImage, ...validRuntimeResourceLimits },
    { uuid: hocuspocusResourceUuid, name: "hivegrid-hocuspocus", docker_registry_image_name: hocuspocusImage, ...validRuntimeResourceLimits },
    {
      uuid: portalDomainControllerResourceUuid,
      name: "hivegrid-portal-domain-controller",
      docker_registry_image_name: portalDomainControllerImage,
      ...validRuntimeResourceLimits,
    },
    ...Object.entries(workerGroupResourceUuids).map(([group, uuid]) => ({
      uuid,
      name: `worker-${group}`,
      docker_registry_image_name: workerImage,
      ...validRuntimeResourceLimits,
    })),
  ];
  const envByUuid = new Map([
    [
      webResourceUuid,
      runtimeDatabaseEnv("hivegrid_app", {
        WEB_RUNTIME_SURFACE: "core",
        PORTAL_APP_HOST: "portal.example.test",
        APP_BASE_DOMAIN: "example.test",
        WEB_APP_HOST: "app.example.test",
        ADMIN_APP_HOST: "admin.example.test",
        API_HOST: "api.example.test",
        CRYPTO_SECRET: sharedCryptoSecret,
        ADMIN_CONNECTION_MASTER_KEY: sharedAdminConnectionMasterKey,
        REDIS_URL: "rediss://core-redis-secret-sentinel@redis.example.test:6380/2",
        WEB_PUSH_VAPID_PUBLIC_KEY: vapidPublicKey,
        ...gdprExportStorageEnv,
      }),
    ],
    [
      adminResourceUuid,
      runtimeDatabaseEnv("hivegrid_app", {
        WEB_RUNTIME_SURFACE: "admin",
        SESSION_SECRET: "dedicated-admin-session-secret-sentinel",
        ADMIN_FALLBACK_LOGIN_ENABLED: "false",
        AUTHENTIK_ISSUER_URL: "https://auth.example.test/application/o/hivegrid-admin/",
        AUTHENTIK_ADMIN_CLIENT_ID: "hivegrid-admin",
        AUTHENTIK_ADMIN_CLIENT_SECRET: "authentik-admin-client-secret-sentinel",
        AUTHENTIK_ADMIN_REDIRECT_URI: "https://admin.example.test/auth/admin/callback",
        AUTHENTIK_ADMIN_JIT_DATABASE_URL: "postgres://hivegrid_authentik_jit:jit-secret-sentinel@db.example.test/hivegrid",
        PORTAL_APP_HOST: "portal.example.test",
        APP_BASE_DOMAIN: "example.test",
        WEB_APP_HOST: "app.example.test",
        ADMIN_APP_HOST: "admin.example.test",
        API_HOST: "api.example.test",
        ADMIN_CONNECTION_MASTER_KEY: sharedAdminConnectionMasterKey,
        REDIS_URL: "rediss://admin-redis-secret-sentinel@redis.example.test:6380/5",
        ...gdprExportStorageEnv,
      }),
    ],
    [
      portalResourceUuid,
      portalRuntimeEnv({
        DATABASE_POOL_MAX: "6",
        PORTAL_APP_HOST: "https://Portal.Example.Test:443",
        APP_PROTOCOL: "https",
        ...portalGdprExportStorageEnv,
      }),
    ],
    [
      apiResourceUuid,
      runtimeDatabaseEnv("hivegrid_app", {
        ADMIN_CONNECTION_MASTER_KEY: sharedAdminConnectionMasterKey,
      }),
    ],
    [mcpResourceUuid, runtimeDatabaseEnv("hivegrid_app")],
    [hocuspocusResourceUuid, runtimeDatabaseEnv("hivegrid_app")],
    ...Object.entries(workerGroupResourceUuids).map(([group, uuid]) => [
      uuid,
      runtimeDatabaseEnv("hivegrid_worker", {
        WORKER_QUEUE_GROUP: group,
        ADMIN_CONNECTION_MASTER_KEY: sharedAdminConnectionMasterKey,
        BULLMQ_REDIS_URL: `rediss://${workerRedisSecrets[group]}@redis.example.test:6380/4`,
        ...(group === "mail" ? { PORTAL_APP_HOST: "portal.example.test:443" } : {}),
        ...(group === "mail" ? { APP_PROTOCOL: "https" } : {}),
        ...(group === "mail" ? { CRYPTO_SECRET: sharedCryptoSecret } : {}),
        ...(group === "general"
          ? {
              ...gdprExportRetentionStorageEnv,
              MEILI_URL: "http://meili.internal.test:7700",
              MEILI_API_KEY: "hivegrid-scoped-meili-key-sentinel",
              EXPORT_RETENTION_QUEUE_ENABLED: "true",
            }
          : {}),
        ...(group === "import"
          ? {
              ...importGdprExportStorageEnv,
              IMPORT_QUEUE_ENABLED: "false",
              IMPORT_QUEUE_RUNTIME_READY: "false",
              PORTAL_GDPR_EXPORT_QUEUE_ENABLED: "true",
            }
          : {}),
        ...(group === "webhook"
          ? {
              WEB_PUSH_VAPID_SUBJECT: vapidSubject,
              WEB_PUSH_VAPID_PUBLIC_KEY: vapidPublicKey,
              WEB_PUSH_VAPID_PRIVATE_KEY: vapidPrivateKey,
            }
          : {}),
      }),
    ]),
  ]);
  return {
    apps,
    envByUuid,
    allowedCsv: apps.map(({ uuid }) => uuid).join(","),
  };
}

function runRuntimePreflight(serverUrl, allowedCsv) {
  return runHelper(serverUrl, "assert-runtime-preflight", allowedCsv, webImage, apiImage, mcpImage, hocuspocusImage, workerImage);
}

function webPushEnvPayload(requestUrl, overrides = {}) {
  if (requestUrl === `/applications/${webResourceUuid}/envs`) {
    return overrides.web ?? [{ key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey }];
  }
  if (requestUrl === `/applications/${workerResourceUuid}/envs`) {
    return (
      overrides.worker ?? [
        { key: "WEB_PUSH_VAPID_SUBJECT", value: vapidSubject },
        { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey },
        { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: vapidPrivateKey },
      ]
    );
  }
  return [];
}

function expectedMcpHealthConfig() {
  return {
    docker_registry_image_tag: deployRevision,
    health_check_enabled: true,
    health_check_type: "cmd",
    health_check_command: `node apps/mcp/dist/deploy-health.js ${deployRevision}`,
    health_check_interval: 30,
    health_check_timeout: 10,
    health_check_retries: 3,
    health_check_start_period: 20,
  };
}

function runHelperWithEnv(serverUrl, envOverrides, ...args) {
  return new Promise((resolve) => {
    const child = spawn(process.execPath, [scriptPath, ...args], {
      env: {
        PATH: process.env.PATH || "",
        NODE_ENV: "test",
        SELECTED_COOLIFY_API_URL: serverUrl,
        SELECTED_COOLIFY_TOKEN: bearerToken,
        MCP_IMAGE: mcpImage,
        WORKER_IMAGE: workerImage,
        ...envOverrides,
      },
      stdio: ["ignore", "pipe", "pipe"],
    });
    let stdout = "";
    let stderr = "";
    child.stdout.setEncoding("utf8");
    child.stderr.setEncoding("utf8");
    child.stdout.on("data", (chunk) => {
      stdout += chunk;
    });
    child.stderr.on("data", (chunk) => {
      stderr += chunk;
    });
    child.on("close", (status) => {
      resolve({ status, stdout, stderr });
    });
  });
}

function runHelper(serverUrl, ...args) {
  return runHelperWithEnv(serverUrl, {}, ...args);
}

async function readBody(request) {
  let body = "";
  request.setEncoding("utf8");
  for await (const chunk of request) {
    body += chunk;
  }
  return body;
}

function startServer(responder) {
  const requests = [];
  const serverErrors = [];
  const server = http.createServer(async (request, response) => {
    const recorded = {
      method: request.method,
      url: request.url,
      authorization: request.headers.authorization,
      body: await readBody(request),
    };
    requests.push(recorded);

    try {
      const result = await responder(recorded, requests.length);
      response.writeHead(result?.status || 200, { "content-type": "application/json" });
      response.end(JSON.stringify(result?.payload ?? {}));
    } catch (error) {
      serverErrors.push(error);
      response.writeHead(500, { "content-type": "application/json" });
      response.end(JSON.stringify({ message: "test server assertion failed" }));
    }
  });

  return new Promise((resolve, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", () => {
      resolve({
        url: `http://127.0.0.1:${server.address().port}`,
        requests,
        serverErrors,
        close: () => new Promise((closeResolve, closeReject) => server.close((error) => (error ? closeReject(error) : closeResolve()))),
      });
    });
  });
}

function assertSecretSafe(result) {
  assert.doesNotMatch(result.stdout, new RegExp(bearerToken));
  assert.doesNotMatch(result.stderr, new RegExp(bearerToken));
  for (const value of [
    gdprExportStorageEnv.STORAGE_ACCESS_KEY_ID,
    gdprExportStorageEnv.STORAGE_SECRET_ACCESS_KEY,
    portalGdprExportStorageEnv.STORAGE_ACCESS_KEY_ID,
    portalGdprExportStorageEnv.STORAGE_SECRET_ACCESS_KEY,
    importGdprExportStorageEnv.STORAGE_ACCESS_KEY_ID,
    importGdprExportStorageEnv.STORAGE_SECRET_ACCESS_KEY,
    gdprExportRetentionStorageEnv.STORAGE_ACCESS_KEY_ID,
    gdprExportRetentionStorageEnv.STORAGE_SECRET_ACCESS_KEY,
    gdprExportSseCKey,
    alternateGdprExportSseCKey,
    ...Object.values(workerRedisSecrets),
  ]) {
    assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(value));
  }
}

function assertWebPushValuesSafe(result) {
  assertSecretSafe(result);
  for (const value of [vapidPublicKey, alternateVapidPublicKey, vapidPrivateKey]) {
    assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(value));
  }
}

describe("coolify-deploy CLI", () => {
  it("triggers the canonical deploy endpoint and emits only the deployment UUID", async () => {
    const server = await startServer(() => ({
      payload: {
        deployments: [
          { resource_uuid: otherResourceUuid, deployment_uuid: otherDeploymentUuid },
          { resource_uuid: resourceUuid, deployment_uuid: deploymentUuid },
        ],
      },
    }));

    try {
      const result = await runHelper(server.url, "trigger", resourceUuid);

      assert.equal(result.status, 0, result.stderr);
      assert.equal(result.stdout.trim(), deploymentUuid);
      assert.equal(result.stderr, "");
      assertSecretSafe(result);
      assert.deepEqual(server.serverErrors, []);
      assert.deepEqual(server.requests, [
        {
          method: "POST",
          url: `/deploy?uuid=${resourceUuid}&force=true`,
          authorization: `Bearer ${bearerToken}`,
          body: "",
        },
      ]);
    } finally {
      await server.close();
    }
  });

  it("fails closed when the deployment array does not contain the requested resource", async () => {
    const server = await startServer(() => ({
      payload: {
        deployments: [{ resource_uuid: otherResourceUuid, deployment_uuid: otherDeploymentUuid }],
      },
    }));

    try {
      const result = await runHelper(server.url, "trigger", resourceUuid);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /deployment UUID was not returned/);
      assert.doesNotMatch(result.stderr, new RegExp(otherDeploymentUuid));
      assertSecretSafe(result);
      assert.deepEqual(server.serverErrors, []);
      assert.equal(server.requests.length, 1);
    } finally {
      await server.close();
    }
  });

  it("does not accept a legacy UUID when Coolify returns an empty deployment array", async () => {
    const server = await startServer(() => ({
      payload: { deployments: [], deployment_uuid: "stale-legacy-deployment" },
    }));

    try {
      const result = await runHelper(server.url, "trigger", resourceUuid);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /deployment UUID was not returned/);
      assert.doesNotMatch(result.stderr, /stale-legacy-deployment/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails closed when the matching deployment entry has no UUID", async () => {
    const server = await startServer(() => ({
      payload: { deployments: [{ resource_uuid: resourceUuid }] },
    }));

    try {
      const result = await runHelper(server.url, "trigger", resourceUuid);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /deployment UUID was not returned/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects malformed resource and deployment IDs without emitting injectable output", async () => {
    const server = await startServer(() => ({
      payload: {
        deployments: [{ resource_uuid: resourceUuid, deployment_uuid: `${deploymentUuid}\nINJECTED=value` }],
      },
    }));

    try {
      const invalidResource = await runHelper(server.url, "trigger", `${resourceUuid}\nINJECTED=value`);
      assert.equal(invalidResource.status, 1);
      assert.equal(invalidResource.stdout, "");
      assert.match(invalidResource.stderr, /valid Coolify CUID/);
      assert.deepEqual(server.requests, []);

      const invalidDeployment = await runHelper(server.url, "trigger", resourceUuid);
      assert.equal(invalidDeployment.status, 1);
      assert.equal(invalidDeployment.stdout, "");
      assert.match(invalidDeployment.stderr, /deployment UUID was not returned/);
      assert.doesNotMatch(invalidDeployment.stderr, /INJECTED/);
      assertSecretSafe(invalidResource);
      assertSecretSafe(invalidDeployment);
    } finally {
      await server.close();
    }
  });

  it("accepts only applications without an APP_REVISION Coolify override", async () => {
    const cleanServer = await startServer(() => ({ payload: [] }));
    try {
      const result = await runHelper(cleanServer.url, "assert-no-app-revision-override", webResourceUuid);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /override absent/);
      assert.deepEqual(
        cleanServer.requests.map(({ method, url }) => ({ method, url })),
        [{ method: "GET", url: `/applications/${webResourceUuid}/envs` }]
      );
      assertSecretSafe(result);
    } finally {
      await cleanServer.close();
    }

    const overrideServer = await startServer(() => ({ payload: [{ key: "APP_REVISION", value: "" }] }));
    try {
      const result = await runHelper(overrideServer.url, "assert-no-app-revision-override", webResourceUuid);
      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /APP_REVISION override must be removed/);
      assert.deepEqual(
        overrideServer.requests.map(({ method }) => method),
        ["GET"]
      );
      assertSecretSafe(result);
    } finally {
      await overrideServer.close();
    }
  });

  it("retries legacy Coolify 4.0 environment reads with explicit decryption", async () => {
    const server = await startServer(({ url }) =>
      url === `/applications/${webResourceUuid}/envs` ? { status: 404, payload: { message: "Not found" } } : { payload: [] }
    );
    try {
      const result = await runHelper(server.url, "assert-no-app-revision-override", webResourceUuid);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /override absent/);
      assert.deepEqual(
        server.requests.map(({ method, url }) => ({ method, url })),
        [
          { method: "GET", url: `/applications/${webResourceUuid}/envs` },
          { method: "GET", url: `/applications/${webResourceUuid}/envs?decrypt=true` },
        ]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails closed on malformed revision-override environment responses", async () => {
    const server = await startServer(() => ({ payload: { unexpected: [] } }));
    try {
      const result = await runHelper(server.url, "assert-no-app-revision-override", webResourceUuid);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /environment response was malformed/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("accepts only a required-RLS runtime credential for the declared role", async () => {
    const server = await startServer(() => ({
      payload: [
        { key: "DATABASE_URL", value: "postgres://hivegrid_app:runtime-secret-sentinel@db.example.test:6432/hivegrid" },
        { key: "DATABASE_READ_URL", value: "postgres://hivegrid_app:read-secret-sentinel@db.example.test/hivegrid" },
        { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_app" },
        { key: "DATABASE_POOL_MAX", value: "10" },
        { key: "RLS_RUNTIME_MODE", value: "required" },
        { key: "WEB_RUNTIME_SURFACE", value: "core" },
      ],
    }));
    try {
      const result = await runHelper(server.url, "assert-runtime-db-env", webResourceUuid, "hivegrid_app", "web");
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /verified for hivegrid_app/);
      assert.doesNotMatch(result.stdout, /runtime-secret-sentinel/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("accepts the dedicated portal role and rejects core, OAuth, billing, migrator, and ingress credentials", async () => {
    const base = portalRuntimeEnv();
    const cleanServer = await startServer(() => ({ payload: base }));
    try {
      const result = await runHelper(cleanServer.url, "assert-runtime-db-env", portalResourceUuid, "hivegrid_portal", "portal");
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /verified for hivegrid_portal/);
      assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
      assertSecretSafe(result);
    } finally {
      await cleanServer.close();
    }

    for (const key of [
      "SESSION_SECRET",
      "AUTHENTIK_ADMIN_CLIENT_SECRET",
      "OAUTH_CLIENT_SECRET",
      "LAGO_API_KEY",
      "MIGRATOR_DATABASE_URL",
      "COOLIFY_TOKEN",
      "SSH_PRIVATE_KEY",
      "TRAEFIK_API_TOKEN",
      "PORTAL_DOMAIN_CONTROLLER_TOKEN",
      "PORTAL_SERVER_API_TOKEN",
    ]) {
      const server = await startServer(() => ({ payload: [...base, { key, value: "forbidden-secret-sentinel" }] }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", portalResourceUuid, "hivegrid_portal", "portal");
        assert.equal(result.status, 1, `${key} unexpectedly passed`);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects an optional DATABASE_READ_URL whose username differs from the declared runtime role", async () => {
    const cases = [
      {
        resource: webResourceUuid,
        role: "hivegrid_app",
        surface: "web",
        payload: [
          { key: "DATABASE_URL", value: "postgres://hivegrid_app:runtime-secret-sentinel@db.example.test/hivegrid" },
          { key: "DATABASE_READ_URL", value: "postgres://hivegrid_worker:read-secret-sentinel@db.example.test/hivegrid" },
          { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_app" },
          { key: "RLS_RUNTIME_MODE", value: "required" },
          { key: "WEB_RUNTIME_SURFACE", value: "core" },
        ],
      },
      {
        resource: portalResourceUuid,
        role: "hivegrid_portal",
        surface: "portal",
        payload: portalRuntimeEnv({
          DATABASE_READ_URL: "postgres://hivegrid_app:read-secret-sentinel@db.example.test/hivegrid",
        }),
      },
      {
        resource: workerResourceUuid,
        role: "hivegrid_worker",
        surface: "service",
        payload: [
          { key: "DATABASE_URL", value: "postgres://hivegrid_worker:runtime-secret-sentinel@db.example.test/hivegrid" },
          { key: "DATABASE_READ_URL", value: "postgres://hivegrid_app:read-secret-sentinel@db.example.test/hivegrid" },
          { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_worker" },
          { key: "RLS_RUNTIME_MODE", value: "required" },
        ],
      },
    ];

    for (const { resource, role, surface, payload } of cases) {
      const server = await startServer(() => ({ payload }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", resource, role, surface);
        assert.equal(result.status, 1, `${role} unexpectedly accepted a mismatched read role`);
        assert.match(result.stderr, /read database credential does not match DATABASE_RUNTIME_ROLE/);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("accepts Cap portal login verification with a login-specific or shared secret", async () => {
    for (const secret of [{ CAP_SITE_SECRET_LOGIN: "cap-login-secret-sentinel" }, { CAP_SITE_SECRET: "cap-shared-secret-sentinel" }]) {
      const payload = portalRuntimeEnv({
        CAPTCHA_PROVIDER: "cap",
        MCAPTCHA_INSTANCE_URL: undefined,
        MCAPTCHA_SITEKEY_LOGIN: undefined,
        MCAPTCHA_ACCOUNT_SECRET: undefined,
        CAP_INSTANCE_URL: "https://cap.example.test",
        CAP_SITEKEY_LOGIN: "portal-cap-login-sitekey",
        ...secret,
      });
      const server = await startServer(() => ({ payload }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", portalResourceUuid, "hivegrid_portal", "portal");
        assert.equal(result.status, 0, result.stderr);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects incomplete provider-specific portal CAPTCHA contracts", async () => {
    const capBase = {
      CAPTCHA_PROVIDER: "cap",
      CAP_INSTANCE_URL: "https://cap.example.test",
      CAP_SITEKEY_LOGIN: "portal-cap-login-sitekey",
      CAP_SITE_SECRET_LOGIN: "cap-login-secret-sentinel",
    };
    const cases = [
      { overrides: { CAPTCHA_PROVIDER: undefined }, expectedKey: "CAPTCHA_PROVIDER" },
      { overrides: { CAPTCHA_PROVIDER: "unsupported" }, expectedKey: "CAPTCHA_PROVIDER" },
      { overrides: { MCAPTCHA_INSTANCE_URL: undefined }, expectedKey: "MCAPTCHA_INSTANCE_URL" },
      { overrides: { MCAPTCHA_SITEKEY_LOGIN: undefined }, expectedKey: "MCAPTCHA_SITEKEY_LOGIN" },
      { overrides: { MCAPTCHA_ACCOUNT_SECRET: undefined }, expectedKey: "MCAPTCHA_ACCOUNT_SECRET" },
      { overrides: { ...capBase, CAP_INSTANCE_URL: undefined }, expectedKey: "CAP_INSTANCE_URL" },
      { overrides: { ...capBase, CAP_SITEKEY_LOGIN: undefined }, expectedKey: "CAP_SITEKEY_LOGIN" },
      { overrides: { ...capBase, CAP_SITE_SECRET_LOGIN: undefined, CAP_SITE_SECRET: undefined }, expectedKey: "CAP_SITE_SECRET_LOGIN" },
    ];

    for (const { overrides, expectedKey } of cases) {
      const server = await startServer(() => ({ payload: portalRuntimeEnv(overrides) }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", portalResourceUuid, "hivegrid_portal", "portal");
        assert.equal(result.status, 1, `${expectedKey} unexpectedly passed`);
        assert.match(result.stderr, new RegExp(expectedKey));
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects missing or invalid portal Redis and BullMQ connections", async () => {
    const cases = [
      { overrides: { REDIS_URL: undefined }, expectedKey: "REDIS_URL" },
      { overrides: { REDIS_URL: "https://redis-secret-sentinel.example.test" }, expectedKey: "REDIS_URL" },
      { overrides: { BULLMQ_REDIS_URL: "redis://redis.example.test/not-a-database" }, expectedKey: "BULLMQ_REDIS_URL" },
    ];

    for (const { overrides, expectedKey } of cases) {
      const server = await startServer(() => ({ payload: portalRuntimeEnv(overrides) }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", portalResourceUuid, "hivegrid_portal", "portal");
        assert.equal(result.status, 1, `${expectedKey} unexpectedly passed`);
        assert.match(result.stderr, new RegExp(expectedKey));
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("requires the explicit core and portal runtime surfaces", async () => {
    for (const [role, surface, entries] of [
      [
        "hivegrid_app",
        "web",
        [
          { key: "DATABASE_URL", value: "postgres://hivegrid_app:secret@db.example.test/hivegrid" },
          { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_app" },
          { key: "RLS_RUNTIME_MODE", value: "required" },
        ],
      ],
      [
        "hivegrid_portal",
        "portal",
        [
          { key: "DATABASE_URL", value: "postgres://hivegrid_portal:secret@db.example.test/hivegrid" },
          { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_portal" },
          { key: "RLS_RUNTIME_MODE", value: "required" },
          { key: "PORTAL_SESSION_SECRET", value: "portal-secret" },
        ],
      ],
    ]) {
      const server = await startServer(() => ({ payload: entries }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", surface === "web" ? webResourceUuid : portalResourceUuid, role, surface);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /WEB_RUNTIME_SURFACE/);
      } finally {
        await server.close();
      }
    }
  });

  it("requires an explicit positive pool cap and PgBouncer primary URL for every runtime", async () => {
    const base = runtimeDatabaseEnv("hivegrid_app", { WEB_RUNTIME_SURFACE: "core" });
    const cases = [
      {
        payload: base.filter(({ key }) => key !== "DATABASE_POOL_MAX"),
        error: /explicit positive DATABASE_POOL_MAX/,
      },
      {
        payload: base.map((entry) => (entry.key === "DATABASE_POOL_MAX" ? { ...entry, value: "0" } : entry)),
        error: /explicit positive DATABASE_POOL_MAX/,
      },
      {
        payload: base.map((entry) =>
          entry.key === "DATABASE_URL" ? { ...entry, value: "postgres://hivegrid_app:runtime-secret-sentinel@db.example.test:5432/hivegrid" } : entry
        ),
        error: /PgBouncer port 6432/,
      },
    ];

    for (const { payload, error } of cases) {
      const server = await startServer(() => ({ payload }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", webResourceUuid, "hivegrid_app", "web");
        assert.equal(result.status, 1);
        assert.match(result.stderr, error);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("verifies positive hard caps on all twelve allowlisted runtimes with GET-only requests before migration", async () => {
    const fixture = runtimePreflightFixture();
    const server = await startServer((request) => ({
      payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
    }));
    try {
      const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /read-only runtime preflight verified/);
      assert.equal(result.stderr, "");
      assert.equal(server.requests.length, 12);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assert.equal(new Set(server.requests.map(({ url }) => url)).size, 12);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects missing or invalid hard caps on API, MCP, and Hocuspocus runtimes", async () => {
    const cases = [
      { uuid: apiResourceUuid, field: "limits_cpus", value: undefined, label: "API" },
      { uuid: mcpResourceUuid, field: "limits_cpus", value: "0", label: "MCP" },
      { uuid: hocuspocusResourceUuid, field: "limits_memory", value: "512", label: "Hocuspocus" },
    ];

    for (const { uuid, field, value, label } of cases) {
      const fixture = runtimePreflightFixture();
      fixture.apps = fixture.apps.map((app) => (app.uuid === uuid ? { ...app, [field]: value } : app));
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1, `${label} resource limits unexpectedly passed`);
        assert.match(result.stderr, new RegExp(`Coolify ${label} runtime must set positive limits_cpus and unit-suffixed limits_memory hard caps`));
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects missing or invalid hard caps on every worker runtime", async () => {
    const invalidLimits = [
      { field: "limits_cpus", value: undefined },
      { field: "limits_cpus", value: "Infinity" },
      { field: "limits_memory", value: undefined },
      { field: "limits_memory", value: "0m" },
      { field: "limits_memory", value: "1.5g" },
    ];

    for (const [index, [group, uuid]] of Object.entries(workerGroupResourceUuids).entries()) {
      const fixture = runtimePreflightFixture();
      const { field, value } = invalidLimits[index];
      fixture.apps = fixture.apps.map((app) => (app.uuid === uuid ? { ...app, [field]: value } : app));
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1, `worker-${group} resource limits unexpectedly passed`);
        assert.match(result.stderr, new RegExp(`Coolify worker-${group} runtime must set positive limits_cpus and unit-suffixed limits_memory hard caps`));
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("requires positive CPU and unit-suffixed memory hard caps on the portal-domain-controller", async () => {
    for (const invalid of [{ limits_cpus: "0" }, { limits_memory: "512mb" }]) {
      const fixture = runtimePreflightFixture();
      fixture.apps = fixture.apps.map((app) => (app.uuid === portalDomainControllerResourceUuid ? { ...app, ...invalid } : app));
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1, "portal-domain-controller resource limits unexpectedly passed");
        assert.match(
          result.stderr,
          /Coolify portal-domain-controller runtime must set positive limits_cpus and unit-suffixed limits_memory hard caps/
        );
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("fails the runtime preflight when core lacks the distributed GDPR download Redis contract", async () => {
    for (const value of [undefined, "https://redis-secret-sentinel.example.test"]) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(
        webResourceUuid,
        fixture.envByUuid
          .get(webResourceUuid)
          .filter(({ key }) => key !== "REDIS_URL")
          .concat(value === undefined ? [] : [{ key: "REDIS_URL", value }])
      );
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /hivegrid-web runtime requires a valid REDIS_URL for distributed GDPR export download gates/);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /redis-secret-sentinel/);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects APP_REVISION overrides on every pre-migration runtime without leaking values", async () => {
    const runtimeUuids = [...runtimePreflightFixture().envByUuid.keys()];
    for (const uuid of runtimeUuids) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(uuid, [...fixture.envByUuid.get(uuid), { key: "APP_REVISION", value: `secret-${uuid}` }]);
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1, `${uuid} APP_REVISION unexpectedly passed`);
        assert.match(result.stderr, /APP_REVISION overrides must be removed before migration/);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(`secret-${uuid}`));
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects the portal session secret on the core web runtime", async () => {
    const fixture = runtimePreflightFixture();
    fixture.envByUuid.set(webResourceUuid, [
      ...fixture.envByUuid.get(webResourceUuid),
      { key: "PORTAL_SESSION_SECRET", value: "core-portal-session-secret-sentinel" },
    ]);
    const server = await startServer((request) => ({
      payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
    }));
    try {
      const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /hivegrid-web runtime must not contain PORTAL_SESSION_SECRET/);
      assert.doesNotMatch(`${result.stdout}${result.stderr}`, /core-portal-session-secret-sentinel/);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails the runtime preflight when a worker group is missing or outside the allowlist", async () => {
    for (const mode of ["missing", "not-allowlisted"]) {
      const fixture = runtimePreflightFixture();
      if (mode === "missing") {
        fixture.apps = fixture.apps.filter(({ uuid }) => uuid !== workerGroupResourceUuids.import);
      } else {
        fixture.allowedCsv = fixture.allowedCsv
          .split(",")
          .filter((uuid) => uuid !== workerGroupResourceUuids.import)
          .join(",");
      }
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1, `${mode} worker unexpectedly passed`);
        assert.match(result.stderr, /worker-import runtime is missing, ambiguous, or outside the resource allowlist/);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("fails the runtime preflight on portal-host or HTTPS drift across boundary consumers", async () => {
    const cases = [
      {
        uuid: webResourceUuid,
        key: "PORTAL_APP_HOST",
        value: "other-portal.example.test",
        error: /same normalized PORTAL_APP_HOST/,
      },
      {
        uuid: workerGroupResourceUuids.mail,
        key: "PORTAL_APP_HOST",
        value: "other-portal.example.test",
        error: /same normalized PORTAL_APP_HOST/,
      },
      {
        uuid: workerGroupResourceUuids.mail,
        key: "PORTAL_APP_HOST",
        value: "portal.example.test:444",
        error: /same normalized PORTAL_APP_HOST/,
      },
      {
        uuid: portalResourceUuid,
        key: "APP_PROTOCOL",
        value: "http",
        error: /must set APP_PROTOCOL=https/,
      },
      {
        uuid: workerGroupResourceUuids.mail,
        key: "APP_PROTOCOL",
        value: "http",
        error: /must set APP_PROTOCOL=https/,
      },
    ];

    for (const testCase of cases) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(
        testCase.uuid,
        fixture.envByUuid.get(testCase.uuid).map((entry) => (entry.key === testCase.key ? { ...entry, value: testCase.value } : entry))
      );
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, testCase.error);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("fails the runtime preflight when worker-mail lacks the exact hivegrid-web envelope key without leaking either value", async () => {
    const cases = [
      { uuid: webResourceUuid, mode: "missing" },
      { uuid: workerGroupResourceUuids.mail, mode: "missing" },
      { uuid: workerGroupResourceUuids.mail, mode: "mismatch" },
    ];

    for (const testCase of cases) {
      const fixture = runtimePreflightFixture();
      const mismatchedSecret = `mismatched-${sharedCryptoSecret}`;
      fixture.envByUuid.set(
        testCase.uuid,
        testCase.mode === "missing"
          ? fixture.envByUuid.get(testCase.uuid).filter(({ key }) => key !== "CRYPTO_SECRET")
          : fixture.envByUuid.get(testCase.uuid).map((entry) => (entry.key === "CRYPTO_SECRET" ? { ...entry, value: mismatchedSecret } : entry))
      );
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /hivegrid-web and worker-mail must share the same non-empty CRYPTO_SECRET/);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(sharedCryptoSecret));
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(mismatchedSecret));
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("fails the runtime preflight when the admin connection master-key boundary is missing, weak, mismatched, reused, or leaked to portal", async () => {
    const cases = [
      {
        uuid: webResourceUuid,
        mutate: (entries) => entries.filter(({ key }) => key !== "ADMIN_CONNECTION_MASTER_KEY"),
        error: /ADMIN_CONNECTION_MASTER_KEY with at least 32 bytes on: hivegrid-web/,
      },
      {
        uuid: apiResourceUuid,
        mutate: (entries) => entries.map((entry) => (entry.key === "ADMIN_CONNECTION_MASTER_KEY" ? { ...entry, value: "too-short" } : entry)),
        error: /ADMIN_CONNECTION_MASTER_KEY with at least 32 bytes on: hivegrid-api/,
      },
      {
        uuid: workerGroupResourceUuids.general,
        mutate: (entries) =>
          entries.map((entry) => (entry.key === "ADMIN_CONNECTION_MASTER_KEY" ? { ...entry, value: "mismatched-admin-connection-key-sentinel" } : entry)),
        error: /must share the same ADMIN_CONNECTION_MASTER_KEY/,
      },
      {
        uuid: webResourceUuid,
        mutate: (entries) => entries.map((entry) => (entry.key === "CRYPTO_SECRET" ? { ...entry, value: sharedAdminConnectionMasterKey } : entry)),
        error: /must be independent from SESSION_SECRET and CRYPTO_SECRET on: hivegrid-web/,
      },
      {
        uuid: portalResourceUuid,
        mutate: (entries) => [...entries, { key: "ADMIN_CONNECTION_MASTER_KEY", value: sharedAdminConnectionMasterKey }],
        error: /hivegrid-portal must not contain ADMIN_CONNECTION_MASTER_KEY/,
      },
    ];

    for (const testCase of cases) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(testCase.uuid, testCase.mutate(fixture.envByUuid.get(testCase.uuid)));
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, testCase.error);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(sharedAdminConnectionMasterKey));
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("fails the runtime preflight on encrypted GDPR export storage, key, or dedicated import-gate drift without leaking values", async () => {
    const cases = [
      {
        uuid: webResourceUuid,
        key: "STORAGE_ENDPOINT",
        mode: "missing",
        error: /hivegrid-web runtime is missing required encrypted GDPR export environment keys: STORAGE_ENDPOINT/,
      },
      {
        uuid: portalResourceUuid,
        key: "STORAGE_ENDPOINT",
        value: "http://storage.example.test",
        error: /hivegrid-portal runtime must set STORAGE_ENDPOINT to a valid HTTPS URL/,
      },
      {
        uuid: workerGroupResourceUuids.import,
        key: "GDPR_EXPORT_SSE_C_KEY_BASE64",
        value: "not-canonical-base64",
        error: /worker-import runtime must set GDPR_EXPORT_SSE_C_KEY_BASE64 to canonical base64 for exactly 32 bytes/,
      },
      {
        uuid: workerGroupResourceUuids.import,
        key: "GDPR_EXPORT_SSE_C_KEY_BASE64",
        value: alternateGdprExportSseCKey,
        error: /encrypted GDPR export runtime values must match after normalization for: GDPR_EXPORT_SSE_C_KEY_BASE64/,
      },
      {
        uuid: workerGroupResourceUuids.import,
        key: "IMPORT_QUEUE_ENABLED",
        value: "true",
        error: /worker-import must use idle mode, full mode.*or dedicated GDPR export mode/,
      },
      {
        uuid: workerGroupResourceUuids.import,
        key: "IMPORT_QUEUE_RUNTIME_READY",
        value: "true",
        error: /worker-import must use idle mode, full mode.*or dedicated GDPR export mode/,
      },
      {
        uuid: workerGroupResourceUuids.import,
        key: "PORTAL_GDPR_EXPORT_QUEUE_ENABLED",
        value: "banana",
        error: /worker-import PORTAL_GDPR_EXPORT_QUEUE_ENABLED must be exactly true or false when configured/,
        redactedValue: true,
      },
      {
        uuid: workerGroupResourceUuids.general,
        key: "EXPORT_RETENTION_QUEUE_ENABLED",
        value: "false",
        error: /worker-general must set EXPORT_RETENTION_QUEUE_ENABLED=true/,
      },
      {
        uuid: workerGroupResourceUuids.general,
        key: "MEILI_URL",
        mode: "missing",
        error: /worker-general runtime is missing required tenant hard-delete search environment keys: MEILI_URL/,
      },
      {
        uuid: workerGroupResourceUuids.general,
        key: "MEILI_API_KEY",
        mode: "missing",
        error: /worker-general runtime is missing required tenant hard-delete search environment keys: MEILI_API_KEY/,
      },
      {
        uuid: workerGroupResourceUuids.general,
        key: "STORAGE_BUCKET",
        value: "other-private-exports",
        error: /GDPR export-retention storage values must match.*STORAGE_BUCKET/,
      },
    ];

    for (const testCase of cases) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(
        testCase.uuid,
        testCase.mode === "missing"
          ? fixture.envByUuid.get(testCase.uuid).filter(({ key }) => key !== testCase.key)
          : fixture.envByUuid.get(testCase.uuid).map((entry) => (entry.key === testCase.key ? { ...entry, value: testCase.value } : entry))
      );
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, testCase.error);
        if (testCase.redactedValue) assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(testCase.value));
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("accepts full worker-import mode with encrypted GDPR export storage and an unset dedicated gate", async () => {
    const fixture = runtimePreflightFixture();
    fixture.envByUuid.set(
      workerGroupResourceUuids.import,
      fixture.envByUuid
        .get(workerGroupResourceUuids.import)
        .filter(({ key }) => key !== "PORTAL_GDPR_EXPORT_QUEUE_ENABLED")
        .map((entry) => (entry.key === "IMPORT_QUEUE_ENABLED" || entry.key === "IMPORT_QUEUE_RUNTIME_READY" ? { ...entry, value: "true" } : entry))
    );
    const server = await startServer((request) => ({
      payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
    }));
    try {
      const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /read-only runtime preflight verified/);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("accepts one shared storage credential across all GDPR export consumers without leaking it", async () => {
    const fixture = runtimePreflightFixture();
    for (const uuid of [portalResourceUuid, workerGroupResourceUuids.import, workerGroupResourceUuids.general]) {
      fixture.envByUuid.set(
        uuid,
        fixture.envByUuid.get(uuid).map((entry) => {
          if (entry.key === "STORAGE_ACCESS_KEY_ID") {
            return { ...entry, value: gdprExportStorageEnv.STORAGE_ACCESS_KEY_ID };
          }
          if (entry.key === "STORAGE_SECRET_ACCESS_KEY") {
            return { ...entry, value: gdprExportStorageEnv.STORAGE_SECRET_ACCESS_KEY };
          }
          return entry;
        })
      );
    }
    const server = await startServer((request) => ({
      payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
    }));
    try {
      const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /read-only runtime preflight verified/);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects an SSE-C read key on the delete-only general worker without leaking it", async () => {
    const fixture = runtimePreflightFixture();
    fixture.envByUuid.set(workerGroupResourceUuids.general, [
      ...fixture.envByUuid.get(workerGroupResourceUuids.general),
      { key: "GDPR_EXPORT_SSE_C_KEY_BASE64", value: alternateGdprExportSseCKey },
    ]);
    const server = await startServer((request) => ({
      payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
    }));
    try {
      const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /worker-general must not contain GDPR_EXPORT_SSE_C_KEY_BASE64 because it is delete-only/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("requires worker-import and worker-general to share a valid tenant-export barrier Redis target without leaking credentials", async () => {
    const cases = [
      {
        uuid: workerGroupResourceUuids.import,
        mutate: (entries) => entries.filter(({ key }) => key !== "BULLMQ_REDIS_URL" && key !== "REDIS_URL"),
        error: /worker-import runtime requires a valid REDIS_URL or BULLMQ_REDIS_URL/,
      },
      {
        uuid: workerGroupResourceUuids.general,
        mutate: (entries) =>
          entries.map((entry) => (entry.key === "BULLMQ_REDIS_URL" ? { ...entry, value: "https://general-redis-secret-sentinel.example.test/4" } : entry)),
        error: /worker-general runtime requires a valid REDIS_URL or BULLMQ_REDIS_URL/,
      },
      {
        uuid: workerGroupResourceUuids.import,
        mutate: (entries) =>
          entries.map((entry) =>
            entry.key === "BULLMQ_REDIS_URL" ? { ...entry, value: "rediss://import-drift-secret-sentinel@redis.example.test:6380/5" } : entry
          ),
        error: /worker-import and worker-general must use the same Redis endpoint and database/,
      },
    ];

    for (const testCase of cases) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(testCase.uuid, testCase.mutate(fixture.envByUuid.get(testCase.uuid)));
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, testCase.error);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /general-redis-secret|import-drift-secret/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("accepts idle worker-import mode without export storage while web surfaces retain the encrypted download contract", async () => {
    const fixture = runtimePreflightFixture();
    fixture.envByUuid.set(
      workerGroupResourceUuids.import,
      fixture.envByUuid
        .get(workerGroupResourceUuids.import)
        .filter(
          ({ key }) =>
            !GDPR_EXPORT_STORAGE_TEST_KEYS.has(key) &&
            key !== "IMPORT_QUEUE_ENABLED" &&
            key !== "IMPORT_QUEUE_RUNTIME_READY" &&
            key !== "PORTAL_GDPR_EXPORT_QUEUE_ENABLED"
        )
    );
    const server = await startServer((request) => ({
      payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
    }));
    try {
      const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /read-only runtime preflight verified/);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects normalized storage-target drift across encrypted GDPR export consumers", async () => {
    const cases = [
      { uuid: portalResourceUuid, key: "STORAGE_ENDPOINT", value: "https://other-storage.example.test", expectedKey: "STORAGE_ENDPOINT" },
      { uuid: workerGroupResourceUuids.import, key: "STORAGE_REGION", value: "eu-west-1", expectedKey: "STORAGE_REGION" },
      { uuid: webResourceUuid, key: "STORAGE_BUCKET", value: "other-private-exports", expectedKey: "STORAGE_BUCKET" },
      { uuid: portalResourceUuid, key: "STORAGE_FORCE_PATH_STYLE", value: "true", expectedKey: "STORAGE_FORCE_PATH_STYLE" },
    ];

    for (const testCase of cases) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(
        testCase.uuid,
        fixture.envByUuid.get(testCase.uuid).map((entry) => (entry.key === testCase.key ? { ...entry, value: testCase.value } : entry))
      );
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, new RegExp(`must match after normalization for: ${testCase.expectedKey}`));
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(testCase.value));
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects a portal host that aliases any canonical HTTPS core origin", async () => {
    for (const key of ["APP_BASE_DOMAIN", "WEB_APP_HOST", "ADMIN_APP_HOST", "API_HOST"]) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(
        webResourceUuid,
        fixture.envByUuid.get(webResourceUuid).map((entry) => (entry.key === key ? { ...entry, value: "https://PORTAL.EXAMPLE.TEST:443" } : entry))
      );
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1, `${key} :443 alias unexpectedly passed`);
        assert.match(result.stderr, /PORTAL_APP_HOST must be distinct from core APP_BASE_DOMAIN, WEB_APP_HOST, ADMIN_APP_HOST, and API_HOST/);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("includes the existing Web Push contract in the full read-only runtime preflight", async () => {
    const fixture = runtimePreflightFixture();
    fixture.envByUuid.set(
      workerGroupResourceUuids.webhook,
      fixture.envByUuid.get(workerGroupResourceUuids.webhook).filter(({ key }) => key !== "WEB_PUSH_VAPID_PRIVATE_KEY")
    );
    const server = await startServer((request) => ({
      payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
    }));
    try {
      const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /WEB_PUSH_VAPID_PRIVATE_KEY/);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails the runtime preflight when any application or worker loses the pool cap", async () => {
    for (const uuid of [apiResourceUuid, workerGroupResourceUuids.general]) {
      const fixture = runtimePreflightFixture();
      fixture.envByUuid.set(
        uuid,
        fixture.envByUuid.get(uuid).filter(({ key }) => key !== "DATABASE_POOL_MAX")
      );
      const server = await startServer((request) => ({
        payload: request.url === "/applications" ? fixture.apps : (fixture.envByUuid.get(request.url.split("/").at(-2)) ?? []),
      }));
      try {
        const result = await runRuntimePreflight(server.url, fixture.allowedCsv);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /explicit positive DATABASE_POOL_MAX/);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("accepts a complete shared Web Push runtime contract without emitting key material", async () => {
    const server = await startServer((request) => ({ payload: webPushEnvPayload(request.url) }));
    try {
      const result = await runHelper(server.url, "assert-web-push-env", webResourceUuid, workerResourceUuid);

      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /Web Push runtime environment verified/);
      assert.equal(result.stderr, "");
      assertWebPushValuesSafe(result);
      assert.deepEqual(
        new Set(server.requests.map(({ method, url }) => `${method} ${url}`)),
        new Set([`GET /applications/${webResourceUuid}/envs`, `GET /applications/${workerResourceUuid}/envs`])
      );
    } finally {
      await server.close();
    }
  });

  it("rejects every missing Web Push runtime key without emitting configured values", async () => {
    const cases = [
      { overrides: { web: [] }, expectedKey: "WEB_PUSH_VAPID_PUBLIC_KEY" },
      {
        overrides: {
          worker: [
            { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey },
            { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: vapidPrivateKey },
          ],
        },
        expectedKey: "WEB_PUSH_VAPID_SUBJECT",
      },
      {
        overrides: {
          worker: [
            { key: "WEB_PUSH_VAPID_SUBJECT", value: vapidSubject },
            { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: vapidPrivateKey },
          ],
        },
        expectedKey: "WEB_PUSH_VAPID_PUBLIC_KEY",
      },
      {
        overrides: {
          worker: [
            { key: "WEB_PUSH_VAPID_SUBJECT", value: vapidSubject },
            { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey },
          ],
        },
        expectedKey: "WEB_PUSH_VAPID_PRIVATE_KEY",
      },
    ];

    for (const { overrides, expectedKey } of cases) {
      const server = await startServer((request) => ({ payload: webPushEnvPayload(request.url, overrides) }));
      try {
        const result = await runHelper(server.url, "assert-web-push-env", webResourceUuid, workerResourceUuid);
        assert.equal(result.status, 1);
        assert.match(result.stderr, new RegExp(expectedKey));
        assertWebPushValuesSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects invalid Web Push subjects and keys without emitting configured values", async () => {
    const cases = [
      {
        worker: [
          { key: "WEB_PUSH_VAPID_SUBJECT", value: "https://user:password@example.test/#fragment" },
          { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey },
          { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: vapidPrivateKey },
        ],
        expectedKey: "WEB_PUSH_VAPID_SUBJECT",
      },
      { web: [{ key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: "not-base64url!" }], expectedKey: "WEB_PUSH_VAPID_PUBLIC_KEY" },
      {
        worker: [
          { key: "WEB_PUSH_VAPID_SUBJECT", value: vapidSubject },
          { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: Buffer.alloc(64, 1).toString("base64url") },
          { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: vapidPrivateKey },
        ],
        expectedKey: "WEB_PUSH_VAPID_PUBLIC_KEY",
      },
      {
        worker: [
          { key: "WEB_PUSH_VAPID_SUBJECT", value: vapidSubject },
          { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey },
          { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: Buffer.alloc(31, 2).toString("base64url") },
        ],
        expectedKey: "WEB_PUSH_VAPID_PRIVATE_KEY",
      },
    ];

    for (const { expectedKey, ...overrides } of cases) {
      const server = await startServer((request) => ({ payload: webPushEnvPayload(request.url, overrides) }));
      try {
        const result = await runHelper(server.url, "assert-web-push-env", webResourceUuid, workerResourceUuid);
        assert.equal(result.status, 1);
        assert.match(result.stderr, new RegExp(expectedKey));
        assertWebPushValuesSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects different valid web and webhook-worker public keys", async () => {
    const worker = [
      { key: "WEB_PUSH_VAPID_SUBJECT", value: vapidSubject },
      { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: alternateVapidPublicKey },
      { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: vapidPrivateKey },
    ];
    const server = await startServer((request) => ({ payload: webPushEnvPayload(request.url, { worker }) }));
    try {
      const result = await runHelper(server.url, "assert-web-push-env", webResourceUuid, workerResourceUuid);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /WEB_PUSH_VAPID_PUBLIC_KEY values must match/);
      assertWebPushValuesSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails closed on preview-only or duplicate Web Push runtime keys", async () => {
    const cases = [
      { web: [{ key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey, is_preview: true }] },
      {
        worker: [
          { key: "WEB_PUSH_VAPID_SUBJECT", value: vapidSubject },
          { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: vapidPublicKey },
          { key: "WEB_PUSH_VAPID_PUBLIC_KEY", value: alternateVapidPublicKey },
          { key: "WEB_PUSH_VAPID_PRIVATE_KEY", value: vapidPrivateKey },
        ],
      },
    ];

    for (const overrides of cases) {
      const server = await startServer((request) => ({ payload: webPushEnvPayload(request.url, overrides) }));
      try {
        const result = await runHelper(server.url, "assert-web-push-env", webResourceUuid, workerResourceUuid);
        assert.equal(result.status, 1);
        assertWebPushValuesSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects privileged env keys, disabled RLS, and a mismatched runtime DSN without leaking credentials", async () => {
    for (const payload of [
      [
        { key: "DATABASE_URL", value: "postgres://hivegrid_app:runtime-secret-sentinel@db.example.test:6432/hivegrid" },
        { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_app" },
        { key: "DATABASE_POOL_MAX", value: "10" },
        { key: "RLS_RUNTIME_MODE", value: "required" },
        { key: "WEB_RUNTIME_SURFACE", value: "core" },
        { key: "DIRECT_URL", value: "postgres://owner:owner-secret-sentinel@db.example.test/hivegrid" },
      ],
      [
        { key: "DATABASE_URL", value: "postgres://hivegrid_worker:runtime-secret-sentinel@db.example.test/hivegrid" },
        { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_app" },
        { key: "RLS_RUNTIME_MODE", value: "off" },
      ],
    ]) {
      const server = await startServer(() => ({ payload }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", webResourceUuid, "hivegrid_app", "web");
        assert.equal(result.status, 1);
        assert.doesNotMatch(result.stdout, /secret-sentinel/);
        assert.doesNotMatch(result.stderr, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("accepts a complete admin-only Authentik Admin JIT credential for the dedicated role", async () => {
    const server = await startServer(() => ({
      payload: [
        { key: "DATABASE_URL", value: "postgres://hivegrid_app:runtime-secret-sentinel@db.example.test:6432/hivegrid" },
        { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_app" },
        { key: "DATABASE_POOL_MAX", value: "10" },
        { key: "RLS_RUNTIME_MODE", value: "required" },
        { key: "WEB_RUNTIME_SURFACE", value: "admin" },
        { key: "SESSION_SECRET", value: "dedicated-admin-session-secret-sentinel" },
        { key: "ADMIN_FALLBACK_LOGIN_ENABLED", value: "false" },
        { key: "AUTHENTIK_ISSUER_URL", value: "https://auth.example.test/application/o/hivegrid-admin-panel/" },
        { key: "AUTHENTIK_ADMIN_CLIENT_ID", value: "hivegrid-admin-panel" },
        { key: "AUTHENTIK_ADMIN_CLIENT_SECRET", value: "oidc-secret-sentinel" },
        { key: "AUTHENTIK_ADMIN_REDIRECT_URI", value: "https://admin.example.test/auth/admin/callback" },
        {
          key: "AUTHENTIK_ADMIN_JIT_DATABASE_URL",
          value: "postgres://hivegrid_authentik_jit:jit-secret-sentinel@db.example.test/hivegrid",
        },
      ],
    }));
    try {
      const result = await runHelper(server.url, "assert-runtime-db-env", adminResourceUuid, "hivegrid_app", "admin");
      assert.equal(result.status, 0, result.stderr);
      assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects incomplete or wrongly authenticated admin JIT configuration without leaking credentials", async () => {
    for (const authentikEntries of [
      [{ key: "AUTHENTIK_ISSUER_URL", value: "https://auth.example.test/application/o/hivegrid-admin-panel/" }],
      [
        { key: "AUTHENTIK_ISSUER_URL", value: "https://auth.example.test/application/o/hivegrid-admin-panel/" },
        { key: "AUTHENTIK_ADMIN_CLIENT_ID", value: "hivegrid-admin-panel" },
        { key: "AUTHENTIK_ADMIN_CLIENT_SECRET", value: "oidc-secret-sentinel" },
        { key: "AUTHENTIK_ADMIN_REDIRECT_URI", value: "https://admin.example.test/auth/admin/callback" },
        {
          key: "AUTHENTIK_ADMIN_JIT_DATABASE_URL",
          value: "postgres://hivegrid_app:jit-secret-sentinel@db.example.test/hivegrid",
        },
      ],
    ]) {
      const server = await startServer(() => ({
        payload: [
          { key: "DATABASE_URL", value: "postgres://hivegrid_app:runtime-secret-sentinel@db.example.test/hivegrid" },
          { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_app" },
          { key: "DATABASE_POOL_MAX", value: "10" },
          { key: "RLS_RUNTIME_MODE", value: "required" },
          { key: "WEB_RUNTIME_SURFACE", value: "admin" },
          { key: "SESSION_SECRET", value: "dedicated-admin-session-secret-sentinel" },
          { key: "ADMIN_FALLBACK_LOGIN_ENABLED", value: "false" },
          ...authentikEntries,
        ],
      }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", adminResourceUuid, "hivegrid_app", "admin");
        assert.equal(result.status, 1);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects every Authentik Admin secret outside the admin resource", async () => {
    for (const authentikEntry of [
      { key: "AUTHENTIK_ADMIN_CLIENT_SECRET", value: "oidc-secret-sentinel" },
      {
        key: "AUTHENTIK_ADMIN_JIT_DATABASE_URL",
        value: "postgres://hivegrid_authentik_jit:jit-secret-sentinel@db.example.test/hivegrid",
      },
    ]) {
      const server = await startServer(() => ({
        payload: [
          { key: "DATABASE_URL", value: "postgres://hivegrid_worker:runtime-secret-sentinel@db.example.test/hivegrid" },
          { key: "DATABASE_RUNTIME_ROLE", value: "hivegrid_worker" },
          { key: "RLS_RUNTIME_MODE", value: "required" },
          authentikEntry,
        ],
      }));
      try {
        const result = await runHelper(server.url, "assert-runtime-db-env", workerResourceUuid, "hivegrid_worker", "service");
        assert.equal(result.status, 1);
        assert.match(result.stderr, /restricted to the admin resource/);
        assert.doesNotMatch(`${result.stdout}${result.stderr}`, /secret-sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("resolves exactly the named hivegrid-web, hivegrid-admin, and hivegrid-portal resources sharing the web image", async () => {
    const server = await startServer((request) => ({
      payload:
        request.url === "/applications"
          ? [
              {
                uuid: webResourceUuid,
                name: "hivegrid-web",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
              {
                uuid: adminResourceUuid,
                name: "hivegrid-admin",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
              {
                uuid: portalResourceUuid,
                name: "hivegrid-portal",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
              { uuid: otherResourceUuid, name: "hivegrid-api", docker_registry_image_name: apiImage },
            ]
          : [],
    }));
    try {
      const result = await runHelper(
        server.url,
        "resolve-web-surfaces",
        `${webResourceUuid},${adminResourceUuid},${portalResourceUuid},${otherResourceUuid}`,
        webImage
      );
      assert.equal(result.status, 0, result.stderr);
      assert.equal(result.stdout, `${webResourceUuid},${adminResourceUuid},${portalResourceUuid}`);
      assert.deepEqual(
        server.requests.map(({ method, url }) => `${method} ${url}`),
        [
          "GET /applications",
          `GET /applications/${webResourceUuid}/envs`,
          `GET /applications/${adminResourceUuid}/envs`,
          `GET /applications/${portalResourceUuid}/envs`,
        ]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects a Dockerfile web surface because digest pinning requires Coolify dockerimage mode", async () => {
    const server = await startServer((request) => ({
      payload:
        request.url === "/applications"
          ? [
              {
                uuid: webResourceUuid,
                name: "hivegrid-web",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
              {
                uuid: adminResourceUuid,
                name: "hivegrid-admin",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
              {
                uuid: portalResourceUuid,
                name: "hivegrid-portal",
                build_pack: "dockerfile",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
            ]
          : [],
    }));
    try {
      const result = await runHelper(server.url, "resolve-web-surfaces", `${webResourceUuid},${adminResourceUuid},${portalResourceUuid}`, webImage);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /must use Coolify dockerimage mode for OCI digest pinning/);
      assert.deepEqual(
        server.requests.map(({ method, url }) => `${method} ${url}`),
        ["GET /applications"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("requires independent positive CPU and unit-suffixed memory hard caps during web-trio inventory", async () => {
    const invalidLimits = [
      { limits_cpus: undefined },
      { limits_cpus: "0" },
      { limits_cpus: "Infinity" },
      { limits_memory: undefined },
      { limits_memory: "0" },
      { limits_memory: "0m" },
      { limits_memory: "512" },
      { limits_memory: ".5g" },
      { limits_memory: "1.5g" },
      { limits_memory: "512mb" },
    ];

    for (const target of ["core", "admin", "portal"]) {
      for (const invalid of invalidLimits) {
        const apps = [
          {
            uuid: webResourceUuid,
            name: "hivegrid-web",
            build_pack: "dockerimage",
            docker_registry_image_name: webImage,
            ...validWebSurfaceResourceLimits,
          },
          {
            uuid: adminResourceUuid,
            name: "hivegrid-admin",
            build_pack: "dockerimage",
            docker_registry_image_name: webImage,
            ...validWebSurfaceResourceLimits,
          },
          {
            uuid: portalResourceUuid,
            name: "hivegrid-portal",
            build_pack: "dockerimage",
            docker_registry_image_name: webImage,
            ...validWebSurfaceResourceLimits,
          },
        ];
        Object.assign(apps[target === "core" ? 0 : target === "admin" ? 1 : 2], invalid);
        const server = await startServer(() => ({ payload: apps }));
        try {
          const result = await runHelper(server.url, "resolve-web-surfaces", `${webResourceUuid},${adminResourceUuid},${portalResourceUuid}`, webImage);
          assert.equal(result.status, 1, `${target} unexpectedly accepted ${JSON.stringify(invalid)}`);
          assert.match(result.stderr, /positive limits_cpus and unit-suffixed limits_memory hard caps/);
          assert.deepEqual(
            server.requests.map(({ method, url }) => `${method} ${url}`),
            ["GET /applications"]
          );
        } finally {
          await server.close();
        }
      }
    }
  });

  it("rejects image-first, missing, extra, or revision-overridden web surface inventories", async () => {
    const cases = [
      [
        { uuid: webResourceUuid, name: "some-web", docker_registry_image_name: webImage },
        { uuid: portalResourceUuid, name: "hivegrid-portal", docker_registry_image_name: webImage },
      ],
      [{ uuid: webResourceUuid, name: "hivegrid-web", docker_registry_image_name: webImage }],
      [
        { uuid: webResourceUuid, name: "hivegrid-web", docker_registry_image_name: webImage },
        { uuid: portalResourceUuid, name: "hivegrid-portal", docker_registry_image_name: webImage },
        { uuid: otherResourceUuid, name: "hivegrid-web-copy", docker_registry_image_name: webImage },
      ],
    ];
    for (const apps of cases) {
      const server = await startServer((request) => ({ payload: request.url === "/applications" ? apps : [] }));
      try {
        const result = await runHelper(server.url, "resolve-web-surfaces", `${webResourceUuid},${portalResourceUuid},${otherResourceUuid}`, webImage);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /exactly the explicit hivegrid-web, hivegrid-admin, and hivegrid-portal/);
        assert.equal(server.requests.filter(({ method }) => method !== "GET").length, 0);
      } finally {
        await server.close();
      }
    }

    const overrideServer = await startServer((request) => ({
      payload:
        request.url === "/applications"
          ? [
              {
                uuid: webResourceUuid,
                name: "hivegrid-web",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
              {
                uuid: adminResourceUuid,
                name: "hivegrid-admin",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
              {
                uuid: portalResourceUuid,
                name: "hivegrid-portal",
                build_pack: "dockerimage",
                docker_registry_image_name: webImage,
                ...validWebSurfaceResourceLimits,
              },
            ]
          : request.url === `/applications/${portalResourceUuid}/envs`
            ? [{ key: "APP_REVISION", value: "" }]
            : [],
    }));
    try {
      const result = await runHelper(overrideServer.url, "resolve-web-surfaces", `${webResourceUuid},${adminResourceUuid},${portalResourceUuid}`, webImage);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /web surface APP_REVISION overrides/);
    } finally {
      await overrideServer.close();
    }
  });

  it("patches and rereads all three explicit web surfaces with one full SHA and digest", async () => {
    const apps = new Map([
      [
        webResourceUuid,
        {
          uuid: webResourceUuid,
          name: "hivegrid-web",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          docker_registry_image_tag: "e".repeat(40),
          ...validWebSurfaceResourceLimits,
        },
      ],
      [
        adminResourceUuid,
        {
          uuid: adminResourceUuid,
          name: "hivegrid-admin",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          docker_registry_image_tag: "e".repeat(40),
          ...validWebSurfaceResourceLimits,
        },
      ],
      [
        portalResourceUuid,
        {
          uuid: portalResourceUuid,
          name: "hivegrid-portal",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          docker_registry_image_tag: "e".repeat(40),
          ...validWebSurfaceResourceLimits,
        },
      ],
    ]);
    const server = await startServer(async (request) => {
      const uuid = request.url.split("/").at(-1);
      const app = apps.get(uuid);
      if (request.method === "PATCH") {
        Object.assign(app, JSON.parse(request.body));
      }
      return { payload: app };
    });
    try {
      const result = await runHelper(
        server.url,
        "set-web-surface-release",
        webResourceUuid,
        adminResourceUuid,
        portalResourceUuid,
        webImage,
        deployRevision,
        webDigest
      );
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, new RegExp(`${webImage}@${webDigest}`));
      const patches = server.requests.filter(({ method }) => method === "PATCH");
      assert.equal(patches.length, 3);
      for (const patchRequest of patches) {
        assert.deepEqual(JSON.parse(patchRequest.body), {
          pre_deployment_command: "",
          post_deployment_command: "",
          docker_registry_image_tag: `sha256-${"f".repeat(64)}`,
        });
      }
      assert.equal(apps.get(webResourceUuid).docker_registry_image_tag, `sha256-${"f".repeat(64)}`);
      assert.equal(apps.get(adminResourceUuid).docker_registry_image_tag, `sha256-${"f".repeat(64)}`);
      assert.equal(apps.get(portalResourceUuid).docker_registry_image_tag, `sha256-${"f".repeat(64)}`);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects missing hard caps when explicit web UUIDs are reread before release mutation", async () => {
    const apps = new Map([
      [
        webResourceUuid,
        {
          uuid: webResourceUuid,
          name: "hivegrid-web",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          ...validWebSurfaceResourceLimits,
        },
      ],
      [
        adminResourceUuid,
        {
          uuid: adminResourceUuid,
          name: "hivegrid-admin",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          ...validWebSurfaceResourceLimits,
        },
      ],
      [
        portalResourceUuid,
        {
          uuid: portalResourceUuid,
          name: "hivegrid-portal",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          limits_cpus: "1",
          limits_memory: "0",
        },
      ],
    ]);
    const server = await startServer((request) => ({ payload: apps.get(request.url.split("/").at(-1)) }));
    try {
      const result = await runHelper(
        server.url,
        "set-web-surface-release",
        webResourceUuid,
        adminResourceUuid,
        portalResourceUuid,
        webImage,
        deployRevision,
        webDigest
      );
      assert.equal(result.status, 1);
      assert.match(result.stderr, /positive limits_cpus and unit-suffixed limits_memory hard caps/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("proves all three completed web deployments are healthy and pinned to the same registry digest", async () => {
    const digestTag = `sha256-${"f".repeat(64)}`;
    const server = await startServer((request) => {
      if (request.url === `/applications/${webResourceUuid}`) {
        return {
          payload: {
            id: 101,
            uuid: webResourceUuid,
            name: "hivegrid-web",
            build_pack: "dockerimage",
            status: "running:healthy",
            docker_registry_image_name: webImage,
            docker_registry_image_tag: digestTag,
            ...validWebSurfaceResourceLimits,
          },
        };
      }
      if (request.url === `/applications/${adminResourceUuid}`) {
        return {
          payload: {
            id: 103,
            uuid: adminResourceUuid,
            name: "hivegrid-admin",
            build_pack: "dockerimage",
            status: "running:healthy",
            docker_registry_image_name: webImage,
            docker_registry_image_tag: digestTag,
            ...validWebSurfaceResourceLimits,
          },
        };
      }
      if (request.url === `/applications/${portalResourceUuid}`) {
        return {
          payload: {
            id: 102,
            uuid: portalResourceUuid,
            name: "hivegrid-portal",
            build_pack: "dockerimage",
            status: "running:healthy",
            docker_registry_image_name: webImage,
            docker_registry_image_tag: digestTag,
            ...validWebSurfaceResourceLimits,
          },
        };
      }
      if (request.url === `/deployments/${deploymentUuid}`) {
        return {
          payload: {
            application_id: "101",
            application_name: "hivegrid-web",
            deployment_uuid: deploymentUuid,
            status: "finished",
          },
        };
      }
      if (request.url === `/deployments/${otherDeploymentUuid}`) {
        return {
          payload: {
            application_id: "102",
            application_name: "hivegrid-portal",
            deployment_uuid: otherDeploymentUuid,
            status: "finished",
          },
        };
      }
      if (request.url === `/deployments/${adminDeploymentUuid}`) {
        return {
          payload: {
            application_id: "103",
            application_name: "hivegrid-admin",
            deployment_uuid: adminDeploymentUuid,
            status: "finished",
          },
        };
      }
      return { status: 404, payload: {} };
    });
    try {
      const result = await runHelper(
        server.url,
        "assert-web-surface-release",
        webResourceUuid,
        adminResourceUuid,
        portalResourceUuid,
        webImage,
        deployRevision,
        webDigest,
        deploymentUuid,
        adminDeploymentUuid,
        otherDeploymentUuid
      );
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, new RegExp(`${webImage}:${deployRevision} at ${webImage}@${webDigest}`));
      const requests = server.requests.map(({ method, url }) => `${method} ${url}`);
      assert.deepEqual(
        requests.slice(0, 3).sort(),
        [`GET /applications/${webResourceUuid}`, `GET /applications/${adminResourceUuid}`, `GET /applications/${portalResourceUuid}`].sort()
      );
      assert.deepEqual(
        requests.slice(3).sort(),
        [`GET /deployments/${deploymentUuid}`, `GET /deployments/${adminDeploymentUuid}`, `GET /deployments/${otherDeploymentUuid}`].sort()
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects digest drift and a deployment that is not bound to the expected surface", async () => {
    for (const drift of ["digest", "deployment"]) {
      const digestTag = `sha256-${"f".repeat(64)}`;
      const server = await startServer((request) => {
        const isPortal = request.url === `/applications/${portalResourceUuid}`;
        const isAdmin = request.url === `/applications/${adminResourceUuid}`;
        if (request.url === `/applications/${webResourceUuid}` || isAdmin || isPortal) {
          return {
            payload: {
              id: isPortal ? 102 : isAdmin ? 103 : 101,
              uuid: isPortal ? portalResourceUuid : isAdmin ? adminResourceUuid : webResourceUuid,
              name: isPortal ? "hivegrid-portal" : isAdmin ? "hivegrid-admin" : "hivegrid-web",
              build_pack: "dockerimage",
              status: "running:healthy",
              docker_registry_image_name: webImage,
              docker_registry_image_tag: drift === "digest" && isPortal ? `sha256-${"e".repeat(64)}` : digestTag,
              ...validWebSurfaceResourceLimits,
            },
          };
        }
        const isPortalDeployment = request.url === `/deployments/${otherDeploymentUuid}`;
        const isAdminDeployment = request.url === `/deployments/${adminDeploymentUuid}`;
        return {
          payload: {
            application_id: isPortalDeployment ? "102" : isAdminDeployment ? "103" : "101",
            application_name: drift === "deployment" && isPortalDeployment ? "hivegrid-web" : isPortalDeployment ? "hivegrid-portal" : "hivegrid-web",
            ...(isAdminDeployment ? { application_name: "hivegrid-admin" } : {}),
            deployment_uuid: isPortalDeployment ? otherDeploymentUuid : isAdminDeployment ? adminDeploymentUuid : deploymentUuid,
            status: "finished",
          },
        };
      });
      try {
        const result = await runHelper(
          server.url,
          "assert-web-surface-release",
          webResourceUuid,
          adminResourceUuid,
          portalResourceUuid,
          webImage,
          deployRevision,
          webDigest,
          deploymentUuid,
          adminDeploymentUuid,
          otherDeploymentUuid
        );
        assert.equal(result.status, 1);
        assert.match(result.stderr, drift === "digest" ? /not pinned to the expected registry digest/ : /expected named application/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("compensates the first web surface when the paired release mutation cannot update the second", async () => {
    const previousRevision = "e".repeat(40);
    const apps = new Map([
      [
        webResourceUuid,
        {
          uuid: webResourceUuid,
          name: "hivegrid-web",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          docker_registry_image_tag: previousRevision,
          ...validWebSurfaceResourceLimits,
          pre_deployment_command: "old-pre",
          post_deployment_command: "old-post",
        },
      ],
      [
        adminResourceUuid,
        {
          uuid: adminResourceUuid,
          name: "hivegrid-admin",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          docker_registry_image_tag: previousRevision,
          ...validWebSurfaceResourceLimits,
        },
      ],
      [
        portalResourceUuid,
        {
          uuid: portalResourceUuid,
          name: "hivegrid-portal",
          build_pack: "dockerimage",
          docker_registry_image_name: webImage,
          docker_registry_image_tag: previousRevision,
          ...validWebSurfaceResourceLimits,
        },
      ],
    ]);
    let portalPatchFailed = false;
    const server = await startServer((request) => {
      const uuid = request.url.split("/").at(-1);
      const app = apps.get(uuid);
      if (request.method === "PATCH" && uuid === portalResourceUuid && !portalPatchFailed) {
        portalPatchFailed = true;
        return { status: 500, payload: { message: "sentinel must stay redacted" } };
      }
      if (request.method === "PATCH") Object.assign(app, JSON.parse(request.body));
      return { payload: app };
    });
    try {
      const result = await runHelper(
        server.url,
        "set-web-surface-release",
        webResourceUuid,
        adminResourceUuid,
        portalResourceUuid,
        webImage,
        deployRevision,
        webDigest
      );
      assert.equal(result.status, 1);
      assert.match(result.stderr, /previous resource configuration was restored/);
      assert.equal(apps.get(webResourceUuid).docker_registry_image_tag, previousRevision);
      assert.equal(apps.get(webResourceUuid).pre_deployment_command, "old-pre");
      assert.equal(apps.get(webResourceUuid).post_deployment_command, "old-post");
      assert.equal(apps.get(adminResourceUuid).docker_registry_image_tag, previousRevision);
      assert.equal(server.requests.filter(({ method }) => method === "PATCH").length, 5);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("validates the exact allowlisted web, API, and MCP image inventory before mutations", async () => {
    const server = await startServer((request) => ({
      payload:
        request.url === "/applications"
          ? [
              { uuid: resourceUuid, docker_registry_image_name: webImage },
              { uuid: otherResourceUuid, docker_registry_image_name: apiImage },
              { uuid: mcpResourceUuid, docker_registry_image_name: mcpImage },
              { uuid: webResourceUuid, docker_registry_image_name: mcpImage },
            ]
          : [],
    }));
    try {
      const result = await runHelper(
        server.url,
        "validate-required-images",
        `${resourceUuid},${otherResourceUuid},${mcpResourceUuid}`,
        webImage,
        apiImage,
        mcpImage
      );
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /inventory validated/);
      assert.deepEqual(
        server.requests.map(({ method, url }) => ({ method, url })),
        [
          { method: "GET", url: "/applications" },
          { method: "GET", url: `/applications/${resourceUuid}/envs` },
          { method: "GET", url: `/applications/${otherResourceUuid}/envs` },
          { method: "GET", url: `/applications/${mcpResourceUuid}/envs` },
        ]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects core APP_REVISION overrides during the read-only inventory preflight", async () => {
    const server = await startServer((request) => ({
      payload:
        request.url === "/applications"
          ? [
              { uuid: resourceUuid, docker_registry_image_name: webImage },
              { uuid: otherResourceUuid, docker_registry_image_name: apiImage },
              { uuid: mcpResourceUuid, docker_registry_image_name: mcpImage },
            ]
          : request.url === `/applications/${mcpResourceUuid}/envs`
            ? [{ key: "APP_REVISION", value: "" }]
            : [],
    }));
    try {
      const result = await runHelper(
        server.url,
        "validate-required-images",
        `${resourceUuid},${otherResourceUuid},${mcpResourceUuid}`,
        webImage,
        apiImage,
        mcpImage
      );
      assert.equal(result.status, 1);
      assert.match(result.stderr, /core application APP_REVISION overrides must be removed/);
      assert.equal(server.requests.filter(({ method }) => method !== "GET").length, 0);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails required image inventory validation when a target is missing or ambiguous", async () => {
    const server = await startServer(() => ({
      payload: [
        { uuid: resourceUuid, docker_registry_image_name: webImage },
        { uuid: otherResourceUuid, docker_registry_image_name: apiImage },
        { uuid: mcpResourceUuid, docker_registry_image_name: apiImage },
      ],
    }));
    try {
      const result = await runHelper(
        server.url,
        "validate-required-images",
        `${resourceUuid},${otherResourceUuid},${mcpResourceUuid}`,
        webImage,
        apiImage,
        mcpImage
      );
      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /inventory is missing, ambiguous, or invalid/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  for (const [order, entries] of [
    [
      "preview-first",
      [
        { key: "MIGRATE_ON_BOOT", value: "true", is_preview: true },
        { key: "MIGRATE_ON_BOOT", value: "false", is_preview: false },
      ],
    ],
    [
      "runtime-first",
      [
        { key: "MIGRATE_ON_BOOT", value: "false", is_preview: false },
        { key: "MIGRATE_ON_BOOT", value: "true", is_preview: true },
      ],
    ],
  ]) {
    it(`updates the non-preview runtime env independent of ${order} Coolify ordering`, async () => {
      const server = await startServer((request) => ({ payload: request.method === "GET" ? entries : {} }));
      try {
        const result = await runHelper(server.url, "set-envs", webResourceUuid, "MIGRATE_ON_BOOT=true");
        assert.equal(result.status, 0, result.stderr);
        assert.match(result.stdout, /env MIGRATE_ON_BOOT updated/);
        assert.deepEqual(
          server.requests.map(({ method }) => method),
          ["GET", "PATCH"]
        );
        assert.deepEqual(JSON.parse(server.requests[1].body), {
          key: "MIGRATE_ON_BOOT",
          value: "true",
          is_literal: true,
          is_preview: false,
          is_multiline: false,
          is_shown_once: false,
        });
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    });
  }

  it("rejects duplicate non-preview env keys before set-envs mutation", async () => {
    const server = await startServer(() => ({
      payload: [
        { key: "MIGRATE_ON_BOOT", value: "false", is_preview: false },
        { key: "MIGRATE_ON_BOOT", value: "true", is_preview: false },
      ],
    }));
    try {
      const result = await runHelper(server.url, "set-envs", webResourceUuid, "MIGRATE_ON_BOOT=true");
      assert.equal(result.status, 1);
      assert.match(result.stderr, /duplicate non-preview keys/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("configures MCP command health and verifies the persisted state", async () => {
    const app = {
      uuid: mcpResourceUuid,
      docker_registry_image_name: mcpImage,
      docker_registry_image_tag: "previous",
      health_check_enabled: false,
      custom_healthcheck_found: false,
    };
    const server = await startServer((request) => {
      if (request.method === "GET" && request.url.endsWith("/envs")) {
        return { payload: [] };
      }
      if (request.method === "GET") {
        return { payload: app };
      }
      if (request.method === "PATCH") {
        Object.assign(app, JSON.parse(request.body));
        return { payload: app };
      }
      return { status: 405, payload: { message: "method not allowed" } };
    });

    try {
      const result = await runHelper(server.url, "configure-mcp-healthcheck", mcpResourceUuid, deployRevision);

      assert.equal(result.status, 0, result.stderr);
      assert.equal(result.stderr, "");
      assertSecretSafe(result);
      assert.deepEqual(server.serverErrors, []);
      assert.equal(server.requests.length, 4);
      assert.deepEqual(
        server.requests.map(({ method, url, authorization }) => ({ method, url, authorization })),
        [
          { method: "GET", url: `/applications/${mcpResourceUuid}/envs`, authorization: `Bearer ${bearerToken}` },
          { method: "GET", url: `/applications/${mcpResourceUuid}`, authorization: `Bearer ${bearerToken}` },
          { method: "PATCH", url: `/applications/${mcpResourceUuid}`, authorization: `Bearer ${bearerToken}` },
          { method: "GET", url: `/applications/${mcpResourceUuid}`, authorization: `Bearer ${bearerToken}` },
        ]
      );
      assert.equal(server.requests[0].body, "");
      assert.equal(server.requests[1].body, "");
      assert.deepEqual(JSON.parse(server.requests[2].body), expectedMcpHealthConfig());
      assert.equal(server.requests[3].body, "");
    } finally {
      await server.close();
    }
  });

  it("does not patch MCP health config that already matches the immutable revision", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? []
        : { uuid: mcpResourceUuid, docker_registry_image_name: mcpImage, ...expectedMcpHealthConfig(), custom_healthcheck_found: false },
    }));

    try {
      const result = await runHelper(server.url, "configure-mcp-healthcheck", mcpResourceUuid, deployRevision);

      assert.equal(result.status, 0, result.stderr);
      assert.equal(result.stderr, "");
      assertSecretSafe(result);
      assert.deepEqual(server.serverErrors, []);
      assert.deepEqual(server.requests, [
        {
          method: "GET",
          url: `/applications/${mcpResourceUuid}/envs`,
          authorization: `Bearer ${bearerToken}`,
          body: "",
        },
        {
          method: "GET",
          url: `/applications/${mcpResourceUuid}`,
          authorization: `Bearer ${bearerToken}`,
          body: "",
        },
      ]);
    } finally {
      await server.close();
    }
  });

  it("fails when MCP command health config is not persisted after the patch", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? []
        : {
            uuid: mcpResourceUuid,
            docker_registry_image_name: mcpImage,
            docker_registry_image_tag: "previous",
            health_check_enabled: false,
            custom_healthcheck_found: false,
          },
    }));

    try {
      const result = await runHelper(server.url, "configure-mcp-healthcheck", mcpResourceUuid, deployRevision);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /configuration was not persisted/);
      assertSecretSafe(result);
      assert.deepEqual(server.serverErrors, []);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET", "PATCH", "GET"]
      );
    } finally {
      await server.close();
    }
  });

  it("rejects MCP APP_REVISION overrides before mutating application config", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs") ? [{ key: "APP_REVISION", real_value: "" }] : {},
    }));

    try {
      const result = await runHelper(server.url, "configure-mcp-healthcheck", mcpResourceUuid, deployRevision);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /APP_REVISION override must be removed/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects malformed MCP environment responses before mutating application config", async () => {
    const server = await startServer(() => ({ payload: { unexpected: [] } }));

    try {
      const result = await runHelper(server.url, "configure-mcp-healthcheck", mcpResourceUuid, deployRevision);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /environment response was malformed/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects MCP images whose built-in healthcheck would override command health", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs") ? [] : { uuid: mcpResourceUuid, docker_registry_image_name: mcpImage, custom_healthcheck_found: true },
    }));

    try {
      const result = await runHelper(server.url, "configure-mcp-healthcheck", mcpResourceUuid, deployRevision);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /image healthcheck must be absent/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects an MCP CUID that resolves to a different image before mutation", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs") ? [] : { uuid: mcpResourceUuid, docker_registry_image_name: webImage, custom_healthcheck_found: false },
    }));
    try {
      const result = await runHelperWithEnv(server.url, { MCP_IMAGE: mcpImage }, "configure-mcp-healthcheck", mcpResourceUuid, deployRevision);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /does not use the expected MCP image/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects MCP targeting when no exact expected image identity is configured", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs") ? [] : { uuid: mcpResourceUuid, docker_registry_image_name: mcpImage, custom_healthcheck_found: false },
    }));
    try {
      const result = await runHelperWithEnv(
        server.url,
        { MCP_IMAGE: "", WORKER_IMAGE: "", REGISTRY: "", IMAGE_NAMESPACE: "", IMAGE_PREFIX: "" },
        "configure-mcp-healthcheck",
        mcpResourceUuid,
        deployRevision
      );
      assert.equal(result.status, 1);
      assert.match(result.stderr, /does not use the expected MCP image/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects a mutable MCP health revision before calling Coolify", async () => {
    const server = await startServer(() => ({ payload: {} }));

    try {
      const result = await runHelper(server.url, "configure-mcp-healthcheck", mcpResourceUuid, "staging");

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /full lowercase Git SHA/);
      assert.deepEqual(server.requests, []);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("atomically configures and rereads degraded legacy rollback health", async () => {
    const app = {
      uuid: workerResourceUuid,
      name: "legacy-worker",
      docker_registry_image_name: workerImage,
      docker_registry_image_tag: "newer",
      health_check_enabled: true,
      health_check_command: "new-command",
    };
    const server = await startServer((request) => {
      if (request.method === "GET" && request.url.endsWith("/envs")) {
        return { payload: [{ key: "WORKER_QUEUE_GROUP", real_value: "'mail'" }] };
      }
      if (request.method === "PATCH") {
        Object.assign(app, JSON.parse(request.body));
        return { payload: app };
      }
      if (request.method === "GET") {
        return { payload: app };
      }
      return { status: 405, payload: {} };
    });

    try {
      const result = await runHelper(server.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision, "mail");

      assert.equal(result.status, 0, result.stderr);
      assert.equal(result.stderr, "");
      assert.match(result.stdout, /Legacy rollback image health configured/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET", "PATCH", "GET"]
      );
      assert.deepEqual(JSON.parse(server.requests[2].body), {
        docker_registry_image_tag: deployRevision,
        health_check_enabled: false,
        health_check_command: "",
      });
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  for (const [order, envs] of [
    [
      "preview-first",
      [
        { key: "WORKER_QUEUE_GROUP", value: "general", is_preview: true },
        { key: "WORKER_QUEUE_GROUP", value: "mail", is_preview: false },
      ],
    ],
    [
      "runtime-first",
      [
        { key: "WORKER_QUEUE_GROUP", value: "mail", is_preview: false },
        { key: "WORKER_QUEUE_GROUP", value: "general", is_preview: true },
      ],
    ],
  ]) {
    it(`uses the non-preview worker group for legacy rollback with ${order} env ordering`, async () => {
      const app = {
        uuid: workerResourceUuid,
        docker_registry_image_name: workerImage,
        docker_registry_image_tag: "newer",
        health_check_enabled: true,
        health_check_command: "new-command",
      };
      const server = await startServer((request) => {
        if (request.url.endsWith("/envs")) return { payload: envs };
        if (request.method === "PATCH") {
          Object.assign(app, JSON.parse(request.body));
          return { payload: app };
        }
        return { payload: app };
      });
      try {
        const result = await runHelper(server.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision, "mail");
        assert.equal(result.status, 0, result.stderr);
        assert.deepEqual(
          server.requests.map(({ method }) => method),
          ["GET", "GET", "PATCH", "GET"]
        );
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    });
  }

  it("does not let a preview worker group authorize legacy runtime rollback", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? [
            { key: "WORKER_QUEUE_GROUP", value: "mail", is_preview: true },
            { key: "WORKER_QUEUE_GROUP", value: "general", is_preview: false },
          ]
        : { docker_registry_image_name: workerImage },
    }));
    try {
      const result = await runHelper(server.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision, "mail");
      assert.equal(result.status, 1);
      assert.match(result.stderr, /worker group does not match/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails closed when legacy rollback health config is not persisted", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? []
        : {
            name: "legacy-mcp",
            docker_registry_image_name: mcpImage,
            docker_registry_image_tag: "newer",
            health_check_enabled: true,
            health_check_command: "new-command",
          },
    }));

    try {
      const result = await runHelper(server.url, "configure-legacy-rollback-healthcheck", legacyMcpResourceUuid, deployRevision);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /configuration was not persisted/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET", "PATCH", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects legacy rollback revision overrides and worker-group drift before mutation", async () => {
    const overrideServer = await startServer((request) => ({
      payload: request.url.endsWith("/envs") ? [{ key: "APP_REVISION", value: "" }] : {},
    }));
    try {
      const result = await runHelper(overrideServer.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision, "mail");
      assert.equal(result.status, 1);
      assert.match(result.stderr, /APP_REVISION override must be removed/);
      assert.deepEqual(
        overrideServer.requests.map(({ method }) => method),
        ["GET"]
      );
      assertSecretSafe(result);
    } finally {
      await overrideServer.close();
    }

    const groupServer = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? [{ key: "WORKER_QUEUE_GROUP", value: "general" }]
        : { name: "legacy-worker", docker_registry_image_name: workerImage },
    }));
    try {
      const result = await runHelper(groupServer.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision, "mail");
      assert.equal(result.status, 1);
      assert.match(result.stderr, /worker group does not match/);
      assert.deepEqual(
        groupServer.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await groupServer.close();
    }
  });

  it("requires a group for legacy workers and rejects non-MCP/non-worker resources", async () => {
    const workerServer = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? [{ key: "WORKER_QUEUE_GROUP", value: "mail" }]
        : { name: "legacy-worker", docker_registry_image_name: workerImage },
    }));
    try {
      const result = await runHelper(workerServer.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /requires the exact worker group/);
      assert.deepEqual(
        workerServer.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await workerServer.close();
    }

    const webServer = await startServer((request) => ({
      payload: request.url.endsWith("/envs") ? [] : { name: "legacy-worker-mcp", description: "worker mcp", docker_registry_image_name: webImage },
    }));
    try {
      const result = await runHelper(webServer.url, "configure-legacy-rollback-healthcheck", webResourceUuid, deployRevision);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /restricted to MCP and worker resources/);
      assert.deepEqual(
        webServer.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await webServer.close();
    }
  });

  it("does not infer a legacy rollback target from spoofed names or descriptions", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? [{ key: "WORKER_QUEUE_GROUP", value: "mail" }]
        : { name: "worker-mail", description: "legacy mcp worker", docker_registry_image_name: webImage },
    }));
    try {
      const result = await runHelper(server.url, "configure-legacy-rollback-healthcheck", webResourceUuid, deployRevision, "mail");
      assert.equal(result.status, 1);
      assert.match(result.stderr, /restricted to MCP and worker resources/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects a different product's worker image before legacy rollback mutation", async () => {
    const server = await startServer((request) => ({
      payload: request.url.endsWith("/envs")
        ? [{ key: "WORKER_QUEUE_GROUP", value: "mail" }]
        : { name: "worker-mail", docker_registry_image_name: "registry.example.test/other-product/worker" },
    }));
    try {
      const result = await runHelper(server.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision, "mail");
      assert.equal(result.status, 1);
      assert.match(result.stderr, /restricted to MCP and worker resources/);
      assert.deepEqual(
        server.requests.map(({ method }) => method),
        ["GET", "GET"]
      );
      assert.equal(server.requests.filter(({ method }) => method === "PATCH").length, 0);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects mutable legacy rollback revisions and invalid worker groups without calling Coolify", async () => {
    const server = await startServer(() => ({ payload: {} }));

    try {
      const mutableResult = await runHelper(server.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, "main", "mail");
      assert.equal(mutableResult.status, 1);
      assert.match(mutableResult.stderr, /full lowercase Git SHA/);

      const groupResult = await runHelper(server.url, "configure-legacy-rollback-healthcheck", workerResourceUuid, deployRevision, "all");
      assert.equal(groupResult.status, 1);
      assert.match(groupResult.stderr, /valid worker group/);
      assert.deepEqual(server.requests, []);
      assertSecretSafe(mutableResult);
      assertSecretSafe(groupResult);
    } finally {
      await server.close();
    }
  });

  it("fails immediately for Coolify cancelled-by-user deployments", async () => {
    const server = await startServer(() => ({ payload: { status: "cancelled-by-user" } }));

    try {
      const result = await runHelper(server.url, "wait", deploymentUuid, "dependent");

      assert.equal(result.status, 1);
      assert.match(result.stdout, /cancelled-by-user/);
      assert.match(result.stderr, /deployment failed/);
      assert.equal(server.requests.length, 1);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("redacts API response bodies when Coolify returns an error", async () => {
    const responseSentinel = "sensitive-coolify-response-sentinel";
    const server = await startServer(() => ({
      status: 401,
      payload: { message: responseSentinel },
    }));

    try {
      const result = await runHelper(server.url, "trigger", resourceUuid);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /HTTP 401/);
      assert.doesNotMatch(result.stderr, new RegExp(responseSentinel));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("bounds and redacts stalled Coolify API requests", async () => {
    const server = await startServer(async () => {
      await new Promise((resolve) => setTimeout(resolve, 100));
      return { payload: { deployments: [] } };
    });

    try {
      const result = await runHelperWithEnv(server.url, { COOLIFY_API_TIMEOUT_MS: "10" }, "trigger", resourceUuid);

      assert.equal(result.status, 1);
      assert.equal(result.stdout, "");
      assert.match(result.stderr, /failed or timed out/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });
});