HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/scripts/coolify-portal-domain-controller.test.mjs

// @vitest-environment node
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import http from "node:http";
import path from "node:path";
import { describe, it } from "vitest";

const scriptPath = path.resolve("scripts/coolify-portal-domain-controller.mjs");
const bearerToken = "controller-coolify-token-sentinel";
const controllerUuid = "c".repeat(24);
const workerGeneralUuid = "g".repeat(24);
const workerMailUuid = "m".repeat(24);
const webCoreUuid = "w".repeat(24);
const webPortalUuid = "p".repeat(24);
const deploymentUuid = "d".repeat(24);
const revision = "a".repeat(40);
const controllerImage = "registry.example.test/toolctrl-labs/hivegrid/portal-domain-controller";
const workerImage = "registry.example.test/toolctrl-labs/hivegrid/worker";
const webImage = "registry.example.test/toolctrl-labs/hivegrid/web";
const commandSecret = "controller-command-secret-sentinel-value";
const otherCommandSecret = "different-controller-command-secret-value";
const metricsToken = "controller-metrics-token-sentinel-value";
const configDirectory = "/data/coolify/proxy/dynamic/hivegrid-portal-domains";
const reloadTriggerFile = "/data/coolify/proxy/dynamic/.hivegrid-portal-domains.reload";
const stagingCname = "staging-domains.portal.hivegrid.de";
const productionCname = "domains.portal.hivegrid.de";
const stagingMetricsBinding = "10.42.0.3:19120:8080";
const productionMetricsBinding = "10.42.0.8:19120:8080";

function expectedControllerHealthConfig(expectedRevision = revision) {
  return {
    health_check_enabled: true,
    health_check_type: "cmd",
    health_check_command: `node apps/portal-domain-controller/dist/deploy-health.js ${expectedRevision}`,
    health_check_interval: 30,
    health_check_timeout: 10,
    health_check_retries: 3,
    health_check_start_period: 20,
  };
}

function controllerEnv(overrides = {}) {
  return envEntries({
    PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: commandSecret,
    PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: metricsToken,
    PORTAL_DOMAIN_CONTROLLER_REDIS_URL: "rediss://controller:secret@redis.example.test:6380/7",
    PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: stagingCname,
    PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: `https-0-${webPortalUuid}@docker`,
    PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "https",
    PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: configDirectory,
    PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: reloadTriggerFile,
    ...overrides,
  });
}

function workerEnv(group, overrides = {}) {
  return envEntries({
    WORKER_QUEUE_GROUP: group,
    BULLMQ_REDIS_URL: "rediss://worker:other-secret@redis.example.test:6380/7",
    ...(group === "general"
      ? {
          PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: commandSecret,
          PORTAL_DOMAIN_ORCHESTRATION_REQUIRED: "true",
        }
      : {}),
    ...overrides,
  });
}

function envEntries(values) {
  return Object.entries(values)
    .filter(([, value]) => value !== undefined)
    .map(([key, value]) => ({ key, value, is_preview: false }));
}

function exactStorages() {
  return {
    file_storages: [
      {
        uuid: "storage-config",
        is_directory: true,
        fs_path: configDirectory,
        mount_path: configDirectory,
      },
    ],
    persistent_storages: [
      {
        uuid: "storage-reload",
        host_path: reloadTriggerFile,
        mount_path: reloadTriggerFile,
      },
    ],
  };
}

function fixture(overrides = {}) {
  const controller = {
    id: 101,
    uuid: controllerUuid,
    name: "hivegrid-portal-domain-controller",
    build_pack: "dockerimage",
    docker_registry_image_name: controllerImage,
    docker_registry_image_tag: "previous",
    pre_deployment_command: "",
    post_deployment_command: "",
    fqdn: "",
    ports_mappings: stagingMetricsBinding,
    status: "stopped",
    custom_healthcheck_found: false,
    health_check_enabled: true,
    health_check_type: "http",
    health_check_command: "",
    health_check_interval: 30,
    health_check_timeout: 10,
    health_check_retries: 3,
    health_check_start_period: 20,
    ...overrides.controller,
  };
  const workers = [
    { uuid: workerGeneralUuid, name: "worker-general", docker_registry_image_name: workerImage },
    { uuid: workerMailUuid, name: "worker-mail", docker_registry_image_name: workerImage },
  ];
  const webCore = {
    uuid: webCoreUuid,
    name: "hivegrid-web",
    build_pack: "dockerimage",
    docker_registry_image_name: webImage,
  };
  const webPortal = {
    uuid: webPortalUuid,
    name: "hivegrid-portal",
    build_pack: "dockerimage",
    docker_registry_image_name: webImage,
  };
  return {
    controller,
    apps: [controller, webCore, webPortal, ...workers],
    envByUuid: new Map([
      [controllerUuid, controllerEnv(overrides.controllerEnv)],
      [
        webCoreUuid,
        envEntries({
          WEB_RUNTIME_SURFACE: "core",
          PORTAL_CUSTOM_DOMAIN_CNAME_TARGET: overrides.webCnameTarget ?? stagingCname,
          PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: overrides.webCoreRuntimeEnabled ?? "true",
          ...overrides.webCoreEnv,
        }),
      ],
      [
        webPortalUuid,
        envEntries({
          WEB_RUNTIME_SURFACE: "portal",
          PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: overrides.webPortalRuntimeEnabled ?? "true",
          ...overrides.webPortalEnv,
        }),
      ],
      [workerGeneralUuid, workerEnv("general", overrides.workerGeneralEnv)],
      [workerMailUuid, workerEnv("mail")],
    ]),
    storages: overrides.storages ?? exactStorages(),
    stopDoesNotTransition: overrides.stopDoesNotTransition ?? false,
    controllerProcessRunning: overrides.controllerProcessRunning ?? true,
    controllerHealthRevision: overrides.controllerHealthRevision ?? controller.docker_registry_image_tag,
    postStopHealthSequence: overrides.postStopHealthSequence ? [...overrides.postStopHealthSequence] : null,
    stopRequested: false,
    releaseMutationDrifts: overrides.releaseMutationDrifts ?? false,
    compensationDoesNotRestore: overrides.compensationDoesNotRestore ?? false,
    releasePatchCount: 0,
    statusSequence: overrides.statusSequence ?? null,
    allowedCsv: [controllerUuid, webCoreUuid, webPortalUuid, workerGeneralUuid, workerMailUuid].join(","),
  };
}

function startFixtureServer(state) {
  return startServer((request) => {
    if (request.url === "/controller-healthz") {
      const reachable = state.stopRequested && state.postStopHealthSequence?.length
        ? state.postStopHealthSequence.shift()
        : state.controllerProcessRunning;
      if (!reachable) return { destroy: true };
      return {
        payload: {
          status: "ok",
          service: "hivegrid-portal-domain-controller",
          revision: state.controllerHealthRevision,
        },
      };
    }
    if (request.url === "/applications") return { payload: state.apps };
    if (request.url === `/applications/${controllerUuid}/storages`) return { payload: state.storages };
    const envMatch = /^\/applications\/([a-z0-9]{24})\/envs$/u.exec(request.url);
    if (envMatch) return { payload: state.envByUuid.get(envMatch[1]) ?? [] };
    if (request.url === `/deployments/${deploymentUuid}`) {
      return {
        payload: {
          deployment_uuid: deploymentUuid,
          application_id: state.controller.id,
          application_name: "hivegrid-portal-domain-controller",
          status: "finished",
        },
      };
    }
    if (request.url === `/applications/${controllerUuid}` && request.method === "PATCH") {
      state.releasePatchCount += 1;
      const patch = JSON.parse(request.body);
      if (state.releaseMutationDrifts && state.releasePatchCount === 1) {
        Object.assign(state.controller, patch, { health_check_command: "drifted-command" });
      } else if (!(state.compensationDoesNotRestore && state.releasePatchCount > 1)) {
        Object.assign(state.controller, patch);
      }
      return { payload: state.controller };
    }
    if (request.url === `/applications/${controllerUuid}/stop` && request.method === "POST") {
      state.stopRequested = true;
      if (!state.stopDoesNotTransition) {
        state.controller.status = "exited:unhealthy";
        state.controllerProcessRunning = false;
      }
      return { payload: { message: "Application stopping request queued." } };
    }
    if (request.url === `/applications/${controllerUuid}`) {
      if (state.statusSequence?.length) state.controller.status = state.statusSequence.shift();
      return { payload: state.controller };
    }
    return { status: 404, payload: {} };
  });
}

function runHelper(serverUrl, ...args) {
  const envOverrides = typeof args[0] === "object" && args[0] !== null ? args.shift() : {};
  return new Promise((resolve) => {
    const child = spawn(process.execPath, [scriptPath, ...args], {
      env: {
        PATH: process.env.PATH || "",
        NODE_ENV: "test",
        SELECTED_COOLIFY_API_URL: serverUrl,
        SELECTED_COOLIFY_TOKEN: bearerToken,
        COOLIFY_DEPLOYMENT_POLL_MS: "1",
        COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "2",
        COOLIFY_CONTROLLER_DRAIN_HEALTH_URL: `${serverUrl}/controller-healthz`,
        COOLIFY_CONTROLLER_DRAIN_FENCE_MS: "1",
        ...envOverrides,
      },
      stdio: ["ignore", "pipe", "pipe"],
    });
    let stdout = "";
    let stderr = "";
    child.stdout.setEncoding("utf8");
    child.stderr.setEncoding("utf8");
    child.stdout.on("data", (chunk) => {
      stdout += chunk;
    });
    child.stderr.on("data", (chunk) => {
      stderr += chunk;
    });
    child.on("close", (status) => resolve({ status, stdout, stderr }));
  });
}

async function readBody(request) {
  let body = "";
  request.setEncoding("utf8");
  for await (const chunk of request) body += chunk;
  return body;
}

function startServer(responder) {
  const requests = [];
  const serverErrors = [];
  const server = http.createServer(async (request, response) => {
    const recorded = {
      method: request.method,
      url: request.url,
      authorization: request.headers.authorization,
      body: await readBody(request),
    };
    requests.push(recorded);
    try {
      const result = await responder(recorded);
      if (result?.destroy) {
        response.destroy();
        return;
      }
      response.writeHead(result?.status || 200, { "content-type": "application/json" });
      response.end(JSON.stringify(result?.payload ?? {}));
    } catch (error) {
      serverErrors.push(error);
      response.writeHead(500, { "content-type": "application/json" });
      response.end("{}");
    }
  });
  return new Promise((resolve, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", () =>
      resolve({
        url: `http://127.0.0.1:${server.address().port}`,
        requests,
        serverErrors,
        close: () => new Promise((done, failed) => server.close((error) => (error ? failed(error) : done()))),
      })
    );
  });
}

function assertSecretSafe(result) {
  for (const secret of [bearerToken, commandSecret, otherCommandSecret, metricsToken, "controller:secret", "worker:other-secret"]) {
    assert.doesNotMatch(`${result.stdout}${result.stderr}`, new RegExp(secret.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")));
  }
}

describe("portal-domain controller Coolify deployment helper", () => {
  it("accepts only the private two-mount runtime with matched worker-general HMAC wiring", async () => {
    const state = fixture();
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /runtime, mounts, and worker-general HMAC wiring verified/);
      assert.equal(result.stderr, "");
      assertSecretSafe(result);
      assert.deepEqual(server.serverErrors, []);
      assert.ok(server.requests.some(({ url }) => url === `/applications/${controllerUuid}/storages`));
      assert.ok(server.requests.some(({ url }) => url === `/applications/${workerGeneralUuid}/envs`));
      assert.ok(server.requests.some(({ url }) => url === `/applications/${webPortalUuid}/envs`));
      assert.ok(server.requests.every(({ authorization }) => authorization === `Bearer ${bearerToken}`));
    } finally {
      await server.close();
    }
  });

  it("accepts only Coolify's exact automatic metadata keys", async () => {
    const accepted = fixture({
      controllerEnv: {
        COOLIFY_BRANCH: "main",
        COOLIFY_CONTAINER_NAME: "controller-container",
        COOLIFY_FQDN: "",
        COOLIFY_RESOURCE_UUID: controllerUuid,
        COOLIFY_URL: "",
      },
    });
    const acceptedServer = await startFixtureServer(accepted);
    try {
      const result = await runHelper(acceptedServer.url, "assert-runtime", "staging", controllerUuid, accepted.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 0, result.stderr);
      assertSecretSafe(result);
    } finally {
      await acceptedServer.close();
    }

    const rejected = fixture({ controllerEnv: { COOLIFY_DEPLOYMENT_TOKEN: "not-allowed" } });
    const rejectedServer = await startFixtureServer(rejected);
    try {
      const result = await runHelper(rejectedServer.url, "assert-runtime", "staging", controllerUuid, rejected.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /forbidden privileged environment keys/);
      assertSecretSafe(result);
    } finally {
      await rejectedServer.close();
    }
  });

  it("keeps the controller probe on the exact internal port behind the mesh metrics binding", async () => {
    const accepted = fixture({ controllerEnv: { PORTAL_DOMAIN_CONTROLLER_PORT: "8080" } });
    const acceptedServer = await startFixtureServer(accepted);
    try {
      const result = await runHelper(acceptedServer.url, "assert-runtime", "staging", controllerUuid, accepted.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 0, result.stderr);
      assertSecretSafe(result);
    } finally {
      await acceptedServer.close();
    }

    const rejected = fixture({ controllerEnv: { PORTAL_DOMAIN_CONTROLLER_PORT: "8081" } });
    const rejectedServer = await startFixtureServer(rejected);
    try {
      const result = await runHelper(rejectedServer.url, "assert-runtime", "staging", controllerUuid, rejected.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /must be exactly 8080/);
      assertSecretSafe(result);
    } finally {
      await rejectedServer.close();
    }
  });

  it("rejects a worker-general HMAC mismatch before any mutation", async () => {
    const state = fixture({ workerGeneralEnv: { PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: otherCommandSecret } });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /HMAC secrets must match/);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects a broad Traefik root mount or missing reload sentinel", async () => {
    for (const storages of [
      [{ type: "file", is_directory: true, fs_path: "/data/coolify/proxy/dynamic", mount_path: "/data/coolify/proxy/dynamic" }],
      {
        file_storages: exactStorages().file_storages,
        persistent_storages: [
          {
            ...exactStorages().persistent_storages[0],
            host_path: "/data/coolify/proxy/dynamic/not-the-reload-sentinel",
          },
        ],
      },
    ]) {
      const state = fixture({ storages });
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /exactly two narrow bind mounts|config subdirectory and RW reload sentinel/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects missing or malformed environment-specific Traefik routing identities", async () => {
    for (const controllerEnvOverride of [
      { PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: undefined },
      { PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "http://hivegrid-portal:8080" },
      { PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "https;malformed" },
      { PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "websecure" },
    ]) {
      const state = fixture({ controllerEnv: controllerEnvOverride });
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /BACKEND_SERVICE|ENTRYPOINT/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects a missing or ambiguous allowlisted portal surface before any mutation", async () => {
    const duplicatePortalUuid = "q".repeat(24);
    const missing = fixture({ webPortalEnv: { WEB_RUNTIME_SURFACE: undefined } });
    const ambiguous = fixture();
    ambiguous.apps.push({
      uuid: duplicatePortalUuid,
      name: "hivegrid-portal-shadow",
      build_pack: "dockerimage",
      docker_registry_image_name: webImage,
    });
    ambiguous.envByUuid.set(duplicatePortalUuid, envEntries({ WEB_RUNTIME_SURFACE: "portal" }));
    ambiguous.allowedCsv = `${ambiguous.allowedCsv},${duplicatePortalUuid}`;

    for (const state of [missing, ambiguous]) {
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /exactly one allowlisted hivegrid-portal surface/);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects a valid Docker service name that does not identify the resolved portal resource", async () => {
    const state = fixture({
      controllerEnv: {
        PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: `https-0-${webCoreUuid}@docker`,
      },
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /must target the exact allowlisted hivegrid-portal Docker service/);
      assert.ok(server.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects CNAME drift and unexpected controller credentials before mutation", async () => {
    for (const state of [
      fixture({ webCnameTarget: "production-ingress.example.test" }),
      fixture({ controllerEnv: { STORAGE_SECRET_ACCESS_KEY: "must-never-reach-controller" } }),
    ]) {
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /hivegrid-web must use exactly|least-privilege environment allowlist/);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("requires the exact custom-domain runtime opt-in on both staging web surfaces", async () => {
    for (const state of [
      fixture({ webCoreEnv: { PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: undefined } }),
      fixture({ webCoreRuntimeEnabled: "false" }),
      fixture({ webCoreRuntimeEnabled: "1" }),
      fixture({ webPortalEnv: { PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED: undefined } }),
      fixture({ webPortalRuntimeEnabled: "false" }),
      fixture({ webPortalRuntimeEnabled: "TRUE" }),
    ]) {
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /PORTAL_CUSTOM_DOMAIN_RUNTIME_ENABLED exactly to true in staging/);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("pins each deployment environment to its exact CNAME target before mutation", async () => {
    const cases = [
      {
        environment: "production",
        state: fixture({ controller: { ports_mappings: productionMetricsBinding } }),
        expected: new RegExp(`EXPECTED_CNAME must be exactly ${productionCname}`),
      },
      {
        environment: "staging",
        state: fixture({
          controllerEnv: { PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: productionCname },
          webCnameTarget: productionCname,
        }),
        expected: new RegExp(`EXPECTED_CNAME must be exactly ${stagingCname}`),
      },
      {
        environment: "preview",
        state: fixture(),
        expected: /deployment environment must be exactly staging or production/,
      },
    ];

    for (const { environment, state, expected } of cases) {
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "set-release", environment, controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
        assert.equal(result.status, 1);
        assert.match(result.stderr, expected);
        assert.ok(server.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("accepts the production environment only with the production CNAME target", async () => {
    const state = fixture({
      controller: { ports_mappings: productionMetricsBinding },
      controllerEnv: { PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: productionCname },
      webCnameTarget: productionCname,
      webCoreRuntimeEnabled: "false",
      webPortalRuntimeEnabled: "false",
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "assert-runtime", "production", controllerUuid, state.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 0, result.stderr);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("pins the explicit resource to the full SHA and clears deployment hooks", async () => {
    const state = fixture();
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /immutable release configured/);
      const patches = server.requests.filter(({ method }) => method === "PATCH");
      assert.equal(patches.length, 1);
      assert.deepEqual(JSON.parse(patches[0].body), {
        pre_deployment_command: "",
        post_deployment_command: "",
        docker_registry_image_tag: revision,
        ...expectedControllerHealthConfig(),
      });
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("verifies compensation after a rejected release mutation", async () => {
    for (const compensationDoesNotRestore of [false, true]) {
      const state = fixture({
        controller: { health_check_interval: 0, health_check_start_period: 0 },
        releaseMutationDrifts: true,
        compensationDoesNotRestore,
      });
      const previousTag = state.controller.docker_registry_image_tag;
      const previousHealthType = state.controller.health_check_type;
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision);
        assert.equal(result.status, 1);
        assert.match(
          result.stderr,
          compensationDoesNotRestore ? /compensation was incomplete/ : /previous configuration was restored/
        );
        assert.equal(server.requests.filter(({ method }) => method === "PATCH").length, 2);
        if (!compensationDoesNotRestore) {
          assert.equal(state.controller.docker_registry_image_tag, previousTag);
          assert.equal(state.controller.health_check_type, previousHealthType);
          assert.equal(state.controller.health_check_interval, 0);
          assert.equal(state.controller.health_check_start_period, 0);
        }
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("stops and drains the old controller before a paired web rollout", async () => {
    const state = fixture({ controller: { docker_registry_image_tag: revision, status: "running:healthy" } });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /stopped and command worker drained/);
      assert.deepEqual(
        server.requests.filter(({ method, url }) => method === "POST" && url.endsWith("/stop")).map(({ url }) => url),
        [`/applications/${controllerUuid}/stop`]
      );
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails closed for operator recovery when pre-stop health is unreachable despite exited status", async () => {
    const state = fixture({
      controller: { docker_registry_image_tag: revision, status: "exited:unhealthy" },
      controllerProcessRunning: false,
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /pre-stop health is unreachable or not revision-bound; operator recovery is required/);
      assert.equal(server.requests.some(({ method, url }) => method === "POST" && url.endsWith("/stop")), false);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("does not trust stale exited status while the queued stop and live health lag", async () => {
    const state = fixture({
      controller: { docker_registry_image_tag: revision, status: "exited:unhealthy" },
      statusSequence: ["exited:unhealthy", "exited:unhealthy", "running:healthy", "exited:unhealthy", "exited:unhealthy"],
      postStopHealthSequence: [true, true, false, false],
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(
        server.url,
        { COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "4" },
        "stop-and-drain",
        "staging",
        controllerUuid,
        state.allowedCsv,
        controllerImage,
        workerImage
      );
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /stopped and command worker drained/);
      assert.deepEqual(
        server.requests.filter(({ method, url }) => method === "POST" && url.endsWith("/stop")).map(({ url }) => url),
        [`/applications/${controllerUuid}/stop`]
      );
      assert.equal(server.requests.filter(({ method, url }) => method === "GET" && url === `/applications/${controllerUuid}`).length, 5);
      assert.equal(server.requests.filter(({ url }) => url === "/controller-healthz").length, 5);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails closed when Coolify cannot prove the controller stopped", async () => {
    const state = fixture({
      controller: { docker_registry_image_tag: revision, status: "running:healthy" },
      stopDoesNotTransition: true,
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /did not stop and drain/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("fails closed before stop when mesh health belongs to another revision", async () => {
    const state = fixture({
      controller: { docker_registry_image_tag: revision, status: "running:healthy" },
      controllerHealthRevision: "b".repeat(40),
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "stop-and-drain", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 1);
      assert.match(result.stderr, /pre-stop health is unreachable or not revision-bound/);
      assert.equal(server.requests.some(({ method, url }) => method === "POST" && url.endsWith("/stop")), false);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("snapshots only a validated immutable controller release without mutation", async () => {
    const valid = fixture({
      controller: {
        docker_registry_image_tag: revision,
        custom_healthcheck_found: false,
        ...expectedControllerHealthConfig(),
      },
    });
    const validServer = await startFixtureServer(valid);
    try {
      const result = await runHelper(validServer.url, "snapshot-release", "staging", controllerUuid, valid.allowedCsv, controllerImage, workerImage);
      assert.equal(result.status, 0, result.stderr);
      assert.equal(result.stdout, revision);
      assert.equal(result.stderr, "");
      assert.ok(validServer.requests.every(({ method }) => method === "GET"));
      assertSecretSafe(result);
    } finally {
      await validServer.close();
    }

    for (const controller of [
      { docker_registry_image_tag: "staging" },
      { docker_registry_image_tag: revision, custom_healthcheck_found: false, ...expectedControllerHealthConfig(), health_check_command: "drifted" },
    ]) {
      const invalid = fixture({ controller });
      const invalidServer = await startFixtureServer(invalid);
      try {
        const result = await runHelper(
          invalidServer.url,
          "snapshot-release",
          "staging",
          controllerUuid,
          invalid.allowedCsv,
          controllerImage,
          workerImage,
        );
        assert.equal(result.status, 1);
        assert.match(result.stderr, /full lowercase Git SHA|revision-bound command health configuration drifted/);
        assert.ok(invalidServer.requests.every(({ method }) => method === "GET"));
        assertSecretSafe(result);
      } finally {
        await invalidServer.close();
      }
    }
  });

  it("rejects a mutable release tag before contacting Coolify", async () => {
    const state = fixture();
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(server.url, "set-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, "staging");
      assert.equal(result.status, 1);
      assert.match(result.stderr, /full lowercase Git SHA/);
      assert.deepEqual(server.requests, []);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("binds the completed deployment to a revision-bound command healthz and readyz contract", async () => {
    const state = fixture({
      controller: {
        docker_registry_image_tag: revision,
        status: "running:healthy",
        custom_healthcheck_found: false,
        ...expectedControllerHealthConfig(),
      },
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(
        server.url,
        "assert-release",
        "staging",
        controllerUuid,
        state.allowedCsv,
        controllerImage,
        workerImage,
        revision,
        deploymentUuid,
      );
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /healthz, readyz, and immutable revision verified/);
      assert.ok(server.requests.some(({ url }) => url === `/deployments/${deploymentUuid}`));
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("tolerates at least eight stale exited statuses within the freshly deployed controller grace period", async () => {
    const state = fixture({
      controller: {
        docker_registry_image_tag: revision,
        status: "exited:unhealthy",
        custom_healthcheck_found: false,
        ...expectedControllerHealthConfig(),
      },
      // First entry feeds the runtime-contract lookup, the rest the poll loop.
      statusSequence: [
        "exited:unhealthy",
        "exited:unhealthy",
        "exited:unhealthy",
        "exited:unhealthy",
        "exited:unhealthy",
        "exited:unhealthy",
        "exited:unhealthy",
        "exited:unhealthy",
        "exited:unhealthy",
        "running:healthy",
      ],
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(
        server.url,
        { COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "10", COOLIFY_FAILED_STATUS_GRACE_MS: "60000" },
        "assert-release",
        "staging",
        controllerUuid,
        state.allowedCsv,
        controllerImage,
        workerImage,
        revision,
        deploymentUuid
      );
      assert.equal(result.status, 0, result.stderr);
      assert.match(result.stdout, /healthz, readyz, and immutable revision verified/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("still fails closed when the controller stays in a failed runtime state", async () => {
    const state = fixture({
      controller: {
        docker_registry_image_tag: revision,
        status: "exited:unhealthy",
        custom_healthcheck_found: false,
        ...expectedControllerHealthConfig(),
      },
    });
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(
        server.url,
        {
          COOLIFY_DEPLOYMENT_POLL_MS: "10",
          COOLIFY_DEPLOYMENT_WAIT_ATTEMPTS: "6",
          COOLIFY_FAILED_STATUS_GRACE_MS: "5",
        },
        "assert-release",
        "staging",
        controllerUuid,
        state.allowedCsv,
        controllerImage,
        workerImage,
        revision,
        deploymentUuid
      );
      assert.equal(result.status, 1);
      assert.match(result.stderr, /entered a terminal runtime state/);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects an invalid failed-status grace before contacting Coolify", async () => {
    const state = fixture();
    const server = await startFixtureServer(state);
    try {
      const result = await runHelper(
        server.url,
        { COOLIFY_FAILED_STATUS_GRACE_MS: "-1" },
        "assert-release",
        "staging",
        controllerUuid,
        state.allowedCsv,
        controllerImage,
        workerImage,
        revision,
        deploymentUuid
      );
      assert.equal(result.status, 1);
      assert.match(result.stderr, /COOLIFY_FAILED_STATUS_GRACE_MS must be a non-negative safe integer/);
      assert.deepEqual(server.requests, []);
      assertSecretSafe(result);
    } finally {
      await server.close();
    }
  });

  it("rejects a public controller route, an image-healthcheck override, or command-health drift", async () => {
    for (const controller of [
      { fqdn: "https://controller.example.test" },
      {
        docker_registry_image_tag: revision,
        status: "running:healthy",
        custom_healthcheck_found: true,
        ...expectedControllerHealthConfig(),
      },
      {
        docker_registry_image_tag: revision,
        status: "running:healthy",
        custom_healthcheck_found: false,
        ...expectedControllerHealthConfig(),
        health_check_command: "curl http://127.0.0.1:8080/readyz",
      },
    ]) {
      const state = fixture({ controller });
      const server = await startFixtureServer(state);
      try {
        const args = controller.fqdn
          ? ["assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage]
          : ["assert-release", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage, revision, deploymentUuid];
        const result = await runHelper(server.url, ...args);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /must remain private|command health configuration drifted/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });

  it("rejects a public or missing metrics port instead of exposing the controller beyond the mesh", async () => {
    for (const ports_mappings of [null, "0.0.0.0:19120:8080", "10.42.0.3:19120:8080,10.42.0.3:19121:8080"]) {
      const state = fixture({ controller: { ports_mappings } });
      const server = await startFixtureServer(state);
      try {
        const result = await runHelper(server.url, "assert-runtime", "staging", controllerUuid, state.allowedCsv, controllerImage, workerImage);
        assert.equal(result.status, 1);
        assert.match(result.stderr, /metrics only through the exact mesh binding/);
        assertSecretSafe(result);
      } finally {
        await server.close();
      }
    }
  });
});