Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -0,0 +1,368 @@
import { existsSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import {
ClaimLockLostError,
ClaimLockUnavailableError,
FencedClaimMutationLock,
FilesystemClaimMutationLock,
RedisClaimMutationLock,
type ClaimLockGuard,
type ClaimLockRedisClient,
type ClaimMutationLock,
} from "./claim-lock.js";
describe("Redis portal-domain claim lock", () => {
it("serializes two controller instances and renews ownership during a long mutation", async () => {
vi.useFakeTimers();
const store = new FakeRedisStore();
const first = lockFor(store.client());
const second = lockFor(store.client());
const entered = deferred<void>();
const finish = deferred<void>();
const order: string[] = [];
try {
const firstOperation = first.withClaimLock("claim_12345678", async () => {
order.push("first:start");
entered.resolve();
await finish.promise;
order.push("first:end");
});
await entered.promise;
const secondOperation = second.withClaimLock("claim_12345678", async () => {
order.push("second");
});
// This is longer than the original lease. The second instance must still
// wait because the first instance renews with its owner token.
await vi.advanceTimersByTimeAsync(70);
expect(order).toEqual(["first:start"]);
finish.resolve();
await firstOperation;
await vi.advanceTimersByTimeAsync(1);
await secondOperation;
expect(order).toEqual(["first:start", "first:end", "second"]);
expect(store.keys()).toEqual([]);
await Promise.all([first.close(), second.close()]);
} finally {
finish.resolve();
vi.useRealTimers();
}
});
it("times out acquisition predictably without exposing Redis error details", async () => {
vi.useFakeTimers();
const store = new FakeRedisStore();
const holder = lockFor(store.client());
const entered = deferred<void>();
const finish = deferred<void>();
try {
const held = holder.withClaimLock("claim_12345678", async () => {
entered.resolve();
await finish.promise;
});
await entered.promise;
const contender = lockFor(store.client(), { acquireTimeoutMs: 15 });
const contenderAttempt = contender.withClaimLock("claim_12345678", async () => undefined).catch((error: unknown) => error);
await vi.advanceTimersByTimeAsync(15);
await expect(contenderAttempt).resolves.toMatchObject({
reasonCode: "claim_lock_unavailable",
retryable: true,
});
const failingClient = store.client();
failingClient.set = async () => {
throw new Error("redis://:do-not-leak@redis.internal:6379");
};
const failing = lockFor(failingClient, { acquireTimeoutMs: 5 });
const failureAttempt = failing.withClaimLock("claim_87654321", async () => undefined).catch((error: unknown) => error);
await vi.advanceTimersByTimeAsync(5);
const failure = await failureAttempt;
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
expect(String(failure)).not.toContain("do-not-leak");
expect(String(failure)).not.toContain("redis.internal");
finish.resolve();
await held;
await Promise.all([holder.close(), contender.close(), failing.close()]);
} finally {
finish.resolve();
vi.useRealTimers();
}
});
it("fences a stale writer after lease loss while a second writer owns the claim", async () => {
const store = new FakeRedisStore();
const first = lockFor(store.client(), { token: () => "original-owner" });
const second = lockFor(store.client(), { token: () => "new-owner" });
const firstEntered = deferred<void>();
const resumeFirst = deferred<void>();
const secondEntered = deferred<void>();
const finishSecond = deferred<void>();
let resource = "initial";
const staleOperation = first.withClaimLock("claim_12345678", async (guard) => {
firstEntered.resolve();
await resumeFirst.promise;
await guard.assertHeld();
resource = "stale";
});
await firstEntered.promise;
store.deleteOnlyValue();
const currentOperation = second.withClaimLock("claim_12345678", async (guard) => {
await guard.assertHeld();
resource = "current";
secondEntered.resolve();
await finishSecond.promise;
});
await secondEntered.promise;
resumeFirst.resolve();
await expect(staleOperation).rejects.toBeInstanceOf(ClaimLockLostError);
expect(resource).toBe("current");
expect(store.onlyValue()).toBe("new-owner");
finishSecond.resolve();
await currentOperation;
expect(store.keys()).toEqual([]);
await Promise.all([first.close(), second.close()]);
});
it("keeps the authoritative fence held after a lease guard passes until the whole mutation returns", async () => {
const filesystem = new SerializedClaimMutationLock();
const lease = new PassthroughClaimMutationLock();
const first = new FencedClaimMutationLock(filesystem, lease);
const second = new FencedClaimMutationLock(filesystem, lease);
const firstGuardPassed = deferred<void>();
const resumeFirst = deferred<void>();
let secondEntered = false;
let resource = "initial";
const staleMutation = first.withClaimLock("claim_12345678", async (guard) => {
await guard.assertHeld();
firstGuardPassed.resolve();
// Represents an event-loop pause after Redis ownership was checked but
// immediately before rename/unlink. The filesystem fence must remain.
await resumeFirst.promise;
resource = "stale-generation";
});
await firstGuardPassed.promise;
const currentMutation = second.withClaimLock("claim_12345678", async () => {
secondEntered = true;
resource = "current-generation";
});
await wait(20);
expect(secondEntered).toBe(false);
resumeFirst.resolve();
await Promise.all([staleMutation, currentMutation]);
expect(secondEntered).toBe(true);
expect(resource).toBe("current-generation");
});
it.skipIf(!existsSync("/usr/bin/flock"))("holds one kernel flock across controller instances until the parent fd closes", async () => {
const directory = await mkdtemp(path.join(os.tmpdir(), "hivegrid-portal-domain-flock-"));
const first = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
const second = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
const entered = deferred<void>();
const finish = deferred<void>();
let secondEntered = false;
try {
const firstMutation = first.withClaimLock("claim_12345678", async () => {
entered.resolve();
await finish.promise;
});
await entered.promise;
const secondMutation = second.withClaimLock("claim_12345678", async () => {
secondEntered = true;
});
await wait(30);
expect(secondEntered).toBe(false);
finish.resolve();
await Promise.all([firstMutation, secondMutation]);
expect(secondEntered).toBe(true);
} finally {
await Promise.all([first.close(), second.close()]);
await rm(directory, { recursive: true, force: true });
}
});
it("hashes claim identifiers, reports readiness and closes the client", async () => {
const store = new FakeRedisStore();
const client = store.client("wait");
const lock = lockFor(client);
await expect(lock.assertReady()).resolves.toBeUndefined();
expect(store.readinessCalls()).toEqual({ sets: 1, renewals: 1, releases: 1 });
await lock.withClaimLock("../../raw-secret-claim", async () => {
expect(store.keys()).toHaveLength(1);
expect(store.keys()[0]).not.toContain("raw-secret-claim");
expect(store.keys()[0]).toMatch(/^hivegrid:portal-domain-controller:claim-lock:[a-f0-9]{64}$/u);
});
await lock.close();
await expect(lock.assertReady()).rejects.toBeInstanceOf(ClaimLockUnavailableError);
expect(client.status).toBe("end");
});
it("fails readiness when Redis permits PING but denies lock writes", async () => {
const store = new FakeRedisStore();
const client = store.client("wait");
client.set = async () => {
throw new Error("NOPERM redis://:do-not-leak@redis.internal");
};
const lock = lockFor(client);
const failure = await lock.assertReady().catch((error: unknown) => error);
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
expect(failure).toMatchObject({ reasonCode: "claim_lock_unavailable", retryable: true });
expect(String(failure)).not.toContain("do-not-leak");
expect(String(failure)).not.toContain("redis.internal");
expect(store.keys()).toEqual([]);
await lock.close();
});
});
class PassthroughClaimMutationLock implements ClaimMutationLock {
async assertReady(): Promise<void> {}
async withClaimLock<T>(_claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
return operation({ assertHeld: async () => undefined });
}
async close(): Promise<void> {}
}
class SerializedClaimMutationLock implements ClaimMutationLock {
readonly #tails = new Map<string, Promise<void>>();
async assertReady(): Promise<void> {}
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
const previous = this.#tails.get(claimId) ?? Promise.resolve();
let release: () => void = () => undefined;
const current = new Promise<void>((resolve) => {
release = resolve;
});
const tail = previous.then(() => current);
this.#tails.set(claimId, tail);
await previous;
try {
return await operation({ assertHeld: async () => undefined });
} finally {
release();
if (this.#tails.get(claimId) === tail) this.#tails.delete(claimId);
}
}
async close(): Promise<void> {}
}
type TestLockOptions = ConstructorParameters<typeof RedisClaimMutationLock>[1];
function lockFor(client: ClaimLockRedisClient, options: TestLockOptions = {}): RedisClaimMutationLock {
return new RedisClaimMutationLock(client, {
leaseMs: 30,
heartbeatIntervalMs: 8,
acquireTimeoutMs: 250,
retryDelayMs: 1,
retryJitterMs: 0,
...options,
});
}
class FakeRedisStore {
readonly #values = new Map<string, { value: string; expiresAt: number }>();
#sets = 0;
#renewals = 0;
#releases = 0;
client(initialStatus: string = "ready"): ClaimLockRedisClient {
let status = initialStatus;
const assertConnected = () => {
if (status !== "ready") throw new Error("redis://:hidden@unavailable.internal");
};
return {
get status() {
return status;
},
connect: async () => {
status = "ready";
},
ping: async () => {
assertConnected();
return "PONG";
},
set: async (key, value, _expiryMode, leaseMs) => {
assertConnected();
this.#sets += 1;
this.#expire(key);
if (this.#values.has(key)) return null;
this.#values.set(key, { value, expiresAt: Date.now() + leaseMs });
return "OK";
},
eval: async (_script, _numberOfKeys, ...args) => {
assertConnected();
const [key, token, leaseMs] = args;
if (typeof key !== "string" || typeof token !== "string") return 0;
this.#expire(key);
const entry = this.#values.get(key);
if (!entry || entry.value !== token) return 0;
if (leaseMs !== undefined) {
this.#renewals += 1;
entry.expiresAt = Date.now() + Number(leaseMs);
return 1;
}
this.#releases += 1;
this.#values.delete(key);
return 1;
},
quit: async () => {
status = "end";
return "OK";
},
disconnect: () => {
status = "end";
},
on: () => undefined,
};
}
keys(): string[] {
for (const key of this.#values.keys()) this.#expire(key);
return [...this.#values.keys()];
}
deleteOnlyValue(): void {
const [key] = this.keys();
if (!key) throw new Error("expected one lock key");
this.#values.delete(key);
}
onlyValue(): string | undefined {
const [key] = this.keys();
return key ? this.#values.get(key)?.value : undefined;
}
readinessCalls(): { sets: number; renewals: number; releases: number } {
return { sets: this.#sets, renewals: this.#renewals, releases: this.#releases };
}
#expire(key: string): void {
const entry = this.#values.get(key);
if (entry && entry.expiresAt <= Date.now()) this.#values.delete(key);
}
}
function deferred<T>() {
let resolve: (value: T | PromiseLike<T>) => void = () => undefined;
const promise = new Promise<T>((settle) => {
resolve = settle;
});
return { promise, resolve };
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs));
}
@@ -0,0 +1,484 @@
import { randomUUID } from "node:crypto";
import { spawn } from "node:child_process";
import { constants as fsConstants } from "node:fs";
import { lstat, open, realpath, type FileHandle } from "node:fs/promises";
import path from "node:path";
import type { ConnectionOptions } from "@hivegrid/queue";
import { Redis } from "ioredis";
import { sha256Hex } from "./contracts.js";
const LOCK_KEY_PREFIX = "hivegrid:portal-domain-controller:claim-lock:";
const DEFAULT_LEASE_MS = 30_000;
const DEFAULT_ACQUIRE_TIMEOUT_MS = 10_000;
const DEFAULT_RETRY_DELAY_MS = 75;
const DEFAULT_RETRY_JITTER_MS = 75;
const DEFAULT_COMMAND_TIMEOUT_MS = 2_500;
const DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS = 10_000;
const FLOCK_BINARY = "/usr/bin/flock";
const RENEW_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('PEXPIRE', KEYS[1], ARGV[2])", "end", "return 0"].join("\n");
const RELEASE_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('DEL', KEYS[1])", "end", "return 0"].join("\n");
export type ClaimMutationLock = {
assertReady(): Promise<void>;
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T>;
close(): Promise<void>;
};
export type ClaimLockGuard = Readonly<{
assertHeld(): Promise<void>;
}>;
export type ClaimLockRedisClient = {
readonly status: string;
connect(): Promise<unknown>;
ping(): Promise<string>;
set(key: string, value: string, expiryMode: "PX", leaseMs: number, condition: "NX"): Promise<"OK" | null>;
eval(script: string, numberOfKeys: number, ...args: Array<string | number>): Promise<unknown>;
quit(): Promise<unknown>;
disconnect(reconnect?: boolean): void;
on(event: "error", listener: (error: unknown) => void): unknown;
};
type ClaimLockOptions = Readonly<{
leaseMs?: number;
acquireTimeoutMs?: number;
retryDelayMs?: number;
retryJitterMs?: number;
heartbeatIntervalMs?: number;
now?: () => number;
random?: () => number;
token?: () => string;
wait?: (delayMs: number) => Promise<void>;
}>;
type NormalizedClaimLockOptions = Readonly<{
leaseMs: number;
acquireTimeoutMs: number;
retryDelayMs: number;
retryJitterMs: number;
heartbeatIntervalMs: number;
now: () => number;
random: () => number;
token: () => string;
wait: (delayMs: number) => Promise<void>;
}>;
export class ClaimLockUnavailableError extends Error {
readonly retryable = true;
readonly reasonCode = "claim_lock_unavailable";
constructor() {
super("Portal-domain claim lock is temporarily unavailable");
this.name = "ClaimLockUnavailableError";
}
}
export class ClaimLockLostError extends Error {
readonly retryable = true;
readonly reasonCode = "claim_lock_lost";
constructor() {
super("Portal-domain claim lock ownership was lost");
this.name = "ClaimLockLostError";
}
}
/**
* Kernel-backed ownership for the shared Traefik directory. Redis remains the
* fast distributed coordinator, while this non-expiring `flock` is the
* authoritative fence around filesystem mutations. The lock is attached to
* the parent's open-file description, so a paused Node event loop cannot let a
* second controller enter after a Redis lease expires. A crash closes the fd
* and lets the kernel release ownership automatically.
*/
export class FilesystemClaimMutationLock implements ClaimMutationLock {
readonly #configuredDirectory: string;
readonly #binary: string;
readonly #acquireTimeoutMs: number;
readonly #activeOperations = new Set<Promise<void>>();
#closing = false;
#closed = false;
#closePromise: Promise<void> | undefined;
constructor(directory: string, options: { binary?: string; acquireTimeoutMs?: number } = {}) {
if (!path.isAbsolute(directory)) throw new ClaimLockUnavailableError();
this.#configuredDirectory = path.resolve(directory);
this.#binary = options.binary ?? FLOCK_BINARY;
this.#acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS, "filesystem claim lock timeout");
}
async assertReady(): Promise<void> {
await this.withClaimLock("__readiness__", async (guard) => guard.assertHeld());
}
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
let finishActive: () => void = () => undefined;
const active = new Promise<void>((resolve) => {
finishActive = resolve;
});
this.#activeOperations.add(active);
try {
return await this.#withClaimLock(claimId, operation);
} finally {
finishActive();
this.#activeOperations.delete(active);
}
}
close(): Promise<void> {
this.#closePromise ??= this.#close();
return this.#closePromise;
}
async #close(): Promise<void> {
if (this.#closed) return;
this.#closing = true;
await Promise.all([...this.#activeOperations]);
this.#closed = true;
}
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
const directory = await this.#resolveSafeDirectory();
const lockPath = path.join(directory, `.portal-${sha256Hex(claimId)}.lock`);
if (path.dirname(lockPath) !== directory) throw new ClaimLockUnavailableError();
let handle: FileHandle | undefined;
let held = false;
try {
handle = await open(
lockPath,
fsConstants.O_CREAT | fsConstants.O_RDWR | noFollowFlag(),
0o600
);
await assertSameRegularFile(lockPath, handle);
await acquireKernelFlock(handle.fd, this.#binary, this.#acquireTimeoutMs);
held = true;
const guard: ClaimLockGuard = Object.freeze({
assertHeld: async () => {
if (!held || !handle) throw new ClaimLockLostError();
await assertSameRegularFile(lockPath, handle);
},
});
return await operation(guard);
} catch (error) {
if (error instanceof ClaimLockLostError || error instanceof ClaimLockUnavailableError) throw error;
throw new ClaimLockUnavailableError();
} finally {
held = false;
await handle?.close().catch(() => undefined);
}
}
async #resolveSafeDirectory(): Promise<string> {
try {
const stats = await lstat(this.#configuredDirectory);
if (!stats.isDirectory() || stats.isSymbolicLink()) throw new ClaimLockUnavailableError();
return await realpath(this.#configuredDirectory);
} catch (error) {
if (error instanceof ClaimLockUnavailableError) throw error;
throw new ClaimLockUnavailableError();
}
}
}
export class FencedClaimMutationLock implements ClaimMutationLock {
readonly #filesystem: ClaimMutationLock;
readonly #lease: ClaimMutationLock;
constructor(filesystem: ClaimMutationLock, lease: ClaimMutationLock) {
this.#filesystem = filesystem;
this.#lease = lease;
}
async assertReady(): Promise<void> {
await this.#filesystem.assertReady();
await this.#lease.assertReady();
}
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
return this.#filesystem.withClaimLock(claimId, () => this.#lease.withClaimLock(claimId, operation));
}
async close(): Promise<void> {
await Promise.all([this.#lease.close(), this.#filesystem.close()]);
}
}
export class RedisClaimMutationLock implements ClaimMutationLock {
readonly #client: ClaimLockRedisClient;
readonly #options: NormalizedClaimLockOptions;
readonly #activeOperations = new Set<Promise<void>>();
#closing = false;
#closed = false;
#closePromise: Promise<void> | undefined;
constructor(client: ClaimLockRedisClient, options: ClaimLockOptions = {}) {
this.#client = client;
this.#options = normalizeOptions(options);
// Redis connection errors are surfaced through bounded commands and the
// readiness endpoint. Keeping this listener intentionally silent also
// prevents connection details from reaching stderr outside our logger.
this.#client.on("error", () => undefined);
}
async assertReady(): Promise<void> {
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
try {
if (this.#client.status === "wait") await this.#client.connect();
if ((await this.#client.ping()) !== "PONG") throw new ClaimLockUnavailableError();
await this.#probeLockCapabilities();
} catch {
throw new ClaimLockUnavailableError();
}
}
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
let finishActive: () => void = () => undefined;
const active = new Promise<void>((resolve) => {
finishActive = resolve;
});
this.#activeOperations.add(active);
try {
return await this.#withClaimLock(claimId, operation);
} finally {
finishActive();
this.#activeOperations.delete(active);
}
}
close(): Promise<void> {
this.#closePromise ??= this.#close();
return this.#closePromise;
}
async #close(): Promise<void> {
if (this.#closed) return;
this.#closing = true;
await Promise.all([...this.#activeOperations]);
try {
if (this.#client.status !== "wait" && this.#client.status !== "end") {
await this.#client.quit();
}
} catch {
// An immediate disconnect is safe after all leases were released. It
// also keeps shutdown bounded when Redis disappears during a rollout.
} finally {
this.#client.disconnect(false);
this.#closed = true;
}
}
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
const key = `${LOCK_KEY_PREFIX}${sha256Hex(claimId)}`;
const token = this.#options.token();
await this.#acquire(key, token);
let heartbeatTimer: NodeJS.Timeout | undefined;
let heartbeatInFlight: Promise<void> = Promise.resolve();
let heartbeatError: ClaimLockLostError | ClaimLockUnavailableError | undefined;
let heartbeatStopped = false;
const scheduleHeartbeat = () => {
if (heartbeatStopped || heartbeatError) return;
heartbeatTimer = setTimeout(() => {
heartbeatInFlight = this.#renew(key, token)
.then(() => scheduleHeartbeat())
.catch((error: unknown) => {
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
});
}, this.#options.heartbeatIntervalMs);
};
scheduleHeartbeat();
const guard: ClaimLockGuard = Object.freeze({
assertHeld: async () => {
if (heartbeatError) throw heartbeatError;
try {
await this.#renew(key, token);
} catch (error) {
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
throw heartbeatError;
}
if (heartbeatError) throw heartbeatError;
},
});
let result: T | undefined;
let operationError: unknown;
let operationFailed = false;
try {
result = await operation(guard);
} catch (error) {
operationFailed = true;
operationError = error;
}
heartbeatStopped = true;
if (heartbeatTimer) clearTimeout(heartbeatTimer);
await heartbeatInFlight.catch(() => undefined);
const observedHeartbeatError = heartbeatError;
let releaseError: ClaimLockUnavailableError | undefined;
let released = false;
try {
released = await this.#release(key, token);
} catch {
releaseError = new ClaimLockUnavailableError();
}
if (operationFailed) throw operationError;
if (observedHeartbeatError) throw observedHeartbeatError;
if (releaseError) throw releaseError;
if (!released) throw new ClaimLockLostError();
return result as T;
}
async #acquire(key: string, token: string): Promise<void> {
const deadline = this.#options.now() + this.#options.acquireTimeoutMs;
while (!this.#closing && !this.#closed) {
try {
if ((await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK") return;
} catch {
// Redis can be transiently unavailable during a rolling restart. The
// bounded deadline below turns that into a retryable controller result.
}
const remainingMs = deadline - this.#options.now();
if (remainingMs <= 0) break;
const jitterMs = Math.floor(this.#options.random() * (this.#options.retryJitterMs + 1));
await this.#options.wait(Math.min(remainingMs, this.#options.retryDelayMs + jitterMs));
}
throw new ClaimLockUnavailableError();
}
async #renew(key: string, token: string): Promise<void> {
let renewed: unknown;
try {
renewed = await this.#client.eval(RENEW_SCRIPT, 1, key, token, this.#options.leaseMs);
} catch {
throw new ClaimLockUnavailableError();
}
if (Number(renewed) !== 1) throw new ClaimLockLostError();
}
async #release(key: string, token: string): Promise<boolean> {
try {
return Number(await this.#client.eval(RELEASE_SCRIPT, 1, key, token)) === 1;
} catch {
throw new ClaimLockUnavailableError();
}
}
async #probeLockCapabilities(): Promise<void> {
const key = `${LOCK_KEY_PREFIX}readiness:${sha256Hex(randomUUID())}`;
const token = randomUUID();
let acquired = false;
let released = false;
try {
acquired = (await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK";
if (!acquired) throw new ClaimLockUnavailableError();
await this.#renew(key, token);
released = await this.#release(key, token);
if (!released) throw new ClaimLockLostError();
} finally {
if (acquired && !released) await this.#release(key, token).catch(() => undefined);
}
}
}
export function createRedisClaimMutationLock(connection: ConnectionOptions): ClaimMutationLock {
const client = new Redis({
...connection,
commandTimeout: DEFAULT_COMMAND_TIMEOUT_MS,
enableOfflineQueue: false,
enableReadyCheck: true,
lazyConnect: true,
maxRetriesPerRequest: 1,
});
return new RedisClaimMutationLock(client);
}
export function createFencedClaimMutationLock(connection: ConnectionOptions, filesystemDirectory: string): ClaimMutationLock {
return new FencedClaimMutationLock(new FilesystemClaimMutationLock(filesystemDirectory), createRedisClaimMutationLock(connection));
}
async function assertSameRegularFile(lockPath: string, handle: FileHandle): Promise<void> {
const [pathStats, openedStats] = await Promise.all([lstat(lockPath), handle.stat()]);
if (
!pathStats.isFile() ||
pathStats.isSymbolicLink() ||
pathStats.nlink !== 1 ||
!openedStats.isFile() ||
openedStats.nlink !== 1 ||
openedStats.dev !== pathStats.dev ||
openedStats.ino !== pathStats.ino
) {
throw new ClaimLockUnavailableError();
}
}
async function acquireKernelFlock(fd: number, binary: string, timeoutMs: number): Promise<void> {
const timeoutSeconds = Math.max(0.001, timeoutMs / 1_000).toFixed(3);
await new Promise<void>((resolve, reject) => {
const child = spawn(binary, ["--exclusive", "--wait", timeoutSeconds, "3"], {
stdio: ["ignore", "ignore", "ignore", fd],
});
let settled = false;
const watchdog = setTimeout(() => {
if (settled) return;
child.kill("SIGKILL");
}, timeoutMs + 1_000);
watchdog.unref();
child.once("error", () => {
if (settled) return;
settled = true;
clearTimeout(watchdog);
reject(new ClaimLockUnavailableError());
});
child.once("close", (code) => {
if (settled) return;
settled = true;
clearTimeout(watchdog);
if (code === 0) resolve();
else reject(new ClaimLockUnavailableError());
});
});
}
function noFollowFlag(): number {
return typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0;
}
function normalizeOptions(options: ClaimLockOptions): NormalizedClaimLockOptions {
const leaseMs = positiveInteger(options.leaseMs ?? DEFAULT_LEASE_MS, "claim lock lease");
const acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_ACQUIRE_TIMEOUT_MS, "claim lock acquisition timeout");
const retryDelayMs = nonNegativeInteger(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS, "claim lock retry delay");
const retryJitterMs = nonNegativeInteger(options.retryJitterMs ?? DEFAULT_RETRY_JITTER_MS, "claim lock retry jitter");
const heartbeatIntervalMs = positiveInteger(options.heartbeatIntervalMs ?? Math.floor(leaseMs / 3), "claim lock heartbeat interval");
if (heartbeatIntervalMs >= leaseMs / 2) throw new Error("claim lock heartbeat interval must be less than half the lease");
return {
leaseMs,
acquireTimeoutMs,
retryDelayMs,
retryJitterMs,
heartbeatIntervalMs,
now: options.now ?? Date.now,
random: options.random ?? Math.random,
token: options.token ?? randomUUID,
wait: options.wait ?? wait,
};
}
function positiveInteger(value: number, name: string): number {
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`);
return value;
}
function nonNegativeInteger(value: number, name: string): number {
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
return value;
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs));
}
@@ -0,0 +1,116 @@
import path from "node:path";
import { describe, expect, it } from "vitest";
import { parsePortalDomainControllerConfig, PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME } from "./config.js";
const baseEnv: NodeJS.ProcessEnv = {
PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: "command-secret-that-is-long-enough-for-hmac",
PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: "metrics-token-that-is-long-enough-to-be-safe",
PORTAL_DOMAIN_CONTROLLER_REDIS_URL: "rediss://controller:secret@redis.internal:6380/7",
PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: `/data/coolify/proxy/dynamic/${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}`,
PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: `/data/coolify/proxy/dynamic/${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`,
PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "https-0-joyyh5b89vpfp8ocri8xr96m@docker",
PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: "portal-ingress.tool-ctrl.de",
GIT_SHA: "abc123",
};
describe("portal-domain controller config", () => {
it("parses the least-privilege runtime contract without retaining a Redis URL", () => {
const config = parsePortalDomainControllerConfig(baseEnv);
expect(config.host).toBe("0.0.0.0");
expect(config.port).toBe(8080);
expect(config.queueConcurrency).toBe(2);
expect(config.queueConnection).toMatchObject({ host: "redis.internal", port: 6380, db: 7, tls: {} });
expect(config.entryPoint).toBe("https");
expect(config.backendService).toBe("https-0-joyyh5b89vpfp8ocri8xr96m@docker");
expect(config.reloadTriggerFile).toBe(`/data/coolify/proxy/dynamic/${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`);
expect(config).not.toHaveProperty("redisUrl");
});
it("accepts a configured safe entry point and rejects injectable routing values", () => {
expect(parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "portal-https" }).entryPoint).toBe("portal-https");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "https\nhttp:" })).toThrow("entry-point");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "http://portal:8080" })).toThrow("Docker service");
expect(() =>
parsePortalDomainControllerConfig({
...baseEnv,
PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "portal@docker\n middlewares: [evil]",
})
).toThrow("Docker service");
});
it("binds probes to the immutable image revision and refuses an unknown revision", () => {
expect(parsePortalDomainControllerConfig({ ...baseEnv, APP_REVISION: "release-from-image" }).revision).toBe("release-from-image");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, GIT_SHA: undefined, APP_REVISION: "unknown" })).toThrow("revision");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, GIT_SHA: undefined })).toThrow("revision");
});
it("accepts only Coolify's benign automatic runtime metadata", () => {
expect(() =>
parsePortalDomainControllerConfig({
...baseEnv,
COOLIFY_BRANCH: "main",
COOLIFY_CONTAINER_NAME: "hivegrid-portal-domain-controller-123",
COOLIFY_FQDN: "",
COOLIFY_RESOURCE_UUID: "joyyh5b89vpfp8ocri8xr96m",
COOLIFY_URL: "",
})
).not.toThrow();
for (const key of ["COOLIFY_API_URL", "COOLIFY_PROJECT_UUID", "COOLIFY_RESOURCE_UUID_EXTRA", "COOLIFY_URL_OVERRIDE"]) {
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, [key]: "must-not-be-here" })).toThrow(key);
}
});
it.each([
"DATABASE_URL",
"DATABASE_READ_URL",
"DIRECT_URL",
"DOCKER_HOST",
"COOLIFY_TOKEN",
"CLOUDFLARE_API_TOKEN",
"SSH_PRIVATE_KEY",
"MIGRATOR_PASSWORD",
"SEED_ON_BOOT",
"AUTHENTIK_ADMIN_JIT_DATABASE_URL",
"STORAGE_SECRET_ACCESS_KEY",
"MAILJET_API_KEY",
"MOLLIE_API_KEY",
"LAGO_API_KEY",
])("fails closed when %s is present", (key) => {
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, [key]: "must-not-be-here" })).toThrow(key);
});
it("requires strong secrets, a dedicated absolute directory and a concrete CNAME", () => {
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: "short" })).toThrow("32");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: "short" })).toThrow("32");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: "relative/path" })).toThrow("absolute");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: path.resolve("/tmp/not-dedicated") })).toThrow(
PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME
);
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: "relative.reload" })).toThrow("absolute");
expect(() =>
parsePortalDomainControllerConfig({
...baseEnv,
PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: `/data/coolify/proxy/${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`,
})
).toThrow("directly beside");
expect(() =>
parsePortalDomainControllerConfig({
...baseEnv,
PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: "/data/coolify/proxy/dynamic/traefik.yaml",
})
).toThrow(PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME);
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: "*.tool-ctrl.de" })).toThrow();
expect(() =>
parsePortalDomainControllerConfig({
...baseEnv,
PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: baseEnv.PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET,
})
).toThrow("must be different");
});
it("rejects injectable bind hosts and invalid concurrency", () => {
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_HOST: "evil.example.com" })).toThrow("bind address");
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_CONCURRENCY: "99" })).toThrow("concurrency");
});
});
@@ -0,0 +1,193 @@
import path from "node:path";
import type { ConnectionOptions } from "@hivegrid/queue";
import { parseRedisUrl } from "@hivegrid/queue";
import { normalizeDomainHostname } from "./contracts.js";
export const PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME = "hivegrid-portal-domains";
export const PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME = ".hivegrid-portal-domains.reload";
export const PORTAL_DOMAIN_DEFAULT_ENTRY_POINT = "https";
export const PORTAL_DOMAIN_TLS_CERT_RESOLVER = "letsencrypt";
export type PortalDomainControllerConfig = Readonly<{
host: string;
port: number;
revision: string;
configDirectory: string;
reloadTriggerFile: string;
entryPoint: string;
backendService: string;
expectedCname: string;
commandSecret: string;
metricsToken: string;
queueConnection: ConnectionOptions;
queueConcurrency: number;
}>;
const FORBIDDEN_EXACT_KEYS = new Set([
"DATABASE_URL",
"DIRECT_URL",
"DOCKER_HOST",
"PGHOST",
"PGPORT",
"PGDATABASE",
"PGUSER",
"PGPASSWORD",
"CLOUDFLARE_API_TOKEN",
"CLOUDFLARE_API_KEY",
"COOLIFY_TOKEN",
"SSH_AUTH_SOCK",
]);
const ALLOWED_COOLIFY_RUNTIME_KEYS = new Set([
"COOLIFY_BRANCH",
"COOLIFY_CONTAINER_NAME",
"COOLIFY_FQDN",
"COOLIFY_RESOURCE_UUID",
"COOLIFY_URL",
]);
const FORBIDDEN_PREFIXES = [
"DATABASE_",
"MIGRATOR_",
"SEED_",
"CLOUDFLARE_",
"SSH_",
"AUTHENTIK_",
"STORAGE_",
"MAILJET_",
"MOLLIE_",
"LAGO_",
"STRIPE_",
"MISTRAL_",
"GOOGLE_",
"MCAPTCHA_",
"CAP_",
"SENTRY_AUTH_",
"API_ACCESS_",
"API_KEY_",
"CRYPTO_",
"JWT_",
"SESSION_",
"AUDIT_",
] as const;
export function parsePortalDomainControllerConfig(env: NodeJS.ProcessEnv = process.env): PortalDomainControllerConfig {
rejectForbiddenCredentials(env);
const commandSecret = requireSecret(env.PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET, "PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET");
const metricsToken = requireSecret(env.PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN, "PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN");
if (commandSecret === metricsToken) throw new Error("controller command and metrics secrets must be different");
const redisUrl = requireValue(env.PORTAL_DOMAIN_CONTROLLER_REDIS_URL, "PORTAL_DOMAIN_CONTROLLER_REDIS_URL");
const configDirectory = parseConfigDirectory(env.PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR);
const reloadTriggerFile = parseReloadTriggerFile(env.PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE, configDirectory);
const entryPoint = normalizePortalDomainEntryPoint(env.PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT ?? PORTAL_DOMAIN_DEFAULT_ENTRY_POINT);
const backendService = normalizePortalDomainBackendService(
requireValue(env.PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE, "PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE")
);
const expectedCname = normalizeDomainHostname(requireValue(env.PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME, "PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME"));
const host = parseHost(env.PORTAL_DOMAIN_CONTROLLER_HOST);
const port = parseInteger(env.PORTAL_DOMAIN_CONTROLLER_PORT ?? env.PORT, 8080, 1, 65_535, "controller port");
const queueConcurrency = parseInteger(env.PORTAL_DOMAIN_CONTROLLER_CONCURRENCY, 2, 1, 16, "controller concurrency");
return Object.freeze({
host,
port,
revision: normalizeRevision(env.APP_REVISION ?? env.GIT_SHA ?? env.SENTRY_RELEASE),
configDirectory,
reloadTriggerFile,
entryPoint,
backendService,
expectedCname,
commandSecret,
metricsToken,
queueConnection: parseRedisUrl(redisUrl),
queueConcurrency,
});
}
export function normalizePortalDomainEntryPoint(value: string): string {
if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,62}$/u.test(value)) {
throw new Error("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT must be a safe Traefik entry-point name");
}
return value;
}
export function normalizePortalDomainBackendService(value: string): string {
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,126}@docker$/u.test(value)) {
throw new Error("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE must be a provider-qualified Docker service name");
}
return value;
}
export function rejectForbiddenCredentials(env: NodeJS.ProcessEnv): void {
const forbidden = Object.keys(env)
.filter((key) => env[key]?.trim())
.filter(
(key) =>
FORBIDDEN_EXACT_KEYS.has(key) ||
(key.startsWith("COOLIFY_") && !ALLOWED_COOLIFY_RUNTIME_KEYS.has(key)) ||
FORBIDDEN_PREFIXES.some((prefix) => key.startsWith(prefix))
)
.sort();
if (forbidden.length > 0) {
throw new Error(`portal domain controller refuses forbidden credentials: ${forbidden.join(", ")}`);
}
}
function parseConfigDirectory(raw: string | undefined): string {
const value = requireValue(raw, "PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR");
if (!path.isAbsolute(value)) throw new Error("PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR must be absolute");
const normalized = path.resolve(value);
if (path.basename(normalized) !== PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME) {
throw new Error(`PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR must end with ${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}`);
}
return normalized;
}
function parseReloadTriggerFile(raw: string | undefined, configDirectory: string): string {
const value = requireValue(raw, "PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE");
if (!path.isAbsolute(value)) throw new Error("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must be absolute");
const normalized = path.resolve(value);
if (path.basename(normalized) !== PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME) {
throw new Error(`PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must end with ${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`);
}
if (path.dirname(normalized) !== path.dirname(configDirectory)) {
throw new Error("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must be directly beside the dedicated config directory");
}
if (/\.ya?ml$/iu.test(normalized)) {
throw new Error("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must not be a Traefik configuration file");
}
return normalized;
}
function parseHost(raw: string | undefined): string {
const host = raw?.trim() || "0.0.0.0";
if (host !== "0.0.0.0" && host !== "127.0.0.1" && host !== "::1") {
throw new Error("PORTAL_DOMAIN_CONTROLLER_HOST must be a loopback or all-interface bind address");
}
return host;
}
function parseInteger(raw: string | undefined, fallback: number, min: number, max: number, name: string): number {
if (!raw?.trim()) return fallback;
const value = Number(raw);
if (!Number.isInteger(value) || value < min || value > max) throw new Error(`${name} must be an integer from ${min} to ${max}`);
return value;
}
function requireValue(raw: string | undefined, name: string): string {
const value = raw?.trim();
if (!value) throw new Error(`${name} is required`);
return value;
}
function requireSecret(raw: string | undefined, name: string): string {
const value = requireValue(raw, name);
const bytes = Buffer.byteLength(value, "utf8");
if (bytes < 32 || bytes > 512) throw new Error(`${name} must contain between 32 and 512 bytes`);
return value;
}
function normalizeRevision(value: string | undefined): string {
const revision = value?.trim();
if (!revision || revision === "unknown" || !/^[A-Za-z0-9._-]{1,128}$/u.test(revision)) {
throw new Error("controller release revision is required");
}
return revision;
}
@@ -0,0 +1,87 @@
import { describe, expect, it } from "vitest";
import {
canonicalJson,
createSignedDomainCommand,
createSignedDomainResult,
normalizeDomainHostname,
sha256Hex,
verifySignedDomainCommand,
verifySignedDomainResult,
} from "./contracts.js";
const SECRET = "test-command-secret-that-is-at-least-thirty-two-bytes";
const command = {
claimId: "claim_12345678",
tenantId: "tenant_12345678",
generation: 3,
hostname: "portal.example.com",
action: "provision" as const,
};
describe("portal-domain signed contracts", () => {
it("produces a deterministic hash and domain-separated signature", () => {
const first = createSignedDomainCommand(command, SECRET);
const second = createSignedDomainCommand({ ...command }, SECRET);
expect(first).toEqual(second);
expect(first.contentHash).toBe(sha256Hex(canonicalJson(first.payload)));
expect(verifySignedDomainCommand(first, SECRET)).toEqual(first.payload);
const result = createSignedDomainResult({
...command,
status: "active",
attempt: 1,
completedAt: "2026-07-16T10:00:00.000Z",
configHash: "a".repeat(64),
certificateExpiresAt: "2026-10-01T00:00:00.000Z",
}, SECRET);
expect(result.signature).not.toBe(first.signature);
expect(verifySignedDomainResult(result, SECRET).status).toBe("active");
});
it("rejects payload, hash, signature, algorithm and unknown-field tampering", () => {
const signed = createSignedDomainCommand(command, SECRET);
expect(() => verifySignedDomainCommand({ ...signed, payload: { ...signed.payload, generation: 4 } }, SECRET)).toThrow("content hash");
expect(() => verifySignedDomainCommand({ ...signed, contentHash: "0".repeat(64) }, SECRET)).toThrow("content hash");
expect(() => verifySignedDomainCommand({ ...signed, signature: "0".repeat(64) }, SECRET)).toThrow("signature");
expect(() => verifySignedDomainCommand({ ...signed, algorithm: "none" }, SECRET)).toThrow("algorithm");
expect(() => verifySignedDomainCommand({ ...signed, injected: true }, SECRET)).toThrow("unknown or missing");
expect(() => verifySignedDomainCommand({ ...signed, payload: { ...signed.payload, injected: true } }, SECRET)).toThrow("unknown or missing");
expect(() => verifySignedDomainCommand(signed, `${SECRET}!`)).toThrow("signature");
});
it("accepts only canonical, concrete fully qualified hostnames and bounded identifiers", () => {
expect(normalizeDomainHostname("Portal.Example.com.")).toBe("portal.example.com");
for (const hostname of ["*.example.com", "127.0.0.1", "localhost", "-bad.example.com", "bad_.example.com", "[::1]"]) {
expect(() => normalizeDomainHostname(hostname)).toThrow();
}
expect(() => createSignedDomainCommand({ ...command, hostname: "Portal.Example.com" }, SECRET)).toThrow("canonical");
expect(() => createSignedDomainCommand({ ...command, claimId: "../x" }, SECRET)).toThrow("claim id");
expect(() => createSignedDomainCommand({ ...command, generation: 0 }, SECRET)).toThrow("generation");
});
it("rejects undersized signing secrets and non-canonical result timestamps", () => {
expect(() => createSignedDomainCommand(command, "short")).toThrow("32");
expect(() => createSignedDomainResult({
...command,
status: "active",
attempt: 1,
completedAt: "2026-07-16",
}, SECRET)).toThrow("timestamp");
expect(() => createSignedDomainResult({
...command,
status: "active",
attempt: 1,
completedAt: "2026-07-16T10:00:00.000Z",
configHash: "not-a-sha256",
certificateExpiresAt: "2026-10-01T00:00:00.000Z",
}, SECRET)).toThrow("configHash");
expect(() => createSignedDomainResult({
...command,
status: "active",
attempt: 1,
completedAt: "2026-07-16T10:00:00.000Z",
configHash: "a".repeat(64),
certificateExpiresAt: "2026-07-16T09:00:00.000Z",
}, SECRET)).toThrow("expire after observation");
});
});
@@ -0,0 +1,248 @@
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
import { isIP } from "node:net";
export const DOMAIN_COMMAND_QUEUE = "hivegrid.portal-domain.commands";
export const DOMAIN_RESULT_QUEUE = "hivegrid.portal-domain.results";
export const DOMAIN_COMMAND_QUEUE_ATTEMPTS = 4;
export const DOMAIN_CONTRACT_VERSION = 1 as const;
export const DOMAIN_COMMAND_ACTIONS = ["provision", "revalidate", "deprovision"] as const;
export const DOMAIN_RESULT_STATUSES = ["active", "healthy", "deprovisioned", "retrying", "failed", "stale"] as const;
export type DomainCommandAction = (typeof DOMAIN_COMMAND_ACTIONS)[number];
export type DomainResultStatus = (typeof DOMAIN_RESULT_STATUSES)[number];
export type DomainCommandPayload = {
version: typeof DOMAIN_CONTRACT_VERSION;
claimId: string;
tenantId: string;
generation: number;
hostname: string;
action: DomainCommandAction;
};
export type DomainResultPayload = DomainCommandPayload & {
status: DomainResultStatus;
attempt: number;
completedAt: string;
retryAfterMs?: number;
reasonCode?: string;
configHash?: string;
certificateExpiresAt?: string;
};
export type SignedDomainEnvelope<Payload> = {
algorithm: "hmac-sha256";
contentHash: string;
payload: Payload;
signature: string;
};
const COMMAND_KEYS = ["action", "claimId", "generation", "hostname", "tenantId", "version"] as const;
const RESULT_REQUIRED_KEYS = [...COMMAND_KEYS, "attempt", "completedAt", "status"] as const;
const RESULT_OPTIONAL_KEYS = ["certificateExpiresAt", "configHash", "reasonCode", "retryAfterMs"] as const;
const ENVELOPE_KEYS = ["algorithm", "contentHash", "payload", "signature"] as const;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{7,127}$/u;
const HEX_SHA256_PATTERN = /^[a-f0-9]{64}$/u;
const HOST_LABEL_PATTERN = /^(?!-)[a-z0-9-]{1,63}(?<!-)$/u;
const MAX_HOSTNAME_BYTES = 253;
export function createSignedDomainCommand(
input: Omit<DomainCommandPayload, "version"> | DomainCommandPayload,
secret: string
): SignedDomainEnvelope<DomainCommandPayload> {
const payload = parseDomainCommandPayload({ ...input, version: DOMAIN_CONTRACT_VERSION });
return signEnvelope(payload, secret, "command");
}
export function createSignedDomainResult(
input: Omit<DomainResultPayload, "version"> | DomainResultPayload,
secret: string
): SignedDomainEnvelope<DomainResultPayload> {
const payload = parseDomainResultPayload({ ...input, version: DOMAIN_CONTRACT_VERSION });
return signEnvelope(payload, secret, "result");
}
export function verifySignedDomainCommand(value: unknown, secret: string): DomainCommandPayload {
const envelope = parseEnvelope(value);
const payload = parseDomainCommandPayload(envelope.payload);
verifyEnvelope(envelope, payload, secret, "command");
return payload;
}
export function verifySignedDomainResult(value: unknown, secret: string): DomainResultPayload {
const envelope = parseEnvelope(value);
const payload = parseDomainResultPayload(envelope.payload);
verifyEnvelope(envelope, payload, secret, "result");
return payload;
}
export function normalizeDomainHostname(value: string): string {
const hostname = value.trim().toLowerCase().replace(/\.$/u, "");
if (!hostname || hostname.includes("*") || hostname.includes(":") || isIP(hostname) !== 0) {
throw new Error("hostname must be a concrete DNS name");
}
if (Buffer.byteLength(hostname, "ascii") > MAX_HOSTNAME_BYTES || !hostname.includes(".")) {
throw new Error("hostname must be a valid fully qualified DNS name");
}
const labels = hostname.split(".");
if (labels.some((label) => !HOST_LABEL_PATTERN.test(label))) {
throw new Error("hostname contains an invalid DNS label");
}
return hostname;
}
export function canonicalJson(value: unknown): string {
if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
if (typeof value === "number") {
if (!Number.isFinite(value)) throw new Error("non-finite numbers are not supported");
return JSON.stringify(value);
}
if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
if (isRecord(value)) {
return `{${Object.keys(value)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
.join(",")}}`;
}
throw new Error("unsupported canonical JSON value");
}
export function sha256Hex(value: string | Buffer): string {
return createHash("sha256").update(value).digest("hex");
}
function signEnvelope<Payload extends DomainCommandPayload | DomainResultPayload>(
payload: Payload,
secret: string,
kind: "command" | "result"
): SignedDomainEnvelope<Payload> {
assertSigningSecret(secret);
const contentHash = sha256Hex(canonicalJson(payload));
return {
algorithm: "hmac-sha256",
contentHash,
payload,
signature: hmac(secret, kind, contentHash),
};
}
function verifyEnvelope<Payload extends DomainCommandPayload | DomainResultPayload>(
envelope: SignedDomainEnvelope<unknown>,
payload: Payload,
secret: string,
kind: "command" | "result"
): void {
assertSigningSecret(secret);
const actualContentHash = sha256Hex(canonicalJson(payload));
if (!safeHexEqual(envelope.contentHash, actualContentHash)) throw new Error("domain envelope content hash mismatch");
const expectedSignature = hmac(secret, kind, actualContentHash);
if (!safeHexEqual(envelope.signature, expectedSignature)) throw new Error("domain envelope signature mismatch");
}
function hmac(secret: string, kind: "command" | "result", contentHash: string): string {
return createHmac("sha256", secret).update(`hivegrid.portal-domain.${kind}.v1:${contentHash}`, "utf8").digest("hex");
}
function safeHexEqual(actual: string, expected: string): boolean {
if (!HEX_SHA256_PATTERN.test(actual) || !HEX_SHA256_PATTERN.test(expected)) return false;
return timingSafeEqual(Buffer.from(actual, "hex"), Buffer.from(expected, "hex"));
}
function parseEnvelope(value: unknown): SignedDomainEnvelope<unknown> {
assertExactKeys(value, ENVELOPE_KEYS, []);
if (value.algorithm !== "hmac-sha256") throw new Error("unsupported domain envelope algorithm");
if (typeof value.contentHash !== "string" || !HEX_SHA256_PATTERN.test(value.contentHash)) throw new Error("invalid domain envelope content hash");
if (typeof value.signature !== "string" || !HEX_SHA256_PATTERN.test(value.signature)) throw new Error("invalid domain envelope signature");
return value as SignedDomainEnvelope<unknown>;
}
function parseDomainCommandPayload(value: unknown): DomainCommandPayload {
assertExactKeys(value, COMMAND_KEYS, []);
return parseCommandFields(value);
}
function parseDomainResultPayload(value: unknown): DomainResultPayload {
assertExactKeys(value, RESULT_REQUIRED_KEYS, RESULT_OPTIONAL_KEYS);
const command = parseCommandFields(value);
if (!DOMAIN_RESULT_STATUSES.includes(value.status as DomainResultStatus)) throw new Error("invalid domain result status");
if (!Number.isInteger(value.attempt) || (value.attempt as number) < 1 || (value.attempt as number) > 100) throw new Error("invalid domain result attempt");
if (typeof value.completedAt !== "string" || !isCanonicalTimestamp(value.completedAt)) throw new Error("invalid domain result timestamp");
if (value.retryAfterMs !== undefined && (!Number.isInteger(value.retryAfterMs) || (value.retryAfterMs as number) < 1)) {
throw new Error("invalid domain result retry delay");
}
for (const key of ["reasonCode", "configHash", "certificateExpiresAt"] as const) {
if (value[key] !== undefined && (typeof value[key] !== "string" || value[key].length < 1 || value[key].length > 128)) {
throw new Error(`invalid domain result ${key}`);
}
}
if (value.reasonCode !== undefined && !/^[a-z0-9_]{1,64}$/u.test(value.reasonCode as string)) {
throw new Error("invalid domain result reasonCode");
}
if (value.configHash !== undefined && !HEX_SHA256_PATTERN.test(value.configHash as string)) {
throw new Error("invalid domain result configHash");
}
if (value.certificateExpiresAt !== undefined && !isCanonicalTimestamp(value.certificateExpiresAt as string)) {
throw new Error("invalid domain result certificateExpiresAt");
}
if (value.status === "active" && value.configHash === undefined) {
throw new Error("active domain result requires configHash");
}
if ((value.status === "active" || value.status === "healthy") && value.certificateExpiresAt === undefined) {
throw new Error("healthy domain result requires certificateExpiresAt");
}
if (value.certificateExpiresAt !== undefined && Date.parse(value.certificateExpiresAt as string) <= Date.parse(value.completedAt as string)) {
throw new Error("domain result certificate must expire after observation");
}
return {
...command,
status: value.status as DomainResultStatus,
attempt: value.attempt as number,
completedAt: value.completedAt,
...(value.retryAfterMs === undefined ? {} : { retryAfterMs: value.retryAfterMs as number }),
...(value.reasonCode === undefined ? {} : { reasonCode: value.reasonCode as string }),
...(value.configHash === undefined ? {} : { configHash: value.configHash as string }),
...(value.certificateExpiresAt === undefined ? {} : { certificateExpiresAt: value.certificateExpiresAt as string }),
};
}
function parseCommandFields(value: Record<string, unknown>): DomainCommandPayload {
if (value.version !== DOMAIN_CONTRACT_VERSION) throw new Error("unsupported domain contract version");
if (typeof value.claimId !== "string" || !IDENTIFIER_PATTERN.test(value.claimId)) throw new Error("invalid domain claim id");
if (typeof value.tenantId !== "string" || !IDENTIFIER_PATTERN.test(value.tenantId)) throw new Error("invalid domain tenant id");
if (!Number.isSafeInteger(value.generation) || (value.generation as number) < 1) throw new Error("invalid domain generation");
if (typeof value.hostname !== "string") throw new Error("invalid domain hostname");
const hostname = normalizeDomainHostname(value.hostname);
if (hostname !== value.hostname) throw new Error("domain hostname must already be canonical");
if (!DOMAIN_COMMAND_ACTIONS.includes(value.action as DomainCommandAction)) throw new Error("invalid domain command action");
return {
version: DOMAIN_CONTRACT_VERSION,
claimId: value.claimId,
tenantId: value.tenantId,
generation: value.generation as number,
hostname,
action: value.action as DomainCommandAction,
};
}
function assertExactKeys(value: unknown, required: readonly string[], optional: readonly string[]): asserts value is Record<string, unknown> {
if (!isRecord(value)) throw new Error("domain contract value must be an object");
const keys = Object.keys(value).sort();
const allowed = new Set([...required, ...optional]);
const unknown = keys.filter((key) => !allowed.has(key));
const missing = required.filter((key) => !Object.hasOwn(value, key));
if (unknown.length > 0 || missing.length > 0) throw new Error("domain contract contains unknown or missing fields");
}
function assertSigningSecret(secret: string): void {
const bytes = Buffer.byteLength(secret, "utf8");
if (bytes < 32 || bytes > 512) throw new Error("domain command secret must contain between 32 and 512 bytes");
}
function isCanonicalTimestamp(value: string): boolean {
const date = new Date(value);
return Number.isFinite(date.getTime()) && date.toISOString() === value;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
@@ -0,0 +1,149 @@
import { describe, expect, it, vi } from "vitest";
import type { DomainCommandPayload } from "./contracts.js";
import { createPortalDomainController, DomainControllerFailure, DOMAIN_MAX_ATTEMPTS, retryDelayForAttempt } from "./controller.js";
import { DomainProbeError, type DomainProbes } from "./probes.js";
import type { TraefikConfigWriter } from "./traefik.js";
const command: DomainCommandPayload = {
version: 1,
claimId: "claim_12345678",
tenantId: "tenant_12345678",
generation: 2,
hostname: "portal.example.com",
action: "provision",
};
const CERTIFICATE_EXPIRES_AT = "2026-10-01T00:00:00.000Z";
describe("portal-domain lifecycle controller", () => {
it("does not acknowledge provision until DNS, atomic config and HTTPS succeed", async () => {
const calls: string[] = [];
const writer = {
applyProvision: vi.fn(async () => {
calls.push("write");
return { disposition: "applied", configHash: "hash" };
}),
readAppliedState: vi.fn(async () => ({ generation: command.generation, action: command.action, hostname: command.hostname, tenantId: command.tenantId })),
} as unknown as TraefikConfigWriter;
const probes: DomainProbes = {
verifyDns: vi.fn(async () => {
calls.push("dns");
}),
verifyHttps: vi.fn(async () => {
calls.push("https");
return { certificateExpiresAt: CERTIFICATE_EXPIRES_AT };
}),
};
const controller = createPortalDomainController({ writer, probes, now: () => new Date("2026-07-16T10:00:00.000Z") });
await expect(controller.handle(command, 1)).resolves.toMatchObject({
status: "active",
configHash: "hash",
certificateExpiresAt: CERTIFICATE_EXPIRES_AT,
attempt: 1,
});
expect(calls).toEqual(["dns", "write", "https"]);
});
it("returns stale without attempting HTTPS and deprovisions without DNS", async () => {
const writer = {
applyProvision: vi.fn().mockResolvedValue({ disposition: "stale" }),
applyDeprovision: vi.fn().mockResolvedValue({ disposition: "applied" }),
} as unknown as TraefikConfigWriter;
const probes: DomainProbes = { verifyDns: vi.fn(), verifyHttps: vi.fn() };
const controller = createPortalDomainController({ writer, probes });
await expect(controller.handle(command, 1)).resolves.toMatchObject({ status: "stale" });
expect(probes.verifyHttps).not.toHaveBeenCalled();
await expect(controller.handle({ ...command, action: "deprovision" }, 1)).resolves.toMatchObject({ status: "deprovisioned" });
expect(probes.verifyDns).toHaveBeenCalledTimes(1);
});
it("repairs routing before HTTPS and rechecks ownership after the probe", async () => {
const calls: string[] = [];
let routingRepaired = false;
const writer = {
readAppliedState: vi.fn(async () => {
calls.push("state");
return { generation: command.generation, action: "provision", hostname: command.hostname };
}),
recordRevalidation: vi.fn(async () => {
calls.push("repair");
routingRepaired = true;
return { disposition: "unchanged" };
}),
} as unknown as TraefikConfigWriter;
const probes: DomainProbes = {
verifyDns: vi.fn(async () => {
calls.push("dns");
}),
verifyHttps: vi.fn(async () => {
calls.push("https");
if (!routingRepaired) throw new DomainProbeError("https_health_failed", "stale backend");
return { certificateExpiresAt: CERTIFICATE_EXPIRES_AT };
}),
};
const controller = createPortalDomainController({ writer, probes });
await expect(controller.handle({ ...command, action: "revalidate" }, 2)).resolves.toMatchObject({
status: "healthy",
certificateExpiresAt: CERTIFICATE_EXPIRES_AT,
});
expect(calls).toEqual(["dns", "repair", "https", "state"]);
});
it("returns stale before HTTPS and after a concurrent deprovision during the probe", async () => {
const staleWriter = {
recordRevalidation: vi.fn().mockResolvedValue({ disposition: "stale" }),
} as unknown as TraefikConfigWriter;
const staleProbes: DomainProbes = { verifyDns: vi.fn(), verifyHttps: vi.fn() };
const staleController = createPortalDomainController({ writer: staleWriter, probes: staleProbes });
await expect(staleController.handle({ ...command, action: "revalidate" }, 1)).resolves.toMatchObject({ status: "stale" });
expect(staleProbes.verifyHttps).not.toHaveBeenCalled();
const racedWriter = {
recordRevalidation: vi.fn().mockResolvedValue({ disposition: "unchanged" }),
readAppliedState: vi.fn().mockResolvedValue({
generation: command.generation + 1,
action: "deprovision",
hostname: command.hostname,
tenantId: command.tenantId,
}),
} as unknown as TraefikConfigWriter;
const racedProbes: DomainProbes = {
verifyDns: vi.fn(),
verifyHttps: vi.fn().mockResolvedValue({ certificateExpiresAt: CERTIFICATE_EXPIRES_AT }),
};
const racedController = createPortalDomainController({ writer: racedWriter, probes: racedProbes });
await expect(racedController.handle({ ...command, action: "revalidate" }, 1)).resolves.toMatchObject({ status: "stale" });
});
it("does not emit an active acknowledgement if a newer generation wins during HTTPS probing", async () => {
const writer = {
applyProvision: vi.fn().mockResolvedValue({ disposition: "applied", configHash: "hash" }),
readAppliedState: vi.fn().mockResolvedValue({ generation: 3, action: "deprovision", hostname: command.hostname, tenantId: command.tenantId }),
} as unknown as TraefikConfigWriter;
const probes: DomainProbes = { verifyDns: vi.fn(), verifyHttps: vi.fn() };
const controller = createPortalDomainController({ writer, probes });
await expect(controller.handle(command, 1)).resolves.toMatchObject({ status: "stale" });
});
it("classifies probe failures as retryable without leaking the underlying error", async () => {
const writer = {} as TraefikConfigWriter;
const probes: DomainProbes = {
verifyDns: vi.fn().mockRejectedValue(new DomainProbeError("dns_lookup_failed", "DNS provider detail")),
verifyHttps: vi.fn(),
};
const controller = createPortalDomainController({ writer, probes });
await expect(controller.handle(command, 1)).rejects.toEqual(
expect.objectContaining({
reasonCode: "dns_lookup_failed",
retryable: true,
})
);
await controller.handle(command, 1).catch((error: DomainControllerFailure) => {
expect(error).toBeInstanceOf(DomainControllerFailure);
});
});
it("uses the required 15m, 1h, 6h retry schedule", () => {
expect(DOMAIN_MAX_ATTEMPTS).toBe(4);
expect([1, 2, 3, 4].map(retryDelayForAttempt)).toEqual([900_000, 3_600_000, 21_600_000, 21_600_000]);
});
});
@@ -0,0 +1,97 @@
import type { DomainCommandPayload, DomainResultPayload } from "./contracts.js";
import type { DomainProbes } from "./probes.js";
import { DomainProbeError } from "./probes.js";
import { GenerationConflictError, TraefikConfigWriter, UnsafeConfigPathError } from "./traefik.js";
export const DOMAIN_RETRY_DELAYS_MS = [15 * 60_000, 60 * 60_000, 6 * 60 * 60_000] as const;
export const DOMAIN_MAX_ATTEMPTS = DOMAIN_RETRY_DELAYS_MS.length + 1;
export class DomainControllerFailure extends Error {
constructor(
readonly reasonCode: string,
readonly retryable: boolean,
message: string
) {
super(message);
}
}
export type PortalDomainController = {
handle(command: DomainCommandPayload, attempt: number): Promise<DomainResultPayload>;
};
export function createPortalDomainController(options: { writer: TraefikConfigWriter; probes: DomainProbes; now?: () => Date }): PortalDomainController {
const now = options.now ?? (() => new Date());
return {
async handle(command, attempt) {
try {
if (command.action === "deprovision") {
const mutation = await options.writer.applyDeprovision(command);
return result(command, attempt, now(), mutation.disposition === "stale" ? "stale" : "deprovisioned");
}
await options.probes.verifyDns(command.hostname);
if (command.action === "provision") {
const mutation = await options.writer.applyProvision(command);
if (mutation.disposition === "stale") return result(command, attempt, now(), "stale");
const health = await options.probes.verifyHttps(command.hostname);
const current = await options.writer.readAppliedState(command.claimId);
if (!current || current.generation !== command.generation || current.action !== command.action) {
return result(command, attempt, now(), "stale");
}
return result(command, attempt, now(), "active", mutation.configHash, health.certificateExpiresAt);
}
const mutation = await options.writer.recordRevalidation(command);
if (mutation.disposition === "stale") return result(command, attempt, now(), "stale");
const health = await options.probes.verifyHttps(command.hostname);
const current = await options.writer.readAppliedState(command.claimId);
if (!current || current.generation !== command.generation || (current.action !== "provision" && current.action !== "revalidate")) {
return result(command, attempt, now(), "stale");
}
return result(command, attempt, now(), "healthy", undefined, health.certificateExpiresAt);
} catch (error) {
throw classifyControllerError(error);
}
},
};
}
export function retryDelayForAttempt(attemptsMade: number): number {
const index = Math.max(0, Math.min(attemptsMade - 1, DOMAIN_RETRY_DELAYS_MS.length - 1));
return DOMAIN_RETRY_DELAYS_MS[index] ?? DOMAIN_RETRY_DELAYS_MS[DOMAIN_RETRY_DELAYS_MS.length - 1];
}
export function classifyControllerError(error: unknown): DomainControllerFailure {
if (error instanceof DomainControllerFailure) return error;
if (error instanceof DomainProbeError) return new DomainControllerFailure(error.reasonCode, true, error.message);
if (error instanceof UnsafeConfigPathError || error instanceof GenerationConflictError) {
return new DomainControllerFailure(error.reasonCode, false, error.message);
}
if (isErrno(error, "EACCES") || isErrno(error, "EPERM") || isErrno(error, "EROFS")) {
return new DomainControllerFailure("config_permission_denied", false, "The dedicated Traefik config directory is not writable");
}
return new DomainControllerFailure("controller_internal_error", true, "The portal domain controller encountered a transient internal error");
}
function result(
command: DomainCommandPayload,
attempt: number,
completedAt: Date,
status: DomainResultPayload["status"],
configHash?: string,
certificateExpiresAt?: string
): DomainResultPayload {
return {
...command,
status,
attempt,
completedAt: completedAt.toISOString(),
...(configHash ? { configHash } : {}),
...(certificateExpiresAt ? { certificateExpiresAt } : {}),
};
}
function isErrno(error: unknown, code: string): boolean {
return typeof error === "object" && error !== null && "code" in error && (error as NodeJS.ErrnoException).code === code;
}
@@ -0,0 +1,134 @@
import { describe, expect, test, vi } from "vitest";
import { runControllerDeployHealthCli } from "./deploy-health.js";
function createIo() {
let stderr = "";
let stdout = "";
return {
io: {
stderr: {
write(chunk: string) {
stderr += chunk;
return true;
},
},
stdout: {
write(chunk: string) {
stdout += chunk;
return true;
},
},
},
readStderr: () => stderr,
readStdout: () => stdout,
};
}
function jsonResponse(payload: Record<string, unknown>, status = 200) {
return { status, json: vi.fn(async () => payload) };
}
function healthyResponses(revision: string) {
return [
jsonResponse({ status: "ok", service: "hivegrid-portal-domain-controller", revision }),
jsonResponse({ status: "ready", service: "hivegrid-portal-domain-controller" }),
];
}
describe("portal-domain controller deployment health", () => {
test("proves liveness, readiness and the exact baked image revision", async () => {
const revision = "a".repeat(40);
const responses = healthyResponses(revision);
const fetchProbe = vi.fn(async (_url: string, _init: RequestInit) => responses.shift()!);
const { io, readStderr, readStdout } = createIo();
const exitCode = await runControllerDeployHealthCli(
revision,
{ APP_REVISION: revision, PORTAL_DOMAIN_CONTROLLER_PORT: "8081" },
io,
{
fetchProbe,
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
}
);
expect(exitCode).toBe(0);
expect(JSON.parse(readStdout())).toEqual({
status: "ready",
service: "hivegrid-portal-domain-controller",
revision,
});
expect(readStderr()).toBe("");
expect(fetchProbe.mock.calls.map(([url]) => url)).toEqual([
"http://127.0.0.1:8081/healthz",
"http://127.0.0.1:8081/readyz",
]);
});
test("rejects malformed revisions, baked-image drift and invalid ports before probing", async () => {
const revision = "a".repeat(40);
const cases = [
{ expected: "not-a-sha", env: { APP_REVISION: revision }, baked: revision },
{ expected: revision, env: { APP_REVISION: revision }, baked: "b".repeat(40) },
{ expected: revision, env: { APP_REVISION: "b".repeat(40) }, baked: revision },
{ expected: revision, env: { APP_REVISION: revision, PORT: "70000" }, baked: revision },
];
for (const scenario of cases) {
const fetchProbe = vi.fn();
const { io, readStderr, readStdout } = createIo();
const exitCode = await runControllerDeployHealthCli(scenario.expected, scenario.env, io, {
fetchProbe,
readImageRevision: () => scenario.baked,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).not.toContain(scenario.expected);
expect(fetchProbe).not.toHaveBeenCalled();
}
});
test("rejects unhealthy or mismatched controller probe responses", async () => {
const revision = "a".repeat(40);
const scenarios = [
[jsonResponse({}, 503), healthyResponses(revision)[1]],
[healthyResponses(revision)[0], jsonResponse({}, 503)],
[jsonResponse({ status: "ok", service: "wrong", revision }), healthyResponses(revision)[1]],
[jsonResponse({ status: "ok", service: "hivegrid-portal-domain-controller", revision: "b".repeat(40) }), healthyResponses(revision)[1]],
[healthyResponses(revision)[0], jsonResponse({ status: "not_ready", service: "hivegrid-portal-domain-controller" })],
];
for (const responses of scenarios) {
const pending = [...responses];
const { io, readStderr, readStdout } = createIo();
const exitCode = await runControllerDeployHealthCli(revision, { APP_REVISION: revision }, io, {
fetchProbe: vi.fn(async () => pending.shift()!),
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/health|probes/);
}
});
test("redacts transport and payload errors", async () => {
const revision = "a".repeat(40);
const { io, readStderr, readStdout } = createIo();
const exitCode = await runControllerDeployHealthCli(revision, { APP_REVISION: revision }, io, {
fetchProbe: vi.fn(async () => {
throw new Error("connection failed with secret details");
}),
readImageRevision: () => revision,
timeoutSignal: () => new AbortController().signal,
});
expect(exitCode).toBe(1);
expect(readStdout()).toBe("");
expect(readStderr()).toMatch(/probe failed/);
expect(readStderr()).not.toContain("secret details");
});
});
@@ -0,0 +1,101 @@
import { readFileSync } from "node:fs";
const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/;
const IMAGE_REVISION_PATH = "/myapp/.image-revision";
const DEFAULT_CONTROLLER_PORT = "8080";
const HEALTH_TIMEOUT_MS = 9_000;
const CONTROLLER_SERVICE = "hivegrid-portal-domain-controller";
type ControllerDeployHealthIo = {
stderr?: Pick<NodeJS.WriteStream, "write">;
stdout: Pick<NodeJS.WriteStream, "write">;
};
type ControllerDeployHealthDependencies = {
fetchProbe?: (url: string, init: RequestInit) => Promise<Pick<Response, "status" | "json">>;
readImageRevision?: () => string;
timeoutSignal?: () => AbortSignal;
};
type ControllerProbeResponse = {
status?: unknown;
service?: unknown;
revision?: unknown;
};
function fail(io: ControllerDeployHealthIo, message: string): number {
io.stderr?.write(`${message}\n`);
return 1;
}
export async function runControllerDeployHealthCli(
expectedRevision: string,
env: NodeJS.ProcessEnv = process.env,
io: ControllerDeployHealthIo = process,
dependencies: ControllerDeployHealthDependencies = {}
): Promise<number> {
if (!FULL_GIT_SHA_PATTERN.test(expectedRevision)) {
return fail(io, "Portal-domain controller deployment health requires a full lowercase Git SHA.");
}
let imageRevision = "";
try {
imageRevision = (dependencies.readImageRevision ?? (() => readFileSync(IMAGE_REVISION_PATH, "utf8")))().trim();
} catch {
return fail(io, "Portal-domain controller deployment image revision is unavailable.");
}
if (imageRevision !== expectedRevision || env.APP_REVISION !== expectedRevision) {
return fail(io, "Portal-domain controller deployment revision does not match the expected image.");
}
const port = env.PORTAL_DOMAIN_CONTROLLER_PORT || env.PORT || DEFAULT_CONTROLLER_PORT;
if (!/^\d{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65_535) {
return fail(io, "Portal-domain controller deployment health port is invalid.");
}
try {
const fetchProbe = dependencies.fetchProbe ?? fetch;
const timeoutSignal = dependencies.timeoutSignal ?? (() => AbortSignal.timeout(HEALTH_TIMEOUT_MS));
const [healthResponse, readinessResponse] = await Promise.all([
fetchProbe(`http://127.0.0.1:${port}/healthz`, {
headers: { Accept: "application/json" },
signal: timeoutSignal(),
}),
fetchProbe(`http://127.0.0.1:${port}/readyz`, {
headers: { Accept: "application/json" },
signal: timeoutSignal(),
}),
]);
if (healthResponse.status !== 200 || readinessResponse.status !== 200) {
return fail(io, "Portal-domain controller deployment health or readiness is not healthy.");
}
const [health, readiness] = (await Promise.all([healthResponse.json(), readinessResponse.json()])) as [
ControllerProbeResponse,
ControllerProbeResponse,
];
if (
health.status !== "ok" ||
health.service !== CONTROLLER_SERVICE ||
health.revision !== expectedRevision ||
readiness.status !== "ready" ||
readiness.service !== CONTROLLER_SERVICE
) {
return fail(io, "Portal-domain controller deployment probes do not match the expected runtime.");
}
} catch {
return fail(io, "Portal-domain controller deployment probe failed.");
}
io.stdout.write(`${JSON.stringify({ status: "ready", service: CONTROLLER_SERVICE, revision: expectedRevision })}\n`);
return 0;
}
if (import.meta.url === `file://${process.argv[1]}`) {
runControllerDeployHealthCli(process.argv[2] || "")
.then((exitCode) => {
process.exitCode = exitCode;
})
.catch(() => {
process.exitCode = 1;
});
}
@@ -0,0 +1,26 @@
import {
createSignedDomainCommand,
DOMAIN_COMMAND_QUEUE,
} from "@hivegrid/portal-domain-controller/contracts";
import {
DOMAIN_COMMAND_JOB_OPTIONS,
} from "@hivegrid/portal-domain-controller/queue";
import { describe, expect, it } from "vitest";
describe("portal-domain controller package exports", () => {
it("exposes one shared signed protocol and its queue retry contract", () => {
const command = createSignedDomainCommand({
claimId: "claim_12345678",
tenantId: "tenant_12345678",
generation: 1,
hostname: "portal.example.com",
action: "provision",
}, "package-export-secret-that-is-at-least-thirty-two-bytes");
expect(command.payload.version).toBe(1);
expect(DOMAIN_COMMAND_QUEUE).toBe("hivegrid.portal-domain.commands");
expect(DOMAIN_COMMAND_JOB_OPTIONS).toMatchObject({
attempts: 4,
backoff: { type: "portal-domain-schedule" },
});
});
});
@@ -0,0 +1,46 @@
import { createLogger } from "@hivegrid/logger";
import { afterEach, describe, expect, it } from "vitest";
import { createControllerHttpRuntime, type ControllerHttpRuntime } from "./http.js";
import { createControllerMetrics } from "./metrics.js";
const METRICS_TOKEN = "metrics-token-that-is-at-least-thirty-two-bytes";
const runtimes: ControllerHttpRuntime[] = [];
afterEach(async () => {
await Promise.all(runtimes.splice(0).map((runtime) => runtime.close()));
});
describe("portal-domain controller HTTP runtime", () => {
it("serves liveness, fail-closed readiness and a bearer-protected metrics endpoint", async () => {
let ready = false;
const runtime = createControllerHttpRuntime({
host: "127.0.0.1",
port: 0,
revision: "abc123",
metricsToken: METRICS_TOKEN,
isReady: async () => ready,
logger: createLogger({ level: "silent" }),
metrics: createControllerMetrics("test", false),
});
runtimes.push(runtime);
const address = await runtime.listen();
const origin = `http://127.0.0.1:${address.port}`;
const health = await fetch(`${origin}/healthz`);
expect(health.status).toBe(200);
expect(await health.json()).toEqual({ status: "ok", service: "hivegrid-portal-domain-controller", revision: "abc123" });
expect(health.headers.get("x-content-type-options")).toBe("nosniff");
expect((await fetch(`${origin}/readyz`)).status).toBe(503);
ready = true;
expect((await fetch(`${origin}/readyz`)).status).toBe(200);
expect((await fetch(`${origin}/metrics`)).status).toBe(401);
expect((await fetch(`${origin}/metrics`, { headers: { authorization: "Bearer wrong" } })).status).toBe(401);
const metrics = await fetch(`${origin}/metrics`, { headers: { authorization: `Bearer ${METRICS_TOKEN}` } });
expect(metrics.status).toBe(200);
expect(await metrics.text()).toContain("hivegrid_portal_domain_controller_ready");
expect((await fetch(`${origin}/unknown`)).status).toBe(404);
expect((await fetch(`${origin}/healthz`, { method: "POST" })).status).toBe(405);
});
});
@@ -0,0 +1,132 @@
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import type { HiveGridLogger } from "@hivegrid/logger";
import { validateMetricsTokenAuth } from "@hivegrid/metrics";
import type { ControllerMetrics } from "./metrics.js";
export type ControllerHttpRuntime = {
server: Server;
listen(): Promise<AddressInfo>;
close(): Promise<void>;
};
export function createControllerHttpRuntime(options: {
host: string;
port: number;
revision: string;
metricsToken: string;
isReady: () => Promise<boolean>;
logger: HiveGridLogger;
metrics: ControllerMetrics;
}): ControllerHttpRuntime {
const server = createServer(async (request, response) => {
try {
const pathname = new URL(request.url ?? "/", "http://controller.invalid").pathname;
setSecurityHeaders(response);
if (request.method !== "GET") {
sendJson(response, 405, { error: "method_not_allowed" }, { allow: "GET" });
return;
}
if (pathname === "/healthz") {
sendJson(response, 200, {
status: "ok",
service: "hivegrid-portal-domain-controller",
revision: options.revision,
});
return;
}
if (pathname === "/readyz") {
let ready = false;
try {
ready = await options.isReady();
} catch (error) {
options.logger.warn({ err: error }, "portal-domain readiness check failed");
}
options.metrics.ready.set(ready ? 1 : 0);
sendJson(response, ready ? 200 : 503, {
status: ready ? "ready" : "not_ready",
service: "hivegrid-portal-domain-controller",
});
return;
}
if (pathname === "/metrics") {
const auth = validateMetricsTokenAuth({
authorizationHeader: request.headers.authorization,
configuredToken: options.metricsToken,
metricsEnabled: true,
});
if (!auth.ok) {
sendJson(response, auth.status, { error: auth.code });
return;
}
const body = await options.metrics.registry.metrics();
response.writeHead(200, {
"cache-control": "no-store",
"content-type": options.metrics.registry.contentType,
});
response.end(body);
return;
}
sendJson(response, 404, { error: "not_found" });
} catch (error) {
options.logger.error({ err: error }, "controller HTTP request failed");
if (!response.headersSent) {
setSecurityHeaders(response);
sendJson(response, 500, { error: "internal_error" });
} else {
response.destroy();
}
}
});
server.on("clientError", (error, socket) => {
options.logger.warn({ err: error }, "rejected malformed controller HTTP request");
socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
});
return {
server,
listen() {
return new Promise((resolve, reject) => {
const onError = (error: Error) => reject(error);
server.once("error", onError);
server.listen(options.port, options.host, () => {
server.off("error", onError);
const address = server.address();
if (!address || typeof address === "string") {
reject(new Error("controller HTTP server did not expose a TCP address"));
return;
}
resolve(address);
});
});
},
close() {
return new Promise((resolve, reject) => {
if (!server.listening) {
resolve();
return;
}
server.close((error) => error ? reject(error) : resolve());
});
},
};
}
function sendJson(
response: Parameters<typeof setSecurityHeaders>[0],
status: number,
payload: Record<string, unknown>,
headers: Record<string, string> = {}
): void {
response.writeHead(status, {
"cache-control": "no-store",
"content-type": "application/json; charset=utf-8",
...headers,
});
response.end(`${JSON.stringify(payload)}\n`);
}
function setSecurityHeaders(response: import("node:http").ServerResponse): void {
response.setHeader("x-content-type-options", "nosniff");
response.setHeader("referrer-policy", "no-referrer");
response.setHeader("content-security-policy", "default-src 'none'");
}
@@ -0,0 +1,102 @@
import { createChildLogger, createLogger } from "@hivegrid/logger";
import { createFencedClaimMutationLock } from "./claim-lock.js";
import { parsePortalDomainControllerConfig } from "./config.js";
import { createPortalDomainController } from "./controller.js";
import { createControllerHttpRuntime } from "./http.js";
import { createControllerMetrics } from "./metrics.js";
import { createDomainProbes } from "./probes.js";
import { createDomainQueueRuntime } from "./queue.js";
import { TraefikConfigWriter } from "./traefik.js";
async function main(): Promise<void> {
const config = parsePortalDomainControllerConfig();
const logger = createChildLogger(createLogger(), {
service: "hivegrid-portal-domain-controller",
revision: config.revision,
});
const claimLock = createFencedClaimMutationLock(config.queueConnection, config.configDirectory);
const writer = new TraefikConfigWriter({
configDirectory: config.configDirectory,
reloadTriggerFile: config.reloadTriggerFile,
routing: {
entryPoint: config.entryPoint,
backendService: config.backendService,
},
claimLock,
});
const metrics = createControllerMetrics(config.revision);
let queue: Awaited<ReturnType<typeof createDomainQueueRuntime>>;
try {
await writer.assertReady();
await writer.reconcilePersistedDomainConfigs();
const probes = createDomainProbes({ expectedCname: config.expectedCname, expectedRevision: config.revision });
const controller = createPortalDomainController({ writer, probes });
queue = await createDomainQueueRuntime({
connection: config.queueConnection,
concurrency: config.queueConcurrency,
controller,
commandSecret: config.commandSecret,
logger,
metrics,
});
} catch (error) {
await claimLock.close().catch(() => undefined);
throw error;
}
const http = createControllerHttpRuntime({
host: config.host,
port: config.port,
revision: config.revision,
metricsToken: config.metricsToken,
logger,
metrics,
isReady: async () => (await queue.isReady()) && (await writable(writer)),
});
let address;
try {
address = await http.listen();
} catch (error) {
await queue.close().catch(() => undefined);
await claimLock.close().catch(() => undefined);
throw error;
}
logger.info({ host: address.address, port: address.port }, "portal-domain controller started");
let shuttingDown = false;
const shutdown = async (signal: string) => {
if (shuttingDown) return;
shuttingDown = true;
logger.info({ signal }, "stopping portal-domain controller");
const timeout = setTimeout(() => {
logger.error("portal-domain controller shutdown timed out");
process.exit(1);
}, 30_000);
timeout.unref();
try {
await queue.close();
await claimLock.close();
await http.close();
clearTimeout(timeout);
} catch (error) {
logger.error({ err: error }, "portal-domain controller shutdown failed");
process.exitCode = 1;
}
};
process.once("SIGTERM", () => void shutdown("SIGTERM"));
process.once("SIGINT", () => void shutdown("SIGINT"));
}
async function writable(writer: TraefikConfigWriter): Promise<boolean> {
try {
await writer.assertReady();
return true;
} catch {
return false;
}
}
main().catch((error) => {
const logger = createChildLogger(createLogger(), { service: "hivegrid-portal-domain-controller" });
logger.error({ err: error }, "portal-domain controller failed to start");
process.exit(1);
});
@@ -0,0 +1,31 @@
import { createMetricsRegistry } from "@hivegrid/metrics";
import { Counter, Gauge, Histogram } from "prom-client";
export function createControllerMetrics(revision: string, collectDefaultProcessMetrics = true) {
const registry = createMetricsRegistry({
serviceName: "portal-domain-controller",
collectDefaultProcessMetrics,
defaultLabels: { revision },
});
const commands = new Counter({
name: "hivegrid_portal_domain_commands_total",
help: "Portal domain commands by action and outcome.",
labelNames: ["action", "status"] as const,
registers: [registry],
});
const duration = new Histogram({
name: "hivegrid_portal_domain_command_duration_seconds",
help: "Portal domain command processing duration.",
labelNames: ["action"] as const,
buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
registers: [registry],
});
const ready = new Gauge({
name: "hivegrid_portal_domain_controller_ready",
help: "Whether Redis and the dedicated Traefik config directory are ready.",
registers: [registry],
});
return { registry, commands, duration, ready };
}
export type ControllerMetrics = ReturnType<typeof createControllerMetrics>;
@@ -0,0 +1,120 @@
import { describe, expect, it, vi } from "vitest";
import { createDomainProbes, DomainProbeError } from "./probes.js";
const REVISION = "0123456789abcdef0123456789abcdef01234567";
function healthResponse(
overrides: Record<string, unknown> = {},
options: { readonly status?: number; readonly url?: string; readonly headers?: Readonly<Record<string, string>> } = {}
): Response {
const response = new Response(
JSON.stringify({
status: "ok",
service: "hivegrid-portal",
surface: "portal",
revision: REVISION,
host: "portal.example.com",
...overrides,
}),
{
status: options.status ?? 200,
headers: options.headers ?? { "content-type": "application/json" },
}
);
Object.defineProperty(response, "url", { value: options.url ?? "https://portal.example.com/healthz" });
return response;
}
describe("portal-domain DNS and HTTPS probes", () => {
it("requires the exact configured CNAME after DNS canonicalization", async () => {
const resolveCnameFn = vi.fn().mockResolvedValue(["INGRESS.TOOL-CTRL.DE."]);
const probes = createDomainProbes({
expectedCname: "ingress.tool-ctrl.de",
expectedRevision: REVISION,
resolveCnameFn: resolveCnameFn as never,
fetchFn: vi.fn() as never,
});
await expect(probes.verifyDns("portal.example.com")).resolves.toBeUndefined();
expect(resolveCnameFn).toHaveBeenCalledWith("portal.example.com");
resolveCnameFn.mockResolvedValueOnce(["attacker.example.com"]);
await expect(probes.verifyDns("portal.example.com")).rejects.toMatchObject({ reasonCode: "dns_cname_mismatch", retryable: true });
resolveCnameFn.mockResolvedValueOnce(["ingress.tool-ctrl.de", "attacker.example.com"]);
await expect(probes.verifyDns("portal.example.com")).rejects.toMatchObject({ reasonCode: "dns_cname_mismatch" });
resolveCnameFn.mockRejectedValueOnce(new Error("ENODATA"));
await expect(probes.verifyDns("portal.example.com")).rejects.toMatchObject({ reasonCode: "dns_lookup_failed" });
});
it("requires custom-host portal health plus an independent SNI certificate acknowledgement", async () => {
const certificateExpiresAt = new Date("2026-10-01T00:00:00.000Z");
const inspectCertificateFn = vi.fn(async () => certificateExpiresAt);
const fetchFn = vi.fn().mockResolvedValue(healthResponse());
const probes = createDomainProbes({
expectedCname: "ingress.tool-ctrl.de",
expectedRevision: REVISION,
resolveCnameFn: vi.fn() as never,
fetchFn: fetchFn as never,
inspectCertificateFn,
now: () => new Date("2026-07-16T10:00:00.000Z"),
});
await expect(probes.verifyHttps("portal.example.com")).resolves.toEqual({ certificateExpiresAt: certificateExpiresAt.toISOString() });
expect(String(fetchFn.mock.calls[0]?.[0])).toBe("https://portal.example.com/healthz");
expect(fetchFn).toHaveBeenCalledWith(expect.any(URL), expect.objectContaining({ redirect: "manual" }));
expect(inspectCertificateFn).toHaveBeenCalledWith("portal.example.com", 10_000);
fetchFn.mockResolvedValueOnce(
healthResponse(
{},
{
status: 302,
url: "https://portal.example.com/healthz",
headers: { "content-type": "application/json", location: "https://attacker.example" },
}
)
);
await expect(probes.verifyHttps("portal.example.com")).rejects.toBeInstanceOf(DomainProbeError);
fetchFn.mockResolvedValueOnce(healthResponse({ service: "wrong" }));
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
});
it("rejects a 200 response whose observed Host, final URL, surface, or revision does not match", async () => {
const fetchFn = vi.fn();
const inspectCertificateFn = vi.fn(async () => new Date("2026-10-01T00:00:00.000Z"));
const probes = createDomainProbes({
expectedCname: "ingress.tool-ctrl.de",
expectedRevision: REVISION,
resolveCnameFn: vi.fn() as never,
fetchFn: fetchFn as never,
inspectCertificateFn,
now: () => new Date("2026-07-16T10:00:00.000Z"),
});
for (const response of [
healthResponse({ host: "attacker.example.com" }),
healthResponse({ surface: "core", service: "hivegrid-web" }),
healthResponse({}, { url: "https://attacker.example.com/healthz" }),
]) {
fetchFn.mockResolvedValueOnce(response);
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
}
fetchFn.mockResolvedValueOnce(healthResponse({ revision: "different-release" }));
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "release_revision_mismatch" });
expect(inspectCertificateFn).not.toHaveBeenCalled();
});
it("fails closed when the explicit SNI certificate observation is missing or expired", async () => {
const fetchFn = vi.fn().mockResolvedValue(healthResponse());
const inspectCertificateFn = vi.fn().mockResolvedValue(new Date("2026-07-16T09:59:59.000Z"));
const probes = createDomainProbes({
expectedCname: "ingress.tool-ctrl.de",
expectedRevision: REVISION,
resolveCnameFn: vi.fn() as never,
fetchFn: fetchFn as never,
inspectCertificateFn,
now: () => new Date("2026-07-16T10:00:00.000Z"),
});
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
inspectCertificateFn.mockRejectedValueOnce(new Error("TLS failure"));
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
});
});
@@ -0,0 +1,185 @@
import { resolveCname } from "node:dns/promises";
import { connect as connectTls } from "node:tls";
import { normalizeDomainHostname } from "./contracts.js";
export class DomainProbeError extends Error {
readonly retryable = true;
constructor(
readonly reasonCode: "dns_cname_mismatch" | "dns_lookup_failed" | "https_health_failed" | "release_revision_mismatch",
message: string
) {
super(message);
}
}
export type DomainProbes = {
verifyDns(hostname: string): Promise<void>;
verifyHttps(hostname: string): Promise<{ readonly certificateExpiresAt: string }>;
};
export type CreateDomainProbesOptions = {
expectedCname: string;
expectedRevision: string;
timeoutMs?: number;
resolveCnameFn?: typeof resolveCname;
fetchFn?: typeof fetch;
inspectCertificateFn?: (hostname: string, timeoutMs: number) => Promise<Date>;
now?: () => Date;
};
export function createDomainProbes(options: CreateDomainProbesOptions): DomainProbes {
const expectedCname = normalizeDomainHostname(options.expectedCname);
const expectedRevision = normalizeExpectedRevision(options.expectedRevision);
const timeoutMs = options.timeoutMs ?? 10_000;
const resolveCnameFn = options.resolveCnameFn ?? resolveCname;
const fetchFn = options.fetchFn ?? fetch;
const inspectCertificateFn = options.inspectCertificateFn ?? inspectTlsCertificate;
const now = options.now ?? (() => new Date());
return {
async verifyDns(hostname) {
const normalizedHostname = normalizeDomainHostname(hostname);
let answers: string[];
try {
answers = await resolveCnameFn(normalizedHostname);
} catch {
throw new DomainProbeError("dns_lookup_failed", "The portal domain CNAME could not be resolved");
}
const normalizedAnswers = answers.flatMap((answer) => {
try {
return [normalizeDomainHostname(answer)];
} catch {
return [];
}
});
if (normalizedAnswers.length !== 1 || normalizedAnswers[0] !== expectedCname) {
throw new DomainProbeError("dns_cname_mismatch", "The portal domain CNAME does not target the HiveGrid ingress");
}
},
async verifyHttps(hostname) {
const normalizedHostname = normalizeDomainHostname(hostname);
const expectedUrl = new URL(`https://${normalizedHostname}/healthz`);
let response: Response;
try {
// This request intentionally traverses the newly rendered exact Host()
// router. The portal runtime exposes only its dependency-free /healthz
// endpoint before DB activation; routes, auth and assets remain 404.
response = await fetchFn(expectedUrl, {
method: "GET",
redirect: "manual",
signal: AbortSignal.timeout(timeoutMs),
headers: { accept: "application/json" },
});
} catch {
throw new DomainProbeError("https_health_failed", "The portal HTTPS health probe failed");
}
if (
response.status !== 200 ||
response.redirected ||
response.headers.get("content-type")?.includes("application/json") !== true ||
!sameHealthUrl(response.url, expectedUrl)
) {
throw new DomainProbeError("https_health_failed", "The portal HTTPS endpoint did not return a valid health response");
}
const contentLength = Number(response.headers.get("content-length") ?? 0);
if (Number.isFinite(contentLength) && contentLength > 4096) {
throw new DomainProbeError("https_health_failed", "The portal HTTPS health response exceeded the size limit");
}
let payload: unknown;
try {
const body = await response.text();
if (Buffer.byteLength(body, "utf8") > 4096) throw new Error("health response too large");
payload = JSON.parse(body);
} catch {
throw new DomainProbeError("https_health_failed", "The portal HTTPS health response was malformed");
}
const observedRevision = portalHealthRevision(payload, normalizedHostname);
if (!observedRevision) {
throw new DomainProbeError("https_health_failed", "The portal HTTPS health response identified the wrong service");
}
if (observedRevision !== expectedRevision) {
throw new DomainProbeError("release_revision_mismatch", "The portal HTTPS health response belongs to a different release");
}
let certificateExpiresAt: Date;
try {
certificateExpiresAt = await inspectCertificateFn(normalizedHostname, timeoutMs);
} catch {
throw new DomainProbeError("https_health_failed", "The portal TLS certificate could not be inspected with SNI");
}
if (!Number.isFinite(certificateExpiresAt.getTime()) || certificateExpiresAt <= now()) {
throw new DomainProbeError("https_health_failed", "The portal TLS certificate is invalid or expired");
}
return { certificateExpiresAt: certificateExpiresAt.toISOString() };
},
};
}
/**
* Opens an independently verified TLS connection with an explicit SNI name.
* The custom-host `fetch()` proves the exact Host router and portal response;
* this narrow probe
* captures the certificate expiry needed for the 21/7-day alert contract.
*/
export function inspectTlsCertificate(hostname: string, timeoutMs: number): Promise<Date> {
const normalizedHostname = normalizeDomainHostname(hostname);
return new Promise<Date>((resolve, reject) => {
let settled = false;
const socket = connectTls({
host: normalizedHostname,
port: 443,
servername: normalizedHostname,
rejectUnauthorized: true,
});
const fail = (error: Error) => {
if (settled) return;
settled = true;
socket.destroy();
reject(error);
};
socket.setTimeout(timeoutMs, () => fail(new Error("portal TLS certificate probe timed out")));
socket.once("error", fail);
socket.once("secureConnect", () => {
if (settled) return;
const certificate = socket.getPeerCertificate();
const expiresAt = new Date(certificate.valid_to ?? "");
if (!certificate.valid_to || !Number.isFinite(expiresAt.getTime())) {
fail(new Error("portal TLS peer certificate has no valid expiry"));
return;
}
settled = true;
socket.end();
resolve(expiresAt);
});
});
}
function portalHealthRevision(value: unknown, hostname: string): string | null {
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
const record = value as Record<string, unknown>;
const matches =
Object.keys(record).sort().join(",") === "host,revision,service,status,surface" &&
record.status === "ok" &&
record.service === "hivegrid-portal" &&
record.surface === "portal" &&
record.host === hostname &&
typeof record.revision === "string" &&
/^[A-Za-z0-9._-]{1,128}$/u.test(record.revision);
return matches ? record.revision as string : null;
}
function sameHealthUrl(actual: string, expected: URL): boolean {
try {
const url = new URL(actual);
return url.protocol === "https:" && url.hostname === expected.hostname && !url.port && url.pathname === "/healthz" && !url.search && !url.hash;
} catch {
return false;
}
}
function normalizeExpectedRevision(value: string): string {
const revision = value.trim();
if (!revision || revision === "unknown" || !/^[A-Za-z0-9._-]{1,128}$/u.test(revision)) {
throw new Error("portal health probe requires an immutable release revision");
}
return revision;
}
@@ -0,0 +1,110 @@
import { createLogger } from "@hivegrid/logger";
import { describe, expect, it, vi } from "vitest";
import {
createSignedDomainCommand,
verifySignedDomainResult,
type DomainResultPayload,
type SignedDomainEnvelope,
} from "./contracts.js";
import { DomainControllerFailure, type PortalDomainController } from "./controller.js";
import { createControllerMetrics } from "./metrics.js";
import { createDomainCommandProcessor } from "./queue.js";
const SECRET = "test-command-secret-that-is-at-least-thirty-two-bytes";
const payload = {
claimId: "claim_12345678",
tenantId: "tenant_12345678",
generation: 1,
hostname: "portal.example.com",
action: "provision" as const,
};
describe("portal-domain queue processor", () => {
it("verifies commands and publishes an independently signed result", async () => {
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
const controller: PortalDomainController = {
handle: vi.fn(async (command, attempt) => ({
...command,
status: "active",
attempt,
completedAt: "2026-07-16T10:00:00.000Z",
configHash: "a".repeat(64),
certificateExpiresAt: "2026-10-01T00:00:00.000Z",
})),
};
const processor = createProcessor(controller, published);
await processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 0 });
expect(published).toHaveLength(1);
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({ status: "active", attempt: 1 });
});
it("rejects unsigned or tampered commands without emitting a reflected result", async () => {
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
const controller: PortalDomainController = { handle: vi.fn() };
const processor = createProcessor(controller, published);
const signed = createSignedDomainCommand(payload, SECRET);
await expect(processor({ data: { ...signed, payload: { ...signed.payload, hostname: "evil.example.com" } }, attemptsMade: 0 })).rejects.toThrow("invalid_domain_command");
expect(published).toEqual([]);
expect(controller.handle).not.toHaveBeenCalled();
});
it("publishes signed retry and terminal outcomes around the four-attempt ceiling", async () => {
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
const controller: PortalDomainController = {
handle: vi.fn().mockRejectedValue(new DomainControllerFailure("https_health_failed", true, "transient")),
};
const processor = createProcessor(controller, published);
const signed = createSignedDomainCommand(payload, SECRET);
await expect(processor({ data: signed, attemptsMade: 0 })).rejects.toThrow("https_health_failed");
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({ status: "retrying", retryAfterMs: 900_000 });
await expect(processor({ data: signed, attemptsMade: 3 })).resolves.toBeUndefined();
expect(verifySignedDomainResult(published[1], SECRET)).toMatchObject({ status: "failed", attempt: 4, reasonCode: "https_health_failed" });
});
it("keeps release mismatches inside the same four-attempt safety budget", async () => {
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
const controller: PortalDomainController = {
handle: vi.fn().mockRejectedValue(new DomainControllerFailure("release_revision_mismatch", true, "release overlap")),
};
const processor = createProcessor(controller, published);
await expect(processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 0 })).rejects.toThrow("release_revision_mismatch");
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({
status: "retrying",
reasonCode: "release_revision_mismatch",
retryAfterMs: 900_000,
});
await expect(processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 3 })).resolves.toBeUndefined();
expect(verifySignedDomainResult(published[1], SECRET)).toMatchObject({
status: "failed",
attempt: 4,
reasonCode: "release_revision_mismatch",
});
});
it("does not retry terminal controller failures", async () => {
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
const controller: PortalDomainController = {
handle: vi.fn().mockRejectedValue(new DomainControllerFailure("unsafe_config_path", false, "terminal")),
};
const processor = createProcessor(controller, published);
await expect(processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 0 })).resolves.toBeUndefined();
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({ status: "failed", attempt: 1 });
});
});
function createProcessor(
controller: PortalDomainController,
published: SignedDomainEnvelope<DomainResultPayload>[]
) {
return createDomainCommandProcessor({
controller,
commandSecret: SECRET,
resultPublisher: async (result) => { published.push(result); },
logger: createLogger({ level: "silent" }),
metrics: createControllerMetrics("test", false),
now: () => new Date("2026-07-16T10:00:00.000Z"),
});
}
@@ -0,0 +1,176 @@
import { Queue, Worker, type JobsOptions } from "bullmq";
import type { HiveGridLogger } from "@hivegrid/logger";
import type { ConnectionOptions } from "@hivegrid/queue";
import {
createSignedDomainResult,
DOMAIN_COMMAND_QUEUE,
DOMAIN_COMMAND_QUEUE_ATTEMPTS,
DOMAIN_RESULT_QUEUE,
sha256Hex,
verifySignedDomainCommand,
type DomainResultPayload,
type SignedDomainEnvelope,
} from "./contracts.js";
import {
DOMAIN_MAX_ATTEMPTS,
DomainControllerFailure,
retryDelayForAttempt,
type PortalDomainController,
} from "./controller.js";
import type { ControllerMetrics } from "./metrics.js";
export const DOMAIN_BACKOFF_TYPE = "portal-domain-schedule";
export const DOMAIN_COMMAND_JOB_OPTIONS: JobsOptions = Object.freeze({
attempts: DOMAIN_COMMAND_QUEUE_ATTEMPTS,
backoff: { type: DOMAIN_BACKOFF_TYPE },
removeOnComplete: { age: 24 * 60 * 60, count: 10_000 },
removeOnFail: { age: 7 * 24 * 60 * 60, count: 50_000 },
});
export type DomainQueueRuntime = {
commandQueue: Queue;
resultQueue: Queue;
worker: Worker;
isReady(): Promise<boolean>;
close(): Promise<void>;
};
export type DomainCommandProcessor = (job: {
data: unknown;
attemptsMade: number;
}) => Promise<void>;
export function createDomainCommandProcessor(options: {
controller: PortalDomainController;
commandSecret: string;
resultPublisher: (result: SignedDomainEnvelope<DomainResultPayload>) => Promise<void>;
logger: HiveGridLogger;
metrics: ControllerMetrics;
now?: () => Date;
}): DomainCommandProcessor {
const now = options.now ?? (() => new Date());
return async (job) => {
const startedAt = performance.now();
let command;
try {
command = verifySignedDomainCommand(job.data, options.commandSecret);
} catch (error) {
options.logger.warn({ err: error }, "rejected unauthenticated portal-domain command");
throw new Error("invalid_domain_command");
}
const attempt = job.attemptsMade + 1;
let status: DomainResultPayload["status"] = "failed";
try {
const result = await options.controller.handle(command, attempt);
status = result.status;
await options.resultPublisher(createSignedDomainResult(result, options.commandSecret));
options.logger.info(
{ claimId: command.claimId, tenantId: command.tenantId, generation: command.generation, action: command.action, status },
"processed portal-domain command"
);
} catch (error) {
const failure = error instanceof DomainControllerFailure
? error
: new DomainControllerFailure("controller_internal_error", true, "Transient controller failure");
const canRetry = failure.retryable && attempt < DOMAIN_MAX_ATTEMPTS;
status = canRetry ? "retrying" : "failed";
const retryAfterMs = canRetry ? retryDelayForAttempt(attempt) : undefined;
const result: DomainResultPayload = {
...command,
status,
attempt,
completedAt: now().toISOString(),
reasonCode: failure.reasonCode,
...(retryAfterMs ? { retryAfterMs } : {}),
};
await options.resultPublisher(createSignedDomainResult(result, options.commandSecret));
options.logger.warn(
{
claimId: command.claimId,
tenantId: command.tenantId,
generation: command.generation,
action: command.action,
status,
reasonCode: failure.reasonCode,
attempt,
},
"portal-domain command did not complete"
);
if (canRetry) throw new Error(failure.reasonCode);
} finally {
options.metrics.commands.inc({ action: command.action, status });
options.metrics.duration.observe({ action: command.action }, (performance.now() - startedAt) / 1000);
}
};
}
export async function createDomainQueueRuntime(options: {
connection: ConnectionOptions;
concurrency: number;
controller: PortalDomainController;
commandSecret: string;
logger: HiveGridLogger;
metrics: ControllerMetrics;
}): Promise<DomainQueueRuntime> {
const commandQueue = new Queue(DOMAIN_COMMAND_QUEUE, {
connection: options.connection,
defaultJobOptions: DOMAIN_COMMAND_JOB_OPTIONS,
});
const resultQueue = new Queue(DOMAIN_RESULT_QUEUE, {
connection: options.connection,
defaultJobOptions: {
attempts: 5,
backoff: { type: "exponential", delay: 1_000 },
removeOnComplete: { age: 7 * 24 * 60 * 60, count: 50_000 },
removeOnFail: { age: 30 * 24 * 60 * 60, count: 100_000 },
},
});
const publishResult = async (result: SignedDomainEnvelope<DomainResultPayload>) => {
const jobId = sha256Hex(`${result.contentHash}:${result.payload.attempt}:${result.payload.status}`);
await resultQueue.add("portal-domain-result", result, { jobId });
};
const processor = createDomainCommandProcessor({ ...options, resultPublisher: publishResult });
let workerReady = false;
const worker = new Worker(
DOMAIN_COMMAND_QUEUE,
async (job) => processor(job),
{
connection: options.connection,
concurrency: options.concurrency,
settings: {
backoffStrategy: (attemptsMade, type, _error) =>
type === DOMAIN_BACKOFF_TYPE ? retryDelayForAttempt(attemptsMade) : -1,
},
}
);
worker.on("ready", () => { workerReady = true; });
worker.on("error", (error) => {
workerReady = false;
options.logger.error({ err: error }, "portal-domain queue worker error");
});
worker.on("closed", () => { workerReady = false; });
await Promise.all([commandQueue.waitUntilReady(), resultQueue.waitUntilReady(), worker.waitUntilReady()]);
workerReady = true;
return {
commandQueue,
resultQueue,
worker,
async isReady() {
if (!workerReady || worker.closing) return false;
try {
const [commandClient, resultClient] = await Promise.all([commandQueue.client, resultQueue.client]);
const replies = await Promise.all([commandClient.ping(), resultClient.ping()]);
return replies.every((reply) => reply === "PONG");
} catch {
return false;
}
},
async close() {
workerReady = false;
await worker.close();
await Promise.all([commandQueue.close(), resultQueue.close()]);
},
};
}
@@ -0,0 +1,527 @@
import { link, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, unlink, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { ClaimLockLostError, type ClaimLockGuard, type ClaimMutationLock } from "./claim-lock.js";
import { PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME } from "./config.js";
import { sha256Hex, type DomainCommandPayload } from "./contracts.js";
import {
GenerationConflictError,
PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY,
portalDomainTombstoneRouterPriority,
renderTraefikDeprovisionedConfig,
renderTraefikDomainConfig,
TraefikConfigWriter,
UnsafeConfigPathError,
type TraefikRoutingConfig,
} from "./traefik.js";
const directories: string[] = [];
const routing = {
entryPoint: "https",
backendService: "https-0-joyyh5b89vpfp8ocri8xr96m@docker",
} as const;
const CONFIGURATION_MUTATION_LOCK_ID = "__portal_domain_configuration_mutation__";
const baseCommand: DomainCommandPayload = {
version: 1,
claimId: "claim_12345678",
tenantId: "tenant_12345678",
generation: 1,
hostname: "portal.example.com",
action: "provision",
};
afterEach(async () => {
await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })));
});
describe("Traefik portal-domain writer", () => {
it("renders one exact Host rule against the configured Docker service without injectable YAML", () => {
const yaml = renderTraefikDomainConfig(baseCommand, routing);
expect(yaml).toContain('rule: "Host(`portal.example.com`)"');
expect(yaml).toContain('- "https"');
expect(yaml).toContain('service: "https-0-joyyh5b89vpfp8ocri8xr96m@docker"');
expect(yaml).toContain(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`);
expect(yaml).toContain("certResolver: letsencrypt");
expect(yaml).not.toContain("loadBalancer");
expect(yaml).not.toContain("HostRegexp");
expect(yaml).not.toContain("*.example.com");
expect(yaml.match(/Host\(/gu)).toHaveLength(1);
expect(() => renderTraefikDomainConfig(baseCommand, { ...routing, entryPoint: "https\nhttp:" })).toThrow("entry-point");
expect(() => renderTraefikDomainConfig(baseCommand, { ...routing, backendService: "portal@docker\n routers:" })).toThrow("Docker service");
const tombstoneCommand: DomainCommandPayload = {
...baseCommand,
tenantId: "tenant-do-not-render",
action: "deprovision",
};
const tombstone = renderTraefikDeprovisionedConfig(tombstoneCommand, routing);
const exactHostRule = "Host(`portal.example.com`)";
const tombstonePriority = portalDomainTombstoneRouterPriority(baseCommand.hostname);
expect(tombstone).toContain('rule: "Host(`portal.example.com`)"');
expect(tombstone).toContain(`priority: ${tombstonePriority}`);
expect(tombstone).toContain('service: "noop@internal"');
expect(tombstone).toContain('"418": 404');
expect(tombstone).toContain("tls: {}");
expect(tombstone).not.toContain("certResolver");
expect(tombstone).not.toContain(routing.backendService);
expect(tombstone).not.toContain("tenant-do-not-render");
expect(tombstonePriority).toBe(Math.max(1, Buffer.byteLength(exactHostRule, "utf8") - 1));
expect(Buffer.byteLength(exactHostRule, "utf8")).toBeGreaterThan(tombstonePriority);
expect(PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY).toBeGreaterThan(tombstonePriority);
});
it("atomically applies, repairs and deprovisions a generation", async () => {
const layout = await tempLayout();
const writer = writerFor(layout);
await writer.assertReady();
const first = await writer.applyProvision(baseCommand);
expect(first.disposition).toBe("applied");
const files = await readdir(layout.configDirectory);
const yamlName = files.find((name) => name.endsWith(".yaml"));
expect(yamlName).toBeDefined();
expect(files.some((name) => name.startsWith(".tmp-"))).toBe(false);
const yamlPath = path.join(layout.configDirectory, yamlName!);
expect((await stat(yamlPath)).mode & 0o777).toBe(0o600);
expect(first.configHash).toBe(sha256Hex(await readFile(yamlPath, "utf8")));
expect(await readFile(layout.reloadTriggerFile, "utf8")).toMatch(/^reload [0-9a-f-]+\n$/u);
expect((await readdir(layout.root)).filter((name) => name.endsWith(".yaml"))).toEqual([]);
await writeFile(yamlPath, "tampered\n", "utf8");
const beforeRepair = await readFile(layout.reloadTriggerFile, "utf8");
expect((await writer.applyProvision(baseCommand)).disposition).toBe("unchanged");
expect(await readFile(yamlPath, "utf8")).toContain("Host(`portal.example.com`)");
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRepair);
const remove = await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" });
expect(remove.disposition).toBe("applied");
expect(await readFile(yamlPath, "utf8")).toContain('service: "noop@internal"');
expect(await readFile(yamlPath, "utf8")).toContain('"418": 404');
expect(await readFile(yamlPath, "utf8")).not.toContain(routing.backendService);
expect(await readFile(yamlPath, "utf8")).not.toContain("certResolver");
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ generation: 2, action: "deprovision" });
await writeFile(yamlPath, "reappeared after deprovision\n", "utf8");
const beforeDeprovisionRepair = await readFile(layout.reloadTriggerFile, "utf8");
expect((await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" })).disposition).toBe("unchanged");
expect(await readFile(yamlPath, "utf8")).toContain('service: "noop@internal"');
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeDeprovisionRepair);
const afterDeprovisionRepair = await readFile(layout.reloadTriggerFile, "utf8");
expect((await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" })).disposition).toBe("unchanged");
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(afterDeprovisionRepair);
});
it("retries the reload before acknowledging a same-generation config repair", async () => {
const layout = await tempLayout();
const writer = writerFor(layout);
await writer.applyProvision(baseCommand);
const yamlPath = configPathForClaim(layout, baseCommand.claimId);
await writeFile(yamlPath, "drifted before replay\n", "utf8");
const secondSentinelLink = path.join(layout.root, "blocked-reload-link");
await link(layout.reloadTriggerFile, secondSentinelLink);
await expect(writer.applyProvision(baseCommand)).rejects.toBeInstanceOf(UnsafeConfigPathError);
expect(await readFile(yamlPath, "utf8")).toBe(renderTraefikDomainConfig(baseCommand, routing));
await unlink(secondSentinelLink);
const beforeRetry = await readFile(layout.reloadTriggerFile, "utf8");
await expect(writer.applyProvision(baseCommand)).resolves.toMatchObject({ disposition: "unchanged" });
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRetry);
});
it("rejects stale and conflicting generations while permitting idempotent replay", async () => {
const writer = writerFor(await tempLayout());
await writer.applyProvision({ ...baseCommand, generation: 4 });
expect((await writer.applyProvision({ ...baseCommand, generation: 3 })).disposition).toBe("stale");
expect((await writer.applyProvision({ ...baseCommand, generation: 4 })).disposition).toBe("unchanged");
await expect(writer.applyDeprovision({ ...baseCommand, generation: 4, action: "deprovision" })).rejects.toBeInstanceOf(GenerationConflictError);
await expect(writer.applyProvision({ ...baseCommand, generation: 5, hostname: "other.example.com" })).rejects.toBeInstanceOf(GenerationConflictError);
await expect(writer.applyProvision({ ...baseCommand, generation: 5, tenantId: "tenant_87654321" })).rejects.toBeInstanceOf(GenerationConflictError);
});
it("repairs backend drift during same-generation revalidation without replacing provision state", async () => {
const layout = await tempLayout();
const coordinator = new TestClaimLockCoordinator();
const oldBackend = "portal-backend-before-recreate@docker";
const currentBackend = "portal-backend-after-recreate@docker";
const initialWriter = writerFor(layout, coordinator.client(), { ...routing, backendService: oldBackend });
await initialWriter.applyProvision(baseCommand);
const yamlName = (await readdir(layout.configDirectory)).find((name) => name.endsWith(".yaml"));
expect(yamlName).toBeDefined();
const yamlPath = path.join(layout.configDirectory, yamlName!);
expect(await readFile(yamlPath, "utf8")).toContain(`service: "${oldBackend}"`);
const beforeRepair = await readFile(layout.reloadTriggerFile, "utf8");
const replacementWriter = writerFor(layout, coordinator.client(), { ...routing, backendService: currentBackend });
const result = await replacementWriter.recordRevalidation({ ...baseCommand, action: "revalidate" });
expect(result.disposition).toBe("unchanged");
expect(await readFile(yamlPath, "utf8")).toContain(`service: "${currentBackend}"`);
expect(await readFile(yamlPath, "utf8")).not.toContain(oldBackend);
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRepair);
expect(await replacementWriter.readAppliedState(baseCommand.claimId)).toMatchObject({
generation: baseCommand.generation,
action: "provision",
});
});
it("serializes two writer instances for one claim so the highest generation wins", async () => {
const layout = await tempLayout();
const coordinator = new TestClaimLockCoordinator();
const firstWriter = writerFor(layout, coordinator.client());
const secondWriter = writerFor(layout, coordinator.client());
const results = await Promise.all([
firstWriter.applyProvision(baseCommand),
secondWriter.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" }),
]);
expect(results.map((result) => result.disposition)).toContain("applied");
expect(coordinator.maxConcurrentFor(baseCommand.claimId)).toBe(1);
expect(await firstWriter.readAppliedState(baseCommand.claimId)).toMatchObject({ generation: 2, action: "deprovision" });
const yamlName = (await readdir(layout.configDirectory)).find((name) => name.endsWith(".yaml"));
expect(yamlName).toBeDefined();
expect(await readFile(path.join(layout.configDirectory, yamlName!), "utf8")).toContain('service: "noop@internal"');
});
it("repairs a legacy deprovision state before queue startup without replaying a command", async () => {
const layout = await tempLayout();
const writer = writerFor(layout);
await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" });
const yamlName = (await readdir(layout.configDirectory)).find((name) => name.endsWith(".yaml"));
expect(yamlName).toBeDefined();
const yamlPath = path.join(layout.configDirectory, yamlName!);
await unlink(yamlPath);
const beforeRepair = await readFile(layout.reloadTriggerFile, "utf8");
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
expect(await readFile(yamlPath, "utf8")).toContain('"418": 404');
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRepair);
const afterRepair = await readFile(layout.reloadTriggerFile, "utf8");
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(0);
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(afterRepair);
});
it("never reopens an active state whose config disappeared or became unknown", async () => {
const layout = await tempLayout();
const writer = writerFor(layout);
await writer.applyProvision(baseCommand);
const yamlPath = configPathForClaim(layout, baseCommand.claimId);
await unlink(yamlPath);
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
const afterCrashWindow = await readFile(yamlPath, "utf8");
expect(afterCrashWindow).toContain('service: "noop@internal"');
expect(afterCrashWindow).toContain('"418": 404');
expect(afterCrashWindow).not.toContain(routing.backendService);
expect(afterCrashWindow).not.toContain("certResolver");
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ action: "provision" });
await writeFile(yamlPath, "unknown legacy contents\n", "utf8");
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
expect(await readFile(yamlPath, "utf8")).toBe(afterCrashWindow);
const beforeTombstoneReplay = await readFile(layout.reloadTriggerFile, "utf8");
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(0);
expect(await readFile(yamlPath, "utf8")).toBe(afterCrashWindow);
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeTombstoneReplay);
});
it("upgrades a known legacy revalidation route without trusting state alone", async () => {
const layout = await tempLayout();
const writer = writerFor(layout);
await writer.recordRevalidation({ ...baseCommand, action: "revalidate" });
const yamlPath = configPathForClaim(layout, baseCommand.claimId);
const legacyActive = (await readFile(yamlPath, "utf8")).replace(` priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}\n`, "");
await writeFile(yamlPath, legacyActive, "utf8");
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
const upgraded = await readFile(yamlPath, "utf8");
expect(upgraded).toContain(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`);
expect(upgraded).toContain(routing.backendService);
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ action: "revalidate" });
});
it(
"upgrades a legacy reclaimed active route above a restored legacy deprovision in one startup pass",
async () => {
const layout = await tempLayout();
const coordinator = new TestClaimLockCoordinator();
const writer = writerFor(layout, coordinator.client());
const retired = { ...baseCommand, generation: 2, action: "deprovision" as const };
const reclaimed = {
...baseCommand,
claimId: "claim_87654321",
tenantId: "tenant_87654321",
generation: 1,
action: "provision" as const,
};
await writer.applyDeprovision(retired);
await writer.applyProvision(reclaimed);
const reclaimedPath = configPathForClaim(layout, reclaimed.claimId);
const retiredPath = configPathForClaim(layout, retired.claimId);
const legacyActive = (await readFile(reclaimedPath, "utf8")).replace(` priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}\n`, "");
await writeFile(reclaimedPath, legacyActive, "utf8");
await unlink(retiredPath);
coordinator.resetObservations();
const beforeReconcile = await readFile(layout.reloadTriggerFile, "utf8");
const tombstonePriority = portalDomainTombstoneRouterPriority(baseCommand.hostname);
const legacyActivePriority = Buffer.byteLength(`Host(\`${baseCommand.hostname}\`)`, "utf8");
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(2);
const yamls = await Promise.all(
(await readdir(layout.configDirectory))
.filter((name) => name.endsWith(".yaml"))
.map((name) => readFile(path.join(layout.configDirectory, name), "utf8"))
);
expect(yamls).toHaveLength(2);
expect(yamls.filter((yaml) => yaml.includes(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`))).toHaveLength(1);
expect(yamls.filter((yaml) => yaml.includes(`priority: ${tombstonePriority}`))).toHaveLength(1);
expect(yamls.find((yaml) => yaml.includes(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`))).toContain(routing.backendService);
expect(yamls.find((yaml) => yaml.includes(`priority: ${tombstonePriority}`))).not.toContain(routing.backendService);
expect(legacyActivePriority).toBeGreaterThan(tombstonePriority);
expect(PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY).toBeGreaterThan(tombstonePriority);
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeReconcile);
expect(coordinator.acquisitionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(1);
expect(coordinator.guardAssertionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(3);
},
60_000
);
it("serializes configuration-wide mutations while different claim locks remain fenced", async () => {
const layout = await tempLayout();
const coordinator = new TestClaimLockCoordinator();
const retiredWriter = writerFor(layout, coordinator.client());
const reclaimedWriter = writerFor(layout, coordinator.client());
await Promise.all([
retiredWriter.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" }),
reclaimedWriter.applyProvision({
...baseCommand,
claimId: "claim_87654321",
tenantId: "tenant_87654321",
}),
]);
expect(coordinator.acquisitionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(2);
expect(coordinator.contentionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(1);
expect(coordinator.maxConcurrentFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(1);
expect(coordinator.acquisitionsFor(baseCommand.claimId)).toBe(1);
expect(coordinator.acquisitionsFor("claim_87654321")).toBe(1);
expect((await readdir(layout.configDirectory)).filter((name) => name.endsWith(".yaml"))).toHaveLength(2);
});
it("fails closed when startup reconciliation sees malformed or symlinked generation state", async () => {
const malformedLayout = await tempLayout();
const malformedPath = path.join(malformedLayout.configDirectory, `.portal-${"a".repeat(32)}.generation`);
await writeFile(malformedPath, "not-json\n", "utf8");
await expect(writerFor(malformedLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
const symlinkLayout = await tempLayout();
const outsidePath = path.join(symlinkLayout.root, "outside-generation-state");
await writeFile(outsidePath, `${JSON.stringify({ generation: 2, action: "deprovision", hostname: baseCommand.hostname, tenantId: baseCommand.tenantId })}\n`);
await symlink(outsidePath, path.join(symlinkLayout.configDirectory, `.portal-${"b".repeat(32)}.generation`));
await expect(writerFor(symlinkLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
const malformedNameLayout = await tempLayout();
await writeFile(path.join(malformedNameLayout.configDirectory, ".portal-not-a-key.generation"), "{}\n", "utf8");
await expect(writerFor(malformedNameLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
});
it("rejects hardlinked generation state and active config during startup reconciliation", async () => {
const stateLayout = await tempLayout();
const stateWriter = writerFor(stateLayout);
await stateWriter.applyProvision(baseCommand);
await link(statePathForClaim(stateLayout, baseCommand.claimId), path.join(stateLayout.root, "linked-generation-state"));
await expect(stateWriter.reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
const configLayout = await tempLayout();
const configWriter = writerFor(configLayout);
await configWriter.applyProvision(baseCommand);
await link(configPathForClaim(configLayout, baseCommand.claimId), path.join(configLayout.root, "linked-active-config"));
await expect(configWriter.reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
});
it("bounds generation state and config reads during startup reconciliation", async () => {
const stateLayout = await tempLayout();
await writeFile(path.join(stateLayout.configDirectory, `.portal-${"c".repeat(32)}.generation`), "x".repeat(4_097), "utf8");
await expect(writerFor(stateLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
const configLayout = await tempLayout();
const configWriter = writerFor(configLayout);
await configWriter.applyProvision(baseCommand);
await writeFile(configPathForClaim(configLayout, baseCommand.claimId), "x".repeat(16_385), "utf8");
await expect(configWriter.reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
});
it("checks distributed ownership immediately before config and state commits", async () => {
const layout = await tempLayout();
const lostLock: ClaimMutationLock = {
assertReady: async () => undefined,
close: async () => undefined,
withClaimLock: async (_claimId, operation) =>
operation({
assertHeld: async () => {
throw new ClaimLockLostError();
},
}),
};
const writer = writerFor(layout, lostLock);
await expect(writer.applyProvision(baseCommand)).rejects.toBeInstanceOf(ClaimLockLostError);
expect((await readdir(layout.configDirectory)).filter((name) => name.endsWith(".yaml") || name.endsWith(".generation"))).toEqual([]);
expect(await writer.readAppliedState(baseCommand.claimId)).toBeUndefined();
expect(await readFile(layout.reloadTriggerFile, "utf8")).toBe("ready\n");
});
it("hashes raw claim ids and never permits path traversal", async () => {
const layout = await tempLayout();
const writer = writerFor(layout);
await writer.applyProvision({ ...baseCommand, claimId: "../../outside-file" });
const entries = await readdir(layout.configDirectory);
expect(entries.every((entry) => !entry.includes("..") && !entry.includes("outside-file"))).toBe(true);
expect(entries.every((entry) => path.dirname(path.join(layout.configDirectory, entry)) === layout.configDirectory)).toBe(true);
});
it("rejects symlink directories and config targets", async () => {
const layout = await tempLayout();
const alternateRoot = await mkdtemp(path.join(os.tmpdir(), "hivegrid-domain-controller-linked-"));
directories.push(alternateRoot);
const alternateDirectory = path.join(alternateRoot, PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME);
await symlink(layout.configDirectory, alternateDirectory, "dir");
const alternateTrigger = path.join(alternateRoot, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME);
await writeFile(alternateTrigger, "ready\n", "utf8");
await expect(
writerFor({ root: alternateRoot, configDirectory: alternateDirectory, reloadTriggerFile: alternateTrigger }).assertReady()
).rejects.toBeInstanceOf(UnsafeConfigPathError);
const key = sha256Hex(baseCommand.claimId).slice(0, 32);
const outside = path.join(layout.root, `outside-${key}`);
await writeFile(outside, "do not replace", "utf8");
await symlink(outside, path.join(layout.configDirectory, `portal-${key}.yaml`));
await expect(writerFor(layout).applyProvision(baseCommand)).rejects.toBeInstanceOf(UnsafeConfigPathError);
expect(await readFile(outside, "utf8")).toBe("do not replace");
});
it("requires one pre-created non-linked reload sentinel and leaves generation retryable when it is unsafe", async () => {
const symlinkLayout = await tempLayout();
const outside = path.join(symlinkLayout.root, "outside-reload");
await writeFile(outside, "outside\n", "utf8");
await unlink(symlinkLayout.reloadTriggerFile);
await symlink(outside, symlinkLayout.reloadTriggerFile);
await expect(writerFor(symlinkLayout).assertReady()).rejects.toBeInstanceOf(UnsafeConfigPathError);
const hardlinkLayout = await tempLayout();
const secondLink = path.join(hardlinkLayout.root, "second-reload-link");
await link(hardlinkLayout.reloadTriggerFile, secondLink);
const writer = writerFor(hardlinkLayout);
await expect(writer.applyProvision(baseCommand)).rejects.toBeInstanceOf(UnsafeConfigPathError);
expect(await writer.readAppliedState(baseCommand.claimId)).toBeUndefined();
await unlink(secondLink);
await expect(writer.applyProvision(baseCommand)).resolves.toMatchObject({ disposition: "applied" });
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ generation: 1, action: "provision" });
});
it("revalidates the configured directory instead of trusting a cached path", async () => {
const layout = await tempLayout();
const writer = writerFor(layout);
await writer.assertReady();
const moved = path.join(layout.root, `${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}-moved`);
await rename(layout.configDirectory, moved);
await symlink(moved, layout.configDirectory, "dir");
await expect(writer.assertReady()).rejects.toBeInstanceOf(UnsafeConfigPathError);
});
});
type TempLayout = Readonly<{ root: string; configDirectory: string; reloadTriggerFile: string }>;
function writerFor(
layout: TempLayout,
claimLock: ClaimMutationLock = new TestClaimLockCoordinator().client(),
writerRouting: TraefikRoutingConfig = routing
): TraefikConfigWriter {
return new TraefikConfigWriter({ ...layout, routing: writerRouting, claimLock });
}
async function tempLayout(): Promise<TempLayout> {
const root = await mkdtemp(path.join(os.tmpdir(), "hivegrid-domain-controller-"));
directories.push(root);
const configDirectory = path.join(root, PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME);
const reloadTriggerFile = path.join(root, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME);
await mkdir(configDirectory);
await writeFile(reloadTriggerFile, "ready\n", { encoding: "utf8", mode: 0o600 });
return { root, configDirectory, reloadTriggerFile };
}
function configPathForClaim(layout: TempLayout, claimId: string): string {
return path.join(layout.configDirectory, `portal-${sha256Hex(claimId).slice(0, 32)}.yaml`);
}
function statePathForClaim(layout: TempLayout, claimId: string): string {
return path.join(layout.configDirectory, `.portal-${sha256Hex(claimId).slice(0, 32)}.generation`);
}
class TestClaimLockCoordinator {
readonly #tails = new Map<string, Promise<void>>();
readonly #active = new Map<string, number>();
readonly #maxConcurrent = new Map<string, number>();
readonly #acquisitions = new Map<string, number>();
readonly #contentions = new Map<string, number>();
readonly #guardAssertions = new Map<string, number>();
acquisitionsFor(claimId: string): number {
return this.#acquisitions.get(claimId) ?? 0;
}
contentionsFor(claimId: string): number {
return this.#contentions.get(claimId) ?? 0;
}
guardAssertionsFor(claimId: string): number {
return this.#guardAssertions.get(claimId) ?? 0;
}
maxConcurrentFor(claimId: string): number {
return this.#maxConcurrent.get(claimId) ?? 0;
}
resetObservations(): void {
this.#maxConcurrent.clear();
this.#acquisitions.clear();
this.#contentions.clear();
this.#guardAssertions.clear();
}
client(): ClaimMutationLock {
return {
assertReady: async () => undefined,
close: async () => undefined,
withClaimLock: async <T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>) => {
this.#acquisitions.set(claimId, this.acquisitionsFor(claimId) + 1);
if (this.#tails.has(claimId)) this.#contentions.set(claimId, this.contentionsFor(claimId) + 1);
const previous = this.#tails.get(claimId) ?? Promise.resolve();
let release: () => void = () => undefined;
const current = new Promise<void>((resolve) => {
release = resolve;
});
const tail = previous.then(() => current);
this.#tails.set(claimId, tail);
await previous;
const active = (this.#active.get(claimId) ?? 0) + 1;
this.#active.set(claimId, active);
this.#maxConcurrent.set(claimId, Math.max(this.maxConcurrentFor(claimId), active));
try {
return await operation({
assertHeld: async () => {
this.#guardAssertions.set(claimId, this.guardAssertionsFor(claimId) + 1);
},
});
} finally {
this.#active.set(claimId, active - 1);
release();
if (this.#tails.get(claimId) === tail) this.#tails.delete(claimId);
}
},
};
}
}
@@ -0,0 +1,613 @@
import { constants as fsConstants, type Stats } from "node:fs";
import { lstat, open, readdir, realpath, rename, unlink } from "node:fs/promises";
import path from "node:path";
import { randomUUID } from "node:crypto";
import {
normalizePortalDomainBackendService,
normalizePortalDomainEntryPoint,
PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME,
PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME,
PORTAL_DOMAIN_TLS_CERT_RESOLVER,
} from "./config.js";
import type { ClaimLockGuard, ClaimMutationLock } from "./claim-lock.js";
import { normalizeDomainHostname, sha256Hex, type DomainCommandAction, type DomainCommandPayload } from "./contracts.js";
export type AppliedDomainState = {
generation: number;
action: DomainCommandAction;
hostname: string;
tenantId: string;
};
export type ConfigMutationResult = {
disposition: "applied" | "unchanged" | "stale";
configHash?: string;
};
export type TraefikRoutingConfig = Readonly<{
entryPoint: string;
backendService: string;
}>;
export type TraefikConfigWriterOptions = Readonly<{
configDirectory: string;
reloadTriggerFile: string;
routing: TraefikRoutingConfig;
claimLock: ClaimMutationLock;
}>;
// A later post-cooldown claim uses a distinct claim file. New active routers
// use an explicit high priority. Tombstones stay one below the exact Host rule
// length so a rollback release using Traefik's default rule-length priority also
// wins after reclaim, without deleting fencing state this DB-free controller
// cannot authoritatively age.
export const PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY = 1_000_001;
const CONFIGURATION_MUTATION_LOCK_ID = "__portal_domain_configuration_mutation__";
const GENERATION_STATE_FILE_PATTERN = /^\.portal-([a-f0-9]{32})\.generation$/u;
const MAX_GENERATION_STATE_BYTES = 4_096;
const MAX_DOMAIN_CONFIG_BYTES = 16_384;
export class UnsafeConfigPathError extends Error {
readonly retryable = false;
readonly reasonCode = "unsafe_config_path";
}
export class GenerationConflictError extends Error {
readonly retryable = false;
readonly reasonCode = "generation_conflict";
}
export class TraefikConfigWriter {
readonly #configuredDirectory: string;
readonly #configuredReloadTriggerFile: string;
readonly #routing: TraefikRoutingConfig;
readonly #distributedClaimLock: ClaimMutationLock;
readonly #claimLocks = new Map<string, Promise<void>>();
#reloadTail: Promise<void> = Promise.resolve();
constructor(options: TraefikConfigWriterOptions) {
if (!path.isAbsolute(options.configDirectory)) throw new UnsafeConfigPathError("Traefik config directory must be absolute");
if (!path.isAbsolute(options.reloadTriggerFile)) throw new UnsafeConfigPathError("Traefik reload trigger file must be absolute");
this.#configuredDirectory = path.resolve(options.configDirectory);
this.#configuredReloadTriggerFile = path.resolve(options.reloadTriggerFile);
if (path.basename(this.#configuredDirectory) !== PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME) {
throw new UnsafeConfigPathError(`Traefik config directory must end with ${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}`);
}
if (path.basename(this.#configuredReloadTriggerFile) !== PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME) {
throw new UnsafeConfigPathError(`Traefik reload trigger file must end with ${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`);
}
if (path.dirname(this.#configuredReloadTriggerFile) !== path.dirname(this.#configuredDirectory)) {
throw new UnsafeConfigPathError("Traefik reload trigger file must be directly beside its dedicated config directory");
}
this.#routing = Object.freeze({
entryPoint: normalizePortalDomainEntryPoint(options.routing.entryPoint),
backendService: normalizePortalDomainBackendService(options.routing.backendService),
});
this.#distributedClaimLock = options.claimLock;
}
async assertReady(): Promise<void> {
await this.#distributedClaimLock.assertReady();
const directory = await this.#resolveSafeDirectory();
await this.#openSafeReloadTrigger(fsConstants.O_WRONLY).then((handle) => handle.close());
const probePath = this.#containedPath(directory, `.write-probe-${randomUUID()}`);
const handle = await open(probePath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY | noFollowFlag(), 0o600);
try {
await handle.writeFile("ready\n", "utf8");
await handle.sync();
} finally {
await handle.close();
await unlink(probePath).catch(() => undefined);
}
}
async applyProvision(command: DomainCommandPayload): Promise<ConfigMutationResult> {
const yaml = renderTraefikDomainConfig(command, this.#routing);
return this.#applyMutation(command, yaml);
}
async applyDeprovision(command: DomainCommandPayload): Promise<ConfigMutationResult> {
return this.#applyMutation(command, renderTraefikDeprovisionedConfig(command, this.#routing));
}
async recordRevalidation(command: DomainCommandPayload): Promise<ConfigMutationResult> {
return this.#applyMutation(command, renderTraefikDomainConfig(command, this.#routing));
}
async readAppliedState(claimId: string): Promise<AppliedDomainState | undefined> {
const directory = await this.#resolveSafeDirectory();
return readAppliedStateFile(this.#statePath(directory, claimId));
}
/**
* Re-renders every durable applied state before the queue worker starts.
* This upgrades byte-identical legacy active routes with explicit priority
* and restores tombstones for releases that represented deprovisioning by
* deleting YAML. Active state alone never reopens a missing or unknown route:
* only a known current/legacy active file is trusted; every other shape is
* replaced with the exact-host fail-closed tombstone.
* The rollout contract drains the previous controller revision first, while
* the configuration-wide fence prevents races with current-release commands.
* A populated state set always triggers one reload so a crash after config
* writes but before the prior sentinel update is repaired on the next start.
*/
async reconcilePersistedDomainConfigs(): Promise<number> {
return this.#distributedClaimLock.withClaimLock(CONFIGURATION_MUTATION_LOCK_ID, (guard) =>
this.#reconcilePersistedDomainConfigsLocked(guard)
);
}
async #applyMutation(command: DomainCommandPayload, yaml?: string): Promise<ConfigMutationResult> {
return this.#withClaimLock(command.claimId, () =>
this.#distributedClaimLock.withClaimLock(CONFIGURATION_MUTATION_LOCK_ID, () =>
this.#distributedClaimLock.withClaimLock(command.claimId, (guard) => this.#applyMutationLocked(command, guard, yaml))
)
);
}
async #reconcilePersistedDomainConfigsLocked(guard: ClaimLockGuard): Promise<number> {
const directory = await this.#resolveSafeDirectory();
const entries = (await readdir(directory, { withFileTypes: true })).sort((left, right) => left.name.localeCompare(right.name));
const states: Array<{ key: string; statePath: string }> = [];
for (const entry of entries) {
const looksLikeGenerationState = entry.name.startsWith(".portal-") && entry.name.endsWith(".generation");
if (!looksLikeGenerationState) continue;
const match = GENERATION_STATE_FILE_PATTERN.exec(entry.name);
if (!match?.[1] || !entry.isFile() || entry.isSymbolicLink()) {
throw new UnsafeConfigPathError("Stored portal-domain generation state path is invalid");
}
states.push({ key: match[1], statePath: this.#containedPath(directory, entry.name) });
}
const persisted: Array<{ key: string; state: AppliedDomainState }> = [];
for (const candidate of states) {
const state = await readAppliedStateFile(candidate.statePath);
if (!state) throw new UnsafeConfigPathError("Stored portal-domain generation state disappeared during reconciliation");
persisted.push({ key: candidate.key, state });
}
let repaired = 0;
for (const candidate of persisted) {
const configPath = this.#configPathForKey(directory, candidate.key);
const tombstone = renderTraefikDeprovisionedConfigForKey(candidate.state.hostname, this.#routing.entryPoint, candidate.key);
let expected = tombstone;
if (candidate.state.action !== "deprovision") {
const active = renderTraefikDomainConfigForKey(candidate.state.hostname, this.#routing, candidate.key);
const legacyActive = renderLegacyTraefikDomainConfigForKey(candidate.state.hostname, this.#routing, candidate.key);
if (await fileMatches(configPath, active)) continue;
if (await fileMatches(configPath, legacyActive)) expected = active;
}
if (await fileMatches(configPath, expected)) continue;
await atomicWrite(directory, configPath, expected, guard);
repaired += 1;
}
if (persisted.length > 0) await this.#triggerReload(guard);
return repaired;
}
async #applyMutationLocked(command: DomainCommandPayload, guard: ClaimLockGuard, yaml?: string): Promise<ConfigMutationResult> {
const directory = await this.#resolveSafeDirectory();
const existing = await this.readAppliedState(command.claimId);
if (existing && (existing.tenantId !== command.tenantId || existing.hostname !== command.hostname)) {
throw new GenerationConflictError("A domain claim cannot change tenant or hostname identity");
}
if (existing && existing.generation > command.generation) return { disposition: "stale" };
if (
existing &&
existing.generation === command.generation &&
existing.action !== command.action &&
!(existing.action === "provision" && command.action === "revalidate")
) {
throw new GenerationConflictError("The same domain generation cannot carry a different action");
}
const configPath = this.#configPath(directory, command.claimId);
const configHash = typeof yaml === "string" ? sha256Hex(yaml) : undefined;
if (existing && existing.generation === command.generation) {
if (typeof yaml === "string") {
if (!(await fileMatches(configPath, yaml))) await atomicWrite(directory, configPath, yaml, guard);
await this.#triggerReload(guard);
} else if (yaml === undefined && (await safeUnlink(configPath, guard))) {
await syncDirectory(directory);
await this.#triggerReload(guard);
}
return { disposition: "unchanged", ...(configHash ? { configHash } : {}) };
}
let reloadRequired = false;
if (typeof yaml === "string") {
await atomicWrite(directory, configPath, yaml, guard);
reloadRequired = true;
} else if (yaml === undefined) {
await safeUnlink(configPath, guard);
await syncDirectory(directory);
reloadRequired = true;
}
if (reloadRequired) await this.#triggerReload(guard);
const state: AppliedDomainState = {
generation: command.generation,
action: command.action,
hostname: command.hostname,
tenantId: command.tenantId,
};
await atomicWrite(directory, this.#statePath(directory, command.claimId), `${JSON.stringify(state)}\n`, guard);
return { disposition: "applied", ...(configHash ? { configHash } : {}) };
}
async #resolveSafeDirectory(): Promise<string> {
const stats = await lstat(this.#configuredDirectory);
if (!stats.isDirectory() || stats.isSymbolicLink()) {
throw new UnsafeConfigPathError("Traefik config path must be a real directory, not a symlink");
}
const canonical = await realpath(this.#configuredDirectory);
return canonical;
}
/**
* Traefik recursively reads the dedicated subdirectory after a watched-root
* event, but does not watch nested directories themselves. Updating this
* non-YAML sibling in place triggers that rescan without granting the
* controller write access to the complete dynamic-config root.
*/
async #triggerReload(guard: ClaimLockGuard): Promise<void> {
const previous = this.#reloadTail;
let release: () => void = () => {};
const current = new Promise<void>((resolve) => {
release = resolve;
});
this.#reloadTail = previous.then(() => current);
await previous;
try {
const handle = await this.#openSafeReloadTrigger(fsConstants.O_WRONLY);
try {
await guard.assertHeld();
await handle.truncate(0);
await handle.writeFile(`reload ${randomUUID()}\n`, "utf8");
await handle.sync();
const [pathStats, openedStats] = await Promise.all([lstat(this.#configuredReloadTriggerFile), handle.stat()]);
if (
!pathStats.isFile() ||
pathStats.isSymbolicLink() ||
pathStats.nlink !== 1 ||
openedStats.nlink !== 1 ||
openedStats.dev !== pathStats.dev ||
openedStats.ino !== pathStats.ino
) {
throw new UnsafeConfigPathError("Traefik reload trigger changed while it was being written");
}
} finally {
await handle.close();
}
} finally {
release();
}
}
async #openSafeReloadTrigger(flags: number) {
const pathStats = await lstat(this.#configuredReloadTriggerFile);
if (!pathStats.isFile() || pathStats.isSymbolicLink() || pathStats.nlink !== 1) {
throw new UnsafeConfigPathError("Traefik reload trigger must be one regular, non-linked file");
}
const canonical = await realpath(this.#configuredReloadTriggerFile);
const canonicalParent = await realpath(path.dirname(this.#configuredReloadTriggerFile));
if (canonical !== path.join(canonicalParent, path.basename(this.#configuredReloadTriggerFile))) {
throw new UnsafeConfigPathError("Traefik reload trigger path must not resolve through a link");
}
const handle = await open(this.#configuredReloadTriggerFile, flags | noFollowFlag());
try {
const openedStats = await handle.stat();
if (!openedStats.isFile() || openedStats.nlink !== 1 || openedStats.dev !== pathStats.dev || openedStats.ino !== pathStats.ino) {
throw new UnsafeConfigPathError("Traefik reload trigger changed while it was being opened");
}
return handle;
} catch (error) {
await handle.close().catch(() => undefined);
throw error;
}
}
async #withClaimLock<T>(claimId: string, operation: () => Promise<T>): Promise<T> {
const previous = this.#claimLocks.get(claimId) ?? Promise.resolve();
let release: () => void = () => {};
const current = new Promise<void>((resolve) => {
release = resolve;
});
const tail = previous.then(() => current);
this.#claimLocks.set(claimId, tail);
await previous;
try {
return await operation();
} finally {
release();
if (this.#claimLocks.get(claimId) === tail) this.#claimLocks.delete(claimId);
}
}
#configPath(directory: string, claimId: string): string {
return this.#configPathForKey(directory, claimFileKey(claimId));
}
#configPathForKey(directory: string, key: string): string {
if (!/^[a-f0-9]{32}$/u.test(key)) throw new UnsafeConfigPathError("Derived portal-domain config key is invalid");
return this.#containedPath(directory, `portal-${key}.yaml`);
}
#statePath(directory: string, claimId: string): string {
return this.#containedPath(directory, `.portal-${claimFileKey(claimId)}.generation`);
}
#containedPath(directory: string, filename: string): string {
const candidate = path.resolve(directory, filename);
if (path.dirname(candidate) !== directory) throw new UnsafeConfigPathError("Derived config path escaped its dedicated directory");
return candidate;
}
}
export function renderTraefikDomainConfig(command: Pick<DomainCommandPayload, "claimId" | "hostname">, routing: TraefikRoutingConfig): string {
return renderTraefikDomainConfigForKey(command.hostname, routing, claimFileKey(command.claimId));
}
function renderTraefikDomainConfigForKey(hostnameValue: string, routing: TraefikRoutingConfig, key: string): string {
return renderTraefikActiveConfigForKey(hostnameValue, routing, key, PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY);
}
function renderLegacyTraefikDomainConfigForKey(hostnameValue: string, routing: TraefikRoutingConfig, key: string): string {
return renderTraefikActiveConfigForKey(hostnameValue, routing, key);
}
function renderTraefikActiveConfigForKey(
hostnameValue: string,
routing: TraefikRoutingConfig,
key: string,
priority?: number
): string {
const hostname = normalizeDomainHostname(hostnameValue);
const entryPoint = normalizePortalDomainEntryPoint(routing.entryPoint);
const backendService = normalizePortalDomainBackendService(routing.backendService);
if (!/^[a-f0-9]{32}$/u.test(key)) throw new UnsafeConfigPathError("Derived portal-domain config key is invalid");
const router = `portal-domain-${key}`;
return [
"# Generated by hivegrid-portal-domain-controller. Manual edits are replaced.",
"http:",
" routers:",
` ${router}:`,
` rule: ${JSON.stringify(`Host(\`${hostname}\`)`)}`,
...(priority === undefined ? [] : [` priority: ${priority}`]),
" entryPoints:",
` - ${JSON.stringify(entryPoint)}`,
` service: ${JSON.stringify(backendService)}`,
" tls:",
` certResolver: ${PORTAL_DOMAIN_TLS_CERT_RESOLVER}`,
"",
].join("\n");
}
export function renderTraefikDeprovisionedConfig(
command: Pick<DomainCommandPayload, "claimId" | "hostname">,
routing: Pick<TraefikRoutingConfig, "entryPoint">
): string {
return renderTraefikDeprovisionedConfigForKey(command.hostname, routing.entryPoint, claimFileKey(command.claimId));
}
export function portalDomainTombstoneRouterPriority(hostnameValue: string): number {
const hostname = normalizeDomainHostname(hostnameValue);
const exactHostRule = `Host(\`${hostname}\`)`;
return tombstonePriorityForExactHostRule(exactHostRule);
}
function renderTraefikDeprovisionedConfigForKey(hostnameValue: string, entryPointValue: string, key: string): string {
const hostname = normalizeDomainHostname(hostnameValue);
const exactHostRule = `Host(\`${hostname}\`)`;
const entryPoint = normalizePortalDomainEntryPoint(entryPointValue);
if (!/^[a-f0-9]{32}$/u.test(key)) throw new UnsafeConfigPathError("Derived portal-domain config key is invalid");
const router = `portal-domain-${key}`;
const middleware = `portal-domain-tombstone-${key}`;
return [
"# Generated by hivegrid-portal-domain-controller. Manual edits are replaced.",
"http:",
" routers:",
` ${router}:`,
` rule: ${JSON.stringify(exactHostRule)}`,
` priority: ${tombstonePriorityForExactHostRule(exactHostRule)}`,
" entryPoints:",
` - ${JSON.stringify(entryPoint)}`,
" middlewares:",
` - ${JSON.stringify(middleware)}`,
' service: "noop@internal"',
" tls: {}",
" middlewares:",
` ${middleware}:`,
" errors:",
" status:",
' - "418"',
" statusRewrites:",
' "418": 404',
' service: "noop@internal"',
' query: "/"',
"",
].join("\n");
}
function tombstonePriorityForExactHostRule(exactHostRule: string): number {
return Math.max(1, Buffer.byteLength(exactHostRule, "utf8") - 1);
}
function claimFileKey(claimId: string): string {
return sha256Hex(claimId).slice(0, 32);
}
async function atomicWrite(directory: string, targetPath: string, contents: string, guard: ClaimLockGuard): Promise<void> {
await assertNotSymlink(targetPath);
const temporaryPath = path.join(directory, `.tmp-${randomUUID()}`);
if (path.dirname(temporaryPath) !== directory) throw new UnsafeConfigPathError("Temporary path escaped config directory");
let handle;
try {
handle = await open(temporaryPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY | noFollowFlag(), 0o600);
await handle.writeFile(contents, "utf8");
await handle.sync();
await handle.close();
handle = undefined;
await guard.assertHeld();
await rename(temporaryPath, targetPath);
await syncDirectory(directory);
} finally {
await handle?.close().catch(() => undefined);
await unlink(temporaryPath).catch(() => undefined);
}
}
async function safeUnlink(targetPath: string, guard: ClaimLockGuard): Promise<boolean> {
await assertNotSymlink(targetPath);
try {
await guard.assertHeld();
await unlink(targetPath);
return true;
} catch (error) {
if (isErrno(error, "ENOENT")) return false;
throw error;
}
}
async function assertNotSymlink(targetPath: string): Promise<void> {
try {
const stats = await lstat(targetPath);
if (stats.isSymbolicLink() || !stats.isFile() || stats.nlink !== 1) {
throw new UnsafeConfigPathError("Portal-domain target must be one regular, non-linked file");
}
} catch (error) {
if (!isErrno(error, "ENOENT")) throw error;
}
}
async function fileMatches(targetPath: string, expected: string): Promise<boolean> {
const contents = await readBoundedRegularFile(targetPath, MAX_DOMAIN_CONFIG_BYTES, "Portal-domain config");
return contents === expected;
}
async function readAppliedStateFile(statePath: string): Promise<AppliedDomainState | undefined> {
const raw = await readBoundedRegularFile(statePath, MAX_GENERATION_STATE_BYTES, "Stored portal-domain generation state");
if (raw === undefined) return undefined;
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
throw new UnsafeConfigPathError("Stored portal-domain generation state is malformed");
}
if (!isAppliedState(parsed)) throw new UnsafeConfigPathError("Stored portal-domain generation state is invalid");
return parsed;
}
async function readBoundedRegularFile(targetPath: string, maxBytes: number, label: string): Promise<string | undefined> {
let pathBefore: Stats;
try {
pathBefore = await lstat(targetPath);
} catch (error) {
if (isErrno(error, "ENOENT")) return undefined;
throw error;
}
assertSafeRegularFile(pathBefore, maxBytes, `${label} path is invalid`);
let handle;
try {
handle = await open(targetPath, fsConstants.O_RDONLY | noFollowFlag());
} catch (error) {
if (isErrno(error, "ENOENT")) throw new UnsafeConfigPathError(`${label} changed while it was being opened`);
throw error;
}
try {
const openedBefore = await handle.stat();
assertStableRegularFile(pathBefore, openedBefore, maxBytes, `${label} changed while it was being opened`);
const buffer = Buffer.alloc(maxBytes + 1);
let bytesRead = 0;
while (bytesRead < buffer.length) {
const chunk = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead);
if (chunk.bytesRead === 0) break;
bytesRead += chunk.bytesRead;
}
if (bytesRead > maxBytes) throw new UnsafeConfigPathError(`${label} exceeds its size limit`);
const openedAfter = await handle.stat();
let pathAfter: Stats;
try {
pathAfter = await lstat(targetPath);
} catch (error) {
if (isErrno(error, "ENOENT")) throw new UnsafeConfigPathError(`${label} changed while it was being read`);
throw error;
}
assertStableRegularFile(pathBefore, openedAfter, maxBytes, `${label} changed while it was being read`);
assertStableRegularFile(openedAfter, pathAfter, maxBytes, `${label} changed while it was being read`);
if (bytesRead !== openedAfter.size) {
throw new UnsafeConfigPathError(`${label} changed while it was being read`);
}
return buffer.subarray(0, bytesRead).toString("utf8");
} finally {
await handle.close();
}
}
function assertSafeRegularFile(stats: Stats, maxBytes: number, message: string): void {
if (!stats.isFile() || stats.isSymbolicLink() || stats.nlink !== 1 || stats.size > maxBytes) {
throw new UnsafeConfigPathError(message);
}
}
function assertStableRegularFile(before: Stats, after: Stats, maxBytes: number, message: string): void {
assertSafeRegularFile(before, maxBytes, message);
assertSafeRegularFile(after, maxBytes, message);
if (
before.dev !== after.dev ||
before.ino !== after.ino ||
before.size !== after.size ||
before.mtimeMs !== after.mtimeMs ||
before.ctimeMs !== after.ctimeMs
) {
throw new UnsafeConfigPathError(message);
}
}
async function syncDirectory(directory: string): Promise<void> {
const handle = await open(directory, fsConstants.O_RDONLY | directoryFlag() | noFollowFlag());
try {
await handle.sync();
} finally {
await handle.close();
}
}
function noFollowFlag(): number {
return fsConstants.O_NOFOLLOW ?? 0;
}
function directoryFlag(): number {
return fsConstants.O_DIRECTORY ?? 0;
}
function isAppliedState(value: unknown): value is AppliedDomainState {
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
const record = value as Record<string, unknown>;
try {
return (
Object.keys(record).sort().join(",") === "action,generation,hostname,tenantId" &&
Number.isSafeInteger(record.generation) &&
(record.generation as number) >= 1 &&
(record.action === "provision" || record.action === "revalidate" || record.action === "deprovision") &&
typeof record.hostname === "string" &&
normalizeDomainHostname(record.hostname) === record.hostname &&
typeof record.tenantId === "string" &&
/^[A-Za-z0-9][A-Za-z0-9_-]{7,127}$/u.test(record.tenantId)
);
} catch {
return false;
}
}
function isErrno(error: unknown, code: string): boolean {
return typeof error === "object" && error !== null && "code" in error && (error as NodeJS.ErrnoException).code === code;
}