Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+368
@@ -0,0 +1,368 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
ClaimLockLostError,
|
||||
ClaimLockUnavailableError,
|
||||
FencedClaimMutationLock,
|
||||
FilesystemClaimMutationLock,
|
||||
RedisClaimMutationLock,
|
||||
type ClaimLockGuard,
|
||||
type ClaimLockRedisClient,
|
||||
type ClaimMutationLock,
|
||||
} from "./claim-lock.js";
|
||||
|
||||
describe("Redis portal-domain claim lock", () => {
|
||||
it("serializes two controller instances and renews ownership during a long mutation", async () => {
|
||||
vi.useFakeTimers();
|
||||
const store = new FakeRedisStore();
|
||||
const first = lockFor(store.client());
|
||||
const second = lockFor(store.client());
|
||||
const entered = deferred<void>();
|
||||
const finish = deferred<void>();
|
||||
const order: string[] = [];
|
||||
try {
|
||||
const firstOperation = first.withClaimLock("claim_12345678", async () => {
|
||||
order.push("first:start");
|
||||
entered.resolve();
|
||||
await finish.promise;
|
||||
order.push("first:end");
|
||||
});
|
||||
await entered.promise;
|
||||
const secondOperation = second.withClaimLock("claim_12345678", async () => {
|
||||
order.push("second");
|
||||
});
|
||||
|
||||
// This is longer than the original lease. The second instance must still
|
||||
// wait because the first instance renews with its owner token.
|
||||
await vi.advanceTimersByTimeAsync(70);
|
||||
expect(order).toEqual(["first:start"]);
|
||||
finish.resolve();
|
||||
await firstOperation;
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await secondOperation;
|
||||
|
||||
expect(order).toEqual(["first:start", "first:end", "second"]);
|
||||
expect(store.keys()).toEqual([]);
|
||||
await Promise.all([first.close(), second.close()]);
|
||||
} finally {
|
||||
finish.resolve();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("times out acquisition predictably without exposing Redis error details", async () => {
|
||||
vi.useFakeTimers();
|
||||
const store = new FakeRedisStore();
|
||||
const holder = lockFor(store.client());
|
||||
const entered = deferred<void>();
|
||||
const finish = deferred<void>();
|
||||
try {
|
||||
const held = holder.withClaimLock("claim_12345678", async () => {
|
||||
entered.resolve();
|
||||
await finish.promise;
|
||||
});
|
||||
await entered.promise;
|
||||
|
||||
const contender = lockFor(store.client(), { acquireTimeoutMs: 15 });
|
||||
const contenderAttempt = contender.withClaimLock("claim_12345678", async () => undefined).catch((error: unknown) => error);
|
||||
await vi.advanceTimersByTimeAsync(15);
|
||||
await expect(contenderAttempt).resolves.toMatchObject({
|
||||
reasonCode: "claim_lock_unavailable",
|
||||
retryable: true,
|
||||
});
|
||||
|
||||
const failingClient = store.client();
|
||||
failingClient.set = async () => {
|
||||
throw new Error("redis://:do-not-leak@redis.internal:6379");
|
||||
};
|
||||
const failing = lockFor(failingClient, { acquireTimeoutMs: 5 });
|
||||
const failureAttempt = failing.withClaimLock("claim_87654321", async () => undefined).catch((error: unknown) => error);
|
||||
await vi.advanceTimersByTimeAsync(5);
|
||||
const failure = await failureAttempt;
|
||||
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
|
||||
expect(String(failure)).not.toContain("do-not-leak");
|
||||
expect(String(failure)).not.toContain("redis.internal");
|
||||
|
||||
finish.resolve();
|
||||
await held;
|
||||
await Promise.all([holder.close(), contender.close(), failing.close()]);
|
||||
} finally {
|
||||
finish.resolve();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("fences a stale writer after lease loss while a second writer owns the claim", async () => {
|
||||
const store = new FakeRedisStore();
|
||||
const first = lockFor(store.client(), { token: () => "original-owner" });
|
||||
const second = lockFor(store.client(), { token: () => "new-owner" });
|
||||
const firstEntered = deferred<void>();
|
||||
const resumeFirst = deferred<void>();
|
||||
const secondEntered = deferred<void>();
|
||||
const finishSecond = deferred<void>();
|
||||
let resource = "initial";
|
||||
|
||||
const staleOperation = first.withClaimLock("claim_12345678", async (guard) => {
|
||||
firstEntered.resolve();
|
||||
await resumeFirst.promise;
|
||||
await guard.assertHeld();
|
||||
resource = "stale";
|
||||
});
|
||||
await firstEntered.promise;
|
||||
store.deleteOnlyValue();
|
||||
|
||||
const currentOperation = second.withClaimLock("claim_12345678", async (guard) => {
|
||||
await guard.assertHeld();
|
||||
resource = "current";
|
||||
secondEntered.resolve();
|
||||
await finishSecond.promise;
|
||||
});
|
||||
await secondEntered.promise;
|
||||
resumeFirst.resolve();
|
||||
|
||||
await expect(staleOperation).rejects.toBeInstanceOf(ClaimLockLostError);
|
||||
expect(resource).toBe("current");
|
||||
expect(store.onlyValue()).toBe("new-owner");
|
||||
finishSecond.resolve();
|
||||
await currentOperation;
|
||||
expect(store.keys()).toEqual([]);
|
||||
await Promise.all([first.close(), second.close()]);
|
||||
});
|
||||
|
||||
it("keeps the authoritative fence held after a lease guard passes until the whole mutation returns", async () => {
|
||||
const filesystem = new SerializedClaimMutationLock();
|
||||
const lease = new PassthroughClaimMutationLock();
|
||||
const first = new FencedClaimMutationLock(filesystem, lease);
|
||||
const second = new FencedClaimMutationLock(filesystem, lease);
|
||||
const firstGuardPassed = deferred<void>();
|
||||
const resumeFirst = deferred<void>();
|
||||
let secondEntered = false;
|
||||
let resource = "initial";
|
||||
|
||||
const staleMutation = first.withClaimLock("claim_12345678", async (guard) => {
|
||||
await guard.assertHeld();
|
||||
firstGuardPassed.resolve();
|
||||
// Represents an event-loop pause after Redis ownership was checked but
|
||||
// immediately before rename/unlink. The filesystem fence must remain.
|
||||
await resumeFirst.promise;
|
||||
resource = "stale-generation";
|
||||
});
|
||||
await firstGuardPassed.promise;
|
||||
const currentMutation = second.withClaimLock("claim_12345678", async () => {
|
||||
secondEntered = true;
|
||||
resource = "current-generation";
|
||||
});
|
||||
|
||||
await wait(20);
|
||||
expect(secondEntered).toBe(false);
|
||||
resumeFirst.resolve();
|
||||
await Promise.all([staleMutation, currentMutation]);
|
||||
expect(secondEntered).toBe(true);
|
||||
expect(resource).toBe("current-generation");
|
||||
});
|
||||
|
||||
it.skipIf(!existsSync("/usr/bin/flock"))("holds one kernel flock across controller instances until the parent fd closes", async () => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "hivegrid-portal-domain-flock-"));
|
||||
const first = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
|
||||
const second = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
|
||||
const entered = deferred<void>();
|
||||
const finish = deferred<void>();
|
||||
let secondEntered = false;
|
||||
try {
|
||||
const firstMutation = first.withClaimLock("claim_12345678", async () => {
|
||||
entered.resolve();
|
||||
await finish.promise;
|
||||
});
|
||||
await entered.promise;
|
||||
const secondMutation = second.withClaimLock("claim_12345678", async () => {
|
||||
secondEntered = true;
|
||||
});
|
||||
await wait(30);
|
||||
expect(secondEntered).toBe(false);
|
||||
finish.resolve();
|
||||
await Promise.all([firstMutation, secondMutation]);
|
||||
expect(secondEntered).toBe(true);
|
||||
} finally {
|
||||
await Promise.all([first.close(), second.close()]);
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("hashes claim identifiers, reports readiness and closes the client", async () => {
|
||||
const store = new FakeRedisStore();
|
||||
const client = store.client("wait");
|
||||
const lock = lockFor(client);
|
||||
|
||||
await expect(lock.assertReady()).resolves.toBeUndefined();
|
||||
expect(store.readinessCalls()).toEqual({ sets: 1, renewals: 1, releases: 1 });
|
||||
await lock.withClaimLock("../../raw-secret-claim", async () => {
|
||||
expect(store.keys()).toHaveLength(1);
|
||||
expect(store.keys()[0]).not.toContain("raw-secret-claim");
|
||||
expect(store.keys()[0]).toMatch(/^hivegrid:portal-domain-controller:claim-lock:[a-f0-9]{64}$/u);
|
||||
});
|
||||
await lock.close();
|
||||
await expect(lock.assertReady()).rejects.toBeInstanceOf(ClaimLockUnavailableError);
|
||||
expect(client.status).toBe("end");
|
||||
});
|
||||
|
||||
it("fails readiness when Redis permits PING but denies lock writes", async () => {
|
||||
const store = new FakeRedisStore();
|
||||
const client = store.client("wait");
|
||||
client.set = async () => {
|
||||
throw new Error("NOPERM redis://:do-not-leak@redis.internal");
|
||||
};
|
||||
const lock = lockFor(client);
|
||||
|
||||
const failure = await lock.assertReady().catch((error: unknown) => error);
|
||||
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
|
||||
expect(failure).toMatchObject({ reasonCode: "claim_lock_unavailable", retryable: true });
|
||||
expect(String(failure)).not.toContain("do-not-leak");
|
||||
expect(String(failure)).not.toContain("redis.internal");
|
||||
expect(store.keys()).toEqual([]);
|
||||
await lock.close();
|
||||
});
|
||||
});
|
||||
|
||||
class PassthroughClaimMutationLock implements ClaimMutationLock {
|
||||
async assertReady(): Promise<void> {}
|
||||
async withClaimLock<T>(_claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
return operation({ assertHeld: async () => undefined });
|
||||
}
|
||||
async close(): Promise<void> {}
|
||||
}
|
||||
|
||||
class SerializedClaimMutationLock implements ClaimMutationLock {
|
||||
readonly #tails = new Map<string, Promise<void>>();
|
||||
|
||||
async assertReady(): Promise<void> {}
|
||||
|
||||
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
const previous = this.#tails.get(claimId) ?? Promise.resolve();
|
||||
let release: () => void = () => undefined;
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const tail = previous.then(() => current);
|
||||
this.#tails.set(claimId, tail);
|
||||
await previous;
|
||||
try {
|
||||
return await operation({ assertHeld: async () => undefined });
|
||||
} finally {
|
||||
release();
|
||||
if (this.#tails.get(claimId) === tail) this.#tails.delete(claimId);
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {}
|
||||
}
|
||||
|
||||
type TestLockOptions = ConstructorParameters<typeof RedisClaimMutationLock>[1];
|
||||
|
||||
function lockFor(client: ClaimLockRedisClient, options: TestLockOptions = {}): RedisClaimMutationLock {
|
||||
return new RedisClaimMutationLock(client, {
|
||||
leaseMs: 30,
|
||||
heartbeatIntervalMs: 8,
|
||||
acquireTimeoutMs: 250,
|
||||
retryDelayMs: 1,
|
||||
retryJitterMs: 0,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
class FakeRedisStore {
|
||||
readonly #values = new Map<string, { value: string; expiresAt: number }>();
|
||||
#sets = 0;
|
||||
#renewals = 0;
|
||||
#releases = 0;
|
||||
|
||||
client(initialStatus: string = "ready"): ClaimLockRedisClient {
|
||||
let status = initialStatus;
|
||||
const assertConnected = () => {
|
||||
if (status !== "ready") throw new Error("redis://:hidden@unavailable.internal");
|
||||
};
|
||||
return {
|
||||
get status() {
|
||||
return status;
|
||||
},
|
||||
connect: async () => {
|
||||
status = "ready";
|
||||
},
|
||||
ping: async () => {
|
||||
assertConnected();
|
||||
return "PONG";
|
||||
},
|
||||
set: async (key, value, _expiryMode, leaseMs) => {
|
||||
assertConnected();
|
||||
this.#sets += 1;
|
||||
this.#expire(key);
|
||||
if (this.#values.has(key)) return null;
|
||||
this.#values.set(key, { value, expiresAt: Date.now() + leaseMs });
|
||||
return "OK";
|
||||
},
|
||||
eval: async (_script, _numberOfKeys, ...args) => {
|
||||
assertConnected();
|
||||
const [key, token, leaseMs] = args;
|
||||
if (typeof key !== "string" || typeof token !== "string") return 0;
|
||||
this.#expire(key);
|
||||
const entry = this.#values.get(key);
|
||||
if (!entry || entry.value !== token) return 0;
|
||||
if (leaseMs !== undefined) {
|
||||
this.#renewals += 1;
|
||||
entry.expiresAt = Date.now() + Number(leaseMs);
|
||||
return 1;
|
||||
}
|
||||
this.#releases += 1;
|
||||
this.#values.delete(key);
|
||||
return 1;
|
||||
},
|
||||
quit: async () => {
|
||||
status = "end";
|
||||
return "OK";
|
||||
},
|
||||
disconnect: () => {
|
||||
status = "end";
|
||||
},
|
||||
on: () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
keys(): string[] {
|
||||
for (const key of this.#values.keys()) this.#expire(key);
|
||||
return [...this.#values.keys()];
|
||||
}
|
||||
|
||||
deleteOnlyValue(): void {
|
||||
const [key] = this.keys();
|
||||
if (!key) throw new Error("expected one lock key");
|
||||
this.#values.delete(key);
|
||||
}
|
||||
|
||||
onlyValue(): string | undefined {
|
||||
const [key] = this.keys();
|
||||
return key ? this.#values.get(key)?.value : undefined;
|
||||
}
|
||||
|
||||
readinessCalls(): { sets: number; renewals: number; releases: number } {
|
||||
return { sets: this.#sets, renewals: this.#renewals, releases: this.#releases };
|
||||
}
|
||||
|
||||
#expire(key: string): void {
|
||||
const entry = this.#values.get(key);
|
||||
if (entry && entry.expiresAt <= Date.now()) this.#values.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve: (value: T | PromiseLike<T>) => void = () => undefined;
|
||||
const promise = new Promise<T>((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function wait(delayMs: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
}
|
||||
+484
@@ -0,0 +1,484 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import { lstat, open, realpath, type FileHandle } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { ConnectionOptions } from "@hivegrid/queue";
|
||||
import { Redis } from "ioredis";
|
||||
import { sha256Hex } from "./contracts.js";
|
||||
|
||||
const LOCK_KEY_PREFIX = "hivegrid:portal-domain-controller:claim-lock:";
|
||||
const DEFAULT_LEASE_MS = 30_000;
|
||||
const DEFAULT_ACQUIRE_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_RETRY_DELAY_MS = 75;
|
||||
const DEFAULT_RETRY_JITTER_MS = 75;
|
||||
const DEFAULT_COMMAND_TIMEOUT_MS = 2_500;
|
||||
const DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS = 10_000;
|
||||
const FLOCK_BINARY = "/usr/bin/flock";
|
||||
|
||||
const RENEW_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('PEXPIRE', KEYS[1], ARGV[2])", "end", "return 0"].join("\n");
|
||||
|
||||
const RELEASE_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('DEL', KEYS[1])", "end", "return 0"].join("\n");
|
||||
|
||||
export type ClaimMutationLock = {
|
||||
assertReady(): Promise<void>;
|
||||
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export type ClaimLockGuard = Readonly<{
|
||||
assertHeld(): Promise<void>;
|
||||
}>;
|
||||
|
||||
export type ClaimLockRedisClient = {
|
||||
readonly status: string;
|
||||
connect(): Promise<unknown>;
|
||||
ping(): Promise<string>;
|
||||
set(key: string, value: string, expiryMode: "PX", leaseMs: number, condition: "NX"): Promise<"OK" | null>;
|
||||
eval(script: string, numberOfKeys: number, ...args: Array<string | number>): Promise<unknown>;
|
||||
quit(): Promise<unknown>;
|
||||
disconnect(reconnect?: boolean): void;
|
||||
on(event: "error", listener: (error: unknown) => void): unknown;
|
||||
};
|
||||
|
||||
type ClaimLockOptions = Readonly<{
|
||||
leaseMs?: number;
|
||||
acquireTimeoutMs?: number;
|
||||
retryDelayMs?: number;
|
||||
retryJitterMs?: number;
|
||||
heartbeatIntervalMs?: number;
|
||||
now?: () => number;
|
||||
random?: () => number;
|
||||
token?: () => string;
|
||||
wait?: (delayMs: number) => Promise<void>;
|
||||
}>;
|
||||
|
||||
type NormalizedClaimLockOptions = Readonly<{
|
||||
leaseMs: number;
|
||||
acquireTimeoutMs: number;
|
||||
retryDelayMs: number;
|
||||
retryJitterMs: number;
|
||||
heartbeatIntervalMs: number;
|
||||
now: () => number;
|
||||
random: () => number;
|
||||
token: () => string;
|
||||
wait: (delayMs: number) => Promise<void>;
|
||||
}>;
|
||||
|
||||
export class ClaimLockUnavailableError extends Error {
|
||||
readonly retryable = true;
|
||||
readonly reasonCode = "claim_lock_unavailable";
|
||||
|
||||
constructor() {
|
||||
super("Portal-domain claim lock is temporarily unavailable");
|
||||
this.name = "ClaimLockUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ClaimLockLostError extends Error {
|
||||
readonly retryable = true;
|
||||
readonly reasonCode = "claim_lock_lost";
|
||||
|
||||
constructor() {
|
||||
super("Portal-domain claim lock ownership was lost");
|
||||
this.name = "ClaimLockLostError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Kernel-backed ownership for the shared Traefik directory. Redis remains the
|
||||
* fast distributed coordinator, while this non-expiring `flock` is the
|
||||
* authoritative fence around filesystem mutations. The lock is attached to
|
||||
* the parent's open-file description, so a paused Node event loop cannot let a
|
||||
* second controller enter after a Redis lease expires. A crash closes the fd
|
||||
* and lets the kernel release ownership automatically.
|
||||
*/
|
||||
export class FilesystemClaimMutationLock implements ClaimMutationLock {
|
||||
readonly #configuredDirectory: string;
|
||||
readonly #binary: string;
|
||||
readonly #acquireTimeoutMs: number;
|
||||
readonly #activeOperations = new Set<Promise<void>>();
|
||||
#closing = false;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
|
||||
constructor(directory: string, options: { binary?: string; acquireTimeoutMs?: number } = {}) {
|
||||
if (!path.isAbsolute(directory)) throw new ClaimLockUnavailableError();
|
||||
this.#configuredDirectory = path.resolve(directory);
|
||||
this.#binary = options.binary ?? FLOCK_BINARY;
|
||||
this.#acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS, "filesystem claim lock timeout");
|
||||
}
|
||||
|
||||
async assertReady(): Promise<void> {
|
||||
await this.withClaimLock("__readiness__", async (guard) => guard.assertHeld());
|
||||
}
|
||||
|
||||
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
||||
let finishActive: () => void = () => undefined;
|
||||
const active = new Promise<void>((resolve) => {
|
||||
finishActive = resolve;
|
||||
});
|
||||
this.#activeOperations.add(active);
|
||||
try {
|
||||
return await this.#withClaimLock(claimId, operation);
|
||||
} finally {
|
||||
finishActive();
|
||||
this.#activeOperations.delete(active);
|
||||
}
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.#closePromise ??= this.#close();
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #close(): Promise<void> {
|
||||
if (this.#closed) return;
|
||||
this.#closing = true;
|
||||
await Promise.all([...this.#activeOperations]);
|
||||
this.#closed = true;
|
||||
}
|
||||
|
||||
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
const directory = await this.#resolveSafeDirectory();
|
||||
const lockPath = path.join(directory, `.portal-${sha256Hex(claimId)}.lock`);
|
||||
if (path.dirname(lockPath) !== directory) throw new ClaimLockUnavailableError();
|
||||
let handle: FileHandle | undefined;
|
||||
let held = false;
|
||||
try {
|
||||
handle = await open(
|
||||
lockPath,
|
||||
fsConstants.O_CREAT | fsConstants.O_RDWR | noFollowFlag(),
|
||||
0o600
|
||||
);
|
||||
await assertSameRegularFile(lockPath, handle);
|
||||
await acquireKernelFlock(handle.fd, this.#binary, this.#acquireTimeoutMs);
|
||||
held = true;
|
||||
const guard: ClaimLockGuard = Object.freeze({
|
||||
assertHeld: async () => {
|
||||
if (!held || !handle) throw new ClaimLockLostError();
|
||||
await assertSameRegularFile(lockPath, handle);
|
||||
},
|
||||
});
|
||||
return await operation(guard);
|
||||
} catch (error) {
|
||||
if (error instanceof ClaimLockLostError || error instanceof ClaimLockUnavailableError) throw error;
|
||||
throw new ClaimLockUnavailableError();
|
||||
} finally {
|
||||
held = false;
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async #resolveSafeDirectory(): Promise<string> {
|
||||
try {
|
||||
const stats = await lstat(this.#configuredDirectory);
|
||||
if (!stats.isDirectory() || stats.isSymbolicLink()) throw new ClaimLockUnavailableError();
|
||||
return await realpath(this.#configuredDirectory);
|
||||
} catch (error) {
|
||||
if (error instanceof ClaimLockUnavailableError) throw error;
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class FencedClaimMutationLock implements ClaimMutationLock {
|
||||
readonly #filesystem: ClaimMutationLock;
|
||||
readonly #lease: ClaimMutationLock;
|
||||
|
||||
constructor(filesystem: ClaimMutationLock, lease: ClaimMutationLock) {
|
||||
this.#filesystem = filesystem;
|
||||
this.#lease = lease;
|
||||
}
|
||||
|
||||
async assertReady(): Promise<void> {
|
||||
await this.#filesystem.assertReady();
|
||||
await this.#lease.assertReady();
|
||||
}
|
||||
|
||||
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
return this.#filesystem.withClaimLock(claimId, () => this.#lease.withClaimLock(claimId, operation));
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await Promise.all([this.#lease.close(), this.#filesystem.close()]);
|
||||
}
|
||||
}
|
||||
|
||||
export class RedisClaimMutationLock implements ClaimMutationLock {
|
||||
readonly #client: ClaimLockRedisClient;
|
||||
readonly #options: NormalizedClaimLockOptions;
|
||||
readonly #activeOperations = new Set<Promise<void>>();
|
||||
#closing = false;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
|
||||
constructor(client: ClaimLockRedisClient, options: ClaimLockOptions = {}) {
|
||||
this.#client = client;
|
||||
this.#options = normalizeOptions(options);
|
||||
// Redis connection errors are surfaced through bounded commands and the
|
||||
// readiness endpoint. Keeping this listener intentionally silent also
|
||||
// prevents connection details from reaching stderr outside our logger.
|
||||
this.#client.on("error", () => undefined);
|
||||
}
|
||||
|
||||
async assertReady(): Promise<void> {
|
||||
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
||||
try {
|
||||
if (this.#client.status === "wait") await this.#client.connect();
|
||||
if ((await this.#client.ping()) !== "PONG") throw new ClaimLockUnavailableError();
|
||||
await this.#probeLockCapabilities();
|
||||
} catch {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
||||
let finishActive: () => void = () => undefined;
|
||||
const active = new Promise<void>((resolve) => {
|
||||
finishActive = resolve;
|
||||
});
|
||||
this.#activeOperations.add(active);
|
||||
try {
|
||||
return await this.#withClaimLock(claimId, operation);
|
||||
} finally {
|
||||
finishActive();
|
||||
this.#activeOperations.delete(active);
|
||||
}
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.#closePromise ??= this.#close();
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #close(): Promise<void> {
|
||||
if (this.#closed) return;
|
||||
this.#closing = true;
|
||||
await Promise.all([...this.#activeOperations]);
|
||||
try {
|
||||
if (this.#client.status !== "wait" && this.#client.status !== "end") {
|
||||
await this.#client.quit();
|
||||
}
|
||||
} catch {
|
||||
// An immediate disconnect is safe after all leases were released. It
|
||||
// also keeps shutdown bounded when Redis disappears during a rollout.
|
||||
} finally {
|
||||
this.#client.disconnect(false);
|
||||
this.#closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
const key = `${LOCK_KEY_PREFIX}${sha256Hex(claimId)}`;
|
||||
const token = this.#options.token();
|
||||
await this.#acquire(key, token);
|
||||
|
||||
let heartbeatTimer: NodeJS.Timeout | undefined;
|
||||
let heartbeatInFlight: Promise<void> = Promise.resolve();
|
||||
let heartbeatError: ClaimLockLostError | ClaimLockUnavailableError | undefined;
|
||||
let heartbeatStopped = false;
|
||||
const scheduleHeartbeat = () => {
|
||||
if (heartbeatStopped || heartbeatError) return;
|
||||
heartbeatTimer = setTimeout(() => {
|
||||
heartbeatInFlight = this.#renew(key, token)
|
||||
.then(() => scheduleHeartbeat())
|
||||
.catch((error: unknown) => {
|
||||
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
|
||||
});
|
||||
}, this.#options.heartbeatIntervalMs);
|
||||
};
|
||||
scheduleHeartbeat();
|
||||
|
||||
const guard: ClaimLockGuard = Object.freeze({
|
||||
assertHeld: async () => {
|
||||
if (heartbeatError) throw heartbeatError;
|
||||
try {
|
||||
await this.#renew(key, token);
|
||||
} catch (error) {
|
||||
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
|
||||
throw heartbeatError;
|
||||
}
|
||||
if (heartbeatError) throw heartbeatError;
|
||||
},
|
||||
});
|
||||
|
||||
let result: T | undefined;
|
||||
let operationError: unknown;
|
||||
let operationFailed = false;
|
||||
try {
|
||||
result = await operation(guard);
|
||||
} catch (error) {
|
||||
operationFailed = true;
|
||||
operationError = error;
|
||||
}
|
||||
|
||||
heartbeatStopped = true;
|
||||
if (heartbeatTimer) clearTimeout(heartbeatTimer);
|
||||
await heartbeatInFlight.catch(() => undefined);
|
||||
const observedHeartbeatError = heartbeatError;
|
||||
let releaseError: ClaimLockUnavailableError | undefined;
|
||||
let released = false;
|
||||
try {
|
||||
released = await this.#release(key, token);
|
||||
} catch {
|
||||
releaseError = new ClaimLockUnavailableError();
|
||||
}
|
||||
|
||||
if (operationFailed) throw operationError;
|
||||
if (observedHeartbeatError) throw observedHeartbeatError;
|
||||
if (releaseError) throw releaseError;
|
||||
if (!released) throw new ClaimLockLostError();
|
||||
return result as T;
|
||||
}
|
||||
|
||||
async #acquire(key: string, token: string): Promise<void> {
|
||||
const deadline = this.#options.now() + this.#options.acquireTimeoutMs;
|
||||
while (!this.#closing && !this.#closed) {
|
||||
try {
|
||||
if ((await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK") return;
|
||||
} catch {
|
||||
// Redis can be transiently unavailable during a rolling restart. The
|
||||
// bounded deadline below turns that into a retryable controller result.
|
||||
}
|
||||
const remainingMs = deadline - this.#options.now();
|
||||
if (remainingMs <= 0) break;
|
||||
const jitterMs = Math.floor(this.#options.random() * (this.#options.retryJitterMs + 1));
|
||||
await this.#options.wait(Math.min(remainingMs, this.#options.retryDelayMs + jitterMs));
|
||||
}
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
|
||||
async #renew(key: string, token: string): Promise<void> {
|
||||
let renewed: unknown;
|
||||
try {
|
||||
renewed = await this.#client.eval(RENEW_SCRIPT, 1, key, token, this.#options.leaseMs);
|
||||
} catch {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
if (Number(renewed) !== 1) throw new ClaimLockLostError();
|
||||
}
|
||||
|
||||
async #release(key: string, token: string): Promise<boolean> {
|
||||
try {
|
||||
return Number(await this.#client.eval(RELEASE_SCRIPT, 1, key, token)) === 1;
|
||||
} catch {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
async #probeLockCapabilities(): Promise<void> {
|
||||
const key = `${LOCK_KEY_PREFIX}readiness:${sha256Hex(randomUUID())}`;
|
||||
const token = randomUUID();
|
||||
let acquired = false;
|
||||
let released = false;
|
||||
try {
|
||||
acquired = (await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK";
|
||||
if (!acquired) throw new ClaimLockUnavailableError();
|
||||
await this.#renew(key, token);
|
||||
released = await this.#release(key, token);
|
||||
if (!released) throw new ClaimLockLostError();
|
||||
} finally {
|
||||
if (acquired && !released) await this.#release(key, token).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createRedisClaimMutationLock(connection: ConnectionOptions): ClaimMutationLock {
|
||||
const client = new Redis({
|
||||
...connection,
|
||||
commandTimeout: DEFAULT_COMMAND_TIMEOUT_MS,
|
||||
enableOfflineQueue: false,
|
||||
enableReadyCheck: true,
|
||||
lazyConnect: true,
|
||||
maxRetriesPerRequest: 1,
|
||||
});
|
||||
return new RedisClaimMutationLock(client);
|
||||
}
|
||||
|
||||
export function createFencedClaimMutationLock(connection: ConnectionOptions, filesystemDirectory: string): ClaimMutationLock {
|
||||
return new FencedClaimMutationLock(new FilesystemClaimMutationLock(filesystemDirectory), createRedisClaimMutationLock(connection));
|
||||
}
|
||||
|
||||
async function assertSameRegularFile(lockPath: string, handle: FileHandle): Promise<void> {
|
||||
const [pathStats, openedStats] = await Promise.all([lstat(lockPath), handle.stat()]);
|
||||
if (
|
||||
!pathStats.isFile() ||
|
||||
pathStats.isSymbolicLink() ||
|
||||
pathStats.nlink !== 1 ||
|
||||
!openedStats.isFile() ||
|
||||
openedStats.nlink !== 1 ||
|
||||
openedStats.dev !== pathStats.dev ||
|
||||
openedStats.ino !== pathStats.ino
|
||||
) {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
async function acquireKernelFlock(fd: number, binary: string, timeoutMs: number): Promise<void> {
|
||||
const timeoutSeconds = Math.max(0.001, timeoutMs / 1_000).toFixed(3);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(binary, ["--exclusive", "--wait", timeoutSeconds, "3"], {
|
||||
stdio: ["ignore", "ignore", "ignore", fd],
|
||||
});
|
||||
let settled = false;
|
||||
const watchdog = setTimeout(() => {
|
||||
if (settled) return;
|
||||
child.kill("SIGKILL");
|
||||
}, timeoutMs + 1_000);
|
||||
watchdog.unref();
|
||||
child.once("error", () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(watchdog);
|
||||
reject(new ClaimLockUnavailableError());
|
||||
});
|
||||
child.once("close", (code) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(watchdog);
|
||||
if (code === 0) resolve();
|
||||
else reject(new ClaimLockUnavailableError());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function noFollowFlag(): number {
|
||||
return typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0;
|
||||
}
|
||||
|
||||
function normalizeOptions(options: ClaimLockOptions): NormalizedClaimLockOptions {
|
||||
const leaseMs = positiveInteger(options.leaseMs ?? DEFAULT_LEASE_MS, "claim lock lease");
|
||||
const acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_ACQUIRE_TIMEOUT_MS, "claim lock acquisition timeout");
|
||||
const retryDelayMs = nonNegativeInteger(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS, "claim lock retry delay");
|
||||
const retryJitterMs = nonNegativeInteger(options.retryJitterMs ?? DEFAULT_RETRY_JITTER_MS, "claim lock retry jitter");
|
||||
const heartbeatIntervalMs = positiveInteger(options.heartbeatIntervalMs ?? Math.floor(leaseMs / 3), "claim lock heartbeat interval");
|
||||
if (heartbeatIntervalMs >= leaseMs / 2) throw new Error("claim lock heartbeat interval must be less than half the lease");
|
||||
return {
|
||||
leaseMs,
|
||||
acquireTimeoutMs,
|
||||
retryDelayMs,
|
||||
retryJitterMs,
|
||||
heartbeatIntervalMs,
|
||||
now: options.now ?? Date.now,
|
||||
random: options.random ?? Math.random,
|
||||
token: options.token ?? randomUUID,
|
||||
wait: options.wait ?? wait,
|
||||
};
|
||||
}
|
||||
|
||||
function positiveInteger(value: number, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value: number, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function wait(delayMs: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parsePortalDomainControllerConfig, PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME } from "./config.js";
|
||||
|
||||
const baseEnv: NodeJS.ProcessEnv = {
|
||||
PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: "command-secret-that-is-long-enough-for-hmac",
|
||||
PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: "metrics-token-that-is-long-enough-to-be-safe",
|
||||
PORTAL_DOMAIN_CONTROLLER_REDIS_URL: "rediss://controller:secret@redis.internal:6380/7",
|
||||
PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: `/data/coolify/proxy/dynamic/${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}`,
|
||||
PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: `/data/coolify/proxy/dynamic/${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`,
|
||||
PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "https-0-joyyh5b89vpfp8ocri8xr96m@docker",
|
||||
PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: "portal-ingress.tool-ctrl.de",
|
||||
GIT_SHA: "abc123",
|
||||
};
|
||||
|
||||
describe("portal-domain controller config", () => {
|
||||
it("parses the least-privilege runtime contract without retaining a Redis URL", () => {
|
||||
const config = parsePortalDomainControllerConfig(baseEnv);
|
||||
expect(config.host).toBe("0.0.0.0");
|
||||
expect(config.port).toBe(8080);
|
||||
expect(config.queueConcurrency).toBe(2);
|
||||
expect(config.queueConnection).toMatchObject({ host: "redis.internal", port: 6380, db: 7, tls: {} });
|
||||
expect(config.entryPoint).toBe("https");
|
||||
expect(config.backendService).toBe("https-0-joyyh5b89vpfp8ocri8xr96m@docker");
|
||||
expect(config.reloadTriggerFile).toBe(`/data/coolify/proxy/dynamic/${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`);
|
||||
expect(config).not.toHaveProperty("redisUrl");
|
||||
});
|
||||
|
||||
it("accepts a configured safe entry point and rejects injectable routing values", () => {
|
||||
expect(parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "portal-https" }).entryPoint).toBe("portal-https");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT: "https\nhttp:" })).toThrow("entry-point");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "http://portal:8080" })).toThrow("Docker service");
|
||||
expect(() =>
|
||||
parsePortalDomainControllerConfig({
|
||||
...baseEnv,
|
||||
PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE: "portal@docker\n middlewares: [evil]",
|
||||
})
|
||||
).toThrow("Docker service");
|
||||
});
|
||||
|
||||
it("binds probes to the immutable image revision and refuses an unknown revision", () => {
|
||||
expect(parsePortalDomainControllerConfig({ ...baseEnv, APP_REVISION: "release-from-image" }).revision).toBe("release-from-image");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, GIT_SHA: undefined, APP_REVISION: "unknown" })).toThrow("revision");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, GIT_SHA: undefined })).toThrow("revision");
|
||||
});
|
||||
|
||||
it("accepts only Coolify's benign automatic runtime metadata", () => {
|
||||
expect(() =>
|
||||
parsePortalDomainControllerConfig({
|
||||
...baseEnv,
|
||||
COOLIFY_BRANCH: "main",
|
||||
COOLIFY_CONTAINER_NAME: "hivegrid-portal-domain-controller-123",
|
||||
COOLIFY_FQDN: "",
|
||||
COOLIFY_RESOURCE_UUID: "joyyh5b89vpfp8ocri8xr96m",
|
||||
COOLIFY_URL: "",
|
||||
})
|
||||
).not.toThrow();
|
||||
|
||||
for (const key of ["COOLIFY_API_URL", "COOLIFY_PROJECT_UUID", "COOLIFY_RESOURCE_UUID_EXTRA", "COOLIFY_URL_OVERRIDE"]) {
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, [key]: "must-not-be-here" })).toThrow(key);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
"DATABASE_URL",
|
||||
"DATABASE_READ_URL",
|
||||
"DIRECT_URL",
|
||||
"DOCKER_HOST",
|
||||
"COOLIFY_TOKEN",
|
||||
"CLOUDFLARE_API_TOKEN",
|
||||
"SSH_PRIVATE_KEY",
|
||||
"MIGRATOR_PASSWORD",
|
||||
"SEED_ON_BOOT",
|
||||
"AUTHENTIK_ADMIN_JIT_DATABASE_URL",
|
||||
"STORAGE_SECRET_ACCESS_KEY",
|
||||
"MAILJET_API_KEY",
|
||||
"MOLLIE_API_KEY",
|
||||
"LAGO_API_KEY",
|
||||
])("fails closed when %s is present", (key) => {
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, [key]: "must-not-be-here" })).toThrow(key);
|
||||
});
|
||||
|
||||
it("requires strong secrets, a dedicated absolute directory and a concrete CNAME", () => {
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET: "short" })).toThrow("32");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: "short" })).toThrow("32");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: "relative/path" })).toThrow("absolute");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR: path.resolve("/tmp/not-dedicated") })).toThrow(
|
||||
PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME
|
||||
);
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: "relative.reload" })).toThrow("absolute");
|
||||
expect(() =>
|
||||
parsePortalDomainControllerConfig({
|
||||
...baseEnv,
|
||||
PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: `/data/coolify/proxy/${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`,
|
||||
})
|
||||
).toThrow("directly beside");
|
||||
expect(() =>
|
||||
parsePortalDomainControllerConfig({
|
||||
...baseEnv,
|
||||
PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE: "/data/coolify/proxy/dynamic/traefik.yaml",
|
||||
})
|
||||
).toThrow(PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME);
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME: "*.tool-ctrl.de" })).toThrow();
|
||||
expect(() =>
|
||||
parsePortalDomainControllerConfig({
|
||||
...baseEnv,
|
||||
PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN: baseEnv.PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET,
|
||||
})
|
||||
).toThrow("must be different");
|
||||
});
|
||||
|
||||
it("rejects injectable bind hosts and invalid concurrency", () => {
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_HOST: "evil.example.com" })).toThrow("bind address");
|
||||
expect(() => parsePortalDomainControllerConfig({ ...baseEnv, PORTAL_DOMAIN_CONTROLLER_CONCURRENCY: "99" })).toThrow("concurrency");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,193 @@
|
||||
import path from "node:path";
|
||||
import type { ConnectionOptions } from "@hivegrid/queue";
|
||||
import { parseRedisUrl } from "@hivegrid/queue";
|
||||
import { normalizeDomainHostname } from "./contracts.js";
|
||||
|
||||
export const PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME = "hivegrid-portal-domains";
|
||||
export const PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME = ".hivegrid-portal-domains.reload";
|
||||
export const PORTAL_DOMAIN_DEFAULT_ENTRY_POINT = "https";
|
||||
export const PORTAL_DOMAIN_TLS_CERT_RESOLVER = "letsencrypt";
|
||||
|
||||
export type PortalDomainControllerConfig = Readonly<{
|
||||
host: string;
|
||||
port: number;
|
||||
revision: string;
|
||||
configDirectory: string;
|
||||
reloadTriggerFile: string;
|
||||
entryPoint: string;
|
||||
backendService: string;
|
||||
expectedCname: string;
|
||||
commandSecret: string;
|
||||
metricsToken: string;
|
||||
queueConnection: ConnectionOptions;
|
||||
queueConcurrency: number;
|
||||
}>;
|
||||
|
||||
const FORBIDDEN_EXACT_KEYS = new Set([
|
||||
"DATABASE_URL",
|
||||
"DIRECT_URL",
|
||||
"DOCKER_HOST",
|
||||
"PGHOST",
|
||||
"PGPORT",
|
||||
"PGDATABASE",
|
||||
"PGUSER",
|
||||
"PGPASSWORD",
|
||||
"CLOUDFLARE_API_TOKEN",
|
||||
"CLOUDFLARE_API_KEY",
|
||||
"COOLIFY_TOKEN",
|
||||
"SSH_AUTH_SOCK",
|
||||
]);
|
||||
const ALLOWED_COOLIFY_RUNTIME_KEYS = new Set([
|
||||
"COOLIFY_BRANCH",
|
||||
"COOLIFY_CONTAINER_NAME",
|
||||
"COOLIFY_FQDN",
|
||||
"COOLIFY_RESOURCE_UUID",
|
||||
"COOLIFY_URL",
|
||||
]);
|
||||
const FORBIDDEN_PREFIXES = [
|
||||
"DATABASE_",
|
||||
"MIGRATOR_",
|
||||
"SEED_",
|
||||
"CLOUDFLARE_",
|
||||
"SSH_",
|
||||
"AUTHENTIK_",
|
||||
"STORAGE_",
|
||||
"MAILJET_",
|
||||
"MOLLIE_",
|
||||
"LAGO_",
|
||||
"STRIPE_",
|
||||
"MISTRAL_",
|
||||
"GOOGLE_",
|
||||
"MCAPTCHA_",
|
||||
"CAP_",
|
||||
"SENTRY_AUTH_",
|
||||
"API_ACCESS_",
|
||||
"API_KEY_",
|
||||
"CRYPTO_",
|
||||
"JWT_",
|
||||
"SESSION_",
|
||||
"AUDIT_",
|
||||
] as const;
|
||||
|
||||
export function parsePortalDomainControllerConfig(env: NodeJS.ProcessEnv = process.env): PortalDomainControllerConfig {
|
||||
rejectForbiddenCredentials(env);
|
||||
const commandSecret = requireSecret(env.PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET, "PORTAL_DOMAIN_CONTROLLER_COMMAND_SECRET");
|
||||
const metricsToken = requireSecret(env.PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN, "PORTAL_DOMAIN_CONTROLLER_METRICS_TOKEN");
|
||||
if (commandSecret === metricsToken) throw new Error("controller command and metrics secrets must be different");
|
||||
const redisUrl = requireValue(env.PORTAL_DOMAIN_CONTROLLER_REDIS_URL, "PORTAL_DOMAIN_CONTROLLER_REDIS_URL");
|
||||
const configDirectory = parseConfigDirectory(env.PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR);
|
||||
const reloadTriggerFile = parseReloadTriggerFile(env.PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE, configDirectory);
|
||||
const entryPoint = normalizePortalDomainEntryPoint(env.PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT ?? PORTAL_DOMAIN_DEFAULT_ENTRY_POINT);
|
||||
const backendService = normalizePortalDomainBackendService(
|
||||
requireValue(env.PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE, "PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE")
|
||||
);
|
||||
const expectedCname = normalizeDomainHostname(requireValue(env.PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME, "PORTAL_DOMAIN_CONTROLLER_EXPECTED_CNAME"));
|
||||
const host = parseHost(env.PORTAL_DOMAIN_CONTROLLER_HOST);
|
||||
const port = parseInteger(env.PORTAL_DOMAIN_CONTROLLER_PORT ?? env.PORT, 8080, 1, 65_535, "controller port");
|
||||
const queueConcurrency = parseInteger(env.PORTAL_DOMAIN_CONTROLLER_CONCURRENCY, 2, 1, 16, "controller concurrency");
|
||||
return Object.freeze({
|
||||
host,
|
||||
port,
|
||||
revision: normalizeRevision(env.APP_REVISION ?? env.GIT_SHA ?? env.SENTRY_RELEASE),
|
||||
configDirectory,
|
||||
reloadTriggerFile,
|
||||
entryPoint,
|
||||
backendService,
|
||||
expectedCname,
|
||||
commandSecret,
|
||||
metricsToken,
|
||||
queueConnection: parseRedisUrl(redisUrl),
|
||||
queueConcurrency,
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizePortalDomainEntryPoint(value: string): string {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,62}$/u.test(value)) {
|
||||
throw new Error("PORTAL_DOMAIN_CONTROLLER_ENTRYPOINT must be a safe Traefik entry-point name");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizePortalDomainBackendService(value: string): string {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,126}@docker$/u.test(value)) {
|
||||
throw new Error("PORTAL_DOMAIN_CONTROLLER_BACKEND_SERVICE must be a provider-qualified Docker service name");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function rejectForbiddenCredentials(env: NodeJS.ProcessEnv): void {
|
||||
const forbidden = Object.keys(env)
|
||||
.filter((key) => env[key]?.trim())
|
||||
.filter(
|
||||
(key) =>
|
||||
FORBIDDEN_EXACT_KEYS.has(key) ||
|
||||
(key.startsWith("COOLIFY_") && !ALLOWED_COOLIFY_RUNTIME_KEYS.has(key)) ||
|
||||
FORBIDDEN_PREFIXES.some((prefix) => key.startsWith(prefix))
|
||||
)
|
||||
.sort();
|
||||
if (forbidden.length > 0) {
|
||||
throw new Error(`portal domain controller refuses forbidden credentials: ${forbidden.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
function parseConfigDirectory(raw: string | undefined): string {
|
||||
const value = requireValue(raw, "PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR");
|
||||
if (!path.isAbsolute(value)) throw new Error("PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR must be absolute");
|
||||
const normalized = path.resolve(value);
|
||||
if (path.basename(normalized) !== PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME) {
|
||||
throw new Error(`PORTAL_DOMAIN_CONTROLLER_CONFIG_DIR must end with ${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseReloadTriggerFile(raw: string | undefined, configDirectory: string): string {
|
||||
const value = requireValue(raw, "PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE");
|
||||
if (!path.isAbsolute(value)) throw new Error("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must be absolute");
|
||||
const normalized = path.resolve(value);
|
||||
if (path.basename(normalized) !== PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME) {
|
||||
throw new Error(`PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must end with ${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`);
|
||||
}
|
||||
if (path.dirname(normalized) !== path.dirname(configDirectory)) {
|
||||
throw new Error("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must be directly beside the dedicated config directory");
|
||||
}
|
||||
if (/\.ya?ml$/iu.test(normalized)) {
|
||||
throw new Error("PORTAL_DOMAIN_CONTROLLER_RELOAD_TRIGGER_FILE must not be a Traefik configuration file");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseHost(raw: string | undefined): string {
|
||||
const host = raw?.trim() || "0.0.0.0";
|
||||
if (host !== "0.0.0.0" && host !== "127.0.0.1" && host !== "::1") {
|
||||
throw new Error("PORTAL_DOMAIN_CONTROLLER_HOST must be a loopback or all-interface bind address");
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
function parseInteger(raw: string | undefined, fallback: number, min: number, max: number, name: string): number {
|
||||
if (!raw?.trim()) return fallback;
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < min || value > max) throw new Error(`${name} must be an integer from ${min} to ${max}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireValue(raw: string | undefined, name: string): string {
|
||||
const value = raw?.trim();
|
||||
if (!value) throw new Error(`${name} is required`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireSecret(raw: string | undefined, name: string): string {
|
||||
const value = requireValue(raw, name);
|
||||
const bytes = Buffer.byteLength(value, "utf8");
|
||||
if (bytes < 32 || bytes > 512) throw new Error(`${name} must contain between 32 and 512 bytes`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeRevision(value: string | undefined): string {
|
||||
const revision = value?.trim();
|
||||
if (!revision || revision === "unknown" || !/^[A-Za-z0-9._-]{1,128}$/u.test(revision)) {
|
||||
throw new Error("controller release revision is required");
|
||||
}
|
||||
return revision;
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
canonicalJson,
|
||||
createSignedDomainCommand,
|
||||
createSignedDomainResult,
|
||||
normalizeDomainHostname,
|
||||
sha256Hex,
|
||||
verifySignedDomainCommand,
|
||||
verifySignedDomainResult,
|
||||
} from "./contracts.js";
|
||||
|
||||
const SECRET = "test-command-secret-that-is-at-least-thirty-two-bytes";
|
||||
const command = {
|
||||
claimId: "claim_12345678",
|
||||
tenantId: "tenant_12345678",
|
||||
generation: 3,
|
||||
hostname: "portal.example.com",
|
||||
action: "provision" as const,
|
||||
};
|
||||
|
||||
describe("portal-domain signed contracts", () => {
|
||||
it("produces a deterministic hash and domain-separated signature", () => {
|
||||
const first = createSignedDomainCommand(command, SECRET);
|
||||
const second = createSignedDomainCommand({ ...command }, SECRET);
|
||||
expect(first).toEqual(second);
|
||||
expect(first.contentHash).toBe(sha256Hex(canonicalJson(first.payload)));
|
||||
expect(verifySignedDomainCommand(first, SECRET)).toEqual(first.payload);
|
||||
|
||||
const result = createSignedDomainResult({
|
||||
...command,
|
||||
status: "active",
|
||||
attempt: 1,
|
||||
completedAt: "2026-07-16T10:00:00.000Z",
|
||||
configHash: "a".repeat(64),
|
||||
certificateExpiresAt: "2026-10-01T00:00:00.000Z",
|
||||
}, SECRET);
|
||||
expect(result.signature).not.toBe(first.signature);
|
||||
expect(verifySignedDomainResult(result, SECRET).status).toBe("active");
|
||||
});
|
||||
|
||||
it("rejects payload, hash, signature, algorithm and unknown-field tampering", () => {
|
||||
const signed = createSignedDomainCommand(command, SECRET);
|
||||
expect(() => verifySignedDomainCommand({ ...signed, payload: { ...signed.payload, generation: 4 } }, SECRET)).toThrow("content hash");
|
||||
expect(() => verifySignedDomainCommand({ ...signed, contentHash: "0".repeat(64) }, SECRET)).toThrow("content hash");
|
||||
expect(() => verifySignedDomainCommand({ ...signed, signature: "0".repeat(64) }, SECRET)).toThrow("signature");
|
||||
expect(() => verifySignedDomainCommand({ ...signed, algorithm: "none" }, SECRET)).toThrow("algorithm");
|
||||
expect(() => verifySignedDomainCommand({ ...signed, injected: true }, SECRET)).toThrow("unknown or missing");
|
||||
expect(() => verifySignedDomainCommand({ ...signed, payload: { ...signed.payload, injected: true } }, SECRET)).toThrow("unknown or missing");
|
||||
expect(() => verifySignedDomainCommand(signed, `${SECRET}!`)).toThrow("signature");
|
||||
});
|
||||
|
||||
it("accepts only canonical, concrete fully qualified hostnames and bounded identifiers", () => {
|
||||
expect(normalizeDomainHostname("Portal.Example.com.")).toBe("portal.example.com");
|
||||
for (const hostname of ["*.example.com", "127.0.0.1", "localhost", "-bad.example.com", "bad_.example.com", "[::1]"]) {
|
||||
expect(() => normalizeDomainHostname(hostname)).toThrow();
|
||||
}
|
||||
expect(() => createSignedDomainCommand({ ...command, hostname: "Portal.Example.com" }, SECRET)).toThrow("canonical");
|
||||
expect(() => createSignedDomainCommand({ ...command, claimId: "../x" }, SECRET)).toThrow("claim id");
|
||||
expect(() => createSignedDomainCommand({ ...command, generation: 0 }, SECRET)).toThrow("generation");
|
||||
});
|
||||
|
||||
it("rejects undersized signing secrets and non-canonical result timestamps", () => {
|
||||
expect(() => createSignedDomainCommand(command, "short")).toThrow("32");
|
||||
expect(() => createSignedDomainResult({
|
||||
...command,
|
||||
status: "active",
|
||||
attempt: 1,
|
||||
completedAt: "2026-07-16",
|
||||
}, SECRET)).toThrow("timestamp");
|
||||
expect(() => createSignedDomainResult({
|
||||
...command,
|
||||
status: "active",
|
||||
attempt: 1,
|
||||
completedAt: "2026-07-16T10:00:00.000Z",
|
||||
configHash: "not-a-sha256",
|
||||
certificateExpiresAt: "2026-10-01T00:00:00.000Z",
|
||||
}, SECRET)).toThrow("configHash");
|
||||
expect(() => createSignedDomainResult({
|
||||
...command,
|
||||
status: "active",
|
||||
attempt: 1,
|
||||
completedAt: "2026-07-16T10:00:00.000Z",
|
||||
configHash: "a".repeat(64),
|
||||
certificateExpiresAt: "2026-07-16T09:00:00.000Z",
|
||||
}, SECRET)).toThrow("expire after observation");
|
||||
});
|
||||
});
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
||||
import { isIP } from "node:net";
|
||||
|
||||
export const DOMAIN_COMMAND_QUEUE = "hivegrid.portal-domain.commands";
|
||||
export const DOMAIN_RESULT_QUEUE = "hivegrid.portal-domain.results";
|
||||
export const DOMAIN_COMMAND_QUEUE_ATTEMPTS = 4;
|
||||
export const DOMAIN_CONTRACT_VERSION = 1 as const;
|
||||
export const DOMAIN_COMMAND_ACTIONS = ["provision", "revalidate", "deprovision"] as const;
|
||||
export const DOMAIN_RESULT_STATUSES = ["active", "healthy", "deprovisioned", "retrying", "failed", "stale"] as const;
|
||||
|
||||
export type DomainCommandAction = (typeof DOMAIN_COMMAND_ACTIONS)[number];
|
||||
export type DomainResultStatus = (typeof DOMAIN_RESULT_STATUSES)[number];
|
||||
|
||||
export type DomainCommandPayload = {
|
||||
version: typeof DOMAIN_CONTRACT_VERSION;
|
||||
claimId: string;
|
||||
tenantId: string;
|
||||
generation: number;
|
||||
hostname: string;
|
||||
action: DomainCommandAction;
|
||||
};
|
||||
|
||||
export type DomainResultPayload = DomainCommandPayload & {
|
||||
status: DomainResultStatus;
|
||||
attempt: number;
|
||||
completedAt: string;
|
||||
retryAfterMs?: number;
|
||||
reasonCode?: string;
|
||||
configHash?: string;
|
||||
certificateExpiresAt?: string;
|
||||
};
|
||||
|
||||
export type SignedDomainEnvelope<Payload> = {
|
||||
algorithm: "hmac-sha256";
|
||||
contentHash: string;
|
||||
payload: Payload;
|
||||
signature: string;
|
||||
};
|
||||
|
||||
const COMMAND_KEYS = ["action", "claimId", "generation", "hostname", "tenantId", "version"] as const;
|
||||
const RESULT_REQUIRED_KEYS = [...COMMAND_KEYS, "attempt", "completedAt", "status"] as const;
|
||||
const RESULT_OPTIONAL_KEYS = ["certificateExpiresAt", "configHash", "reasonCode", "retryAfterMs"] as const;
|
||||
const ENVELOPE_KEYS = ["algorithm", "contentHash", "payload", "signature"] as const;
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{7,127}$/u;
|
||||
const HEX_SHA256_PATTERN = /^[a-f0-9]{64}$/u;
|
||||
const HOST_LABEL_PATTERN = /^(?!-)[a-z0-9-]{1,63}(?<!-)$/u;
|
||||
const MAX_HOSTNAME_BYTES = 253;
|
||||
|
||||
export function createSignedDomainCommand(
|
||||
input: Omit<DomainCommandPayload, "version"> | DomainCommandPayload,
|
||||
secret: string
|
||||
): SignedDomainEnvelope<DomainCommandPayload> {
|
||||
const payload = parseDomainCommandPayload({ ...input, version: DOMAIN_CONTRACT_VERSION });
|
||||
return signEnvelope(payload, secret, "command");
|
||||
}
|
||||
|
||||
export function createSignedDomainResult(
|
||||
input: Omit<DomainResultPayload, "version"> | DomainResultPayload,
|
||||
secret: string
|
||||
): SignedDomainEnvelope<DomainResultPayload> {
|
||||
const payload = parseDomainResultPayload({ ...input, version: DOMAIN_CONTRACT_VERSION });
|
||||
return signEnvelope(payload, secret, "result");
|
||||
}
|
||||
|
||||
export function verifySignedDomainCommand(value: unknown, secret: string): DomainCommandPayload {
|
||||
const envelope = parseEnvelope(value);
|
||||
const payload = parseDomainCommandPayload(envelope.payload);
|
||||
verifyEnvelope(envelope, payload, secret, "command");
|
||||
return payload;
|
||||
}
|
||||
|
||||
export function verifySignedDomainResult(value: unknown, secret: string): DomainResultPayload {
|
||||
const envelope = parseEnvelope(value);
|
||||
const payload = parseDomainResultPayload(envelope.payload);
|
||||
verifyEnvelope(envelope, payload, secret, "result");
|
||||
return payload;
|
||||
}
|
||||
|
||||
export function normalizeDomainHostname(value: string): string {
|
||||
const hostname = value.trim().toLowerCase().replace(/\.$/u, "");
|
||||
if (!hostname || hostname.includes("*") || hostname.includes(":") || isIP(hostname) !== 0) {
|
||||
throw new Error("hostname must be a concrete DNS name");
|
||||
}
|
||||
if (Buffer.byteLength(hostname, "ascii") > MAX_HOSTNAME_BYTES || !hostname.includes(".")) {
|
||||
throw new Error("hostname must be a valid fully qualified DNS name");
|
||||
}
|
||||
const labels = hostname.split(".");
|
||||
if (labels.some((label) => !HOST_LABEL_PATTERN.test(label))) {
|
||||
throw new Error("hostname contains an invalid DNS label");
|
||||
}
|
||||
return hostname;
|
||||
}
|
||||
|
||||
export function canonicalJson(value: unknown): string {
|
||||
if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value);
|
||||
if (typeof value === "number") {
|
||||
if (!Number.isFinite(value)) throw new Error("non-finite numbers are not supported");
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
|
||||
if (isRecord(value)) {
|
||||
return `{${Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
throw new Error("unsupported canonical JSON value");
|
||||
}
|
||||
|
||||
export function sha256Hex(value: string | Buffer): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function signEnvelope<Payload extends DomainCommandPayload | DomainResultPayload>(
|
||||
payload: Payload,
|
||||
secret: string,
|
||||
kind: "command" | "result"
|
||||
): SignedDomainEnvelope<Payload> {
|
||||
assertSigningSecret(secret);
|
||||
const contentHash = sha256Hex(canonicalJson(payload));
|
||||
return {
|
||||
algorithm: "hmac-sha256",
|
||||
contentHash,
|
||||
payload,
|
||||
signature: hmac(secret, kind, contentHash),
|
||||
};
|
||||
}
|
||||
|
||||
function verifyEnvelope<Payload extends DomainCommandPayload | DomainResultPayload>(
|
||||
envelope: SignedDomainEnvelope<unknown>,
|
||||
payload: Payload,
|
||||
secret: string,
|
||||
kind: "command" | "result"
|
||||
): void {
|
||||
assertSigningSecret(secret);
|
||||
const actualContentHash = sha256Hex(canonicalJson(payload));
|
||||
if (!safeHexEqual(envelope.contentHash, actualContentHash)) throw new Error("domain envelope content hash mismatch");
|
||||
const expectedSignature = hmac(secret, kind, actualContentHash);
|
||||
if (!safeHexEqual(envelope.signature, expectedSignature)) throw new Error("domain envelope signature mismatch");
|
||||
}
|
||||
|
||||
function hmac(secret: string, kind: "command" | "result", contentHash: string): string {
|
||||
return createHmac("sha256", secret).update(`hivegrid.portal-domain.${kind}.v1:${contentHash}`, "utf8").digest("hex");
|
||||
}
|
||||
|
||||
function safeHexEqual(actual: string, expected: string): boolean {
|
||||
if (!HEX_SHA256_PATTERN.test(actual) || !HEX_SHA256_PATTERN.test(expected)) return false;
|
||||
return timingSafeEqual(Buffer.from(actual, "hex"), Buffer.from(expected, "hex"));
|
||||
}
|
||||
|
||||
function parseEnvelope(value: unknown): SignedDomainEnvelope<unknown> {
|
||||
assertExactKeys(value, ENVELOPE_KEYS, []);
|
||||
if (value.algorithm !== "hmac-sha256") throw new Error("unsupported domain envelope algorithm");
|
||||
if (typeof value.contentHash !== "string" || !HEX_SHA256_PATTERN.test(value.contentHash)) throw new Error("invalid domain envelope content hash");
|
||||
if (typeof value.signature !== "string" || !HEX_SHA256_PATTERN.test(value.signature)) throw new Error("invalid domain envelope signature");
|
||||
return value as SignedDomainEnvelope<unknown>;
|
||||
}
|
||||
|
||||
function parseDomainCommandPayload(value: unknown): DomainCommandPayload {
|
||||
assertExactKeys(value, COMMAND_KEYS, []);
|
||||
return parseCommandFields(value);
|
||||
}
|
||||
|
||||
function parseDomainResultPayload(value: unknown): DomainResultPayload {
|
||||
assertExactKeys(value, RESULT_REQUIRED_KEYS, RESULT_OPTIONAL_KEYS);
|
||||
const command = parseCommandFields(value);
|
||||
if (!DOMAIN_RESULT_STATUSES.includes(value.status as DomainResultStatus)) throw new Error("invalid domain result status");
|
||||
if (!Number.isInteger(value.attempt) || (value.attempt as number) < 1 || (value.attempt as number) > 100) throw new Error("invalid domain result attempt");
|
||||
if (typeof value.completedAt !== "string" || !isCanonicalTimestamp(value.completedAt)) throw new Error("invalid domain result timestamp");
|
||||
if (value.retryAfterMs !== undefined && (!Number.isInteger(value.retryAfterMs) || (value.retryAfterMs as number) < 1)) {
|
||||
throw new Error("invalid domain result retry delay");
|
||||
}
|
||||
for (const key of ["reasonCode", "configHash", "certificateExpiresAt"] as const) {
|
||||
if (value[key] !== undefined && (typeof value[key] !== "string" || value[key].length < 1 || value[key].length > 128)) {
|
||||
throw new Error(`invalid domain result ${key}`);
|
||||
}
|
||||
}
|
||||
if (value.reasonCode !== undefined && !/^[a-z0-9_]{1,64}$/u.test(value.reasonCode as string)) {
|
||||
throw new Error("invalid domain result reasonCode");
|
||||
}
|
||||
if (value.configHash !== undefined && !HEX_SHA256_PATTERN.test(value.configHash as string)) {
|
||||
throw new Error("invalid domain result configHash");
|
||||
}
|
||||
if (value.certificateExpiresAt !== undefined && !isCanonicalTimestamp(value.certificateExpiresAt as string)) {
|
||||
throw new Error("invalid domain result certificateExpiresAt");
|
||||
}
|
||||
if (value.status === "active" && value.configHash === undefined) {
|
||||
throw new Error("active domain result requires configHash");
|
||||
}
|
||||
if ((value.status === "active" || value.status === "healthy") && value.certificateExpiresAt === undefined) {
|
||||
throw new Error("healthy domain result requires certificateExpiresAt");
|
||||
}
|
||||
if (value.certificateExpiresAt !== undefined && Date.parse(value.certificateExpiresAt as string) <= Date.parse(value.completedAt as string)) {
|
||||
throw new Error("domain result certificate must expire after observation");
|
||||
}
|
||||
return {
|
||||
...command,
|
||||
status: value.status as DomainResultStatus,
|
||||
attempt: value.attempt as number,
|
||||
completedAt: value.completedAt,
|
||||
...(value.retryAfterMs === undefined ? {} : { retryAfterMs: value.retryAfterMs as number }),
|
||||
...(value.reasonCode === undefined ? {} : { reasonCode: value.reasonCode as string }),
|
||||
...(value.configHash === undefined ? {} : { configHash: value.configHash as string }),
|
||||
...(value.certificateExpiresAt === undefined ? {} : { certificateExpiresAt: value.certificateExpiresAt as string }),
|
||||
};
|
||||
}
|
||||
|
||||
function parseCommandFields(value: Record<string, unknown>): DomainCommandPayload {
|
||||
if (value.version !== DOMAIN_CONTRACT_VERSION) throw new Error("unsupported domain contract version");
|
||||
if (typeof value.claimId !== "string" || !IDENTIFIER_PATTERN.test(value.claimId)) throw new Error("invalid domain claim id");
|
||||
if (typeof value.tenantId !== "string" || !IDENTIFIER_PATTERN.test(value.tenantId)) throw new Error("invalid domain tenant id");
|
||||
if (!Number.isSafeInteger(value.generation) || (value.generation as number) < 1) throw new Error("invalid domain generation");
|
||||
if (typeof value.hostname !== "string") throw new Error("invalid domain hostname");
|
||||
const hostname = normalizeDomainHostname(value.hostname);
|
||||
if (hostname !== value.hostname) throw new Error("domain hostname must already be canonical");
|
||||
if (!DOMAIN_COMMAND_ACTIONS.includes(value.action as DomainCommandAction)) throw new Error("invalid domain command action");
|
||||
return {
|
||||
version: DOMAIN_CONTRACT_VERSION,
|
||||
claimId: value.claimId,
|
||||
tenantId: value.tenantId,
|
||||
generation: value.generation as number,
|
||||
hostname,
|
||||
action: value.action as DomainCommandAction,
|
||||
};
|
||||
}
|
||||
|
||||
function assertExactKeys(value: unknown, required: readonly string[], optional: readonly string[]): asserts value is Record<string, unknown> {
|
||||
if (!isRecord(value)) throw new Error("domain contract value must be an object");
|
||||
const keys = Object.keys(value).sort();
|
||||
const allowed = new Set([...required, ...optional]);
|
||||
const unknown = keys.filter((key) => !allowed.has(key));
|
||||
const missing = required.filter((key) => !Object.hasOwn(value, key));
|
||||
if (unknown.length > 0 || missing.length > 0) throw new Error("domain contract contains unknown or missing fields");
|
||||
}
|
||||
|
||||
function assertSigningSecret(secret: string): void {
|
||||
const bytes = Buffer.byteLength(secret, "utf8");
|
||||
if (bytes < 32 || bytes > 512) throw new Error("domain command secret must contain between 32 and 512 bytes");
|
||||
}
|
||||
|
||||
function isCanonicalTimestamp(value: string): boolean {
|
||||
const date = new Date(value);
|
||||
return Number.isFinite(date.getTime()) && date.toISOString() === value;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { DomainCommandPayload } from "./contracts.js";
|
||||
import { createPortalDomainController, DomainControllerFailure, DOMAIN_MAX_ATTEMPTS, retryDelayForAttempt } from "./controller.js";
|
||||
import { DomainProbeError, type DomainProbes } from "./probes.js";
|
||||
import type { TraefikConfigWriter } from "./traefik.js";
|
||||
|
||||
const command: DomainCommandPayload = {
|
||||
version: 1,
|
||||
claimId: "claim_12345678",
|
||||
tenantId: "tenant_12345678",
|
||||
generation: 2,
|
||||
hostname: "portal.example.com",
|
||||
action: "provision",
|
||||
};
|
||||
const CERTIFICATE_EXPIRES_AT = "2026-10-01T00:00:00.000Z";
|
||||
|
||||
describe("portal-domain lifecycle controller", () => {
|
||||
it("does not acknowledge provision until DNS, atomic config and HTTPS succeed", async () => {
|
||||
const calls: string[] = [];
|
||||
const writer = {
|
||||
applyProvision: vi.fn(async () => {
|
||||
calls.push("write");
|
||||
return { disposition: "applied", configHash: "hash" };
|
||||
}),
|
||||
readAppliedState: vi.fn(async () => ({ generation: command.generation, action: command.action, hostname: command.hostname, tenantId: command.tenantId })),
|
||||
} as unknown as TraefikConfigWriter;
|
||||
const probes: DomainProbes = {
|
||||
verifyDns: vi.fn(async () => {
|
||||
calls.push("dns");
|
||||
}),
|
||||
verifyHttps: vi.fn(async () => {
|
||||
calls.push("https");
|
||||
return { certificateExpiresAt: CERTIFICATE_EXPIRES_AT };
|
||||
}),
|
||||
};
|
||||
const controller = createPortalDomainController({ writer, probes, now: () => new Date("2026-07-16T10:00:00.000Z") });
|
||||
await expect(controller.handle(command, 1)).resolves.toMatchObject({
|
||||
status: "active",
|
||||
configHash: "hash",
|
||||
certificateExpiresAt: CERTIFICATE_EXPIRES_AT,
|
||||
attempt: 1,
|
||||
});
|
||||
expect(calls).toEqual(["dns", "write", "https"]);
|
||||
});
|
||||
|
||||
it("returns stale without attempting HTTPS and deprovisions without DNS", async () => {
|
||||
const writer = {
|
||||
applyProvision: vi.fn().mockResolvedValue({ disposition: "stale" }),
|
||||
applyDeprovision: vi.fn().mockResolvedValue({ disposition: "applied" }),
|
||||
} as unknown as TraefikConfigWriter;
|
||||
const probes: DomainProbes = { verifyDns: vi.fn(), verifyHttps: vi.fn() };
|
||||
const controller = createPortalDomainController({ writer, probes });
|
||||
await expect(controller.handle(command, 1)).resolves.toMatchObject({ status: "stale" });
|
||||
expect(probes.verifyHttps).not.toHaveBeenCalled();
|
||||
await expect(controller.handle({ ...command, action: "deprovision" }, 1)).resolves.toMatchObject({ status: "deprovisioned" });
|
||||
expect(probes.verifyDns).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("repairs routing before HTTPS and rechecks ownership after the probe", async () => {
|
||||
const calls: string[] = [];
|
||||
let routingRepaired = false;
|
||||
const writer = {
|
||||
readAppliedState: vi.fn(async () => {
|
||||
calls.push("state");
|
||||
return { generation: command.generation, action: "provision", hostname: command.hostname };
|
||||
}),
|
||||
recordRevalidation: vi.fn(async () => {
|
||||
calls.push("repair");
|
||||
routingRepaired = true;
|
||||
return { disposition: "unchanged" };
|
||||
}),
|
||||
} as unknown as TraefikConfigWriter;
|
||||
const probes: DomainProbes = {
|
||||
verifyDns: vi.fn(async () => {
|
||||
calls.push("dns");
|
||||
}),
|
||||
verifyHttps: vi.fn(async () => {
|
||||
calls.push("https");
|
||||
if (!routingRepaired) throw new DomainProbeError("https_health_failed", "stale backend");
|
||||
return { certificateExpiresAt: CERTIFICATE_EXPIRES_AT };
|
||||
}),
|
||||
};
|
||||
const controller = createPortalDomainController({ writer, probes });
|
||||
await expect(controller.handle({ ...command, action: "revalidate" }, 2)).resolves.toMatchObject({
|
||||
status: "healthy",
|
||||
certificateExpiresAt: CERTIFICATE_EXPIRES_AT,
|
||||
});
|
||||
expect(calls).toEqual(["dns", "repair", "https", "state"]);
|
||||
});
|
||||
|
||||
it("returns stale before HTTPS and after a concurrent deprovision during the probe", async () => {
|
||||
const staleWriter = {
|
||||
recordRevalidation: vi.fn().mockResolvedValue({ disposition: "stale" }),
|
||||
} as unknown as TraefikConfigWriter;
|
||||
const staleProbes: DomainProbes = { verifyDns: vi.fn(), verifyHttps: vi.fn() };
|
||||
const staleController = createPortalDomainController({ writer: staleWriter, probes: staleProbes });
|
||||
await expect(staleController.handle({ ...command, action: "revalidate" }, 1)).resolves.toMatchObject({ status: "stale" });
|
||||
expect(staleProbes.verifyHttps).not.toHaveBeenCalled();
|
||||
|
||||
const racedWriter = {
|
||||
recordRevalidation: vi.fn().mockResolvedValue({ disposition: "unchanged" }),
|
||||
readAppliedState: vi.fn().mockResolvedValue({
|
||||
generation: command.generation + 1,
|
||||
action: "deprovision",
|
||||
hostname: command.hostname,
|
||||
tenantId: command.tenantId,
|
||||
}),
|
||||
} as unknown as TraefikConfigWriter;
|
||||
const racedProbes: DomainProbes = {
|
||||
verifyDns: vi.fn(),
|
||||
verifyHttps: vi.fn().mockResolvedValue({ certificateExpiresAt: CERTIFICATE_EXPIRES_AT }),
|
||||
};
|
||||
const racedController = createPortalDomainController({ writer: racedWriter, probes: racedProbes });
|
||||
await expect(racedController.handle({ ...command, action: "revalidate" }, 1)).resolves.toMatchObject({ status: "stale" });
|
||||
});
|
||||
|
||||
it("does not emit an active acknowledgement if a newer generation wins during HTTPS probing", async () => {
|
||||
const writer = {
|
||||
applyProvision: vi.fn().mockResolvedValue({ disposition: "applied", configHash: "hash" }),
|
||||
readAppliedState: vi.fn().mockResolvedValue({ generation: 3, action: "deprovision", hostname: command.hostname, tenantId: command.tenantId }),
|
||||
} as unknown as TraefikConfigWriter;
|
||||
const probes: DomainProbes = { verifyDns: vi.fn(), verifyHttps: vi.fn() };
|
||||
const controller = createPortalDomainController({ writer, probes });
|
||||
await expect(controller.handle(command, 1)).resolves.toMatchObject({ status: "stale" });
|
||||
});
|
||||
|
||||
it("classifies probe failures as retryable without leaking the underlying error", async () => {
|
||||
const writer = {} as TraefikConfigWriter;
|
||||
const probes: DomainProbes = {
|
||||
verifyDns: vi.fn().mockRejectedValue(new DomainProbeError("dns_lookup_failed", "DNS provider detail")),
|
||||
verifyHttps: vi.fn(),
|
||||
};
|
||||
const controller = createPortalDomainController({ writer, probes });
|
||||
await expect(controller.handle(command, 1)).rejects.toEqual(
|
||||
expect.objectContaining({
|
||||
reasonCode: "dns_lookup_failed",
|
||||
retryable: true,
|
||||
})
|
||||
);
|
||||
await controller.handle(command, 1).catch((error: DomainControllerFailure) => {
|
||||
expect(error).toBeInstanceOf(DomainControllerFailure);
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the required 15m, 1h, 6h retry schedule", () => {
|
||||
expect(DOMAIN_MAX_ATTEMPTS).toBe(4);
|
||||
expect([1, 2, 3, 4].map(retryDelayForAttempt)).toEqual([900_000, 3_600_000, 21_600_000, 21_600_000]);
|
||||
});
|
||||
});
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
import type { DomainCommandPayload, DomainResultPayload } from "./contracts.js";
|
||||
import type { DomainProbes } from "./probes.js";
|
||||
import { DomainProbeError } from "./probes.js";
|
||||
import { GenerationConflictError, TraefikConfigWriter, UnsafeConfigPathError } from "./traefik.js";
|
||||
|
||||
export const DOMAIN_RETRY_DELAYS_MS = [15 * 60_000, 60 * 60_000, 6 * 60 * 60_000] as const;
|
||||
export const DOMAIN_MAX_ATTEMPTS = DOMAIN_RETRY_DELAYS_MS.length + 1;
|
||||
|
||||
export class DomainControllerFailure extends Error {
|
||||
constructor(
|
||||
readonly reasonCode: string,
|
||||
readonly retryable: boolean,
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export type PortalDomainController = {
|
||||
handle(command: DomainCommandPayload, attempt: number): Promise<DomainResultPayload>;
|
||||
};
|
||||
|
||||
export function createPortalDomainController(options: { writer: TraefikConfigWriter; probes: DomainProbes; now?: () => Date }): PortalDomainController {
|
||||
const now = options.now ?? (() => new Date());
|
||||
return {
|
||||
async handle(command, attempt) {
|
||||
try {
|
||||
if (command.action === "deprovision") {
|
||||
const mutation = await options.writer.applyDeprovision(command);
|
||||
return result(command, attempt, now(), mutation.disposition === "stale" ? "stale" : "deprovisioned");
|
||||
}
|
||||
|
||||
await options.probes.verifyDns(command.hostname);
|
||||
if (command.action === "provision") {
|
||||
const mutation = await options.writer.applyProvision(command);
|
||||
if (mutation.disposition === "stale") return result(command, attempt, now(), "stale");
|
||||
const health = await options.probes.verifyHttps(command.hostname);
|
||||
const current = await options.writer.readAppliedState(command.claimId);
|
||||
if (!current || current.generation !== command.generation || current.action !== command.action) {
|
||||
return result(command, attempt, now(), "stale");
|
||||
}
|
||||
return result(command, attempt, now(), "active", mutation.configHash, health.certificateExpiresAt);
|
||||
}
|
||||
|
||||
const mutation = await options.writer.recordRevalidation(command);
|
||||
if (mutation.disposition === "stale") return result(command, attempt, now(), "stale");
|
||||
const health = await options.probes.verifyHttps(command.hostname);
|
||||
const current = await options.writer.readAppliedState(command.claimId);
|
||||
if (!current || current.generation !== command.generation || (current.action !== "provision" && current.action !== "revalidate")) {
|
||||
return result(command, attempt, now(), "stale");
|
||||
}
|
||||
return result(command, attempt, now(), "healthy", undefined, health.certificateExpiresAt);
|
||||
} catch (error) {
|
||||
throw classifyControllerError(error);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function retryDelayForAttempt(attemptsMade: number): number {
|
||||
const index = Math.max(0, Math.min(attemptsMade - 1, DOMAIN_RETRY_DELAYS_MS.length - 1));
|
||||
return DOMAIN_RETRY_DELAYS_MS[index] ?? DOMAIN_RETRY_DELAYS_MS[DOMAIN_RETRY_DELAYS_MS.length - 1];
|
||||
}
|
||||
|
||||
export function classifyControllerError(error: unknown): DomainControllerFailure {
|
||||
if (error instanceof DomainControllerFailure) return error;
|
||||
if (error instanceof DomainProbeError) return new DomainControllerFailure(error.reasonCode, true, error.message);
|
||||
if (error instanceof UnsafeConfigPathError || error instanceof GenerationConflictError) {
|
||||
return new DomainControllerFailure(error.reasonCode, false, error.message);
|
||||
}
|
||||
if (isErrno(error, "EACCES") || isErrno(error, "EPERM") || isErrno(error, "EROFS")) {
|
||||
return new DomainControllerFailure("config_permission_denied", false, "The dedicated Traefik config directory is not writable");
|
||||
}
|
||||
return new DomainControllerFailure("controller_internal_error", true, "The portal domain controller encountered a transient internal error");
|
||||
}
|
||||
|
||||
function result(
|
||||
command: DomainCommandPayload,
|
||||
attempt: number,
|
||||
completedAt: Date,
|
||||
status: DomainResultPayload["status"],
|
||||
configHash?: string,
|
||||
certificateExpiresAt?: string
|
||||
): DomainResultPayload {
|
||||
return {
|
||||
...command,
|
||||
status,
|
||||
attempt,
|
||||
completedAt: completedAt.toISOString(),
|
||||
...(configHash ? { configHash } : {}),
|
||||
...(certificateExpiresAt ? { certificateExpiresAt } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function isErrno(error: unknown, code: string): boolean {
|
||||
return typeof error === "object" && error !== null && "code" in error && (error as NodeJS.ErrnoException).code === code;
|
||||
}
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import { runControllerDeployHealthCli } from "./deploy-health.js";
|
||||
|
||||
function createIo() {
|
||||
let stderr = "";
|
||||
let stdout = "";
|
||||
return {
|
||||
io: {
|
||||
stderr: {
|
||||
write(chunk: string) {
|
||||
stderr += chunk;
|
||||
return true;
|
||||
},
|
||||
},
|
||||
stdout: {
|
||||
write(chunk: string) {
|
||||
stdout += chunk;
|
||||
return true;
|
||||
},
|
||||
},
|
||||
},
|
||||
readStderr: () => stderr,
|
||||
readStdout: () => stdout,
|
||||
};
|
||||
}
|
||||
|
||||
function jsonResponse(payload: Record<string, unknown>, status = 200) {
|
||||
return { status, json: vi.fn(async () => payload) };
|
||||
}
|
||||
|
||||
function healthyResponses(revision: string) {
|
||||
return [
|
||||
jsonResponse({ status: "ok", service: "hivegrid-portal-domain-controller", revision }),
|
||||
jsonResponse({ status: "ready", service: "hivegrid-portal-domain-controller" }),
|
||||
];
|
||||
}
|
||||
|
||||
describe("portal-domain controller deployment health", () => {
|
||||
test("proves liveness, readiness and the exact baked image revision", async () => {
|
||||
const revision = "a".repeat(40);
|
||||
const responses = healthyResponses(revision);
|
||||
const fetchProbe = vi.fn(async (_url: string, _init: RequestInit) => responses.shift()!);
|
||||
const { io, readStderr, readStdout } = createIo();
|
||||
|
||||
const exitCode = await runControllerDeployHealthCli(
|
||||
revision,
|
||||
{ APP_REVISION: revision, PORTAL_DOMAIN_CONTROLLER_PORT: "8081" },
|
||||
io,
|
||||
{
|
||||
fetchProbe,
|
||||
readImageRevision: () => revision,
|
||||
timeoutSignal: () => new AbortController().signal,
|
||||
}
|
||||
);
|
||||
|
||||
expect(exitCode).toBe(0);
|
||||
expect(JSON.parse(readStdout())).toEqual({
|
||||
status: "ready",
|
||||
service: "hivegrid-portal-domain-controller",
|
||||
revision,
|
||||
});
|
||||
expect(readStderr()).toBe("");
|
||||
expect(fetchProbe.mock.calls.map(([url]) => url)).toEqual([
|
||||
"http://127.0.0.1:8081/healthz",
|
||||
"http://127.0.0.1:8081/readyz",
|
||||
]);
|
||||
});
|
||||
|
||||
test("rejects malformed revisions, baked-image drift and invalid ports before probing", async () => {
|
||||
const revision = "a".repeat(40);
|
||||
const cases = [
|
||||
{ expected: "not-a-sha", env: { APP_REVISION: revision }, baked: revision },
|
||||
{ expected: revision, env: { APP_REVISION: revision }, baked: "b".repeat(40) },
|
||||
{ expected: revision, env: { APP_REVISION: "b".repeat(40) }, baked: revision },
|
||||
{ expected: revision, env: { APP_REVISION: revision, PORT: "70000" }, baked: revision },
|
||||
];
|
||||
|
||||
for (const scenario of cases) {
|
||||
const fetchProbe = vi.fn();
|
||||
const { io, readStderr, readStdout } = createIo();
|
||||
const exitCode = await runControllerDeployHealthCli(scenario.expected, scenario.env, io, {
|
||||
fetchProbe,
|
||||
readImageRevision: () => scenario.baked,
|
||||
});
|
||||
|
||||
expect(exitCode).toBe(1);
|
||||
expect(readStdout()).toBe("");
|
||||
expect(readStderr()).not.toContain(scenario.expected);
|
||||
expect(fetchProbe).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects unhealthy or mismatched controller probe responses", async () => {
|
||||
const revision = "a".repeat(40);
|
||||
const scenarios = [
|
||||
[jsonResponse({}, 503), healthyResponses(revision)[1]],
|
||||
[healthyResponses(revision)[0], jsonResponse({}, 503)],
|
||||
[jsonResponse({ status: "ok", service: "wrong", revision }), healthyResponses(revision)[1]],
|
||||
[jsonResponse({ status: "ok", service: "hivegrid-portal-domain-controller", revision: "b".repeat(40) }), healthyResponses(revision)[1]],
|
||||
[healthyResponses(revision)[0], jsonResponse({ status: "not_ready", service: "hivegrid-portal-domain-controller" })],
|
||||
];
|
||||
|
||||
for (const responses of scenarios) {
|
||||
const pending = [...responses];
|
||||
const { io, readStderr, readStdout } = createIo();
|
||||
const exitCode = await runControllerDeployHealthCli(revision, { APP_REVISION: revision }, io, {
|
||||
fetchProbe: vi.fn(async () => pending.shift()!),
|
||||
readImageRevision: () => revision,
|
||||
timeoutSignal: () => new AbortController().signal,
|
||||
});
|
||||
|
||||
expect(exitCode).toBe(1);
|
||||
expect(readStdout()).toBe("");
|
||||
expect(readStderr()).toMatch(/health|probes/);
|
||||
}
|
||||
});
|
||||
|
||||
test("redacts transport and payload errors", async () => {
|
||||
const revision = "a".repeat(40);
|
||||
const { io, readStderr, readStdout } = createIo();
|
||||
const exitCode = await runControllerDeployHealthCli(revision, { APP_REVISION: revision }, io, {
|
||||
fetchProbe: vi.fn(async () => {
|
||||
throw new Error("connection failed with secret details");
|
||||
}),
|
||||
readImageRevision: () => revision,
|
||||
timeoutSignal: () => new AbortController().signal,
|
||||
});
|
||||
|
||||
expect(exitCode).toBe(1);
|
||||
expect(readStdout()).toBe("");
|
||||
expect(readStderr()).toMatch(/probe failed/);
|
||||
expect(readStderr()).not.toContain("secret details");
|
||||
});
|
||||
});
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const FULL_GIT_SHA_PATTERN = /^[0-9a-f]{40}$/;
|
||||
const IMAGE_REVISION_PATH = "/myapp/.image-revision";
|
||||
const DEFAULT_CONTROLLER_PORT = "8080";
|
||||
const HEALTH_TIMEOUT_MS = 9_000;
|
||||
const CONTROLLER_SERVICE = "hivegrid-portal-domain-controller";
|
||||
|
||||
type ControllerDeployHealthIo = {
|
||||
stderr?: Pick<NodeJS.WriteStream, "write">;
|
||||
stdout: Pick<NodeJS.WriteStream, "write">;
|
||||
};
|
||||
|
||||
type ControllerDeployHealthDependencies = {
|
||||
fetchProbe?: (url: string, init: RequestInit) => Promise<Pick<Response, "status" | "json">>;
|
||||
readImageRevision?: () => string;
|
||||
timeoutSignal?: () => AbortSignal;
|
||||
};
|
||||
|
||||
type ControllerProbeResponse = {
|
||||
status?: unknown;
|
||||
service?: unknown;
|
||||
revision?: unknown;
|
||||
};
|
||||
|
||||
function fail(io: ControllerDeployHealthIo, message: string): number {
|
||||
io.stderr?.write(`${message}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
export async function runControllerDeployHealthCli(
|
||||
expectedRevision: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
io: ControllerDeployHealthIo = process,
|
||||
dependencies: ControllerDeployHealthDependencies = {}
|
||||
): Promise<number> {
|
||||
if (!FULL_GIT_SHA_PATTERN.test(expectedRevision)) {
|
||||
return fail(io, "Portal-domain controller deployment health requires a full lowercase Git SHA.");
|
||||
}
|
||||
|
||||
let imageRevision = "";
|
||||
try {
|
||||
imageRevision = (dependencies.readImageRevision ?? (() => readFileSync(IMAGE_REVISION_PATH, "utf8")))().trim();
|
||||
} catch {
|
||||
return fail(io, "Portal-domain controller deployment image revision is unavailable.");
|
||||
}
|
||||
if (imageRevision !== expectedRevision || env.APP_REVISION !== expectedRevision) {
|
||||
return fail(io, "Portal-domain controller deployment revision does not match the expected image.");
|
||||
}
|
||||
|
||||
const port = env.PORTAL_DOMAIN_CONTROLLER_PORT || env.PORT || DEFAULT_CONTROLLER_PORT;
|
||||
if (!/^\d{1,5}$/.test(port) || Number(port) < 1 || Number(port) > 65_535) {
|
||||
return fail(io, "Portal-domain controller deployment health port is invalid.");
|
||||
}
|
||||
|
||||
try {
|
||||
const fetchProbe = dependencies.fetchProbe ?? fetch;
|
||||
const timeoutSignal = dependencies.timeoutSignal ?? (() => AbortSignal.timeout(HEALTH_TIMEOUT_MS));
|
||||
const [healthResponse, readinessResponse] = await Promise.all([
|
||||
fetchProbe(`http://127.0.0.1:${port}/healthz`, {
|
||||
headers: { Accept: "application/json" },
|
||||
signal: timeoutSignal(),
|
||||
}),
|
||||
fetchProbe(`http://127.0.0.1:${port}/readyz`, {
|
||||
headers: { Accept: "application/json" },
|
||||
signal: timeoutSignal(),
|
||||
}),
|
||||
]);
|
||||
if (healthResponse.status !== 200 || readinessResponse.status !== 200) {
|
||||
return fail(io, "Portal-domain controller deployment health or readiness is not healthy.");
|
||||
}
|
||||
const [health, readiness] = (await Promise.all([healthResponse.json(), readinessResponse.json()])) as [
|
||||
ControllerProbeResponse,
|
||||
ControllerProbeResponse,
|
||||
];
|
||||
if (
|
||||
health.status !== "ok" ||
|
||||
health.service !== CONTROLLER_SERVICE ||
|
||||
health.revision !== expectedRevision ||
|
||||
readiness.status !== "ready" ||
|
||||
readiness.service !== CONTROLLER_SERVICE
|
||||
) {
|
||||
return fail(io, "Portal-domain controller deployment probes do not match the expected runtime.");
|
||||
}
|
||||
} catch {
|
||||
return fail(io, "Portal-domain controller deployment probe failed.");
|
||||
}
|
||||
|
||||
io.stdout.write(`${JSON.stringify({ status: "ready", service: CONTROLLER_SERVICE, revision: expectedRevision })}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
runControllerDeployHealthCli(process.argv[2] || "")
|
||||
.then((exitCode) => {
|
||||
process.exitCode = exitCode;
|
||||
})
|
||||
.catch(() => {
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
import {
|
||||
createSignedDomainCommand,
|
||||
DOMAIN_COMMAND_QUEUE,
|
||||
} from "@hivegrid/portal-domain-controller/contracts";
|
||||
import {
|
||||
DOMAIN_COMMAND_JOB_OPTIONS,
|
||||
} from "@hivegrid/portal-domain-controller/queue";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
describe("portal-domain controller package exports", () => {
|
||||
it("exposes one shared signed protocol and its queue retry contract", () => {
|
||||
const command = createSignedDomainCommand({
|
||||
claimId: "claim_12345678",
|
||||
tenantId: "tenant_12345678",
|
||||
generation: 1,
|
||||
hostname: "portal.example.com",
|
||||
action: "provision",
|
||||
}, "package-export-secret-that-is-at-least-thirty-two-bytes");
|
||||
expect(command.payload.version).toBe(1);
|
||||
expect(DOMAIN_COMMAND_QUEUE).toBe("hivegrid.portal-domain.commands");
|
||||
expect(DOMAIN_COMMAND_JOB_OPTIONS).toMatchObject({
|
||||
attempts: 4,
|
||||
backoff: { type: "portal-domain-schedule" },
|
||||
});
|
||||
});
|
||||
});
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
import { createLogger } from "@hivegrid/logger";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createControllerHttpRuntime, type ControllerHttpRuntime } from "./http.js";
|
||||
import { createControllerMetrics } from "./metrics.js";
|
||||
|
||||
const METRICS_TOKEN = "metrics-token-that-is-at-least-thirty-two-bytes";
|
||||
const runtimes: ControllerHttpRuntime[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(runtimes.splice(0).map((runtime) => runtime.close()));
|
||||
});
|
||||
|
||||
describe("portal-domain controller HTTP runtime", () => {
|
||||
it("serves liveness, fail-closed readiness and a bearer-protected metrics endpoint", async () => {
|
||||
let ready = false;
|
||||
const runtime = createControllerHttpRuntime({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
revision: "abc123",
|
||||
metricsToken: METRICS_TOKEN,
|
||||
isReady: async () => ready,
|
||||
logger: createLogger({ level: "silent" }),
|
||||
metrics: createControllerMetrics("test", false),
|
||||
});
|
||||
runtimes.push(runtime);
|
||||
const address = await runtime.listen();
|
||||
const origin = `http://127.0.0.1:${address.port}`;
|
||||
|
||||
const health = await fetch(`${origin}/healthz`);
|
||||
expect(health.status).toBe(200);
|
||||
expect(await health.json()).toEqual({ status: "ok", service: "hivegrid-portal-domain-controller", revision: "abc123" });
|
||||
expect(health.headers.get("x-content-type-options")).toBe("nosniff");
|
||||
|
||||
expect((await fetch(`${origin}/readyz`)).status).toBe(503);
|
||||
ready = true;
|
||||
expect((await fetch(`${origin}/readyz`)).status).toBe(200);
|
||||
|
||||
expect((await fetch(`${origin}/metrics`)).status).toBe(401);
|
||||
expect((await fetch(`${origin}/metrics`, { headers: { authorization: "Bearer wrong" } })).status).toBe(401);
|
||||
const metrics = await fetch(`${origin}/metrics`, { headers: { authorization: `Bearer ${METRICS_TOKEN}` } });
|
||||
expect(metrics.status).toBe(200);
|
||||
expect(await metrics.text()).toContain("hivegrid_portal_domain_controller_ready");
|
||||
expect((await fetch(`${origin}/unknown`)).status).toBe(404);
|
||||
expect((await fetch(`${origin}/healthz`, { method: "POST" })).status).toBe(405);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import type { HiveGridLogger } from "@hivegrid/logger";
|
||||
import { validateMetricsTokenAuth } from "@hivegrid/metrics";
|
||||
import type { ControllerMetrics } from "./metrics.js";
|
||||
|
||||
export type ControllerHttpRuntime = {
|
||||
server: Server;
|
||||
listen(): Promise<AddressInfo>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export function createControllerHttpRuntime(options: {
|
||||
host: string;
|
||||
port: number;
|
||||
revision: string;
|
||||
metricsToken: string;
|
||||
isReady: () => Promise<boolean>;
|
||||
logger: HiveGridLogger;
|
||||
metrics: ControllerMetrics;
|
||||
}): ControllerHttpRuntime {
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const pathname = new URL(request.url ?? "/", "http://controller.invalid").pathname;
|
||||
setSecurityHeaders(response);
|
||||
if (request.method !== "GET") {
|
||||
sendJson(response, 405, { error: "method_not_allowed" }, { allow: "GET" });
|
||||
return;
|
||||
}
|
||||
if (pathname === "/healthz") {
|
||||
sendJson(response, 200, {
|
||||
status: "ok",
|
||||
service: "hivegrid-portal-domain-controller",
|
||||
revision: options.revision,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (pathname === "/readyz") {
|
||||
let ready = false;
|
||||
try {
|
||||
ready = await options.isReady();
|
||||
} catch (error) {
|
||||
options.logger.warn({ err: error }, "portal-domain readiness check failed");
|
||||
}
|
||||
options.metrics.ready.set(ready ? 1 : 0);
|
||||
sendJson(response, ready ? 200 : 503, {
|
||||
status: ready ? "ready" : "not_ready",
|
||||
service: "hivegrid-portal-domain-controller",
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (pathname === "/metrics") {
|
||||
const auth = validateMetricsTokenAuth({
|
||||
authorizationHeader: request.headers.authorization,
|
||||
configuredToken: options.metricsToken,
|
||||
metricsEnabled: true,
|
||||
});
|
||||
if (!auth.ok) {
|
||||
sendJson(response, auth.status, { error: auth.code });
|
||||
return;
|
||||
}
|
||||
const body = await options.metrics.registry.metrics();
|
||||
response.writeHead(200, {
|
||||
"cache-control": "no-store",
|
||||
"content-type": options.metrics.registry.contentType,
|
||||
});
|
||||
response.end(body);
|
||||
return;
|
||||
}
|
||||
sendJson(response, 404, { error: "not_found" });
|
||||
} catch (error) {
|
||||
options.logger.error({ err: error }, "controller HTTP request failed");
|
||||
if (!response.headersSent) {
|
||||
setSecurityHeaders(response);
|
||||
sendJson(response, 500, { error: "internal_error" });
|
||||
} else {
|
||||
response.destroy();
|
||||
}
|
||||
}
|
||||
});
|
||||
server.on("clientError", (error, socket) => {
|
||||
options.logger.warn({ err: error }, "rejected malformed controller HTTP request");
|
||||
socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n");
|
||||
});
|
||||
return {
|
||||
server,
|
||||
listen() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const onError = (error: Error) => reject(error);
|
||||
server.once("error", onError);
|
||||
server.listen(options.port, options.host, () => {
|
||||
server.off("error", onError);
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
reject(new Error("controller HTTP server did not expose a TCP address"));
|
||||
return;
|
||||
}
|
||||
resolve(address);
|
||||
});
|
||||
});
|
||||
},
|
||||
close() {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!server.listening) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
server.close((error) => error ? reject(error) : resolve());
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sendJson(
|
||||
response: Parameters<typeof setSecurityHeaders>[0],
|
||||
status: number,
|
||||
payload: Record<string, unknown>,
|
||||
headers: Record<string, string> = {}
|
||||
): void {
|
||||
response.writeHead(status, {
|
||||
"cache-control": "no-store",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
...headers,
|
||||
});
|
||||
response.end(`${JSON.stringify(payload)}\n`);
|
||||
}
|
||||
|
||||
function setSecurityHeaders(response: import("node:http").ServerResponse): void {
|
||||
response.setHeader("x-content-type-options", "nosniff");
|
||||
response.setHeader("referrer-policy", "no-referrer");
|
||||
response.setHeader("content-security-policy", "default-src 'none'");
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import { createChildLogger, createLogger } from "@hivegrid/logger";
|
||||
import { createFencedClaimMutationLock } from "./claim-lock.js";
|
||||
import { parsePortalDomainControllerConfig } from "./config.js";
|
||||
import { createPortalDomainController } from "./controller.js";
|
||||
import { createControllerHttpRuntime } from "./http.js";
|
||||
import { createControllerMetrics } from "./metrics.js";
|
||||
import { createDomainProbes } from "./probes.js";
|
||||
import { createDomainQueueRuntime } from "./queue.js";
|
||||
import { TraefikConfigWriter } from "./traefik.js";
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const config = parsePortalDomainControllerConfig();
|
||||
const logger = createChildLogger(createLogger(), {
|
||||
service: "hivegrid-portal-domain-controller",
|
||||
revision: config.revision,
|
||||
});
|
||||
const claimLock = createFencedClaimMutationLock(config.queueConnection, config.configDirectory);
|
||||
const writer = new TraefikConfigWriter({
|
||||
configDirectory: config.configDirectory,
|
||||
reloadTriggerFile: config.reloadTriggerFile,
|
||||
routing: {
|
||||
entryPoint: config.entryPoint,
|
||||
backendService: config.backendService,
|
||||
},
|
||||
claimLock,
|
||||
});
|
||||
const metrics = createControllerMetrics(config.revision);
|
||||
let queue: Awaited<ReturnType<typeof createDomainQueueRuntime>>;
|
||||
try {
|
||||
await writer.assertReady();
|
||||
await writer.reconcilePersistedDomainConfigs();
|
||||
const probes = createDomainProbes({ expectedCname: config.expectedCname, expectedRevision: config.revision });
|
||||
const controller = createPortalDomainController({ writer, probes });
|
||||
queue = await createDomainQueueRuntime({
|
||||
connection: config.queueConnection,
|
||||
concurrency: config.queueConcurrency,
|
||||
controller,
|
||||
commandSecret: config.commandSecret,
|
||||
logger,
|
||||
metrics,
|
||||
});
|
||||
} catch (error) {
|
||||
await claimLock.close().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
const http = createControllerHttpRuntime({
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
revision: config.revision,
|
||||
metricsToken: config.metricsToken,
|
||||
logger,
|
||||
metrics,
|
||||
isReady: async () => (await queue.isReady()) && (await writable(writer)),
|
||||
});
|
||||
let address;
|
||||
try {
|
||||
address = await http.listen();
|
||||
} catch (error) {
|
||||
await queue.close().catch(() => undefined);
|
||||
await claimLock.close().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
logger.info({ host: address.address, port: address.port }, "portal-domain controller started");
|
||||
|
||||
let shuttingDown = false;
|
||||
const shutdown = async (signal: string) => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
logger.info({ signal }, "stopping portal-domain controller");
|
||||
const timeout = setTimeout(() => {
|
||||
logger.error("portal-domain controller shutdown timed out");
|
||||
process.exit(1);
|
||||
}, 30_000);
|
||||
timeout.unref();
|
||||
try {
|
||||
await queue.close();
|
||||
await claimLock.close();
|
||||
await http.close();
|
||||
clearTimeout(timeout);
|
||||
} catch (error) {
|
||||
logger.error({ err: error }, "portal-domain controller shutdown failed");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
};
|
||||
process.once("SIGTERM", () => void shutdown("SIGTERM"));
|
||||
process.once("SIGINT", () => void shutdown("SIGINT"));
|
||||
}
|
||||
|
||||
async function writable(writer: TraefikConfigWriter): Promise<boolean> {
|
||||
try {
|
||||
await writer.assertReady();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
const logger = createChildLogger(createLogger(), { service: "hivegrid-portal-domain-controller" });
|
||||
logger.error({ err: error }, "portal-domain controller failed to start");
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import { createMetricsRegistry } from "@hivegrid/metrics";
|
||||
import { Counter, Gauge, Histogram } from "prom-client";
|
||||
|
||||
export function createControllerMetrics(revision: string, collectDefaultProcessMetrics = true) {
|
||||
const registry = createMetricsRegistry({
|
||||
serviceName: "portal-domain-controller",
|
||||
collectDefaultProcessMetrics,
|
||||
defaultLabels: { revision },
|
||||
});
|
||||
const commands = new Counter({
|
||||
name: "hivegrid_portal_domain_commands_total",
|
||||
help: "Portal domain commands by action and outcome.",
|
||||
labelNames: ["action", "status"] as const,
|
||||
registers: [registry],
|
||||
});
|
||||
const duration = new Histogram({
|
||||
name: "hivegrid_portal_domain_command_duration_seconds",
|
||||
help: "Portal domain command processing duration.",
|
||||
labelNames: ["action"] as const,
|
||||
buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
|
||||
registers: [registry],
|
||||
});
|
||||
const ready = new Gauge({
|
||||
name: "hivegrid_portal_domain_controller_ready",
|
||||
help: "Whether Redis and the dedicated Traefik config directory are ready.",
|
||||
registers: [registry],
|
||||
});
|
||||
return { registry, commands, duration, ready };
|
||||
}
|
||||
|
||||
export type ControllerMetrics = ReturnType<typeof createControllerMetrics>;
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createDomainProbes, DomainProbeError } from "./probes.js";
|
||||
|
||||
const REVISION = "0123456789abcdef0123456789abcdef01234567";
|
||||
|
||||
function healthResponse(
|
||||
overrides: Record<string, unknown> = {},
|
||||
options: { readonly status?: number; readonly url?: string; readonly headers?: Readonly<Record<string, string>> } = {}
|
||||
): Response {
|
||||
const response = new Response(
|
||||
JSON.stringify({
|
||||
status: "ok",
|
||||
service: "hivegrid-portal",
|
||||
surface: "portal",
|
||||
revision: REVISION,
|
||||
host: "portal.example.com",
|
||||
...overrides,
|
||||
}),
|
||||
{
|
||||
status: options.status ?? 200,
|
||||
headers: options.headers ?? { "content-type": "application/json" },
|
||||
}
|
||||
);
|
||||
Object.defineProperty(response, "url", { value: options.url ?? "https://portal.example.com/healthz" });
|
||||
return response;
|
||||
}
|
||||
|
||||
describe("portal-domain DNS and HTTPS probes", () => {
|
||||
it("requires the exact configured CNAME after DNS canonicalization", async () => {
|
||||
const resolveCnameFn = vi.fn().mockResolvedValue(["INGRESS.TOOL-CTRL.DE."]);
|
||||
const probes = createDomainProbes({
|
||||
expectedCname: "ingress.tool-ctrl.de",
|
||||
expectedRevision: REVISION,
|
||||
resolveCnameFn: resolveCnameFn as never,
|
||||
fetchFn: vi.fn() as never,
|
||||
});
|
||||
await expect(probes.verifyDns("portal.example.com")).resolves.toBeUndefined();
|
||||
expect(resolveCnameFn).toHaveBeenCalledWith("portal.example.com");
|
||||
|
||||
resolveCnameFn.mockResolvedValueOnce(["attacker.example.com"]);
|
||||
await expect(probes.verifyDns("portal.example.com")).rejects.toMatchObject({ reasonCode: "dns_cname_mismatch", retryable: true });
|
||||
resolveCnameFn.mockResolvedValueOnce(["ingress.tool-ctrl.de", "attacker.example.com"]);
|
||||
await expect(probes.verifyDns("portal.example.com")).rejects.toMatchObject({ reasonCode: "dns_cname_mismatch" });
|
||||
resolveCnameFn.mockRejectedValueOnce(new Error("ENODATA"));
|
||||
await expect(probes.verifyDns("portal.example.com")).rejects.toMatchObject({ reasonCode: "dns_lookup_failed" });
|
||||
});
|
||||
|
||||
it("requires custom-host portal health plus an independent SNI certificate acknowledgement", async () => {
|
||||
const certificateExpiresAt = new Date("2026-10-01T00:00:00.000Z");
|
||||
const inspectCertificateFn = vi.fn(async () => certificateExpiresAt);
|
||||
const fetchFn = vi.fn().mockResolvedValue(healthResponse());
|
||||
const probes = createDomainProbes({
|
||||
expectedCname: "ingress.tool-ctrl.de",
|
||||
expectedRevision: REVISION,
|
||||
resolveCnameFn: vi.fn() as never,
|
||||
fetchFn: fetchFn as never,
|
||||
inspectCertificateFn,
|
||||
now: () => new Date("2026-07-16T10:00:00.000Z"),
|
||||
});
|
||||
await expect(probes.verifyHttps("portal.example.com")).resolves.toEqual({ certificateExpiresAt: certificateExpiresAt.toISOString() });
|
||||
expect(String(fetchFn.mock.calls[0]?.[0])).toBe("https://portal.example.com/healthz");
|
||||
expect(fetchFn).toHaveBeenCalledWith(expect.any(URL), expect.objectContaining({ redirect: "manual" }));
|
||||
expect(inspectCertificateFn).toHaveBeenCalledWith("portal.example.com", 10_000);
|
||||
|
||||
fetchFn.mockResolvedValueOnce(
|
||||
healthResponse(
|
||||
{},
|
||||
{
|
||||
status: 302,
|
||||
url: "https://portal.example.com/healthz",
|
||||
headers: { "content-type": "application/json", location: "https://attacker.example" },
|
||||
}
|
||||
)
|
||||
);
|
||||
await expect(probes.verifyHttps("portal.example.com")).rejects.toBeInstanceOf(DomainProbeError);
|
||||
fetchFn.mockResolvedValueOnce(healthResponse({ service: "wrong" }));
|
||||
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
|
||||
});
|
||||
|
||||
it("rejects a 200 response whose observed Host, final URL, surface, or revision does not match", async () => {
|
||||
const fetchFn = vi.fn();
|
||||
const inspectCertificateFn = vi.fn(async () => new Date("2026-10-01T00:00:00.000Z"));
|
||||
const probes = createDomainProbes({
|
||||
expectedCname: "ingress.tool-ctrl.de",
|
||||
expectedRevision: REVISION,
|
||||
resolveCnameFn: vi.fn() as never,
|
||||
fetchFn: fetchFn as never,
|
||||
inspectCertificateFn,
|
||||
now: () => new Date("2026-07-16T10:00:00.000Z"),
|
||||
});
|
||||
|
||||
for (const response of [
|
||||
healthResponse({ host: "attacker.example.com" }),
|
||||
healthResponse({ surface: "core", service: "hivegrid-web" }),
|
||||
healthResponse({}, { url: "https://attacker.example.com/healthz" }),
|
||||
]) {
|
||||
fetchFn.mockResolvedValueOnce(response);
|
||||
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
|
||||
}
|
||||
fetchFn.mockResolvedValueOnce(healthResponse({ revision: "different-release" }));
|
||||
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "release_revision_mismatch" });
|
||||
expect(inspectCertificateFn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails closed when the explicit SNI certificate observation is missing or expired", async () => {
|
||||
const fetchFn = vi.fn().mockResolvedValue(healthResponse());
|
||||
const inspectCertificateFn = vi.fn().mockResolvedValue(new Date("2026-07-16T09:59:59.000Z"));
|
||||
const probes = createDomainProbes({
|
||||
expectedCname: "ingress.tool-ctrl.de",
|
||||
expectedRevision: REVISION,
|
||||
resolveCnameFn: vi.fn() as never,
|
||||
fetchFn: fetchFn as never,
|
||||
inspectCertificateFn,
|
||||
now: () => new Date("2026-07-16T10:00:00.000Z"),
|
||||
});
|
||||
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
|
||||
inspectCertificateFn.mockRejectedValueOnce(new Error("TLS failure"));
|
||||
await expect(probes.verifyHttps("portal.example.com")).rejects.toMatchObject({ reasonCode: "https_health_failed" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
import { resolveCname } from "node:dns/promises";
|
||||
import { connect as connectTls } from "node:tls";
|
||||
import { normalizeDomainHostname } from "./contracts.js";
|
||||
|
||||
export class DomainProbeError extends Error {
|
||||
readonly retryable = true;
|
||||
|
||||
constructor(
|
||||
readonly reasonCode: "dns_cname_mismatch" | "dns_lookup_failed" | "https_health_failed" | "release_revision_mismatch",
|
||||
message: string
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
export type DomainProbes = {
|
||||
verifyDns(hostname: string): Promise<void>;
|
||||
verifyHttps(hostname: string): Promise<{ readonly certificateExpiresAt: string }>;
|
||||
};
|
||||
|
||||
export type CreateDomainProbesOptions = {
|
||||
expectedCname: string;
|
||||
expectedRevision: string;
|
||||
timeoutMs?: number;
|
||||
resolveCnameFn?: typeof resolveCname;
|
||||
fetchFn?: typeof fetch;
|
||||
inspectCertificateFn?: (hostname: string, timeoutMs: number) => Promise<Date>;
|
||||
now?: () => Date;
|
||||
};
|
||||
|
||||
export function createDomainProbes(options: CreateDomainProbesOptions): DomainProbes {
|
||||
const expectedCname = normalizeDomainHostname(options.expectedCname);
|
||||
const expectedRevision = normalizeExpectedRevision(options.expectedRevision);
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
const resolveCnameFn = options.resolveCnameFn ?? resolveCname;
|
||||
const fetchFn = options.fetchFn ?? fetch;
|
||||
const inspectCertificateFn = options.inspectCertificateFn ?? inspectTlsCertificate;
|
||||
const now = options.now ?? (() => new Date());
|
||||
return {
|
||||
async verifyDns(hostname) {
|
||||
const normalizedHostname = normalizeDomainHostname(hostname);
|
||||
let answers: string[];
|
||||
try {
|
||||
answers = await resolveCnameFn(normalizedHostname);
|
||||
} catch {
|
||||
throw new DomainProbeError("dns_lookup_failed", "The portal domain CNAME could not be resolved");
|
||||
}
|
||||
const normalizedAnswers = answers.flatMap((answer) => {
|
||||
try {
|
||||
return [normalizeDomainHostname(answer)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
if (normalizedAnswers.length !== 1 || normalizedAnswers[0] !== expectedCname) {
|
||||
throw new DomainProbeError("dns_cname_mismatch", "The portal domain CNAME does not target the HiveGrid ingress");
|
||||
}
|
||||
},
|
||||
async verifyHttps(hostname) {
|
||||
const normalizedHostname = normalizeDomainHostname(hostname);
|
||||
const expectedUrl = new URL(`https://${normalizedHostname}/healthz`);
|
||||
let response: Response;
|
||||
try {
|
||||
// This request intentionally traverses the newly rendered exact Host()
|
||||
// router. The portal runtime exposes only its dependency-free /healthz
|
||||
// endpoint before DB activation; routes, auth and assets remain 404.
|
||||
response = await fetchFn(expectedUrl, {
|
||||
method: "GET",
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
headers: { accept: "application/json" },
|
||||
});
|
||||
} catch {
|
||||
throw new DomainProbeError("https_health_failed", "The portal HTTPS health probe failed");
|
||||
}
|
||||
if (
|
||||
response.status !== 200 ||
|
||||
response.redirected ||
|
||||
response.headers.get("content-type")?.includes("application/json") !== true ||
|
||||
!sameHealthUrl(response.url, expectedUrl)
|
||||
) {
|
||||
throw new DomainProbeError("https_health_failed", "The portal HTTPS endpoint did not return a valid health response");
|
||||
}
|
||||
const contentLength = Number(response.headers.get("content-length") ?? 0);
|
||||
if (Number.isFinite(contentLength) && contentLength > 4096) {
|
||||
throw new DomainProbeError("https_health_failed", "The portal HTTPS health response exceeded the size limit");
|
||||
}
|
||||
let payload: unknown;
|
||||
try {
|
||||
const body = await response.text();
|
||||
if (Buffer.byteLength(body, "utf8") > 4096) throw new Error("health response too large");
|
||||
payload = JSON.parse(body);
|
||||
} catch {
|
||||
throw new DomainProbeError("https_health_failed", "The portal HTTPS health response was malformed");
|
||||
}
|
||||
const observedRevision = portalHealthRevision(payload, normalizedHostname);
|
||||
if (!observedRevision) {
|
||||
throw new DomainProbeError("https_health_failed", "The portal HTTPS health response identified the wrong service");
|
||||
}
|
||||
if (observedRevision !== expectedRevision) {
|
||||
throw new DomainProbeError("release_revision_mismatch", "The portal HTTPS health response belongs to a different release");
|
||||
}
|
||||
let certificateExpiresAt: Date;
|
||||
try {
|
||||
certificateExpiresAt = await inspectCertificateFn(normalizedHostname, timeoutMs);
|
||||
} catch {
|
||||
throw new DomainProbeError("https_health_failed", "The portal TLS certificate could not be inspected with SNI");
|
||||
}
|
||||
if (!Number.isFinite(certificateExpiresAt.getTime()) || certificateExpiresAt <= now()) {
|
||||
throw new DomainProbeError("https_health_failed", "The portal TLS certificate is invalid or expired");
|
||||
}
|
||||
return { certificateExpiresAt: certificateExpiresAt.toISOString() };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens an independently verified TLS connection with an explicit SNI name.
|
||||
* The custom-host `fetch()` proves the exact Host router and portal response;
|
||||
* this narrow probe
|
||||
* captures the certificate expiry needed for the 21/7-day alert contract.
|
||||
*/
|
||||
export function inspectTlsCertificate(hostname: string, timeoutMs: number): Promise<Date> {
|
||||
const normalizedHostname = normalizeDomainHostname(hostname);
|
||||
return new Promise<Date>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const socket = connectTls({
|
||||
host: normalizedHostname,
|
||||
port: 443,
|
||||
servername: normalizedHostname,
|
||||
rejectUnauthorized: true,
|
||||
});
|
||||
const fail = (error: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
socket.destroy();
|
||||
reject(error);
|
||||
};
|
||||
socket.setTimeout(timeoutMs, () => fail(new Error("portal TLS certificate probe timed out")));
|
||||
socket.once("error", fail);
|
||||
socket.once("secureConnect", () => {
|
||||
if (settled) return;
|
||||
const certificate = socket.getPeerCertificate();
|
||||
const expiresAt = new Date(certificate.valid_to ?? "");
|
||||
if (!certificate.valid_to || !Number.isFinite(expiresAt.getTime())) {
|
||||
fail(new Error("portal TLS peer certificate has no valid expiry"));
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
socket.end();
|
||||
resolve(expiresAt);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function portalHealthRevision(value: unknown, hostname: string): string | null {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
||||
const record = value as Record<string, unknown>;
|
||||
const matches =
|
||||
Object.keys(record).sort().join(",") === "host,revision,service,status,surface" &&
|
||||
record.status === "ok" &&
|
||||
record.service === "hivegrid-portal" &&
|
||||
record.surface === "portal" &&
|
||||
record.host === hostname &&
|
||||
typeof record.revision === "string" &&
|
||||
/^[A-Za-z0-9._-]{1,128}$/u.test(record.revision);
|
||||
return matches ? record.revision as string : null;
|
||||
}
|
||||
|
||||
function sameHealthUrl(actual: string, expected: URL): boolean {
|
||||
try {
|
||||
const url = new URL(actual);
|
||||
return url.protocol === "https:" && url.hostname === expected.hostname && !url.port && url.pathname === "/healthz" && !url.search && !url.hash;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeExpectedRevision(value: string): string {
|
||||
const revision = value.trim();
|
||||
if (!revision || revision === "unknown" || !/^[A-Za-z0-9._-]{1,128}$/u.test(revision)) {
|
||||
throw new Error("portal health probe requires an immutable release revision");
|
||||
}
|
||||
return revision;
|
||||
}
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
import { createLogger } from "@hivegrid/logger";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createSignedDomainCommand,
|
||||
verifySignedDomainResult,
|
||||
type DomainResultPayload,
|
||||
type SignedDomainEnvelope,
|
||||
} from "./contracts.js";
|
||||
import { DomainControllerFailure, type PortalDomainController } from "./controller.js";
|
||||
import { createControllerMetrics } from "./metrics.js";
|
||||
import { createDomainCommandProcessor } from "./queue.js";
|
||||
|
||||
const SECRET = "test-command-secret-that-is-at-least-thirty-two-bytes";
|
||||
const payload = {
|
||||
claimId: "claim_12345678",
|
||||
tenantId: "tenant_12345678",
|
||||
generation: 1,
|
||||
hostname: "portal.example.com",
|
||||
action: "provision" as const,
|
||||
};
|
||||
|
||||
describe("portal-domain queue processor", () => {
|
||||
it("verifies commands and publishes an independently signed result", async () => {
|
||||
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
|
||||
const controller: PortalDomainController = {
|
||||
handle: vi.fn(async (command, attempt) => ({
|
||||
...command,
|
||||
status: "active",
|
||||
attempt,
|
||||
completedAt: "2026-07-16T10:00:00.000Z",
|
||||
configHash: "a".repeat(64),
|
||||
certificateExpiresAt: "2026-10-01T00:00:00.000Z",
|
||||
})),
|
||||
};
|
||||
const processor = createProcessor(controller, published);
|
||||
await processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 0 });
|
||||
expect(published).toHaveLength(1);
|
||||
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({ status: "active", attempt: 1 });
|
||||
});
|
||||
|
||||
it("rejects unsigned or tampered commands without emitting a reflected result", async () => {
|
||||
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
|
||||
const controller: PortalDomainController = { handle: vi.fn() };
|
||||
const processor = createProcessor(controller, published);
|
||||
const signed = createSignedDomainCommand(payload, SECRET);
|
||||
await expect(processor({ data: { ...signed, payload: { ...signed.payload, hostname: "evil.example.com" } }, attemptsMade: 0 })).rejects.toThrow("invalid_domain_command");
|
||||
expect(published).toEqual([]);
|
||||
expect(controller.handle).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("publishes signed retry and terminal outcomes around the four-attempt ceiling", async () => {
|
||||
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
|
||||
const controller: PortalDomainController = {
|
||||
handle: vi.fn().mockRejectedValue(new DomainControllerFailure("https_health_failed", true, "transient")),
|
||||
};
|
||||
const processor = createProcessor(controller, published);
|
||||
const signed = createSignedDomainCommand(payload, SECRET);
|
||||
await expect(processor({ data: signed, attemptsMade: 0 })).rejects.toThrow("https_health_failed");
|
||||
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({ status: "retrying", retryAfterMs: 900_000 });
|
||||
|
||||
await expect(processor({ data: signed, attemptsMade: 3 })).resolves.toBeUndefined();
|
||||
expect(verifySignedDomainResult(published[1], SECRET)).toMatchObject({ status: "failed", attempt: 4, reasonCode: "https_health_failed" });
|
||||
});
|
||||
|
||||
it("keeps release mismatches inside the same four-attempt safety budget", async () => {
|
||||
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
|
||||
const controller: PortalDomainController = {
|
||||
handle: vi.fn().mockRejectedValue(new DomainControllerFailure("release_revision_mismatch", true, "release overlap")),
|
||||
};
|
||||
const processor = createProcessor(controller, published);
|
||||
|
||||
await expect(processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 0 })).rejects.toThrow("release_revision_mismatch");
|
||||
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({
|
||||
status: "retrying",
|
||||
reasonCode: "release_revision_mismatch",
|
||||
retryAfterMs: 900_000,
|
||||
});
|
||||
|
||||
await expect(processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 3 })).resolves.toBeUndefined();
|
||||
expect(verifySignedDomainResult(published[1], SECRET)).toMatchObject({
|
||||
status: "failed",
|
||||
attempt: 4,
|
||||
reasonCode: "release_revision_mismatch",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not retry terminal controller failures", async () => {
|
||||
const published: SignedDomainEnvelope<DomainResultPayload>[] = [];
|
||||
const controller: PortalDomainController = {
|
||||
handle: vi.fn().mockRejectedValue(new DomainControllerFailure("unsafe_config_path", false, "terminal")),
|
||||
};
|
||||
const processor = createProcessor(controller, published);
|
||||
await expect(processor({ data: createSignedDomainCommand(payload, SECRET), attemptsMade: 0 })).resolves.toBeUndefined();
|
||||
expect(verifySignedDomainResult(published[0], SECRET)).toMatchObject({ status: "failed", attempt: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
function createProcessor(
|
||||
controller: PortalDomainController,
|
||||
published: SignedDomainEnvelope<DomainResultPayload>[]
|
||||
) {
|
||||
return createDomainCommandProcessor({
|
||||
controller,
|
||||
commandSecret: SECRET,
|
||||
resultPublisher: async (result) => { published.push(result); },
|
||||
logger: createLogger({ level: "silent" }),
|
||||
metrics: createControllerMetrics("test", false),
|
||||
now: () => new Date("2026-07-16T10:00:00.000Z"),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { Queue, Worker, type JobsOptions } from "bullmq";
|
||||
import type { HiveGridLogger } from "@hivegrid/logger";
|
||||
import type { ConnectionOptions } from "@hivegrid/queue";
|
||||
import {
|
||||
createSignedDomainResult,
|
||||
DOMAIN_COMMAND_QUEUE,
|
||||
DOMAIN_COMMAND_QUEUE_ATTEMPTS,
|
||||
DOMAIN_RESULT_QUEUE,
|
||||
sha256Hex,
|
||||
verifySignedDomainCommand,
|
||||
type DomainResultPayload,
|
||||
type SignedDomainEnvelope,
|
||||
} from "./contracts.js";
|
||||
import {
|
||||
DOMAIN_MAX_ATTEMPTS,
|
||||
DomainControllerFailure,
|
||||
retryDelayForAttempt,
|
||||
type PortalDomainController,
|
||||
} from "./controller.js";
|
||||
import type { ControllerMetrics } from "./metrics.js";
|
||||
|
||||
export const DOMAIN_BACKOFF_TYPE = "portal-domain-schedule";
|
||||
export const DOMAIN_COMMAND_JOB_OPTIONS: JobsOptions = Object.freeze({
|
||||
attempts: DOMAIN_COMMAND_QUEUE_ATTEMPTS,
|
||||
backoff: { type: DOMAIN_BACKOFF_TYPE },
|
||||
removeOnComplete: { age: 24 * 60 * 60, count: 10_000 },
|
||||
removeOnFail: { age: 7 * 24 * 60 * 60, count: 50_000 },
|
||||
});
|
||||
|
||||
export type DomainQueueRuntime = {
|
||||
commandQueue: Queue;
|
||||
resultQueue: Queue;
|
||||
worker: Worker;
|
||||
isReady(): Promise<boolean>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export type DomainCommandProcessor = (job: {
|
||||
data: unknown;
|
||||
attemptsMade: number;
|
||||
}) => Promise<void>;
|
||||
|
||||
export function createDomainCommandProcessor(options: {
|
||||
controller: PortalDomainController;
|
||||
commandSecret: string;
|
||||
resultPublisher: (result: SignedDomainEnvelope<DomainResultPayload>) => Promise<void>;
|
||||
logger: HiveGridLogger;
|
||||
metrics: ControllerMetrics;
|
||||
now?: () => Date;
|
||||
}): DomainCommandProcessor {
|
||||
const now = options.now ?? (() => new Date());
|
||||
return async (job) => {
|
||||
const startedAt = performance.now();
|
||||
let command;
|
||||
try {
|
||||
command = verifySignedDomainCommand(job.data, options.commandSecret);
|
||||
} catch (error) {
|
||||
options.logger.warn({ err: error }, "rejected unauthenticated portal-domain command");
|
||||
throw new Error("invalid_domain_command");
|
||||
}
|
||||
|
||||
const attempt = job.attemptsMade + 1;
|
||||
let status: DomainResultPayload["status"] = "failed";
|
||||
try {
|
||||
const result = await options.controller.handle(command, attempt);
|
||||
status = result.status;
|
||||
await options.resultPublisher(createSignedDomainResult(result, options.commandSecret));
|
||||
options.logger.info(
|
||||
{ claimId: command.claimId, tenantId: command.tenantId, generation: command.generation, action: command.action, status },
|
||||
"processed portal-domain command"
|
||||
);
|
||||
} catch (error) {
|
||||
const failure = error instanceof DomainControllerFailure
|
||||
? error
|
||||
: new DomainControllerFailure("controller_internal_error", true, "Transient controller failure");
|
||||
const canRetry = failure.retryable && attempt < DOMAIN_MAX_ATTEMPTS;
|
||||
status = canRetry ? "retrying" : "failed";
|
||||
const retryAfterMs = canRetry ? retryDelayForAttempt(attempt) : undefined;
|
||||
const result: DomainResultPayload = {
|
||||
...command,
|
||||
status,
|
||||
attempt,
|
||||
completedAt: now().toISOString(),
|
||||
reasonCode: failure.reasonCode,
|
||||
...(retryAfterMs ? { retryAfterMs } : {}),
|
||||
};
|
||||
await options.resultPublisher(createSignedDomainResult(result, options.commandSecret));
|
||||
options.logger.warn(
|
||||
{
|
||||
claimId: command.claimId,
|
||||
tenantId: command.tenantId,
|
||||
generation: command.generation,
|
||||
action: command.action,
|
||||
status,
|
||||
reasonCode: failure.reasonCode,
|
||||
attempt,
|
||||
},
|
||||
"portal-domain command did not complete"
|
||||
);
|
||||
if (canRetry) throw new Error(failure.reasonCode);
|
||||
} finally {
|
||||
options.metrics.commands.inc({ action: command.action, status });
|
||||
options.metrics.duration.observe({ action: command.action }, (performance.now() - startedAt) / 1000);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function createDomainQueueRuntime(options: {
|
||||
connection: ConnectionOptions;
|
||||
concurrency: number;
|
||||
controller: PortalDomainController;
|
||||
commandSecret: string;
|
||||
logger: HiveGridLogger;
|
||||
metrics: ControllerMetrics;
|
||||
}): Promise<DomainQueueRuntime> {
|
||||
const commandQueue = new Queue(DOMAIN_COMMAND_QUEUE, {
|
||||
connection: options.connection,
|
||||
defaultJobOptions: DOMAIN_COMMAND_JOB_OPTIONS,
|
||||
});
|
||||
const resultQueue = new Queue(DOMAIN_RESULT_QUEUE, {
|
||||
connection: options.connection,
|
||||
defaultJobOptions: {
|
||||
attempts: 5,
|
||||
backoff: { type: "exponential", delay: 1_000 },
|
||||
removeOnComplete: { age: 7 * 24 * 60 * 60, count: 50_000 },
|
||||
removeOnFail: { age: 30 * 24 * 60 * 60, count: 100_000 },
|
||||
},
|
||||
});
|
||||
const publishResult = async (result: SignedDomainEnvelope<DomainResultPayload>) => {
|
||||
const jobId = sha256Hex(`${result.contentHash}:${result.payload.attempt}:${result.payload.status}`);
|
||||
await resultQueue.add("portal-domain-result", result, { jobId });
|
||||
};
|
||||
const processor = createDomainCommandProcessor({ ...options, resultPublisher: publishResult });
|
||||
let workerReady = false;
|
||||
const worker = new Worker(
|
||||
DOMAIN_COMMAND_QUEUE,
|
||||
async (job) => processor(job),
|
||||
{
|
||||
connection: options.connection,
|
||||
concurrency: options.concurrency,
|
||||
settings: {
|
||||
backoffStrategy: (attemptsMade, type, _error) =>
|
||||
type === DOMAIN_BACKOFF_TYPE ? retryDelayForAttempt(attemptsMade) : -1,
|
||||
},
|
||||
}
|
||||
);
|
||||
worker.on("ready", () => { workerReady = true; });
|
||||
worker.on("error", (error) => {
|
||||
workerReady = false;
|
||||
options.logger.error({ err: error }, "portal-domain queue worker error");
|
||||
});
|
||||
worker.on("closed", () => { workerReady = false; });
|
||||
await Promise.all([commandQueue.waitUntilReady(), resultQueue.waitUntilReady(), worker.waitUntilReady()]);
|
||||
workerReady = true;
|
||||
|
||||
return {
|
||||
commandQueue,
|
||||
resultQueue,
|
||||
worker,
|
||||
async isReady() {
|
||||
if (!workerReady || worker.closing) return false;
|
||||
try {
|
||||
const [commandClient, resultClient] = await Promise.all([commandQueue.client, resultQueue.client]);
|
||||
const replies = await Promise.all([commandClient.ping(), resultClient.ping()]);
|
||||
return replies.every((reply) => reply === "PONG");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
async close() {
|
||||
workerReady = false;
|
||||
await worker.close();
|
||||
await Promise.all([commandQueue.close(), resultQueue.close()]);
|
||||
},
|
||||
};
|
||||
}
|
||||
+527
@@ -0,0 +1,527 @@
|
||||
import { link, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, symlink, unlink, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { ClaimLockLostError, type ClaimLockGuard, type ClaimMutationLock } from "./claim-lock.js";
|
||||
import { PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME } from "./config.js";
|
||||
import { sha256Hex, type DomainCommandPayload } from "./contracts.js";
|
||||
import {
|
||||
GenerationConflictError,
|
||||
PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY,
|
||||
portalDomainTombstoneRouterPriority,
|
||||
renderTraefikDeprovisionedConfig,
|
||||
renderTraefikDomainConfig,
|
||||
TraefikConfigWriter,
|
||||
UnsafeConfigPathError,
|
||||
type TraefikRoutingConfig,
|
||||
} from "./traefik.js";
|
||||
|
||||
const directories: string[] = [];
|
||||
const routing = {
|
||||
entryPoint: "https",
|
||||
backendService: "https-0-joyyh5b89vpfp8ocri8xr96m@docker",
|
||||
} as const;
|
||||
const CONFIGURATION_MUTATION_LOCK_ID = "__portal_domain_configuration_mutation__";
|
||||
const baseCommand: DomainCommandPayload = {
|
||||
version: 1,
|
||||
claimId: "claim_12345678",
|
||||
tenantId: "tenant_12345678",
|
||||
generation: 1,
|
||||
hostname: "portal.example.com",
|
||||
action: "provision",
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("Traefik portal-domain writer", () => {
|
||||
it("renders one exact Host rule against the configured Docker service without injectable YAML", () => {
|
||||
const yaml = renderTraefikDomainConfig(baseCommand, routing);
|
||||
expect(yaml).toContain('rule: "Host(`portal.example.com`)"');
|
||||
expect(yaml).toContain('- "https"');
|
||||
expect(yaml).toContain('service: "https-0-joyyh5b89vpfp8ocri8xr96m@docker"');
|
||||
expect(yaml).toContain(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`);
|
||||
expect(yaml).toContain("certResolver: letsencrypt");
|
||||
expect(yaml).not.toContain("loadBalancer");
|
||||
expect(yaml).not.toContain("HostRegexp");
|
||||
expect(yaml).not.toContain("*.example.com");
|
||||
expect(yaml.match(/Host\(/gu)).toHaveLength(1);
|
||||
expect(() => renderTraefikDomainConfig(baseCommand, { ...routing, entryPoint: "https\nhttp:" })).toThrow("entry-point");
|
||||
expect(() => renderTraefikDomainConfig(baseCommand, { ...routing, backendService: "portal@docker\n routers:" })).toThrow("Docker service");
|
||||
|
||||
const tombstoneCommand: DomainCommandPayload = {
|
||||
...baseCommand,
|
||||
tenantId: "tenant-do-not-render",
|
||||
action: "deprovision",
|
||||
};
|
||||
const tombstone = renderTraefikDeprovisionedConfig(tombstoneCommand, routing);
|
||||
const exactHostRule = "Host(`portal.example.com`)";
|
||||
const tombstonePriority = portalDomainTombstoneRouterPriority(baseCommand.hostname);
|
||||
expect(tombstone).toContain('rule: "Host(`portal.example.com`)"');
|
||||
expect(tombstone).toContain(`priority: ${tombstonePriority}`);
|
||||
expect(tombstone).toContain('service: "noop@internal"');
|
||||
expect(tombstone).toContain('"418": 404');
|
||||
expect(tombstone).toContain("tls: {}");
|
||||
expect(tombstone).not.toContain("certResolver");
|
||||
expect(tombstone).not.toContain(routing.backendService);
|
||||
expect(tombstone).not.toContain("tenant-do-not-render");
|
||||
expect(tombstonePriority).toBe(Math.max(1, Buffer.byteLength(exactHostRule, "utf8") - 1));
|
||||
expect(Buffer.byteLength(exactHostRule, "utf8")).toBeGreaterThan(tombstonePriority);
|
||||
expect(PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY).toBeGreaterThan(tombstonePriority);
|
||||
});
|
||||
|
||||
it("atomically applies, repairs and deprovisions a generation", async () => {
|
||||
const layout = await tempLayout();
|
||||
const writer = writerFor(layout);
|
||||
await writer.assertReady();
|
||||
const first = await writer.applyProvision(baseCommand);
|
||||
expect(first.disposition).toBe("applied");
|
||||
const files = await readdir(layout.configDirectory);
|
||||
const yamlName = files.find((name) => name.endsWith(".yaml"));
|
||||
expect(yamlName).toBeDefined();
|
||||
expect(files.some((name) => name.startsWith(".tmp-"))).toBe(false);
|
||||
const yamlPath = path.join(layout.configDirectory, yamlName!);
|
||||
expect((await stat(yamlPath)).mode & 0o777).toBe(0o600);
|
||||
expect(first.configHash).toBe(sha256Hex(await readFile(yamlPath, "utf8")));
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).toMatch(/^reload [0-9a-f-]+\n$/u);
|
||||
expect((await readdir(layout.root)).filter((name) => name.endsWith(".yaml"))).toEqual([]);
|
||||
|
||||
await writeFile(yamlPath, "tampered\n", "utf8");
|
||||
const beforeRepair = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
expect((await writer.applyProvision(baseCommand)).disposition).toBe("unchanged");
|
||||
expect(await readFile(yamlPath, "utf8")).toContain("Host(`portal.example.com`)");
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRepair);
|
||||
|
||||
const remove = await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" });
|
||||
expect(remove.disposition).toBe("applied");
|
||||
expect(await readFile(yamlPath, "utf8")).toContain('service: "noop@internal"');
|
||||
expect(await readFile(yamlPath, "utf8")).toContain('"418": 404');
|
||||
expect(await readFile(yamlPath, "utf8")).not.toContain(routing.backendService);
|
||||
expect(await readFile(yamlPath, "utf8")).not.toContain("certResolver");
|
||||
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ generation: 2, action: "deprovision" });
|
||||
|
||||
await writeFile(yamlPath, "reappeared after deprovision\n", "utf8");
|
||||
const beforeDeprovisionRepair = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
expect((await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" })).disposition).toBe("unchanged");
|
||||
expect(await readFile(yamlPath, "utf8")).toContain('service: "noop@internal"');
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeDeprovisionRepair);
|
||||
const afterDeprovisionRepair = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
expect((await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" })).disposition).toBe("unchanged");
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(afterDeprovisionRepair);
|
||||
});
|
||||
|
||||
it("retries the reload before acknowledging a same-generation config repair", async () => {
|
||||
const layout = await tempLayout();
|
||||
const writer = writerFor(layout);
|
||||
await writer.applyProvision(baseCommand);
|
||||
const yamlPath = configPathForClaim(layout, baseCommand.claimId);
|
||||
await writeFile(yamlPath, "drifted before replay\n", "utf8");
|
||||
const secondSentinelLink = path.join(layout.root, "blocked-reload-link");
|
||||
await link(layout.reloadTriggerFile, secondSentinelLink);
|
||||
|
||||
await expect(writer.applyProvision(baseCommand)).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
expect(await readFile(yamlPath, "utf8")).toBe(renderTraefikDomainConfig(baseCommand, routing));
|
||||
|
||||
await unlink(secondSentinelLink);
|
||||
const beforeRetry = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
await expect(writer.applyProvision(baseCommand)).resolves.toMatchObject({ disposition: "unchanged" });
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRetry);
|
||||
});
|
||||
|
||||
it("rejects stale and conflicting generations while permitting idempotent replay", async () => {
|
||||
const writer = writerFor(await tempLayout());
|
||||
await writer.applyProvision({ ...baseCommand, generation: 4 });
|
||||
expect((await writer.applyProvision({ ...baseCommand, generation: 3 })).disposition).toBe("stale");
|
||||
expect((await writer.applyProvision({ ...baseCommand, generation: 4 })).disposition).toBe("unchanged");
|
||||
await expect(writer.applyDeprovision({ ...baseCommand, generation: 4, action: "deprovision" })).rejects.toBeInstanceOf(GenerationConflictError);
|
||||
await expect(writer.applyProvision({ ...baseCommand, generation: 5, hostname: "other.example.com" })).rejects.toBeInstanceOf(GenerationConflictError);
|
||||
await expect(writer.applyProvision({ ...baseCommand, generation: 5, tenantId: "tenant_87654321" })).rejects.toBeInstanceOf(GenerationConflictError);
|
||||
});
|
||||
|
||||
it("repairs backend drift during same-generation revalidation without replacing provision state", async () => {
|
||||
const layout = await tempLayout();
|
||||
const coordinator = new TestClaimLockCoordinator();
|
||||
const oldBackend = "portal-backend-before-recreate@docker";
|
||||
const currentBackend = "portal-backend-after-recreate@docker";
|
||||
const initialWriter = writerFor(layout, coordinator.client(), { ...routing, backendService: oldBackend });
|
||||
await initialWriter.applyProvision(baseCommand);
|
||||
const yamlName = (await readdir(layout.configDirectory)).find((name) => name.endsWith(".yaml"));
|
||||
expect(yamlName).toBeDefined();
|
||||
const yamlPath = path.join(layout.configDirectory, yamlName!);
|
||||
expect(await readFile(yamlPath, "utf8")).toContain(`service: "${oldBackend}"`);
|
||||
const beforeRepair = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
|
||||
const replacementWriter = writerFor(layout, coordinator.client(), { ...routing, backendService: currentBackend });
|
||||
const result = await replacementWriter.recordRevalidation({ ...baseCommand, action: "revalidate" });
|
||||
|
||||
expect(result.disposition).toBe("unchanged");
|
||||
expect(await readFile(yamlPath, "utf8")).toContain(`service: "${currentBackend}"`);
|
||||
expect(await readFile(yamlPath, "utf8")).not.toContain(oldBackend);
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRepair);
|
||||
expect(await replacementWriter.readAppliedState(baseCommand.claimId)).toMatchObject({
|
||||
generation: baseCommand.generation,
|
||||
action: "provision",
|
||||
});
|
||||
});
|
||||
|
||||
it("serializes two writer instances for one claim so the highest generation wins", async () => {
|
||||
const layout = await tempLayout();
|
||||
const coordinator = new TestClaimLockCoordinator();
|
||||
const firstWriter = writerFor(layout, coordinator.client());
|
||||
const secondWriter = writerFor(layout, coordinator.client());
|
||||
const results = await Promise.all([
|
||||
firstWriter.applyProvision(baseCommand),
|
||||
secondWriter.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" }),
|
||||
]);
|
||||
expect(results.map((result) => result.disposition)).toContain("applied");
|
||||
expect(coordinator.maxConcurrentFor(baseCommand.claimId)).toBe(1);
|
||||
expect(await firstWriter.readAppliedState(baseCommand.claimId)).toMatchObject({ generation: 2, action: "deprovision" });
|
||||
const yamlName = (await readdir(layout.configDirectory)).find((name) => name.endsWith(".yaml"));
|
||||
expect(yamlName).toBeDefined();
|
||||
expect(await readFile(path.join(layout.configDirectory, yamlName!), "utf8")).toContain('service: "noop@internal"');
|
||||
});
|
||||
|
||||
it("repairs a legacy deprovision state before queue startup without replaying a command", async () => {
|
||||
const layout = await tempLayout();
|
||||
const writer = writerFor(layout);
|
||||
await writer.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" });
|
||||
const yamlName = (await readdir(layout.configDirectory)).find((name) => name.endsWith(".yaml"));
|
||||
expect(yamlName).toBeDefined();
|
||||
const yamlPath = path.join(layout.configDirectory, yamlName!);
|
||||
await unlink(yamlPath);
|
||||
const beforeRepair = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
|
||||
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
|
||||
expect(await readFile(yamlPath, "utf8")).toContain('"418": 404');
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeRepair);
|
||||
|
||||
const afterRepair = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(0);
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(afterRepair);
|
||||
});
|
||||
|
||||
it("never reopens an active state whose config disappeared or became unknown", async () => {
|
||||
const layout = await tempLayout();
|
||||
const writer = writerFor(layout);
|
||||
await writer.applyProvision(baseCommand);
|
||||
const yamlPath = configPathForClaim(layout, baseCommand.claimId);
|
||||
await unlink(yamlPath);
|
||||
|
||||
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
|
||||
const afterCrashWindow = await readFile(yamlPath, "utf8");
|
||||
expect(afterCrashWindow).toContain('service: "noop@internal"');
|
||||
expect(afterCrashWindow).toContain('"418": 404');
|
||||
expect(afterCrashWindow).not.toContain(routing.backendService);
|
||||
expect(afterCrashWindow).not.toContain("certResolver");
|
||||
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ action: "provision" });
|
||||
|
||||
await writeFile(yamlPath, "unknown legacy contents\n", "utf8");
|
||||
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
|
||||
expect(await readFile(yamlPath, "utf8")).toBe(afterCrashWindow);
|
||||
|
||||
const beforeTombstoneReplay = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(0);
|
||||
expect(await readFile(yamlPath, "utf8")).toBe(afterCrashWindow);
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeTombstoneReplay);
|
||||
});
|
||||
|
||||
it("upgrades a known legacy revalidation route without trusting state alone", async () => {
|
||||
const layout = await tempLayout();
|
||||
const writer = writerFor(layout);
|
||||
await writer.recordRevalidation({ ...baseCommand, action: "revalidate" });
|
||||
const yamlPath = configPathForClaim(layout, baseCommand.claimId);
|
||||
const legacyActive = (await readFile(yamlPath, "utf8")).replace(` priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}\n`, "");
|
||||
await writeFile(yamlPath, legacyActive, "utf8");
|
||||
|
||||
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(1);
|
||||
const upgraded = await readFile(yamlPath, "utf8");
|
||||
expect(upgraded).toContain(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`);
|
||||
expect(upgraded).toContain(routing.backendService);
|
||||
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ action: "revalidate" });
|
||||
});
|
||||
|
||||
it(
|
||||
"upgrades a legacy reclaimed active route above a restored legacy deprovision in one startup pass",
|
||||
async () => {
|
||||
const layout = await tempLayout();
|
||||
const coordinator = new TestClaimLockCoordinator();
|
||||
const writer = writerFor(layout, coordinator.client());
|
||||
const retired = { ...baseCommand, generation: 2, action: "deprovision" as const };
|
||||
const reclaimed = {
|
||||
...baseCommand,
|
||||
claimId: "claim_87654321",
|
||||
tenantId: "tenant_87654321",
|
||||
generation: 1,
|
||||
action: "provision" as const,
|
||||
};
|
||||
|
||||
await writer.applyDeprovision(retired);
|
||||
await writer.applyProvision(reclaimed);
|
||||
const reclaimedPath = configPathForClaim(layout, reclaimed.claimId);
|
||||
const retiredPath = configPathForClaim(layout, retired.claimId);
|
||||
const legacyActive = (await readFile(reclaimedPath, "utf8")).replace(` priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}\n`, "");
|
||||
await writeFile(reclaimedPath, legacyActive, "utf8");
|
||||
await unlink(retiredPath);
|
||||
coordinator.resetObservations();
|
||||
const beforeReconcile = await readFile(layout.reloadTriggerFile, "utf8");
|
||||
const tombstonePriority = portalDomainTombstoneRouterPriority(baseCommand.hostname);
|
||||
const legacyActivePriority = Buffer.byteLength(`Host(\`${baseCommand.hostname}\`)`, "utf8");
|
||||
|
||||
await expect(writer.reconcilePersistedDomainConfigs()).resolves.toBe(2);
|
||||
const yamls = await Promise.all(
|
||||
(await readdir(layout.configDirectory))
|
||||
.filter((name) => name.endsWith(".yaml"))
|
||||
.map((name) => readFile(path.join(layout.configDirectory, name), "utf8"))
|
||||
);
|
||||
expect(yamls).toHaveLength(2);
|
||||
expect(yamls.filter((yaml) => yaml.includes(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`))).toHaveLength(1);
|
||||
expect(yamls.filter((yaml) => yaml.includes(`priority: ${tombstonePriority}`))).toHaveLength(1);
|
||||
expect(yamls.find((yaml) => yaml.includes(`priority: ${PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY}`))).toContain(routing.backendService);
|
||||
expect(yamls.find((yaml) => yaml.includes(`priority: ${tombstonePriority}`))).not.toContain(routing.backendService);
|
||||
expect(legacyActivePriority).toBeGreaterThan(tombstonePriority);
|
||||
expect(PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY).toBeGreaterThan(tombstonePriority);
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).not.toBe(beforeReconcile);
|
||||
expect(coordinator.acquisitionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(1);
|
||||
expect(coordinator.guardAssertionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(3);
|
||||
},
|
||||
60_000
|
||||
);
|
||||
|
||||
it("serializes configuration-wide mutations while different claim locks remain fenced", async () => {
|
||||
const layout = await tempLayout();
|
||||
const coordinator = new TestClaimLockCoordinator();
|
||||
const retiredWriter = writerFor(layout, coordinator.client());
|
||||
const reclaimedWriter = writerFor(layout, coordinator.client());
|
||||
|
||||
await Promise.all([
|
||||
retiredWriter.applyDeprovision({ ...baseCommand, generation: 2, action: "deprovision" }),
|
||||
reclaimedWriter.applyProvision({
|
||||
...baseCommand,
|
||||
claimId: "claim_87654321",
|
||||
tenantId: "tenant_87654321",
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(coordinator.acquisitionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(2);
|
||||
expect(coordinator.contentionsFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(1);
|
||||
expect(coordinator.maxConcurrentFor(CONFIGURATION_MUTATION_LOCK_ID)).toBe(1);
|
||||
expect(coordinator.acquisitionsFor(baseCommand.claimId)).toBe(1);
|
||||
expect(coordinator.acquisitionsFor("claim_87654321")).toBe(1);
|
||||
expect((await readdir(layout.configDirectory)).filter((name) => name.endsWith(".yaml"))).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("fails closed when startup reconciliation sees malformed or symlinked generation state", async () => {
|
||||
const malformedLayout = await tempLayout();
|
||||
const malformedPath = path.join(malformedLayout.configDirectory, `.portal-${"a".repeat(32)}.generation`);
|
||||
await writeFile(malformedPath, "not-json\n", "utf8");
|
||||
await expect(writerFor(malformedLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
|
||||
const symlinkLayout = await tempLayout();
|
||||
const outsidePath = path.join(symlinkLayout.root, "outside-generation-state");
|
||||
await writeFile(outsidePath, `${JSON.stringify({ generation: 2, action: "deprovision", hostname: baseCommand.hostname, tenantId: baseCommand.tenantId })}\n`);
|
||||
await symlink(outsidePath, path.join(symlinkLayout.configDirectory, `.portal-${"b".repeat(32)}.generation`));
|
||||
await expect(writerFor(symlinkLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
|
||||
const malformedNameLayout = await tempLayout();
|
||||
await writeFile(path.join(malformedNameLayout.configDirectory, ".portal-not-a-key.generation"), "{}\n", "utf8");
|
||||
await expect(writerFor(malformedNameLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
});
|
||||
|
||||
it("rejects hardlinked generation state and active config during startup reconciliation", async () => {
|
||||
const stateLayout = await tempLayout();
|
||||
const stateWriter = writerFor(stateLayout);
|
||||
await stateWriter.applyProvision(baseCommand);
|
||||
await link(statePathForClaim(stateLayout, baseCommand.claimId), path.join(stateLayout.root, "linked-generation-state"));
|
||||
await expect(stateWriter.reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
|
||||
const configLayout = await tempLayout();
|
||||
const configWriter = writerFor(configLayout);
|
||||
await configWriter.applyProvision(baseCommand);
|
||||
await link(configPathForClaim(configLayout, baseCommand.claimId), path.join(configLayout.root, "linked-active-config"));
|
||||
await expect(configWriter.reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
});
|
||||
|
||||
it("bounds generation state and config reads during startup reconciliation", async () => {
|
||||
const stateLayout = await tempLayout();
|
||||
await writeFile(path.join(stateLayout.configDirectory, `.portal-${"c".repeat(32)}.generation`), "x".repeat(4_097), "utf8");
|
||||
await expect(writerFor(stateLayout).reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
|
||||
const configLayout = await tempLayout();
|
||||
const configWriter = writerFor(configLayout);
|
||||
await configWriter.applyProvision(baseCommand);
|
||||
await writeFile(configPathForClaim(configLayout, baseCommand.claimId), "x".repeat(16_385), "utf8");
|
||||
await expect(configWriter.reconcilePersistedDomainConfigs()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
});
|
||||
|
||||
it("checks distributed ownership immediately before config and state commits", async () => {
|
||||
const layout = await tempLayout();
|
||||
const lostLock: ClaimMutationLock = {
|
||||
assertReady: async () => undefined,
|
||||
close: async () => undefined,
|
||||
withClaimLock: async (_claimId, operation) =>
|
||||
operation({
|
||||
assertHeld: async () => {
|
||||
throw new ClaimLockLostError();
|
||||
},
|
||||
}),
|
||||
};
|
||||
const writer = writerFor(layout, lostLock);
|
||||
|
||||
await expect(writer.applyProvision(baseCommand)).rejects.toBeInstanceOf(ClaimLockLostError);
|
||||
expect((await readdir(layout.configDirectory)).filter((name) => name.endsWith(".yaml") || name.endsWith(".generation"))).toEqual([]);
|
||||
expect(await writer.readAppliedState(baseCommand.claimId)).toBeUndefined();
|
||||
expect(await readFile(layout.reloadTriggerFile, "utf8")).toBe("ready\n");
|
||||
});
|
||||
|
||||
it("hashes raw claim ids and never permits path traversal", async () => {
|
||||
const layout = await tempLayout();
|
||||
const writer = writerFor(layout);
|
||||
await writer.applyProvision({ ...baseCommand, claimId: "../../outside-file" });
|
||||
const entries = await readdir(layout.configDirectory);
|
||||
expect(entries.every((entry) => !entry.includes("..") && !entry.includes("outside-file"))).toBe(true);
|
||||
expect(entries.every((entry) => path.dirname(path.join(layout.configDirectory, entry)) === layout.configDirectory)).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects symlink directories and config targets", async () => {
|
||||
const layout = await tempLayout();
|
||||
const alternateRoot = await mkdtemp(path.join(os.tmpdir(), "hivegrid-domain-controller-linked-"));
|
||||
directories.push(alternateRoot);
|
||||
const alternateDirectory = path.join(alternateRoot, PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME);
|
||||
await symlink(layout.configDirectory, alternateDirectory, "dir");
|
||||
const alternateTrigger = path.join(alternateRoot, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME);
|
||||
await writeFile(alternateTrigger, "ready\n", "utf8");
|
||||
await expect(
|
||||
writerFor({ root: alternateRoot, configDirectory: alternateDirectory, reloadTriggerFile: alternateTrigger }).assertReady()
|
||||
).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
|
||||
const key = sha256Hex(baseCommand.claimId).slice(0, 32);
|
||||
const outside = path.join(layout.root, `outside-${key}`);
|
||||
await writeFile(outside, "do not replace", "utf8");
|
||||
await symlink(outside, path.join(layout.configDirectory, `portal-${key}.yaml`));
|
||||
await expect(writerFor(layout).applyProvision(baseCommand)).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
expect(await readFile(outside, "utf8")).toBe("do not replace");
|
||||
});
|
||||
|
||||
it("requires one pre-created non-linked reload sentinel and leaves generation retryable when it is unsafe", async () => {
|
||||
const symlinkLayout = await tempLayout();
|
||||
const outside = path.join(symlinkLayout.root, "outside-reload");
|
||||
await writeFile(outside, "outside\n", "utf8");
|
||||
await unlink(symlinkLayout.reloadTriggerFile);
|
||||
await symlink(outside, symlinkLayout.reloadTriggerFile);
|
||||
await expect(writerFor(symlinkLayout).assertReady()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
|
||||
const hardlinkLayout = await tempLayout();
|
||||
const secondLink = path.join(hardlinkLayout.root, "second-reload-link");
|
||||
await link(hardlinkLayout.reloadTriggerFile, secondLink);
|
||||
const writer = writerFor(hardlinkLayout);
|
||||
await expect(writer.applyProvision(baseCommand)).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
expect(await writer.readAppliedState(baseCommand.claimId)).toBeUndefined();
|
||||
await unlink(secondLink);
|
||||
await expect(writer.applyProvision(baseCommand)).resolves.toMatchObject({ disposition: "applied" });
|
||||
expect(await writer.readAppliedState(baseCommand.claimId)).toMatchObject({ generation: 1, action: "provision" });
|
||||
});
|
||||
|
||||
it("revalidates the configured directory instead of trusting a cached path", async () => {
|
||||
const layout = await tempLayout();
|
||||
const writer = writerFor(layout);
|
||||
await writer.assertReady();
|
||||
const moved = path.join(layout.root, `${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}-moved`);
|
||||
await rename(layout.configDirectory, moved);
|
||||
await symlink(moved, layout.configDirectory, "dir");
|
||||
await expect(writer.assertReady()).rejects.toBeInstanceOf(UnsafeConfigPathError);
|
||||
});
|
||||
});
|
||||
|
||||
type TempLayout = Readonly<{ root: string; configDirectory: string; reloadTriggerFile: string }>;
|
||||
|
||||
function writerFor(
|
||||
layout: TempLayout,
|
||||
claimLock: ClaimMutationLock = new TestClaimLockCoordinator().client(),
|
||||
writerRouting: TraefikRoutingConfig = routing
|
||||
): TraefikConfigWriter {
|
||||
return new TraefikConfigWriter({ ...layout, routing: writerRouting, claimLock });
|
||||
}
|
||||
|
||||
async function tempLayout(): Promise<TempLayout> {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "hivegrid-domain-controller-"));
|
||||
directories.push(root);
|
||||
const configDirectory = path.join(root, PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME);
|
||||
const reloadTriggerFile = path.join(root, PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME);
|
||||
await mkdir(configDirectory);
|
||||
await writeFile(reloadTriggerFile, "ready\n", { encoding: "utf8", mode: 0o600 });
|
||||
return { root, configDirectory, reloadTriggerFile };
|
||||
}
|
||||
|
||||
function configPathForClaim(layout: TempLayout, claimId: string): string {
|
||||
return path.join(layout.configDirectory, `portal-${sha256Hex(claimId).slice(0, 32)}.yaml`);
|
||||
}
|
||||
|
||||
function statePathForClaim(layout: TempLayout, claimId: string): string {
|
||||
return path.join(layout.configDirectory, `.portal-${sha256Hex(claimId).slice(0, 32)}.generation`);
|
||||
}
|
||||
|
||||
class TestClaimLockCoordinator {
|
||||
readonly #tails = new Map<string, Promise<void>>();
|
||||
readonly #active = new Map<string, number>();
|
||||
readonly #maxConcurrent = new Map<string, number>();
|
||||
readonly #acquisitions = new Map<string, number>();
|
||||
readonly #contentions = new Map<string, number>();
|
||||
readonly #guardAssertions = new Map<string, number>();
|
||||
|
||||
acquisitionsFor(claimId: string): number {
|
||||
return this.#acquisitions.get(claimId) ?? 0;
|
||||
}
|
||||
|
||||
contentionsFor(claimId: string): number {
|
||||
return this.#contentions.get(claimId) ?? 0;
|
||||
}
|
||||
|
||||
guardAssertionsFor(claimId: string): number {
|
||||
return this.#guardAssertions.get(claimId) ?? 0;
|
||||
}
|
||||
|
||||
maxConcurrentFor(claimId: string): number {
|
||||
return this.#maxConcurrent.get(claimId) ?? 0;
|
||||
}
|
||||
|
||||
resetObservations(): void {
|
||||
this.#maxConcurrent.clear();
|
||||
this.#acquisitions.clear();
|
||||
this.#contentions.clear();
|
||||
this.#guardAssertions.clear();
|
||||
}
|
||||
|
||||
client(): ClaimMutationLock {
|
||||
return {
|
||||
assertReady: async () => undefined,
|
||||
close: async () => undefined,
|
||||
withClaimLock: async <T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>) => {
|
||||
this.#acquisitions.set(claimId, this.acquisitionsFor(claimId) + 1);
|
||||
if (this.#tails.has(claimId)) this.#contentions.set(claimId, this.contentionsFor(claimId) + 1);
|
||||
const previous = this.#tails.get(claimId) ?? Promise.resolve();
|
||||
let release: () => void = () => undefined;
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const tail = previous.then(() => current);
|
||||
this.#tails.set(claimId, tail);
|
||||
await previous;
|
||||
const active = (this.#active.get(claimId) ?? 0) + 1;
|
||||
this.#active.set(claimId, active);
|
||||
this.#maxConcurrent.set(claimId, Math.max(this.maxConcurrentFor(claimId), active));
|
||||
try {
|
||||
return await operation({
|
||||
assertHeld: async () => {
|
||||
this.#guardAssertions.set(claimId, this.guardAssertionsFor(claimId) + 1);
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
this.#active.set(claimId, active - 1);
|
||||
release();
|
||||
if (this.#tails.get(claimId) === tail) this.#tails.delete(claimId);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
+613
@@ -0,0 +1,613 @@
|
||||
import { constants as fsConstants, type Stats } from "node:fs";
|
||||
import { lstat, open, readdir, realpath, rename, unlink } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
normalizePortalDomainBackendService,
|
||||
normalizePortalDomainEntryPoint,
|
||||
PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME,
|
||||
PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME,
|
||||
PORTAL_DOMAIN_TLS_CERT_RESOLVER,
|
||||
} from "./config.js";
|
||||
import type { ClaimLockGuard, ClaimMutationLock } from "./claim-lock.js";
|
||||
import { normalizeDomainHostname, sha256Hex, type DomainCommandAction, type DomainCommandPayload } from "./contracts.js";
|
||||
|
||||
export type AppliedDomainState = {
|
||||
generation: number;
|
||||
action: DomainCommandAction;
|
||||
hostname: string;
|
||||
tenantId: string;
|
||||
};
|
||||
|
||||
export type ConfigMutationResult = {
|
||||
disposition: "applied" | "unchanged" | "stale";
|
||||
configHash?: string;
|
||||
};
|
||||
|
||||
export type TraefikRoutingConfig = Readonly<{
|
||||
entryPoint: string;
|
||||
backendService: string;
|
||||
}>;
|
||||
|
||||
export type TraefikConfigWriterOptions = Readonly<{
|
||||
configDirectory: string;
|
||||
reloadTriggerFile: string;
|
||||
routing: TraefikRoutingConfig;
|
||||
claimLock: ClaimMutationLock;
|
||||
}>;
|
||||
|
||||
// A later post-cooldown claim uses a distinct claim file. New active routers
|
||||
// use an explicit high priority. Tombstones stay one below the exact Host rule
|
||||
// length so a rollback release using Traefik's default rule-length priority also
|
||||
// wins after reclaim, without deleting fencing state this DB-free controller
|
||||
// cannot authoritatively age.
|
||||
export const PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY = 1_000_001;
|
||||
|
||||
const CONFIGURATION_MUTATION_LOCK_ID = "__portal_domain_configuration_mutation__";
|
||||
const GENERATION_STATE_FILE_PATTERN = /^\.portal-([a-f0-9]{32})\.generation$/u;
|
||||
const MAX_GENERATION_STATE_BYTES = 4_096;
|
||||
const MAX_DOMAIN_CONFIG_BYTES = 16_384;
|
||||
|
||||
export class UnsafeConfigPathError extends Error {
|
||||
readonly retryable = false;
|
||||
readonly reasonCode = "unsafe_config_path";
|
||||
}
|
||||
|
||||
export class GenerationConflictError extends Error {
|
||||
readonly retryable = false;
|
||||
readonly reasonCode = "generation_conflict";
|
||||
}
|
||||
|
||||
export class TraefikConfigWriter {
|
||||
readonly #configuredDirectory: string;
|
||||
readonly #configuredReloadTriggerFile: string;
|
||||
readonly #routing: TraefikRoutingConfig;
|
||||
readonly #distributedClaimLock: ClaimMutationLock;
|
||||
readonly #claimLocks = new Map<string, Promise<void>>();
|
||||
#reloadTail: Promise<void> = Promise.resolve();
|
||||
|
||||
constructor(options: TraefikConfigWriterOptions) {
|
||||
if (!path.isAbsolute(options.configDirectory)) throw new UnsafeConfigPathError("Traefik config directory must be absolute");
|
||||
if (!path.isAbsolute(options.reloadTriggerFile)) throw new UnsafeConfigPathError("Traefik reload trigger file must be absolute");
|
||||
this.#configuredDirectory = path.resolve(options.configDirectory);
|
||||
this.#configuredReloadTriggerFile = path.resolve(options.reloadTriggerFile);
|
||||
if (path.basename(this.#configuredDirectory) !== PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME) {
|
||||
throw new UnsafeConfigPathError(`Traefik config directory must end with ${PORTAL_DOMAIN_CONFIG_DIRECTORY_NAME}`);
|
||||
}
|
||||
if (path.basename(this.#configuredReloadTriggerFile) !== PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME) {
|
||||
throw new UnsafeConfigPathError(`Traefik reload trigger file must end with ${PORTAL_DOMAIN_RELOAD_TRIGGER_FILE_NAME}`);
|
||||
}
|
||||
if (path.dirname(this.#configuredReloadTriggerFile) !== path.dirname(this.#configuredDirectory)) {
|
||||
throw new UnsafeConfigPathError("Traefik reload trigger file must be directly beside its dedicated config directory");
|
||||
}
|
||||
this.#routing = Object.freeze({
|
||||
entryPoint: normalizePortalDomainEntryPoint(options.routing.entryPoint),
|
||||
backendService: normalizePortalDomainBackendService(options.routing.backendService),
|
||||
});
|
||||
this.#distributedClaimLock = options.claimLock;
|
||||
}
|
||||
|
||||
async assertReady(): Promise<void> {
|
||||
await this.#distributedClaimLock.assertReady();
|
||||
const directory = await this.#resolveSafeDirectory();
|
||||
await this.#openSafeReloadTrigger(fsConstants.O_WRONLY).then((handle) => handle.close());
|
||||
const probePath = this.#containedPath(directory, `.write-probe-${randomUUID()}`);
|
||||
const handle = await open(probePath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY | noFollowFlag(), 0o600);
|
||||
try {
|
||||
await handle.writeFile("ready\n", "utf8");
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
await unlink(probePath).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async applyProvision(command: DomainCommandPayload): Promise<ConfigMutationResult> {
|
||||
const yaml = renderTraefikDomainConfig(command, this.#routing);
|
||||
return this.#applyMutation(command, yaml);
|
||||
}
|
||||
|
||||
async applyDeprovision(command: DomainCommandPayload): Promise<ConfigMutationResult> {
|
||||
return this.#applyMutation(command, renderTraefikDeprovisionedConfig(command, this.#routing));
|
||||
}
|
||||
|
||||
async recordRevalidation(command: DomainCommandPayload): Promise<ConfigMutationResult> {
|
||||
return this.#applyMutation(command, renderTraefikDomainConfig(command, this.#routing));
|
||||
}
|
||||
|
||||
async readAppliedState(claimId: string): Promise<AppliedDomainState | undefined> {
|
||||
const directory = await this.#resolveSafeDirectory();
|
||||
return readAppliedStateFile(this.#statePath(directory, claimId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-renders every durable applied state before the queue worker starts.
|
||||
* This upgrades byte-identical legacy active routes with explicit priority
|
||||
* and restores tombstones for releases that represented deprovisioning by
|
||||
* deleting YAML. Active state alone never reopens a missing or unknown route:
|
||||
* only a known current/legacy active file is trusted; every other shape is
|
||||
* replaced with the exact-host fail-closed tombstone.
|
||||
* The rollout contract drains the previous controller revision first, while
|
||||
* the configuration-wide fence prevents races with current-release commands.
|
||||
* A populated state set always triggers one reload so a crash after config
|
||||
* writes but before the prior sentinel update is repaired on the next start.
|
||||
*/
|
||||
async reconcilePersistedDomainConfigs(): Promise<number> {
|
||||
return this.#distributedClaimLock.withClaimLock(CONFIGURATION_MUTATION_LOCK_ID, (guard) =>
|
||||
this.#reconcilePersistedDomainConfigsLocked(guard)
|
||||
);
|
||||
}
|
||||
|
||||
async #applyMutation(command: DomainCommandPayload, yaml?: string): Promise<ConfigMutationResult> {
|
||||
return this.#withClaimLock(command.claimId, () =>
|
||||
this.#distributedClaimLock.withClaimLock(CONFIGURATION_MUTATION_LOCK_ID, () =>
|
||||
this.#distributedClaimLock.withClaimLock(command.claimId, (guard) => this.#applyMutationLocked(command, guard, yaml))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async #reconcilePersistedDomainConfigsLocked(guard: ClaimLockGuard): Promise<number> {
|
||||
const directory = await this.#resolveSafeDirectory();
|
||||
const entries = (await readdir(directory, { withFileTypes: true })).sort((left, right) => left.name.localeCompare(right.name));
|
||||
const states: Array<{ key: string; statePath: string }> = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
const looksLikeGenerationState = entry.name.startsWith(".portal-") && entry.name.endsWith(".generation");
|
||||
if (!looksLikeGenerationState) continue;
|
||||
const match = GENERATION_STATE_FILE_PATTERN.exec(entry.name);
|
||||
if (!match?.[1] || !entry.isFile() || entry.isSymbolicLink()) {
|
||||
throw new UnsafeConfigPathError("Stored portal-domain generation state path is invalid");
|
||||
}
|
||||
states.push({ key: match[1], statePath: this.#containedPath(directory, entry.name) });
|
||||
}
|
||||
|
||||
const persisted: Array<{ key: string; state: AppliedDomainState }> = [];
|
||||
for (const candidate of states) {
|
||||
const state = await readAppliedStateFile(candidate.statePath);
|
||||
if (!state) throw new UnsafeConfigPathError("Stored portal-domain generation state disappeared during reconciliation");
|
||||
persisted.push({ key: candidate.key, state });
|
||||
}
|
||||
|
||||
let repaired = 0;
|
||||
for (const candidate of persisted) {
|
||||
const configPath = this.#configPathForKey(directory, candidate.key);
|
||||
const tombstone = renderTraefikDeprovisionedConfigForKey(candidate.state.hostname, this.#routing.entryPoint, candidate.key);
|
||||
let expected = tombstone;
|
||||
if (candidate.state.action !== "deprovision") {
|
||||
const active = renderTraefikDomainConfigForKey(candidate.state.hostname, this.#routing, candidate.key);
|
||||
const legacyActive = renderLegacyTraefikDomainConfigForKey(candidate.state.hostname, this.#routing, candidate.key);
|
||||
if (await fileMatches(configPath, active)) continue;
|
||||
if (await fileMatches(configPath, legacyActive)) expected = active;
|
||||
}
|
||||
if (await fileMatches(configPath, expected)) continue;
|
||||
await atomicWrite(directory, configPath, expected, guard);
|
||||
repaired += 1;
|
||||
}
|
||||
|
||||
if (persisted.length > 0) await this.#triggerReload(guard);
|
||||
return repaired;
|
||||
}
|
||||
|
||||
async #applyMutationLocked(command: DomainCommandPayload, guard: ClaimLockGuard, yaml?: string): Promise<ConfigMutationResult> {
|
||||
const directory = await this.#resolveSafeDirectory();
|
||||
const existing = await this.readAppliedState(command.claimId);
|
||||
if (existing && (existing.tenantId !== command.tenantId || existing.hostname !== command.hostname)) {
|
||||
throw new GenerationConflictError("A domain claim cannot change tenant or hostname identity");
|
||||
}
|
||||
if (existing && existing.generation > command.generation) return { disposition: "stale" };
|
||||
if (
|
||||
existing &&
|
||||
existing.generation === command.generation &&
|
||||
existing.action !== command.action &&
|
||||
!(existing.action === "provision" && command.action === "revalidate")
|
||||
) {
|
||||
throw new GenerationConflictError("The same domain generation cannot carry a different action");
|
||||
}
|
||||
|
||||
const configPath = this.#configPath(directory, command.claimId);
|
||||
const configHash = typeof yaml === "string" ? sha256Hex(yaml) : undefined;
|
||||
if (existing && existing.generation === command.generation) {
|
||||
if (typeof yaml === "string") {
|
||||
if (!(await fileMatches(configPath, yaml))) await atomicWrite(directory, configPath, yaml, guard);
|
||||
await this.#triggerReload(guard);
|
||||
} else if (yaml === undefined && (await safeUnlink(configPath, guard))) {
|
||||
await syncDirectory(directory);
|
||||
await this.#triggerReload(guard);
|
||||
}
|
||||
return { disposition: "unchanged", ...(configHash ? { configHash } : {}) };
|
||||
}
|
||||
|
||||
let reloadRequired = false;
|
||||
if (typeof yaml === "string") {
|
||||
await atomicWrite(directory, configPath, yaml, guard);
|
||||
reloadRequired = true;
|
||||
} else if (yaml === undefined) {
|
||||
await safeUnlink(configPath, guard);
|
||||
await syncDirectory(directory);
|
||||
reloadRequired = true;
|
||||
}
|
||||
|
||||
if (reloadRequired) await this.#triggerReload(guard);
|
||||
const state: AppliedDomainState = {
|
||||
generation: command.generation,
|
||||
action: command.action,
|
||||
hostname: command.hostname,
|
||||
tenantId: command.tenantId,
|
||||
};
|
||||
await atomicWrite(directory, this.#statePath(directory, command.claimId), `${JSON.stringify(state)}\n`, guard);
|
||||
return { disposition: "applied", ...(configHash ? { configHash } : {}) };
|
||||
}
|
||||
|
||||
async #resolveSafeDirectory(): Promise<string> {
|
||||
const stats = await lstat(this.#configuredDirectory);
|
||||
if (!stats.isDirectory() || stats.isSymbolicLink()) {
|
||||
throw new UnsafeConfigPathError("Traefik config path must be a real directory, not a symlink");
|
||||
}
|
||||
const canonical = await realpath(this.#configuredDirectory);
|
||||
return canonical;
|
||||
}
|
||||
|
||||
/**
|
||||
* Traefik recursively reads the dedicated subdirectory after a watched-root
|
||||
* event, but does not watch nested directories themselves. Updating this
|
||||
* non-YAML sibling in place triggers that rescan without granting the
|
||||
* controller write access to the complete dynamic-config root.
|
||||
*/
|
||||
async #triggerReload(guard: ClaimLockGuard): Promise<void> {
|
||||
const previous = this.#reloadTail;
|
||||
let release: () => void = () => {};
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
this.#reloadTail = previous.then(() => current);
|
||||
await previous;
|
||||
try {
|
||||
const handle = await this.#openSafeReloadTrigger(fsConstants.O_WRONLY);
|
||||
try {
|
||||
await guard.assertHeld();
|
||||
await handle.truncate(0);
|
||||
await handle.writeFile(`reload ${randomUUID()}\n`, "utf8");
|
||||
await handle.sync();
|
||||
const [pathStats, openedStats] = await Promise.all([lstat(this.#configuredReloadTriggerFile), handle.stat()]);
|
||||
if (
|
||||
!pathStats.isFile() ||
|
||||
pathStats.isSymbolicLink() ||
|
||||
pathStats.nlink !== 1 ||
|
||||
openedStats.nlink !== 1 ||
|
||||
openedStats.dev !== pathStats.dev ||
|
||||
openedStats.ino !== pathStats.ino
|
||||
) {
|
||||
throw new UnsafeConfigPathError("Traefik reload trigger changed while it was being written");
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
async #openSafeReloadTrigger(flags: number) {
|
||||
const pathStats = await lstat(this.#configuredReloadTriggerFile);
|
||||
if (!pathStats.isFile() || pathStats.isSymbolicLink() || pathStats.nlink !== 1) {
|
||||
throw new UnsafeConfigPathError("Traefik reload trigger must be one regular, non-linked file");
|
||||
}
|
||||
const canonical = await realpath(this.#configuredReloadTriggerFile);
|
||||
const canonicalParent = await realpath(path.dirname(this.#configuredReloadTriggerFile));
|
||||
if (canonical !== path.join(canonicalParent, path.basename(this.#configuredReloadTriggerFile))) {
|
||||
throw new UnsafeConfigPathError("Traefik reload trigger path must not resolve through a link");
|
||||
}
|
||||
const handle = await open(this.#configuredReloadTriggerFile, flags | noFollowFlag());
|
||||
try {
|
||||
const openedStats = await handle.stat();
|
||||
if (!openedStats.isFile() || openedStats.nlink !== 1 || openedStats.dev !== pathStats.dev || openedStats.ino !== pathStats.ino) {
|
||||
throw new UnsafeConfigPathError("Traefik reload trigger changed while it was being opened");
|
||||
}
|
||||
return handle;
|
||||
} catch (error) {
|
||||
await handle.close().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async #withClaimLock<T>(claimId: string, operation: () => Promise<T>): Promise<T> {
|
||||
const previous = this.#claimLocks.get(claimId) ?? Promise.resolve();
|
||||
let release: () => void = () => {};
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const tail = previous.then(() => current);
|
||||
this.#claimLocks.set(claimId, tail);
|
||||
await previous;
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
if (this.#claimLocks.get(claimId) === tail) this.#claimLocks.delete(claimId);
|
||||
}
|
||||
}
|
||||
|
||||
#configPath(directory: string, claimId: string): string {
|
||||
return this.#configPathForKey(directory, claimFileKey(claimId));
|
||||
}
|
||||
|
||||
#configPathForKey(directory: string, key: string): string {
|
||||
if (!/^[a-f0-9]{32}$/u.test(key)) throw new UnsafeConfigPathError("Derived portal-domain config key is invalid");
|
||||
return this.#containedPath(directory, `portal-${key}.yaml`);
|
||||
}
|
||||
|
||||
#statePath(directory: string, claimId: string): string {
|
||||
return this.#containedPath(directory, `.portal-${claimFileKey(claimId)}.generation`);
|
||||
}
|
||||
|
||||
#containedPath(directory: string, filename: string): string {
|
||||
const candidate = path.resolve(directory, filename);
|
||||
if (path.dirname(candidate) !== directory) throw new UnsafeConfigPathError("Derived config path escaped its dedicated directory");
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
export function renderTraefikDomainConfig(command: Pick<DomainCommandPayload, "claimId" | "hostname">, routing: TraefikRoutingConfig): string {
|
||||
return renderTraefikDomainConfigForKey(command.hostname, routing, claimFileKey(command.claimId));
|
||||
}
|
||||
|
||||
function renderTraefikDomainConfigForKey(hostnameValue: string, routing: TraefikRoutingConfig, key: string): string {
|
||||
return renderTraefikActiveConfigForKey(hostnameValue, routing, key, PORTAL_DOMAIN_ACTIVE_ROUTER_PRIORITY);
|
||||
}
|
||||
|
||||
function renderLegacyTraefikDomainConfigForKey(hostnameValue: string, routing: TraefikRoutingConfig, key: string): string {
|
||||
return renderTraefikActiveConfigForKey(hostnameValue, routing, key);
|
||||
}
|
||||
|
||||
function renderTraefikActiveConfigForKey(
|
||||
hostnameValue: string,
|
||||
routing: TraefikRoutingConfig,
|
||||
key: string,
|
||||
priority?: number
|
||||
): string {
|
||||
const hostname = normalizeDomainHostname(hostnameValue);
|
||||
const entryPoint = normalizePortalDomainEntryPoint(routing.entryPoint);
|
||||
const backendService = normalizePortalDomainBackendService(routing.backendService);
|
||||
if (!/^[a-f0-9]{32}$/u.test(key)) throw new UnsafeConfigPathError("Derived portal-domain config key is invalid");
|
||||
const router = `portal-domain-${key}`;
|
||||
return [
|
||||
"# Generated by hivegrid-portal-domain-controller. Manual edits are replaced.",
|
||||
"http:",
|
||||
" routers:",
|
||||
` ${router}:`,
|
||||
` rule: ${JSON.stringify(`Host(\`${hostname}\`)`)}`,
|
||||
...(priority === undefined ? [] : [` priority: ${priority}`]),
|
||||
" entryPoints:",
|
||||
` - ${JSON.stringify(entryPoint)}`,
|
||||
` service: ${JSON.stringify(backendService)}`,
|
||||
" tls:",
|
||||
` certResolver: ${PORTAL_DOMAIN_TLS_CERT_RESOLVER}`,
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function renderTraefikDeprovisionedConfig(
|
||||
command: Pick<DomainCommandPayload, "claimId" | "hostname">,
|
||||
routing: Pick<TraefikRoutingConfig, "entryPoint">
|
||||
): string {
|
||||
return renderTraefikDeprovisionedConfigForKey(command.hostname, routing.entryPoint, claimFileKey(command.claimId));
|
||||
}
|
||||
|
||||
export function portalDomainTombstoneRouterPriority(hostnameValue: string): number {
|
||||
const hostname = normalizeDomainHostname(hostnameValue);
|
||||
const exactHostRule = `Host(\`${hostname}\`)`;
|
||||
return tombstonePriorityForExactHostRule(exactHostRule);
|
||||
}
|
||||
|
||||
function renderTraefikDeprovisionedConfigForKey(hostnameValue: string, entryPointValue: string, key: string): string {
|
||||
const hostname = normalizeDomainHostname(hostnameValue);
|
||||
const exactHostRule = `Host(\`${hostname}\`)`;
|
||||
const entryPoint = normalizePortalDomainEntryPoint(entryPointValue);
|
||||
if (!/^[a-f0-9]{32}$/u.test(key)) throw new UnsafeConfigPathError("Derived portal-domain config key is invalid");
|
||||
const router = `portal-domain-${key}`;
|
||||
const middleware = `portal-domain-tombstone-${key}`;
|
||||
return [
|
||||
"# Generated by hivegrid-portal-domain-controller. Manual edits are replaced.",
|
||||
"http:",
|
||||
" routers:",
|
||||
` ${router}:`,
|
||||
` rule: ${JSON.stringify(exactHostRule)}`,
|
||||
` priority: ${tombstonePriorityForExactHostRule(exactHostRule)}`,
|
||||
" entryPoints:",
|
||||
` - ${JSON.stringify(entryPoint)}`,
|
||||
" middlewares:",
|
||||
` - ${JSON.stringify(middleware)}`,
|
||||
' service: "noop@internal"',
|
||||
" tls: {}",
|
||||
" middlewares:",
|
||||
` ${middleware}:`,
|
||||
" errors:",
|
||||
" status:",
|
||||
' - "418"',
|
||||
" statusRewrites:",
|
||||
' "418": 404',
|
||||
' service: "noop@internal"',
|
||||
' query: "/"',
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function tombstonePriorityForExactHostRule(exactHostRule: string): number {
|
||||
return Math.max(1, Buffer.byteLength(exactHostRule, "utf8") - 1);
|
||||
}
|
||||
|
||||
function claimFileKey(claimId: string): string {
|
||||
return sha256Hex(claimId).slice(0, 32);
|
||||
}
|
||||
|
||||
async function atomicWrite(directory: string, targetPath: string, contents: string, guard: ClaimLockGuard): Promise<void> {
|
||||
await assertNotSymlink(targetPath);
|
||||
const temporaryPath = path.join(directory, `.tmp-${randomUUID()}`);
|
||||
if (path.dirname(temporaryPath) !== directory) throw new UnsafeConfigPathError("Temporary path escaped config directory");
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(temporaryPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY | noFollowFlag(), 0o600);
|
||||
await handle.writeFile(contents, "utf8");
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
await guard.assertHeld();
|
||||
await rename(temporaryPath, targetPath);
|
||||
await syncDirectory(directory);
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
await unlink(temporaryPath).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async function safeUnlink(targetPath: string, guard: ClaimLockGuard): Promise<boolean> {
|
||||
await assertNotSymlink(targetPath);
|
||||
try {
|
||||
await guard.assertHeld();
|
||||
await unlink(targetPath);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isErrno(error, "ENOENT")) return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function assertNotSymlink(targetPath: string): Promise<void> {
|
||||
try {
|
||||
const stats = await lstat(targetPath);
|
||||
if (stats.isSymbolicLink() || !stats.isFile() || stats.nlink !== 1) {
|
||||
throw new UnsafeConfigPathError("Portal-domain target must be one regular, non-linked file");
|
||||
}
|
||||
} catch (error) {
|
||||
if (!isErrno(error, "ENOENT")) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function fileMatches(targetPath: string, expected: string): Promise<boolean> {
|
||||
const contents = await readBoundedRegularFile(targetPath, MAX_DOMAIN_CONFIG_BYTES, "Portal-domain config");
|
||||
return contents === expected;
|
||||
}
|
||||
|
||||
async function readAppliedStateFile(statePath: string): Promise<AppliedDomainState | undefined> {
|
||||
const raw = await readBoundedRegularFile(statePath, MAX_GENERATION_STATE_BYTES, "Stored portal-domain generation state");
|
||||
if (raw === undefined) return undefined;
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new UnsafeConfigPathError("Stored portal-domain generation state is malformed");
|
||||
}
|
||||
if (!isAppliedState(parsed)) throw new UnsafeConfigPathError("Stored portal-domain generation state is invalid");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function readBoundedRegularFile(targetPath: string, maxBytes: number, label: string): Promise<string | undefined> {
|
||||
let pathBefore: Stats;
|
||||
try {
|
||||
pathBefore = await lstat(targetPath);
|
||||
} catch (error) {
|
||||
if (isErrno(error, "ENOENT")) return undefined;
|
||||
throw error;
|
||||
}
|
||||
assertSafeRegularFile(pathBefore, maxBytes, `${label} path is invalid`);
|
||||
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(targetPath, fsConstants.O_RDONLY | noFollowFlag());
|
||||
} catch (error) {
|
||||
if (isErrno(error, "ENOENT")) throw new UnsafeConfigPathError(`${label} changed while it was being opened`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
try {
|
||||
const openedBefore = await handle.stat();
|
||||
assertStableRegularFile(pathBefore, openedBefore, maxBytes, `${label} changed while it was being opened`);
|
||||
|
||||
const buffer = Buffer.alloc(maxBytes + 1);
|
||||
let bytesRead = 0;
|
||||
while (bytesRead < buffer.length) {
|
||||
const chunk = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead);
|
||||
if (chunk.bytesRead === 0) break;
|
||||
bytesRead += chunk.bytesRead;
|
||||
}
|
||||
if (bytesRead > maxBytes) throw new UnsafeConfigPathError(`${label} exceeds its size limit`);
|
||||
|
||||
const openedAfter = await handle.stat();
|
||||
let pathAfter: Stats;
|
||||
try {
|
||||
pathAfter = await lstat(targetPath);
|
||||
} catch (error) {
|
||||
if (isErrno(error, "ENOENT")) throw new UnsafeConfigPathError(`${label} changed while it was being read`);
|
||||
throw error;
|
||||
}
|
||||
assertStableRegularFile(pathBefore, openedAfter, maxBytes, `${label} changed while it was being read`);
|
||||
assertStableRegularFile(openedAfter, pathAfter, maxBytes, `${label} changed while it was being read`);
|
||||
if (bytesRead !== openedAfter.size) {
|
||||
throw new UnsafeConfigPathError(`${label} changed while it was being read`);
|
||||
}
|
||||
return buffer.subarray(0, bytesRead).toString("utf8");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
function assertSafeRegularFile(stats: Stats, maxBytes: number, message: string): void {
|
||||
if (!stats.isFile() || stats.isSymbolicLink() || stats.nlink !== 1 || stats.size > maxBytes) {
|
||||
throw new UnsafeConfigPathError(message);
|
||||
}
|
||||
}
|
||||
|
||||
function assertStableRegularFile(before: Stats, after: Stats, maxBytes: number, message: string): void {
|
||||
assertSafeRegularFile(before, maxBytes, message);
|
||||
assertSafeRegularFile(after, maxBytes, message);
|
||||
if (
|
||||
before.dev !== after.dev ||
|
||||
before.ino !== after.ino ||
|
||||
before.size !== after.size ||
|
||||
before.mtimeMs !== after.mtimeMs ||
|
||||
before.ctimeMs !== after.ctimeMs
|
||||
) {
|
||||
throw new UnsafeConfigPathError(message);
|
||||
}
|
||||
}
|
||||
|
||||
async function syncDirectory(directory: string): Promise<void> {
|
||||
const handle = await open(directory, fsConstants.O_RDONLY | directoryFlag() | noFollowFlag());
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
function noFollowFlag(): number {
|
||||
return fsConstants.O_NOFOLLOW ?? 0;
|
||||
}
|
||||
|
||||
function directoryFlag(): number {
|
||||
return fsConstants.O_DIRECTORY ?? 0;
|
||||
}
|
||||
|
||||
function isAppliedState(value: unknown): value is AppliedDomainState {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
||||
const record = value as Record<string, unknown>;
|
||||
try {
|
||||
return (
|
||||
Object.keys(record).sort().join(",") === "action,generation,hostname,tenantId" &&
|
||||
Number.isSafeInteger(record.generation) &&
|
||||
(record.generation as number) >= 1 &&
|
||||
(record.action === "provision" || record.action === "revalidate" || record.action === "deprovision") &&
|
||||
typeof record.hostname === "string" &&
|
||||
normalizeDomainHostname(record.hostname) === record.hostname &&
|
||||
typeof record.tenantId === "string" &&
|
||||
/^[A-Za-z0-9][A-Za-z0-9_-]{7,127}$/u.test(record.tenantId)
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isErrno(error: unknown, code: string): boolean {
|
||||
return typeof error === "object" && error !== null && "code" in error && (error as NodeJS.ErrnoException).code === code;
|
||||
}
|
||||
Reference in new issue
Block a user