import { randomUUID } from "node:crypto"; import { spawn } from "node:child_process"; import { constants as fsConstants } from "node:fs"; import { lstat, open, realpath, type FileHandle } from "node:fs/promises"; import path from "node:path"; import type { ConnectionOptions } from "@hivegrid/queue"; import { Redis } from "ioredis"; import { sha256Hex } from "./contracts.js"; const LOCK_KEY_PREFIX = "hivegrid:portal-domain-controller:claim-lock:"; const DEFAULT_LEASE_MS = 30_000; const DEFAULT_ACQUIRE_TIMEOUT_MS = 10_000; const DEFAULT_RETRY_DELAY_MS = 75; const DEFAULT_RETRY_JITTER_MS = 75; const DEFAULT_COMMAND_TIMEOUT_MS = 2_500; const DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS = 10_000; const FLOCK_BINARY = "/usr/bin/flock"; const RENEW_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('PEXPIRE', KEYS[1], ARGV[2])", "end", "return 0"].join("\n"); const RELEASE_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('DEL', KEYS[1])", "end", "return 0"].join("\n"); export type ClaimMutationLock = { assertReady(): Promise; withClaimLock(claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise; close(): Promise; }; export type ClaimLockGuard = Readonly<{ assertHeld(): Promise; }>; export type ClaimLockRedisClient = { readonly status: string; connect(): Promise; ping(): Promise; set(key: string, value: string, expiryMode: "PX", leaseMs: number, condition: "NX"): Promise<"OK" | null>; eval(script: string, numberOfKeys: number, ...args: Array): Promise; quit(): Promise; disconnect(reconnect?: boolean): void; on(event: "error", listener: (error: unknown) => void): unknown; }; type ClaimLockOptions = Readonly<{ leaseMs?: number; acquireTimeoutMs?: number; retryDelayMs?: number; retryJitterMs?: number; heartbeatIntervalMs?: number; now?: () => number; random?: () => number; token?: () => string; wait?: (delayMs: number) => Promise; }>; type NormalizedClaimLockOptions = Readonly<{ leaseMs: number; acquireTimeoutMs: number; retryDelayMs: number; retryJitterMs: number; heartbeatIntervalMs: number; now: () => number; random: () => number; token: () => string; wait: (delayMs: number) => Promise; }>; export class ClaimLockUnavailableError extends Error { readonly retryable = true; readonly reasonCode = "claim_lock_unavailable"; constructor() { super("Portal-domain claim lock is temporarily unavailable"); this.name = "ClaimLockUnavailableError"; } } export class ClaimLockLostError extends Error { readonly retryable = true; readonly reasonCode = "claim_lock_lost"; constructor() { super("Portal-domain claim lock ownership was lost"); this.name = "ClaimLockLostError"; } } /** * Kernel-backed ownership for the shared Traefik directory. Redis remains the * fast distributed coordinator, while this non-expiring `flock` is the * authoritative fence around filesystem mutations. The lock is attached to * the parent's open-file description, so a paused Node event loop cannot let a * second controller enter after a Redis lease expires. A crash closes the fd * and lets the kernel release ownership automatically. */ export class FilesystemClaimMutationLock implements ClaimMutationLock { readonly #configuredDirectory: string; readonly #binary: string; readonly #acquireTimeoutMs: number; readonly #activeOperations = new Set>(); #closing = false; #closed = false; #closePromise: Promise | undefined; constructor(directory: string, options: { binary?: string; acquireTimeoutMs?: number } = {}) { if (!path.isAbsolute(directory)) throw new ClaimLockUnavailableError(); this.#configuredDirectory = path.resolve(directory); this.#binary = options.binary ?? FLOCK_BINARY; this.#acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS, "filesystem claim lock timeout"); } async assertReady(): Promise { await this.withClaimLock("__readiness__", async (guard) => guard.assertHeld()); } async withClaimLock(claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise { if (this.#closing || this.#closed) throw new ClaimLockUnavailableError(); let finishActive: () => void = () => undefined; const active = new Promise((resolve) => { finishActive = resolve; }); this.#activeOperations.add(active); try { return await this.#withClaimLock(claimId, operation); } finally { finishActive(); this.#activeOperations.delete(active); } } close(): Promise { this.#closePromise ??= this.#close(); return this.#closePromise; } async #close(): Promise { if (this.#closed) return; this.#closing = true; await Promise.all([...this.#activeOperations]); this.#closed = true; } async #withClaimLock(claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise { const directory = await this.#resolveSafeDirectory(); const lockPath = path.join(directory, `.portal-${sha256Hex(claimId)}.lock`); if (path.dirname(lockPath) !== directory) throw new ClaimLockUnavailableError(); let handle: FileHandle | undefined; let held = false; try { handle = await open( lockPath, fsConstants.O_CREAT | fsConstants.O_RDWR | noFollowFlag(), 0o600 ); await assertSameRegularFile(lockPath, handle); await acquireKernelFlock(handle.fd, this.#binary, this.#acquireTimeoutMs); held = true; const guard: ClaimLockGuard = Object.freeze({ assertHeld: async () => { if (!held || !handle) throw new ClaimLockLostError(); await assertSameRegularFile(lockPath, handle); }, }); return await operation(guard); } catch (error) { if (error instanceof ClaimLockLostError || error instanceof ClaimLockUnavailableError) throw error; throw new ClaimLockUnavailableError(); } finally { held = false; await handle?.close().catch(() => undefined); } } async #resolveSafeDirectory(): Promise { try { const stats = await lstat(this.#configuredDirectory); if (!stats.isDirectory() || stats.isSymbolicLink()) throw new ClaimLockUnavailableError(); return await realpath(this.#configuredDirectory); } catch (error) { if (error instanceof ClaimLockUnavailableError) throw error; throw new ClaimLockUnavailableError(); } } } export class FencedClaimMutationLock implements ClaimMutationLock { readonly #filesystem: ClaimMutationLock; readonly #lease: ClaimMutationLock; constructor(filesystem: ClaimMutationLock, lease: ClaimMutationLock) { this.#filesystem = filesystem; this.#lease = lease; } async assertReady(): Promise { await this.#filesystem.assertReady(); await this.#lease.assertReady(); } withClaimLock(claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise { return this.#filesystem.withClaimLock(claimId, () => this.#lease.withClaimLock(claimId, operation)); } async close(): Promise { await Promise.all([this.#lease.close(), this.#filesystem.close()]); } } export class RedisClaimMutationLock implements ClaimMutationLock { readonly #client: ClaimLockRedisClient; readonly #options: NormalizedClaimLockOptions; readonly #activeOperations = new Set>(); #closing = false; #closed = false; #closePromise: Promise | undefined; constructor(client: ClaimLockRedisClient, options: ClaimLockOptions = {}) { this.#client = client; this.#options = normalizeOptions(options); // Redis connection errors are surfaced through bounded commands and the // readiness endpoint. Keeping this listener intentionally silent also // prevents connection details from reaching stderr outside our logger. this.#client.on("error", () => undefined); } async assertReady(): Promise { if (this.#closing || this.#closed) throw new ClaimLockUnavailableError(); try { if (this.#client.status === "wait") await this.#client.connect(); if ((await this.#client.ping()) !== "PONG") throw new ClaimLockUnavailableError(); await this.#probeLockCapabilities(); } catch { throw new ClaimLockUnavailableError(); } } async withClaimLock(claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise { if (this.#closing || this.#closed) throw new ClaimLockUnavailableError(); let finishActive: () => void = () => undefined; const active = new Promise((resolve) => { finishActive = resolve; }); this.#activeOperations.add(active); try { return await this.#withClaimLock(claimId, operation); } finally { finishActive(); this.#activeOperations.delete(active); } } close(): Promise { this.#closePromise ??= this.#close(); return this.#closePromise; } async #close(): Promise { if (this.#closed) return; this.#closing = true; await Promise.all([...this.#activeOperations]); try { if (this.#client.status !== "wait" && this.#client.status !== "end") { await this.#client.quit(); } } catch { // An immediate disconnect is safe after all leases were released. It // also keeps shutdown bounded when Redis disappears during a rollout. } finally { this.#client.disconnect(false); this.#closed = true; } } async #withClaimLock(claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise { const key = `${LOCK_KEY_PREFIX}${sha256Hex(claimId)}`; const token = this.#options.token(); await this.#acquire(key, token); let heartbeatTimer: NodeJS.Timeout | undefined; let heartbeatInFlight: Promise = Promise.resolve(); let heartbeatError: ClaimLockLostError | ClaimLockUnavailableError | undefined; let heartbeatStopped = false; const scheduleHeartbeat = () => { if (heartbeatStopped || heartbeatError) return; heartbeatTimer = setTimeout(() => { heartbeatInFlight = this.#renew(key, token) .then(() => scheduleHeartbeat()) .catch((error: unknown) => { heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError(); }); }, this.#options.heartbeatIntervalMs); }; scheduleHeartbeat(); const guard: ClaimLockGuard = Object.freeze({ assertHeld: async () => { if (heartbeatError) throw heartbeatError; try { await this.#renew(key, token); } catch (error) { heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError(); throw heartbeatError; } if (heartbeatError) throw heartbeatError; }, }); let result: T | undefined; let operationError: unknown; let operationFailed = false; try { result = await operation(guard); } catch (error) { operationFailed = true; operationError = error; } heartbeatStopped = true; if (heartbeatTimer) clearTimeout(heartbeatTimer); await heartbeatInFlight.catch(() => undefined); const observedHeartbeatError = heartbeatError; let releaseError: ClaimLockUnavailableError | undefined; let released = false; try { released = await this.#release(key, token); } catch { releaseError = new ClaimLockUnavailableError(); } if (operationFailed) throw operationError; if (observedHeartbeatError) throw observedHeartbeatError; if (releaseError) throw releaseError; if (!released) throw new ClaimLockLostError(); return result as T; } async #acquire(key: string, token: string): Promise { const deadline = this.#options.now() + this.#options.acquireTimeoutMs; while (!this.#closing && !this.#closed) { try { if ((await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK") return; } catch { // Redis can be transiently unavailable during a rolling restart. The // bounded deadline below turns that into a retryable controller result. } const remainingMs = deadline - this.#options.now(); if (remainingMs <= 0) break; const jitterMs = Math.floor(this.#options.random() * (this.#options.retryJitterMs + 1)); await this.#options.wait(Math.min(remainingMs, this.#options.retryDelayMs + jitterMs)); } throw new ClaimLockUnavailableError(); } async #renew(key: string, token: string): Promise { let renewed: unknown; try { renewed = await this.#client.eval(RENEW_SCRIPT, 1, key, token, this.#options.leaseMs); } catch { throw new ClaimLockUnavailableError(); } if (Number(renewed) !== 1) throw new ClaimLockLostError(); } async #release(key: string, token: string): Promise { try { return Number(await this.#client.eval(RELEASE_SCRIPT, 1, key, token)) === 1; } catch { throw new ClaimLockUnavailableError(); } } async #probeLockCapabilities(): Promise { const key = `${LOCK_KEY_PREFIX}readiness:${sha256Hex(randomUUID())}`; const token = randomUUID(); let acquired = false; let released = false; try { acquired = (await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK"; if (!acquired) throw new ClaimLockUnavailableError(); await this.#renew(key, token); released = await this.#release(key, token); if (!released) throw new ClaimLockLostError(); } finally { if (acquired && !released) await this.#release(key, token).catch(() => undefined); } } } export function createRedisClaimMutationLock(connection: ConnectionOptions): ClaimMutationLock { const client = new Redis({ ...connection, commandTimeout: DEFAULT_COMMAND_TIMEOUT_MS, enableOfflineQueue: false, enableReadyCheck: true, lazyConnect: true, maxRetriesPerRequest: 1, }); return new RedisClaimMutationLock(client); } export function createFencedClaimMutationLock(connection: ConnectionOptions, filesystemDirectory: string): ClaimMutationLock { return new FencedClaimMutationLock(new FilesystemClaimMutationLock(filesystemDirectory), createRedisClaimMutationLock(connection)); } async function assertSameRegularFile(lockPath: string, handle: FileHandle): Promise { const [pathStats, openedStats] = await Promise.all([lstat(lockPath), handle.stat()]); if ( !pathStats.isFile() || pathStats.isSymbolicLink() || pathStats.nlink !== 1 || !openedStats.isFile() || openedStats.nlink !== 1 || openedStats.dev !== pathStats.dev || openedStats.ino !== pathStats.ino ) { throw new ClaimLockUnavailableError(); } } async function acquireKernelFlock(fd: number, binary: string, timeoutMs: number): Promise { const timeoutSeconds = Math.max(0.001, timeoutMs / 1_000).toFixed(3); await new Promise((resolve, reject) => { const child = spawn(binary, ["--exclusive", "--wait", timeoutSeconds, "3"], { stdio: ["ignore", "ignore", "ignore", fd], }); let settled = false; const watchdog = setTimeout(() => { if (settled) return; child.kill("SIGKILL"); }, timeoutMs + 1_000); watchdog.unref(); child.once("error", () => { if (settled) return; settled = true; clearTimeout(watchdog); reject(new ClaimLockUnavailableError()); }); child.once("close", (code) => { if (settled) return; settled = true; clearTimeout(watchdog); if (code === 0) resolve(); else reject(new ClaimLockUnavailableError()); }); }); } function noFollowFlag(): number { return typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0; } function normalizeOptions(options: ClaimLockOptions): NormalizedClaimLockOptions { const leaseMs = positiveInteger(options.leaseMs ?? DEFAULT_LEASE_MS, "claim lock lease"); const acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_ACQUIRE_TIMEOUT_MS, "claim lock acquisition timeout"); const retryDelayMs = nonNegativeInteger(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS, "claim lock retry delay"); const retryJitterMs = nonNegativeInteger(options.retryJitterMs ?? DEFAULT_RETRY_JITTER_MS, "claim lock retry jitter"); const heartbeatIntervalMs = positiveInteger(options.heartbeatIntervalMs ?? Math.floor(leaseMs / 3), "claim lock heartbeat interval"); if (heartbeatIntervalMs >= leaseMs / 2) throw new Error("claim lock heartbeat interval must be less than half the lease"); return { leaseMs, acquireTimeoutMs, retryDelayMs, retryJitterMs, heartbeatIntervalMs, now: options.now ?? Date.now, random: options.random ?? Math.random, token: options.token ?? randomUUID, wait: options.wait ?? wait, }; } function positiveInteger(value: number, name: string): number { if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); return value; } function nonNegativeInteger(value: number, name: string): number { if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return value; } function wait(delayMs: number): Promise { return new Promise((resolve) => setTimeout(resolve, delayMs)); }