485 lines
17 KiB
TypeScript
485 lines
17 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
import { spawn } from "node:child_process";
|
|
import { constants as fsConstants } from "node:fs";
|
|
import { lstat, open, realpath, type FileHandle } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import type { ConnectionOptions } from "@hivegrid/queue";
|
|
import { Redis } from "ioredis";
|
|
import { sha256Hex } from "./contracts.js";
|
|
|
|
const LOCK_KEY_PREFIX = "hivegrid:portal-domain-controller:claim-lock:";
|
|
const DEFAULT_LEASE_MS = 30_000;
|
|
const DEFAULT_ACQUIRE_TIMEOUT_MS = 10_000;
|
|
const DEFAULT_RETRY_DELAY_MS = 75;
|
|
const DEFAULT_RETRY_JITTER_MS = 75;
|
|
const DEFAULT_COMMAND_TIMEOUT_MS = 2_500;
|
|
const DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS = 10_000;
|
|
const FLOCK_BINARY = "/usr/bin/flock";
|
|
|
|
const RENEW_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('PEXPIRE', KEYS[1], ARGV[2])", "end", "return 0"].join("\n");
|
|
|
|
const RELEASE_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('DEL', KEYS[1])", "end", "return 0"].join("\n");
|
|
|
|
export type ClaimMutationLock = {
|
|
assertReady(): Promise<void>;
|
|
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T>;
|
|
close(): Promise<void>;
|
|
};
|
|
|
|
export type ClaimLockGuard = Readonly<{
|
|
assertHeld(): Promise<void>;
|
|
}>;
|
|
|
|
export type ClaimLockRedisClient = {
|
|
readonly status: string;
|
|
connect(): Promise<unknown>;
|
|
ping(): Promise<string>;
|
|
set(key: string, value: string, expiryMode: "PX", leaseMs: number, condition: "NX"): Promise<"OK" | null>;
|
|
eval(script: string, numberOfKeys: number, ...args: Array<string | number>): Promise<unknown>;
|
|
quit(): Promise<unknown>;
|
|
disconnect(reconnect?: boolean): void;
|
|
on(event: "error", listener: (error: unknown) => void): unknown;
|
|
};
|
|
|
|
type ClaimLockOptions = Readonly<{
|
|
leaseMs?: number;
|
|
acquireTimeoutMs?: number;
|
|
retryDelayMs?: number;
|
|
retryJitterMs?: number;
|
|
heartbeatIntervalMs?: number;
|
|
now?: () => number;
|
|
random?: () => number;
|
|
token?: () => string;
|
|
wait?: (delayMs: number) => Promise<void>;
|
|
}>;
|
|
|
|
type NormalizedClaimLockOptions = Readonly<{
|
|
leaseMs: number;
|
|
acquireTimeoutMs: number;
|
|
retryDelayMs: number;
|
|
retryJitterMs: number;
|
|
heartbeatIntervalMs: number;
|
|
now: () => number;
|
|
random: () => number;
|
|
token: () => string;
|
|
wait: (delayMs: number) => Promise<void>;
|
|
}>;
|
|
|
|
export class ClaimLockUnavailableError extends Error {
|
|
readonly retryable = true;
|
|
readonly reasonCode = "claim_lock_unavailable";
|
|
|
|
constructor() {
|
|
super("Portal-domain claim lock is temporarily unavailable");
|
|
this.name = "ClaimLockUnavailableError";
|
|
}
|
|
}
|
|
|
|
export class ClaimLockLostError extends Error {
|
|
readonly retryable = true;
|
|
readonly reasonCode = "claim_lock_lost";
|
|
|
|
constructor() {
|
|
super("Portal-domain claim lock ownership was lost");
|
|
this.name = "ClaimLockLostError";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Kernel-backed ownership for the shared Traefik directory. Redis remains the
|
|
* fast distributed coordinator, while this non-expiring `flock` is the
|
|
* authoritative fence around filesystem mutations. The lock is attached to
|
|
* the parent's open-file description, so a paused Node event loop cannot let a
|
|
* second controller enter after a Redis lease expires. A crash closes the fd
|
|
* and lets the kernel release ownership automatically.
|
|
*/
|
|
export class FilesystemClaimMutationLock implements ClaimMutationLock {
|
|
readonly #configuredDirectory: string;
|
|
readonly #binary: string;
|
|
readonly #acquireTimeoutMs: number;
|
|
readonly #activeOperations = new Set<Promise<void>>();
|
|
#closing = false;
|
|
#closed = false;
|
|
#closePromise: Promise<void> | undefined;
|
|
|
|
constructor(directory: string, options: { binary?: string; acquireTimeoutMs?: number } = {}) {
|
|
if (!path.isAbsolute(directory)) throw new ClaimLockUnavailableError();
|
|
this.#configuredDirectory = path.resolve(directory);
|
|
this.#binary = options.binary ?? FLOCK_BINARY;
|
|
this.#acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS, "filesystem claim lock timeout");
|
|
}
|
|
|
|
async assertReady(): Promise<void> {
|
|
await this.withClaimLock("__readiness__", async (guard) => guard.assertHeld());
|
|
}
|
|
|
|
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
|
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
|
let finishActive: () => void = () => undefined;
|
|
const active = new Promise<void>((resolve) => {
|
|
finishActive = resolve;
|
|
});
|
|
this.#activeOperations.add(active);
|
|
try {
|
|
return await this.#withClaimLock(claimId, operation);
|
|
} finally {
|
|
finishActive();
|
|
this.#activeOperations.delete(active);
|
|
}
|
|
}
|
|
|
|
close(): Promise<void> {
|
|
this.#closePromise ??= this.#close();
|
|
return this.#closePromise;
|
|
}
|
|
|
|
async #close(): Promise<void> {
|
|
if (this.#closed) return;
|
|
this.#closing = true;
|
|
await Promise.all([...this.#activeOperations]);
|
|
this.#closed = true;
|
|
}
|
|
|
|
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
|
const directory = await this.#resolveSafeDirectory();
|
|
const lockPath = path.join(directory, `.portal-${sha256Hex(claimId)}.lock`);
|
|
if (path.dirname(lockPath) !== directory) throw new ClaimLockUnavailableError();
|
|
let handle: FileHandle | undefined;
|
|
let held = false;
|
|
try {
|
|
handle = await open(
|
|
lockPath,
|
|
fsConstants.O_CREAT | fsConstants.O_RDWR | noFollowFlag(),
|
|
0o600
|
|
);
|
|
await assertSameRegularFile(lockPath, handle);
|
|
await acquireKernelFlock(handle.fd, this.#binary, this.#acquireTimeoutMs);
|
|
held = true;
|
|
const guard: ClaimLockGuard = Object.freeze({
|
|
assertHeld: async () => {
|
|
if (!held || !handle) throw new ClaimLockLostError();
|
|
await assertSameRegularFile(lockPath, handle);
|
|
},
|
|
});
|
|
return await operation(guard);
|
|
} catch (error) {
|
|
if (error instanceof ClaimLockLostError || error instanceof ClaimLockUnavailableError) throw error;
|
|
throw new ClaimLockUnavailableError();
|
|
} finally {
|
|
held = false;
|
|
await handle?.close().catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
async #resolveSafeDirectory(): Promise<string> {
|
|
try {
|
|
const stats = await lstat(this.#configuredDirectory);
|
|
if (!stats.isDirectory() || stats.isSymbolicLink()) throw new ClaimLockUnavailableError();
|
|
return await realpath(this.#configuredDirectory);
|
|
} catch (error) {
|
|
if (error instanceof ClaimLockUnavailableError) throw error;
|
|
throw new ClaimLockUnavailableError();
|
|
}
|
|
}
|
|
}
|
|
|
|
export class FencedClaimMutationLock implements ClaimMutationLock {
|
|
readonly #filesystem: ClaimMutationLock;
|
|
readonly #lease: ClaimMutationLock;
|
|
|
|
constructor(filesystem: ClaimMutationLock, lease: ClaimMutationLock) {
|
|
this.#filesystem = filesystem;
|
|
this.#lease = lease;
|
|
}
|
|
|
|
async assertReady(): Promise<void> {
|
|
await this.#filesystem.assertReady();
|
|
await this.#lease.assertReady();
|
|
}
|
|
|
|
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
|
return this.#filesystem.withClaimLock(claimId, () => this.#lease.withClaimLock(claimId, operation));
|
|
}
|
|
|
|
async close(): Promise<void> {
|
|
await Promise.all([this.#lease.close(), this.#filesystem.close()]);
|
|
}
|
|
}
|
|
|
|
export class RedisClaimMutationLock implements ClaimMutationLock {
|
|
readonly #client: ClaimLockRedisClient;
|
|
readonly #options: NormalizedClaimLockOptions;
|
|
readonly #activeOperations = new Set<Promise<void>>();
|
|
#closing = false;
|
|
#closed = false;
|
|
#closePromise: Promise<void> | undefined;
|
|
|
|
constructor(client: ClaimLockRedisClient, options: ClaimLockOptions = {}) {
|
|
this.#client = client;
|
|
this.#options = normalizeOptions(options);
|
|
// Redis connection errors are surfaced through bounded commands and the
|
|
// readiness endpoint. Keeping this listener intentionally silent also
|
|
// prevents connection details from reaching stderr outside our logger.
|
|
this.#client.on("error", () => undefined);
|
|
}
|
|
|
|
async assertReady(): Promise<void> {
|
|
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
|
try {
|
|
if (this.#client.status === "wait") await this.#client.connect();
|
|
if ((await this.#client.ping()) !== "PONG") throw new ClaimLockUnavailableError();
|
|
await this.#probeLockCapabilities();
|
|
} catch {
|
|
throw new ClaimLockUnavailableError();
|
|
}
|
|
}
|
|
|
|
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
|
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
|
let finishActive: () => void = () => undefined;
|
|
const active = new Promise<void>((resolve) => {
|
|
finishActive = resolve;
|
|
});
|
|
this.#activeOperations.add(active);
|
|
try {
|
|
return await this.#withClaimLock(claimId, operation);
|
|
} finally {
|
|
finishActive();
|
|
this.#activeOperations.delete(active);
|
|
}
|
|
}
|
|
|
|
close(): Promise<void> {
|
|
this.#closePromise ??= this.#close();
|
|
return this.#closePromise;
|
|
}
|
|
|
|
async #close(): Promise<void> {
|
|
if (this.#closed) return;
|
|
this.#closing = true;
|
|
await Promise.all([...this.#activeOperations]);
|
|
try {
|
|
if (this.#client.status !== "wait" && this.#client.status !== "end") {
|
|
await this.#client.quit();
|
|
}
|
|
} catch {
|
|
// An immediate disconnect is safe after all leases were released. It
|
|
// also keeps shutdown bounded when Redis disappears during a rollout.
|
|
} finally {
|
|
this.#client.disconnect(false);
|
|
this.#closed = true;
|
|
}
|
|
}
|
|
|
|
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
|
const key = `${LOCK_KEY_PREFIX}${sha256Hex(claimId)}`;
|
|
const token = this.#options.token();
|
|
await this.#acquire(key, token);
|
|
|
|
let heartbeatTimer: NodeJS.Timeout | undefined;
|
|
let heartbeatInFlight: Promise<void> = Promise.resolve();
|
|
let heartbeatError: ClaimLockLostError | ClaimLockUnavailableError | undefined;
|
|
let heartbeatStopped = false;
|
|
const scheduleHeartbeat = () => {
|
|
if (heartbeatStopped || heartbeatError) return;
|
|
heartbeatTimer = setTimeout(() => {
|
|
heartbeatInFlight = this.#renew(key, token)
|
|
.then(() => scheduleHeartbeat())
|
|
.catch((error: unknown) => {
|
|
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
|
|
});
|
|
}, this.#options.heartbeatIntervalMs);
|
|
};
|
|
scheduleHeartbeat();
|
|
|
|
const guard: ClaimLockGuard = Object.freeze({
|
|
assertHeld: async () => {
|
|
if (heartbeatError) throw heartbeatError;
|
|
try {
|
|
await this.#renew(key, token);
|
|
} catch (error) {
|
|
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
|
|
throw heartbeatError;
|
|
}
|
|
if (heartbeatError) throw heartbeatError;
|
|
},
|
|
});
|
|
|
|
let result: T | undefined;
|
|
let operationError: unknown;
|
|
let operationFailed = false;
|
|
try {
|
|
result = await operation(guard);
|
|
} catch (error) {
|
|
operationFailed = true;
|
|
operationError = error;
|
|
}
|
|
|
|
heartbeatStopped = true;
|
|
if (heartbeatTimer) clearTimeout(heartbeatTimer);
|
|
await heartbeatInFlight.catch(() => undefined);
|
|
const observedHeartbeatError = heartbeatError;
|
|
let releaseError: ClaimLockUnavailableError | undefined;
|
|
let released = false;
|
|
try {
|
|
released = await this.#release(key, token);
|
|
} catch {
|
|
releaseError = new ClaimLockUnavailableError();
|
|
}
|
|
|
|
if (operationFailed) throw operationError;
|
|
if (observedHeartbeatError) throw observedHeartbeatError;
|
|
if (releaseError) throw releaseError;
|
|
if (!released) throw new ClaimLockLostError();
|
|
return result as T;
|
|
}
|
|
|
|
async #acquire(key: string, token: string): Promise<void> {
|
|
const deadline = this.#options.now() + this.#options.acquireTimeoutMs;
|
|
while (!this.#closing && !this.#closed) {
|
|
try {
|
|
if ((await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK") return;
|
|
} catch {
|
|
// Redis can be transiently unavailable during a rolling restart. The
|
|
// bounded deadline below turns that into a retryable controller result.
|
|
}
|
|
const remainingMs = deadline - this.#options.now();
|
|
if (remainingMs <= 0) break;
|
|
const jitterMs = Math.floor(this.#options.random() * (this.#options.retryJitterMs + 1));
|
|
await this.#options.wait(Math.min(remainingMs, this.#options.retryDelayMs + jitterMs));
|
|
}
|
|
throw new ClaimLockUnavailableError();
|
|
}
|
|
|
|
async #renew(key: string, token: string): Promise<void> {
|
|
let renewed: unknown;
|
|
try {
|
|
renewed = await this.#client.eval(RENEW_SCRIPT, 1, key, token, this.#options.leaseMs);
|
|
} catch {
|
|
throw new ClaimLockUnavailableError();
|
|
}
|
|
if (Number(renewed) !== 1) throw new ClaimLockLostError();
|
|
}
|
|
|
|
async #release(key: string, token: string): Promise<boolean> {
|
|
try {
|
|
return Number(await this.#client.eval(RELEASE_SCRIPT, 1, key, token)) === 1;
|
|
} catch {
|
|
throw new ClaimLockUnavailableError();
|
|
}
|
|
}
|
|
|
|
async #probeLockCapabilities(): Promise<void> {
|
|
const key = `${LOCK_KEY_PREFIX}readiness:${sha256Hex(randomUUID())}`;
|
|
const token = randomUUID();
|
|
let acquired = false;
|
|
let released = false;
|
|
try {
|
|
acquired = (await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK";
|
|
if (!acquired) throw new ClaimLockUnavailableError();
|
|
await this.#renew(key, token);
|
|
released = await this.#release(key, token);
|
|
if (!released) throw new ClaimLockLostError();
|
|
} finally {
|
|
if (acquired && !released) await this.#release(key, token).catch(() => undefined);
|
|
}
|
|
}
|
|
}
|
|
|
|
export function createRedisClaimMutationLock(connection: ConnectionOptions): ClaimMutationLock {
|
|
const client = new Redis({
|
|
...connection,
|
|
commandTimeout: DEFAULT_COMMAND_TIMEOUT_MS,
|
|
enableOfflineQueue: false,
|
|
enableReadyCheck: true,
|
|
lazyConnect: true,
|
|
maxRetriesPerRequest: 1,
|
|
});
|
|
return new RedisClaimMutationLock(client);
|
|
}
|
|
|
|
export function createFencedClaimMutationLock(connection: ConnectionOptions, filesystemDirectory: string): ClaimMutationLock {
|
|
return new FencedClaimMutationLock(new FilesystemClaimMutationLock(filesystemDirectory), createRedisClaimMutationLock(connection));
|
|
}
|
|
|
|
async function assertSameRegularFile(lockPath: string, handle: FileHandle): Promise<void> {
|
|
const [pathStats, openedStats] = await Promise.all([lstat(lockPath), handle.stat()]);
|
|
if (
|
|
!pathStats.isFile() ||
|
|
pathStats.isSymbolicLink() ||
|
|
pathStats.nlink !== 1 ||
|
|
!openedStats.isFile() ||
|
|
openedStats.nlink !== 1 ||
|
|
openedStats.dev !== pathStats.dev ||
|
|
openedStats.ino !== pathStats.ino
|
|
) {
|
|
throw new ClaimLockUnavailableError();
|
|
}
|
|
}
|
|
|
|
async function acquireKernelFlock(fd: number, binary: string, timeoutMs: number): Promise<void> {
|
|
const timeoutSeconds = Math.max(0.001, timeoutMs / 1_000).toFixed(3);
|
|
await new Promise<void>((resolve, reject) => {
|
|
const child = spawn(binary, ["--exclusive", "--wait", timeoutSeconds, "3"], {
|
|
stdio: ["ignore", "ignore", "ignore", fd],
|
|
});
|
|
let settled = false;
|
|
const watchdog = setTimeout(() => {
|
|
if (settled) return;
|
|
child.kill("SIGKILL");
|
|
}, timeoutMs + 1_000);
|
|
watchdog.unref();
|
|
child.once("error", () => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(watchdog);
|
|
reject(new ClaimLockUnavailableError());
|
|
});
|
|
child.once("close", (code) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(watchdog);
|
|
if (code === 0) resolve();
|
|
else reject(new ClaimLockUnavailableError());
|
|
});
|
|
});
|
|
}
|
|
|
|
function noFollowFlag(): number {
|
|
return typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0;
|
|
}
|
|
|
|
function normalizeOptions(options: ClaimLockOptions): NormalizedClaimLockOptions {
|
|
const leaseMs = positiveInteger(options.leaseMs ?? DEFAULT_LEASE_MS, "claim lock lease");
|
|
const acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_ACQUIRE_TIMEOUT_MS, "claim lock acquisition timeout");
|
|
const retryDelayMs = nonNegativeInteger(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS, "claim lock retry delay");
|
|
const retryJitterMs = nonNegativeInteger(options.retryJitterMs ?? DEFAULT_RETRY_JITTER_MS, "claim lock retry jitter");
|
|
const heartbeatIntervalMs = positiveInteger(options.heartbeatIntervalMs ?? Math.floor(leaseMs / 3), "claim lock heartbeat interval");
|
|
if (heartbeatIntervalMs >= leaseMs / 2) throw new Error("claim lock heartbeat interval must be less than half the lease");
|
|
return {
|
|
leaseMs,
|
|
acquireTimeoutMs,
|
|
retryDelayMs,
|
|
retryJitterMs,
|
|
heartbeatIntervalMs,
|
|
now: options.now ?? Date.now,
|
|
random: options.random ?? Math.random,
|
|
token: options.token ?? randomUUID,
|
|
wait: options.wait ?? wait,
|
|
};
|
|
}
|
|
|
|
function positiveInteger(value: number, name: string): number {
|
|
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
function nonNegativeInteger(value: number, name: string): number {
|
|
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
|
return value;
|
|
}
|
|
|
|
function wait(delayMs: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
}
|