Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/portal-domain-controller/src/claim-lock.ts
T

485 lines
17 KiB
TypeScript

import { randomUUID } from "node:crypto";
import { spawn } from "node:child_process";
import { constants as fsConstants } from "node:fs";
import { lstat, open, realpath, type FileHandle } from "node:fs/promises";
import path from "node:path";
import type { ConnectionOptions } from "@hivegrid/queue";
import { Redis } from "ioredis";
import { sha256Hex } from "./contracts.js";
const LOCK_KEY_PREFIX = "hivegrid:portal-domain-controller:claim-lock:";
const DEFAULT_LEASE_MS = 30_000;
const DEFAULT_ACQUIRE_TIMEOUT_MS = 10_000;
const DEFAULT_RETRY_DELAY_MS = 75;
const DEFAULT_RETRY_JITTER_MS = 75;
const DEFAULT_COMMAND_TIMEOUT_MS = 2_500;
const DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS = 10_000;
const FLOCK_BINARY = "/usr/bin/flock";
const RENEW_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('PEXPIRE', KEYS[1], ARGV[2])", "end", "return 0"].join("\n");
const RELEASE_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('DEL', KEYS[1])", "end", "return 0"].join("\n");
export type ClaimMutationLock = {
assertReady(): Promise<void>;
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T>;
close(): Promise<void>;
};
export type ClaimLockGuard = Readonly<{
assertHeld(): Promise<void>;
}>;
export type ClaimLockRedisClient = {
readonly status: string;
connect(): Promise<unknown>;
ping(): Promise<string>;
set(key: string, value: string, expiryMode: "PX", leaseMs: number, condition: "NX"): Promise<"OK" | null>;
eval(script: string, numberOfKeys: number, ...args: Array<string | number>): Promise<unknown>;
quit(): Promise<unknown>;
disconnect(reconnect?: boolean): void;
on(event: "error", listener: (error: unknown) => void): unknown;
};
type ClaimLockOptions = Readonly<{
leaseMs?: number;
acquireTimeoutMs?: number;
retryDelayMs?: number;
retryJitterMs?: number;
heartbeatIntervalMs?: number;
now?: () => number;
random?: () => number;
token?: () => string;
wait?: (delayMs: number) => Promise<void>;
}>;
type NormalizedClaimLockOptions = Readonly<{
leaseMs: number;
acquireTimeoutMs: number;
retryDelayMs: number;
retryJitterMs: number;
heartbeatIntervalMs: number;
now: () => number;
random: () => number;
token: () => string;
wait: (delayMs: number) => Promise<void>;
}>;
export class ClaimLockUnavailableError extends Error {
readonly retryable = true;
readonly reasonCode = "claim_lock_unavailable";
constructor() {
super("Portal-domain claim lock is temporarily unavailable");
this.name = "ClaimLockUnavailableError";
}
}
export class ClaimLockLostError extends Error {
readonly retryable = true;
readonly reasonCode = "claim_lock_lost";
constructor() {
super("Portal-domain claim lock ownership was lost");
this.name = "ClaimLockLostError";
}
}
/**
* Kernel-backed ownership for the shared Traefik directory. Redis remains the
* fast distributed coordinator, while this non-expiring `flock` is the
* authoritative fence around filesystem mutations. The lock is attached to
* the parent's open-file description, so a paused Node event loop cannot let a
* second controller enter after a Redis lease expires. A crash closes the fd
* and lets the kernel release ownership automatically.
*/
export class FilesystemClaimMutationLock implements ClaimMutationLock {
readonly #configuredDirectory: string;
readonly #binary: string;
readonly #acquireTimeoutMs: number;
readonly #activeOperations = new Set<Promise<void>>();
#closing = false;
#closed = false;
#closePromise: Promise<void> | undefined;
constructor(directory: string, options: { binary?: string; acquireTimeoutMs?: number } = {}) {
if (!path.isAbsolute(directory)) throw new ClaimLockUnavailableError();
this.#configuredDirectory = path.resolve(directory);
this.#binary = options.binary ?? FLOCK_BINARY;
this.#acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS, "filesystem claim lock timeout");
}
async assertReady(): Promise<void> {
await this.withClaimLock("__readiness__", async (guard) => guard.assertHeld());
}
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
let finishActive: () => void = () => undefined;
const active = new Promise<void>((resolve) => {
finishActive = resolve;
});
this.#activeOperations.add(active);
try {
return await this.#withClaimLock(claimId, operation);
} finally {
finishActive();
this.#activeOperations.delete(active);
}
}
close(): Promise<void> {
this.#closePromise ??= this.#close();
return this.#closePromise;
}
async #close(): Promise<void> {
if (this.#closed) return;
this.#closing = true;
await Promise.all([...this.#activeOperations]);
this.#closed = true;
}
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
const directory = await this.#resolveSafeDirectory();
const lockPath = path.join(directory, `.portal-${sha256Hex(claimId)}.lock`);
if (path.dirname(lockPath) !== directory) throw new ClaimLockUnavailableError();
let handle: FileHandle | undefined;
let held = false;
try {
handle = await open(
lockPath,
fsConstants.O_CREAT | fsConstants.O_RDWR | noFollowFlag(),
0o600
);
await assertSameRegularFile(lockPath, handle);
await acquireKernelFlock(handle.fd, this.#binary, this.#acquireTimeoutMs);
held = true;
const guard: ClaimLockGuard = Object.freeze({
assertHeld: async () => {
if (!held || !handle) throw new ClaimLockLostError();
await assertSameRegularFile(lockPath, handle);
},
});
return await operation(guard);
} catch (error) {
if (error instanceof ClaimLockLostError || error instanceof ClaimLockUnavailableError) throw error;
throw new ClaimLockUnavailableError();
} finally {
held = false;
await handle?.close().catch(() => undefined);
}
}
async #resolveSafeDirectory(): Promise<string> {
try {
const stats = await lstat(this.#configuredDirectory);
if (!stats.isDirectory() || stats.isSymbolicLink()) throw new ClaimLockUnavailableError();
return await realpath(this.#configuredDirectory);
} catch (error) {
if (error instanceof ClaimLockUnavailableError) throw error;
throw new ClaimLockUnavailableError();
}
}
}
export class FencedClaimMutationLock implements ClaimMutationLock {
readonly #filesystem: ClaimMutationLock;
readonly #lease: ClaimMutationLock;
constructor(filesystem: ClaimMutationLock, lease: ClaimMutationLock) {
this.#filesystem = filesystem;
this.#lease = lease;
}
async assertReady(): Promise<void> {
await this.#filesystem.assertReady();
await this.#lease.assertReady();
}
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
return this.#filesystem.withClaimLock(claimId, () => this.#lease.withClaimLock(claimId, operation));
}
async close(): Promise<void> {
await Promise.all([this.#lease.close(), this.#filesystem.close()]);
}
}
export class RedisClaimMutationLock implements ClaimMutationLock {
readonly #client: ClaimLockRedisClient;
readonly #options: NormalizedClaimLockOptions;
readonly #activeOperations = new Set<Promise<void>>();
#closing = false;
#closed = false;
#closePromise: Promise<void> | undefined;
constructor(client: ClaimLockRedisClient, options: ClaimLockOptions = {}) {
this.#client = client;
this.#options = normalizeOptions(options);
// Redis connection errors are surfaced through bounded commands and the
// readiness endpoint. Keeping this listener intentionally silent also
// prevents connection details from reaching stderr outside our logger.
this.#client.on("error", () => undefined);
}
async assertReady(): Promise<void> {
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
try {
if (this.#client.status === "wait") await this.#client.connect();
if ((await this.#client.ping()) !== "PONG") throw new ClaimLockUnavailableError();
await this.#probeLockCapabilities();
} catch {
throw new ClaimLockUnavailableError();
}
}
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
let finishActive: () => void = () => undefined;
const active = new Promise<void>((resolve) => {
finishActive = resolve;
});
this.#activeOperations.add(active);
try {
return await this.#withClaimLock(claimId, operation);
} finally {
finishActive();
this.#activeOperations.delete(active);
}
}
close(): Promise<void> {
this.#closePromise ??= this.#close();
return this.#closePromise;
}
async #close(): Promise<void> {
if (this.#closed) return;
this.#closing = true;
await Promise.all([...this.#activeOperations]);
try {
if (this.#client.status !== "wait" && this.#client.status !== "end") {
await this.#client.quit();
}
} catch {
// An immediate disconnect is safe after all leases were released. It
// also keeps shutdown bounded when Redis disappears during a rollout.
} finally {
this.#client.disconnect(false);
this.#closed = true;
}
}
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
const key = `${LOCK_KEY_PREFIX}${sha256Hex(claimId)}`;
const token = this.#options.token();
await this.#acquire(key, token);
let heartbeatTimer: NodeJS.Timeout | undefined;
let heartbeatInFlight: Promise<void> = Promise.resolve();
let heartbeatError: ClaimLockLostError | ClaimLockUnavailableError | undefined;
let heartbeatStopped = false;
const scheduleHeartbeat = () => {
if (heartbeatStopped || heartbeatError) return;
heartbeatTimer = setTimeout(() => {
heartbeatInFlight = this.#renew(key, token)
.then(() => scheduleHeartbeat())
.catch((error: unknown) => {
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
});
}, this.#options.heartbeatIntervalMs);
};
scheduleHeartbeat();
const guard: ClaimLockGuard = Object.freeze({
assertHeld: async () => {
if (heartbeatError) throw heartbeatError;
try {
await this.#renew(key, token);
} catch (error) {
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
throw heartbeatError;
}
if (heartbeatError) throw heartbeatError;
},
});
let result: T | undefined;
let operationError: unknown;
let operationFailed = false;
try {
result = await operation(guard);
} catch (error) {
operationFailed = true;
operationError = error;
}
heartbeatStopped = true;
if (heartbeatTimer) clearTimeout(heartbeatTimer);
await heartbeatInFlight.catch(() => undefined);
const observedHeartbeatError = heartbeatError;
let releaseError: ClaimLockUnavailableError | undefined;
let released = false;
try {
released = await this.#release(key, token);
} catch {
releaseError = new ClaimLockUnavailableError();
}
if (operationFailed) throw operationError;
if (observedHeartbeatError) throw observedHeartbeatError;
if (releaseError) throw releaseError;
if (!released) throw new ClaimLockLostError();
return result as T;
}
async #acquire(key: string, token: string): Promise<void> {
const deadline = this.#options.now() + this.#options.acquireTimeoutMs;
while (!this.#closing && !this.#closed) {
try {
if ((await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK") return;
} catch {
// Redis can be transiently unavailable during a rolling restart. The
// bounded deadline below turns that into a retryable controller result.
}
const remainingMs = deadline - this.#options.now();
if (remainingMs <= 0) break;
const jitterMs = Math.floor(this.#options.random() * (this.#options.retryJitterMs + 1));
await this.#options.wait(Math.min(remainingMs, this.#options.retryDelayMs + jitterMs));
}
throw new ClaimLockUnavailableError();
}
async #renew(key: string, token: string): Promise<void> {
let renewed: unknown;
try {
renewed = await this.#client.eval(RENEW_SCRIPT, 1, key, token, this.#options.leaseMs);
} catch {
throw new ClaimLockUnavailableError();
}
if (Number(renewed) !== 1) throw new ClaimLockLostError();
}
async #release(key: string, token: string): Promise<boolean> {
try {
return Number(await this.#client.eval(RELEASE_SCRIPT, 1, key, token)) === 1;
} catch {
throw new ClaimLockUnavailableError();
}
}
async #probeLockCapabilities(): Promise<void> {
const key = `${LOCK_KEY_PREFIX}readiness:${sha256Hex(randomUUID())}`;
const token = randomUUID();
let acquired = false;
let released = false;
try {
acquired = (await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK";
if (!acquired) throw new ClaimLockUnavailableError();
await this.#renew(key, token);
released = await this.#release(key, token);
if (!released) throw new ClaimLockLostError();
} finally {
if (acquired && !released) await this.#release(key, token).catch(() => undefined);
}
}
}
export function createRedisClaimMutationLock(connection: ConnectionOptions): ClaimMutationLock {
const client = new Redis({
...connection,
commandTimeout: DEFAULT_COMMAND_TIMEOUT_MS,
enableOfflineQueue: false,
enableReadyCheck: true,
lazyConnect: true,
maxRetriesPerRequest: 1,
});
return new RedisClaimMutationLock(client);
}
export function createFencedClaimMutationLock(connection: ConnectionOptions, filesystemDirectory: string): ClaimMutationLock {
return new FencedClaimMutationLock(new FilesystemClaimMutationLock(filesystemDirectory), createRedisClaimMutationLock(connection));
}
async function assertSameRegularFile(lockPath: string, handle: FileHandle): Promise<void> {
const [pathStats, openedStats] = await Promise.all([lstat(lockPath), handle.stat()]);
if (
!pathStats.isFile() ||
pathStats.isSymbolicLink() ||
pathStats.nlink !== 1 ||
!openedStats.isFile() ||
openedStats.nlink !== 1 ||
openedStats.dev !== pathStats.dev ||
openedStats.ino !== pathStats.ino
) {
throw new ClaimLockUnavailableError();
}
}
async function acquireKernelFlock(fd: number, binary: string, timeoutMs: number): Promise<void> {
const timeoutSeconds = Math.max(0.001, timeoutMs / 1_000).toFixed(3);
await new Promise<void>((resolve, reject) => {
const child = spawn(binary, ["--exclusive", "--wait", timeoutSeconds, "3"], {
stdio: ["ignore", "ignore", "ignore", fd],
});
let settled = false;
const watchdog = setTimeout(() => {
if (settled) return;
child.kill("SIGKILL");
}, timeoutMs + 1_000);
watchdog.unref();
child.once("error", () => {
if (settled) return;
settled = true;
clearTimeout(watchdog);
reject(new ClaimLockUnavailableError());
});
child.once("close", (code) => {
if (settled) return;
settled = true;
clearTimeout(watchdog);
if (code === 0) resolve();
else reject(new ClaimLockUnavailableError());
});
});
}
function noFollowFlag(): number {
return typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0;
}
function normalizeOptions(options: ClaimLockOptions): NormalizedClaimLockOptions {
const leaseMs = positiveInteger(options.leaseMs ?? DEFAULT_LEASE_MS, "claim lock lease");
const acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_ACQUIRE_TIMEOUT_MS, "claim lock acquisition timeout");
const retryDelayMs = nonNegativeInteger(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS, "claim lock retry delay");
const retryJitterMs = nonNegativeInteger(options.retryJitterMs ?? DEFAULT_RETRY_JITTER_MS, "claim lock retry jitter");
const heartbeatIntervalMs = positiveInteger(options.heartbeatIntervalMs ?? Math.floor(leaseMs / 3), "claim lock heartbeat interval");
if (heartbeatIntervalMs >= leaseMs / 2) throw new Error("claim lock heartbeat interval must be less than half the lease");
return {
leaseMs,
acquireTimeoutMs,
retryDelayMs,
retryJitterMs,
heartbeatIntervalMs,
now: options.now ?? Date.now,
random: options.random ?? Math.random,
token: options.token ?? randomUUID,
wait: options.wait ?? wait,
};
}
function positiveInteger(value: number, name: string): number {
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`);
return value;
}
function nonNegativeInteger(value: number, name: string): number {
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
return value;
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs));
}