Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+484
@@ -0,0 +1,484 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import { lstat, open, realpath, type FileHandle } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { ConnectionOptions } from "@hivegrid/queue";
|
||||
import { Redis } from "ioredis";
|
||||
import { sha256Hex } from "./contracts.js";
|
||||
|
||||
const LOCK_KEY_PREFIX = "hivegrid:portal-domain-controller:claim-lock:";
|
||||
const DEFAULT_LEASE_MS = 30_000;
|
||||
const DEFAULT_ACQUIRE_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_RETRY_DELAY_MS = 75;
|
||||
const DEFAULT_RETRY_JITTER_MS = 75;
|
||||
const DEFAULT_COMMAND_TIMEOUT_MS = 2_500;
|
||||
const DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS = 10_000;
|
||||
const FLOCK_BINARY = "/usr/bin/flock";
|
||||
|
||||
const RENEW_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('PEXPIRE', KEYS[1], ARGV[2])", "end", "return 0"].join("\n");
|
||||
|
||||
const RELEASE_SCRIPT = ["if redis.call('GET', KEYS[1]) == ARGV[1] then", " return redis.call('DEL', KEYS[1])", "end", "return 0"].join("\n");
|
||||
|
||||
export type ClaimMutationLock = {
|
||||
assertReady(): Promise<void>;
|
||||
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T>;
|
||||
close(): Promise<void>;
|
||||
};
|
||||
|
||||
export type ClaimLockGuard = Readonly<{
|
||||
assertHeld(): Promise<void>;
|
||||
}>;
|
||||
|
||||
export type ClaimLockRedisClient = {
|
||||
readonly status: string;
|
||||
connect(): Promise<unknown>;
|
||||
ping(): Promise<string>;
|
||||
set(key: string, value: string, expiryMode: "PX", leaseMs: number, condition: "NX"): Promise<"OK" | null>;
|
||||
eval(script: string, numberOfKeys: number, ...args: Array<string | number>): Promise<unknown>;
|
||||
quit(): Promise<unknown>;
|
||||
disconnect(reconnect?: boolean): void;
|
||||
on(event: "error", listener: (error: unknown) => void): unknown;
|
||||
};
|
||||
|
||||
type ClaimLockOptions = Readonly<{
|
||||
leaseMs?: number;
|
||||
acquireTimeoutMs?: number;
|
||||
retryDelayMs?: number;
|
||||
retryJitterMs?: number;
|
||||
heartbeatIntervalMs?: number;
|
||||
now?: () => number;
|
||||
random?: () => number;
|
||||
token?: () => string;
|
||||
wait?: (delayMs: number) => Promise<void>;
|
||||
}>;
|
||||
|
||||
type NormalizedClaimLockOptions = Readonly<{
|
||||
leaseMs: number;
|
||||
acquireTimeoutMs: number;
|
||||
retryDelayMs: number;
|
||||
retryJitterMs: number;
|
||||
heartbeatIntervalMs: number;
|
||||
now: () => number;
|
||||
random: () => number;
|
||||
token: () => string;
|
||||
wait: (delayMs: number) => Promise<void>;
|
||||
}>;
|
||||
|
||||
export class ClaimLockUnavailableError extends Error {
|
||||
readonly retryable = true;
|
||||
readonly reasonCode = "claim_lock_unavailable";
|
||||
|
||||
constructor() {
|
||||
super("Portal-domain claim lock is temporarily unavailable");
|
||||
this.name = "ClaimLockUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ClaimLockLostError extends Error {
|
||||
readonly retryable = true;
|
||||
readonly reasonCode = "claim_lock_lost";
|
||||
|
||||
constructor() {
|
||||
super("Portal-domain claim lock ownership was lost");
|
||||
this.name = "ClaimLockLostError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Kernel-backed ownership for the shared Traefik directory. Redis remains the
|
||||
* fast distributed coordinator, while this non-expiring `flock` is the
|
||||
* authoritative fence around filesystem mutations. The lock is attached to
|
||||
* the parent's open-file description, so a paused Node event loop cannot let a
|
||||
* second controller enter after a Redis lease expires. A crash closes the fd
|
||||
* and lets the kernel release ownership automatically.
|
||||
*/
|
||||
export class FilesystemClaimMutationLock implements ClaimMutationLock {
|
||||
readonly #configuredDirectory: string;
|
||||
readonly #binary: string;
|
||||
readonly #acquireTimeoutMs: number;
|
||||
readonly #activeOperations = new Set<Promise<void>>();
|
||||
#closing = false;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
|
||||
constructor(directory: string, options: { binary?: string; acquireTimeoutMs?: number } = {}) {
|
||||
if (!path.isAbsolute(directory)) throw new ClaimLockUnavailableError();
|
||||
this.#configuredDirectory = path.resolve(directory);
|
||||
this.#binary = options.binary ?? FLOCK_BINARY;
|
||||
this.#acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_FILESYSTEM_LOCK_TIMEOUT_MS, "filesystem claim lock timeout");
|
||||
}
|
||||
|
||||
async assertReady(): Promise<void> {
|
||||
await this.withClaimLock("__readiness__", async (guard) => guard.assertHeld());
|
||||
}
|
||||
|
||||
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
||||
let finishActive: () => void = () => undefined;
|
||||
const active = new Promise<void>((resolve) => {
|
||||
finishActive = resolve;
|
||||
});
|
||||
this.#activeOperations.add(active);
|
||||
try {
|
||||
return await this.#withClaimLock(claimId, operation);
|
||||
} finally {
|
||||
finishActive();
|
||||
this.#activeOperations.delete(active);
|
||||
}
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.#closePromise ??= this.#close();
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #close(): Promise<void> {
|
||||
if (this.#closed) return;
|
||||
this.#closing = true;
|
||||
await Promise.all([...this.#activeOperations]);
|
||||
this.#closed = true;
|
||||
}
|
||||
|
||||
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
const directory = await this.#resolveSafeDirectory();
|
||||
const lockPath = path.join(directory, `.portal-${sha256Hex(claimId)}.lock`);
|
||||
if (path.dirname(lockPath) !== directory) throw new ClaimLockUnavailableError();
|
||||
let handle: FileHandle | undefined;
|
||||
let held = false;
|
||||
try {
|
||||
handle = await open(
|
||||
lockPath,
|
||||
fsConstants.O_CREAT | fsConstants.O_RDWR | noFollowFlag(),
|
||||
0o600
|
||||
);
|
||||
await assertSameRegularFile(lockPath, handle);
|
||||
await acquireKernelFlock(handle.fd, this.#binary, this.#acquireTimeoutMs);
|
||||
held = true;
|
||||
const guard: ClaimLockGuard = Object.freeze({
|
||||
assertHeld: async () => {
|
||||
if (!held || !handle) throw new ClaimLockLostError();
|
||||
await assertSameRegularFile(lockPath, handle);
|
||||
},
|
||||
});
|
||||
return await operation(guard);
|
||||
} catch (error) {
|
||||
if (error instanceof ClaimLockLostError || error instanceof ClaimLockUnavailableError) throw error;
|
||||
throw new ClaimLockUnavailableError();
|
||||
} finally {
|
||||
held = false;
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async #resolveSafeDirectory(): Promise<string> {
|
||||
try {
|
||||
const stats = await lstat(this.#configuredDirectory);
|
||||
if (!stats.isDirectory() || stats.isSymbolicLink()) throw new ClaimLockUnavailableError();
|
||||
return await realpath(this.#configuredDirectory);
|
||||
} catch (error) {
|
||||
if (error instanceof ClaimLockUnavailableError) throw error;
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class FencedClaimMutationLock implements ClaimMutationLock {
|
||||
readonly #filesystem: ClaimMutationLock;
|
||||
readonly #lease: ClaimMutationLock;
|
||||
|
||||
constructor(filesystem: ClaimMutationLock, lease: ClaimMutationLock) {
|
||||
this.#filesystem = filesystem;
|
||||
this.#lease = lease;
|
||||
}
|
||||
|
||||
async assertReady(): Promise<void> {
|
||||
await this.#filesystem.assertReady();
|
||||
await this.#lease.assertReady();
|
||||
}
|
||||
|
||||
withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
return this.#filesystem.withClaimLock(claimId, () => this.#lease.withClaimLock(claimId, operation));
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await Promise.all([this.#lease.close(), this.#filesystem.close()]);
|
||||
}
|
||||
}
|
||||
|
||||
export class RedisClaimMutationLock implements ClaimMutationLock {
|
||||
readonly #client: ClaimLockRedisClient;
|
||||
readonly #options: NormalizedClaimLockOptions;
|
||||
readonly #activeOperations = new Set<Promise<void>>();
|
||||
#closing = false;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
|
||||
constructor(client: ClaimLockRedisClient, options: ClaimLockOptions = {}) {
|
||||
this.#client = client;
|
||||
this.#options = normalizeOptions(options);
|
||||
// Redis connection errors are surfaced through bounded commands and the
|
||||
// readiness endpoint. Keeping this listener intentionally silent also
|
||||
// prevents connection details from reaching stderr outside our logger.
|
||||
this.#client.on("error", () => undefined);
|
||||
}
|
||||
|
||||
async assertReady(): Promise<void> {
|
||||
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
||||
try {
|
||||
if (this.#client.status === "wait") await this.#client.connect();
|
||||
if ((await this.#client.ping()) !== "PONG") throw new ClaimLockUnavailableError();
|
||||
await this.#probeLockCapabilities();
|
||||
} catch {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
if (this.#closing || this.#closed) throw new ClaimLockUnavailableError();
|
||||
let finishActive: () => void = () => undefined;
|
||||
const active = new Promise<void>((resolve) => {
|
||||
finishActive = resolve;
|
||||
});
|
||||
this.#activeOperations.add(active);
|
||||
try {
|
||||
return await this.#withClaimLock(claimId, operation);
|
||||
} finally {
|
||||
finishActive();
|
||||
this.#activeOperations.delete(active);
|
||||
}
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.#closePromise ??= this.#close();
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #close(): Promise<void> {
|
||||
if (this.#closed) return;
|
||||
this.#closing = true;
|
||||
await Promise.all([...this.#activeOperations]);
|
||||
try {
|
||||
if (this.#client.status !== "wait" && this.#client.status !== "end") {
|
||||
await this.#client.quit();
|
||||
}
|
||||
} catch {
|
||||
// An immediate disconnect is safe after all leases were released. It
|
||||
// also keeps shutdown bounded when Redis disappears during a rollout.
|
||||
} finally {
|
||||
this.#client.disconnect(false);
|
||||
this.#closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
async #withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
||||
const key = `${LOCK_KEY_PREFIX}${sha256Hex(claimId)}`;
|
||||
const token = this.#options.token();
|
||||
await this.#acquire(key, token);
|
||||
|
||||
let heartbeatTimer: NodeJS.Timeout | undefined;
|
||||
let heartbeatInFlight: Promise<void> = Promise.resolve();
|
||||
let heartbeatError: ClaimLockLostError | ClaimLockUnavailableError | undefined;
|
||||
let heartbeatStopped = false;
|
||||
const scheduleHeartbeat = () => {
|
||||
if (heartbeatStopped || heartbeatError) return;
|
||||
heartbeatTimer = setTimeout(() => {
|
||||
heartbeatInFlight = this.#renew(key, token)
|
||||
.then(() => scheduleHeartbeat())
|
||||
.catch((error: unknown) => {
|
||||
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
|
||||
});
|
||||
}, this.#options.heartbeatIntervalMs);
|
||||
};
|
||||
scheduleHeartbeat();
|
||||
|
||||
const guard: ClaimLockGuard = Object.freeze({
|
||||
assertHeld: async () => {
|
||||
if (heartbeatError) throw heartbeatError;
|
||||
try {
|
||||
await this.#renew(key, token);
|
||||
} catch (error) {
|
||||
heartbeatError = error instanceof ClaimLockLostError ? error : new ClaimLockUnavailableError();
|
||||
throw heartbeatError;
|
||||
}
|
||||
if (heartbeatError) throw heartbeatError;
|
||||
},
|
||||
});
|
||||
|
||||
let result: T | undefined;
|
||||
let operationError: unknown;
|
||||
let operationFailed = false;
|
||||
try {
|
||||
result = await operation(guard);
|
||||
} catch (error) {
|
||||
operationFailed = true;
|
||||
operationError = error;
|
||||
}
|
||||
|
||||
heartbeatStopped = true;
|
||||
if (heartbeatTimer) clearTimeout(heartbeatTimer);
|
||||
await heartbeatInFlight.catch(() => undefined);
|
||||
const observedHeartbeatError = heartbeatError;
|
||||
let releaseError: ClaimLockUnavailableError | undefined;
|
||||
let released = false;
|
||||
try {
|
||||
released = await this.#release(key, token);
|
||||
} catch {
|
||||
releaseError = new ClaimLockUnavailableError();
|
||||
}
|
||||
|
||||
if (operationFailed) throw operationError;
|
||||
if (observedHeartbeatError) throw observedHeartbeatError;
|
||||
if (releaseError) throw releaseError;
|
||||
if (!released) throw new ClaimLockLostError();
|
||||
return result as T;
|
||||
}
|
||||
|
||||
async #acquire(key: string, token: string): Promise<void> {
|
||||
const deadline = this.#options.now() + this.#options.acquireTimeoutMs;
|
||||
while (!this.#closing && !this.#closed) {
|
||||
try {
|
||||
if ((await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK") return;
|
||||
} catch {
|
||||
// Redis can be transiently unavailable during a rolling restart. The
|
||||
// bounded deadline below turns that into a retryable controller result.
|
||||
}
|
||||
const remainingMs = deadline - this.#options.now();
|
||||
if (remainingMs <= 0) break;
|
||||
const jitterMs = Math.floor(this.#options.random() * (this.#options.retryJitterMs + 1));
|
||||
await this.#options.wait(Math.min(remainingMs, this.#options.retryDelayMs + jitterMs));
|
||||
}
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
|
||||
async #renew(key: string, token: string): Promise<void> {
|
||||
let renewed: unknown;
|
||||
try {
|
||||
renewed = await this.#client.eval(RENEW_SCRIPT, 1, key, token, this.#options.leaseMs);
|
||||
} catch {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
if (Number(renewed) !== 1) throw new ClaimLockLostError();
|
||||
}
|
||||
|
||||
async #release(key: string, token: string): Promise<boolean> {
|
||||
try {
|
||||
return Number(await this.#client.eval(RELEASE_SCRIPT, 1, key, token)) === 1;
|
||||
} catch {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
async #probeLockCapabilities(): Promise<void> {
|
||||
const key = `${LOCK_KEY_PREFIX}readiness:${sha256Hex(randomUUID())}`;
|
||||
const token = randomUUID();
|
||||
let acquired = false;
|
||||
let released = false;
|
||||
try {
|
||||
acquired = (await this.#client.set(key, token, "PX", this.#options.leaseMs, "NX")) === "OK";
|
||||
if (!acquired) throw new ClaimLockUnavailableError();
|
||||
await this.#renew(key, token);
|
||||
released = await this.#release(key, token);
|
||||
if (!released) throw new ClaimLockLostError();
|
||||
} finally {
|
||||
if (acquired && !released) await this.#release(key, token).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createRedisClaimMutationLock(connection: ConnectionOptions): ClaimMutationLock {
|
||||
const client = new Redis({
|
||||
...connection,
|
||||
commandTimeout: DEFAULT_COMMAND_TIMEOUT_MS,
|
||||
enableOfflineQueue: false,
|
||||
enableReadyCheck: true,
|
||||
lazyConnect: true,
|
||||
maxRetriesPerRequest: 1,
|
||||
});
|
||||
return new RedisClaimMutationLock(client);
|
||||
}
|
||||
|
||||
export function createFencedClaimMutationLock(connection: ConnectionOptions, filesystemDirectory: string): ClaimMutationLock {
|
||||
return new FencedClaimMutationLock(new FilesystemClaimMutationLock(filesystemDirectory), createRedisClaimMutationLock(connection));
|
||||
}
|
||||
|
||||
async function assertSameRegularFile(lockPath: string, handle: FileHandle): Promise<void> {
|
||||
const [pathStats, openedStats] = await Promise.all([lstat(lockPath), handle.stat()]);
|
||||
if (
|
||||
!pathStats.isFile() ||
|
||||
pathStats.isSymbolicLink() ||
|
||||
pathStats.nlink !== 1 ||
|
||||
!openedStats.isFile() ||
|
||||
openedStats.nlink !== 1 ||
|
||||
openedStats.dev !== pathStats.dev ||
|
||||
openedStats.ino !== pathStats.ino
|
||||
) {
|
||||
throw new ClaimLockUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
async function acquireKernelFlock(fd: number, binary: string, timeoutMs: number): Promise<void> {
|
||||
const timeoutSeconds = Math.max(0.001, timeoutMs / 1_000).toFixed(3);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(binary, ["--exclusive", "--wait", timeoutSeconds, "3"], {
|
||||
stdio: ["ignore", "ignore", "ignore", fd],
|
||||
});
|
||||
let settled = false;
|
||||
const watchdog = setTimeout(() => {
|
||||
if (settled) return;
|
||||
child.kill("SIGKILL");
|
||||
}, timeoutMs + 1_000);
|
||||
watchdog.unref();
|
||||
child.once("error", () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(watchdog);
|
||||
reject(new ClaimLockUnavailableError());
|
||||
});
|
||||
child.once("close", (code) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(watchdog);
|
||||
if (code === 0) resolve();
|
||||
else reject(new ClaimLockUnavailableError());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function noFollowFlag(): number {
|
||||
return typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_NOFOLLOW : 0;
|
||||
}
|
||||
|
||||
function normalizeOptions(options: ClaimLockOptions): NormalizedClaimLockOptions {
|
||||
const leaseMs = positiveInteger(options.leaseMs ?? DEFAULT_LEASE_MS, "claim lock lease");
|
||||
const acquireTimeoutMs = positiveInteger(options.acquireTimeoutMs ?? DEFAULT_ACQUIRE_TIMEOUT_MS, "claim lock acquisition timeout");
|
||||
const retryDelayMs = nonNegativeInteger(options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS, "claim lock retry delay");
|
||||
const retryJitterMs = nonNegativeInteger(options.retryJitterMs ?? DEFAULT_RETRY_JITTER_MS, "claim lock retry jitter");
|
||||
const heartbeatIntervalMs = positiveInteger(options.heartbeatIntervalMs ?? Math.floor(leaseMs / 3), "claim lock heartbeat interval");
|
||||
if (heartbeatIntervalMs >= leaseMs / 2) throw new Error("claim lock heartbeat interval must be less than half the lease");
|
||||
return {
|
||||
leaseMs,
|
||||
acquireTimeoutMs,
|
||||
retryDelayMs,
|
||||
retryJitterMs,
|
||||
heartbeatIntervalMs,
|
||||
now: options.now ?? Date.now,
|
||||
random: options.random ?? Math.random,
|
||||
token: options.token ?? randomUUID,
|
||||
wait: options.wait ?? wait,
|
||||
};
|
||||
}
|
||||
|
||||
function positiveInteger(value: number, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value: number, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function wait(delayMs: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
}
|
||||
Reference in new issue
Block a user