import { existsSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { ClaimLockLostError, ClaimLockUnavailableError, FencedClaimMutationLock, FilesystemClaimMutationLock, RedisClaimMutationLock, type ClaimLockGuard, type ClaimLockRedisClient, type ClaimMutationLock, } from "./claim-lock.js"; describe("Redis portal-domain claim lock", () => { it("serializes two controller instances and renews ownership during a long mutation", async () => { vi.useFakeTimers(); const store = new FakeRedisStore(); const first = lockFor(store.client()); const second = lockFor(store.client()); const entered = deferred(); const finish = deferred(); const order: string[] = []; try { const firstOperation = first.withClaimLock("claim_12345678", async () => { order.push("first:start"); entered.resolve(); await finish.promise; order.push("first:end"); }); await entered.promise; const secondOperation = second.withClaimLock("claim_12345678", async () => { order.push("second"); }); // This is longer than the original lease. The second instance must still // wait because the first instance renews with its owner token. await vi.advanceTimersByTimeAsync(70); expect(order).toEqual(["first:start"]); finish.resolve(); await firstOperation; await vi.advanceTimersByTimeAsync(1); await secondOperation; expect(order).toEqual(["first:start", "first:end", "second"]); expect(store.keys()).toEqual([]); await Promise.all([first.close(), second.close()]); } finally { finish.resolve(); vi.useRealTimers(); } }); it("times out acquisition predictably without exposing Redis error details", async () => { vi.useFakeTimers(); const store = new FakeRedisStore(); const holder = lockFor(store.client()); const entered = deferred(); const finish = deferred(); try { const held = holder.withClaimLock("claim_12345678", async () => { entered.resolve(); await finish.promise; }); await entered.promise; const contender = lockFor(store.client(), { acquireTimeoutMs: 15 }); const contenderAttempt = contender.withClaimLock("claim_12345678", async () => undefined).catch((error: unknown) => error); await vi.advanceTimersByTimeAsync(15); await expect(contenderAttempt).resolves.toMatchObject({ reasonCode: "claim_lock_unavailable", retryable: true, }); const failingClient = store.client(); failingClient.set = async () => { throw new Error("redis://:do-not-leak@redis.internal:6379"); }; const failing = lockFor(failingClient, { acquireTimeoutMs: 5 }); const failureAttempt = failing.withClaimLock("claim_87654321", async () => undefined).catch((error: unknown) => error); await vi.advanceTimersByTimeAsync(5); const failure = await failureAttempt; expect(failure).toBeInstanceOf(ClaimLockUnavailableError); expect(String(failure)).not.toContain("do-not-leak"); expect(String(failure)).not.toContain("redis.internal"); finish.resolve(); await held; await Promise.all([holder.close(), contender.close(), failing.close()]); } finally { finish.resolve(); vi.useRealTimers(); } }); it("fences a stale writer after lease loss while a second writer owns the claim", async () => { const store = new FakeRedisStore(); const first = lockFor(store.client(), { token: () => "original-owner" }); const second = lockFor(store.client(), { token: () => "new-owner" }); const firstEntered = deferred(); const resumeFirst = deferred(); const secondEntered = deferred(); const finishSecond = deferred(); let resource = "initial"; const staleOperation = first.withClaimLock("claim_12345678", async (guard) => { firstEntered.resolve(); await resumeFirst.promise; await guard.assertHeld(); resource = "stale"; }); await firstEntered.promise; store.deleteOnlyValue(); const currentOperation = second.withClaimLock("claim_12345678", async (guard) => { await guard.assertHeld(); resource = "current"; secondEntered.resolve(); await finishSecond.promise; }); await secondEntered.promise; resumeFirst.resolve(); await expect(staleOperation).rejects.toBeInstanceOf(ClaimLockLostError); expect(resource).toBe("current"); expect(store.onlyValue()).toBe("new-owner"); finishSecond.resolve(); await currentOperation; expect(store.keys()).toEqual([]); await Promise.all([first.close(), second.close()]); }); it("keeps the authoritative fence held after a lease guard passes until the whole mutation returns", async () => { const filesystem = new SerializedClaimMutationLock(); const lease = new PassthroughClaimMutationLock(); const first = new FencedClaimMutationLock(filesystem, lease); const second = new FencedClaimMutationLock(filesystem, lease); const firstGuardPassed = deferred(); const resumeFirst = deferred(); let secondEntered = false; let resource = "initial"; const staleMutation = first.withClaimLock("claim_12345678", async (guard) => { await guard.assertHeld(); firstGuardPassed.resolve(); // Represents an event-loop pause after Redis ownership was checked but // immediately before rename/unlink. The filesystem fence must remain. await resumeFirst.promise; resource = "stale-generation"; }); await firstGuardPassed.promise; const currentMutation = second.withClaimLock("claim_12345678", async () => { secondEntered = true; resource = "current-generation"; }); await wait(20); expect(secondEntered).toBe(false); resumeFirst.resolve(); await Promise.all([staleMutation, currentMutation]); expect(secondEntered).toBe(true); expect(resource).toBe("current-generation"); }); it.skipIf(!existsSync("/usr/bin/flock"))("holds one kernel flock across controller instances until the parent fd closes", async () => { const directory = await mkdtemp(path.join(os.tmpdir(), "hivegrid-portal-domain-flock-")); const first = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 }); const second = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 }); const entered = deferred(); const finish = deferred(); let secondEntered = false; try { const firstMutation = first.withClaimLock("claim_12345678", async () => { entered.resolve(); await finish.promise; }); await entered.promise; const secondMutation = second.withClaimLock("claim_12345678", async () => { secondEntered = true; }); await wait(30); expect(secondEntered).toBe(false); finish.resolve(); await Promise.all([firstMutation, secondMutation]); expect(secondEntered).toBe(true); } finally { await Promise.all([first.close(), second.close()]); await rm(directory, { recursive: true, force: true }); } }); it("hashes claim identifiers, reports readiness and closes the client", async () => { const store = new FakeRedisStore(); const client = store.client("wait"); const lock = lockFor(client); await expect(lock.assertReady()).resolves.toBeUndefined(); expect(store.readinessCalls()).toEqual({ sets: 1, renewals: 1, releases: 1 }); await lock.withClaimLock("../../raw-secret-claim", async () => { expect(store.keys()).toHaveLength(1); expect(store.keys()[0]).not.toContain("raw-secret-claim"); expect(store.keys()[0]).toMatch(/^hivegrid:portal-domain-controller:claim-lock:[a-f0-9]{64}$/u); }); await lock.close(); await expect(lock.assertReady()).rejects.toBeInstanceOf(ClaimLockUnavailableError); expect(client.status).toBe("end"); }); it("fails readiness when Redis permits PING but denies lock writes", async () => { const store = new FakeRedisStore(); const client = store.client("wait"); client.set = async () => { throw new Error("NOPERM redis://:do-not-leak@redis.internal"); }; const lock = lockFor(client); const failure = await lock.assertReady().catch((error: unknown) => error); expect(failure).toBeInstanceOf(ClaimLockUnavailableError); expect(failure).toMatchObject({ reasonCode: "claim_lock_unavailable", retryable: true }); expect(String(failure)).not.toContain("do-not-leak"); expect(String(failure)).not.toContain("redis.internal"); expect(store.keys()).toEqual([]); await lock.close(); }); }); class PassthroughClaimMutationLock implements ClaimMutationLock { async assertReady(): Promise {} async withClaimLock(_claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise { return operation({ assertHeld: async () => undefined }); } async close(): Promise {} } class SerializedClaimMutationLock implements ClaimMutationLock { readonly #tails = new Map>(); async assertReady(): Promise {} async withClaimLock(claimId: string, operation: (guard: ClaimLockGuard) => Promise): Promise { const previous = this.#tails.get(claimId) ?? Promise.resolve(); let release: () => void = () => undefined; const current = new Promise((resolve) => { release = resolve; }); const tail = previous.then(() => current); this.#tails.set(claimId, tail); await previous; try { return await operation({ assertHeld: async () => undefined }); } finally { release(); if (this.#tails.get(claimId) === tail) this.#tails.delete(claimId); } } async close(): Promise {} } type TestLockOptions = ConstructorParameters[1]; function lockFor(client: ClaimLockRedisClient, options: TestLockOptions = {}): RedisClaimMutationLock { return new RedisClaimMutationLock(client, { leaseMs: 30, heartbeatIntervalMs: 8, acquireTimeoutMs: 250, retryDelayMs: 1, retryJitterMs: 0, ...options, }); } class FakeRedisStore { readonly #values = new Map(); #sets = 0; #renewals = 0; #releases = 0; client(initialStatus: string = "ready"): ClaimLockRedisClient { let status = initialStatus; const assertConnected = () => { if (status !== "ready") throw new Error("redis://:hidden@unavailable.internal"); }; return { get status() { return status; }, connect: async () => { status = "ready"; }, ping: async () => { assertConnected(); return "PONG"; }, set: async (key, value, _expiryMode, leaseMs) => { assertConnected(); this.#sets += 1; this.#expire(key); if (this.#values.has(key)) return null; this.#values.set(key, { value, expiresAt: Date.now() + leaseMs }); return "OK"; }, eval: async (_script, _numberOfKeys, ...args) => { assertConnected(); const [key, token, leaseMs] = args; if (typeof key !== "string" || typeof token !== "string") return 0; this.#expire(key); const entry = this.#values.get(key); if (!entry || entry.value !== token) return 0; if (leaseMs !== undefined) { this.#renewals += 1; entry.expiresAt = Date.now() + Number(leaseMs); return 1; } this.#releases += 1; this.#values.delete(key); return 1; }, quit: async () => { status = "end"; return "OK"; }, disconnect: () => { status = "end"; }, on: () => undefined, }; } keys(): string[] { for (const key of this.#values.keys()) this.#expire(key); return [...this.#values.keys()]; } deleteOnlyValue(): void { const [key] = this.keys(); if (!key) throw new Error("expected one lock key"); this.#values.delete(key); } onlyValue(): string | undefined { const [key] = this.keys(); return key ? this.#values.get(key)?.value : undefined; } readinessCalls(): { sets: number; renewals: number; releases: number } { return { sets: this.#sets, renewals: this.#renewals, releases: this.#releases }; } #expire(key: string): void { const entry = this.#values.get(key); if (entry && entry.expiresAt <= Date.now()) this.#values.delete(key); } } function deferred() { let resolve: (value: T | PromiseLike) => void = () => undefined; const promise = new Promise((settle) => { resolve = settle; }); return { promise, resolve }; } function wait(delayMs: number): Promise { return new Promise((resolve) => setTimeout(resolve, delayMs)); }