Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/portal-domain-controller/src/claim-lock.test.ts
T

369 lines
13 KiB
TypeScript

import { existsSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import {
ClaimLockLostError,
ClaimLockUnavailableError,
FencedClaimMutationLock,
FilesystemClaimMutationLock,
RedisClaimMutationLock,
type ClaimLockGuard,
type ClaimLockRedisClient,
type ClaimMutationLock,
} from "./claim-lock.js";
describe("Redis portal-domain claim lock", () => {
it("serializes two controller instances and renews ownership during a long mutation", async () => {
vi.useFakeTimers();
const store = new FakeRedisStore();
const first = lockFor(store.client());
const second = lockFor(store.client());
const entered = deferred<void>();
const finish = deferred<void>();
const order: string[] = [];
try {
const firstOperation = first.withClaimLock("claim_12345678", async () => {
order.push("first:start");
entered.resolve();
await finish.promise;
order.push("first:end");
});
await entered.promise;
const secondOperation = second.withClaimLock("claim_12345678", async () => {
order.push("second");
});
// This is longer than the original lease. The second instance must still
// wait because the first instance renews with its owner token.
await vi.advanceTimersByTimeAsync(70);
expect(order).toEqual(["first:start"]);
finish.resolve();
await firstOperation;
await vi.advanceTimersByTimeAsync(1);
await secondOperation;
expect(order).toEqual(["first:start", "first:end", "second"]);
expect(store.keys()).toEqual([]);
await Promise.all([first.close(), second.close()]);
} finally {
finish.resolve();
vi.useRealTimers();
}
});
it("times out acquisition predictably without exposing Redis error details", async () => {
vi.useFakeTimers();
const store = new FakeRedisStore();
const holder = lockFor(store.client());
const entered = deferred<void>();
const finish = deferred<void>();
try {
const held = holder.withClaimLock("claim_12345678", async () => {
entered.resolve();
await finish.promise;
});
await entered.promise;
const contender = lockFor(store.client(), { acquireTimeoutMs: 15 });
const contenderAttempt = contender.withClaimLock("claim_12345678", async () => undefined).catch((error: unknown) => error);
await vi.advanceTimersByTimeAsync(15);
await expect(contenderAttempt).resolves.toMatchObject({
reasonCode: "claim_lock_unavailable",
retryable: true,
});
const failingClient = store.client();
failingClient.set = async () => {
throw new Error("redis://:do-not-leak@redis.internal:6379");
};
const failing = lockFor(failingClient, { acquireTimeoutMs: 5 });
const failureAttempt = failing.withClaimLock("claim_87654321", async () => undefined).catch((error: unknown) => error);
await vi.advanceTimersByTimeAsync(5);
const failure = await failureAttempt;
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
expect(String(failure)).not.toContain("do-not-leak");
expect(String(failure)).not.toContain("redis.internal");
finish.resolve();
await held;
await Promise.all([holder.close(), contender.close(), failing.close()]);
} finally {
finish.resolve();
vi.useRealTimers();
}
});
it("fences a stale writer after lease loss while a second writer owns the claim", async () => {
const store = new FakeRedisStore();
const first = lockFor(store.client(), { token: () => "original-owner" });
const second = lockFor(store.client(), { token: () => "new-owner" });
const firstEntered = deferred<void>();
const resumeFirst = deferred<void>();
const secondEntered = deferred<void>();
const finishSecond = deferred<void>();
let resource = "initial";
const staleOperation = first.withClaimLock("claim_12345678", async (guard) => {
firstEntered.resolve();
await resumeFirst.promise;
await guard.assertHeld();
resource = "stale";
});
await firstEntered.promise;
store.deleteOnlyValue();
const currentOperation = second.withClaimLock("claim_12345678", async (guard) => {
await guard.assertHeld();
resource = "current";
secondEntered.resolve();
await finishSecond.promise;
});
await secondEntered.promise;
resumeFirst.resolve();
await expect(staleOperation).rejects.toBeInstanceOf(ClaimLockLostError);
expect(resource).toBe("current");
expect(store.onlyValue()).toBe("new-owner");
finishSecond.resolve();
await currentOperation;
expect(store.keys()).toEqual([]);
await Promise.all([first.close(), second.close()]);
});
it("keeps the authoritative fence held after a lease guard passes until the whole mutation returns", async () => {
const filesystem = new SerializedClaimMutationLock();
const lease = new PassthroughClaimMutationLock();
const first = new FencedClaimMutationLock(filesystem, lease);
const second = new FencedClaimMutationLock(filesystem, lease);
const firstGuardPassed = deferred<void>();
const resumeFirst = deferred<void>();
let secondEntered = false;
let resource = "initial";
const staleMutation = first.withClaimLock("claim_12345678", async (guard) => {
await guard.assertHeld();
firstGuardPassed.resolve();
// Represents an event-loop pause after Redis ownership was checked but
// immediately before rename/unlink. The filesystem fence must remain.
await resumeFirst.promise;
resource = "stale-generation";
});
await firstGuardPassed.promise;
const currentMutation = second.withClaimLock("claim_12345678", async () => {
secondEntered = true;
resource = "current-generation";
});
await wait(20);
expect(secondEntered).toBe(false);
resumeFirst.resolve();
await Promise.all([staleMutation, currentMutation]);
expect(secondEntered).toBe(true);
expect(resource).toBe("current-generation");
});
it.skipIf(!existsSync("/usr/bin/flock"))("holds one kernel flock across controller instances until the parent fd closes", async () => {
const directory = await mkdtemp(path.join(os.tmpdir(), "hivegrid-portal-domain-flock-"));
const first = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
const second = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
const entered = deferred<void>();
const finish = deferred<void>();
let secondEntered = false;
try {
const firstMutation = first.withClaimLock("claim_12345678", async () => {
entered.resolve();
await finish.promise;
});
await entered.promise;
const secondMutation = second.withClaimLock("claim_12345678", async () => {
secondEntered = true;
});
await wait(30);
expect(secondEntered).toBe(false);
finish.resolve();
await Promise.all([firstMutation, secondMutation]);
expect(secondEntered).toBe(true);
} finally {
await Promise.all([first.close(), second.close()]);
await rm(directory, { recursive: true, force: true });
}
});
it("hashes claim identifiers, reports readiness and closes the client", async () => {
const store = new FakeRedisStore();
const client = store.client("wait");
const lock = lockFor(client);
await expect(lock.assertReady()).resolves.toBeUndefined();
expect(store.readinessCalls()).toEqual({ sets: 1, renewals: 1, releases: 1 });
await lock.withClaimLock("../../raw-secret-claim", async () => {
expect(store.keys()).toHaveLength(1);
expect(store.keys()[0]).not.toContain("raw-secret-claim");
expect(store.keys()[0]).toMatch(/^hivegrid:portal-domain-controller:claim-lock:[a-f0-9]{64}$/u);
});
await lock.close();
await expect(lock.assertReady()).rejects.toBeInstanceOf(ClaimLockUnavailableError);
expect(client.status).toBe("end");
});
it("fails readiness when Redis permits PING but denies lock writes", async () => {
const store = new FakeRedisStore();
const client = store.client("wait");
client.set = async () => {
throw new Error("NOPERM redis://:do-not-leak@redis.internal");
};
const lock = lockFor(client);
const failure = await lock.assertReady().catch((error: unknown) => error);
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
expect(failure).toMatchObject({ reasonCode: "claim_lock_unavailable", retryable: true });
expect(String(failure)).not.toContain("do-not-leak");
expect(String(failure)).not.toContain("redis.internal");
expect(store.keys()).toEqual([]);
await lock.close();
});
});
class PassthroughClaimMutationLock implements ClaimMutationLock {
async assertReady(): Promise<void> {}
async withClaimLock<T>(_claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
return operation({ assertHeld: async () => undefined });
}
async close(): Promise<void> {}
}
class SerializedClaimMutationLock implements ClaimMutationLock {
readonly #tails = new Map<string, Promise<void>>();
async assertReady(): Promise<void> {}
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
const previous = this.#tails.get(claimId) ?? Promise.resolve();
let release: () => void = () => undefined;
const current = new Promise<void>((resolve) => {
release = resolve;
});
const tail = previous.then(() => current);
this.#tails.set(claimId, tail);
await previous;
try {
return await operation({ assertHeld: async () => undefined });
} finally {
release();
if (this.#tails.get(claimId) === tail) this.#tails.delete(claimId);
}
}
async close(): Promise<void> {}
}
type TestLockOptions = ConstructorParameters<typeof RedisClaimMutationLock>[1];
function lockFor(client: ClaimLockRedisClient, options: TestLockOptions = {}): RedisClaimMutationLock {
return new RedisClaimMutationLock(client, {
leaseMs: 30,
heartbeatIntervalMs: 8,
acquireTimeoutMs: 250,
retryDelayMs: 1,
retryJitterMs: 0,
...options,
});
}
class FakeRedisStore {
readonly #values = new Map<string, { value: string; expiresAt: number }>();
#sets = 0;
#renewals = 0;
#releases = 0;
client(initialStatus: string = "ready"): ClaimLockRedisClient {
let status = initialStatus;
const assertConnected = () => {
if (status !== "ready") throw new Error("redis://:hidden@unavailable.internal");
};
return {
get status() {
return status;
},
connect: async () => {
status = "ready";
},
ping: async () => {
assertConnected();
return "PONG";
},
set: async (key, value, _expiryMode, leaseMs) => {
assertConnected();
this.#sets += 1;
this.#expire(key);
if (this.#values.has(key)) return null;
this.#values.set(key, { value, expiresAt: Date.now() + leaseMs });
return "OK";
},
eval: async (_script, _numberOfKeys, ...args) => {
assertConnected();
const [key, token, leaseMs] = args;
if (typeof key !== "string" || typeof token !== "string") return 0;
this.#expire(key);
const entry = this.#values.get(key);
if (!entry || entry.value !== token) return 0;
if (leaseMs !== undefined) {
this.#renewals += 1;
entry.expiresAt = Date.now() + Number(leaseMs);
return 1;
}
this.#releases += 1;
this.#values.delete(key);
return 1;
},
quit: async () => {
status = "end";
return "OK";
},
disconnect: () => {
status = "end";
},
on: () => undefined,
};
}
keys(): string[] {
for (const key of this.#values.keys()) this.#expire(key);
return [...this.#values.keys()];
}
deleteOnlyValue(): void {
const [key] = this.keys();
if (!key) throw new Error("expected one lock key");
this.#values.delete(key);
}
onlyValue(): string | undefined {
const [key] = this.keys();
return key ? this.#values.get(key)?.value : undefined;
}
readinessCalls(): { sets: number; renewals: number; releases: number } {
return { sets: this.#sets, renewals: this.#renewals, releases: this.#releases };
}
#expire(key: string): void {
const entry = this.#values.get(key);
if (entry && entry.expiresAt <= Date.now()) this.#values.delete(key);
}
}
function deferred<T>() {
let resolve: (value: T | PromiseLike<T>) => void = () => undefined;
const promise = new Promise<T>((settle) => {
resolve = settle;
});
return { promise, resolve };
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs));
}