369 lines
13 KiB
TypeScript
369 lines
13 KiB
TypeScript
import { existsSync } from "node:fs";
|
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import {
|
|
ClaimLockLostError,
|
|
ClaimLockUnavailableError,
|
|
FencedClaimMutationLock,
|
|
FilesystemClaimMutationLock,
|
|
RedisClaimMutationLock,
|
|
type ClaimLockGuard,
|
|
type ClaimLockRedisClient,
|
|
type ClaimMutationLock,
|
|
} from "./claim-lock.js";
|
|
|
|
describe("Redis portal-domain claim lock", () => {
|
|
it("serializes two controller instances and renews ownership during a long mutation", async () => {
|
|
vi.useFakeTimers();
|
|
const store = new FakeRedisStore();
|
|
const first = lockFor(store.client());
|
|
const second = lockFor(store.client());
|
|
const entered = deferred<void>();
|
|
const finish = deferred<void>();
|
|
const order: string[] = [];
|
|
try {
|
|
const firstOperation = first.withClaimLock("claim_12345678", async () => {
|
|
order.push("first:start");
|
|
entered.resolve();
|
|
await finish.promise;
|
|
order.push("first:end");
|
|
});
|
|
await entered.promise;
|
|
const secondOperation = second.withClaimLock("claim_12345678", async () => {
|
|
order.push("second");
|
|
});
|
|
|
|
// This is longer than the original lease. The second instance must still
|
|
// wait because the first instance renews with its owner token.
|
|
await vi.advanceTimersByTimeAsync(70);
|
|
expect(order).toEqual(["first:start"]);
|
|
finish.resolve();
|
|
await firstOperation;
|
|
await vi.advanceTimersByTimeAsync(1);
|
|
await secondOperation;
|
|
|
|
expect(order).toEqual(["first:start", "first:end", "second"]);
|
|
expect(store.keys()).toEqual([]);
|
|
await Promise.all([first.close(), second.close()]);
|
|
} finally {
|
|
finish.resolve();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("times out acquisition predictably without exposing Redis error details", async () => {
|
|
vi.useFakeTimers();
|
|
const store = new FakeRedisStore();
|
|
const holder = lockFor(store.client());
|
|
const entered = deferred<void>();
|
|
const finish = deferred<void>();
|
|
try {
|
|
const held = holder.withClaimLock("claim_12345678", async () => {
|
|
entered.resolve();
|
|
await finish.promise;
|
|
});
|
|
await entered.promise;
|
|
|
|
const contender = lockFor(store.client(), { acquireTimeoutMs: 15 });
|
|
const contenderAttempt = contender.withClaimLock("claim_12345678", async () => undefined).catch((error: unknown) => error);
|
|
await vi.advanceTimersByTimeAsync(15);
|
|
await expect(contenderAttempt).resolves.toMatchObject({
|
|
reasonCode: "claim_lock_unavailable",
|
|
retryable: true,
|
|
});
|
|
|
|
const failingClient = store.client();
|
|
failingClient.set = async () => {
|
|
throw new Error("redis://:do-not-leak@redis.internal:6379");
|
|
};
|
|
const failing = lockFor(failingClient, { acquireTimeoutMs: 5 });
|
|
const failureAttempt = failing.withClaimLock("claim_87654321", async () => undefined).catch((error: unknown) => error);
|
|
await vi.advanceTimersByTimeAsync(5);
|
|
const failure = await failureAttempt;
|
|
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
|
|
expect(String(failure)).not.toContain("do-not-leak");
|
|
expect(String(failure)).not.toContain("redis.internal");
|
|
|
|
finish.resolve();
|
|
await held;
|
|
await Promise.all([holder.close(), contender.close(), failing.close()]);
|
|
} finally {
|
|
finish.resolve();
|
|
vi.useRealTimers();
|
|
}
|
|
});
|
|
|
|
it("fences a stale writer after lease loss while a second writer owns the claim", async () => {
|
|
const store = new FakeRedisStore();
|
|
const first = lockFor(store.client(), { token: () => "original-owner" });
|
|
const second = lockFor(store.client(), { token: () => "new-owner" });
|
|
const firstEntered = deferred<void>();
|
|
const resumeFirst = deferred<void>();
|
|
const secondEntered = deferred<void>();
|
|
const finishSecond = deferred<void>();
|
|
let resource = "initial";
|
|
|
|
const staleOperation = first.withClaimLock("claim_12345678", async (guard) => {
|
|
firstEntered.resolve();
|
|
await resumeFirst.promise;
|
|
await guard.assertHeld();
|
|
resource = "stale";
|
|
});
|
|
await firstEntered.promise;
|
|
store.deleteOnlyValue();
|
|
|
|
const currentOperation = second.withClaimLock("claim_12345678", async (guard) => {
|
|
await guard.assertHeld();
|
|
resource = "current";
|
|
secondEntered.resolve();
|
|
await finishSecond.promise;
|
|
});
|
|
await secondEntered.promise;
|
|
resumeFirst.resolve();
|
|
|
|
await expect(staleOperation).rejects.toBeInstanceOf(ClaimLockLostError);
|
|
expect(resource).toBe("current");
|
|
expect(store.onlyValue()).toBe("new-owner");
|
|
finishSecond.resolve();
|
|
await currentOperation;
|
|
expect(store.keys()).toEqual([]);
|
|
await Promise.all([first.close(), second.close()]);
|
|
});
|
|
|
|
it("keeps the authoritative fence held after a lease guard passes until the whole mutation returns", async () => {
|
|
const filesystem = new SerializedClaimMutationLock();
|
|
const lease = new PassthroughClaimMutationLock();
|
|
const first = new FencedClaimMutationLock(filesystem, lease);
|
|
const second = new FencedClaimMutationLock(filesystem, lease);
|
|
const firstGuardPassed = deferred<void>();
|
|
const resumeFirst = deferred<void>();
|
|
let secondEntered = false;
|
|
let resource = "initial";
|
|
|
|
const staleMutation = first.withClaimLock("claim_12345678", async (guard) => {
|
|
await guard.assertHeld();
|
|
firstGuardPassed.resolve();
|
|
// Represents an event-loop pause after Redis ownership was checked but
|
|
// immediately before rename/unlink. The filesystem fence must remain.
|
|
await resumeFirst.promise;
|
|
resource = "stale-generation";
|
|
});
|
|
await firstGuardPassed.promise;
|
|
const currentMutation = second.withClaimLock("claim_12345678", async () => {
|
|
secondEntered = true;
|
|
resource = "current-generation";
|
|
});
|
|
|
|
await wait(20);
|
|
expect(secondEntered).toBe(false);
|
|
resumeFirst.resolve();
|
|
await Promise.all([staleMutation, currentMutation]);
|
|
expect(secondEntered).toBe(true);
|
|
expect(resource).toBe("current-generation");
|
|
});
|
|
|
|
it.skipIf(!existsSync("/usr/bin/flock"))("holds one kernel flock across controller instances until the parent fd closes", async () => {
|
|
const directory = await mkdtemp(path.join(os.tmpdir(), "hivegrid-portal-domain-flock-"));
|
|
const first = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
|
|
const second = new FilesystemClaimMutationLock(directory, { acquireTimeoutMs: 1_000 });
|
|
const entered = deferred<void>();
|
|
const finish = deferred<void>();
|
|
let secondEntered = false;
|
|
try {
|
|
const firstMutation = first.withClaimLock("claim_12345678", async () => {
|
|
entered.resolve();
|
|
await finish.promise;
|
|
});
|
|
await entered.promise;
|
|
const secondMutation = second.withClaimLock("claim_12345678", async () => {
|
|
secondEntered = true;
|
|
});
|
|
await wait(30);
|
|
expect(secondEntered).toBe(false);
|
|
finish.resolve();
|
|
await Promise.all([firstMutation, secondMutation]);
|
|
expect(secondEntered).toBe(true);
|
|
} finally {
|
|
await Promise.all([first.close(), second.close()]);
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it("hashes claim identifiers, reports readiness and closes the client", async () => {
|
|
const store = new FakeRedisStore();
|
|
const client = store.client("wait");
|
|
const lock = lockFor(client);
|
|
|
|
await expect(lock.assertReady()).resolves.toBeUndefined();
|
|
expect(store.readinessCalls()).toEqual({ sets: 1, renewals: 1, releases: 1 });
|
|
await lock.withClaimLock("../../raw-secret-claim", async () => {
|
|
expect(store.keys()).toHaveLength(1);
|
|
expect(store.keys()[0]).not.toContain("raw-secret-claim");
|
|
expect(store.keys()[0]).toMatch(/^hivegrid:portal-domain-controller:claim-lock:[a-f0-9]{64}$/u);
|
|
});
|
|
await lock.close();
|
|
await expect(lock.assertReady()).rejects.toBeInstanceOf(ClaimLockUnavailableError);
|
|
expect(client.status).toBe("end");
|
|
});
|
|
|
|
it("fails readiness when Redis permits PING but denies lock writes", async () => {
|
|
const store = new FakeRedisStore();
|
|
const client = store.client("wait");
|
|
client.set = async () => {
|
|
throw new Error("NOPERM redis://:do-not-leak@redis.internal");
|
|
};
|
|
const lock = lockFor(client);
|
|
|
|
const failure = await lock.assertReady().catch((error: unknown) => error);
|
|
expect(failure).toBeInstanceOf(ClaimLockUnavailableError);
|
|
expect(failure).toMatchObject({ reasonCode: "claim_lock_unavailable", retryable: true });
|
|
expect(String(failure)).not.toContain("do-not-leak");
|
|
expect(String(failure)).not.toContain("redis.internal");
|
|
expect(store.keys()).toEqual([]);
|
|
await lock.close();
|
|
});
|
|
});
|
|
|
|
class PassthroughClaimMutationLock implements ClaimMutationLock {
|
|
async assertReady(): Promise<void> {}
|
|
async withClaimLock<T>(_claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
|
return operation({ assertHeld: async () => undefined });
|
|
}
|
|
async close(): Promise<void> {}
|
|
}
|
|
|
|
class SerializedClaimMutationLock implements ClaimMutationLock {
|
|
readonly #tails = new Map<string, Promise<void>>();
|
|
|
|
async assertReady(): Promise<void> {}
|
|
|
|
async withClaimLock<T>(claimId: string, operation: (guard: ClaimLockGuard) => Promise<T>): Promise<T> {
|
|
const previous = this.#tails.get(claimId) ?? Promise.resolve();
|
|
let release: () => void = () => undefined;
|
|
const current = new Promise<void>((resolve) => {
|
|
release = resolve;
|
|
});
|
|
const tail = previous.then(() => current);
|
|
this.#tails.set(claimId, tail);
|
|
await previous;
|
|
try {
|
|
return await operation({ assertHeld: async () => undefined });
|
|
} finally {
|
|
release();
|
|
if (this.#tails.get(claimId) === tail) this.#tails.delete(claimId);
|
|
}
|
|
}
|
|
|
|
async close(): Promise<void> {}
|
|
}
|
|
|
|
type TestLockOptions = ConstructorParameters<typeof RedisClaimMutationLock>[1];
|
|
|
|
function lockFor(client: ClaimLockRedisClient, options: TestLockOptions = {}): RedisClaimMutationLock {
|
|
return new RedisClaimMutationLock(client, {
|
|
leaseMs: 30,
|
|
heartbeatIntervalMs: 8,
|
|
acquireTimeoutMs: 250,
|
|
retryDelayMs: 1,
|
|
retryJitterMs: 0,
|
|
...options,
|
|
});
|
|
}
|
|
|
|
class FakeRedisStore {
|
|
readonly #values = new Map<string, { value: string; expiresAt: number }>();
|
|
#sets = 0;
|
|
#renewals = 0;
|
|
#releases = 0;
|
|
|
|
client(initialStatus: string = "ready"): ClaimLockRedisClient {
|
|
let status = initialStatus;
|
|
const assertConnected = () => {
|
|
if (status !== "ready") throw new Error("redis://:hidden@unavailable.internal");
|
|
};
|
|
return {
|
|
get status() {
|
|
return status;
|
|
},
|
|
connect: async () => {
|
|
status = "ready";
|
|
},
|
|
ping: async () => {
|
|
assertConnected();
|
|
return "PONG";
|
|
},
|
|
set: async (key, value, _expiryMode, leaseMs) => {
|
|
assertConnected();
|
|
this.#sets += 1;
|
|
this.#expire(key);
|
|
if (this.#values.has(key)) return null;
|
|
this.#values.set(key, { value, expiresAt: Date.now() + leaseMs });
|
|
return "OK";
|
|
},
|
|
eval: async (_script, _numberOfKeys, ...args) => {
|
|
assertConnected();
|
|
const [key, token, leaseMs] = args;
|
|
if (typeof key !== "string" || typeof token !== "string") return 0;
|
|
this.#expire(key);
|
|
const entry = this.#values.get(key);
|
|
if (!entry || entry.value !== token) return 0;
|
|
if (leaseMs !== undefined) {
|
|
this.#renewals += 1;
|
|
entry.expiresAt = Date.now() + Number(leaseMs);
|
|
return 1;
|
|
}
|
|
this.#releases += 1;
|
|
this.#values.delete(key);
|
|
return 1;
|
|
},
|
|
quit: async () => {
|
|
status = "end";
|
|
return "OK";
|
|
},
|
|
disconnect: () => {
|
|
status = "end";
|
|
},
|
|
on: () => undefined,
|
|
};
|
|
}
|
|
|
|
keys(): string[] {
|
|
for (const key of this.#values.keys()) this.#expire(key);
|
|
return [...this.#values.keys()];
|
|
}
|
|
|
|
deleteOnlyValue(): void {
|
|
const [key] = this.keys();
|
|
if (!key) throw new Error("expected one lock key");
|
|
this.#values.delete(key);
|
|
}
|
|
|
|
onlyValue(): string | undefined {
|
|
const [key] = this.keys();
|
|
return key ? this.#values.get(key)?.value : undefined;
|
|
}
|
|
|
|
readinessCalls(): { sets: number; renewals: number; releases: number } {
|
|
return { sets: this.#sets, renewals: this.#renewals, releases: this.#releases };
|
|
}
|
|
|
|
#expire(key: string): void {
|
|
const entry = this.#values.get(key);
|
|
if (entry && entry.expiresAt <= Date.now()) this.#values.delete(key);
|
|
}
|
|
}
|
|
|
|
function deferred<T>() {
|
|
let resolve: (value: T | PromiseLike<T>) => void = () => undefined;
|
|
const promise = new Promise<T>((settle) => {
|
|
resolve = settle;
|
|
});
|
|
return { promise, resolve };
|
|
}
|
|
|
|
function wait(delayMs: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
}
|