91 lines
3.0 KiB
TypeScript
91 lines
3.0 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
|
|
import { MemoryAdapter, type SharedStore } from "@hivegrid/shared-store";
|
|
|
|
export const DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS = 120;
|
|
|
|
const RENEW_LEASE_SCRIPT = `
|
|
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
return redis.call("expire", KEYS[1], ARGV[2])
|
|
end
|
|
return 0
|
|
`;
|
|
|
|
const RELEASE_LEASE_SCRIPT = `
|
|
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
return redis.call("del", KEYS[1])
|
|
end
|
|
return 0
|
|
`;
|
|
|
|
MemoryAdapter.registerScript(RENEW_LEASE_SCRIPT, async (adapter, keys, args) => {
|
|
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
|
|
return (await adapter.expire(keys[0]!, Number(args[1]))) ? 1 : 0;
|
|
});
|
|
|
|
MemoryAdapter.registerScript(RELEASE_LEASE_SCRIPT, async (adapter, keys, args) => {
|
|
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
|
|
return adapter.del(keys[0]!);
|
|
});
|
|
|
|
export class DistributedJobLeaseLostError extends Error {
|
|
constructor(jobKey: string) {
|
|
super(`Distributed lease was lost while ${jobKey} was running`);
|
|
this.name = "DistributedJobLeaseLostError";
|
|
}
|
|
}
|
|
|
|
export type DistributedJobLeaseResult<Value> = { acquired: true; value: Value } | { acquired: false };
|
|
|
|
/**
|
|
* Run a cron body under a replica-wide Redis lease. SET NX prevents overlap;
|
|
* token-checked Lua renewal/release prevents an expired owner from touching a
|
|
* successor's lease. Store failures fail closed instead of running unlocked.
|
|
*/
|
|
export async function withDistributedJobLease<Value>(
|
|
store: SharedStore,
|
|
jobKey: string,
|
|
worker: () => Promise<Value>,
|
|
options: { leaseSeconds?: number; ownerToken?: string } = {}
|
|
): Promise<DistributedJobLeaseResult<Value>> {
|
|
const leaseSeconds = Math.max(10, Math.min(Math.floor(options.leaseSeconds ?? DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS), 3_600));
|
|
const ownerToken = options.ownerToken ?? randomUUID();
|
|
const key = `job-lease:${jobKey}`;
|
|
const acquired = await store.set(key, ownerToken, { nx: true, ttlSeconds: leaseSeconds });
|
|
if (!acquired) return { acquired: false };
|
|
|
|
let stopped = false;
|
|
let lost = false;
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
const heartbeatMs = Math.max(1_000, Math.floor((leaseSeconds * 1_000) / 3));
|
|
const scheduleHeartbeat = () => {
|
|
timer = setTimeout(async () => {
|
|
if (stopped) return;
|
|
try {
|
|
const renewed = await store.eval(RENEW_LEASE_SCRIPT, [key], [ownerToken, String(leaseSeconds)]);
|
|
if (Number(renewed) !== 1) lost = true;
|
|
} catch {
|
|
lost = true;
|
|
}
|
|
if (!stopped && !lost) scheduleHeartbeat();
|
|
}, heartbeatMs);
|
|
timer.unref?.();
|
|
};
|
|
scheduleHeartbeat();
|
|
|
|
try {
|
|
const value = await worker();
|
|
if (lost) throw new DistributedJobLeaseLostError(jobKey);
|
|
return { acquired: true, value };
|
|
} finally {
|
|
stopped = true;
|
|
if (timer) clearTimeout(timer);
|
|
try {
|
|
await store.eval(RELEASE_LEASE_SCRIPT, [key], [ownerToken]);
|
|
} catch {
|
|
// The bounded TTL is the final safety net when Redis disappears during
|
|
// release. Never issue an unconditional DEL that could remove a new owner.
|
|
}
|
|
}
|
|
}
|