import { randomUUID } from "node:crypto"; import { MemoryAdapter, type SharedStore } from "@hivegrid/shared-store"; export const DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS = 120; const RENEW_LEASE_SCRIPT = ` if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("expire", KEYS[1], ARGV[2]) end return 0 `; const RELEASE_LEASE_SCRIPT = ` if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) end return 0 `; MemoryAdapter.registerScript(RENEW_LEASE_SCRIPT, async (adapter, keys, args) => { if ((await adapter.get(keys[0]!)) !== args[0]) return 0; return (await adapter.expire(keys[0]!, Number(args[1]))) ? 1 : 0; }); MemoryAdapter.registerScript(RELEASE_LEASE_SCRIPT, async (adapter, keys, args) => { if ((await adapter.get(keys[0]!)) !== args[0]) return 0; return adapter.del(keys[0]!); }); export class DistributedJobLeaseLostError extends Error { constructor(jobKey: string) { super(`Distributed lease was lost while ${jobKey} was running`); this.name = "DistributedJobLeaseLostError"; } } export type DistributedJobLeaseResult = { acquired: true; value: Value } | { acquired: false }; /** * Run a cron body under a replica-wide Redis lease. SET NX prevents overlap; * token-checked Lua renewal/release prevents an expired owner from touching a * successor's lease. Store failures fail closed instead of running unlocked. */ export async function withDistributedJobLease( store: SharedStore, jobKey: string, worker: () => Promise, options: { leaseSeconds?: number; ownerToken?: string } = {} ): Promise> { const leaseSeconds = Math.max(10, Math.min(Math.floor(options.leaseSeconds ?? DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS), 3_600)); const ownerToken = options.ownerToken ?? randomUUID(); const key = `job-lease:${jobKey}`; const acquired = await store.set(key, ownerToken, { nx: true, ttlSeconds: leaseSeconds }); if (!acquired) return { acquired: false }; let stopped = false; let lost = false; let timer: ReturnType | undefined; const heartbeatMs = Math.max(1_000, Math.floor((leaseSeconds * 1_000) / 3)); const scheduleHeartbeat = () => { timer = setTimeout(async () => { if (stopped) return; try { const renewed = await store.eval(RENEW_LEASE_SCRIPT, [key], [ownerToken, String(leaseSeconds)]); if (Number(renewed) !== 1) lost = true; } catch { lost = true; } if (!stopped && !lost) scheduleHeartbeat(); }, heartbeatMs); timer.unref?.(); }; scheduleHeartbeat(); try { const value = await worker(); if (lost) throw new DistributedJobLeaseLostError(jobKey); return { acquired: true, value }; } finally { stopped = true; if (timer) clearTimeout(timer); try { await store.eval(RELEASE_LEASE_SCRIPT, [key], [ownerToken]); } catch { // The bounded TTL is the final safety net when Redis disappears during // release. Never issue an unconditional DEL that could remove a new owner. } } }