Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/apps/worker/src/jobs/distributed-job-lease.ts
T

91 lines
3.0 KiB
TypeScript

import { randomUUID } from "node:crypto";
import { MemoryAdapter, type SharedStore } from "@hivegrid/shared-store";
export const DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS = 120;
const RENEW_LEASE_SCRIPT = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("expire", KEYS[1], ARGV[2])
end
return 0
`;
const RELEASE_LEASE_SCRIPT = `
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
end
return 0
`;
MemoryAdapter.registerScript(RENEW_LEASE_SCRIPT, async (adapter, keys, args) => {
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
return (await adapter.expire(keys[0]!, Number(args[1]))) ? 1 : 0;
});
MemoryAdapter.registerScript(RELEASE_LEASE_SCRIPT, async (adapter, keys, args) => {
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
return adapter.del(keys[0]!);
});
export class DistributedJobLeaseLostError extends Error {
constructor(jobKey: string) {
super(`Distributed lease was lost while ${jobKey} was running`);
this.name = "DistributedJobLeaseLostError";
}
}
export type DistributedJobLeaseResult<Value> = { acquired: true; value: Value } | { acquired: false };
/**
* Run a cron body under a replica-wide Redis lease. SET NX prevents overlap;
* token-checked Lua renewal/release prevents an expired owner from touching a
* successor's lease. Store failures fail closed instead of running unlocked.
*/
export async function withDistributedJobLease<Value>(
store: SharedStore,
jobKey: string,
worker: () => Promise<Value>,
options: { leaseSeconds?: number; ownerToken?: string } = {}
): Promise<DistributedJobLeaseResult<Value>> {
const leaseSeconds = Math.max(10, Math.min(Math.floor(options.leaseSeconds ?? DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS), 3_600));
const ownerToken = options.ownerToken ?? randomUUID();
const key = `job-lease:${jobKey}`;
const acquired = await store.set(key, ownerToken, { nx: true, ttlSeconds: leaseSeconds });
if (!acquired) return { acquired: false };
let stopped = false;
let lost = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const heartbeatMs = Math.max(1_000, Math.floor((leaseSeconds * 1_000) / 3));
const scheduleHeartbeat = () => {
timer = setTimeout(async () => {
if (stopped) return;
try {
const renewed = await store.eval(RENEW_LEASE_SCRIPT, [key], [ownerToken, String(leaseSeconds)]);
if (Number(renewed) !== 1) lost = true;
} catch {
lost = true;
}
if (!stopped && !lost) scheduleHeartbeat();
}, heartbeatMs);
timer.unref?.();
};
scheduleHeartbeat();
try {
const value = await worker();
if (lost) throw new DistributedJobLeaseLostError(jobKey);
return { acquired: true, value };
} finally {
stopped = true;
if (timer) clearTimeout(timer);
try {
await store.eval(RELEASE_LEASE_SCRIPT, [key], [ownerToken]);
} catch {
// The bounded TTL is the final safety net when Redis disappears during
// release. Never issue an unconditional DEL that could remove a new owner.
}
}
}