Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+90
@@ -0,0 +1,90 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import { MemoryAdapter, type SharedStore } from "@hivegrid/shared-store";
|
||||
|
||||
export const DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS = 120;
|
||||
|
||||
const RENEW_LEASE_SCRIPT = `
|
||||
if redis.call("get", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("expire", KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
const RELEASE_LEASE_SCRIPT = `
|
||||
if redis.call("get", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("del", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`;
|
||||
|
||||
MemoryAdapter.registerScript(RENEW_LEASE_SCRIPT, async (adapter, keys, args) => {
|
||||
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
|
||||
return (await adapter.expire(keys[0]!, Number(args[1]))) ? 1 : 0;
|
||||
});
|
||||
|
||||
MemoryAdapter.registerScript(RELEASE_LEASE_SCRIPT, async (adapter, keys, args) => {
|
||||
if ((await adapter.get(keys[0]!)) !== args[0]) return 0;
|
||||
return adapter.del(keys[0]!);
|
||||
});
|
||||
|
||||
export class DistributedJobLeaseLostError extends Error {
|
||||
constructor(jobKey: string) {
|
||||
super(`Distributed lease was lost while ${jobKey} was running`);
|
||||
this.name = "DistributedJobLeaseLostError";
|
||||
}
|
||||
}
|
||||
|
||||
export type DistributedJobLeaseResult<Value> = { acquired: true; value: Value } | { acquired: false };
|
||||
|
||||
/**
|
||||
* Run a cron body under a replica-wide Redis lease. SET NX prevents overlap;
|
||||
* token-checked Lua renewal/release prevents an expired owner from touching a
|
||||
* successor's lease. Store failures fail closed instead of running unlocked.
|
||||
*/
|
||||
export async function withDistributedJobLease<Value>(
|
||||
store: SharedStore,
|
||||
jobKey: string,
|
||||
worker: () => Promise<Value>,
|
||||
options: { leaseSeconds?: number; ownerToken?: string } = {}
|
||||
): Promise<DistributedJobLeaseResult<Value>> {
|
||||
const leaseSeconds = Math.max(10, Math.min(Math.floor(options.leaseSeconds ?? DEFAULT_DISTRIBUTED_JOB_LEASE_SECONDS), 3_600));
|
||||
const ownerToken = options.ownerToken ?? randomUUID();
|
||||
const key = `job-lease:${jobKey}`;
|
||||
const acquired = await store.set(key, ownerToken, { nx: true, ttlSeconds: leaseSeconds });
|
||||
if (!acquired) return { acquired: false };
|
||||
|
||||
let stopped = false;
|
||||
let lost = false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const heartbeatMs = Math.max(1_000, Math.floor((leaseSeconds * 1_000) / 3));
|
||||
const scheduleHeartbeat = () => {
|
||||
timer = setTimeout(async () => {
|
||||
if (stopped) return;
|
||||
try {
|
||||
const renewed = await store.eval(RENEW_LEASE_SCRIPT, [key], [ownerToken, String(leaseSeconds)]);
|
||||
if (Number(renewed) !== 1) lost = true;
|
||||
} catch {
|
||||
lost = true;
|
||||
}
|
||||
if (!stopped && !lost) scheduleHeartbeat();
|
||||
}, heartbeatMs);
|
||||
timer.unref?.();
|
||||
};
|
||||
scheduleHeartbeat();
|
||||
|
||||
try {
|
||||
const value = await worker();
|
||||
if (lost) throw new DistributedJobLeaseLostError(jobKey);
|
||||
return { acquired: true, value };
|
||||
} finally {
|
||||
stopped = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
try {
|
||||
await store.eval(RELEASE_LEASE_SCRIPT, [key], [ownerToken]);
|
||||
} catch {
|
||||
// The bounded TTL is the final safety net when Redis disappears during
|
||||
// release. Never issue an unconditional DEL that could remove a new owner.
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user