30 lines
1.1 KiB
TypeScript
30 lines
1.1 KiB
TypeScript
import { OUTBOX_MAX_PAYLOAD_BYTES } from "@hivegrid/db/outbox";
|
|
|
|
const SEARCH_OUTBOX_SAFETY_MARGIN_BYTES = 8 * 1024;
|
|
export const SEARCH_OUTBOX_TARGET_BYTES = OUTBOX_MAX_PAYLOAD_BYTES - SEARCH_OUTBOX_SAFETY_MARGIN_BYTES;
|
|
|
|
function payloadBytes(value: unknown): number {
|
|
return Buffer.byteLength(JSON.stringify(value), "utf8");
|
|
}
|
|
|
|
export function splitSearchPayload<T>(values: readonly T[], payloadFor: (batch: readonly T[]) => unknown, maxItems = 250): T[][] {
|
|
const batches: T[][] = [];
|
|
let current: T[] = [];
|
|
|
|
for (const value of values) {
|
|
const candidate = [...current, value];
|
|
if (candidate.length <= maxItems && payloadBytes(payloadFor(candidate)) <= SEARCH_OUTBOX_TARGET_BYTES) {
|
|
current = candidate;
|
|
continue;
|
|
}
|
|
if (current.length === 0) throw new Error("demo-pro seed: one search document exceeds the safe outbox payload size");
|
|
batches.push(current);
|
|
current = [value];
|
|
if (payloadBytes(payloadFor(current)) > SEARCH_OUTBOX_TARGET_BYTES) {
|
|
throw new Error("demo-pro seed: one search document exceeds the safe outbox payload size");
|
|
}
|
|
}
|
|
if (current.length > 0) batches.push(current);
|
|
return batches;
|
|
}
|