Refresh HiveGrid Cloud reference snapshot

This commit is contained in:
david committed 2026-08-03 15:31:14 +02:00
1 parent 5e99e14772
commit 7f88026f79
5955 files changed
+1177325 -188362

No files matched your search

@@ -43,6 +43,8 @@ const DEFAULT_TAKE = 25;
const INITIAL_SYNC_DAYS = 90;
/** Pages per initial-sync run per folder — keeps a run bounded + resumable. */
const INITIAL_SYNC_MAX_PAGES_PER_RUN = 5;
/** Keep Gmail latency bounded without creating a provider rate-limit burst. */
const GMAIL_MESSAGE_FETCH_CONCURRENCY = 5;
/** Estimated total for the Klartext percent line when the provider gives one. */
const WEBHOOK_RETENTION_DAYS = 30;
const WEBHOOK_FAILED_RETENTION_DAYS = 90;
@@ -159,7 +161,7 @@ async function ensureFreshCreds(
client: EmailWorkerClient,
mailbox: { id: string; tenantId: string; provider: EmailProvider },
creds: EmailCreds,
now = new Date(),
now = new Date()
): Promise<EmailCreds> {
if (creds.expiresAt && creds.expiresAt.getTime() > now.getTime() + 60_000) {
return creds;
@@ -208,7 +210,7 @@ export type EmailMessageUpsertedHook = (input: {
export async function upsertNeutralMessage(
client: EmailWorkerClient,
ctx: { tenantId: string; mailboxId: string },
message: MappedEmailMessage,
message: MappedEmailMessage
): Promise<{ id: string; created: boolean }> {
const baseFields = {
providerThreadId: message.providerThreadId,
@@ -264,15 +266,21 @@ export async function upsertNeutralMessage(
async function fetchGmailMapped(accessToken: string, messageIds: readonly string[]): Promise<MappedEmailMessage[]> {
const mapped: MappedEmailMessage[] = [];
for (const id of messageIds) {
try {
mapped.push(mapGmailMessage(await getGmailMessage(accessToken, id, "full")));
} catch (error) {
if (error instanceof GmailError && error.code === "message_not_found") {
continue; // deleted between listing and fetching — not an error
}
throw error;
}
for (let offset = 0; offset < messageIds.length; offset += GMAIL_MESSAGE_FETCH_CONCURRENCY) {
const batch = messageIds.slice(offset, offset + GMAIL_MESSAGE_FETCH_CONCURRENCY);
const results = await Promise.all(
batch.map(async (id) => {
try {
return mapGmailMessage(await getGmailMessage(accessToken, id, "full"));
} catch (error) {
if (error instanceof GmailError && error.code === "message_not_found") {
return null; // deleted between listing and fetching — not an error
}
throw error;
}
})
);
mapped.push(...results.filter((message): message is MappedEmailMessage => message !== null));
}
return mapped;
}
@@ -281,7 +289,7 @@ async function persistMessages(
client: EmailWorkerClient,
mailbox: EmailMailboxRow,
messages: readonly MappedEmailMessage[],
hook: EmailMessageUpsertedHook | undefined,
hook: EmailMessageUpsertedHook | undefined
): Promise<number> {
let upserted = 0;
for (const message of messages) {
@@ -321,7 +329,7 @@ export async function syncMailboxIncremental(
client: EmailWorkerClient,
tenantId: string,
mailboxId: string,
opts: { now?: Date; hook?: EmailMessageUpsertedHook } = {},
opts: { now?: Date; hook?: EmailMessageUpsertedHook } = {}
): Promise<{ upserted: number; resyncScheduled: boolean }> {
const now = opts.now ?? new Date();
const mailbox = await client.emailIntegrationMailbox.findFirst({
@@ -422,7 +430,7 @@ export async function syncMailboxIncremental(
export async function runEmailInitialSyncForTenant(
client: EmailWorkerClient,
tenantId: string,
opts: { now?: Date; take?: number; initialDays?: number; hook?: EmailMessageUpsertedHook } = {},
opts: { now?: Date; take?: number; initialDays?: number; hook?: EmailMessageUpsertedHook } = {}
): Promise<{ folders: number; fetched: number; completed: number; failed: number }> {
const now = opts.now ?? new Date();
const initialSince = new Date(now.getTime() - (opts.initialDays ?? INITIAL_SYNC_DAYS) * 86_400_000);
@@ -521,7 +529,7 @@ export async function runEmailInitialSyncForTenant(
export async function refreshEmailTokensForTenant(
client: EmailWorkerClient,
tenantId: string,
opts: { now?: Date; windowMs?: number } = {},
opts: { now?: Date; windowMs?: number } = {}
): Promise<{ refreshed: number; failed: number }> {
const now = opts.now ?? new Date();
const horizon = new Date(now.getTime() + (opts.windowMs ?? 10 * 60_000));
@@ -557,7 +565,7 @@ export async function refreshEmailTokensForTenant(
export async function renewEmailWebhooksForTenant(
client: EmailWorkerClient,
tenantId: string,
opts: { now?: Date; thresholdMs?: number; graphWebhookUrl?: string } = {},
opts: { now?: Date; thresholdMs?: number; graphWebhookUrl?: string } = {}
): Promise<{ checked: number; renewed: number; failed: number; skipped: number }> {
const now = opts.now ?? new Date();
const thresholdMs = opts.thresholdMs ?? 24 * 60 * 60_000;
@@ -632,7 +640,7 @@ export async function renewEmailWebhooksForTenant(
export async function drainEmailWebhookEventsForTenant(
client: EmailWorkerClient,
tenantId: string,
opts: { now?: Date; take?: number; hook?: EmailMessageUpsertedHook } = {},
opts: { now?: Date; take?: number; hook?: EmailMessageUpsertedHook } = {}
): Promise<{ claimed: number; completed: number; failed: number }> {
const now = opts.now ?? new Date();
const rows = await client.emailIntegrationWebhookEvent.findMany({
@@ -684,7 +692,7 @@ export async function drainEmailWebhookEventsForTenant(
export async function pollStaleEmailMailboxesForTenant(
client: EmailWorkerClient,
tenantId: string,
opts: { now?: Date; staleMs?: number; take?: number; hook?: EmailMessageUpsertedHook } = {},
opts: { now?: Date; staleMs?: number; take?: number; hook?: EmailMessageUpsertedHook } = {}
): Promise<{ polled: number; failed: number }> {
const now = opts.now ?? new Date();
const staleBefore = new Date(now.getTime() - (opts.staleMs ?? 4 * 60 * 60_000));
@@ -710,11 +718,7 @@ export async function pollStaleEmailMailboxesForTenant(
}
/** 30-day webhook-event retention (failed rows: 90 days for diagnosis). */
export async function cleanupEmailWebhookEventsForTenant(
client: EmailWorkerClient,
tenantId: string,
opts: { now?: Date } = {},
): Promise<{ deleted: number }> {
export async function cleanupEmailWebhookEventsForTenant(client: EmailWorkerClient, tenantId: string, opts: { now?: Date } = {}): Promise<{ deleted: number }> {
const now = opts.now ?? new Date();
const doneBefore = new Date(now.getTime() - WEBHOOK_RETENTION_DAYS * 86_400_000);
const failedBefore = new Date(now.getTime() - WEBHOOK_FAILED_RETENTION_DAYS * 86_400_000);