Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
+27
-23
@@ -43,6 +43,8 @@ const DEFAULT_TAKE = 25;
|
||||
const INITIAL_SYNC_DAYS = 90;
|
||||
/** Pages per initial-sync run per folder — keeps a run bounded + resumable. */
|
||||
const INITIAL_SYNC_MAX_PAGES_PER_RUN = 5;
|
||||
/** Keep Gmail latency bounded without creating a provider rate-limit burst. */
|
||||
const GMAIL_MESSAGE_FETCH_CONCURRENCY = 5;
|
||||
/** Estimated total for the Klartext percent line when the provider gives one. */
|
||||
const WEBHOOK_RETENTION_DAYS = 30;
|
||||
const WEBHOOK_FAILED_RETENTION_DAYS = 90;
|
||||
@@ -159,7 +161,7 @@ async function ensureFreshCreds(
|
||||
client: EmailWorkerClient,
|
||||
mailbox: { id: string; tenantId: string; provider: EmailProvider },
|
||||
creds: EmailCreds,
|
||||
now = new Date(),
|
||||
now = new Date()
|
||||
): Promise<EmailCreds> {
|
||||
if (creds.expiresAt && creds.expiresAt.getTime() > now.getTime() + 60_000) {
|
||||
return creds;
|
||||
@@ -208,7 +210,7 @@ export type EmailMessageUpsertedHook = (input: {
|
||||
export async function upsertNeutralMessage(
|
||||
client: EmailWorkerClient,
|
||||
ctx: { tenantId: string; mailboxId: string },
|
||||
message: MappedEmailMessage,
|
||||
message: MappedEmailMessage
|
||||
): Promise<{ id: string; created: boolean }> {
|
||||
const baseFields = {
|
||||
providerThreadId: message.providerThreadId,
|
||||
@@ -264,15 +266,21 @@ export async function upsertNeutralMessage(
|
||||
|
||||
async function fetchGmailMapped(accessToken: string, messageIds: readonly string[]): Promise<MappedEmailMessage[]> {
|
||||
const mapped: MappedEmailMessage[] = [];
|
||||
for (const id of messageIds) {
|
||||
try {
|
||||
mapped.push(mapGmailMessage(await getGmailMessage(accessToken, id, "full")));
|
||||
} catch (error) {
|
||||
if (error instanceof GmailError && error.code === "message_not_found") {
|
||||
continue; // deleted between listing and fetching — not an error
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
for (let offset = 0; offset < messageIds.length; offset += GMAIL_MESSAGE_FETCH_CONCURRENCY) {
|
||||
const batch = messageIds.slice(offset, offset + GMAIL_MESSAGE_FETCH_CONCURRENCY);
|
||||
const results = await Promise.all(
|
||||
batch.map(async (id) => {
|
||||
try {
|
||||
return mapGmailMessage(await getGmailMessage(accessToken, id, "full"));
|
||||
} catch (error) {
|
||||
if (error instanceof GmailError && error.code === "message_not_found") {
|
||||
return null; // deleted between listing and fetching — not an error
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
})
|
||||
);
|
||||
mapped.push(...results.filter((message): message is MappedEmailMessage => message !== null));
|
||||
}
|
||||
return mapped;
|
||||
}
|
||||
@@ -281,7 +289,7 @@ async function persistMessages(
|
||||
client: EmailWorkerClient,
|
||||
mailbox: EmailMailboxRow,
|
||||
messages: readonly MappedEmailMessage[],
|
||||
hook: EmailMessageUpsertedHook | undefined,
|
||||
hook: EmailMessageUpsertedHook | undefined
|
||||
): Promise<number> {
|
||||
let upserted = 0;
|
||||
for (const message of messages) {
|
||||
@@ -321,7 +329,7 @@ export async function syncMailboxIncremental(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
mailboxId: string,
|
||||
opts: { now?: Date; hook?: EmailMessageUpsertedHook } = {},
|
||||
opts: { now?: Date; hook?: EmailMessageUpsertedHook } = {}
|
||||
): Promise<{ upserted: number; resyncScheduled: boolean }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const mailbox = await client.emailIntegrationMailbox.findFirst({
|
||||
@@ -422,7 +430,7 @@ export async function syncMailboxIncremental(
|
||||
export async function runEmailInitialSyncForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; take?: number; initialDays?: number; hook?: EmailMessageUpsertedHook } = {},
|
||||
opts: { now?: Date; take?: number; initialDays?: number; hook?: EmailMessageUpsertedHook } = {}
|
||||
): Promise<{ folders: number; fetched: number; completed: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const initialSince = new Date(now.getTime() - (opts.initialDays ?? INITIAL_SYNC_DAYS) * 86_400_000);
|
||||
@@ -521,7 +529,7 @@ export async function runEmailInitialSyncForTenant(
|
||||
export async function refreshEmailTokensForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; windowMs?: number } = {},
|
||||
opts: { now?: Date; windowMs?: number } = {}
|
||||
): Promise<{ refreshed: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const horizon = new Date(now.getTime() + (opts.windowMs ?? 10 * 60_000));
|
||||
@@ -557,7 +565,7 @@ export async function refreshEmailTokensForTenant(
|
||||
export async function renewEmailWebhooksForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; thresholdMs?: number; graphWebhookUrl?: string } = {},
|
||||
opts: { now?: Date; thresholdMs?: number; graphWebhookUrl?: string } = {}
|
||||
): Promise<{ checked: number; renewed: number; failed: number; skipped: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const thresholdMs = opts.thresholdMs ?? 24 * 60 * 60_000;
|
||||
@@ -632,7 +640,7 @@ export async function renewEmailWebhooksForTenant(
|
||||
export async function drainEmailWebhookEventsForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; take?: number; hook?: EmailMessageUpsertedHook } = {},
|
||||
opts: { now?: Date; take?: number; hook?: EmailMessageUpsertedHook } = {}
|
||||
): Promise<{ claimed: number; completed: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const rows = await client.emailIntegrationWebhookEvent.findMany({
|
||||
@@ -684,7 +692,7 @@ export async function drainEmailWebhookEventsForTenant(
|
||||
export async function pollStaleEmailMailboxesForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; staleMs?: number; take?: number; hook?: EmailMessageUpsertedHook } = {},
|
||||
opts: { now?: Date; staleMs?: number; take?: number; hook?: EmailMessageUpsertedHook } = {}
|
||||
): Promise<{ polled: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const staleBefore = new Date(now.getTime() - (opts.staleMs ?? 4 * 60 * 60_000));
|
||||
@@ -710,11 +718,7 @@ export async function pollStaleEmailMailboxesForTenant(
|
||||
}
|
||||
|
||||
/** 30-day webhook-event retention (failed rows: 90 days for diagnosis). */
|
||||
export async function cleanupEmailWebhookEventsForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date } = {},
|
||||
): Promise<{ deleted: number }> {
|
||||
export async function cleanupEmailWebhookEventsForTenant(client: EmailWorkerClient, tenantId: string, opts: { now?: Date } = {}): Promise<{ deleted: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const doneBefore = new Date(now.getTime() - WEBHOOK_RETENTION_DAYS * 86_400_000);
|
||||
const failedBefore = new Date(now.getTime() - WEBHOOK_FAILED_RETENTION_DAYS * 86_400_000);
|
||||
|
||||
Reference in new issue
Block a user