Update hivegrid-cloud reference to latest main
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e98d38e716
commit
7b21524c86
3781 files changed
+373484
-225607
No files matched your search
+736
@@ -0,0 +1,736 @@
|
||||
// VA-P23 (P23) email-inbox worker tasks. Shared by apps/worker so the background
|
||||
// sync engine runs without importing apps/web modules (mirror of
|
||||
// calendar-worker-tasks.server.ts). Provider I/O goes through the raw clients
|
||||
// (gmailApi/graphMailApi) + the shared neutral mapping — the same mapping the
|
||||
// web adapters use, so connect-time and cron-time sync cannot drift.
|
||||
//
|
||||
// Responsibilities (M09 Block D):
|
||||
// - refreshEmailTokensForTenant — access-token refresh window sweep
|
||||
// - renewEmailWebhooksForTenant — Graph subscription (~3d) AND Gmail watch
|
||||
// (~7d) renewal, keyed by webhookSubscriptionExpiresAt
|
||||
// - drainEmailWebhookEventsForTenant — claim pending pushes → incremental sync
|
||||
// - runEmailInitialSyncForTenant — bounded 90-day backfill, RESUMABLE via the
|
||||
// per-folder cursor progress JSON
|
||||
// - pollStaleEmailMailboxesForTenant — 4h-silence fallback poll (safety net,
|
||||
// push stays the primary channel)
|
||||
// - cleanupEmailWebhookEventsForTenant — 30d retention (failed: 90d)
|
||||
|
||||
import { decryptTenantField, encryptTenantField } from "@hivegrid/db/crypto/tenant-envelope.server";
|
||||
import {
|
||||
GmailError,
|
||||
getMessage as getGmailMessage,
|
||||
listHistory,
|
||||
listMessageIds,
|
||||
refreshGmailAccessToken,
|
||||
resolveGmailOAuthConfig,
|
||||
watchMailbox,
|
||||
} from "./google/gmailApi.js";
|
||||
import {
|
||||
GraphMailError,
|
||||
buildInitialDeltaUrl,
|
||||
createMailSubscription,
|
||||
fetchDeltaPage,
|
||||
refreshMicrosoftAccessToken,
|
||||
renewMailSubscription,
|
||||
resolveMicrosoftMailOAuthConfig,
|
||||
} from "./microsoft/graphMailApi.js";
|
||||
import { mapGmailMessage, mapGraphMessage, type MappedEmailMessage } from "./email-message-mapping.js";
|
||||
|
||||
export const EMAIL_GMAIL_FOLDER = "*";
|
||||
export const EMAIL_GRAPH_FOLDERS = ["inbox", "sentitems"] as const;
|
||||
|
||||
const DEFAULT_TAKE = 25;
|
||||
const INITIAL_SYNC_DAYS = 90;
|
||||
/** Pages per initial-sync run per folder — keeps a run bounded + resumable. */
|
||||
const INITIAL_SYNC_MAX_PAGES_PER_RUN = 5;
|
||||
/** Estimated total for the Klartext percent line when the provider gives one. */
|
||||
const WEBHOOK_RETENTION_DAYS = 30;
|
||||
const WEBHOOK_FAILED_RETENTION_DAYS = 90;
|
||||
|
||||
type EmailProvider = "gmail" | "microsoft";
|
||||
|
||||
type EmailMailboxRow = {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
ownerUserId: string;
|
||||
provider: EmailProvider;
|
||||
emailAddress: string;
|
||||
oauthRefreshTokenCiphertext: string | null;
|
||||
oauthAccessTokenCiphertext: string | null;
|
||||
oauthAccessTokenExpiresAt: Date | null;
|
||||
webhookSubscriptionId: string | null;
|
||||
webhookSubscriptionExpiresAt: Date | null;
|
||||
webhookClientState: string | null;
|
||||
status: "connecting" | "active" | "paused" | "error" | "revoked";
|
||||
lastSyncAt: Date | null;
|
||||
};
|
||||
|
||||
type EmailSyncCursorRow = {
|
||||
id: string;
|
||||
mailboxId: string;
|
||||
externalFolderId: string;
|
||||
cursorToken: string | null;
|
||||
initialSyncStatus: "pending" | "running" | "completed" | "failed";
|
||||
initialSyncProgress: unknown;
|
||||
};
|
||||
|
||||
type EmailWebhookEventRow = {
|
||||
id: string;
|
||||
tenantId: string;
|
||||
mailboxId: string;
|
||||
};
|
||||
|
||||
type EmailMessageIdRow = { id: string; providerMessageId: string };
|
||||
|
||||
export type EmailWorkerClient = {
|
||||
emailIntegrationMailbox: {
|
||||
findMany(args: Record<string, unknown>): Promise<EmailMailboxRow[]>;
|
||||
findFirst(args: Record<string, unknown>): Promise<EmailMailboxRow | null>;
|
||||
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
||||
};
|
||||
emailIntegrationMessage: {
|
||||
findFirst(args: Record<string, unknown>): Promise<EmailMessageIdRow | null>;
|
||||
create(args: Record<string, unknown>): Promise<EmailMessageIdRow>;
|
||||
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
||||
};
|
||||
emailIntegrationSyncCursor: {
|
||||
findMany(args: Record<string, unknown>): Promise<EmailSyncCursorRow[]>;
|
||||
findFirst(args: Record<string, unknown>): Promise<EmailSyncCursorRow | null>;
|
||||
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
||||
upsert(args: Record<string, unknown>): Promise<unknown>;
|
||||
};
|
||||
emailIntegrationWebhookEvent: {
|
||||
findMany(args: Record<string, unknown>): Promise<EmailWebhookEventRow[]>;
|
||||
update(args: Record<string, unknown>): Promise<unknown>;
|
||||
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
||||
deleteMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
||||
};
|
||||
};
|
||||
|
||||
type EmailCreds = {
|
||||
accessToken: string;
|
||||
refreshToken: string;
|
||||
expiresAt: Date | null;
|
||||
};
|
||||
|
||||
const MAILBOX_SELECT = {
|
||||
id: true,
|
||||
tenantId: true,
|
||||
ownerUserId: true,
|
||||
provider: true,
|
||||
emailAddress: true,
|
||||
oauthRefreshTokenCiphertext: true,
|
||||
oauthAccessTokenCiphertext: true,
|
||||
oauthAccessTokenExpiresAt: true,
|
||||
webhookSubscriptionId: true,
|
||||
webhookSubscriptionExpiresAt: true,
|
||||
webhookClientState: true,
|
||||
status: true,
|
||||
lastSyncAt: true,
|
||||
} as const;
|
||||
|
||||
async function loadCreds(client: EmailWorkerClient, tenantId: string, mailboxId: string): Promise<EmailCreds | null> {
|
||||
const row = await client.emailIntegrationMailbox.findFirst({
|
||||
where: { id: mailboxId, tenantId, archivedAt: null },
|
||||
select: MAILBOX_SELECT,
|
||||
});
|
||||
if (!row?.oauthRefreshTokenCiphertext || !row.oauthAccessTokenCiphertext) {
|
||||
return null;
|
||||
}
|
||||
const [refreshToken, accessToken] = await Promise.all([
|
||||
decryptTenantField(tenantId, row.oauthRefreshTokenCiphertext, { client: client as never }),
|
||||
decryptTenantField(tenantId, row.oauthAccessTokenCiphertext, { client: client as never }),
|
||||
]);
|
||||
return { refreshToken, accessToken, expiresAt: row.oauthAccessTokenExpiresAt };
|
||||
}
|
||||
|
||||
async function refreshProviderTokens(provider: EmailProvider, refreshToken: string) {
|
||||
if (provider === "gmail") {
|
||||
const config = resolveGmailOAuthConfig();
|
||||
if (!config) throw new GmailError("not_configured");
|
||||
return refreshGmailAccessToken(config, refreshToken);
|
||||
}
|
||||
const config = resolveMicrosoftMailOAuthConfig();
|
||||
if (!config) throw new GraphMailError("not_configured");
|
||||
return refreshMicrosoftAccessToken(config, refreshToken);
|
||||
}
|
||||
|
||||
async function ensureFreshCreds(
|
||||
client: EmailWorkerClient,
|
||||
mailbox: { id: string; tenantId: string; provider: EmailProvider },
|
||||
creds: EmailCreds,
|
||||
now = new Date(),
|
||||
): Promise<EmailCreds> {
|
||||
if (creds.expiresAt && creds.expiresAt.getTime() > now.getTime() + 60_000) {
|
||||
return creds;
|
||||
}
|
||||
const refreshed = await refreshProviderTokens(mailbox.provider, creds.refreshToken);
|
||||
const nextRefreshToken = refreshed.refreshToken ?? creds.refreshToken;
|
||||
const [accessCiphertext, refreshCiphertext] = await Promise.all([
|
||||
encryptTenantField(mailbox.tenantId, refreshed.accessToken, { client: client as never }),
|
||||
encryptTenantField(mailbox.tenantId, nextRefreshToken, { client: client as never }),
|
||||
]);
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: mailbox.id, tenantId: mailbox.tenantId },
|
||||
data: {
|
||||
oauthAccessTokenCiphertext: accessCiphertext,
|
||||
oauthRefreshTokenCiphertext: refreshCiphertext,
|
||||
oauthAccessTokenExpiresAt: refreshed.expiresAt,
|
||||
status: "active",
|
||||
lastErrorText: null,
|
||||
},
|
||||
});
|
||||
return { accessToken: refreshed.accessToken, refreshToken: nextRefreshToken, expiresAt: refreshed.expiresAt };
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook invoked after every message upsert — S6 wires the auto-match + activity
|
||||
* logging through this seam so the sync engine stays match-agnostic.
|
||||
*/
|
||||
export type EmailMessageUpsertedHook = (input: {
|
||||
client: EmailWorkerClient;
|
||||
tenantId: string;
|
||||
mailboxId: string;
|
||||
ownerUserId: string;
|
||||
ownerEmailAddress: string;
|
||||
messageId: string;
|
||||
created: boolean;
|
||||
message: MappedEmailMessage;
|
||||
}) => Promise<void>;
|
||||
|
||||
/**
|
||||
* Upsert one neutral message. Dedupe order (M09):
|
||||
* 1. internetMessageId within the mailbox (collapses the Graph sent-items
|
||||
* re-ingest onto the local outbound mirror; providerMessageId is adopted)
|
||||
* 2. the [tenantId, mailboxId, providerMessageId] unique key
|
||||
* Returns the row id + whether a new row was created.
|
||||
*/
|
||||
export async function upsertNeutralMessage(
|
||||
client: EmailWorkerClient,
|
||||
ctx: { tenantId: string; mailboxId: string },
|
||||
message: MappedEmailMessage,
|
||||
): Promise<{ id: string; created: boolean }> {
|
||||
const baseFields = {
|
||||
providerThreadId: message.providerThreadId,
|
||||
internetMessageId: message.internetMessageId,
|
||||
direction: message.direction,
|
||||
fromAddress: message.from.address,
|
||||
fromName: message.from.name ?? null,
|
||||
toAddresses: message.to as never,
|
||||
ccAddresses: message.cc as never,
|
||||
bccAddresses: message.bcc as never,
|
||||
subject: message.subject,
|
||||
snippet: message.snippet,
|
||||
bodyHtml: message.bodyHtml,
|
||||
bodyText: message.bodyText,
|
||||
sentAt: message.sentAt,
|
||||
receivedAt: message.receivedAt,
|
||||
isRead: message.isRead,
|
||||
isNewsletter: message.isNewsletter,
|
||||
hasAttachments: message.hasAttachments,
|
||||
providerLabels: message.providerLabels as never,
|
||||
};
|
||||
|
||||
if (message.internetMessageId) {
|
||||
const existingByRfcId = await client.emailIntegrationMessage.findFirst({
|
||||
where: { tenantId: ctx.tenantId, mailboxId: ctx.mailboxId, internetMessageId: message.internetMessageId },
|
||||
select: { id: true, providerMessageId: true },
|
||||
});
|
||||
if (existingByRfcId) {
|
||||
await client.emailIntegrationMessage.updateMany({
|
||||
where: { id: existingByRfcId.id, tenantId: ctx.tenantId },
|
||||
data: { ...baseFields, providerMessageId: message.providerMessageId },
|
||||
});
|
||||
return { id: existingByRfcId.id, created: false };
|
||||
}
|
||||
}
|
||||
|
||||
const existing = await client.emailIntegrationMessage.findFirst({
|
||||
where: { tenantId: ctx.tenantId, mailboxId: ctx.mailboxId, providerMessageId: message.providerMessageId },
|
||||
select: { id: true, providerMessageId: true },
|
||||
});
|
||||
if (existing) {
|
||||
await client.emailIntegrationMessage.updateMany({
|
||||
where: { id: existing.id, tenantId: ctx.tenantId },
|
||||
data: baseFields,
|
||||
});
|
||||
return { id: existing.id, created: false };
|
||||
}
|
||||
const created = await client.emailIntegrationMessage.create({
|
||||
data: { tenantId: ctx.tenantId, mailboxId: ctx.mailboxId, providerMessageId: message.providerMessageId, ...baseFields },
|
||||
});
|
||||
return { id: created.id, created: true };
|
||||
}
|
||||
|
||||
async function fetchGmailMapped(accessToken: string, messageIds: readonly string[]): Promise<MappedEmailMessage[]> {
|
||||
const mapped: MappedEmailMessage[] = [];
|
||||
for (const id of messageIds) {
|
||||
try {
|
||||
mapped.push(mapGmailMessage(await getGmailMessage(accessToken, id, "full")));
|
||||
} catch (error) {
|
||||
if (error instanceof GmailError && error.code === "message_not_found") {
|
||||
continue; // deleted between listing and fetching — not an error
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return mapped;
|
||||
}
|
||||
|
||||
async function persistMessages(
|
||||
client: EmailWorkerClient,
|
||||
mailbox: EmailMailboxRow,
|
||||
messages: readonly MappedEmailMessage[],
|
||||
hook: EmailMessageUpsertedHook | undefined,
|
||||
): Promise<number> {
|
||||
let upserted = 0;
|
||||
for (const message of messages) {
|
||||
const result = await upsertNeutralMessage(client, { tenantId: mailbox.tenantId, mailboxId: mailbox.id }, message);
|
||||
upserted += 1;
|
||||
if (hook) {
|
||||
await hook({
|
||||
client,
|
||||
tenantId: mailbox.tenantId,
|
||||
mailboxId: mailbox.id,
|
||||
ownerUserId: mailbox.ownerUserId,
|
||||
ownerEmailAddress: mailbox.emailAddress,
|
||||
messageId: result.id,
|
||||
created: result.created,
|
||||
message,
|
||||
});
|
||||
}
|
||||
}
|
||||
return upserted;
|
||||
}
|
||||
|
||||
/** Reset a folder cursor for a bounded re-backfill after a provider 404/410. */
|
||||
async function resetCursorForResync(client: EmailWorkerClient, tenantId: string, mailboxId: string, externalFolderId: string): Promise<void> {
|
||||
await client.emailIntegrationSyncCursor.updateMany({
|
||||
where: { tenantId, mailboxId, externalFolderId },
|
||||
data: { cursorToken: null, initialSyncStatus: "pending", initialSyncProgress: null, errorText: null },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Incremental sync of ONE mailbox (all its folders) from the stored cursors.
|
||||
* Gmail: history.list from the historyId anchor; Graph: delta per folder. An
|
||||
* expired cursor resets the folder to a bounded initial re-sync (next initial
|
||||
* run picks it up) instead of failing the drain.
|
||||
*/
|
||||
export async function syncMailboxIncremental(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
mailboxId: string,
|
||||
opts: { now?: Date; hook?: EmailMessageUpsertedHook } = {},
|
||||
): Promise<{ upserted: number; resyncScheduled: boolean }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const mailbox = await client.emailIntegrationMailbox.findFirst({
|
||||
where: { id: mailboxId, tenantId, archivedAt: null },
|
||||
select: MAILBOX_SELECT,
|
||||
});
|
||||
if (!mailbox || mailbox.status === "revoked" || mailbox.status === "paused") {
|
||||
return { upserted: 0, resyncScheduled: false };
|
||||
}
|
||||
const baseCreds = await loadCreds(client, tenantId, mailboxId);
|
||||
if (!baseCreds) {
|
||||
return { upserted: 0, resyncScheduled: false };
|
||||
}
|
||||
const creds = await ensureFreshCreds(client, mailbox, baseCreds, now);
|
||||
|
||||
let upserted = 0;
|
||||
let resyncScheduled = false;
|
||||
|
||||
if (mailbox.provider === "gmail") {
|
||||
const cursor = await client.emailIntegrationSyncCursor.findFirst({
|
||||
where: { tenantId, mailboxId, externalFolderId: EMAIL_GMAIL_FOLDER },
|
||||
select: { id: true, mailboxId: true, externalFolderId: true, cursorToken: true, initialSyncStatus: true, initialSyncProgress: true },
|
||||
});
|
||||
if (cursor?.cursorToken) {
|
||||
let pageToken: string | undefined;
|
||||
let latestHistoryId: string | undefined;
|
||||
try {
|
||||
do {
|
||||
const page = await listHistory(creds.accessToken, cursor.cursorToken, pageToken);
|
||||
upserted += await persistMessages(client, mailbox, await fetchGmailMapped(creds.accessToken, page.addedMessageIds), opts.hook);
|
||||
pageToken = page.nextPageToken;
|
||||
latestHistoryId = page.historyId ?? latestHistoryId;
|
||||
} while (pageToken);
|
||||
if (latestHistoryId) {
|
||||
await client.emailIntegrationSyncCursor.updateMany({
|
||||
where: { tenantId, mailboxId, externalFolderId: EMAIL_GMAIL_FOLDER },
|
||||
data: { cursorToken: latestHistoryId, lastCursorAdvanceAt: now },
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof GmailError && error.fullResyncRequired) {
|
||||
await resetCursorForResync(client, tenantId, mailboxId, EMAIL_GMAIL_FOLDER);
|
||||
resyncScheduled = true;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const folderId of EMAIL_GRAPH_FOLDERS) {
|
||||
const cursor = await client.emailIntegrationSyncCursor.findFirst({
|
||||
where: { tenantId, mailboxId, externalFolderId: folderId },
|
||||
select: { id: true, mailboxId: true, externalFolderId: true, cursorToken: true, initialSyncStatus: true, initialSyncProgress: true },
|
||||
});
|
||||
if (!cursor?.cursorToken) {
|
||||
continue; // initial delta not completed yet — the initial-sync run owns it
|
||||
}
|
||||
let url: string | undefined = cursor.cursorToken;
|
||||
let newDeltaLink: string | undefined;
|
||||
try {
|
||||
while (url) {
|
||||
const page = await fetchDeltaPage(creds.accessToken, url);
|
||||
const mapped = page.messages.filter((m) => !m["@removed"]).map((m) => mapGraphMessage(m, folderId));
|
||||
upserted += await persistMessages(client, mailbox, mapped, opts.hook);
|
||||
url = page.nextLink;
|
||||
newDeltaLink = page.deltaLink ?? newDeltaLink;
|
||||
}
|
||||
if (newDeltaLink) {
|
||||
await client.emailIntegrationSyncCursor.updateMany({
|
||||
where: { tenantId, mailboxId, externalFolderId: folderId },
|
||||
data: { cursorToken: newDeltaLink, lastCursorAdvanceAt: now },
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof GraphMailError && error.fullResyncRequired) {
|
||||
await resetCursorForResync(client, tenantId, mailboxId, folderId);
|
||||
resyncScheduled = true;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: mailboxId, tenantId },
|
||||
data: { lastSyncAt: now, status: "active", lastErrorText: null },
|
||||
});
|
||||
return { upserted, resyncScheduled };
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded, RESUMABLE initial sync (M09 Block D): processes at most
|
||||
* `INITIAL_SYNC_MAX_PAGES_PER_RUN` pages per pending/running folder cursor per
|
||||
* run and persists `{pageToken|nextLink, fetchedCount}` in the progress JSON so
|
||||
* an aborted run continues exactly where it stopped.
|
||||
*/
|
||||
export async function runEmailInitialSyncForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; take?: number; initialDays?: number; hook?: EmailMessageUpsertedHook } = {},
|
||||
): Promise<{ folders: number; fetched: number; completed: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const initialSince = new Date(now.getTime() - (opts.initialDays ?? INITIAL_SYNC_DAYS) * 86_400_000);
|
||||
const cursors = await client.emailIntegrationSyncCursor.findMany({
|
||||
where: { tenantId, initialSyncStatus: { in: ["pending", "running"] } },
|
||||
select: { id: true, mailboxId: true, externalFolderId: true, cursorToken: true, initialSyncStatus: true, initialSyncProgress: true },
|
||||
take: opts.take ?? DEFAULT_TAKE,
|
||||
});
|
||||
|
||||
let fetched = 0;
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const cursor of cursors) {
|
||||
const mailbox = await client.emailIntegrationMailbox.findFirst({
|
||||
where: { id: cursor.mailboxId, tenantId, archivedAt: null },
|
||||
select: MAILBOX_SELECT,
|
||||
});
|
||||
if (!mailbox || mailbox.status === "revoked" || mailbox.status === "paused") {
|
||||
continue;
|
||||
}
|
||||
const progress = (cursor.initialSyncProgress ?? {}) as { pageToken?: string; fetchedCount?: number };
|
||||
let pageToken: string | undefined = progress.pageToken;
|
||||
let fetchedCount = progress.fetchedCount ?? 0;
|
||||
|
||||
try {
|
||||
const baseCreds = await loadCreds(client, tenantId, mailbox.id);
|
||||
if (!baseCreds) {
|
||||
continue;
|
||||
}
|
||||
const creds = await ensureFreshCreds(client, mailbox, baseCreds, now);
|
||||
let done = false;
|
||||
|
||||
for (let page = 0; page < INITIAL_SYNC_MAX_PAGES_PER_RUN; page += 1) {
|
||||
if (mailbox.provider === "gmail") {
|
||||
const days = Math.max(1, Math.ceil((now.getTime() - initialSince.getTime()) / 86_400_000));
|
||||
const idPage = await listMessageIds(creds.accessToken, {
|
||||
q: `newer_than:${days}d -in:spam -in:trash`,
|
||||
pageToken,
|
||||
maxResults: 50,
|
||||
});
|
||||
const mapped = await fetchGmailMapped(creds.accessToken, idPage.messageIds);
|
||||
fetched += await persistMessages(client, mailbox, mapped, opts.hook);
|
||||
fetchedCount += mapped.length;
|
||||
pageToken = idPage.nextPageToken;
|
||||
} else {
|
||||
const url = pageToken ?? buildInitialDeltaUrl(cursor.externalFolderId, initialSince.toISOString());
|
||||
const deltaPage = await fetchDeltaPage(creds.accessToken, url);
|
||||
const mapped = deltaPage.messages.filter((m) => !m["@removed"]).map((m) => mapGraphMessage(m, cursor.externalFolderId));
|
||||
fetched += await persistMessages(client, mailbox, mapped, opts.hook);
|
||||
fetchedCount += mapped.length;
|
||||
pageToken = deltaPage.nextLink;
|
||||
if (!pageToken && deltaPage.deltaLink) {
|
||||
// Initial delta finished — the deltaLink becomes the incremental cursor.
|
||||
await client.emailIntegrationSyncCursor.updateMany({
|
||||
where: { id: cursor.id, tenantId },
|
||||
data: { cursorToken: deltaPage.deltaLink, lastCursorAdvanceAt: now },
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!pageToken) {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await client.emailIntegrationSyncCursor.updateMany({
|
||||
where: { id: cursor.id, tenantId },
|
||||
data: done
|
||||
? { initialSyncStatus: "completed", initialSyncProgress: { fetchedCount, percent: 100 }, lastFullSyncAt: now, errorText: null }
|
||||
: { initialSyncStatus: "running", initialSyncProgress: { pageToken, fetchedCount }, errorText: null },
|
||||
});
|
||||
if (done) {
|
||||
completed += 1;
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: mailbox.id, tenantId },
|
||||
data: { lastSyncAt: now, status: "active", lastErrorText: null },
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
await client.emailIntegrationSyncCursor.updateMany({
|
||||
where: { id: cursor.id, tenantId },
|
||||
data: { initialSyncStatus: "failed", errorText: redactedError(error) },
|
||||
});
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: cursor.mailboxId, tenantId },
|
||||
data: { status: "error", lastErrorText: "emailInbox.health.unknown" },
|
||||
});
|
||||
}
|
||||
}
|
||||
return { folders: cursors.length, fetched, completed, failed };
|
||||
}
|
||||
|
||||
/** Access-token refresh sweep (both providers; Graph rotates refresh tokens). */
|
||||
export async function refreshEmailTokensForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; windowMs?: number } = {},
|
||||
): Promise<{ refreshed: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const horizon = new Date(now.getTime() + (opts.windowMs ?? 10 * 60_000));
|
||||
const rows = await client.emailIntegrationMailbox.findMany({
|
||||
where: { tenantId, archivedAt: null, status: { in: ["active", "error", "connecting"] }, oauthAccessTokenExpiresAt: { not: null, lte: horizon } },
|
||||
select: MAILBOX_SELECT,
|
||||
});
|
||||
let refreshed = 0;
|
||||
let failed = 0;
|
||||
for (const row of rows) {
|
||||
const creds = await loadCreds(client, tenantId, row.id);
|
||||
if (!creds) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await ensureFreshCreds(client, row, creds, now);
|
||||
refreshed += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: row.id, tenantId },
|
||||
data: { status: "error", lastErrorText: "emailInbox.health.tokenExpired" },
|
||||
});
|
||||
}
|
||||
}
|
||||
return { refreshed, failed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Push-subscription renewal for BOTH providers (Graph ~3 days, Gmail watch ~7
|
||||
* days), keyed by webhookSubscriptionExpiresAt within `thresholdMs`.
|
||||
*/
|
||||
export async function renewEmailWebhooksForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; thresholdMs?: number; graphWebhookUrl?: string } = {},
|
||||
): Promise<{ checked: number; renewed: number; failed: number; skipped: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const thresholdMs = opts.thresholdMs ?? 24 * 60 * 60_000;
|
||||
const rows = await client.emailIntegrationMailbox.findMany({
|
||||
where: { tenantId, archivedAt: null, status: "active" },
|
||||
select: MAILBOX_SELECT,
|
||||
});
|
||||
|
||||
let renewed = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
for (const row of rows) {
|
||||
const needsRenewal =
|
||||
row.webhookSubscriptionId !== null &&
|
||||
(row.webhookSubscriptionExpiresAt === null || row.webhookSubscriptionExpiresAt.getTime() - now.getTime() <= thresholdMs);
|
||||
if (!needsRenewal) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
const baseCreds = await loadCreds(client, tenantId, row.id);
|
||||
if (!baseCreds) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const creds = await ensureFreshCreds(client, row, baseCreds, now);
|
||||
if (row.provider === "gmail") {
|
||||
const config = resolveGmailOAuthConfig();
|
||||
if (!config) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
const watch = await watchMailbox(creds.accessToken, config.pubsubTopic);
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: row.id, tenantId },
|
||||
data: { webhookSubscriptionExpiresAt: watch.expiresAt, lastErrorText: null },
|
||||
});
|
||||
} else {
|
||||
if (!opts.graphWebhookUrl || !row.webhookClientState) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
let subscription;
|
||||
try {
|
||||
subscription = await renewMailSubscription(creds.accessToken, row.webhookSubscriptionId ?? "");
|
||||
} catch (error) {
|
||||
// Expired at Microsoft → re-create with the stored clientState.
|
||||
if (error instanceof GraphMailError && error.code === "subscription_failed") {
|
||||
subscription = await createMailSubscription(creds.accessToken, opts.graphWebhookUrl, row.webhookClientState);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: row.id, tenantId },
|
||||
data: { webhookSubscriptionId: subscription.id, webhookSubscriptionExpiresAt: subscription.expiresAt, lastErrorText: null },
|
||||
});
|
||||
}
|
||||
renewed += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: row.id, tenantId },
|
||||
data: { lastErrorText: "emailInbox.health.webhookSubscriptionFailed" },
|
||||
});
|
||||
}
|
||||
}
|
||||
return { checked: rows.length, renewed, failed, skipped };
|
||||
}
|
||||
|
||||
/** Claim + process pending provider pushes (fast-ack rows from the webhook routes). */
|
||||
export async function drainEmailWebhookEventsForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; take?: number; hook?: EmailMessageUpsertedHook } = {},
|
||||
): Promise<{ claimed: number; completed: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const rows = await client.emailIntegrationWebhookEvent.findMany({
|
||||
where: { tenantId, processingStatus: "pending" },
|
||||
select: { id: true, tenantId: true, mailboxId: true },
|
||||
orderBy: { receivedAt: "asc" },
|
||||
take: opts.take ?? DEFAULT_TAKE,
|
||||
});
|
||||
let claimed = 0;
|
||||
let completed = 0;
|
||||
let failed = 0;
|
||||
// Multiple pushes for the same mailbox collapse into one incremental sync.
|
||||
const syncedMailboxes = new Set<string>();
|
||||
for (const row of rows) {
|
||||
const claim = await client.emailIntegrationWebhookEvent.updateMany({
|
||||
where: { id: row.id, tenantId, processingStatus: "pending" },
|
||||
data: { processingStatus: "processing" },
|
||||
});
|
||||
if (claim.count !== 1) {
|
||||
continue;
|
||||
}
|
||||
claimed += 1;
|
||||
try {
|
||||
if (!syncedMailboxes.has(row.mailboxId)) {
|
||||
await syncMailboxIncremental(client, tenantId, row.mailboxId, { now, hook: opts.hook });
|
||||
syncedMailboxes.add(row.mailboxId);
|
||||
}
|
||||
await client.emailIntegrationWebhookEvent.update({
|
||||
where: { id: row.id },
|
||||
data: { processingStatus: "completed", processedAt: now, processingErrorText: null },
|
||||
});
|
||||
completed += 1;
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
await client.emailIntegrationWebhookEvent.update({
|
||||
where: { id: row.id },
|
||||
data: { processingStatus: "failed", processedAt: now, processingErrorText: redactedError(error) },
|
||||
});
|
||||
}
|
||||
}
|
||||
return { claimed, completed, failed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback poll (M09 Block D): mailboxes that saw no successful sync for
|
||||
* `staleMs` (default 4h) get one incremental pull — the safety net when a push
|
||||
* channel silently dies. Push remains the primary mechanism.
|
||||
*/
|
||||
export async function pollStaleEmailMailboxesForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date; staleMs?: number; take?: number; hook?: EmailMessageUpsertedHook } = {},
|
||||
): Promise<{ polled: number; failed: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const staleBefore = new Date(now.getTime() - (opts.staleMs ?? 4 * 60 * 60_000));
|
||||
const rows = await client.emailIntegrationMailbox.findMany({
|
||||
where: { tenantId, archivedAt: null, status: "active", lastSyncAt: { lte: staleBefore } },
|
||||
select: MAILBOX_SELECT,
|
||||
});
|
||||
let polled = 0;
|
||||
let failed = 0;
|
||||
for (const row of rows.slice(0, opts.take ?? DEFAULT_TAKE)) {
|
||||
try {
|
||||
await syncMailboxIncremental(client, tenantId, row.id, { now, hook: opts.hook });
|
||||
polled += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
await client.emailIntegrationMailbox.updateMany({
|
||||
where: { id: row.id, tenantId },
|
||||
data: { status: "error", lastErrorText: "emailInbox.health.unknown" },
|
||||
});
|
||||
}
|
||||
}
|
||||
return { polled, failed };
|
||||
}
|
||||
|
||||
/** 30-day webhook-event retention (failed rows: 90 days for diagnosis). */
|
||||
export async function cleanupEmailWebhookEventsForTenant(
|
||||
client: EmailWorkerClient,
|
||||
tenantId: string,
|
||||
opts: { now?: Date } = {},
|
||||
): Promise<{ deleted: number }> {
|
||||
const now = opts.now ?? new Date();
|
||||
const doneBefore = new Date(now.getTime() - WEBHOOK_RETENTION_DAYS * 86_400_000);
|
||||
const failedBefore = new Date(now.getTime() - WEBHOOK_FAILED_RETENTION_DAYS * 86_400_000);
|
||||
const done = await client.emailIntegrationWebhookEvent.deleteMany({
|
||||
where: { tenantId, processingStatus: { in: ["completed", "skipped"] }, receivedAt: { lte: doneBefore } },
|
||||
});
|
||||
const failedRows = await client.emailIntegrationWebhookEvent.deleteMany({
|
||||
where: { tenantId, processingStatus: "failed", receivedAt: { lte: failedBefore } },
|
||||
});
|
||||
return { deleted: done.count + failedRows.count };
|
||||
}
|
||||
|
||||
function redactedError(error: unknown): string {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message
|
||||
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
||||
.replace(/(token|secret|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
|
||||
.slice(0, 500);
|
||||
}
|
||||
Reference in new issue
Block a user