737 lines
28 KiB
TypeScript
737 lines
28 KiB
TypeScript
// VA-P23 (P23) email-inbox worker tasks. Shared by apps/worker so the background
|
|
// sync engine runs without importing apps/web modules (mirror of
|
|
// calendar-worker-tasks.server.ts). Provider I/O goes through the raw clients
|
|
// (gmailApi/graphMailApi) + the shared neutral mapping — the same mapping the
|
|
// web adapters use, so connect-time and cron-time sync cannot drift.
|
|
//
|
|
// Responsibilities (M09 Block D):
|
|
// - refreshEmailTokensForTenant — access-token refresh window sweep
|
|
// - renewEmailWebhooksForTenant — Graph subscription (~3d) AND Gmail watch
|
|
// (~7d) renewal, keyed by webhookSubscriptionExpiresAt
|
|
// - drainEmailWebhookEventsForTenant — claim pending pushes → incremental sync
|
|
// - runEmailInitialSyncForTenant — bounded 90-day backfill, RESUMABLE via the
|
|
// per-folder cursor progress JSON
|
|
// - pollStaleEmailMailboxesForTenant — 4h-silence fallback poll (safety net,
|
|
// push stays the primary channel)
|
|
// - cleanupEmailWebhookEventsForTenant — 30d retention (failed: 90d)
|
|
|
|
import { decryptTenantField, encryptTenantField } from "@hivegrid/db/crypto/tenant-envelope.server";
|
|
import {
|
|
GmailError,
|
|
getMessage as getGmailMessage,
|
|
listHistory,
|
|
listMessageIds,
|
|
refreshGmailAccessToken,
|
|
resolveGmailOAuthConfig,
|
|
watchMailbox,
|
|
} from "./google/gmailApi.js";
|
|
import {
|
|
GraphMailError,
|
|
buildInitialDeltaUrl,
|
|
createMailSubscription,
|
|
fetchDeltaPage,
|
|
refreshMicrosoftAccessToken,
|
|
renewMailSubscription,
|
|
resolveMicrosoftMailOAuthConfig,
|
|
} from "./microsoft/graphMailApi.js";
|
|
import { mapGmailMessage, mapGraphMessage, type MappedEmailMessage } from "./email-message-mapping.js";
|
|
|
|
export const EMAIL_GMAIL_FOLDER = "*";
|
|
export const EMAIL_GRAPH_FOLDERS = ["inbox", "sentitems"] as const;
|
|
|
|
const DEFAULT_TAKE = 25;
|
|
const INITIAL_SYNC_DAYS = 90;
|
|
/** Pages per initial-sync run per folder — keeps a run bounded + resumable. */
|
|
const INITIAL_SYNC_MAX_PAGES_PER_RUN = 5;
|
|
/** Estimated total for the Klartext percent line when the provider gives one. */
|
|
const WEBHOOK_RETENTION_DAYS = 30;
|
|
const WEBHOOK_FAILED_RETENTION_DAYS = 90;
|
|
|
|
type EmailProvider = "gmail" | "microsoft";
|
|
|
|
type EmailMailboxRow = {
|
|
id: string;
|
|
tenantId: string;
|
|
ownerUserId: string;
|
|
provider: EmailProvider;
|
|
emailAddress: string;
|
|
oauthRefreshTokenCiphertext: string | null;
|
|
oauthAccessTokenCiphertext: string | null;
|
|
oauthAccessTokenExpiresAt: Date | null;
|
|
webhookSubscriptionId: string | null;
|
|
webhookSubscriptionExpiresAt: Date | null;
|
|
webhookClientState: string | null;
|
|
status: "connecting" | "active" | "paused" | "error" | "revoked";
|
|
lastSyncAt: Date | null;
|
|
};
|
|
|
|
type EmailSyncCursorRow = {
|
|
id: string;
|
|
mailboxId: string;
|
|
externalFolderId: string;
|
|
cursorToken: string | null;
|
|
initialSyncStatus: "pending" | "running" | "completed" | "failed";
|
|
initialSyncProgress: unknown;
|
|
};
|
|
|
|
type EmailWebhookEventRow = {
|
|
id: string;
|
|
tenantId: string;
|
|
mailboxId: string;
|
|
};
|
|
|
|
type EmailMessageIdRow = { id: string; providerMessageId: string };
|
|
|
|
export type EmailWorkerClient = {
|
|
emailIntegrationMailbox: {
|
|
findMany(args: Record<string, unknown>): Promise<EmailMailboxRow[]>;
|
|
findFirst(args: Record<string, unknown>): Promise<EmailMailboxRow | null>;
|
|
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
|
};
|
|
emailIntegrationMessage: {
|
|
findFirst(args: Record<string, unknown>): Promise<EmailMessageIdRow | null>;
|
|
create(args: Record<string, unknown>): Promise<EmailMessageIdRow>;
|
|
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
|
};
|
|
emailIntegrationSyncCursor: {
|
|
findMany(args: Record<string, unknown>): Promise<EmailSyncCursorRow[]>;
|
|
findFirst(args: Record<string, unknown>): Promise<EmailSyncCursorRow | null>;
|
|
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
|
upsert(args: Record<string, unknown>): Promise<unknown>;
|
|
};
|
|
emailIntegrationWebhookEvent: {
|
|
findMany(args: Record<string, unknown>): Promise<EmailWebhookEventRow[]>;
|
|
update(args: Record<string, unknown>): Promise<unknown>;
|
|
updateMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
|
deleteMany(args: Record<string, unknown>): Promise<{ count: number }>;
|
|
};
|
|
};
|
|
|
|
type EmailCreds = {
|
|
accessToken: string;
|
|
refreshToken: string;
|
|
expiresAt: Date | null;
|
|
};
|
|
|
|
const MAILBOX_SELECT = {
|
|
id: true,
|
|
tenantId: true,
|
|
ownerUserId: true,
|
|
provider: true,
|
|
emailAddress: true,
|
|
oauthRefreshTokenCiphertext: true,
|
|
oauthAccessTokenCiphertext: true,
|
|
oauthAccessTokenExpiresAt: true,
|
|
webhookSubscriptionId: true,
|
|
webhookSubscriptionExpiresAt: true,
|
|
webhookClientState: true,
|
|
status: true,
|
|
lastSyncAt: true,
|
|
} as const;
|
|
|
|
async function loadCreds(client: EmailWorkerClient, tenantId: string, mailboxId: string): Promise<EmailCreds | null> {
|
|
const row = await client.emailIntegrationMailbox.findFirst({
|
|
where: { id: mailboxId, tenantId, archivedAt: null },
|
|
select: MAILBOX_SELECT,
|
|
});
|
|
if (!row?.oauthRefreshTokenCiphertext || !row.oauthAccessTokenCiphertext) {
|
|
return null;
|
|
}
|
|
const [refreshToken, accessToken] = await Promise.all([
|
|
decryptTenantField(tenantId, row.oauthRefreshTokenCiphertext, { client: client as never }),
|
|
decryptTenantField(tenantId, row.oauthAccessTokenCiphertext, { client: client as never }),
|
|
]);
|
|
return { refreshToken, accessToken, expiresAt: row.oauthAccessTokenExpiresAt };
|
|
}
|
|
|
|
async function refreshProviderTokens(provider: EmailProvider, refreshToken: string) {
|
|
if (provider === "gmail") {
|
|
const config = resolveGmailOAuthConfig();
|
|
if (!config) throw new GmailError("not_configured");
|
|
return refreshGmailAccessToken(config, refreshToken);
|
|
}
|
|
const config = resolveMicrosoftMailOAuthConfig();
|
|
if (!config) throw new GraphMailError("not_configured");
|
|
return refreshMicrosoftAccessToken(config, refreshToken);
|
|
}
|
|
|
|
async function ensureFreshCreds(
|
|
client: EmailWorkerClient,
|
|
mailbox: { id: string; tenantId: string; provider: EmailProvider },
|
|
creds: EmailCreds,
|
|
now = new Date(),
|
|
): Promise<EmailCreds> {
|
|
if (creds.expiresAt && creds.expiresAt.getTime() > now.getTime() + 60_000) {
|
|
return creds;
|
|
}
|
|
const refreshed = await refreshProviderTokens(mailbox.provider, creds.refreshToken);
|
|
const nextRefreshToken = refreshed.refreshToken ?? creds.refreshToken;
|
|
const [accessCiphertext, refreshCiphertext] = await Promise.all([
|
|
encryptTenantField(mailbox.tenantId, refreshed.accessToken, { client: client as never }),
|
|
encryptTenantField(mailbox.tenantId, nextRefreshToken, { client: client as never }),
|
|
]);
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: mailbox.id, tenantId: mailbox.tenantId },
|
|
data: {
|
|
oauthAccessTokenCiphertext: accessCiphertext,
|
|
oauthRefreshTokenCiphertext: refreshCiphertext,
|
|
oauthAccessTokenExpiresAt: refreshed.expiresAt,
|
|
status: "active",
|
|
lastErrorText: null,
|
|
},
|
|
});
|
|
return { accessToken: refreshed.accessToken, refreshToken: nextRefreshToken, expiresAt: refreshed.expiresAt };
|
|
}
|
|
|
|
/**
|
|
* Hook invoked after every message upsert — S6 wires the auto-match + activity
|
|
* logging through this seam so the sync engine stays match-agnostic.
|
|
*/
|
|
export type EmailMessageUpsertedHook = (input: {
|
|
client: EmailWorkerClient;
|
|
tenantId: string;
|
|
mailboxId: string;
|
|
ownerUserId: string;
|
|
ownerEmailAddress: string;
|
|
messageId: string;
|
|
created: boolean;
|
|
message: MappedEmailMessage;
|
|
}) => Promise<void>;
|
|
|
|
/**
|
|
* Upsert one neutral message. Dedupe order (M09):
|
|
* 1. internetMessageId within the mailbox (collapses the Graph sent-items
|
|
* re-ingest onto the local outbound mirror; providerMessageId is adopted)
|
|
* 2. the [tenantId, mailboxId, providerMessageId] unique key
|
|
* Returns the row id + whether a new row was created.
|
|
*/
|
|
export async function upsertNeutralMessage(
|
|
client: EmailWorkerClient,
|
|
ctx: { tenantId: string; mailboxId: string },
|
|
message: MappedEmailMessage,
|
|
): Promise<{ id: string; created: boolean }> {
|
|
const baseFields = {
|
|
providerThreadId: message.providerThreadId,
|
|
internetMessageId: message.internetMessageId,
|
|
direction: message.direction,
|
|
fromAddress: message.from.address,
|
|
fromName: message.from.name ?? null,
|
|
toAddresses: message.to as never,
|
|
ccAddresses: message.cc as never,
|
|
bccAddresses: message.bcc as never,
|
|
subject: message.subject,
|
|
snippet: message.snippet,
|
|
bodyHtml: message.bodyHtml,
|
|
bodyText: message.bodyText,
|
|
sentAt: message.sentAt,
|
|
receivedAt: message.receivedAt,
|
|
isRead: message.isRead,
|
|
isNewsletter: message.isNewsletter,
|
|
hasAttachments: message.hasAttachments,
|
|
providerLabels: message.providerLabels as never,
|
|
};
|
|
|
|
if (message.internetMessageId) {
|
|
const existingByRfcId = await client.emailIntegrationMessage.findFirst({
|
|
where: { tenantId: ctx.tenantId, mailboxId: ctx.mailboxId, internetMessageId: message.internetMessageId },
|
|
select: { id: true, providerMessageId: true },
|
|
});
|
|
if (existingByRfcId) {
|
|
await client.emailIntegrationMessage.updateMany({
|
|
where: { id: existingByRfcId.id, tenantId: ctx.tenantId },
|
|
data: { ...baseFields, providerMessageId: message.providerMessageId },
|
|
});
|
|
return { id: existingByRfcId.id, created: false };
|
|
}
|
|
}
|
|
|
|
const existing = await client.emailIntegrationMessage.findFirst({
|
|
where: { tenantId: ctx.tenantId, mailboxId: ctx.mailboxId, providerMessageId: message.providerMessageId },
|
|
select: { id: true, providerMessageId: true },
|
|
});
|
|
if (existing) {
|
|
await client.emailIntegrationMessage.updateMany({
|
|
where: { id: existing.id, tenantId: ctx.tenantId },
|
|
data: baseFields,
|
|
});
|
|
return { id: existing.id, created: false };
|
|
}
|
|
const created = await client.emailIntegrationMessage.create({
|
|
data: { tenantId: ctx.tenantId, mailboxId: ctx.mailboxId, providerMessageId: message.providerMessageId, ...baseFields },
|
|
});
|
|
return { id: created.id, created: true };
|
|
}
|
|
|
|
async function fetchGmailMapped(accessToken: string, messageIds: readonly string[]): Promise<MappedEmailMessage[]> {
|
|
const mapped: MappedEmailMessage[] = [];
|
|
for (const id of messageIds) {
|
|
try {
|
|
mapped.push(mapGmailMessage(await getGmailMessage(accessToken, id, "full")));
|
|
} catch (error) {
|
|
if (error instanceof GmailError && error.code === "message_not_found") {
|
|
continue; // deleted between listing and fetching — not an error
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
return mapped;
|
|
}
|
|
|
|
async function persistMessages(
|
|
client: EmailWorkerClient,
|
|
mailbox: EmailMailboxRow,
|
|
messages: readonly MappedEmailMessage[],
|
|
hook: EmailMessageUpsertedHook | undefined,
|
|
): Promise<number> {
|
|
let upserted = 0;
|
|
for (const message of messages) {
|
|
const result = await upsertNeutralMessage(client, { tenantId: mailbox.tenantId, mailboxId: mailbox.id }, message);
|
|
upserted += 1;
|
|
if (hook) {
|
|
await hook({
|
|
client,
|
|
tenantId: mailbox.tenantId,
|
|
mailboxId: mailbox.id,
|
|
ownerUserId: mailbox.ownerUserId,
|
|
ownerEmailAddress: mailbox.emailAddress,
|
|
messageId: result.id,
|
|
created: result.created,
|
|
message,
|
|
});
|
|
}
|
|
}
|
|
return upserted;
|
|
}
|
|
|
|
/** Reset a folder cursor for a bounded re-backfill after a provider 404/410. */
|
|
async function resetCursorForResync(client: EmailWorkerClient, tenantId: string, mailboxId: string, externalFolderId: string): Promise<void> {
|
|
await client.emailIntegrationSyncCursor.updateMany({
|
|
where: { tenantId, mailboxId, externalFolderId },
|
|
data: { cursorToken: null, initialSyncStatus: "pending", initialSyncProgress: null, errorText: null },
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Incremental sync of ONE mailbox (all its folders) from the stored cursors.
|
|
* Gmail: history.list from the historyId anchor; Graph: delta per folder. An
|
|
* expired cursor resets the folder to a bounded initial re-sync (next initial
|
|
* run picks it up) instead of failing the drain.
|
|
*/
|
|
export async function syncMailboxIncremental(
|
|
client: EmailWorkerClient,
|
|
tenantId: string,
|
|
mailboxId: string,
|
|
opts: { now?: Date; hook?: EmailMessageUpsertedHook } = {},
|
|
): Promise<{ upserted: number; resyncScheduled: boolean }> {
|
|
const now = opts.now ?? new Date();
|
|
const mailbox = await client.emailIntegrationMailbox.findFirst({
|
|
where: { id: mailboxId, tenantId, archivedAt: null },
|
|
select: MAILBOX_SELECT,
|
|
});
|
|
if (!mailbox || mailbox.status === "revoked" || mailbox.status === "paused") {
|
|
return { upserted: 0, resyncScheduled: false };
|
|
}
|
|
const baseCreds = await loadCreds(client, tenantId, mailboxId);
|
|
if (!baseCreds) {
|
|
return { upserted: 0, resyncScheduled: false };
|
|
}
|
|
const creds = await ensureFreshCreds(client, mailbox, baseCreds, now);
|
|
|
|
let upserted = 0;
|
|
let resyncScheduled = false;
|
|
|
|
if (mailbox.provider === "gmail") {
|
|
const cursor = await client.emailIntegrationSyncCursor.findFirst({
|
|
where: { tenantId, mailboxId, externalFolderId: EMAIL_GMAIL_FOLDER },
|
|
select: { id: true, mailboxId: true, externalFolderId: true, cursorToken: true, initialSyncStatus: true, initialSyncProgress: true },
|
|
});
|
|
if (cursor?.cursorToken) {
|
|
let pageToken: string | undefined;
|
|
let latestHistoryId: string | undefined;
|
|
try {
|
|
do {
|
|
const page = await listHistory(creds.accessToken, cursor.cursorToken, pageToken);
|
|
upserted += await persistMessages(client, mailbox, await fetchGmailMapped(creds.accessToken, page.addedMessageIds), opts.hook);
|
|
pageToken = page.nextPageToken;
|
|
latestHistoryId = page.historyId ?? latestHistoryId;
|
|
} while (pageToken);
|
|
if (latestHistoryId) {
|
|
await client.emailIntegrationSyncCursor.updateMany({
|
|
where: { tenantId, mailboxId, externalFolderId: EMAIL_GMAIL_FOLDER },
|
|
data: { cursorToken: latestHistoryId, lastCursorAdvanceAt: now },
|
|
});
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof GmailError && error.fullResyncRequired) {
|
|
await resetCursorForResync(client, tenantId, mailboxId, EMAIL_GMAIL_FOLDER);
|
|
resyncScheduled = true;
|
|
} else {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
for (const folderId of EMAIL_GRAPH_FOLDERS) {
|
|
const cursor = await client.emailIntegrationSyncCursor.findFirst({
|
|
where: { tenantId, mailboxId, externalFolderId: folderId },
|
|
select: { id: true, mailboxId: true, externalFolderId: true, cursorToken: true, initialSyncStatus: true, initialSyncProgress: true },
|
|
});
|
|
if (!cursor?.cursorToken) {
|
|
continue; // initial delta not completed yet — the initial-sync run owns it
|
|
}
|
|
let url: string | undefined = cursor.cursorToken;
|
|
let newDeltaLink: string | undefined;
|
|
try {
|
|
while (url) {
|
|
const page = await fetchDeltaPage(creds.accessToken, url);
|
|
const mapped = page.messages.filter((m) => !m["@removed"]).map((m) => mapGraphMessage(m, folderId));
|
|
upserted += await persistMessages(client, mailbox, mapped, opts.hook);
|
|
url = page.nextLink;
|
|
newDeltaLink = page.deltaLink ?? newDeltaLink;
|
|
}
|
|
if (newDeltaLink) {
|
|
await client.emailIntegrationSyncCursor.updateMany({
|
|
where: { tenantId, mailboxId, externalFolderId: folderId },
|
|
data: { cursorToken: newDeltaLink, lastCursorAdvanceAt: now },
|
|
});
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof GraphMailError && error.fullResyncRequired) {
|
|
await resetCursorForResync(client, tenantId, mailboxId, folderId);
|
|
resyncScheduled = true;
|
|
} else {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: mailboxId, tenantId },
|
|
data: { lastSyncAt: now, status: "active", lastErrorText: null },
|
|
});
|
|
return { upserted, resyncScheduled };
|
|
}
|
|
|
|
/**
|
|
* Bounded, RESUMABLE initial sync (M09 Block D): processes at most
|
|
* `INITIAL_SYNC_MAX_PAGES_PER_RUN` pages per pending/running folder cursor per
|
|
* run and persists `{pageToken|nextLink, fetchedCount}` in the progress JSON so
|
|
* an aborted run continues exactly where it stopped.
|
|
*/
|
|
export async function runEmailInitialSyncForTenant(
|
|
client: EmailWorkerClient,
|
|
tenantId: string,
|
|
opts: { now?: Date; take?: number; initialDays?: number; hook?: EmailMessageUpsertedHook } = {},
|
|
): Promise<{ folders: number; fetched: number; completed: number; failed: number }> {
|
|
const now = opts.now ?? new Date();
|
|
const initialSince = new Date(now.getTime() - (opts.initialDays ?? INITIAL_SYNC_DAYS) * 86_400_000);
|
|
const cursors = await client.emailIntegrationSyncCursor.findMany({
|
|
where: { tenantId, initialSyncStatus: { in: ["pending", "running"] } },
|
|
select: { id: true, mailboxId: true, externalFolderId: true, cursorToken: true, initialSyncStatus: true, initialSyncProgress: true },
|
|
take: opts.take ?? DEFAULT_TAKE,
|
|
});
|
|
|
|
let fetched = 0;
|
|
let completed = 0;
|
|
let failed = 0;
|
|
|
|
for (const cursor of cursors) {
|
|
const mailbox = await client.emailIntegrationMailbox.findFirst({
|
|
where: { id: cursor.mailboxId, tenantId, archivedAt: null },
|
|
select: MAILBOX_SELECT,
|
|
});
|
|
if (!mailbox || mailbox.status === "revoked" || mailbox.status === "paused") {
|
|
continue;
|
|
}
|
|
const progress = (cursor.initialSyncProgress ?? {}) as { pageToken?: string; fetchedCount?: number };
|
|
let pageToken: string | undefined = progress.pageToken;
|
|
let fetchedCount = progress.fetchedCount ?? 0;
|
|
|
|
try {
|
|
const baseCreds = await loadCreds(client, tenantId, mailbox.id);
|
|
if (!baseCreds) {
|
|
continue;
|
|
}
|
|
const creds = await ensureFreshCreds(client, mailbox, baseCreds, now);
|
|
let done = false;
|
|
|
|
for (let page = 0; page < INITIAL_SYNC_MAX_PAGES_PER_RUN; page += 1) {
|
|
if (mailbox.provider === "gmail") {
|
|
const days = Math.max(1, Math.ceil((now.getTime() - initialSince.getTime()) / 86_400_000));
|
|
const idPage = await listMessageIds(creds.accessToken, {
|
|
q: `newer_than:${days}d -in:spam -in:trash`,
|
|
pageToken,
|
|
maxResults: 50,
|
|
});
|
|
const mapped = await fetchGmailMapped(creds.accessToken, idPage.messageIds);
|
|
fetched += await persistMessages(client, mailbox, mapped, opts.hook);
|
|
fetchedCount += mapped.length;
|
|
pageToken = idPage.nextPageToken;
|
|
} else {
|
|
const url = pageToken ?? buildInitialDeltaUrl(cursor.externalFolderId, initialSince.toISOString());
|
|
const deltaPage = await fetchDeltaPage(creds.accessToken, url);
|
|
const mapped = deltaPage.messages.filter((m) => !m["@removed"]).map((m) => mapGraphMessage(m, cursor.externalFolderId));
|
|
fetched += await persistMessages(client, mailbox, mapped, opts.hook);
|
|
fetchedCount += mapped.length;
|
|
pageToken = deltaPage.nextLink;
|
|
if (!pageToken && deltaPage.deltaLink) {
|
|
// Initial delta finished — the deltaLink becomes the incremental cursor.
|
|
await client.emailIntegrationSyncCursor.updateMany({
|
|
where: { id: cursor.id, tenantId },
|
|
data: { cursorToken: deltaPage.deltaLink, lastCursorAdvanceAt: now },
|
|
});
|
|
}
|
|
}
|
|
if (!pageToken) {
|
|
done = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
await client.emailIntegrationSyncCursor.updateMany({
|
|
where: { id: cursor.id, tenantId },
|
|
data: done
|
|
? { initialSyncStatus: "completed", initialSyncProgress: { fetchedCount, percent: 100 }, lastFullSyncAt: now, errorText: null }
|
|
: { initialSyncStatus: "running", initialSyncProgress: { pageToken, fetchedCount }, errorText: null },
|
|
});
|
|
if (done) {
|
|
completed += 1;
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: mailbox.id, tenantId },
|
|
data: { lastSyncAt: now, status: "active", lastErrorText: null },
|
|
});
|
|
}
|
|
} catch (error) {
|
|
failed += 1;
|
|
await client.emailIntegrationSyncCursor.updateMany({
|
|
where: { id: cursor.id, tenantId },
|
|
data: { initialSyncStatus: "failed", errorText: redactedError(error) },
|
|
});
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: cursor.mailboxId, tenantId },
|
|
data: { status: "error", lastErrorText: "emailInbox.health.unknown" },
|
|
});
|
|
}
|
|
}
|
|
return { folders: cursors.length, fetched, completed, failed };
|
|
}
|
|
|
|
/** Access-token refresh sweep (both providers; Graph rotates refresh tokens). */
|
|
export async function refreshEmailTokensForTenant(
|
|
client: EmailWorkerClient,
|
|
tenantId: string,
|
|
opts: { now?: Date; windowMs?: number } = {},
|
|
): Promise<{ refreshed: number; failed: number }> {
|
|
const now = opts.now ?? new Date();
|
|
const horizon = new Date(now.getTime() + (opts.windowMs ?? 10 * 60_000));
|
|
const rows = await client.emailIntegrationMailbox.findMany({
|
|
where: { tenantId, archivedAt: null, status: { in: ["active", "error", "connecting"] }, oauthAccessTokenExpiresAt: { not: null, lte: horizon } },
|
|
select: MAILBOX_SELECT,
|
|
});
|
|
let refreshed = 0;
|
|
let failed = 0;
|
|
for (const row of rows) {
|
|
const creds = await loadCreds(client, tenantId, row.id);
|
|
if (!creds) {
|
|
continue;
|
|
}
|
|
try {
|
|
await ensureFreshCreds(client, row, creds, now);
|
|
refreshed += 1;
|
|
} catch {
|
|
failed += 1;
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: row.id, tenantId },
|
|
data: { status: "error", lastErrorText: "emailInbox.health.tokenExpired" },
|
|
});
|
|
}
|
|
}
|
|
return { refreshed, failed };
|
|
}
|
|
|
|
/**
|
|
* Push-subscription renewal for BOTH providers (Graph ~3 days, Gmail watch ~7
|
|
* days), keyed by webhookSubscriptionExpiresAt within `thresholdMs`.
|
|
*/
|
|
export async function renewEmailWebhooksForTenant(
|
|
client: EmailWorkerClient,
|
|
tenantId: string,
|
|
opts: { now?: Date; thresholdMs?: number; graphWebhookUrl?: string } = {},
|
|
): Promise<{ checked: number; renewed: number; failed: number; skipped: number }> {
|
|
const now = opts.now ?? new Date();
|
|
const thresholdMs = opts.thresholdMs ?? 24 * 60 * 60_000;
|
|
const rows = await client.emailIntegrationMailbox.findMany({
|
|
where: { tenantId, archivedAt: null, status: "active" },
|
|
select: MAILBOX_SELECT,
|
|
});
|
|
|
|
let renewed = 0;
|
|
let failed = 0;
|
|
let skipped = 0;
|
|
for (const row of rows) {
|
|
const needsRenewal =
|
|
row.webhookSubscriptionId !== null &&
|
|
(row.webhookSubscriptionExpiresAt === null || row.webhookSubscriptionExpiresAt.getTime() - now.getTime() <= thresholdMs);
|
|
if (!needsRenewal) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
const baseCreds = await loadCreds(client, tenantId, row.id);
|
|
if (!baseCreds) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
try {
|
|
const creds = await ensureFreshCreds(client, row, baseCreds, now);
|
|
if (row.provider === "gmail") {
|
|
const config = resolveGmailOAuthConfig();
|
|
if (!config) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
const watch = await watchMailbox(creds.accessToken, config.pubsubTopic);
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: row.id, tenantId },
|
|
data: { webhookSubscriptionExpiresAt: watch.expiresAt, lastErrorText: null },
|
|
});
|
|
} else {
|
|
if (!opts.graphWebhookUrl || !row.webhookClientState) {
|
|
skipped += 1;
|
|
continue;
|
|
}
|
|
let subscription;
|
|
try {
|
|
subscription = await renewMailSubscription(creds.accessToken, row.webhookSubscriptionId ?? "");
|
|
} catch (error) {
|
|
// Expired at Microsoft → re-create with the stored clientState.
|
|
if (error instanceof GraphMailError && error.code === "subscription_failed") {
|
|
subscription = await createMailSubscription(creds.accessToken, opts.graphWebhookUrl, row.webhookClientState);
|
|
} else {
|
|
throw error;
|
|
}
|
|
}
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: row.id, tenantId },
|
|
data: { webhookSubscriptionId: subscription.id, webhookSubscriptionExpiresAt: subscription.expiresAt, lastErrorText: null },
|
|
});
|
|
}
|
|
renewed += 1;
|
|
} catch {
|
|
failed += 1;
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: row.id, tenantId },
|
|
data: { lastErrorText: "emailInbox.health.webhookSubscriptionFailed" },
|
|
});
|
|
}
|
|
}
|
|
return { checked: rows.length, renewed, failed, skipped };
|
|
}
|
|
|
|
/** Claim + process pending provider pushes (fast-ack rows from the webhook routes). */
|
|
export async function drainEmailWebhookEventsForTenant(
|
|
client: EmailWorkerClient,
|
|
tenantId: string,
|
|
opts: { now?: Date; take?: number; hook?: EmailMessageUpsertedHook } = {},
|
|
): Promise<{ claimed: number; completed: number; failed: number }> {
|
|
const now = opts.now ?? new Date();
|
|
const rows = await client.emailIntegrationWebhookEvent.findMany({
|
|
where: { tenantId, processingStatus: "pending" },
|
|
select: { id: true, tenantId: true, mailboxId: true },
|
|
orderBy: { receivedAt: "asc" },
|
|
take: opts.take ?? DEFAULT_TAKE,
|
|
});
|
|
let claimed = 0;
|
|
let completed = 0;
|
|
let failed = 0;
|
|
// Multiple pushes for the same mailbox collapse into one incremental sync.
|
|
const syncedMailboxes = new Set<string>();
|
|
for (const row of rows) {
|
|
const claim = await client.emailIntegrationWebhookEvent.updateMany({
|
|
where: { id: row.id, tenantId, processingStatus: "pending" },
|
|
data: { processingStatus: "processing" },
|
|
});
|
|
if (claim.count !== 1) {
|
|
continue;
|
|
}
|
|
claimed += 1;
|
|
try {
|
|
if (!syncedMailboxes.has(row.mailboxId)) {
|
|
await syncMailboxIncremental(client, tenantId, row.mailboxId, { now, hook: opts.hook });
|
|
syncedMailboxes.add(row.mailboxId);
|
|
}
|
|
await client.emailIntegrationWebhookEvent.update({
|
|
where: { id: row.id },
|
|
data: { processingStatus: "completed", processedAt: now, processingErrorText: null },
|
|
});
|
|
completed += 1;
|
|
} catch (error) {
|
|
failed += 1;
|
|
await client.emailIntegrationWebhookEvent.update({
|
|
where: { id: row.id },
|
|
data: { processingStatus: "failed", processedAt: now, processingErrorText: redactedError(error) },
|
|
});
|
|
}
|
|
}
|
|
return { claimed, completed, failed };
|
|
}
|
|
|
|
/**
|
|
* Fallback poll (M09 Block D): mailboxes that saw no successful sync for
|
|
* `staleMs` (default 4h) get one incremental pull — the safety net when a push
|
|
* channel silently dies. Push remains the primary mechanism.
|
|
*/
|
|
export async function pollStaleEmailMailboxesForTenant(
|
|
client: EmailWorkerClient,
|
|
tenantId: string,
|
|
opts: { now?: Date; staleMs?: number; take?: number; hook?: EmailMessageUpsertedHook } = {},
|
|
): Promise<{ polled: number; failed: number }> {
|
|
const now = opts.now ?? new Date();
|
|
const staleBefore = new Date(now.getTime() - (opts.staleMs ?? 4 * 60 * 60_000));
|
|
const rows = await client.emailIntegrationMailbox.findMany({
|
|
where: { tenantId, archivedAt: null, status: "active", lastSyncAt: { lte: staleBefore } },
|
|
select: MAILBOX_SELECT,
|
|
});
|
|
let polled = 0;
|
|
let failed = 0;
|
|
for (const row of rows.slice(0, opts.take ?? DEFAULT_TAKE)) {
|
|
try {
|
|
await syncMailboxIncremental(client, tenantId, row.id, { now, hook: opts.hook });
|
|
polled += 1;
|
|
} catch {
|
|
failed += 1;
|
|
await client.emailIntegrationMailbox.updateMany({
|
|
where: { id: row.id, tenantId },
|
|
data: { status: "error", lastErrorText: "emailInbox.health.unknown" },
|
|
});
|
|
}
|
|
}
|
|
return { polled, failed };
|
|
}
|
|
|
|
/** 30-day webhook-event retention (failed rows: 90 days for diagnosis). */
|
|
export async function cleanupEmailWebhookEventsForTenant(
|
|
client: EmailWorkerClient,
|
|
tenantId: string,
|
|
opts: { now?: Date } = {},
|
|
): Promise<{ deleted: number }> {
|
|
const now = opts.now ?? new Date();
|
|
const doneBefore = new Date(now.getTime() - WEBHOOK_RETENTION_DAYS * 86_400_000);
|
|
const failedBefore = new Date(now.getTime() - WEBHOOK_FAILED_RETENTION_DAYS * 86_400_000);
|
|
const done = await client.emailIntegrationWebhookEvent.deleteMany({
|
|
where: { tenantId, processingStatus: { in: ["completed", "skipped"] }, receivedAt: { lte: doneBefore } },
|
|
});
|
|
const failedRows = await client.emailIntegrationWebhookEvent.deleteMany({
|
|
where: { tenantId, processingStatus: "failed", receivedAt: { lte: failedBefore } },
|
|
});
|
|
return { deleted: done.count + failedRows.count };
|
|
}
|
|
|
|
function redactedError(error: unknown): string {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
return message
|
|
.replace(/(authorization\s*:\s*Bearer\s+)\S+/gi, "$1[redacted]")
|
|
.replace(/(token|secret|password)(\s*[=:]\s*)\S+/gi, "$1$2[redacted]")
|
|
.slice(0, 500);
|
|
}
|