1027 lines
38 KiB
TypeScript
1027 lines
38 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
import type { Server } from "node:http";
|
|
|
|
import { chatErrors } from "@hivegrid/chat";
|
|
import { CHAT_GATEWAY_MIN_SECRET_BYTES, verifyChatGatewayToken, type ChatGatewayTokenIdentity } from "@hivegrid/chat/gateway-auth.server";
|
|
import {
|
|
assertMemberChannelWriteAccess,
|
|
fillChatEventGap,
|
|
markChatMessageRead,
|
|
revalidateChatGatewayAccess,
|
|
sendTopLevelChatMessage,
|
|
} from "@hivegrid/chat/repository.server";
|
|
import { CatalogError, foundationErrors } from "@hivegrid/foundation";
|
|
import type { HiveGridLogger } from "@hivegrid/logger";
|
|
import {
|
|
CHAT_CLIENT_FRAME_TYPES,
|
|
CHAT_EVENT_NAMES,
|
|
CHAT_MAX_ACTIVE_SUBSCRIPTIONS,
|
|
CHAT_SERVER_FRAME_TYPES,
|
|
CHAT_WIRE_MAX_CLIENT_JSON_LENGTH,
|
|
CHAT_WIRE_MAX_GAP_EVENTS,
|
|
isPersistentChatEvent,
|
|
parseChatClientFrameJson,
|
|
serializeChatServerFrame,
|
|
type ChatClientFrame,
|
|
type ChatConnectionPresence,
|
|
type ChatEphemeralEvent,
|
|
type ChatServerEvent,
|
|
type ChatServerFrame,
|
|
} from "@hivegrid/realtime";
|
|
import type { IncomingMessage } from "node:http";
|
|
import type { Duplex } from "node:stream";
|
|
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
|
|
|
import { getAllowedCorsOrigin } from "../cors.js";
|
|
import { apiLogger } from "../logger.js";
|
|
import { getSharedStore } from "../shared-store.js";
|
|
import { createChatConnectionAdmission, type ChatConnectionAdmission, type ChatConnectionIdentity } from "./admission.js";
|
|
import { CHAT_TYPING_LEASE_TTL_MS, createChatRoomHub, type ChatRoomHub, type ChatRoomSubscription } from "./room-hub.js";
|
|
|
|
export const CHAT_GATEWAY_PATH = "/ws/chat";
|
|
export const CHAT_GATEWAY_AUTH_TIMEOUT_MS = 5_000;
|
|
export const CHAT_GATEWAY_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
export const CHAT_GATEWAY_MAX_BUFFERED_BYTES = 1024 * 1024;
|
|
export const CHAT_GATEWAY_MAX_CATCH_UP_EVENTS = 2_000;
|
|
export const CHAT_GATEWAY_MAX_LIVE_RESYNC_EVENTS = 2_000;
|
|
export const CHAT_GATEWAY_MAX_EPHEMERAL_BUFFER = 100;
|
|
export const CHAT_GATEWAY_MAX_PENDING_COMMANDS = 64;
|
|
export const CHAT_GATEWAY_MAX_SUBSCRIPTIONS = CHAT_MAX_ACTIVE_SUBSCRIPTIONS;
|
|
export const CHAT_GATEWAY_AUTH_CACHE_TTL_MS = 30_000;
|
|
export const CHAT_GATEWAY_MAX_PROCESS_CONNECTIONS = 1_000;
|
|
|
|
const CLOSE_UNAUTHORIZED = 4_401;
|
|
const CLOSE_TOKEN_EXPIRED = 4_402;
|
|
const CLOSE_POLICY_VIOLATION = 1_008;
|
|
const CLOSE_RETRY_LATER = 1_013;
|
|
const CLOSE_STREAM_RESET = 1_012;
|
|
|
|
type GatewayRepositoryContext = {
|
|
tenantId: string;
|
|
userId: string;
|
|
sessionId: string;
|
|
};
|
|
|
|
type ChatGatewayRepository = {
|
|
revalidateAccess(context: GatewayRepositoryContext & { chatIds: readonly string[] }): Promise<void>;
|
|
assertWriteAccess(context: GatewayRepositoryContext & { chatId: string }): Promise<void>;
|
|
fillGap(context: GatewayRepositoryContext & { chatId: string; afterCursor: string; limit?: number }): ReturnType<typeof fillChatEventGap>;
|
|
sendMessage(
|
|
context: GatewayRepositoryContext & {
|
|
chatId: string;
|
|
requestId: string;
|
|
clientMessageId: string;
|
|
body: string;
|
|
mentions: Parameters<typeof sendTopLevelChatMessage>[0]["mentions"];
|
|
}
|
|
): ReturnType<typeof sendTopLevelChatMessage>;
|
|
markRead(context: GatewayRepositoryContext & { chatId: string; messageId: string; requestId: string }): ReturnType<typeof markChatMessageRead>;
|
|
};
|
|
|
|
export type ChatGatewayDependencies = {
|
|
secret: string;
|
|
roomHub: ChatRoomHub;
|
|
admission: ChatConnectionAdmission;
|
|
repository: ChatGatewayRepository;
|
|
isAllowedOrigin(origin: string | undefined): boolean;
|
|
logger: HiveGridLogger;
|
|
now(): Date;
|
|
heartbeatIntervalMs: number;
|
|
maxProcessConnections: number;
|
|
typingLeaseTtlMs: number;
|
|
};
|
|
|
|
export type ChatGatewayHandle = {
|
|
close(): Promise<void>;
|
|
};
|
|
|
|
type ChannelSubscriptionState = {
|
|
chatId: string;
|
|
roomSubscription: ChatRoomSubscription;
|
|
catchingUp: boolean;
|
|
closed: boolean;
|
|
lastCursor: string;
|
|
authorizationValidatedAtMs: number;
|
|
persistentSignalPending: boolean;
|
|
pendingEphemeral: Map<string, ChatEphemeralEvent>;
|
|
deliveryDrain: Promise<void> | null;
|
|
};
|
|
|
|
type TypingLeaseState = {
|
|
generation: number;
|
|
timer: ReturnType<typeof setTimeout>;
|
|
};
|
|
|
|
type ConnectionState = {
|
|
id: string;
|
|
socket: WebSocket;
|
|
identity: ChatGatewayTokenIdentity | null;
|
|
tokenExpiryTimer: ReturnType<typeof setTimeout> | null;
|
|
authTimer: ReturnType<typeof setTimeout>;
|
|
pingOutstanding: boolean;
|
|
closing: boolean;
|
|
closed: boolean;
|
|
subscriptions: Map<string, ChannelSubscriptionState>;
|
|
presence: Exclude<ChatConnectionPresence, "offline">;
|
|
typingLeases: Map<string, TypingLeaseState>;
|
|
leaseOperations: Promise<void>;
|
|
authorizationRefresh: Promise<void> | null;
|
|
presenceRefresh: Promise<void> | null;
|
|
admitted: boolean;
|
|
commands: Promise<void>;
|
|
pendingCommands: number;
|
|
remainingInitialCatchUpEvents: number;
|
|
};
|
|
|
|
const defaultRepository: ChatGatewayRepository = {
|
|
revalidateAccess: revalidateChatGatewayAccess,
|
|
assertWriteAccess: assertMemberChannelWriteAccess,
|
|
fillGap: fillChatEventGap,
|
|
sendMessage: sendTopLevelChatMessage,
|
|
markRead: markChatMessageRead,
|
|
};
|
|
|
|
export function attachChatGateway(server: Server, overrides: Partial<ChatGatewayDependencies> = {}): ChatGatewayHandle | null {
|
|
const secret = overrides.secret ?? process.env.CHAT_GATEWAY_AUTH_SECRET ?? "";
|
|
if (secret !== secret.trim() || Buffer.byteLength(secret, "utf8") < CHAT_GATEWAY_MIN_SECRET_BYTES) {
|
|
apiLogger.warn({ component: "chat-gateway" }, "Chat gateway is disabled because its auth secret is unavailable");
|
|
return null;
|
|
}
|
|
|
|
let sharedStore: ReturnType<typeof getSharedStore> | null = null;
|
|
const resolveSharedStore = () => (sharedStore ??= getSharedStore());
|
|
const roomHub =
|
|
overrides.roomHub ??
|
|
createChatRoomHub(resolveSharedStore(), {
|
|
onDiagnostic: (diagnostic) => apiLogger.warn({ component: "chat-gateway", diagnostic }, "Chat room fan-out rejected an event"),
|
|
});
|
|
const dependencies: ChatGatewayDependencies = {
|
|
secret,
|
|
roomHub,
|
|
admission: overrides.admission ?? createChatConnectionAdmission(resolveSharedStore()),
|
|
repository: overrides.repository ?? defaultRepository,
|
|
isAllowedOrigin: overrides.isAllowedOrigin ?? ((origin) => origin !== undefined && getAllowedCorsOrigin(origin) === origin),
|
|
logger: overrides.logger ?? apiLogger,
|
|
now: overrides.now ?? (() => new Date()),
|
|
heartbeatIntervalMs: overrides.heartbeatIntervalMs ?? CHAT_GATEWAY_HEARTBEAT_INTERVAL_MS,
|
|
maxProcessConnections: overrides.maxProcessConnections ?? CHAT_GATEWAY_MAX_PROCESS_CONNECTIONS,
|
|
typingLeaseTtlMs: overrides.typingLeaseTtlMs ?? CHAT_TYPING_LEASE_TTL_MS,
|
|
};
|
|
const webSocketServer = new WebSocketServer({ noServer: true, maxPayload: CHAT_WIRE_MAX_CLIENT_JSON_LENGTH });
|
|
const connections = new Set<ConnectionState>();
|
|
let closed = false;
|
|
let closePromise: Promise<void> | null = null;
|
|
|
|
const onUpgrade = (request: IncomingMessage, socket: Duplex, head: Buffer): void => {
|
|
if (closed || connections.size >= dependencies.maxProcessConnections || !isAcceptedUpgradeRequest(request, dependencies)) {
|
|
socket.destroy();
|
|
return;
|
|
}
|
|
webSocketServer.handleUpgrade(request, socket, head, (webSocket) => {
|
|
webSocketServer.emit("connection", webSocket, request);
|
|
});
|
|
};
|
|
|
|
const heartbeat = setInterval(() => {
|
|
for (const connection of connections) {
|
|
if (connection.pingOutstanding) {
|
|
terminateConnection(connection);
|
|
continue;
|
|
}
|
|
connection.pingOutstanding = true;
|
|
try {
|
|
connection.socket.ping();
|
|
} catch {
|
|
terminateConnection(connection);
|
|
}
|
|
}
|
|
}, dependencies.heartbeatIntervalMs);
|
|
heartbeat.unref();
|
|
|
|
webSocketServer.on("connection", (socket) => {
|
|
const state = createConnectionState(socket);
|
|
connections.add(state);
|
|
socket.on("pong", () => {
|
|
if (!state.pingOutstanding || state.closing || state.closed) return;
|
|
state.pingOutstanding = false;
|
|
if (!state.identity) return;
|
|
void refreshPresenceLeasesCoalesced(state, dependencies).catch((error) => handleOperationError(state, error, dependencies));
|
|
});
|
|
socket.on("message", (raw, isBinary) => {
|
|
if (state.closing || state.closed || state.socket.readyState !== WebSocket.OPEN) return;
|
|
if (state.pendingCommands >= CHAT_GATEWAY_MAX_PENDING_COMMANDS) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
state.pendingCommands += 1;
|
|
state.commands = state.commands
|
|
.then(() => handleSocketMessage(state, raw, isBinary, dependencies))
|
|
.catch((error) => handleOperationError(state, error, dependencies))
|
|
.finally(() => {
|
|
state.pendingCommands = Math.max(0, state.pendingCommands - 1);
|
|
});
|
|
});
|
|
socket.on("close", () => {
|
|
connections.delete(state);
|
|
void closeConnection(state, dependencies);
|
|
});
|
|
socket.on("error", () => {
|
|
dependencies.logger.warn({ component: "chat-gateway", connectionId: state.id }, "Chat WebSocket failed");
|
|
});
|
|
});
|
|
|
|
server.on("upgrade", onUpgrade);
|
|
const handle: ChatGatewayHandle = {
|
|
close() {
|
|
if (closePromise) return closePromise;
|
|
closed = true;
|
|
clearInterval(heartbeat);
|
|
server.off("upgrade", onUpgrade);
|
|
closePromise = (async () => {
|
|
await Promise.allSettled([...connections].map((connection) => closeConnection(connection, dependencies)));
|
|
for (const connection of connections) connection.socket.terminate();
|
|
connections.clear();
|
|
await dependencies.roomHub.close();
|
|
await new Promise<void>((resolve) => webSocketServer.close(() => resolve()));
|
|
})();
|
|
return closePromise;
|
|
},
|
|
};
|
|
server.once("close", () => {
|
|
void handle.close();
|
|
});
|
|
return handle;
|
|
}
|
|
|
|
function createConnectionState(socket: WebSocket): ConnectionState {
|
|
const state: ConnectionState = {
|
|
id: randomUUID(),
|
|
socket,
|
|
identity: null,
|
|
tokenExpiryTimer: null,
|
|
authTimer: null as unknown as ReturnType<typeof setTimeout>,
|
|
pingOutstanding: false,
|
|
closing: false,
|
|
closed: false,
|
|
subscriptions: new Map(),
|
|
presence: "online",
|
|
typingLeases: new Map(),
|
|
leaseOperations: Promise.resolve(),
|
|
authorizationRefresh: null,
|
|
presenceRefresh: null,
|
|
admitted: false,
|
|
commands: Promise.resolve(),
|
|
pendingCommands: 0,
|
|
remainingInitialCatchUpEvents: CHAT_GATEWAY_MAX_CATCH_UP_EVENTS,
|
|
};
|
|
state.authTimer = setTimeout(() => beginClose(state, CLOSE_UNAUTHORIZED), CHAT_GATEWAY_AUTH_TIMEOUT_MS);
|
|
state.authTimer.unref();
|
|
return state;
|
|
}
|
|
|
|
function isAcceptedUpgradeRequest(request: IncomingMessage, dependencies: ChatGatewayDependencies): boolean {
|
|
if (request.method !== "GET" || request.headers["sec-websocket-protocol"] !== undefined) return false;
|
|
const origin = request.headers.origin;
|
|
if (!origin || !dependencies.isAllowedOrigin(origin)) return false;
|
|
try {
|
|
const url = new URL(request.url ?? "", "http://chat-gateway.invalid");
|
|
return url.pathname === CHAT_GATEWAY_PATH && url.search === "" && url.hash === "";
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function handleSocketMessage(state: ConnectionState, raw: RawData, isBinary: boolean, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (state.closing || state.closed) return;
|
|
if (isBinary) {
|
|
beginClose(state, CLOSE_POLICY_VIOLATION);
|
|
return;
|
|
}
|
|
|
|
let frame: ChatClientFrame;
|
|
try {
|
|
frame = parseChatClientFrameJson(rawDataToUtf8(raw));
|
|
} catch {
|
|
sendError(state, chatErrors.invalidInput.i18nKey, false);
|
|
beginClose(state, CLOSE_POLICY_VIOLATION);
|
|
return;
|
|
}
|
|
|
|
if (!state.identity) {
|
|
if (frame.type !== CHAT_CLIENT_FRAME_TYPES.authenticate) {
|
|
beginClose(state, CLOSE_UNAUTHORIZED);
|
|
return;
|
|
}
|
|
await authenticateConnection(state, frame, dependencies);
|
|
return;
|
|
}
|
|
if (frame.type === CHAT_CLIENT_FRAME_TYPES.authenticate) {
|
|
beginClose(state, CLOSE_POLICY_VIOLATION);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
await handleAuthenticatedFrame(state, frame, dependencies);
|
|
} catch (error) {
|
|
handleOperationError(state, error, dependencies, frame);
|
|
}
|
|
}
|
|
|
|
async function authenticateConnection(
|
|
state: ConnectionState,
|
|
frame: Extract<ChatClientFrame, { type: "authenticate" }>,
|
|
dependencies: ChatGatewayDependencies
|
|
): Promise<void> {
|
|
if (frame.resumeCursors.length > CHAT_GATEWAY_MAX_SUBSCRIPTIONS) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
const identity = verifyChatGatewayToken(frame.token, { secret: dependencies.secret, now: dependencies.now() });
|
|
if (!identity) {
|
|
beginClose(state, CLOSE_UNAUTHORIZED);
|
|
return;
|
|
}
|
|
|
|
const admissionIdentity = connectionIdentity(state, identity);
|
|
let admitted = false;
|
|
try {
|
|
admitted = await dependencies.admission.acquire(admissionIdentity);
|
|
} catch (error) {
|
|
logAdmissionFailure(state, error, dependencies, "acquire");
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
if (!admitted) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
if (state.closing || state.closed) {
|
|
await releaseAdmissionLease(state, admissionIdentity, dependencies);
|
|
return;
|
|
}
|
|
|
|
const context = repositoryContext(identity);
|
|
try {
|
|
await dependencies.repository.revalidateAccess({
|
|
...context,
|
|
chatIds: [...new Set(frame.resumeCursors.map((cursor) => cursor.chatId))],
|
|
});
|
|
} catch {
|
|
await releaseAdmissionLease(state, admissionIdentity, dependencies);
|
|
beginClose(state, CLOSE_UNAUTHORIZED);
|
|
return;
|
|
}
|
|
if (state.closing || state.closed) {
|
|
await releaseAdmissionLease(state, admissionIdentity, dependencies);
|
|
return;
|
|
}
|
|
|
|
state.identity = identity;
|
|
state.admitted = true;
|
|
clearTimeout(state.authTimer);
|
|
const expiresInMs = Math.max(0, identity.expiresAt * 1_000 - dependencies.now().getTime());
|
|
state.tokenExpiryTimer = setTimeout(() => beginClose(state, CLOSE_TOKEN_EXPIRED), expiresInMs);
|
|
state.tokenExpiryTimer.unref();
|
|
if (!sendFrame(state, { type: CHAT_SERVER_FRAME_TYPES.ready, connectionId: state.id }, "durable")) return;
|
|
|
|
for (const cursor of frame.resumeCursors) {
|
|
if (state.closing || state.closed) return;
|
|
await subscribeAndCatchUp(state, cursor.chatId, cursor.afterCursor, dependencies);
|
|
}
|
|
}
|
|
|
|
async function handleAuthenticatedFrame(
|
|
state: ConnectionState,
|
|
frame: Exclude<ChatClientFrame, { type: "authenticate" }>,
|
|
dependencies: ChatGatewayDependencies
|
|
): Promise<void> {
|
|
if (state.closing || state.closed) return;
|
|
const identity = requireIdentity(state);
|
|
const context = repositoryContext(identity);
|
|
switch (frame.type) {
|
|
case CHAT_CLIENT_FRAME_TYPES.subscribe:
|
|
await subscribeAndCatchUp(state, frame.chatId, frame.afterCursor, dependencies);
|
|
return;
|
|
case CHAT_CLIENT_FRAME_TYPES.unsubscribe:
|
|
await removeSubscription(state, frame.chatId, dependencies);
|
|
return;
|
|
case CHAT_CLIENT_FRAME_TYPES.sendMessage: {
|
|
requireSubscription(state, frame.chatId);
|
|
await stopTypingForChannel(state, frame.chatId, dependencies);
|
|
const result = await dependencies.repository.sendMessage({
|
|
...context,
|
|
chatId: frame.chatId,
|
|
requestId: frame.requestId,
|
|
clientMessageId: frame.clientMessageId,
|
|
body: frame.body,
|
|
mentions: frame.mentions,
|
|
});
|
|
if (result.replayed) {
|
|
sendFrame(state, { type: CHAT_SERVER_FRAME_TYPES.event, event: result.event }, "durable");
|
|
} else {
|
|
await dependencies.roomHub.publish(result.event);
|
|
}
|
|
return;
|
|
}
|
|
case CHAT_CLIENT_FRAME_TYPES.markRead: {
|
|
requireSubscription(state, frame.chatId);
|
|
const result = await dependencies.repository.markRead({
|
|
...context,
|
|
chatId: frame.chatId,
|
|
messageId: frame.messageId,
|
|
requestId: frame.requestId,
|
|
});
|
|
await dependencies.roomHub.publish(result.event);
|
|
return;
|
|
}
|
|
case CHAT_CLIENT_FRAME_TYPES.typing: {
|
|
const subscription = requireSubscription(state, frame.chatId);
|
|
if (!(await ensureSubscriptionAuthorizationFresh(state, subscription, dependencies))) return;
|
|
if (frame.status === "started") {
|
|
await dependencies.repository.assertWriteAccess({ ...context, chatId: frame.chatId });
|
|
await startTypingForChannel(state, frame.chatId, dependencies);
|
|
} else {
|
|
await stopTypingForChannel(state, frame.chatId, dependencies);
|
|
}
|
|
return;
|
|
}
|
|
case CHAT_CLIENT_FRAME_TYPES.presence:
|
|
await updatePresence(state, frame.status, dependencies);
|
|
return;
|
|
case CHAT_CLIENT_FRAME_TYPES.ping:
|
|
sendFrame(state, { type: CHAT_SERVER_FRAME_TYPES.pong }, "ephemeral");
|
|
return;
|
|
}
|
|
}
|
|
|
|
async function subscribeAndCatchUp(state: ConnectionState, chatId: string, afterCursor: string, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (state.closing || state.closed) return;
|
|
if (!state.subscriptions.has(chatId) && state.subscriptions.size >= CHAT_GATEWAY_MAX_SUBSCRIPTIONS) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
const identity = requireIdentity(state);
|
|
await removeSubscription(state, chatId, dependencies, { releaseLeases: false });
|
|
if (state.closing || state.closed) return;
|
|
|
|
const subscriptionState: ChannelSubscriptionState = {
|
|
chatId,
|
|
roomSubscription: { unsubscribe: async () => undefined },
|
|
catchingUp: true,
|
|
closed: false,
|
|
lastCursor: afterCursor,
|
|
authorizationValidatedAtMs: 0,
|
|
persistentSignalPending: false,
|
|
pendingEphemeral: new Map(),
|
|
deliveryDrain: null,
|
|
};
|
|
const roomSubscription = await dependencies.roomHub.subscribe({ tenantId: identity.tenantId, chatId }, async (event) =>
|
|
handleRoomEvent(state, subscriptionState, event, dependencies)
|
|
);
|
|
if (state.closing || state.closed) {
|
|
await roomSubscription.unsubscribe();
|
|
return;
|
|
}
|
|
subscriptionState.roomSubscription = roomSubscription;
|
|
state.subscriptions.set(chatId, subscriptionState);
|
|
|
|
try {
|
|
const caughtUp = await sendInitialGap(state, subscriptionState, dependencies);
|
|
if (!caughtUp) {
|
|
await removeSubscription(state, chatId, dependencies);
|
|
return;
|
|
}
|
|
subscriptionState.authorizationValidatedAtMs = dependencies.now().getTime();
|
|
if (state.closing || state.closed || subscriptionState.closed) return;
|
|
await touchPresenceForChannel(state, chatId, dependencies);
|
|
if (state.closing || state.closed || subscriptionState.closed) return;
|
|
const presenceSnapshot = await dependencies.roomHub.getPresenceSnapshot({ tenantId: identity.tenantId, chatId }, dependencies.now());
|
|
for (const event of presenceSnapshot) {
|
|
sendFrame(state, { type: CHAT_SERVER_FRAME_TYPES.event, event }, "ephemeral");
|
|
if (state.closing || state.closed || subscriptionState.closed) return;
|
|
}
|
|
subscriptionState.catchingUp = false;
|
|
if (
|
|
!sendFrame(
|
|
state,
|
|
{
|
|
type: CHAT_SERVER_FRAME_TYPES.subscribed,
|
|
chatId,
|
|
result: { status: "ok", events: [], nextCursor: subscriptionState.lastCursor, hasMore: false },
|
|
},
|
|
"durable"
|
|
)
|
|
) {
|
|
await removeSubscription(state, chatId, dependencies);
|
|
return;
|
|
}
|
|
if (hasPendingDelivery(subscriptionState)) await scheduleDeliveryDrain(state, subscriptionState, dependencies);
|
|
} catch (error) {
|
|
await removeSubscription(state, chatId, dependencies);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function sendInitialGap(state: ConnectionState, subscription: ChannelSubscriptionState, dependencies: ChatGatewayDependencies): Promise<boolean> {
|
|
const identity = requireIdentity(state);
|
|
let cursor = subscription.lastCursor;
|
|
|
|
while (!state.closing && !state.closed && !subscription.closed) {
|
|
const result = await dependencies.repository.fillGap({
|
|
...repositoryContext(identity),
|
|
chatId: subscription.chatId,
|
|
afterCursor: cursor,
|
|
limit: gapPageLimit(state.remainingInitialCatchUpEvents),
|
|
});
|
|
if (state.closing || state.closed || subscription.closed) return false;
|
|
if (result.status === "reset_required") {
|
|
if (
|
|
!sendFrame(
|
|
state,
|
|
{
|
|
type: CHAT_SERVER_FRAME_TYPES.subscribed,
|
|
chatId: subscription.chatId,
|
|
result,
|
|
},
|
|
"durable"
|
|
)
|
|
) {
|
|
return false;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
if (!consumeInitialCatchUpBudget(state, result.events.length)) return false;
|
|
subscription.lastCursor = result.nextCursor;
|
|
for (const event of result.events) {
|
|
if (!sendFrame(state, { type: CHAT_SERVER_FRAME_TYPES.event, event }, "durable")) return false;
|
|
}
|
|
cursor = result.nextCursor;
|
|
if (!result.hasMore) {
|
|
return true;
|
|
}
|
|
if (state.remainingInitialCatchUpEvents === 0) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return false;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function handleRoomEvent(state: ConnectionState, subscription: ChannelSubscriptionState, event: ChatServerEvent, dependencies: ChatGatewayDependencies): void {
|
|
if (state.closing || state.closed || subscription.closed) return;
|
|
if (isPersistentChatEvent(event)) subscription.persistentSignalPending = true;
|
|
else enqueueEphemeral(subscription, event);
|
|
if (!subscription.catchingUp) void scheduleDeliveryDrain(state, subscription, dependencies);
|
|
}
|
|
|
|
function enqueueEphemeral(subscription: ChannelSubscriptionState, event: ChatEphemeralEvent): void {
|
|
const key = ephemeralQueueKey(event);
|
|
if (subscription.pendingEphemeral.has(key)) {
|
|
subscription.pendingEphemeral.delete(key);
|
|
} else if (subscription.pendingEphemeral.size >= CHAT_GATEWAY_MAX_EPHEMERAL_BUFFER) {
|
|
const oldestKey = subscription.pendingEphemeral.keys().next().value as string | undefined;
|
|
if (oldestKey !== undefined) subscription.pendingEphemeral.delete(oldestKey);
|
|
}
|
|
subscription.pendingEphemeral.set(key, event);
|
|
}
|
|
|
|
function ephemeralQueueKey(event: ChatEphemeralEvent): string {
|
|
const family = event.event === CHAT_EVENT_NAMES.typingStarted || event.event === CHAT_EVENT_NAMES.typingStopped ? "typing" : "presence";
|
|
return `${event.chatId}:${event.data.userId}:${family}`;
|
|
}
|
|
|
|
function hasPendingDelivery(subscription: ChannelSubscriptionState): boolean {
|
|
return subscription.persistentSignalPending || subscription.pendingEphemeral.size > 0;
|
|
}
|
|
|
|
function scheduleDeliveryDrain(state: ConnectionState, subscription: ChannelSubscriptionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (subscription.deliveryDrain) return subscription.deliveryDrain;
|
|
let tracked: Promise<void>;
|
|
tracked = drainPendingDeliveries(state, subscription, dependencies)
|
|
.catch((error) => handleOperationError(state, error, dependencies))
|
|
.finally(() => {
|
|
if (subscription.deliveryDrain === tracked) subscription.deliveryDrain = null;
|
|
if (!subscription.catchingUp && !subscription.closed && !state.closing && !state.closed && hasPendingDelivery(subscription)) {
|
|
void scheduleDeliveryDrain(state, subscription, dependencies);
|
|
}
|
|
});
|
|
subscription.deliveryDrain = tracked;
|
|
return tracked;
|
|
}
|
|
|
|
async function drainPendingDeliveries(state: ConnectionState, subscription: ChannelSubscriptionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
while (!state.closing && !state.closed && !subscription.closed && !subscription.catchingUp) {
|
|
if (subscription.persistentSignalPending) {
|
|
subscription.persistentSignalPending = false;
|
|
await synchronizePersistentGap(state, subscription, dependencies);
|
|
continue;
|
|
}
|
|
const next = subscription.pendingEphemeral.entries().next();
|
|
if (next.done) return;
|
|
const [key, event] = next.value;
|
|
subscription.pendingEphemeral.delete(key);
|
|
await deliverEphemeral(state, subscription, event, dependencies);
|
|
}
|
|
}
|
|
|
|
async function synchronizePersistentGap(state: ConnectionState, subscription: ChannelSubscriptionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
const identity = requireIdentity(state);
|
|
let remainingLiveResyncEvents = CHAT_GATEWAY_MAX_LIVE_RESYNC_EVENTS;
|
|
while (!state.closing && !state.closed && !subscription.closed) {
|
|
const result = await dependencies.repository.fillGap({
|
|
...repositoryContext(identity),
|
|
chatId: subscription.chatId,
|
|
afterCursor: subscription.lastCursor,
|
|
limit: gapPageLimit(remainingLiveResyncEvents),
|
|
});
|
|
if (state.closing || state.closed || subscription.closed) return;
|
|
if (result.status === "reset_required") {
|
|
beginClose(state, CLOSE_STREAM_RESET);
|
|
return;
|
|
}
|
|
if (result.events.length > remainingLiveResyncEvents) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
remainingLiveResyncEvents -= result.events.length;
|
|
subscription.authorizationValidatedAtMs = dependencies.now().getTime();
|
|
for (const event of result.events) {
|
|
if (!sendFrame(state, { type: CHAT_SERVER_FRAME_TYPES.event, event }, "durable")) return;
|
|
}
|
|
subscription.lastCursor = result.nextCursor;
|
|
if (!result.hasMore) return;
|
|
if (remainingLiveResyncEvents === 0) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function deliverEphemeral(
|
|
state: ConnectionState,
|
|
subscription: ChannelSubscriptionState,
|
|
event: ChatEphemeralEvent,
|
|
dependencies: ChatGatewayDependencies
|
|
): Promise<void> {
|
|
if (state.closing || state.closed || subscription.closed) return;
|
|
if (!(await ensureSubscriptionAuthorizationFresh(state, subscription, dependencies))) return;
|
|
if (state.closing || state.closed || subscription.closed) return;
|
|
sendFrame(state, { type: CHAT_SERVER_FRAME_TYPES.event, event }, "ephemeral");
|
|
}
|
|
|
|
async function updatePresence(
|
|
state: ConnectionState,
|
|
presence: Exclude<ChatConnectionPresence, "offline">,
|
|
dependencies: ChatGatewayDependencies
|
|
): Promise<void> {
|
|
if (state.closing || state.closed) return;
|
|
state.presence = presence;
|
|
await refreshPresenceLeasesCoalesced(state, dependencies);
|
|
}
|
|
|
|
async function refreshPresenceLeases(state: ConnectionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (state.closing || state.closed || !state.identity) return;
|
|
let admitted = false;
|
|
try {
|
|
admitted = await dependencies.admission.refresh(connectionIdentity(state, state.identity));
|
|
} catch (error) {
|
|
logAdmissionFailure(state, error, dependencies, "refresh");
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
throw error;
|
|
}
|
|
if (!admitted) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return;
|
|
}
|
|
await refreshAuthorizationCoalesced(state, dependencies);
|
|
for (const chatId of [...state.subscriptions.keys()]) {
|
|
if (state.closing || state.closed) return;
|
|
await touchPresenceForChannel(state, chatId, dependencies);
|
|
}
|
|
}
|
|
|
|
function refreshPresenceLeasesCoalesced(state: ConnectionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (state.presenceRefresh) return state.presenceRefresh;
|
|
let tracked: Promise<void>;
|
|
tracked = refreshPresenceLeases(state, dependencies).finally(() => {
|
|
if (state.presenceRefresh === tracked) state.presenceRefresh = null;
|
|
});
|
|
state.presenceRefresh = tracked;
|
|
return tracked;
|
|
}
|
|
|
|
function refreshAuthorizationCoalesced(state: ConnectionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (state.authorizationRefresh) return state.authorizationRefresh;
|
|
let tracked: Promise<void>;
|
|
tracked = revalidateCurrentSubscriptions(state, dependencies).finally(() => {
|
|
if (state.authorizationRefresh === tracked) state.authorizationRefresh = null;
|
|
});
|
|
state.authorizationRefresh = tracked;
|
|
return tracked;
|
|
}
|
|
|
|
async function revalidateCurrentSubscriptions(state: ConnectionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (state.closing || state.closed || !state.identity) return;
|
|
const subscriptions = [...state.subscriptions.values()].filter((subscription) => !subscription.closed);
|
|
await dependencies.repository.revalidateAccess({
|
|
...repositoryContext(state.identity),
|
|
chatIds: subscriptions.map((subscription) => subscription.chatId),
|
|
});
|
|
const validatedAtMs = dependencies.now().getTime();
|
|
for (const subscription of subscriptions) {
|
|
if (state.subscriptions.get(subscription.chatId) === subscription && !subscription.closed) {
|
|
subscription.authorizationValidatedAtMs = validatedAtMs;
|
|
}
|
|
}
|
|
}
|
|
|
|
async function ensureSubscriptionAuthorizationFresh(
|
|
state: ConnectionState,
|
|
subscription: ChannelSubscriptionState,
|
|
dependencies: ChatGatewayDependencies
|
|
): Promise<boolean> {
|
|
if (state.closing || state.closed || subscription.closed || state.subscriptions.get(subscription.chatId) !== subscription) return false;
|
|
const ageMs = dependencies.now().getTime() - subscription.authorizationValidatedAtMs;
|
|
if (ageMs >= 0 && ageMs < CHAT_GATEWAY_AUTH_CACHE_TTL_MS) return true;
|
|
await refreshAuthorizationCoalesced(state, dependencies);
|
|
return !state.closing && !state.closed && !subscription.closed && state.subscriptions.get(subscription.chatId) === subscription;
|
|
}
|
|
|
|
function touchPresenceForChannel(state: ConnectionState, chatId: string, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
const identity = requireIdentity(state);
|
|
return queueLeaseOperation(state, async () => {
|
|
if (state.closed || !state.subscriptions.has(chatId)) return;
|
|
await dependencies.roomHub.touchPresence({
|
|
tenantId: identity.tenantId,
|
|
chatId,
|
|
userId: identity.userId,
|
|
connectionId: state.id,
|
|
presence: state.presence,
|
|
occurredAt: dependencies.now(),
|
|
});
|
|
});
|
|
}
|
|
|
|
async function startTypingForChannel(state: ConnectionState, chatId: string, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
const identity = requireIdentity(state);
|
|
const previous = state.typingLeases.get(chatId);
|
|
if (previous) clearTimeout(previous.timer);
|
|
const generation = (previous?.generation ?? 0) + 1;
|
|
const timer = setTimeout(() => {
|
|
const current = state.typingLeases.get(chatId);
|
|
if (!current || current.generation !== generation) return;
|
|
state.typingLeases.delete(chatId);
|
|
void releaseTypingLease(state, chatId, identity, dependencies).catch(() => {
|
|
dependencies.logger.warn({ component: "chat-gateway", connectionId: state.id }, "Chat typing lease expiry failed");
|
|
});
|
|
}, dependencies.typingLeaseTtlMs);
|
|
timer.unref();
|
|
state.typingLeases.set(chatId, { generation, timer });
|
|
await queueLeaseOperation(state, async () => {
|
|
if (state.closed || !state.subscriptions.has(chatId)) return;
|
|
await dependencies.roomHub.touchTyping({
|
|
tenantId: identity.tenantId,
|
|
chatId,
|
|
userId: identity.userId,
|
|
connectionId: state.id,
|
|
occurredAt: dependencies.now(),
|
|
});
|
|
});
|
|
}
|
|
|
|
async function stopTypingForChannel(state: ConnectionState, chatId: string, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
const identity = requireIdentity(state);
|
|
const current = state.typingLeases.get(chatId);
|
|
if (current) clearTimeout(current.timer);
|
|
state.typingLeases.delete(chatId);
|
|
await releaseTypingLease(state, chatId, identity, dependencies);
|
|
}
|
|
|
|
function releaseTypingLease(state: ConnectionState, chatId: string, identity: ChatGatewayTokenIdentity, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
return queueLeaseOperation(state, async () => {
|
|
await dependencies.roomHub.releaseTyping({
|
|
tenantId: identity.tenantId,
|
|
chatId,
|
|
userId: identity.userId,
|
|
connectionId: state.id,
|
|
occurredAt: dependencies.now(),
|
|
});
|
|
});
|
|
}
|
|
|
|
function releasePresenceLease(
|
|
state: ConnectionState,
|
|
chatId: string,
|
|
identity: ChatGatewayTokenIdentity,
|
|
dependencies: ChatGatewayDependencies
|
|
): Promise<void> {
|
|
return queueLeaseOperation(state, async () => {
|
|
await dependencies.roomHub.releasePresence({
|
|
tenantId: identity.tenantId,
|
|
chatId,
|
|
userId: identity.userId,
|
|
connectionId: state.id,
|
|
occurredAt: dependencies.now(),
|
|
});
|
|
});
|
|
}
|
|
|
|
function queueLeaseOperation(state: ConnectionState, operation: () => Promise<void>): Promise<void> {
|
|
const current = state.leaseOperations.then(operation);
|
|
state.leaseOperations = current.catch(() => undefined);
|
|
return current;
|
|
}
|
|
|
|
async function removeSubscription(
|
|
state: ConnectionState,
|
|
chatId: string,
|
|
dependencies: ChatGatewayDependencies,
|
|
options: { releaseLeases?: boolean } = {}
|
|
): Promise<void> {
|
|
const subscription = state.subscriptions.get(chatId);
|
|
if (!subscription) return;
|
|
state.subscriptions.delete(chatId);
|
|
subscription.closed = true;
|
|
subscription.persistentSignalPending = false;
|
|
subscription.pendingEphemeral.clear();
|
|
if (options.releaseLeases !== false && state.identity) {
|
|
const typing = state.typingLeases.get(chatId);
|
|
if (typing) clearTimeout(typing.timer);
|
|
state.typingLeases.delete(chatId);
|
|
await Promise.allSettled([
|
|
releaseTypingLease(state, chatId, state.identity, dependencies),
|
|
releasePresenceLease(state, chatId, state.identity, dependencies),
|
|
]);
|
|
}
|
|
await subscription.roomSubscription.unsubscribe();
|
|
}
|
|
|
|
async function closeConnection(state: ConnectionState, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
if (state.closed) return;
|
|
state.closing = true;
|
|
state.closed = true;
|
|
clearTimeout(state.authTimer);
|
|
if (state.tokenExpiryTimer) clearTimeout(state.tokenExpiryTimer);
|
|
const identity = state.identity;
|
|
const admitted = state.admitted;
|
|
state.admitted = false;
|
|
const chatIds = [...state.subscriptions.keys()];
|
|
for (const typing of state.typingLeases.values()) clearTimeout(typing.timer);
|
|
state.typingLeases.clear();
|
|
await Promise.allSettled(chatIds.map((chatId) => removeSubscription(state, chatId, dependencies, { releaseLeases: false })));
|
|
if (identity) {
|
|
await Promise.allSettled([
|
|
...chatIds.flatMap((chatId) => [releaseTypingLease(state, chatId, identity, dependencies), releasePresenceLease(state, chatId, identity, dependencies)]),
|
|
...(admitted ? [releaseAdmissionLease(state, connectionIdentity(state, identity), dependencies)] : []),
|
|
]);
|
|
}
|
|
}
|
|
|
|
async function releaseAdmissionLease(state: ConnectionState, identity: ChatConnectionIdentity, dependencies: ChatGatewayDependencies): Promise<void> {
|
|
try {
|
|
await dependencies.admission.release(identity);
|
|
} catch (error) {
|
|
logAdmissionFailure(state, error, dependencies, "release");
|
|
}
|
|
}
|
|
|
|
function logAdmissionFailure(
|
|
state: ConnectionState,
|
|
error: unknown,
|
|
dependencies: ChatGatewayDependencies,
|
|
operation: "acquire" | "refresh" | "release"
|
|
): void {
|
|
dependencies.logger.warn(
|
|
{
|
|
component: "chat-gateway",
|
|
connectionId: state.id,
|
|
operation,
|
|
errorName: error instanceof Error ? error.name : "Error",
|
|
},
|
|
"Chat connection admission failed"
|
|
);
|
|
}
|
|
|
|
function repositoryContext(identity: ChatGatewayTokenIdentity): GatewayRepositoryContext {
|
|
return { tenantId: identity.tenantId, userId: identity.userId, sessionId: identity.sessionId };
|
|
}
|
|
|
|
function connectionIdentity(state: ConnectionState, identity: ChatGatewayTokenIdentity): ChatConnectionIdentity {
|
|
return {
|
|
tenantId: identity.tenantId,
|
|
userId: identity.userId,
|
|
sessionId: identity.sessionId,
|
|
connectionId: state.id,
|
|
};
|
|
}
|
|
|
|
function requireIdentity(state: ConnectionState): ChatGatewayTokenIdentity {
|
|
if (!state.identity) throw new CatalogError(foundationErrors.notFound);
|
|
return state.identity;
|
|
}
|
|
|
|
function requireSubscription(state: ConnectionState, chatId: string): ChannelSubscriptionState {
|
|
const subscription = state.subscriptions.get(chatId);
|
|
if (!subscription || subscription.closed || subscription.catchingUp) {
|
|
throw new CatalogError(foundationErrors.notFound, { chat: "not-subscribed" });
|
|
}
|
|
return subscription;
|
|
}
|
|
|
|
function sendError(state: ConnectionState, i18nKey: string, retryable: boolean, correlation: { requestId?: string; clientMessageId?: string } = {}): void {
|
|
sendFrame(
|
|
state,
|
|
{
|
|
type: CHAT_SERVER_FRAME_TYPES.error,
|
|
i18nKey,
|
|
retryable,
|
|
...correlation,
|
|
},
|
|
"durable"
|
|
);
|
|
}
|
|
|
|
function sendFrame(state: ConnectionState, frame: ChatServerFrame, priority: "durable" | "ephemeral"): boolean {
|
|
if (state.closing || state.closed) return false;
|
|
if (state.socket.readyState !== WebSocket.OPEN) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return false;
|
|
}
|
|
if (state.socket.bufferedAmount > CHAT_GATEWAY_MAX_BUFFERED_BYTES) {
|
|
if (priority === "ephemeral") return false;
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return false;
|
|
}
|
|
try {
|
|
state.socket.send(serializeChatServerFrame(frame));
|
|
return true;
|
|
} catch {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function handleOperationError(
|
|
state: ConnectionState,
|
|
error: unknown,
|
|
dependencies: ChatGatewayDependencies,
|
|
frame?: Exclude<ChatClientFrame, { type: "authenticate" }>
|
|
): void {
|
|
if (state.closing || state.closed) return;
|
|
const correlation =
|
|
frame?.type === CHAT_CLIENT_FRAME_TYPES.sendMessage
|
|
? { requestId: frame.requestId, clientMessageId: frame.clientMessageId }
|
|
: frame?.type === CHAT_CLIENT_FRAME_TYPES.markRead
|
|
? { requestId: frame.requestId }
|
|
: {};
|
|
if (error instanceof CatalogError) {
|
|
sendError(state, error.i18nKey, error.status >= 500, correlation);
|
|
if (error.status === 404) beginClose(state, CLOSE_UNAUTHORIZED);
|
|
return;
|
|
}
|
|
dependencies.logger.error(
|
|
{ component: "chat-gateway", connectionId: state.id, errorName: error instanceof Error ? error.name : "Error" },
|
|
"Chat gateway operation failed"
|
|
);
|
|
sendError(state, chatErrors.realtimeUnavailable.i18nKey, true, correlation);
|
|
}
|
|
|
|
function rawDataToUtf8(raw: RawData): string {
|
|
if (typeof raw === "string") return raw;
|
|
if (Buffer.isBuffer(raw)) return raw.toString("utf8");
|
|
if (Array.isArray(raw)) return Buffer.concat(raw).toString("utf8");
|
|
return Buffer.from(raw).toString("utf8");
|
|
}
|
|
|
|
function closeSocket(socket: WebSocket, code: number): void {
|
|
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
socket.close(code);
|
|
}
|
|
}
|
|
|
|
function beginClose(state: ConnectionState, code: number): void {
|
|
if (state.closing || state.closed) return;
|
|
state.closing = true;
|
|
closeSocket(state.socket, code);
|
|
}
|
|
|
|
function terminateConnection(state: ConnectionState): void {
|
|
if (state.closed) return;
|
|
state.closing = true;
|
|
state.socket.terminate();
|
|
}
|
|
|
|
function gapPageLimit(remainingEvents: number): number {
|
|
return Math.min(CHAT_WIRE_MAX_GAP_EVENTS, Math.max(1, remainingEvents + 1));
|
|
}
|
|
|
|
function consumeInitialCatchUpBudget(state: ConnectionState, eventCount: number): boolean {
|
|
if (eventCount > state.remainingInitialCatchUpEvents) {
|
|
beginClose(state, CLOSE_RETRY_LATER);
|
|
return false;
|
|
}
|
|
state.remainingInitialCatchUpEvents -= eventCount;
|
|
return true;
|
|
}
|