#!/usr/bin/env tsx /* eslint-disable no-console */ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { i18nNamespaces, type SupportedLocale } from "../packages/i18n/src/index.ts"; type TargetLocale = Exclude; type JsonValue = boolean | null | number | string | JsonValue[] | { [key: string]: JsonValue }; type JsonObject = { [key: string]: JsonValue }; type MaskedText = { text: string; tokens: string[] }; const targetLocales = new Set(["pl", "ru", "uk", "hu"]); const locale = process.argv[2] as TargetLocale | undefined; if (!locale || !targetLocales.has(locale)) { console.error("Usage: pnpm exec tsx scripts/i18n-translate-catalog.ts "); process.exit(1); } const root = process.cwd(); const localesRoot = path.join(root, "apps/web/public/locales"); const sourceRoot = path.join(localesRoot, "de"); const formalSourceRoot = path.join(localesRoot, "de-formal"); const targetRoot = path.join(localesRoot, locale); const formalTargetRoot = path.join(localesRoot, `${locale}-formal`); const cacheRoot = path.join(root, ".cache/i18n-translations"); const cachePath = path.join(cacheRoot, `${locale}.json`); const reviewPath = path.join(root, `docs/contributing/i18n/reviews/${locale}.json`); mkdirSync(targetRoot, { recursive: true }); mkdirSync(formalTargetRoot, { recursive: true }); mkdirSync(cacheRoot, { recursive: true }); const cache = readJson>(cachePath, {}); for (const [sourceValue, translatedValue] of Object.entries(cache)) { if (!hasSameProtectedTokens(sourceValue, translatedValue)) delete cache[sourceValue]; } let translatedCount = 0; let cacheHitCount = 0; for (const namespace of i18nNamespaces) { const sourcePath = path.join(sourceRoot, `${namespace}.json`); const targetPath = path.join(targetRoot, `${namespace}.json`); const source = readJson(sourcePath); const translated = await translateObject(source); applyPluralOverrides(namespace, translated); writeJson(targetPath, translated); const formalSourcePath = path.join(formalSourceRoot, `${namespace}.json`); const formalTargetPath = path.join(formalTargetRoot, `${namespace}.json`); const formalSource = existsSync(formalSourcePath) ? readJson(formalSourcePath) : {}; writeJson(formalTargetPath, pruneMatchingValues(await translateObject(formalSource), translated) as JsonObject); writeJson(cachePath, cache); console.log(`[${locale}] ${namespace}: translated=${translatedCount} cacheHits=${cacheHitCount}`); } const review = readJson>(reviewPath, {}); const sourceHashes = Object.fromEntries( i18nNamespaces.map((namespace) => { const bytes = readFileSync(path.join(sourceRoot, `${namespace}.json`)); return [namespace, createHash("sha256").update(bytes).digest("hex")]; }) ); writeJson(reviewPath, { ...review, locale, sourceLocale: "de", sourceHashes, machineTranslation: { status: "complete", provider: "Google Translate static build tool", completedAt: new Date().toISOString(), }, automatedReview: { status: "pending", checkedAt: null }, }); console.log(`[${locale}] complete: ${translatedCount} new translations, ${cacheHitCount} cache hits`); async function translateObject(source: JsonObject): Promise { const leaves: Array<{ path: string[]; value: string }> = []; collectStringLeaves(source, [], leaves); const output = structuredClone(source); const missing = leaves.filter(({ value }) => shouldTranslate(value) && cache[value] === undefined); for (const batch of createBatches(missing, 3_000)) { const translated = await translateBatch(batch); for (const [sourceValue, targetValue] of translated) { cache[sourceValue] = targetValue.normalize("NFC"); translatedCount += 1; } writeJson(cachePath, cache); } for (const leaf of leaves) { const translated = shouldTranslate(leaf.value) ? cache[leaf.value] : leaf.value; if (translated === undefined) throw new Error(`Missing cached translation for ${leaf.path.join(".")}`); if (cache[leaf.value] !== undefined && !missing.some((candidate) => candidate.value === leaf.value)) cacheHitCount += 1; setAtPath(output, leaf.path, translated.normalize("NFC")); } return output; } function collectStringLeaves(value: JsonValue, currentPath: string[], output: Array<{ path: string[]; value: string }>): void { if (typeof value === "string") { output.push({ path: currentPath, value }); return; } if (!value || typeof value !== "object" || Array.isArray(value)) return; for (const [key, child] of Object.entries(value)) collectStringLeaves(child, [...currentPath, key], output); } function setAtPath(rootValue: JsonObject, keyPath: string[], value: string): void { let current: JsonObject = rootValue; for (const key of keyPath.slice(0, -1)) current = current[key] as JsonObject; current[keyPath.at(-1)!] = value; } function pruneMatchingValues(overrides: JsonValue, base: JsonValue | undefined): JsonValue | undefined { if (JSON.stringify(overrides) === JSON.stringify(base)) return undefined; if (!overrides || typeof overrides !== "object" || Array.isArray(overrides)) return overrides; const pruned = Object.fromEntries( Object.entries(overrides) .map(([key, value]) => [key, pruneMatchingValues(value, base && typeof base === "object" && !Array.isArray(base) ? base[key] : undefined)] as const) .filter((entry): entry is readonly [string, JsonValue] => entry[1] !== undefined) ); return Object.keys(pruned).length > 0 ? pruned : undefined; } function shouldTranslate(value: string): boolean { const trimmed = value.trim(); if (!/\p{L}/u.test(trimmed)) return false; if (/^(?:https?:\/\/|mailto:|tel:)/iu.test(trimmed)) return false; if (/^[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}$/u.test(trimmed)) return false; if (/^[A-Z0-9_.:/+-]{2,}$/u.test(trimmed)) return false; if (/^\{[^{}]+,\s*plural,/u.test(trimmed)) return false; return true; } function createBatches(entries: Array<{ value: string }>, maxCharacters: number): string[][] { const unique = [...new Set(entries.map((entry) => entry.value))]; const batches: string[][] = []; let current: string[] = []; let size = 0; for (const value of unique) { const maskedLength = maskTokens(value).text.length + 20; if (current.length > 0 && size + maskedLength > maxCharacters) { batches.push(current); current = []; size = 0; } current.push(value); size += maskedLength; } if (current.length > 0) batches.push(current); return batches; } async function translateBatch(values: string[]): Promise> { const masked = values.map(maskTokens); const protectedValues = values.filter((_value, index) => masked[index]!.tokens.length > 0); if (protectedValues.length > 0) { const result = new Map(); for (const value of protectedValues) result.set(value, await translatePreservingTokens(value)); const plainValues = values.filter((_value, index) => masked[index]!.tokens.length === 0); if (plainValues.length > 0) { for (const entry of await translateBatch(plainValues)) result.set(...entry); } return result; } if (values.length === 1) { return new Map([[values[0]!, restoreTokens(await requestTranslation(masked[0]!.text), masked[0]!.tokens)]]); } const requestText = masked.map((entry, index) => `HG${String(index).padStart(6, "0")}|||${entry.text}`).join("\n"); let segments: string[] | undefined; for (let attempt = 1; attempt <= 3; attempt += 1) { try { segments = parseBatchResponse(await requestTranslation(requestText), values.length); break; } catch { if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, attempt * 500)); } } if (!segments) { const midpoint = Math.ceil(values.length / 2); return new Map([...(await translateBatch(values.slice(0, midpoint))), ...(await translateBatch(values.slice(midpoint)))]); } const result = new Map(); for (let index = 0; index < values.length; index += 1) { result.set(values[index]!, restoreTokens(segments[index]!, masked[index]!.tokens)); } return result; } function maskTokens(value: string): MaskedText { const tokens: string[] = []; return { text: value.replace(protectedTokenPattern(), (token) => { const index = tokens.push(token) - 1; return `HGTOKEN${String(index).padStart(4, "0")}TOKENHG`; }), tokens, }; } async function translatePreservingTokens(value: string): Promise { const matches = [...value.matchAll(protectedTokenPattern())]; let cursor = 0; let translated = ""; for (const match of matches) { const index = match.index ?? cursor; translated += await translateFragment(value.slice(cursor, index)); translated += match[0]; cursor = index + match[0].length; } translated += await translateFragment(value.slice(cursor)); return translated; } async function translateFragment(value: string): Promise { if (!/\p{L}/u.test(value)) return value; const leadingWhitespace = value.match(/^\s*/u)?.[0] ?? ""; const trailingWhitespace = value.match(/\s*$/u)?.[0] ?? ""; const core = value.slice(leadingWhitespace.length, value.length - trailingWhitespace.length); if (!core) return value; return `${leadingWhitespace}${(await requestTranslation(core)).trim()}${trailingWhitespace}`; } function protectedTokenPattern(): RegExp { return /https?:\/\/[^\s<>"']+|mailto:[^\s<>"']+|tel:[^\s<>"']+|\{\{[^{}]+\}\}|<\/?[A-Za-z][^>]*>|`[^`]+`|\{[A-Za-z_][\w.]*\}|[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}|\r?\n/gu; } function restoreTokens(value: string, tokens: string[]): string { let restored = value.trim(); for (let index = 0; index < tokens.length; index += 1) { const marker = new RegExp(`HGTOKEN\\s*${String(index).padStart(4, "0")}\\s*TOKENHG`, "giu"); restored = restored.replace(marker, tokens[index]!); } if (/HGTOKEN\s*\d{4}\s*TOKENHG/iu.test(restored)) throw new Error(`Unrestored token in translated value: ${restored.slice(0, 160)}`); return restored; } function hasSameProtectedTokens(sourceValue: string, translatedValue: string): boolean { return JSON.stringify(maskTokens(sourceValue).tokens) === JSON.stringify(maskTokens(translatedValue).tokens); } function parseBatchResponse(value: string, expected: number): string[] { const markers = [...value.matchAll(/HG\s*(\d{6})\s*\|\|\|\s*/gu)]; if (markers.length !== expected) { throw new Error(`Translation batch marker mismatch: expected ${expected}, received ${markers.length}`); } const result = Array(expected); for (let markerIndex = 0; markerIndex < markers.length; markerIndex += 1) { const marker = markers[markerIndex]!; const index = Number(marker[1]); const start = (marker.index ?? 0) + marker[0].length; const end = markers[markerIndex + 1]?.index ?? value.length; result[index] = value.slice(start, end).trim(); } if (result.some((entry) => entry === undefined)) throw new Error("Translation batch returned non-contiguous markers."); return result; } async function requestTranslation(text: string): Promise { const query = new URLSearchParams({ client: "gtx", sl: "de", tl: locale!, dt: "t", q: text }); const url = `https://translate.googleapis.com/translate_a/single?${query.toString()}`; let lastError: unknown; for (let attempt = 1; attempt <= 5; attempt += 1) { try { const response = await fetch(url, { headers: { "user-agent": "HiveGrid-static-i18n-build/1.0" }, signal: AbortSignal.timeout(30_000) }); if (!response.ok) throw new Error(`Translation request failed with ${response.status}`); const payload = (await response.json()) as unknown[][]; const fragments = payload[0]; if (!Array.isArray(fragments)) throw new Error("Translation response has no fragments."); return fragments.map((fragment) => (Array.isArray(fragment) ? String(fragment[0] ?? "") : "")).join(""); } catch (error) { lastError = error; await new Promise((resolve) => setTimeout(resolve, attempt * 750)); } } throw lastError; } function applyPluralOverrides(namespace: string, target: JsonObject): void { if (namespace !== "common") return; const overrides: Record = { pl: { itemsSelected: "{count, plural, =0 {Nic nie wybrano} one {Wybrano # element} few {Wybrano # elementy} many {Wybrano # elementów} other {Wybrano # elementu}}", memberCount: "{count, plural, one {# członek} few {# członków} many {# członków} other {# członka}}", }, ru: { itemsSelected: "{count, plural, =0 {Ничего не выбрано} one {Выбран # элемент} few {Выбрано # элемента} many {Выбрано # элементов} other {Выбрано # элемента}}", memberCount: "{count, plural, one {# участник} few {# участника} many {# участников} other {# участника}}", }, uk: { itemsSelected: "{count, plural, =0 {Нічого не вибрано} one {Вибрано # елемент} few {Вибрано # елементи} many {Вибрано # елементів} other {Вибрано # елемента}}", memberCount: "{count, plural, one {# учасник} few {# учасники} many {# учасників} other {# учасника}}", }, hu: { itemsSelected: "{count, plural, =0 {Nincs kijelölt elem} one {# elem kijelölve} other {# elem kijelölve}}", memberCount: "{count, plural, one {# tag} other {# tag}}", }, }; target.itemsSelected = overrides[locale!].itemsSelected; target.memberCount = overrides[locale!].memberCount; } function readJson(filePath: string, fallback?: T): T { if (!existsSync(filePath) && fallback !== undefined) return fallback; return JSON.parse(readFileSync(filePath, "utf8")) as T; } function writeJson(filePath: string, value: unknown): void { mkdirSync(path.dirname(filePath), { recursive: true }); writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); }