Files

319 lines
14 KiB
TypeScript

#!/usr/bin/env tsx
/* eslint-disable no-console */
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import path from "node:path";
import { i18nNamespaces, type SupportedLocale } from "../packages/i18n/src/index.ts";
type TargetLocale = Exclude<SupportedLocale, "de" | "en">;
type JsonValue = boolean | null | number | string | JsonValue[] | { [key: string]: JsonValue };
type JsonObject = { [key: string]: JsonValue };
type MaskedText = { text: string; tokens: string[] };
const targetLocales = new Set<TargetLocale>(["pl", "ru", "uk", "hu"]);
const locale = process.argv[2] as TargetLocale | undefined;
if (!locale || !targetLocales.has(locale)) {
console.error("Usage: pnpm exec tsx scripts/i18n-translate-catalog.ts <pl|ru|uk|hu>");
process.exit(1);
}
const root = process.cwd();
const localesRoot = path.join(root, "apps/web/public/locales");
const sourceRoot = path.join(localesRoot, "de");
const formalSourceRoot = path.join(localesRoot, "de-formal");
const targetRoot = path.join(localesRoot, locale);
const formalTargetRoot = path.join(localesRoot, `${locale}-formal`);
const cacheRoot = path.join(root, ".cache/i18n-translations");
const cachePath = path.join(cacheRoot, `${locale}.json`);
const reviewPath = path.join(root, `docs/contributing/i18n/reviews/${locale}.json`);
mkdirSync(targetRoot, { recursive: true });
mkdirSync(formalTargetRoot, { recursive: true });
mkdirSync(cacheRoot, { recursive: true });
const cache = readJson<Record<string, string>>(cachePath, {});
for (const [sourceValue, translatedValue] of Object.entries(cache)) {
if (!hasSameProtectedTokens(sourceValue, translatedValue)) delete cache[sourceValue];
}
let translatedCount = 0;
let cacheHitCount = 0;
for (const namespace of i18nNamespaces) {
const sourcePath = path.join(sourceRoot, `${namespace}.json`);
const targetPath = path.join(targetRoot, `${namespace}.json`);
const source = readJson<JsonObject>(sourcePath);
const translated = await translateObject(source);
applyPluralOverrides(namespace, translated);
writeJson(targetPath, translated);
const formalSourcePath = path.join(formalSourceRoot, `${namespace}.json`);
const formalTargetPath = path.join(formalTargetRoot, `${namespace}.json`);
const formalSource = existsSync(formalSourcePath) ? readJson<JsonObject>(formalSourcePath) : {};
writeJson(formalTargetPath, pruneMatchingValues(await translateObject(formalSource), translated) as JsonObject);
writeJson(cachePath, cache);
console.log(`[${locale}] ${namespace}: translated=${translatedCount} cacheHits=${cacheHitCount}`);
}
const review = readJson<Record<string, unknown>>(reviewPath, {});
const sourceHashes = Object.fromEntries(
i18nNamespaces.map((namespace) => {
const bytes = readFileSync(path.join(sourceRoot, `${namespace}.json`));
return [namespace, createHash("sha256").update(bytes).digest("hex")];
})
);
writeJson(reviewPath, {
...review,
locale,
sourceLocale: "de",
sourceHashes,
machineTranslation: {
status: "complete",
provider: "Google Translate static build tool",
completedAt: new Date().toISOString(),
},
automatedReview: { status: "pending", checkedAt: null },
});
console.log(`[${locale}] complete: ${translatedCount} new translations, ${cacheHitCount} cache hits`);
async function translateObject(source: JsonObject): Promise<JsonObject> {
const leaves: Array<{ path: string[]; value: string }> = [];
collectStringLeaves(source, [], leaves);
const output = structuredClone(source);
const missing = leaves.filter(({ value }) => shouldTranslate(value) && cache[value] === undefined);
for (const batch of createBatches(missing, 3_000)) {
const translated = await translateBatch(batch);
for (const [sourceValue, targetValue] of translated) {
cache[sourceValue] = targetValue.normalize("NFC");
translatedCount += 1;
}
writeJson(cachePath, cache);
}
for (const leaf of leaves) {
const translated = shouldTranslate(leaf.value) ? cache[leaf.value] : leaf.value;
if (translated === undefined) throw new Error(`Missing cached translation for ${leaf.path.join(".")}`);
if (cache[leaf.value] !== undefined && !missing.some((candidate) => candidate.value === leaf.value)) cacheHitCount += 1;
setAtPath(output, leaf.path, translated.normalize("NFC"));
}
return output;
}
function collectStringLeaves(value: JsonValue, currentPath: string[], output: Array<{ path: string[]; value: string }>): void {
if (typeof value === "string") {
output.push({ path: currentPath, value });
return;
}
if (!value || typeof value !== "object" || Array.isArray(value)) return;
for (const [key, child] of Object.entries(value)) collectStringLeaves(child, [...currentPath, key], output);
}
function setAtPath(rootValue: JsonObject, keyPath: string[], value: string): void {
let current: JsonObject = rootValue;
for (const key of keyPath.slice(0, -1)) current = current[key] as JsonObject;
current[keyPath.at(-1)!] = value;
}
function pruneMatchingValues(overrides: JsonValue, base: JsonValue | undefined): JsonValue | undefined {
if (JSON.stringify(overrides) === JSON.stringify(base)) return undefined;
if (!overrides || typeof overrides !== "object" || Array.isArray(overrides)) return overrides;
const pruned = Object.fromEntries(
Object.entries(overrides)
.map(([key, value]) => [key, pruneMatchingValues(value, base && typeof base === "object" && !Array.isArray(base) ? base[key] : undefined)] as const)
.filter((entry): entry is readonly [string, JsonValue] => entry[1] !== undefined)
);
return Object.keys(pruned).length > 0 ? pruned : undefined;
}
function shouldTranslate(value: string): boolean {
const trimmed = value.trim();
if (!/\p{L}/u.test(trimmed)) return false;
if (/^(?:https?:\/\/|mailto:|tel:)/iu.test(trimmed)) return false;
if (/^[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}$/u.test(trimmed)) return false;
if (/^[A-Z0-9_.:/+-]{2,}$/u.test(trimmed)) return false;
if (/^\{[^{}]+,\s*plural,/u.test(trimmed)) return false;
return true;
}
function createBatches(entries: Array<{ value: string }>, maxCharacters: number): string[][] {
const unique = [...new Set(entries.map((entry) => entry.value))];
const batches: string[][] = [];
let current: string[] = [];
let size = 0;
for (const value of unique) {
const maskedLength = maskTokens(value).text.length + 20;
if (current.length > 0 && size + maskedLength > maxCharacters) {
batches.push(current);
current = [];
size = 0;
}
current.push(value);
size += maskedLength;
}
if (current.length > 0) batches.push(current);
return batches;
}
async function translateBatch(values: string[]): Promise<Map<string, string>> {
const masked = values.map(maskTokens);
const protectedValues = values.filter((_value, index) => masked[index]!.tokens.length > 0);
if (protectedValues.length > 0) {
const result = new Map<string, string>();
for (const value of protectedValues) result.set(value, await translatePreservingTokens(value));
const plainValues = values.filter((_value, index) => masked[index]!.tokens.length === 0);
if (plainValues.length > 0) {
for (const entry of await translateBatch(plainValues)) result.set(...entry);
}
return result;
}
if (values.length === 1) {
return new Map([[values[0]!, restoreTokens(await requestTranslation(masked[0]!.text), masked[0]!.tokens)]]);
}
const requestText = masked.map((entry, index) => `HG${String(index).padStart(6, "0")}|||${entry.text}`).join("\n");
let segments: string[] | undefined;
for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
segments = parseBatchResponse(await requestTranslation(requestText), values.length);
break;
} catch {
if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, attempt * 500));
}
}
if (!segments) {
const midpoint = Math.ceil(values.length / 2);
return new Map([...(await translateBatch(values.slice(0, midpoint))), ...(await translateBatch(values.slice(midpoint)))]);
}
const result = new Map<string, string>();
for (let index = 0; index < values.length; index += 1) {
result.set(values[index]!, restoreTokens(segments[index]!, masked[index]!.tokens));
}
return result;
}
function maskTokens(value: string): MaskedText {
const tokens: string[] = [];
return {
text: value.replace(protectedTokenPattern(), (token) => {
const index = tokens.push(token) - 1;
return `HGTOKEN${String(index).padStart(4, "0")}TOKENHG`;
}),
tokens,
};
}
async function translatePreservingTokens(value: string): Promise<string> {
const matches = [...value.matchAll(protectedTokenPattern())];
let cursor = 0;
let translated = "";
for (const match of matches) {
const index = match.index ?? cursor;
translated += await translateFragment(value.slice(cursor, index));
translated += match[0];
cursor = index + match[0].length;
}
translated += await translateFragment(value.slice(cursor));
return translated;
}
async function translateFragment(value: string): Promise<string> {
if (!/\p{L}/u.test(value)) return value;
const leadingWhitespace = value.match(/^\s*/u)?.[0] ?? "";
const trailingWhitespace = value.match(/\s*$/u)?.[0] ?? "";
const core = value.slice(leadingWhitespace.length, value.length - trailingWhitespace.length);
if (!core) return value;
return `${leadingWhitespace}${(await requestTranslation(core)).trim()}${trailingWhitespace}`;
}
function protectedTokenPattern(): RegExp {
return /https?:\/\/[^\s<>"']+|mailto:[^\s<>"']+|tel:[^\s<>"']+|\{\{[^{}]+\}\}|<\/?[A-Za-z][^>]*>|`[^`]+`|\{[A-Za-z_][\w.]*\}|[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}|\r?\n/gu;
}
function restoreTokens(value: string, tokens: string[]): string {
let restored = value.trim();
for (let index = 0; index < tokens.length; index += 1) {
const marker = new RegExp(`HGTOKEN\\s*${String(index).padStart(4, "0")}\\s*TOKENHG`, "giu");
restored = restored.replace(marker, tokens[index]!);
}
if (/HGTOKEN\s*\d{4}\s*TOKENHG/iu.test(restored)) throw new Error(`Unrestored token in translated value: ${restored.slice(0, 160)}`);
return restored;
}
function hasSameProtectedTokens(sourceValue: string, translatedValue: string): boolean {
return JSON.stringify(maskTokens(sourceValue).tokens) === JSON.stringify(maskTokens(translatedValue).tokens);
}
function parseBatchResponse(value: string, expected: number): string[] {
const markers = [...value.matchAll(/HG\s*(\d{6})\s*\|\|\|\s*/gu)];
if (markers.length !== expected) {
throw new Error(`Translation batch marker mismatch: expected ${expected}, received ${markers.length}`);
}
const result = Array<string>(expected);
for (let markerIndex = 0; markerIndex < markers.length; markerIndex += 1) {
const marker = markers[markerIndex]!;
const index = Number(marker[1]);
const start = (marker.index ?? 0) + marker[0].length;
const end = markers[markerIndex + 1]?.index ?? value.length;
result[index] = value.slice(start, end).trim();
}
if (result.some((entry) => entry === undefined)) throw new Error("Translation batch returned non-contiguous markers.");
return result;
}
async function requestTranslation(text: string): Promise<string> {
const query = new URLSearchParams({ client: "gtx", sl: "de", tl: locale!, dt: "t", q: text });
const url = `https://translate.googleapis.com/translate_a/single?${query.toString()}`;
let lastError: unknown;
for (let attempt = 1; attempt <= 5; attempt += 1) {
try {
const response = await fetch(url, { headers: { "user-agent": "HiveGrid-static-i18n-build/1.0" }, signal: AbortSignal.timeout(30_000) });
if (!response.ok) throw new Error(`Translation request failed with ${response.status}`);
const payload = (await response.json()) as unknown[][];
const fragments = payload[0];
if (!Array.isArray(fragments)) throw new Error("Translation response has no fragments.");
return fragments.map((fragment) => (Array.isArray(fragment) ? String(fragment[0] ?? "") : "")).join("");
} catch (error) {
lastError = error;
await new Promise((resolve) => setTimeout(resolve, attempt * 750));
}
}
throw lastError;
}
function applyPluralOverrides(namespace: string, target: JsonObject): void {
if (namespace !== "common") return;
const overrides: Record<TargetLocale, { itemsSelected: string; memberCount: string }> = {
pl: {
itemsSelected: "{count, plural, =0 {Nic nie wybrano} one {Wybrano # element} few {Wybrano # elementy} many {Wybrano # elementów} other {Wybrano # elementu}}",
memberCount: "{count, plural, one {# członek} few {# członków} many {# członków} other {# członka}}",
},
ru: {
itemsSelected: "{count, plural, =0 {Ничего не выбрано} one {Выбран # элемент} few {Выбрано # элемента} many {Выбрано # элементов} other {Выбрано # элемента}}",
memberCount: "{count, plural, one {# участник} few {# участника} many {# участников} other {# участника}}",
},
uk: {
itemsSelected: "{count, plural, =0 {Нічого не вибрано} one {Вибрано # елемент} few {Вибрано # елементи} many {Вибрано # елементів} other {Вибрано # елемента}}",
memberCount: "{count, plural, one {# учасник} few {# учасники} many {# учасників} other {# учасника}}",
},
hu: {
itemsSelected: "{count, plural, =0 {Nincs kijelölt elem} one {# elem kijelölve} other {# elem kijelölve}}",
memberCount: "{count, plural, one {# tag} other {# tag}}",
},
};
target.itemsSelected = overrides[locale!].itemsSelected;
target.memberCount = overrides[locale!].memberCount;
}
function readJson<T>(filePath: string, fallback?: T): T {
if (!existsSync(filePath) && fallback !== undefined) return fallback;
return JSON.parse(readFileSync(filePath, "utf8")) as T;
}
function writeJson(filePath: string, value: unknown): void {
mkdirSync(path.dirname(filePath), { recursive: true });
writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
}