319 lines
14 KiB
TypeScript
319 lines
14 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/* eslint-disable no-console */
|
|
import { createHash } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import path from "node:path";
|
|
|
|
import { i18nNamespaces, type SupportedLocale } from "../packages/i18n/src/index.ts";
|
|
|
|
type TargetLocale = Exclude<SupportedLocale, "de" | "en">;
|
|
type JsonValue = boolean | null | number | string | JsonValue[] | { [key: string]: JsonValue };
|
|
type JsonObject = { [key: string]: JsonValue };
|
|
type MaskedText = { text: string; tokens: string[] };
|
|
|
|
const targetLocales = new Set<TargetLocale>(["pl", "ru", "uk", "hu"]);
|
|
const locale = process.argv[2] as TargetLocale | undefined;
|
|
if (!locale || !targetLocales.has(locale)) {
|
|
console.error("Usage: pnpm exec tsx scripts/i18n-translate-catalog.ts <pl|ru|uk|hu>");
|
|
process.exit(1);
|
|
}
|
|
|
|
const root = process.cwd();
|
|
const localesRoot = path.join(root, "apps/web/public/locales");
|
|
const sourceRoot = path.join(localesRoot, "de");
|
|
const formalSourceRoot = path.join(localesRoot, "de-formal");
|
|
const targetRoot = path.join(localesRoot, locale);
|
|
const formalTargetRoot = path.join(localesRoot, `${locale}-formal`);
|
|
const cacheRoot = path.join(root, ".cache/i18n-translations");
|
|
const cachePath = path.join(cacheRoot, `${locale}.json`);
|
|
const reviewPath = path.join(root, `docs/contributing/i18n/reviews/${locale}.json`);
|
|
|
|
mkdirSync(targetRoot, { recursive: true });
|
|
mkdirSync(formalTargetRoot, { recursive: true });
|
|
mkdirSync(cacheRoot, { recursive: true });
|
|
|
|
const cache = readJson<Record<string, string>>(cachePath, {});
|
|
for (const [sourceValue, translatedValue] of Object.entries(cache)) {
|
|
if (!hasSameProtectedTokens(sourceValue, translatedValue)) delete cache[sourceValue];
|
|
}
|
|
let translatedCount = 0;
|
|
let cacheHitCount = 0;
|
|
|
|
for (const namespace of i18nNamespaces) {
|
|
const sourcePath = path.join(sourceRoot, `${namespace}.json`);
|
|
const targetPath = path.join(targetRoot, `${namespace}.json`);
|
|
const source = readJson<JsonObject>(sourcePath);
|
|
const translated = await translateObject(source);
|
|
applyPluralOverrides(namespace, translated);
|
|
writeJson(targetPath, translated);
|
|
|
|
const formalSourcePath = path.join(formalSourceRoot, `${namespace}.json`);
|
|
const formalTargetPath = path.join(formalTargetRoot, `${namespace}.json`);
|
|
const formalSource = existsSync(formalSourcePath) ? readJson<JsonObject>(formalSourcePath) : {};
|
|
writeJson(formalTargetPath, pruneMatchingValues(await translateObject(formalSource), translated) as JsonObject);
|
|
|
|
writeJson(cachePath, cache);
|
|
console.log(`[${locale}] ${namespace}: translated=${translatedCount} cacheHits=${cacheHitCount}`);
|
|
}
|
|
|
|
const review = readJson<Record<string, unknown>>(reviewPath, {});
|
|
const sourceHashes = Object.fromEntries(
|
|
i18nNamespaces.map((namespace) => {
|
|
const bytes = readFileSync(path.join(sourceRoot, `${namespace}.json`));
|
|
return [namespace, createHash("sha256").update(bytes).digest("hex")];
|
|
})
|
|
);
|
|
writeJson(reviewPath, {
|
|
...review,
|
|
locale,
|
|
sourceLocale: "de",
|
|
sourceHashes,
|
|
machineTranslation: {
|
|
status: "complete",
|
|
provider: "Google Translate static build tool",
|
|
completedAt: new Date().toISOString(),
|
|
},
|
|
automatedReview: { status: "pending", checkedAt: null },
|
|
});
|
|
|
|
console.log(`[${locale}] complete: ${translatedCount} new translations, ${cacheHitCount} cache hits`);
|
|
|
|
async function translateObject(source: JsonObject): Promise<JsonObject> {
|
|
const leaves: Array<{ path: string[]; value: string }> = [];
|
|
collectStringLeaves(source, [], leaves);
|
|
const output = structuredClone(source);
|
|
const missing = leaves.filter(({ value }) => shouldTranslate(value) && cache[value] === undefined);
|
|
|
|
for (const batch of createBatches(missing, 3_000)) {
|
|
const translated = await translateBatch(batch);
|
|
for (const [sourceValue, targetValue] of translated) {
|
|
cache[sourceValue] = targetValue.normalize("NFC");
|
|
translatedCount += 1;
|
|
}
|
|
writeJson(cachePath, cache);
|
|
}
|
|
|
|
for (const leaf of leaves) {
|
|
const translated = shouldTranslate(leaf.value) ? cache[leaf.value] : leaf.value;
|
|
if (translated === undefined) throw new Error(`Missing cached translation for ${leaf.path.join(".")}`);
|
|
if (cache[leaf.value] !== undefined && !missing.some((candidate) => candidate.value === leaf.value)) cacheHitCount += 1;
|
|
setAtPath(output, leaf.path, translated.normalize("NFC"));
|
|
}
|
|
return output;
|
|
}
|
|
|
|
function collectStringLeaves(value: JsonValue, currentPath: string[], output: Array<{ path: string[]; value: string }>): void {
|
|
if (typeof value === "string") {
|
|
output.push({ path: currentPath, value });
|
|
return;
|
|
}
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
|
for (const [key, child] of Object.entries(value)) collectStringLeaves(child, [...currentPath, key], output);
|
|
}
|
|
|
|
function setAtPath(rootValue: JsonObject, keyPath: string[], value: string): void {
|
|
let current: JsonObject = rootValue;
|
|
for (const key of keyPath.slice(0, -1)) current = current[key] as JsonObject;
|
|
current[keyPath.at(-1)!] = value;
|
|
}
|
|
|
|
function pruneMatchingValues(overrides: JsonValue, base: JsonValue | undefined): JsonValue | undefined {
|
|
if (JSON.stringify(overrides) === JSON.stringify(base)) return undefined;
|
|
if (!overrides || typeof overrides !== "object" || Array.isArray(overrides)) return overrides;
|
|
|
|
const pruned = Object.fromEntries(
|
|
Object.entries(overrides)
|
|
.map(([key, value]) => [key, pruneMatchingValues(value, base && typeof base === "object" && !Array.isArray(base) ? base[key] : undefined)] as const)
|
|
.filter((entry): entry is readonly [string, JsonValue] => entry[1] !== undefined)
|
|
);
|
|
return Object.keys(pruned).length > 0 ? pruned : undefined;
|
|
}
|
|
|
|
function shouldTranslate(value: string): boolean {
|
|
const trimmed = value.trim();
|
|
if (!/\p{L}/u.test(trimmed)) return false;
|
|
if (/^(?:https?:\/\/|mailto:|tel:)/iu.test(trimmed)) return false;
|
|
if (/^[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}$/u.test(trimmed)) return false;
|
|
if (/^[A-Z0-9_.:/+-]{2,}$/u.test(trimmed)) return false;
|
|
if (/^\{[^{}]+,\s*plural,/u.test(trimmed)) return false;
|
|
return true;
|
|
}
|
|
|
|
function createBatches(entries: Array<{ value: string }>, maxCharacters: number): string[][] {
|
|
const unique = [...new Set(entries.map((entry) => entry.value))];
|
|
const batches: string[][] = [];
|
|
let current: string[] = [];
|
|
let size = 0;
|
|
for (const value of unique) {
|
|
const maskedLength = maskTokens(value).text.length + 20;
|
|
if (current.length > 0 && size + maskedLength > maxCharacters) {
|
|
batches.push(current);
|
|
current = [];
|
|
size = 0;
|
|
}
|
|
current.push(value);
|
|
size += maskedLength;
|
|
}
|
|
if (current.length > 0) batches.push(current);
|
|
return batches;
|
|
}
|
|
|
|
async function translateBatch(values: string[]): Promise<Map<string, string>> {
|
|
const masked = values.map(maskTokens);
|
|
const protectedValues = values.filter((_value, index) => masked[index]!.tokens.length > 0);
|
|
if (protectedValues.length > 0) {
|
|
const result = new Map<string, string>();
|
|
for (const value of protectedValues) result.set(value, await translatePreservingTokens(value));
|
|
const plainValues = values.filter((_value, index) => masked[index]!.tokens.length === 0);
|
|
if (plainValues.length > 0) {
|
|
for (const entry of await translateBatch(plainValues)) result.set(...entry);
|
|
}
|
|
return result;
|
|
}
|
|
if (values.length === 1) {
|
|
return new Map([[values[0]!, restoreTokens(await requestTranslation(masked[0]!.text), masked[0]!.tokens)]]);
|
|
}
|
|
const requestText = masked.map((entry, index) => `HG${String(index).padStart(6, "0")}|||${entry.text}`).join("\n");
|
|
let segments: string[] | undefined;
|
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
try {
|
|
segments = parseBatchResponse(await requestTranslation(requestText), values.length);
|
|
break;
|
|
} catch {
|
|
if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, attempt * 500));
|
|
}
|
|
}
|
|
if (!segments) {
|
|
const midpoint = Math.ceil(values.length / 2);
|
|
return new Map([...(await translateBatch(values.slice(0, midpoint))), ...(await translateBatch(values.slice(midpoint)))]);
|
|
}
|
|
const result = new Map<string, string>();
|
|
for (let index = 0; index < values.length; index += 1) {
|
|
result.set(values[index]!, restoreTokens(segments[index]!, masked[index]!.tokens));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function maskTokens(value: string): MaskedText {
|
|
const tokens: string[] = [];
|
|
return {
|
|
text: value.replace(protectedTokenPattern(), (token) => {
|
|
const index = tokens.push(token) - 1;
|
|
return `HGTOKEN${String(index).padStart(4, "0")}TOKENHG`;
|
|
}),
|
|
tokens,
|
|
};
|
|
}
|
|
|
|
async function translatePreservingTokens(value: string): Promise<string> {
|
|
const matches = [...value.matchAll(protectedTokenPattern())];
|
|
let cursor = 0;
|
|
let translated = "";
|
|
for (const match of matches) {
|
|
const index = match.index ?? cursor;
|
|
translated += await translateFragment(value.slice(cursor, index));
|
|
translated += match[0];
|
|
cursor = index + match[0].length;
|
|
}
|
|
translated += await translateFragment(value.slice(cursor));
|
|
return translated;
|
|
}
|
|
|
|
async function translateFragment(value: string): Promise<string> {
|
|
if (!/\p{L}/u.test(value)) return value;
|
|
const leadingWhitespace = value.match(/^\s*/u)?.[0] ?? "";
|
|
const trailingWhitespace = value.match(/\s*$/u)?.[0] ?? "";
|
|
const core = value.slice(leadingWhitespace.length, value.length - trailingWhitespace.length);
|
|
if (!core) return value;
|
|
return `${leadingWhitespace}${(await requestTranslation(core)).trim()}${trailingWhitespace}`;
|
|
}
|
|
|
|
function protectedTokenPattern(): RegExp {
|
|
return /https?:\/\/[^\s<>"']+|mailto:[^\s<>"']+|tel:[^\s<>"']+|\{\{[^{}]+\}\}|<\/?[A-Za-z][^>]*>|`[^`]+`|\{[A-Za-z_][\w.]*\}|[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}|\r?\n/gu;
|
|
}
|
|
|
|
function restoreTokens(value: string, tokens: string[]): string {
|
|
let restored = value.trim();
|
|
for (let index = 0; index < tokens.length; index += 1) {
|
|
const marker = new RegExp(`HGTOKEN\\s*${String(index).padStart(4, "0")}\\s*TOKENHG`, "giu");
|
|
restored = restored.replace(marker, tokens[index]!);
|
|
}
|
|
if (/HGTOKEN\s*\d{4}\s*TOKENHG/iu.test(restored)) throw new Error(`Unrestored token in translated value: ${restored.slice(0, 160)}`);
|
|
return restored;
|
|
}
|
|
|
|
function hasSameProtectedTokens(sourceValue: string, translatedValue: string): boolean {
|
|
return JSON.stringify(maskTokens(sourceValue).tokens) === JSON.stringify(maskTokens(translatedValue).tokens);
|
|
}
|
|
|
|
function parseBatchResponse(value: string, expected: number): string[] {
|
|
const markers = [...value.matchAll(/HG\s*(\d{6})\s*\|\|\|\s*/gu)];
|
|
if (markers.length !== expected) {
|
|
throw new Error(`Translation batch marker mismatch: expected ${expected}, received ${markers.length}`);
|
|
}
|
|
const result = Array<string>(expected);
|
|
for (let markerIndex = 0; markerIndex < markers.length; markerIndex += 1) {
|
|
const marker = markers[markerIndex]!;
|
|
const index = Number(marker[1]);
|
|
const start = (marker.index ?? 0) + marker[0].length;
|
|
const end = markers[markerIndex + 1]?.index ?? value.length;
|
|
result[index] = value.slice(start, end).trim();
|
|
}
|
|
if (result.some((entry) => entry === undefined)) throw new Error("Translation batch returned non-contiguous markers.");
|
|
return result;
|
|
}
|
|
|
|
async function requestTranslation(text: string): Promise<string> {
|
|
const query = new URLSearchParams({ client: "gtx", sl: "de", tl: locale!, dt: "t", q: text });
|
|
const url = `https://translate.googleapis.com/translate_a/single?${query.toString()}`;
|
|
let lastError: unknown;
|
|
for (let attempt = 1; attempt <= 5; attempt += 1) {
|
|
try {
|
|
const response = await fetch(url, { headers: { "user-agent": "HiveGrid-static-i18n-build/1.0" }, signal: AbortSignal.timeout(30_000) });
|
|
if (!response.ok) throw new Error(`Translation request failed with ${response.status}`);
|
|
const payload = (await response.json()) as unknown[][];
|
|
const fragments = payload[0];
|
|
if (!Array.isArray(fragments)) throw new Error("Translation response has no fragments.");
|
|
return fragments.map((fragment) => (Array.isArray(fragment) ? String(fragment[0] ?? "") : "")).join("");
|
|
} catch (error) {
|
|
lastError = error;
|
|
await new Promise((resolve) => setTimeout(resolve, attempt * 750));
|
|
}
|
|
}
|
|
throw lastError;
|
|
}
|
|
|
|
function applyPluralOverrides(namespace: string, target: JsonObject): void {
|
|
if (namespace !== "common") return;
|
|
const overrides: Record<TargetLocale, { itemsSelected: string; memberCount: string }> = {
|
|
pl: {
|
|
itemsSelected: "{count, plural, =0 {Nic nie wybrano} one {Wybrano # element} few {Wybrano # elementy} many {Wybrano # elementów} other {Wybrano # elementu}}",
|
|
memberCount: "{count, plural, one {# członek} few {# członków} many {# członków} other {# członka}}",
|
|
},
|
|
ru: {
|
|
itemsSelected: "{count, plural, =0 {Ничего не выбрано} one {Выбран # элемент} few {Выбрано # элемента} many {Выбрано # элементов} other {Выбрано # элемента}}",
|
|
memberCount: "{count, plural, one {# участник} few {# участника} many {# участников} other {# участника}}",
|
|
},
|
|
uk: {
|
|
itemsSelected: "{count, plural, =0 {Нічого не вибрано} one {Вибрано # елемент} few {Вибрано # елементи} many {Вибрано # елементів} other {Вибрано # елемента}}",
|
|
memberCount: "{count, plural, one {# учасник} few {# учасники} many {# учасників} other {# учасника}}",
|
|
},
|
|
hu: {
|
|
itemsSelected: "{count, plural, =0 {Nincs kijelölt elem} one {# elem kijelölve} other {# elem kijelölve}}",
|
|
memberCount: "{count, plural, one {# tag} other {# tag}}",
|
|
},
|
|
};
|
|
target.itemsSelected = overrides[locale!].itemsSelected;
|
|
target.memberCount = overrides[locale!].memberCount;
|
|
}
|
|
|
|
function readJson<T>(filePath: string, fallback?: T): T {
|
|
if (!existsSync(filePath) && fallback !== undefined) return fallback;
|
|
return JSON.parse(readFileSync(filePath, "utf8")) as T;
|
|
}
|
|
|
|
function writeJson(filePath: string, value: unknown): void {
|
|
mkdirSync(path.dirname(filePath), { recursive: true });
|
|
writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
}
|