Refresh HiveGrid Cloud reference snapshot
This commit is contained in:
1 parent
5e99e14772
commit
7f88026f79
5955 files changed
+1177325
-188362
No files matched your search
@@ -0,0 +1,318 @@
|
||||
#!/usr/bin/env tsx
|
||||
/* eslint-disable no-console */
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import { i18nNamespaces, type SupportedLocale } from "../packages/i18n/src/index.ts";
|
||||
|
||||
type TargetLocale = Exclude<SupportedLocale, "de" | "en">;
|
||||
type JsonValue = boolean | null | number | string | JsonValue[] | { [key: string]: JsonValue };
|
||||
type JsonObject = { [key: string]: JsonValue };
|
||||
type MaskedText = { text: string; tokens: string[] };
|
||||
|
||||
const targetLocales = new Set<TargetLocale>(["pl", "ru", "uk", "hu"]);
|
||||
const locale = process.argv[2] as TargetLocale | undefined;
|
||||
if (!locale || !targetLocales.has(locale)) {
|
||||
console.error("Usage: pnpm exec tsx scripts/i18n-translate-catalog.ts <pl|ru|uk|hu>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const root = process.cwd();
|
||||
const localesRoot = path.join(root, "apps/web/public/locales");
|
||||
const sourceRoot = path.join(localesRoot, "de");
|
||||
const formalSourceRoot = path.join(localesRoot, "de-formal");
|
||||
const targetRoot = path.join(localesRoot, locale);
|
||||
const formalTargetRoot = path.join(localesRoot, `${locale}-formal`);
|
||||
const cacheRoot = path.join(root, ".cache/i18n-translations");
|
||||
const cachePath = path.join(cacheRoot, `${locale}.json`);
|
||||
const reviewPath = path.join(root, `docs/contributing/i18n/reviews/${locale}.json`);
|
||||
|
||||
mkdirSync(targetRoot, { recursive: true });
|
||||
mkdirSync(formalTargetRoot, { recursive: true });
|
||||
mkdirSync(cacheRoot, { recursive: true });
|
||||
|
||||
const cache = readJson<Record<string, string>>(cachePath, {});
|
||||
for (const [sourceValue, translatedValue] of Object.entries(cache)) {
|
||||
if (!hasSameProtectedTokens(sourceValue, translatedValue)) delete cache[sourceValue];
|
||||
}
|
||||
let translatedCount = 0;
|
||||
let cacheHitCount = 0;
|
||||
|
||||
for (const namespace of i18nNamespaces) {
|
||||
const sourcePath = path.join(sourceRoot, `${namespace}.json`);
|
||||
const targetPath = path.join(targetRoot, `${namespace}.json`);
|
||||
const source = readJson<JsonObject>(sourcePath);
|
||||
const translated = await translateObject(source);
|
||||
applyPluralOverrides(namespace, translated);
|
||||
writeJson(targetPath, translated);
|
||||
|
||||
const formalSourcePath = path.join(formalSourceRoot, `${namespace}.json`);
|
||||
const formalTargetPath = path.join(formalTargetRoot, `${namespace}.json`);
|
||||
const formalSource = existsSync(formalSourcePath) ? readJson<JsonObject>(formalSourcePath) : {};
|
||||
writeJson(formalTargetPath, pruneMatchingValues(await translateObject(formalSource), translated) as JsonObject);
|
||||
|
||||
writeJson(cachePath, cache);
|
||||
console.log(`[${locale}] ${namespace}: translated=${translatedCount} cacheHits=${cacheHitCount}`);
|
||||
}
|
||||
|
||||
const review = readJson<Record<string, unknown>>(reviewPath, {});
|
||||
const sourceHashes = Object.fromEntries(
|
||||
i18nNamespaces.map((namespace) => {
|
||||
const bytes = readFileSync(path.join(sourceRoot, `${namespace}.json`));
|
||||
return [namespace, createHash("sha256").update(bytes).digest("hex")];
|
||||
})
|
||||
);
|
||||
writeJson(reviewPath, {
|
||||
...review,
|
||||
locale,
|
||||
sourceLocale: "de",
|
||||
sourceHashes,
|
||||
machineTranslation: {
|
||||
status: "complete",
|
||||
provider: "Google Translate static build tool",
|
||||
completedAt: new Date().toISOString(),
|
||||
},
|
||||
automatedReview: { status: "pending", checkedAt: null },
|
||||
});
|
||||
|
||||
console.log(`[${locale}] complete: ${translatedCount} new translations, ${cacheHitCount} cache hits`);
|
||||
|
||||
async function translateObject(source: JsonObject): Promise<JsonObject> {
|
||||
const leaves: Array<{ path: string[]; value: string }> = [];
|
||||
collectStringLeaves(source, [], leaves);
|
||||
const output = structuredClone(source);
|
||||
const missing = leaves.filter(({ value }) => shouldTranslate(value) && cache[value] === undefined);
|
||||
|
||||
for (const batch of createBatches(missing, 3_000)) {
|
||||
const translated = await translateBatch(batch);
|
||||
for (const [sourceValue, targetValue] of translated) {
|
||||
cache[sourceValue] = targetValue.normalize("NFC");
|
||||
translatedCount += 1;
|
||||
}
|
||||
writeJson(cachePath, cache);
|
||||
}
|
||||
|
||||
for (const leaf of leaves) {
|
||||
const translated = shouldTranslate(leaf.value) ? cache[leaf.value] : leaf.value;
|
||||
if (translated === undefined) throw new Error(`Missing cached translation for ${leaf.path.join(".")}`);
|
||||
if (cache[leaf.value] !== undefined && !missing.some((candidate) => candidate.value === leaf.value)) cacheHitCount += 1;
|
||||
setAtPath(output, leaf.path, translated.normalize("NFC"));
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function collectStringLeaves(value: JsonValue, currentPath: string[], output: Array<{ path: string[]; value: string }>): void {
|
||||
if (typeof value === "string") {
|
||||
output.push({ path: currentPath, value });
|
||||
return;
|
||||
}
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return;
|
||||
for (const [key, child] of Object.entries(value)) collectStringLeaves(child, [...currentPath, key], output);
|
||||
}
|
||||
|
||||
function setAtPath(rootValue: JsonObject, keyPath: string[], value: string): void {
|
||||
let current: JsonObject = rootValue;
|
||||
for (const key of keyPath.slice(0, -1)) current = current[key] as JsonObject;
|
||||
current[keyPath.at(-1)!] = value;
|
||||
}
|
||||
|
||||
function pruneMatchingValues(overrides: JsonValue, base: JsonValue | undefined): JsonValue | undefined {
|
||||
if (JSON.stringify(overrides) === JSON.stringify(base)) return undefined;
|
||||
if (!overrides || typeof overrides !== "object" || Array.isArray(overrides)) return overrides;
|
||||
|
||||
const pruned = Object.fromEntries(
|
||||
Object.entries(overrides)
|
||||
.map(([key, value]) => [key, pruneMatchingValues(value, base && typeof base === "object" && !Array.isArray(base) ? base[key] : undefined)] as const)
|
||||
.filter((entry): entry is readonly [string, JsonValue] => entry[1] !== undefined)
|
||||
);
|
||||
return Object.keys(pruned).length > 0 ? pruned : undefined;
|
||||
}
|
||||
|
||||
function shouldTranslate(value: string): boolean {
|
||||
const trimmed = value.trim();
|
||||
if (!/\p{L}/u.test(trimmed)) return false;
|
||||
if (/^(?:https?:\/\/|mailto:|tel:)/iu.test(trimmed)) return false;
|
||||
if (/^[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}$/u.test(trimmed)) return false;
|
||||
if (/^[A-Z0-9_.:/+-]{2,}$/u.test(trimmed)) return false;
|
||||
if (/^\{[^{}]+,\s*plural,/u.test(trimmed)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function createBatches(entries: Array<{ value: string }>, maxCharacters: number): string[][] {
|
||||
const unique = [...new Set(entries.map((entry) => entry.value))];
|
||||
const batches: string[][] = [];
|
||||
let current: string[] = [];
|
||||
let size = 0;
|
||||
for (const value of unique) {
|
||||
const maskedLength = maskTokens(value).text.length + 20;
|
||||
if (current.length > 0 && size + maskedLength > maxCharacters) {
|
||||
batches.push(current);
|
||||
current = [];
|
||||
size = 0;
|
||||
}
|
||||
current.push(value);
|
||||
size += maskedLength;
|
||||
}
|
||||
if (current.length > 0) batches.push(current);
|
||||
return batches;
|
||||
}
|
||||
|
||||
async function translateBatch(values: string[]): Promise<Map<string, string>> {
|
||||
const masked = values.map(maskTokens);
|
||||
const protectedValues = values.filter((_value, index) => masked[index]!.tokens.length > 0);
|
||||
if (protectedValues.length > 0) {
|
||||
const result = new Map<string, string>();
|
||||
for (const value of protectedValues) result.set(value, await translatePreservingTokens(value));
|
||||
const plainValues = values.filter((_value, index) => masked[index]!.tokens.length === 0);
|
||||
if (plainValues.length > 0) {
|
||||
for (const entry of await translateBatch(plainValues)) result.set(...entry);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
if (values.length === 1) {
|
||||
return new Map([[values[0]!, restoreTokens(await requestTranslation(masked[0]!.text), masked[0]!.tokens)]]);
|
||||
}
|
||||
const requestText = masked.map((entry, index) => `HG${String(index).padStart(6, "0")}|||${entry.text}`).join("\n");
|
||||
let segments: string[] | undefined;
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
try {
|
||||
segments = parseBatchResponse(await requestTranslation(requestText), values.length);
|
||||
break;
|
||||
} catch {
|
||||
if (attempt < 3) await new Promise((resolve) => setTimeout(resolve, attempt * 500));
|
||||
}
|
||||
}
|
||||
if (!segments) {
|
||||
const midpoint = Math.ceil(values.length / 2);
|
||||
return new Map([...(await translateBatch(values.slice(0, midpoint))), ...(await translateBatch(values.slice(midpoint)))]);
|
||||
}
|
||||
const result = new Map<string, string>();
|
||||
for (let index = 0; index < values.length; index += 1) {
|
||||
result.set(values[index]!, restoreTokens(segments[index]!, masked[index]!.tokens));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function maskTokens(value: string): MaskedText {
|
||||
const tokens: string[] = [];
|
||||
return {
|
||||
text: value.replace(protectedTokenPattern(), (token) => {
|
||||
const index = tokens.push(token) - 1;
|
||||
return `HGTOKEN${String(index).padStart(4, "0")}TOKENHG`;
|
||||
}),
|
||||
tokens,
|
||||
};
|
||||
}
|
||||
|
||||
async function translatePreservingTokens(value: string): Promise<string> {
|
||||
const matches = [...value.matchAll(protectedTokenPattern())];
|
||||
let cursor = 0;
|
||||
let translated = "";
|
||||
for (const match of matches) {
|
||||
const index = match.index ?? cursor;
|
||||
translated += await translateFragment(value.slice(cursor, index));
|
||||
translated += match[0];
|
||||
cursor = index + match[0].length;
|
||||
}
|
||||
translated += await translateFragment(value.slice(cursor));
|
||||
return translated;
|
||||
}
|
||||
|
||||
async function translateFragment(value: string): Promise<string> {
|
||||
if (!/\p{L}/u.test(value)) return value;
|
||||
const leadingWhitespace = value.match(/^\s*/u)?.[0] ?? "";
|
||||
const trailingWhitespace = value.match(/\s*$/u)?.[0] ?? "";
|
||||
const core = value.slice(leadingWhitespace.length, value.length - trailingWhitespace.length);
|
||||
if (!core) return value;
|
||||
return `${leadingWhitespace}${(await requestTranslation(core)).trim()}${trailingWhitespace}`;
|
||||
}
|
||||
|
||||
function protectedTokenPattern(): RegExp {
|
||||
return /https?:\/\/[^\s<>"']+|mailto:[^\s<>"']+|tel:[^\s<>"']+|\{\{[^{}]+\}\}|<\/?[A-Za-z][^>]*>|`[^`]+`|\{[A-Za-z_][\w.]*\}|[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}|\r?\n/gu;
|
||||
}
|
||||
|
||||
function restoreTokens(value: string, tokens: string[]): string {
|
||||
let restored = value.trim();
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
const marker = new RegExp(`HGTOKEN\\s*${String(index).padStart(4, "0")}\\s*TOKENHG`, "giu");
|
||||
restored = restored.replace(marker, tokens[index]!);
|
||||
}
|
||||
if (/HGTOKEN\s*\d{4}\s*TOKENHG/iu.test(restored)) throw new Error(`Unrestored token in translated value: ${restored.slice(0, 160)}`);
|
||||
return restored;
|
||||
}
|
||||
|
||||
function hasSameProtectedTokens(sourceValue: string, translatedValue: string): boolean {
|
||||
return JSON.stringify(maskTokens(sourceValue).tokens) === JSON.stringify(maskTokens(translatedValue).tokens);
|
||||
}
|
||||
|
||||
function parseBatchResponse(value: string, expected: number): string[] {
|
||||
const markers = [...value.matchAll(/HG\s*(\d{6})\s*\|\|\|\s*/gu)];
|
||||
if (markers.length !== expected) {
|
||||
throw new Error(`Translation batch marker mismatch: expected ${expected}, received ${markers.length}`);
|
||||
}
|
||||
const result = Array<string>(expected);
|
||||
for (let markerIndex = 0; markerIndex < markers.length; markerIndex += 1) {
|
||||
const marker = markers[markerIndex]!;
|
||||
const index = Number(marker[1]);
|
||||
const start = (marker.index ?? 0) + marker[0].length;
|
||||
const end = markers[markerIndex + 1]?.index ?? value.length;
|
||||
result[index] = value.slice(start, end).trim();
|
||||
}
|
||||
if (result.some((entry) => entry === undefined)) throw new Error("Translation batch returned non-contiguous markers.");
|
||||
return result;
|
||||
}
|
||||
|
||||
async function requestTranslation(text: string): Promise<string> {
|
||||
const query = new URLSearchParams({ client: "gtx", sl: "de", tl: locale!, dt: "t", q: text });
|
||||
const url = `https://translate.googleapis.com/translate_a/single?${query.toString()}`;
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= 5; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(url, { headers: { "user-agent": "HiveGrid-static-i18n-build/1.0" }, signal: AbortSignal.timeout(30_000) });
|
||||
if (!response.ok) throw new Error(`Translation request failed with ${response.status}`);
|
||||
const payload = (await response.json()) as unknown[][];
|
||||
const fragments = payload[0];
|
||||
if (!Array.isArray(fragments)) throw new Error("Translation response has no fragments.");
|
||||
return fragments.map((fragment) => (Array.isArray(fragment) ? String(fragment[0] ?? "") : "")).join("");
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await new Promise((resolve) => setTimeout(resolve, attempt * 750));
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
function applyPluralOverrides(namespace: string, target: JsonObject): void {
|
||||
if (namespace !== "common") return;
|
||||
const overrides: Record<TargetLocale, { itemsSelected: string; memberCount: string }> = {
|
||||
pl: {
|
||||
itemsSelected: "{count, plural, =0 {Nic nie wybrano} one {Wybrano # element} few {Wybrano # elementy} many {Wybrano # elementów} other {Wybrano # elementu}}",
|
||||
memberCount: "{count, plural, one {# członek} few {# członków} many {# członków} other {# członka}}",
|
||||
},
|
||||
ru: {
|
||||
itemsSelected: "{count, plural, =0 {Ничего не выбрано} one {Выбран # элемент} few {Выбрано # элемента} many {Выбрано # элементов} other {Выбрано # элемента}}",
|
||||
memberCount: "{count, plural, one {# участник} few {# участника} many {# участников} other {# участника}}",
|
||||
},
|
||||
uk: {
|
||||
itemsSelected: "{count, plural, =0 {Нічого не вибрано} one {Вибрано # елемент} few {Вибрано # елементи} many {Вибрано # елементів} other {Вибрано # елемента}}",
|
||||
memberCount: "{count, plural, one {# учасник} few {# учасники} many {# учасників} other {# учасника}}",
|
||||
},
|
||||
hu: {
|
||||
itemsSelected: "{count, plural, =0 {Nincs kijelölt elem} one {# elem kijelölve} other {# elem kijelölve}}",
|
||||
memberCount: "{count, plural, one {# tag} other {# tag}}",
|
||||
},
|
||||
};
|
||||
target.itemsSelected = overrides[locale!].itemsSelected;
|
||||
target.memberCount = overrides[locale!].memberCount;
|
||||
}
|
||||
|
||||
function readJson<T>(filePath: string, fallback?: T): T {
|
||||
if (!existsSync(filePath) && fallback !== undefined) return fallback;
|
||||
return JSON.parse(readFileSync(filePath, "utf8")) as T;
|
||||
}
|
||||
|
||||
function writeJson(filePath: string, value: unknown): void {
|
||||
mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
Reference in new issue
Block a user