173 lines
7.6 KiB
TypeScript
173 lines
7.6 KiB
TypeScript
#!/usr/bin/env tsx
|
|
/* eslint-disable no-console */
|
|
import { createHash } from "node:crypto";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import path from "node:path";
|
|
|
|
import { catalogReadyLocales, i18nNamespaces, type SupportedLocale } from "../packages/i18n/src/index.ts";
|
|
|
|
type JsonValue = boolean | null | number | string | JsonValue[] | { [key: string]: JsonValue };
|
|
type JsonObject = { [key: string]: JsonValue };
|
|
type TargetLocale = Exclude<SupportedLocale, "de" | "en">;
|
|
|
|
const targetLocales = new Set<TargetLocale>(["pl", "ru", "uk", "hu"]);
|
|
const localesRoot = path.join(process.cwd(), "apps/web/public/locales");
|
|
const reviewRoot = path.join(process.cwd(), "docs/contributing/i18n/reviews");
|
|
const requiredPluralCategories: Readonly<Record<TargetLocale, readonly string[]>> = {
|
|
pl: ["one", "few", "many", "other"],
|
|
ru: ["one", "few", "many", "other"],
|
|
uk: ["one", "few", "many", "other"],
|
|
hu: ["one", "other"],
|
|
};
|
|
|
|
const issues: string[] = [];
|
|
const germanCatalogs = loadLocale("de");
|
|
const englishCatalogs = loadLocale("en");
|
|
|
|
for (const locale of catalogReadyLocales) {
|
|
if (!targetLocales.has(locale as TargetLocale)) continue;
|
|
const targetLocale = locale as TargetLocale;
|
|
const targetCatalogs = loadLocale(targetLocale);
|
|
validateManifest(targetLocale);
|
|
|
|
for (const namespace of i18nNamespaces) {
|
|
const german = flatten(germanCatalogs[namespace]!);
|
|
const english = flatten(englishCatalogs[namespace]!);
|
|
const target = flatten(targetCatalogs[namespace]!);
|
|
for (const [key, sourceValue] of german) {
|
|
const targetValue = target.get(key);
|
|
if (typeof sourceValue !== "string" || typeof targetValue !== "string") continue;
|
|
validateText(`${targetLocale}/${namespace}:${key}`, sourceValue, english.get(key), targetValue);
|
|
if (sourceValue.includes(", plural,")) validatePlural(`${targetLocale}/${namespace}:${key}`, targetLocale, targetValue);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const locale of catalogReadyLocales) validateFormalBundle(locale, loadLocale(locale));
|
|
|
|
if (issues.length > 0) {
|
|
console.error(["FAIL i18n-quality:", ...issues.map((issue) => `- ${issue}`)].join("\n"));
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(`PASS i18n-quality: ${catalogReadyLocales.length} catalog-ready locales preserve tokens, ICU plurals, Unicode and source hashes`);
|
|
|
|
function loadLocale(locale: string): Record<string, JsonObject> {
|
|
return Object.fromEntries(
|
|
i18nNamespaces.map((namespace) => {
|
|
const filePath = path.join(localesRoot, locale, `${namespace}.json`);
|
|
if (!existsSync(filePath)) return [namespace, {}];
|
|
return [namespace, JSON.parse(readFileSync(filePath, "utf8")) as JsonObject];
|
|
})
|
|
);
|
|
}
|
|
|
|
function flatten(value: JsonValue, prefix = "", output = new Map<string, JsonValue>()): Map<string, JsonValue> {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
if (prefix) output.set(prefix, value);
|
|
return output;
|
|
}
|
|
for (const [key, child] of Object.entries(value)) flatten(child, prefix ? `${prefix}.${key}` : key, output);
|
|
return output;
|
|
}
|
|
|
|
function validateText(id: string, german: string, english: JsonValue | undefined, target: string): void {
|
|
if (target !== target.normalize("NFC")) issues.push(`${id} is not NFC-normalized`);
|
|
if (/�|Ã.|Â.|â€|Ð.|Ñ./u.test(target)) issues.push(`${id} contains mojibake`);
|
|
if (/[⟦⟧⟟⦧⧦⦦]|(?:HG\p{L}{0,12}O{1,2}KEN|O{1,2}KEN\p{L}{0,12}HG)/iu.test(target)) {
|
|
issues.push(`${id} contains an unrestored translation marker`);
|
|
}
|
|
|
|
const sourceSignature = protectedSignature(german);
|
|
const targetSignature = protectedSignature(target);
|
|
if (JSON.stringify(sourceSignature) !== JSON.stringify(targetSignature)) {
|
|
issues.push(`${id} changed placeholders, ICU arguments, tags or URLs`);
|
|
}
|
|
|
|
if (isHumanSentence(german) && target.trim() === german.trim() && target.trim() !== String(english ?? "").trim()) {
|
|
issues.push(`${id} is unchanged German source text`);
|
|
}
|
|
if (typeof english === "string" && isHumanSentence(english) && target.trim() === english.trim() && target.trim() !== german.trim()) {
|
|
issues.push(`${id} is unchanged English reference text`);
|
|
}
|
|
}
|
|
|
|
function protectedSignature(value: string): Record<string, string[]> {
|
|
return {
|
|
interpolation: sortedMatches(value, /\{\{[^{}]+\}\}/gu),
|
|
arguments: sortedMatches(value, /\{[A-Za-z_][\w.]*(?=\s*(?:,|\}))/gu),
|
|
emails: sortedMatches(value, /[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}/gu),
|
|
lineBreaks: sortedMatches(value, /\r?\n/gu),
|
|
tags: sortedMatches(value, /<\/?[A-Za-z][^>]*>/gu),
|
|
urls: sortedMatches(value, /(?:https?:\/\/|mailto:|tel:)[^\s<>"']+/gu),
|
|
};
|
|
}
|
|
|
|
function sortedMatches(value: string, pattern: RegExp): string[] {
|
|
return [...value.matchAll(pattern)].map((match) => match[0]).sort((left, right) => left.localeCompare(right));
|
|
}
|
|
|
|
function isHumanSentence(value: string): boolean {
|
|
if (value.length < 18 || !/\s/u.test(value) || /^(?:https?:\/\/|mailto:|tel:)/u.test(value.trim())) return false;
|
|
const withoutProtected = value
|
|
.replaceAll(/\{\{[^{}]+\}\}/gu, "")
|
|
.replaceAll(/<\/?[A-Za-z][^>]*>/gu, "")
|
|
.replaceAll(/[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}/gu, "")
|
|
.replaceAll(/(?:https?:\/\/|mailto:|tel:)[^\s<>"']+/gu, "");
|
|
return (withoutProtected.match(/\p{L}+/gu)?.length ?? 0) >= 4;
|
|
}
|
|
|
|
function validatePlural(id: string, locale: TargetLocale, value: string): void {
|
|
const categories = new Set([...value.matchAll(/(?:^|[\s,{])(zero|one|two|few|many|other)\s*\{/gu)].map((match) => match[1]));
|
|
for (const category of requiredPluralCategories[locale]) {
|
|
if (!categories.has(category)) issues.push(`${id} misses CLDR plural category ${category}`);
|
|
}
|
|
}
|
|
|
|
function validateFormalBundle(locale: SupportedLocale, baseCatalogs: Record<string, JsonObject>): void {
|
|
for (const namespace of i18nNamespaces) {
|
|
const filePath = path.join(localesRoot, `${locale}-formal`, `${namespace}.json`);
|
|
if (!existsSync(filePath)) {
|
|
issues.push(`${locale}-formal/${namespace}.json is missing`);
|
|
continue;
|
|
}
|
|
const formal = flatten(JSON.parse(readFileSync(filePath, "utf8")) as JsonObject);
|
|
const base = flatten(baseCatalogs[namespace]!);
|
|
for (const [key, value] of formal) {
|
|
if (!base.has(key)) {
|
|
issues.push(`${locale}-formal/${namespace}:${key} is not present in the base locale`);
|
|
continue;
|
|
}
|
|
if (JSON.stringify(value) === JSON.stringify(base.get(key))) {
|
|
issues.push(`${locale}-formal/${namespace}:${key} duplicates the base locale instead of being a formal override`);
|
|
continue;
|
|
}
|
|
if (typeof value === "string" && typeof base.get(key) === "string") {
|
|
const formalSignature = protectedSignature(value);
|
|
const baseSignature = protectedSignature(base.get(key) as string);
|
|
if (JSON.stringify(formalSignature) !== JSON.stringify(baseSignature)) {
|
|
issues.push(`${locale}-formal/${namespace}:${key} changed protected tokens`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function validateManifest(locale: TargetLocale): void {
|
|
const manifestPath = path.join(reviewRoot, `${locale}.json`);
|
|
if (!existsSync(manifestPath)) {
|
|
issues.push(`${locale} review manifest is missing`);
|
|
return;
|
|
}
|
|
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
|
|
sourceHashes?: Record<string, string>;
|
|
machineTranslation?: { status?: string };
|
|
};
|
|
if (manifest.machineTranslation?.status !== "complete") issues.push(`${locale} machine-translation status is not complete`);
|
|
for (const namespace of i18nNamespaces) {
|
|
const sourcePath = path.join(localesRoot, "de", `${namespace}.json`);
|
|
const expected = createHash("sha256").update(readFileSync(sourcePath)).digest("hex");
|
|
if (manifest.sourceHashes?.[namespace] !== expected) issues.push(`${locale}/${namespace} source hash is stale`);
|
|
}
|
|
}
|