feat(website): add multilingual editions and translation workflow

This commit is contained in:
N0zoM1z0
2026-10-11 17:24:46 +08:00
committed by Ysopwnable
parent ff57b75e22
commit a2d3338e90
79 changed files with 250648 additions and 471 deletions
+1
View File
@@ -123,6 +123,7 @@ if (
file.startsWith("docs/")
) {
selected.docs = true;
selected.website ||= matchesAny(file, SCOPES.website);
continue;
}
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
+2
View File
@@ -222,6 +222,8 @@ export const SCOPES = {
website: [
"website/**",
"scripts/*website*.py",
"README*.md",
".github/workflows/website-check.yml",
".github/workflows/website-pages.yml",
],
};
+8 -3
View File
@@ -76,9 +76,14 @@ def prepare_notes_example(site):
print(f"Prepared examples/notes-example.zip ({len(EXAMPLE_FILES)} files).")
if __name__ == "__main__":
root = Path(__file__).resolve().parent.parent
site = root / "website/public"
def prepare_assets(site):
"""Finish the shared publish artifact before it replaces the current preview."""
prepare_sharing_image(site)
prepare_sitemap(site)
prepare_notes_example(site)
if __name__ == "__main__":
root = Path(__file__).resolve().parent.parent
from website_i18n import build_translations
build_translations(root, root / "website/_site", finalize=prepare_assets)
+163 -3
View File
@@ -3,6 +3,7 @@
import contextlib
import importlib.util
import io
import json
from pathlib import Path
import shutil
import tempfile
@@ -13,6 +14,9 @@ from urllib.error import HTTPError
from urllib.request import urlopen
import xml.etree.ElementTree as ET
import polib
from website_i18n import Document, build_translations, checked_catalog, extract_catalogs, languages, reading_pages, read_runtime
ROOT = Path(__file__).resolve().parent.parent
@@ -113,7 +117,7 @@ class WebsiteSearchChecks(unittest.TestCase):
for name, old, new, message in cases:
with self.subTest(name=name), tempfile.TemporaryDirectory() as directory:
site = Path(directory) / "public"
shutil.copytree(ROOT / "website/public", site)
shutil.copytree(ROOT / "website/_site", site)
path = site / name
source = path.read_text()
self.assertIn(old, source)
@@ -123,7 +127,7 @@ class WebsiteSearchChecks(unittest.TestCase):
)
with tempfile.TemporaryDirectory() as directory:
site = Path(directory) / "public"
shutil.copytree(ROOT / "website/public", site)
shutil.copytree(ROOT / "website/_site", site)
(site / "assets/social-card.png").unlink()
self.assertIn(
"Social preview PNG is missing; run scripts/prepare-website.py",
@@ -162,6 +166,9 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
for name in ("public", "templates"):
shutil.copytree(ROOT / "website" / name, root / "website" / name)
site = root / "website/public"
WEBSITE.sync_pages(root)
shutil.copyfile(ROOT / "website/_site/assets/social-card.png", site / "assets/social-card.png")
PREPARE.prepare_notes_example(site)
page = WEBSITE.new_page(root, "blog/new-example", 'A <test> & "result"', 'Check "quotes" & <markup>.')
source = page.read_text()
self.assertIn('href="../../blog/" aria-current="page"', source)
@@ -182,7 +189,7 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
def test_links_in_prompts_fragments_and_copy_controls_are_verified(self):
with tempfile.TemporaryDirectory() as directory:
site = Path(directory) / "public"
shutil.copytree(ROOT / "website/public", site)
shutil.copytree(ROOT / "website/_site", site)
page = site / "faq/index.html"
original = page.read_text()
cases = (
@@ -242,5 +249,158 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
worker.join()
class WebsiteTranslationChecks(unittest.TestCase):
"""Exercise source edits, translated evidence and language editions at their consumers."""
@contextlib.contextmanager
def translated_project(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("public", "templates", "locales"):
shutil.copytree(ROOT / "website" / name, root / "website" / name)
inventory = [item for item in languages(ROOT) if item["code"] in {"en", "zh-CN", "ar"}]
(root / "website/locales/languages.json").write_text(json.dumps(inventory))
for item in inventory:
(root / item["readme"]).write_text("# REA\n")
WEBSITE.sync_pages(root)
extract_catalogs(root)
yield root
def test_source_change_preserves_review_and_excludes_stale_translation_from_search(self):
with self.translated_project() as root:
path = root / "website/locales/zh-CN/messages.po"
catalog = polib.pofile(path)
old = catalog.find("How do I install REA?", msgctxt="page:faq/index.html")
old.tcomment = "Reviewed installation wording."
old_translation = old.msgstr
catalog.save(path)
page = root / "website/public/faq/index.html"
page.write_text(page.read_text().replace("How do I install REA?", "How do I install REA today?"))
with self.assertRaisesRegex(ValueError, "Translation sources changed"):
extract_catalogs(root, check=True)
extract_catalogs(root)
updated = polib.pofile(path)
archived = next(entry for entry in updated if entry.obsolete and entry.msgid == old.msgid)
self.assertEqual((archived.msgstr, archived.tcomment), (old_translation, old.tcomment))
new = updated.find("How do I install REA today?", msgctxt=old.msgctxt)
self.assertEqual(new.msgstr, "")
build_translations(root, root / "website/_site")
site = root / "website/_site"
PREPARE.prepare_sitemap(site)
fallback = VERIFY.HtmlReferences()
fallback.feed((site / "zh-CN/faq/index.html").read_text())
self.assertTrue(fallback.translation_fallback)
self.assertTrue(fallback.noindex())
self.assertEqual(fallback.lang, "en")
self.assertNotIn("https://rea.tools/zh-CN/faq/", (site / "sitemap.xml").read_text())
self.assertNotIn('hreflang="zh-CN"', (site / "faq/index.html").read_text())
new.msgstr = "现在如何安装 REA?"
new.flags.append("fuzzy")
updated.save(path)
build_translations(root, site)
self.assertIn('data-translation-fallback="true"', (site / "zh-CN/faq/index.html").read_text())
new.flags.remove("fuzzy")
updated.save(path)
build_translations(root, site)
PREPARE.prepare_sitemap(site)
result = (site / "zh-CN/faq/index.html").read_text()
self.assertIn("现在如何安装 REA?", result)
self.assertNotIn("data-translation-fallback", result)
self.assertIn('href="../assets/styles.css"', (site / "zh-CN/index.html").read_text())
self.assertIn('hreflang="zh-CN"', (site / "faq/index.html").read_text())
self.assertIn("https://rea.tools/zh-CN/faq/", (site / "sitemap.xml").read_text())
def test_broken_placeholders_and_inline_markup_fail_before_replacing_preview(self):
with self.translated_project() as root:
site = root / "website/_site"
build_translations(root, site)
before = (site / "zh-CN/index.html").read_bytes()
path = root / "website/locales/zh-CN/messages.po"
catalog = polib.pofile(path)
entry = catalog.find("Already know RE? {ph0}Skip to analysis guides →{ph1}", msgctxt="page:index.html")
original = entry.msgstr
for bad, reason in (("丢失链接", "changed placeholders"), ("{ph1}错误的链接{ph0}", "invalid inline markup")):
with self.subTest(reason=reason):
entry.msgstr = bad
catalog.save(path)
with self.assertRaisesRegex(ValueError, reason):
build_translations(root, site)
self.assertEqual((site / "zh-CN/index.html").read_bytes(), before)
entry.msgstr = original
catalog.save(path)
(root / "website/locales/languages.json").unlink()
with self.assertRaisesRegex(ValueError, "Missing languages.json"):
build_translations(root, site)
self.assertEqual((site / "zh-CN/index.html").read_bytes(), before)
def test_runtime_message_drift_and_page_specific_completeness(self):
with self.translated_project() as root:
script = root / "website/public/assets/dino-demo.js"
original = script.read_text()
script.write_text(original.replace('t("Pause")', 't("Pause game")'))
self.assertIn("Pause game", read_runtime(root))
with self.assertRaisesRegex(ValueError, "Translation sources changed"):
extract_catalogs(root, check=True)
script.write_text(original.replace('t("Pause")', 't(dynamicMessage)'))
with self.assertRaisesRegex(ValueError, "one literal message"):
read_runtime(root)
script.write_text(original)
path = root / "website/locales/zh-CN/messages.po"
catalog = polib.pofile(path)
catalog.find("Pause", msgctxt="runtime").msgstr = ""
catalog.save(path)
site = root / "website/_site"
build_translations(root, site)
self.assertIn('data-translation-fallback="true"', (site / "zh-CN/examples/dino-lab/index.html").read_text())
self.assertNotIn('data-translation-fallback="true"', (site / "zh-CN/faq/index.html").read_text())
def test_all_readme_editions_preserve_code_targets_downloads_and_reciprocal_routes(self):
site = ROOT / "website/_site"
english_pages = reading_pages(ROOT / "website/public")
for language in languages(ROOT):
code = language["code"]
for original in english_pages:
relative = original.relative_to(ROOT / "website/public")
path = site / relative if code == "en" else site / code / relative
with self.subTest(language=code, page=str(relative)):
source = original.read_text()
translated = path.read_text()
def evidence(html):
doc = Document(html)
return sorted(doc.source[node.start:node.end] for node in doc.nodes if node.tag in {"pre", "code", "kbd"} or node.attrs.get("translate") == "no")
self.assertEqual(evidence(source), evidence(translated))
parser = VERIFY.HtmlReferences()
parser.feed(translated)
self.assertEqual(parser.lang, "en" if parser.translation_fallback else code)
if code != "en" and not parser.translation_fallback:
self.assertEqual(parser.direction, language["dir"])
if parser.translation_fallback:
self.assertTrue(parser.noindex())
self.assertNotIn(code, parser.alternates)
else:
self.assertEqual(parser.alternates[code], WEBSITE.page_url(site, path))
english = VERIFY.HtmlReferences()
english.feed((site / relative).read_text())
self.assertEqual(parser.alternates, english.alternates)
self.assertTrue(set(parser.copy_targets) <= parser.ids)
for fixture in WEBSITE.NOTES_FIXTURES:
self.assertEqual((site / fixture).read_bytes(), (ROOT / "website/public" / fixture).read_bytes())
with ThreadingHTTPServer(("127.0.0.1", 0), WEBSITE.preview_handler(site, "/rea/")) as server:
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
base = f"http://127.0.0.1:{server.server_port}/rea/"
with urlopen(base + "ar/faq?test=1") as response:
self.assertEqual(response.url, base + "ar/faq/?test=1")
self.assertIn(b'lang="ar"', response.read())
with urlopen(base + "assets/i18n.js") as response:
self.assertIn(b"export function t", response.read())
with urlopen(base + "examples/notes-example.zip") as response:
self.assertEqual(response.read(), (site / "examples/notes-example.zip").read_bytes())
finally:
server.shutdown()
worker.join()
if __name__ == "__main__":
unittest.main()
+43 -7
View File
@@ -95,6 +95,9 @@ class HtmlReferences(HTMLParser):
self.properties = {}
self.canonicals = []
self.lang = None
self.direction = None
self.translation_fallback = False
self.alternates = {}
self.h1_count = 0
self.text = []
self.copy_targets = []
@@ -103,11 +106,18 @@ class HtmlReferences(HTMLParser):
attributes = dict(attrs)
if tag == "html":
self.lang = attributes.get("lang")
self.direction = attributes.get("dir")
self.translation_fallback = attributes.get("data-translation-fallback") == "true"
if tag == "h1":
self.h1_count += 1
if tag == "head":
self.in_head = True
if self.in_head:
if tag == "link" and attributes.get("rel") == "alternate" and attributes.get("hreflang"):
language = attributes["hreflang"]
if language in self.alternates:
self.duplicate_ids.append("hreflang:" + language)
self.alternates[language] = attributes.get("href") or ""
if tag == "title":
self.titles.append("")
self.in_title = True
@@ -167,8 +177,17 @@ def check_search_metadata(site, pages):
title = page.titles[0].strip() if len(page.titles) == 1 else ""
if not title:
errors.append(f"{label}: needs exactly one nonempty head title")
if page.lang != "en" or page.h1_count != 1:
errors.append(f"{label}: needs lang=en and exactly one h1")
first = path.relative_to(site).parts[0]
expected_language = first if re.fullmatch(r"[a-z]{2}(?:-[A-Z]{2})?", first) else "en"
if page.translation_fallback:
if page.lang != "en" or not page.noindex():
errors.append(f"{label}: English translation fallback needs lang=en and noindex")
elif page.lang != expected_language:
errors.append(f"{label}: needs lang={expected_language} and exactly one h1")
if page.h1_count != 1:
errors.append(f"{label}: needs exactly one h1")
if expected_language in ("ar", "fa") and not page.translation_fallback and page.direction != "rtl":
errors.append(f"{label}: needs dir=rtl")
if label in NOTES_FIXTURES and not page.noindex():
errors.append(
f"{label}: example application must have a head noindex directive"
@@ -179,9 +198,10 @@ def check_search_metadata(site, pages):
indexable.add(url)
if page.canonicals != [url]:
errors.append(f"{label}: needs exactly one head canonical for {url}")
if title in titles:
title_identity = (page.lang, title)
if title_identity in titles:
errors.append(f"{label}: duplicate search title {title!r}")
titles.add(title)
titles.add(title_identity)
descriptions = page.meta.get("description", [])
description = descriptions[0].strip() if len(descriptions) == 1 else ""
if not description:
@@ -219,6 +239,22 @@ def check_search_metadata(site, pages):
values = collection.get(key, [])
if len(values) != 1 or not values[0].strip():
errors.append(f"{label}: needs exactly one nonempty {key}")
if page.alternates:
if page.alternates.get(page.lang) != url:
errors.append(f"{label}: hreflang must include its own canonical")
for language, target in page.alternates.items():
parsed = urlsplit(target)
if parsed.scheme != "https" or parsed.netloc != "rea.tools":
errors.append(f"{label}: hreflang needs absolute rea.tools URLs")
continue
target_path = (site / unquote(parsed.path).lstrip("/")).resolve()
if target_path.is_dir():
target_path /= "index.html"
other = pages.get(target_path)
if other is None or other.noindex() or other.alternates != page.alternates:
errors.append(f"{label}: nonreciprocal or nonindexable hreflang: {target}")
elif language != "x-default" and other.lang != language:
errors.append(f"{label}: hreflang={language} points to lang={other.lang}")
sitemap_path = site / "sitemap.xml"
try:
@@ -290,7 +326,7 @@ def check_site(site):
parser.feed(path.read_text(encoding="utf-8"))
pages[path] = parser
if site / "index.html" not in pages:
errors.append("website/public/index.html is missing")
errors.append(f"{site}/index.html is missing; run scripts/prepare-website.py")
for path, page in pages.items():
label = path.relative_to(site)
@@ -400,8 +436,8 @@ def check_example_archive(site):
def main():
"""Run the same checks locally, on website PRs and before publishing."""
root = Path(__file__).resolve().parent.parent
pages, references, indexable, errors = check_site(root / "website/public")
errors.extend(check_example_archive(root / "website/public"))
pages, references, indexable, errors = check_site(root / "website/_site")
errors.extend(check_example_archive(root / "website/_site"))
errors.extend(check_publisher(root))
errors.extend(check_repository_links(root))
try:
+23 -2
View File
@@ -31,6 +31,7 @@ def authored_markdown(root):
path for path in sorted(site.rglob("*.md"))
if not any(part.startswith(".") or part in ("node_modules", "__pycache__")
for part in path.relative_to(site).parts)
and "_site" not in path.relative_to(site).parts
]
@@ -111,6 +112,10 @@ def shared_blocks(root, path, source):
"title": escape(metadata.titles[0].strip(), quote=True),
"description": escape(metadata.descriptions[0].strip(), quote=True),
}
from website_i18n import language_markup
selector, alternates = language_markup(root, path.relative_to(site).as_posix())
values["language_picker"] = selector if not metadata.noindex else ""
values["alternates"] = alternates if not metadata.noindex else ""
templates = root / "website/templates"
return {
name: indent(Template((templates / f"{name}.html.tmpl").read_text(encoding="utf-8")).substitute(values), " ")
@@ -207,13 +212,29 @@ def main():
create.add_argument("route")
create.add_argument("--title", required=True)
create.add_argument("--description", required=True)
translations = commands.add_parser("translations", help="Maintain gettext catalogs")
translations.add_argument("action", choices=("extract", "status", "check"))
translations.add_argument("--language", help="Check one language, such as zh-CN")
translations.add_argument("--strict", action="store_true", help="Fail on missing or fuzzy translations")
serve = commands.add_parser("serve", help="Prepare and preview the static site")
serve.add_argument("--port", type=int, default=4173)
serve.add_argument("--base-path", choices=("/", "/rea/"), default="/")
args = parser.parse_args()
root = Path(__file__).resolve().parent.parent
try:
if args.command == "new-page":
if args.command == "translations":
from website_i18n import checked_catalog, extract_catalogs, languages
template = extract_catalogs(root, check=args.action != "extract")
if args.action != "extract":
inventory = languages(root)[1:]
if args.language:
inventory = [item for item in inventory if item["code"] == args.language]
if not inventory:
raise ValueError(f"Unknown translation language: {args.language}")
for language in inventory:
messages, missing = checked_catalog(root, language, template, strict=args.strict)
print(f"{language['code']}: {len(messages)}/{len(template)} translated; {len(missing)} missing or fuzzy")
elif args.command == "new-page":
new_page(root, args.route, args.title, args.description)
elif args.command == "sync":
sync_pages(root, check=args.check)
@@ -228,7 +249,7 @@ def main():
run_script(root, "prepare-website.py")
prefix = args.base_path
handler = preview_handler(root / "website/public", prefix)
handler = preview_handler(root / "website/_site", prefix)
with ThreadingHTTPServer(("127.0.0.1", args.port), handler) as server:
print(f"Preview: http://127.0.0.1:{server.server_port}{prefix}", flush=True)
server.serve_forever()
+637
View File
@@ -0,0 +1,637 @@
"""Extract gettext catalogs and render language editions of authored static HTML."""
from collections import Counter
from html import escape, unescape
from html.parser import HTMLParser
import json
import os
from pathlib import Path
import re
import shutil
from string import Formatter
from tempfile import TemporaryDirectory
from urllib.parse import quote, unquote, urlsplit, urlunsplit
import polib
VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
INLINE = {"a", "abbr", "b", "bdi", "br", "cite", "code", "em", "i", "kbd", "mark", "s", "small", "span", "strong", "sub", "sup", "time", "u"}
PROSE = {"a", "p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "dt", "dd", "figcaption", "summary", "label", "button", "th", "td", "span", "strong", "small", "cite", "blockquote", "output"}
PROTECTED = {"code", "kbd", "pre", "script", "style", "svg", "math"}
ATTRIBUTES = {"alt", "aria-label", "aria-description", "title", "placeholder"}
PLACEHOLDER = re.compile(r"\{([a-zA-Z][a-zA-Z0-9_]*)\}")
LITERALS = {"REA", "Reverse Engineer Anything", "REA · Reverse Engineer Anything", "GitHub", "Discord", "FAQ", "MCP", "CLI", "CSV", "JSON", "HTML", "CSS", "JavaScript", "Electron", "Hopper", "Ghidra", "IDA", "Windows", "Linux", "macOS", "Notion", "DX-Ball", "TH04", "TH08", "TH095", "Aegis", "SHA-1", "HMAC", "Node.js", "npm", "npx", "DOS", "PC-98"}
FALLBACK_NOTICE = "Translation is being updated. Read this page in English for now."
def languages(root):
"""Load the website language inventory and check it against README editions."""
path = root / "website/locales/languages.json"
if not path.exists():
if any(root.glob("README_*.md")):
raise ValueError("Missing languages.json for the translated README editions")
return [{"code": "en", "name": "English", "readme": "README.md", "dir": "ltr"}]
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, list) or not data or not isinstance(data[0], dict) or data[0].get("code") != "en":
raise ValueError("languages.json must start with English")
codes, filenames = set(), set()
for item in data:
if not isinstance(item, dict) or set(item) != {"code", "name", "readme", "dir"}:
raise ValueError("Each language needs code, name, readme and dir")
if not all(isinstance(value, str) and value for value in item.values()):
raise ValueError("Language fields must be nonempty strings")
if not re.fullmatch(r"[a-z]{2}(?:-[A-Z]{2})?", item["code"]) or item["code"] in codes:
raise ValueError(f"Invalid or duplicate language: {item['code']}")
if item["dir"] not in ("ltr", "rtl"):
raise ValueError(f"Invalid text direction: {item['code']}")
if item["readme"] in filenames:
raise ValueError(f"Duplicate README edition: {item['readme']}")
codes.add(item["code"])
filenames.add(item["readme"])
readmes = {path.name for path in root.glob("README*.md")}
if readmes and ({item["readme"] for item in data} != readmes):
raise ValueError("Website languages must match the README language editions")
return data
def reading_pages(site):
"""Discover authored reading pages, leaving target applications untouched."""
from website import NOTES_FIXTURES, PageMetadata
pages = []
for path in sorted(site.rglob("*.html")):
if path.relative_to(site).as_posix() in NOTES_FIXTURES:
continue
metadata = PageMetadata()
metadata.feed(path.read_text(encoding="utf-8"))
if not metadata.noindex:
pages.append(path)
return pages
def localized_path(relative, code):
"""Keep English's existing routes and prefix other editions with their language."""
return Path(relative) if code == "en" else Path(code) / relative
def language_markup(root, relative, code="en", available=None):
"""Render a keyboard-accessible same-page selector and reciprocal hreflang links."""
from website import SITE_ORIGIN, page_route
inventory = languages(root)
available = set(available) if available is not None else {item["code"] for item in inventory}
current = next(item for item in inventory if item["code"] == code)
route = page_route(Path("."), Path(".") / relative)
depth = len(localized_path(relative, code).parent.parts)
base = "../" * depth or "./"
choices, alternates = [], []
for item in inventory:
target = ("" if item["code"] == "en" else item["code"] + "/") + route
href = escape(base + target, quote=True)
active = ' aria-current="true"' if item["code"] == code else ""
fallback = " · English" if item["code"] not in available else ""
choices.append(f'<a href="{href}" lang="{item["code"]}" dir="{item["dir"]}" data-language="{item["code"]}"{active}>{escape(item["name"])}{fallback}</a>')
if item["code"] in available:
alternates.append(f'<!-- prettier-ignore -->\n<link rel="alternate" hreflang="{item["code"]}" href="{SITE_ORIGIN}/{quote(target, safe="/")}" />')
alternates.append(f'<!-- prettier-ignore -->\n<link rel="alternate" hreflang="x-default" href="{SITE_ORIGIN}/{quote(route, safe="/")}" />')
selector = '<details class="language-picker" data-i18n-skip><summary><span aria-hidden="true">文 / A</span> <span>' + escape(current["name"]) + '</span></summary><nav aria-label="' + escape(current["name"], quote=True) + '">' + "".join(choices) + "</nav></details>"
return selector, "\n".join(alternates)
class Node:
"""Keep exact source spans so translating prose cannot reserialize code blocks."""
def __init__(self, tag, attrs, start, open_end, line):
self.tag, self.attrs, self.start, self.open_end = tag, dict(attrs), start, open_end
self.end_start, self.end, self.line = open_end, open_end, line
self.children = []
self.parent = None
class Document(HTMLParser):
"""Parse structure while retaining original start tags, text and source positions."""
def __init__(self, source):
super().__init__(convert_charrefs=False)
self.source = source
self.offsets = [0]
for match in re.finditer("\n", source):
self.offsets.append(match.end())
self.root = Node("document", [], 0, 0, 1)
self.root.end = self.root.end_start = len(source)
self.stack, self.nodes = [self.root], []
self.feed(source)
self.close()
if len(self.stack) != 1:
raise ValueError(f"Unclosed HTML element: {self.stack[-1].tag}")
def position(self):
line, column = self.getpos()
return self.offsets[line - 1] + column
def handle_starttag(self, tag, attrs):
start = self.position()
node = Node(tag, attrs, start, start + len(self.get_starttag_text()), self.getpos()[0])
node.parent = self.stack[-1]
node.parent.children.append(node)
self.nodes.append(node)
if tag not in VOID:
self.stack.append(node)
def handle_startendtag(self, tag, attrs):
self.handle_starttag(tag, attrs)
if tag not in VOID:
self.stack.pop()
def handle_endtag(self, tag):
if len(self.stack) < 2 or self.stack[-1].tag != tag:
raise ValueError(f"Unbalanced HTML at line {self.getpos()[0]}: </{tag}>")
node = self.stack.pop()
node.end_start = self.position()
node.end = self.source.index(">", node.end_start) + 1
def handle_data(self, data):
self.stack[-1].children.append((self.position(), self.position() + len(data)))
def handle_entityref(self, name):
self.handle_data("&" + name + ";")
def handle_charref(self, name):
self.handle_data("&#" + name + ";")
def skipped(node):
"""Respect protected evidence and explicit exclusions throughout a subtree."""
while node:
if node.tag in PROTECTED or "data-i18n-skip" in node.attrs or node.attrs.get("translate") == "no":
return True
node = node.parent
return False
def translatable(text):
"""Leave identifiers, product names and numeric evidence as literal source."""
literal = re.fullmatch(r"0x[0-9a-fA-F]+|code\[\d+\]\s*=\s*\d+|× pan_scale", text)
return bool(re.search(r"[a-zA-Z]", text)) and text not in LITERALS and not literal and not re.fullmatch(r"[\w./\\:-]+\.(?:exe|dll|js|json|html|asar|apk|zip|cpp|h|svg)", text, flags=re.IGNORECASE)
def inline(node):
return all(not isinstance(child, Node) or (child.tag in INLINE and inline(child)) for child in node.children)
def unit_text(document, node):
"""Represent inline markup and code as named placeholders in a complete sentence."""
slots, parts = {}, []
def slot(value):
key = f"ph{len(slots)}"
slots[key] = value
return "{" + key + "}"
def visit(child):
if not isinstance(child, Node):
parts.append(unescape(document.source[child[0]:child[1]]))
elif child.tag in {"code", "kbd"} or "data-i18n-skip" in child.attrs or child.attrs.get("translate") == "no":
parts.append(slot(document.source[child.start:child.end]))
else:
parts.append(slot(document.source[child.start:child.open_end]))
for descendant in child.children:
visit(descendant)
if child.end > child.end_start:
parts.append(slot(document.source[child.end_start:child.end]))
for child in node.children:
visit(child)
return re.sub(r"\s+", " ", "".join(parts)).strip(), slots
def prose_units(source, path):
"""Extract paragraph-sized units, or individual orphan text when structure requires it."""
document = Document(source)
units = []
def context(start):
for name in ("sharing", "header", "footer"):
begin, end = source.find(f"<!-- website:{name}:start -->"), source.find(f"<!-- website:{name}:end -->")
if begin >= 0 and begin < start < end:
return "shared"
return "page:" + path
def visit(node):
if skipped(node) or node.tag in {"head", "document"}:
if node.tag == "document":
for child in node.children:
if isinstance(child, Node):
visit(child)
return
if node.tag in PROSE and inline(node) and node.children:
text, slots = unit_text(document, node)
if translatable(PLACEHOLDER.sub("", text).strip()):
units.append((context(node.start), text, slots, node.open_end, node.end_start, node.line))
return
for child in node.children:
if isinstance(child, Node):
visit(child)
else:
raw = source[child[0]:child[1]]
text = re.sub(r"\s+", " ", unescape(raw)).strip()
if translatable(text):
units.append((context(child[0]), text, {}, child[0], child[1], source.count("\n", 0, child[0]) + 1))
visit(document.root)
# The title is prose even though it lives in the head.
for node in document.nodes:
if node.tag == "title" and not skipped(node):
text, slots = unit_text(document, node)
if translatable(text):
units.append(("page:" + path, text, slots, node.open_end, node.end_start, node.line))
return units
def attribute_units(source, path):
"""Extract accessibility labels and authored metadata without translating URLs."""
units = []
document = Document(source)
for node in document.nodes:
if skipped(node):
continue
names = set(ATTRIBUTES)
if node.tag == "meta" and (node.attrs.get("name") == "description" or node.attrs.get("property") == "og:image:alt" or node.attrs.get("name") == "twitter:image:alt"):
names.add("content")
for name in names:
text = node.attrs.get(name)
if text and translatable(text):
context = "shared" if "<!-- website:header:start -->" in source[:node.start] and "<!-- website:header:end -->" not in source[:node.start] else "page:" + path
if name == "content" and node.attrs.get("name") != "description":
context = "shared"
units.append((context, text, {}, node, name))
return units
def runtime_units(root):
"""Extract the constrained t("literal", namedValues) authoring form directly."""
yield "fallback", FALLBACK_NOTICE, "scripts/website_i18n.py", ""
site = root / "website/public"
for path in sorted((site / "assets").glob("*.js")):
if path.name == "i18n.js": # Owns t itself, rather than reader-facing calls.
continue
source = path.read_text(encoding="utf-8")
calls = list(re.finditer(r'\bt\(\s*("(?:[^"\\]|\\.)*")', source))
if len(calls) != len(re.findall(r"\bt\(", source)):
raise ValueError(f'{path.name}: use t("one literal message", namedValues)')
script = path.relative_to(site).as_posix()
for call in calls:
yield script, json.loads(call[1]), "website/public/" + script, source.count("\n", 0, call.start()) + 1
def read_runtime(root, page=None):
"""Scope automatically extracted interaction messages to a page's scripts."""
units = list(runtime_units(root))
if page is None:
return list(dict.fromkeys(text for _, text, _, _ in units))
site = root / "website/public"
scripts = set()
for node in Document(page.read_text(encoding="utf-8")).nodes:
if node.tag == "script" and node.attrs.get("src"):
reference = urlsplit(node.attrs["src"])
if not reference.scheme and not reference.netloc:
scripts.add((page.parent / unquote(reference.path)).resolve().relative_to(site.resolve()).as_posix())
return list(dict.fromkeys(text for script, text, _, _ in units if script in scripts))
def source_catalog(root):
"""Build a deterministic POT with source references and protected-placeholder notes."""
site = root / "website/public"
catalog = polib.POFile(wrapwidth=88)
catalog.metadata = {"Project-Id-Version": "REA website", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit"}
entries = {}
def add(context, text, slots, reference, line):
key = context, text
if key not in entries:
entry = polib.POEntry(msgctxt=context, msgid=text, msgstr="", flags=["python-brace-format"] if PLACEHOLDER.search(text) else [])
entry.comment = "\n".join(f"{{{key}}}: {re.sub(r'\s+', ' ', value)}" for key, value in slots.items())
entries[key] = entry
catalog.append(entry)
occurrence = (reference, str(line))
if occurrence not in entries[key].occurrences:
entries[key].occurrences.append(occurrence)
for path in reading_pages(site):
relative = path.relative_to(site).as_posix()
source = path.read_text(encoding="utf-8")
for context, text, slots, _, _, line in prose_units(source, relative):
add(context, text, slots, "website/public/" + relative, line)
for context, text, slots, node, _ in attribute_units(source, relative):
add(context, text, slots, "website/public/" + relative, node.line)
for _, text, reference, line in runtime_units(root):
add("runtime", text, {}, reference, line)
for path in sorted((site / "assets/figures").glob("*.svg")):
source = path.read_text(encoding="utf-8")
for node in Document(source).nodes:
if node.tag in {"text", "title", "desc"} and not any(isinstance(child, Node) for child in node.children):
text = unescape(source[node.open_end:node.end_start]).strip()
if translatable(text):
add("figure:" + path.name, text, {}, "website/public/assets/figures/" + path.name, node.line)
return catalog
def extract_catalogs(root, check=False):
"""Refresh standard catalogs, preserving translations/comments and archiving removed text."""
inventory = languages(root)
template = source_catalog(root)
directory = root / "website/locales"
target = directory / "messages.pot"
changed = []
if not target.exists() or target.read_text(encoding="utf-8") != str(template):
changed.append(target)
if not check:
template.save(target)
expected = {(entry.msgctxt, entry.msgid) for entry in template}
for language in inventory[1:]:
path = directory / language["code"] / "messages.po"
catalog = polib.pofile(path, encoding="utf-8") if path.exists() else polib.POFile(wrapwidth=88)
active = {(entry.msgctxt, entry.msgid) for entry in catalog if not entry.obsolete}
if active != expected:
changed.append(path)
if not check:
catalog.merge(template)
catalog.metadata.update(template.metadata)
catalog.metadata["Language"] = language["code"]
path.parent.mkdir(parents=True, exist_ok=True)
catalog.save(path)
if check and changed:
raise ValueError("Translation sources changed: " + ", ".join(str(path.relative_to(root)) for path in changed[:8]) + ". Run python3 scripts/website.py translations extract, then update the affected msgstr entries.")
print(f"{'Checked' if check else 'Extracted'} {len(template)} source messages for {len(inventory) - 1} translation catalogs.")
return template
def fields(text):
"""Validate named format fields without accepting conversion or format instructions."""
result = []
for _, name, specification, conversion in Formatter().parse(text):
if name is not None:
if not re.fullmatch(r"[a-zA-Z][a-zA-Z0-9_]*", name) or specification or conversion:
raise ValueError(f"Unsupported translation placeholder: {name}")
result.append(name)
return Counter(result)
def checked_catalog(root, language, template, strict=False):
"""Reject malformed catalogs and report missing or fuzzy messages independently."""
path = root / "website/locales" / language["code"] / "messages.po"
catalog = polib.pofile(path, encoding="utf-8", check_for_duplicates=True)
if catalog.metadata.get("Language") != language["code"]:
raise ValueError(f"{path}: wrong Language header")
messages, missing = {}, []
source = {(entry.msgctxt, entry.msgid): entry for entry in template}
for entry in catalog:
if entry.obsolete:
continue
key = entry.msgctxt, entry.msgid
if key not in source:
raise ValueError(f"{path}: obsolete source still active: {entry.msgid[:80]}")
if entry.msgid_plural or entry.msgstr_plural:
raise ValueError(f"{path}: use the declared named-count messages, not undeclared plural forms")
if not entry.msgstr.strip() or "fuzzy" in entry.flags:
missing.append(key)
continue
if fields(entry.msgid) != fields(entry.msgstr):
raise ValueError(f"{path}: translation changed placeholders: {entry.msgid[:100]}")
slots = dict(re.findall(r"^\{(ph\d+)\}: (.*)$", source[key].comment, re.MULTILINE))
if slots:
fragment = escape(entry.msgstr, quote=False)
for name, value in slots.items():
fragment = fragment.replace("{" + name + "}", value)
try:
Document("<div>" + fragment + "</div>")
except ValueError as error:
raise ValueError(f"{path}: {entry.msgctxt}: invalid inline markup in {entry.msgid[:100]!r}: {error}") from error
messages[key] = entry.msgstr.strip()
missing.extend(key for key in source if key not in messages and key not in missing)
if strict and missing:
raise ValueError(f"{path}: {len(missing)} untranslated/fuzzy messages; first: {missing[0][1][:100]}")
return messages, missing
def replace_attribute(raw, name, value):
"""Change one parsed start-tag attribute without modifying adjacent markup."""
pattern = re.compile(r"(\s" + re.escape(name) + r"\s*=\s*)([\"'])(.*?)\2", re.DOTALL | re.IGNORECASE)
updated, count = pattern.subn(lambda match: match[1] + '"' + escape(value, quote=True) + '"', raw)
if count != 1:
raise ValueError(f"Expected one quoted {name} attribute: {raw[:100]}")
return updated
def apply_edits(source, edits):
"""Apply non-overlapping spans from the end, preserving untouched evidence byte-for-byte."""
end = len(source)
for start, stop, replacement in sorted(edits, reverse=True):
if stop > end:
raise ValueError("Overlapping translation spans")
source = source[:start] + replacement + source[stop:]
end = start
return source
def render_prose(source, relative, messages):
"""Translate complete units and restore validated original inline markup."""
edits = []
for context, text, slots, start, end, _ in prose_units(source, relative):
translated = messages[(context, text)]
if fields(text) != fields(translated):
raise ValueError(f"{relative}: changed placeholders")
rendered = escape(translated, quote=False)
for key, value in slots.items():
rendered = rendered.replace("{" + key + "}", value)
# Check translator-reordered markup before accepting it into the document.
try:
Document("<div>" + rendered + "</div>")
except ValueError as error:
raise ValueError(f"{relative}: invalid translated markup in {text[:100]!r}: {error}") from error
raw = source[start:end]
leading = re.match(r"\s*", raw)[0]
trailing = re.search(r"\s*$", raw)[0]
edits.append((start, end, leading + rendered + trailing))
return apply_edits(source, edits)
def relative_reference(reference, source_path, output_path, site, available, code, figures, language_link=False):
"""Resolve links once, keeping assets shared and reading links in the selected edition."""
parsed = urlsplit(reference)
if parsed.scheme or parsed.netloc or not parsed.path:
return reference
target = (source_path.parent / unquote(parsed.path)).resolve()
if not target.is_relative_to(site.resolve()):
raise ValueError(f"Reference leaves the website: {reference}")
relative = target.relative_to(site.resolve())
lookup = relative / "index.html" if target.is_dir() else relative
if not language_link and lookup.as_posix() in available and code != "en":
relative = Path(code) / relative
if relative.as_posix() in figures and code != "en":
relative = Path("assets/localized") / code / relative.relative_to("assets")
destination = os.path.relpath(site / relative, output_path.parent).replace(os.sep, "/")
if parsed.path.endswith("/") and not destination.endswith("/"):
destination += "/"
return urlunsplit(("", "", quote(destination, safe="/"), parsed.query, parsed.fragment))
def replace_language_markup(source, root, relative, code, available):
"""Derive selectors and alternates from the same page-level completeness decision."""
selector, alternates = language_markup(root, relative, code, available)
document = Document(source)
picker = next(node for node in document.nodes if node.tag == "details" and node.attrs.get("class") == "language-picker")
source = apply_edits(source, [(picker.start, picker.end, selector)])
pattern = r"(<!-- website:languages:start -->).*?(<!-- website:languages:end -->)"
return re.sub(pattern, lambda match: match[1] + "\n" + alternates + "\n" + match[2], source, flags=re.DOTALL)
def render_page(root, source_path, language, messages, available, figures, page_languages):
"""Create one complete translation, preserving routes, code, IDs and reference data."""
from website import SITE_ORIGIN, PageMetadata, page_url
site = root / "website/public"
relative = source_path.relative_to(site).as_posix()
code = language["code"]
output_path = site / localized_path(relative, code)
source = source_path.read_text(encoding="utf-8")
source = render_prose(source, relative, messages)
metadata = PageMetadata()
# Translate attributes separately; units inside inline markup keep their source tags.
replacements = {}
for context, text, _, node, attribute in attribute_units(source, relative):
raw = replacements.get(node.start, source[node.start:node.open_end])
replacements[node.start] = replace_attribute(raw, attribute, messages[(context, text)])
source = apply_edits(source, [(node.start, node.open_end, replacements[node.start]) for node in Document(source).nodes if node.start in replacements])
metadata.feed(source)
canonical = page_url(site, output_path)
replacements = []
for node in Document(source).nodes:
raw = source[node.start:node.open_end]
if node.tag == "html":
raw = replace_attribute(raw, "lang", code)
raw = raw[:-1] + f' dir="{language["dir"]}">'
if node.tag == "meta":
key = node.attrs.get("property") or node.attrs.get("name")
if key in {"og:title", "twitter:title"}:
raw = replace_attribute(raw, "content", metadata.titles[0])
elif key in {"og:description", "twitter:description"}:
raw = replace_attribute(raw, "content", metadata.descriptions[0])
elif key == "og:url":
raw = replace_attribute(raw, "content", canonical)
for attribute in ("href", "src"):
if attribute not in node.attrs:
continue
reference = node.attrs[attribute]
if node.tag == "link" and node.attrs.get("rel") == "canonical":
value = canonical
else:
value = relative_reference(reference, source_path, output_path, site, available, code, figures, "data-language" in node.attrs)
raw = replace_attribute(raw, attribute, value)
if raw != source[node.start:node.open_end]:
replacements.append((node.start, node.open_end, raw))
source = apply_edits(source, replacements)
source = replace_language_markup(source, root, relative, code, page_languages)
runtime = {text: messages[("runtime", text)] for text in read_runtime(root, source_path)}
payload = json.dumps(runtime, ensure_ascii=False).replace("<", "\\u003c").replace("&", "\\u0026")
source = source.replace("</head>", f'<script id="website-messages" type="application/json">{payload}</script>\n </head>')
return source
def required_messages(root, path, template):
"""Bind a page to its prose, attributes, runtime labels and referenced figures."""
site = root / "website/public"
relative = path.relative_to(site).as_posix()
source = path.read_text(encoding="utf-8")
required = {(unit[0], unit[1]) for unit in prose_units(source, relative)}
required.update((unit[0], unit[1]) for unit in attribute_units(source, relative))
required.update(("runtime", text) for text in read_runtime(root, path))
for node in Document(source).nodes:
reference = node.attrs.get("src")
if reference and not urlsplit(reference).scheme:
target = (path.parent / unquote(urlsplit(reference).path)).resolve()
if target.parent == (site / "assets/figures").resolve():
required.update((entry.msgctxt, entry.msgid) for entry in template if entry.msgctxt == "figure:" + target.name)
return required
def render_fallback(root, path, language, page_languages, messages):
"""Expose a clear English fallback instead of a stale or partially translated article."""
from website import page_url
site = root / "website/public"
relative = path.relative_to(site).as_posix()
code = language["code"]
output_path = site / localized_path(relative, code)
source = path.read_text(encoding="utf-8")
edits = []
canonical = page_url(site, output_path)
for node in Document(source).nodes:
raw = source[node.start:node.open_end]
for attribute in ("href", "src"):
if attribute in node.attrs:
value = canonical if node.tag == "link" and node.attrs.get("rel") == "canonical" else relative_reference(node.attrs[attribute], path, output_path, site, set(), code, set(), "data-language" in node.attrs)
raw = replace_attribute(raw, attribute, value)
if node.tag == "meta" and node.attrs.get("property") == "og:url":
raw = replace_attribute(raw, "content", canonical)
if node.tag == "html":
raw = raw[:-1] + ' data-translation-fallback="true">'
if raw != source[node.start:node.open_end]:
edits.append((node.start, node.open_end, raw))
source = apply_edits(source, edits)
source = source.replace("</head>", '<meta name="robots" content="noindex, follow" />\n </head>')
source = replace_language_markup(source, root, relative, code, page_languages)
notice = messages.get(("runtime", FALLBACK_NOTICE), FALLBACK_NOTICE)
notice_lang = code if ("runtime", FALLBACK_NOTICE) in messages else "en"
main = next(node for node in Document(source).nodes if node.tag == "main")
source = source[:main.open_end] + f'<aside class="container translation-notice" lang="{notice_lang}" dir="{language["dir"]}">{escape(notice)}</aside>' + source[main.open_end:]
return source
def build_translations(root, destination, finalize=None):
"""Build one clean publish directory from English sources and versioned PO catalogs."""
site = root / "website/public"
if destination.resolve() != (root / "website/_site").resolve():
raise ValueError("Generated output must be website/_site")
template = extract_catalogs(root, check=True)
inventory = languages(root)
pages = reading_pages(site)
catalogs = {language["code"]: checked_catalog(root, language, template)[0] for language in inventory[1:]}
required = {path.relative_to(site).as_posix(): required_messages(root, path, template) for path in pages}
page_languages = {relative: {"en"} | {code for code, messages in catalogs.items() if keys <= messages.keys()} for relative, keys in required.items()}
# Finish rendering before replacing the previous preview/publish artifact.
with TemporaryDirectory(prefix=".i18n-", dir=root / "website") as directory:
staging = Path(directory) / "site"
shutil.copytree(site, staging, ignore=shutil.ignore_patterns("__pycache__"))
for path in pages:
relative = path.relative_to(site).as_posix()
output = staging / relative
output.write_text(replace_language_markup(path.read_text(encoding="utf-8"), root, relative, "en", page_languages[relative]), encoding="utf-8")
for language in inventory[1:]:
messages = catalogs[language["code"]]
available = {relative for relative, codes in page_languages.items() if language["code"] in codes}
figures = set()
for path in sorted((site / "assets/figures").glob("*.svg")):
source = path.read_text(encoding="utf-8")
edits = []
for node in Document(source).nodes:
if node.tag in {"text", "title", "desc"} and not any(isinstance(child, Node) for child in node.children):
text = unescape(source[node.open_end:node.end_start]).strip()
if translatable(text) and ("figure:" + path.name, text) in messages:
translated = messages[("figure:" + path.name, text)]
edits.append((node.open_end, node.end_start, escape(translated, quote=False)))
translated = apply_edits(source, edits)
output = staging / "assets/localized" / language["code"] / path.relative_to(site / "assets")
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(translated, encoding="utf-8")
figures.add(path.relative_to(site).as_posix())
for path in pages:
relative = path.relative_to(site).as_posix()
output = staging / localized_path(relative, language["code"])
output.parent.mkdir(parents=True, exist_ok=True)
rendered = render_page(root, path, language, messages, available, figures, page_languages[relative]) if relative in available else render_fallback(root, path, language, page_languages[relative], messages)
output.write_text(rendered, encoding="utf-8")
fallback_count = len(pages) - len(available)
if fallback_count:
print(f"{language['code']}: {fallback_count} English fallback pages, marked noindex; run translations status.")
print(f"Prepared {len(pages) * len(inventory)} reading pages in {len(inventory)} languages.")
if finalize is not None:
finalize(staging)
if destination.exists():
shutil.rmtree(destination)
staging.rename(destination)