feat(website): add multilingual editions and translation workflow
This commit is contained in:
@@ -123,6 +123,7 @@ if (
|
||||
file.startsWith("docs/")
|
||||
) {
|
||||
selected.docs = true;
|
||||
selected.website ||= matchesAny(file, SCOPES.website);
|
||||
continue;
|
||||
}
|
||||
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
|
||||
|
||||
@@ -222,6 +222,8 @@ export const SCOPES = {
|
||||
website: [
|
||||
"website/**",
|
||||
"scripts/*website*.py",
|
||||
"README*.md",
|
||||
".github/workflows/website-check.yml",
|
||||
".github/workflows/website-pages.yml",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -76,9 +76,14 @@ def prepare_notes_example(site):
|
||||
print(f"Prepared examples/notes-example.zip ({len(EXAMPLE_FILES)} files).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
site = root / "website/public"
|
||||
def prepare_assets(site):
|
||||
"""Finish the shared publish artifact before it replaces the current preview."""
|
||||
prepare_sharing_image(site)
|
||||
prepare_sitemap(site)
|
||||
prepare_notes_example(site)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
from website_i18n import build_translations
|
||||
build_translations(root, root / "website/_site", finalize=prepare_assets)
|
||||
|
||||
+163
-3
@@ -3,6 +3,7 @@
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
@@ -13,6 +14,9 @@ from urllib.error import HTTPError
|
||||
from urllib.request import urlopen
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import polib
|
||||
from website_i18n import Document, build_translations, checked_catalog, extract_catalogs, languages, reading_pages, read_runtime
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
@@ -113,7 +117,7 @@ class WebsiteSearchChecks(unittest.TestCase):
|
||||
for name, old, new, message in cases:
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as directory:
|
||||
site = Path(directory) / "public"
|
||||
shutil.copytree(ROOT / "website/public", site)
|
||||
shutil.copytree(ROOT / "website/_site", site)
|
||||
path = site / name
|
||||
source = path.read_text()
|
||||
self.assertIn(old, source)
|
||||
@@ -123,7 +127,7 @@ class WebsiteSearchChecks(unittest.TestCase):
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
site = Path(directory) / "public"
|
||||
shutil.copytree(ROOT / "website/public", site)
|
||||
shutil.copytree(ROOT / "website/_site", site)
|
||||
(site / "assets/social-card.png").unlink()
|
||||
self.assertIn(
|
||||
"Social preview PNG is missing; run scripts/prepare-website.py",
|
||||
@@ -162,6 +166,9 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
for name in ("public", "templates"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
site = root / "website/public"
|
||||
WEBSITE.sync_pages(root)
|
||||
shutil.copyfile(ROOT / "website/_site/assets/social-card.png", site / "assets/social-card.png")
|
||||
PREPARE.prepare_notes_example(site)
|
||||
page = WEBSITE.new_page(root, "blog/new-example", 'A <test> & "result"', 'Check "quotes" & <markup>.')
|
||||
source = page.read_text()
|
||||
self.assertIn('href="../../blog/" aria-current="page"', source)
|
||||
@@ -182,7 +189,7 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
def test_links_in_prompts_fragments_and_copy_controls_are_verified(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
site = Path(directory) / "public"
|
||||
shutil.copytree(ROOT / "website/public", site)
|
||||
shutil.copytree(ROOT / "website/_site", site)
|
||||
page = site / "faq/index.html"
|
||||
original = page.read_text()
|
||||
cases = (
|
||||
@@ -242,5 +249,158 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
worker.join()
|
||||
|
||||
|
||||
class WebsiteTranslationChecks(unittest.TestCase):
|
||||
"""Exercise source edits, translated evidence and language editions at their consumers."""
|
||||
|
||||
@contextlib.contextmanager
|
||||
def translated_project(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ("public", "templates", "locales"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
inventory = [item for item in languages(ROOT) if item["code"] in {"en", "zh-CN", "ar"}]
|
||||
(root / "website/locales/languages.json").write_text(json.dumps(inventory))
|
||||
for item in inventory:
|
||||
(root / item["readme"]).write_text("# REA\n")
|
||||
WEBSITE.sync_pages(root)
|
||||
extract_catalogs(root)
|
||||
yield root
|
||||
|
||||
def test_source_change_preserves_review_and_excludes_stale_translation_from_search(self):
|
||||
with self.translated_project() as root:
|
||||
path = root / "website/locales/zh-CN/messages.po"
|
||||
catalog = polib.pofile(path)
|
||||
old = catalog.find("How do I install REA?", msgctxt="page:faq/index.html")
|
||||
old.tcomment = "Reviewed installation wording."
|
||||
old_translation = old.msgstr
|
||||
catalog.save(path)
|
||||
page = root / "website/public/faq/index.html"
|
||||
page.write_text(page.read_text().replace("How do I install REA?", "How do I install REA today?"))
|
||||
with self.assertRaisesRegex(ValueError, "Translation sources changed"):
|
||||
extract_catalogs(root, check=True)
|
||||
extract_catalogs(root)
|
||||
updated = polib.pofile(path)
|
||||
archived = next(entry for entry in updated if entry.obsolete and entry.msgid == old.msgid)
|
||||
self.assertEqual((archived.msgstr, archived.tcomment), (old_translation, old.tcomment))
|
||||
new = updated.find("How do I install REA today?", msgctxt=old.msgctxt)
|
||||
self.assertEqual(new.msgstr, "")
|
||||
build_translations(root, root / "website/_site")
|
||||
site = root / "website/_site"
|
||||
PREPARE.prepare_sitemap(site)
|
||||
fallback = VERIFY.HtmlReferences()
|
||||
fallback.feed((site / "zh-CN/faq/index.html").read_text())
|
||||
self.assertTrue(fallback.translation_fallback)
|
||||
self.assertTrue(fallback.noindex())
|
||||
self.assertEqual(fallback.lang, "en")
|
||||
self.assertNotIn("https://rea.tools/zh-CN/faq/", (site / "sitemap.xml").read_text())
|
||||
self.assertNotIn('hreflang="zh-CN"', (site / "faq/index.html").read_text())
|
||||
new.msgstr = "现在如何安装 REA?"
|
||||
new.flags.append("fuzzy")
|
||||
updated.save(path)
|
||||
build_translations(root, site)
|
||||
self.assertIn('data-translation-fallback="true"', (site / "zh-CN/faq/index.html").read_text())
|
||||
new.flags.remove("fuzzy")
|
||||
updated.save(path)
|
||||
build_translations(root, site)
|
||||
PREPARE.prepare_sitemap(site)
|
||||
result = (site / "zh-CN/faq/index.html").read_text()
|
||||
self.assertIn("现在如何安装 REA?", result)
|
||||
self.assertNotIn("data-translation-fallback", result)
|
||||
self.assertIn('href="../assets/styles.css"', (site / "zh-CN/index.html").read_text())
|
||||
self.assertIn('hreflang="zh-CN"', (site / "faq/index.html").read_text())
|
||||
self.assertIn("https://rea.tools/zh-CN/faq/", (site / "sitemap.xml").read_text())
|
||||
|
||||
def test_broken_placeholders_and_inline_markup_fail_before_replacing_preview(self):
|
||||
with self.translated_project() as root:
|
||||
site = root / "website/_site"
|
||||
build_translations(root, site)
|
||||
before = (site / "zh-CN/index.html").read_bytes()
|
||||
path = root / "website/locales/zh-CN/messages.po"
|
||||
catalog = polib.pofile(path)
|
||||
entry = catalog.find("Already know RE? {ph0}Skip to analysis guides →{ph1}", msgctxt="page:index.html")
|
||||
original = entry.msgstr
|
||||
for bad, reason in (("丢失链接", "changed placeholders"), ("{ph1}错误的链接{ph0}", "invalid inline markup")):
|
||||
with self.subTest(reason=reason):
|
||||
entry.msgstr = bad
|
||||
catalog.save(path)
|
||||
with self.assertRaisesRegex(ValueError, reason):
|
||||
build_translations(root, site)
|
||||
self.assertEqual((site / "zh-CN/index.html").read_bytes(), before)
|
||||
entry.msgstr = original
|
||||
catalog.save(path)
|
||||
(root / "website/locales/languages.json").unlink()
|
||||
with self.assertRaisesRegex(ValueError, "Missing languages.json"):
|
||||
build_translations(root, site)
|
||||
self.assertEqual((site / "zh-CN/index.html").read_bytes(), before)
|
||||
|
||||
def test_runtime_message_drift_and_page_specific_completeness(self):
|
||||
with self.translated_project() as root:
|
||||
script = root / "website/public/assets/dino-demo.js"
|
||||
original = script.read_text()
|
||||
script.write_text(original.replace('t("Pause")', 't("Pause game")'))
|
||||
self.assertIn("Pause game", read_runtime(root))
|
||||
with self.assertRaisesRegex(ValueError, "Translation sources changed"):
|
||||
extract_catalogs(root, check=True)
|
||||
script.write_text(original.replace('t("Pause")', 't(dynamicMessage)'))
|
||||
with self.assertRaisesRegex(ValueError, "one literal message"):
|
||||
read_runtime(root)
|
||||
script.write_text(original)
|
||||
path = root / "website/locales/zh-CN/messages.po"
|
||||
catalog = polib.pofile(path)
|
||||
catalog.find("Pause", msgctxt="runtime").msgstr = ""
|
||||
catalog.save(path)
|
||||
site = root / "website/_site"
|
||||
build_translations(root, site)
|
||||
self.assertIn('data-translation-fallback="true"', (site / "zh-CN/examples/dino-lab/index.html").read_text())
|
||||
self.assertNotIn('data-translation-fallback="true"', (site / "zh-CN/faq/index.html").read_text())
|
||||
|
||||
def test_all_readme_editions_preserve_code_targets_downloads_and_reciprocal_routes(self):
|
||||
site = ROOT / "website/_site"
|
||||
english_pages = reading_pages(ROOT / "website/public")
|
||||
for language in languages(ROOT):
|
||||
code = language["code"]
|
||||
for original in english_pages:
|
||||
relative = original.relative_to(ROOT / "website/public")
|
||||
path = site / relative if code == "en" else site / code / relative
|
||||
with self.subTest(language=code, page=str(relative)):
|
||||
source = original.read_text()
|
||||
translated = path.read_text()
|
||||
def evidence(html):
|
||||
doc = Document(html)
|
||||
return sorted(doc.source[node.start:node.end] for node in doc.nodes if node.tag in {"pre", "code", "kbd"} or node.attrs.get("translate") == "no")
|
||||
self.assertEqual(evidence(source), evidence(translated))
|
||||
parser = VERIFY.HtmlReferences()
|
||||
parser.feed(translated)
|
||||
self.assertEqual(parser.lang, "en" if parser.translation_fallback else code)
|
||||
if code != "en" and not parser.translation_fallback:
|
||||
self.assertEqual(parser.direction, language["dir"])
|
||||
if parser.translation_fallback:
|
||||
self.assertTrue(parser.noindex())
|
||||
self.assertNotIn(code, parser.alternates)
|
||||
else:
|
||||
self.assertEqual(parser.alternates[code], WEBSITE.page_url(site, path))
|
||||
english = VERIFY.HtmlReferences()
|
||||
english.feed((site / relative).read_text())
|
||||
self.assertEqual(parser.alternates, english.alternates)
|
||||
self.assertTrue(set(parser.copy_targets) <= parser.ids)
|
||||
for fixture in WEBSITE.NOTES_FIXTURES:
|
||||
self.assertEqual((site / fixture).read_bytes(), (ROOT / "website/public" / fixture).read_bytes())
|
||||
with ThreadingHTTPServer(("127.0.0.1", 0), WEBSITE.preview_handler(site, "/rea/")) as server:
|
||||
worker = Thread(target=server.serve_forever, daemon=True)
|
||||
worker.start()
|
||||
try:
|
||||
base = f"http://127.0.0.1:{server.server_port}/rea/"
|
||||
with urlopen(base + "ar/faq?test=1") as response:
|
||||
self.assertEqual(response.url, base + "ar/faq/?test=1")
|
||||
self.assertIn(b'lang="ar"', response.read())
|
||||
with urlopen(base + "assets/i18n.js") as response:
|
||||
self.assertIn(b"export function t", response.read())
|
||||
with urlopen(base + "examples/notes-example.zip") as response:
|
||||
self.assertEqual(response.read(), (site / "examples/notes-example.zip").read_bytes())
|
||||
finally:
|
||||
server.shutdown()
|
||||
worker.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -95,6 +95,9 @@ class HtmlReferences(HTMLParser):
|
||||
self.properties = {}
|
||||
self.canonicals = []
|
||||
self.lang = None
|
||||
self.direction = None
|
||||
self.translation_fallback = False
|
||||
self.alternates = {}
|
||||
self.h1_count = 0
|
||||
self.text = []
|
||||
self.copy_targets = []
|
||||
@@ -103,11 +106,18 @@ class HtmlReferences(HTMLParser):
|
||||
attributes = dict(attrs)
|
||||
if tag == "html":
|
||||
self.lang = attributes.get("lang")
|
||||
self.direction = attributes.get("dir")
|
||||
self.translation_fallback = attributes.get("data-translation-fallback") == "true"
|
||||
if tag == "h1":
|
||||
self.h1_count += 1
|
||||
if tag == "head":
|
||||
self.in_head = True
|
||||
if self.in_head:
|
||||
if tag == "link" and attributes.get("rel") == "alternate" and attributes.get("hreflang"):
|
||||
language = attributes["hreflang"]
|
||||
if language in self.alternates:
|
||||
self.duplicate_ids.append("hreflang:" + language)
|
||||
self.alternates[language] = attributes.get("href") or ""
|
||||
if tag == "title":
|
||||
self.titles.append("")
|
||||
self.in_title = True
|
||||
@@ -167,8 +177,17 @@ def check_search_metadata(site, pages):
|
||||
title = page.titles[0].strip() if len(page.titles) == 1 else ""
|
||||
if not title:
|
||||
errors.append(f"{label}: needs exactly one nonempty head title")
|
||||
if page.lang != "en" or page.h1_count != 1:
|
||||
errors.append(f"{label}: needs lang=en and exactly one h1")
|
||||
first = path.relative_to(site).parts[0]
|
||||
expected_language = first if re.fullmatch(r"[a-z]{2}(?:-[A-Z]{2})?", first) else "en"
|
||||
if page.translation_fallback:
|
||||
if page.lang != "en" or not page.noindex():
|
||||
errors.append(f"{label}: English translation fallback needs lang=en and noindex")
|
||||
elif page.lang != expected_language:
|
||||
errors.append(f"{label}: needs lang={expected_language} and exactly one h1")
|
||||
if page.h1_count != 1:
|
||||
errors.append(f"{label}: needs exactly one h1")
|
||||
if expected_language in ("ar", "fa") and not page.translation_fallback and page.direction != "rtl":
|
||||
errors.append(f"{label}: needs dir=rtl")
|
||||
if label in NOTES_FIXTURES and not page.noindex():
|
||||
errors.append(
|
||||
f"{label}: example application must have a head noindex directive"
|
||||
@@ -179,9 +198,10 @@ def check_search_metadata(site, pages):
|
||||
indexable.add(url)
|
||||
if page.canonicals != [url]:
|
||||
errors.append(f"{label}: needs exactly one head canonical for {url}")
|
||||
if title in titles:
|
||||
title_identity = (page.lang, title)
|
||||
if title_identity in titles:
|
||||
errors.append(f"{label}: duplicate search title {title!r}")
|
||||
titles.add(title)
|
||||
titles.add(title_identity)
|
||||
descriptions = page.meta.get("description", [])
|
||||
description = descriptions[0].strip() if len(descriptions) == 1 else ""
|
||||
if not description:
|
||||
@@ -219,6 +239,22 @@ def check_search_metadata(site, pages):
|
||||
values = collection.get(key, [])
|
||||
if len(values) != 1 or not values[0].strip():
|
||||
errors.append(f"{label}: needs exactly one nonempty {key}")
|
||||
if page.alternates:
|
||||
if page.alternates.get(page.lang) != url:
|
||||
errors.append(f"{label}: hreflang must include its own canonical")
|
||||
for language, target in page.alternates.items():
|
||||
parsed = urlsplit(target)
|
||||
if parsed.scheme != "https" or parsed.netloc != "rea.tools":
|
||||
errors.append(f"{label}: hreflang needs absolute rea.tools URLs")
|
||||
continue
|
||||
target_path = (site / unquote(parsed.path).lstrip("/")).resolve()
|
||||
if target_path.is_dir():
|
||||
target_path /= "index.html"
|
||||
other = pages.get(target_path)
|
||||
if other is None or other.noindex() or other.alternates != page.alternates:
|
||||
errors.append(f"{label}: nonreciprocal or nonindexable hreflang: {target}")
|
||||
elif language != "x-default" and other.lang != language:
|
||||
errors.append(f"{label}: hreflang={language} points to lang={other.lang}")
|
||||
|
||||
sitemap_path = site / "sitemap.xml"
|
||||
try:
|
||||
@@ -290,7 +326,7 @@ def check_site(site):
|
||||
parser.feed(path.read_text(encoding="utf-8"))
|
||||
pages[path] = parser
|
||||
if site / "index.html" not in pages:
|
||||
errors.append("website/public/index.html is missing")
|
||||
errors.append(f"{site}/index.html is missing; run scripts/prepare-website.py")
|
||||
|
||||
for path, page in pages.items():
|
||||
label = path.relative_to(site)
|
||||
@@ -400,8 +436,8 @@ def check_example_archive(site):
|
||||
def main():
|
||||
"""Run the same checks locally, on website PRs and before publishing."""
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
pages, references, indexable, errors = check_site(root / "website/public")
|
||||
errors.extend(check_example_archive(root / "website/public"))
|
||||
pages, references, indexable, errors = check_site(root / "website/_site")
|
||||
errors.extend(check_example_archive(root / "website/_site"))
|
||||
errors.extend(check_publisher(root))
|
||||
errors.extend(check_repository_links(root))
|
||||
try:
|
||||
|
||||
+23
-2
@@ -31,6 +31,7 @@ def authored_markdown(root):
|
||||
path for path in sorted(site.rglob("*.md"))
|
||||
if not any(part.startswith(".") or part in ("node_modules", "__pycache__")
|
||||
for part in path.relative_to(site).parts)
|
||||
and "_site" not in path.relative_to(site).parts
|
||||
]
|
||||
|
||||
|
||||
@@ -111,6 +112,10 @@ def shared_blocks(root, path, source):
|
||||
"title": escape(metadata.titles[0].strip(), quote=True),
|
||||
"description": escape(metadata.descriptions[0].strip(), quote=True),
|
||||
}
|
||||
from website_i18n import language_markup
|
||||
selector, alternates = language_markup(root, path.relative_to(site).as_posix())
|
||||
values["language_picker"] = selector if not metadata.noindex else ""
|
||||
values["alternates"] = alternates if not metadata.noindex else ""
|
||||
templates = root / "website/templates"
|
||||
return {
|
||||
name: indent(Template((templates / f"{name}.html.tmpl").read_text(encoding="utf-8")).substitute(values), " ")
|
||||
@@ -207,13 +212,29 @@ def main():
|
||||
create.add_argument("route")
|
||||
create.add_argument("--title", required=True)
|
||||
create.add_argument("--description", required=True)
|
||||
translations = commands.add_parser("translations", help="Maintain gettext catalogs")
|
||||
translations.add_argument("action", choices=("extract", "status", "check"))
|
||||
translations.add_argument("--language", help="Check one language, such as zh-CN")
|
||||
translations.add_argument("--strict", action="store_true", help="Fail on missing or fuzzy translations")
|
||||
serve = commands.add_parser("serve", help="Prepare and preview the static site")
|
||||
serve.add_argument("--port", type=int, default=4173)
|
||||
serve.add_argument("--base-path", choices=("/", "/rea/"), default="/")
|
||||
args = parser.parse_args()
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
try:
|
||||
if args.command == "new-page":
|
||||
if args.command == "translations":
|
||||
from website_i18n import checked_catalog, extract_catalogs, languages
|
||||
template = extract_catalogs(root, check=args.action != "extract")
|
||||
if args.action != "extract":
|
||||
inventory = languages(root)[1:]
|
||||
if args.language:
|
||||
inventory = [item for item in inventory if item["code"] == args.language]
|
||||
if not inventory:
|
||||
raise ValueError(f"Unknown translation language: {args.language}")
|
||||
for language in inventory:
|
||||
messages, missing = checked_catalog(root, language, template, strict=args.strict)
|
||||
print(f"{language['code']}: {len(messages)}/{len(template)} translated; {len(missing)} missing or fuzzy")
|
||||
elif args.command == "new-page":
|
||||
new_page(root, args.route, args.title, args.description)
|
||||
elif args.command == "sync":
|
||||
sync_pages(root, check=args.check)
|
||||
@@ -228,7 +249,7 @@ def main():
|
||||
run_script(root, "prepare-website.py")
|
||||
prefix = args.base_path
|
||||
|
||||
handler = preview_handler(root / "website/public", prefix)
|
||||
handler = preview_handler(root / "website/_site", prefix)
|
||||
with ThreadingHTTPServer(("127.0.0.1", args.port), handler) as server:
|
||||
print(f"Preview: http://127.0.0.1:{server.server_port}{prefix}", flush=True)
|
||||
server.serve_forever()
|
||||
|
||||
@@ -0,0 +1,637 @@
|
||||
"""Extract gettext catalogs and render language editions of authored static HTML."""
|
||||
|
||||
from collections import Counter
|
||||
from html import escape, unescape
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
from string import Formatter
|
||||
from tempfile import TemporaryDirectory
|
||||
from urllib.parse import quote, unquote, urlsplit, urlunsplit
|
||||
|
||||
import polib
|
||||
|
||||
|
||||
VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
|
||||
INLINE = {"a", "abbr", "b", "bdi", "br", "cite", "code", "em", "i", "kbd", "mark", "s", "small", "span", "strong", "sub", "sup", "time", "u"}
|
||||
PROSE = {"a", "p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "dt", "dd", "figcaption", "summary", "label", "button", "th", "td", "span", "strong", "small", "cite", "blockquote", "output"}
|
||||
PROTECTED = {"code", "kbd", "pre", "script", "style", "svg", "math"}
|
||||
ATTRIBUTES = {"alt", "aria-label", "aria-description", "title", "placeholder"}
|
||||
PLACEHOLDER = re.compile(r"\{([a-zA-Z][a-zA-Z0-9_]*)\}")
|
||||
LITERALS = {"REA", "Reverse Engineer Anything", "REA · Reverse Engineer Anything", "GitHub", "Discord", "FAQ", "MCP", "CLI", "CSV", "JSON", "HTML", "CSS", "JavaScript", "Electron", "Hopper", "Ghidra", "IDA", "Windows", "Linux", "macOS", "Notion", "DX-Ball", "TH04", "TH08", "TH095", "Aegis", "SHA-1", "HMAC", "Node.js", "npm", "npx", "DOS", "PC-98"}
|
||||
FALLBACK_NOTICE = "Translation is being updated. Read this page in English for now."
|
||||
|
||||
|
||||
def languages(root):
|
||||
"""Load the website language inventory and check it against README editions."""
|
||||
path = root / "website/locales/languages.json"
|
||||
if not path.exists():
|
||||
if any(root.glob("README_*.md")):
|
||||
raise ValueError("Missing languages.json for the translated README editions")
|
||||
return [{"code": "en", "name": "English", "readme": "README.md", "dir": "ltr"}]
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, list) or not data or not isinstance(data[0], dict) or data[0].get("code") != "en":
|
||||
raise ValueError("languages.json must start with English")
|
||||
codes, filenames = set(), set()
|
||||
for item in data:
|
||||
if not isinstance(item, dict) or set(item) != {"code", "name", "readme", "dir"}:
|
||||
raise ValueError("Each language needs code, name, readme and dir")
|
||||
if not all(isinstance(value, str) and value for value in item.values()):
|
||||
raise ValueError("Language fields must be nonempty strings")
|
||||
if not re.fullmatch(r"[a-z]{2}(?:-[A-Z]{2})?", item["code"]) or item["code"] in codes:
|
||||
raise ValueError(f"Invalid or duplicate language: {item['code']}")
|
||||
if item["dir"] not in ("ltr", "rtl"):
|
||||
raise ValueError(f"Invalid text direction: {item['code']}")
|
||||
if item["readme"] in filenames:
|
||||
raise ValueError(f"Duplicate README edition: {item['readme']}")
|
||||
codes.add(item["code"])
|
||||
filenames.add(item["readme"])
|
||||
readmes = {path.name for path in root.glob("README*.md")}
|
||||
if readmes and ({item["readme"] for item in data} != readmes):
|
||||
raise ValueError("Website languages must match the README language editions")
|
||||
return data
|
||||
|
||||
|
||||
def reading_pages(site):
|
||||
"""Discover authored reading pages, leaving target applications untouched."""
|
||||
from website import NOTES_FIXTURES, PageMetadata
|
||||
pages = []
|
||||
for path in sorted(site.rglob("*.html")):
|
||||
if path.relative_to(site).as_posix() in NOTES_FIXTURES:
|
||||
continue
|
||||
metadata = PageMetadata()
|
||||
metadata.feed(path.read_text(encoding="utf-8"))
|
||||
if not metadata.noindex:
|
||||
pages.append(path)
|
||||
return pages
|
||||
|
||||
|
||||
def localized_path(relative, code):
|
||||
"""Keep English's existing routes and prefix other editions with their language."""
|
||||
return Path(relative) if code == "en" else Path(code) / relative
|
||||
|
||||
|
||||
def language_markup(root, relative, code="en", available=None):
|
||||
"""Render a keyboard-accessible same-page selector and reciprocal hreflang links."""
|
||||
from website import SITE_ORIGIN, page_route
|
||||
inventory = languages(root)
|
||||
available = set(available) if available is not None else {item["code"] for item in inventory}
|
||||
current = next(item for item in inventory if item["code"] == code)
|
||||
route = page_route(Path("."), Path(".") / relative)
|
||||
depth = len(localized_path(relative, code).parent.parts)
|
||||
base = "../" * depth or "./"
|
||||
choices, alternates = [], []
|
||||
for item in inventory:
|
||||
target = ("" if item["code"] == "en" else item["code"] + "/") + route
|
||||
href = escape(base + target, quote=True)
|
||||
active = ' aria-current="true"' if item["code"] == code else ""
|
||||
fallback = " · English" if item["code"] not in available else ""
|
||||
choices.append(f'<a href="{href}" lang="{item["code"]}" dir="{item["dir"]}" data-language="{item["code"]}"{active}>{escape(item["name"])}{fallback}</a>')
|
||||
if item["code"] in available:
|
||||
alternates.append(f'<!-- prettier-ignore -->\n<link rel="alternate" hreflang="{item["code"]}" href="{SITE_ORIGIN}/{quote(target, safe="/")}" />')
|
||||
alternates.append(f'<!-- prettier-ignore -->\n<link rel="alternate" hreflang="x-default" href="{SITE_ORIGIN}/{quote(route, safe="/")}" />')
|
||||
selector = '<details class="language-picker" data-i18n-skip><summary><span aria-hidden="true">文 / A</span> <span>' + escape(current["name"]) + '</span></summary><nav aria-label="' + escape(current["name"], quote=True) + '">' + "".join(choices) + "</nav></details>"
|
||||
return selector, "\n".join(alternates)
|
||||
|
||||
|
||||
class Node:
|
||||
"""Keep exact source spans so translating prose cannot reserialize code blocks."""
|
||||
|
||||
def __init__(self, tag, attrs, start, open_end, line):
|
||||
self.tag, self.attrs, self.start, self.open_end = tag, dict(attrs), start, open_end
|
||||
self.end_start, self.end, self.line = open_end, open_end, line
|
||||
self.children = []
|
||||
self.parent = None
|
||||
|
||||
|
||||
class Document(HTMLParser):
|
||||
"""Parse structure while retaining original start tags, text and source positions."""
|
||||
|
||||
def __init__(self, source):
|
||||
super().__init__(convert_charrefs=False)
|
||||
self.source = source
|
||||
self.offsets = [0]
|
||||
for match in re.finditer("\n", source):
|
||||
self.offsets.append(match.end())
|
||||
self.root = Node("document", [], 0, 0, 1)
|
||||
self.root.end = self.root.end_start = len(source)
|
||||
self.stack, self.nodes = [self.root], []
|
||||
self.feed(source)
|
||||
self.close()
|
||||
if len(self.stack) != 1:
|
||||
raise ValueError(f"Unclosed HTML element: {self.stack[-1].tag}")
|
||||
|
||||
def position(self):
|
||||
line, column = self.getpos()
|
||||
return self.offsets[line - 1] + column
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
start = self.position()
|
||||
node = Node(tag, attrs, start, start + len(self.get_starttag_text()), self.getpos()[0])
|
||||
node.parent = self.stack[-1]
|
||||
node.parent.children.append(node)
|
||||
self.nodes.append(node)
|
||||
if tag not in VOID:
|
||||
self.stack.append(node)
|
||||
|
||||
def handle_startendtag(self, tag, attrs):
|
||||
self.handle_starttag(tag, attrs)
|
||||
if tag not in VOID:
|
||||
self.stack.pop()
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if len(self.stack) < 2 or self.stack[-1].tag != tag:
|
||||
raise ValueError(f"Unbalanced HTML at line {self.getpos()[0]}: </{tag}>")
|
||||
node = self.stack.pop()
|
||||
node.end_start = self.position()
|
||||
node.end = self.source.index(">", node.end_start) + 1
|
||||
|
||||
def handle_data(self, data):
|
||||
self.stack[-1].children.append((self.position(), self.position() + len(data)))
|
||||
|
||||
def handle_entityref(self, name):
|
||||
self.handle_data("&" + name + ";")
|
||||
|
||||
def handle_charref(self, name):
|
||||
self.handle_data("&#" + name + ";")
|
||||
|
||||
|
||||
def skipped(node):
|
||||
"""Respect protected evidence and explicit exclusions throughout a subtree."""
|
||||
while node:
|
||||
if node.tag in PROTECTED or "data-i18n-skip" in node.attrs or node.attrs.get("translate") == "no":
|
||||
return True
|
||||
node = node.parent
|
||||
return False
|
||||
|
||||
|
||||
def translatable(text):
|
||||
"""Leave identifiers, product names and numeric evidence as literal source."""
|
||||
literal = re.fullmatch(r"0x[0-9a-fA-F]+|code\[\d+\]\s*=\s*\d+|× pan_scale", text)
|
||||
return bool(re.search(r"[a-zA-Z]", text)) and text not in LITERALS and not literal and not re.fullmatch(r"[\w./\\:-]+\.(?:exe|dll|js|json|html|asar|apk|zip|cpp|h|svg)", text, flags=re.IGNORECASE)
|
||||
|
||||
|
||||
def inline(node):
|
||||
return all(not isinstance(child, Node) or (child.tag in INLINE and inline(child)) for child in node.children)
|
||||
|
||||
|
||||
def unit_text(document, node):
|
||||
"""Represent inline markup and code as named placeholders in a complete sentence."""
|
||||
slots, parts = {}, []
|
||||
|
||||
def slot(value):
|
||||
key = f"ph{len(slots)}"
|
||||
slots[key] = value
|
||||
return "{" + key + "}"
|
||||
|
||||
def visit(child):
|
||||
if not isinstance(child, Node):
|
||||
parts.append(unescape(document.source[child[0]:child[1]]))
|
||||
elif child.tag in {"code", "kbd"} or "data-i18n-skip" in child.attrs or child.attrs.get("translate") == "no":
|
||||
parts.append(slot(document.source[child.start:child.end]))
|
||||
else:
|
||||
parts.append(slot(document.source[child.start:child.open_end]))
|
||||
for descendant in child.children:
|
||||
visit(descendant)
|
||||
if child.end > child.end_start:
|
||||
parts.append(slot(document.source[child.end_start:child.end]))
|
||||
for child in node.children:
|
||||
visit(child)
|
||||
return re.sub(r"\s+", " ", "".join(parts)).strip(), slots
|
||||
|
||||
|
||||
def prose_units(source, path):
|
||||
"""Extract paragraph-sized units, or individual orphan text when structure requires it."""
|
||||
document = Document(source)
|
||||
units = []
|
||||
|
||||
def context(start):
|
||||
for name in ("sharing", "header", "footer"):
|
||||
begin, end = source.find(f"<!-- website:{name}:start -->"), source.find(f"<!-- website:{name}:end -->")
|
||||
if begin >= 0 and begin < start < end:
|
||||
return "shared"
|
||||
return "page:" + path
|
||||
|
||||
def visit(node):
|
||||
if skipped(node) or node.tag in {"head", "document"}:
|
||||
if node.tag == "document":
|
||||
for child in node.children:
|
||||
if isinstance(child, Node):
|
||||
visit(child)
|
||||
return
|
||||
if node.tag in PROSE and inline(node) and node.children:
|
||||
text, slots = unit_text(document, node)
|
||||
if translatable(PLACEHOLDER.sub("", text).strip()):
|
||||
units.append((context(node.start), text, slots, node.open_end, node.end_start, node.line))
|
||||
return
|
||||
for child in node.children:
|
||||
if isinstance(child, Node):
|
||||
visit(child)
|
||||
else:
|
||||
raw = source[child[0]:child[1]]
|
||||
text = re.sub(r"\s+", " ", unescape(raw)).strip()
|
||||
if translatable(text):
|
||||
units.append((context(child[0]), text, {}, child[0], child[1], source.count("\n", 0, child[0]) + 1))
|
||||
visit(document.root)
|
||||
# The title is prose even though it lives in the head.
|
||||
for node in document.nodes:
|
||||
if node.tag == "title" and not skipped(node):
|
||||
text, slots = unit_text(document, node)
|
||||
if translatable(text):
|
||||
units.append(("page:" + path, text, slots, node.open_end, node.end_start, node.line))
|
||||
return units
|
||||
|
||||
|
||||
def attribute_units(source, path):
|
||||
"""Extract accessibility labels and authored metadata without translating URLs."""
|
||||
units = []
|
||||
document = Document(source)
|
||||
for node in document.nodes:
|
||||
if skipped(node):
|
||||
continue
|
||||
names = set(ATTRIBUTES)
|
||||
if node.tag == "meta" and (node.attrs.get("name") == "description" or node.attrs.get("property") == "og:image:alt" or node.attrs.get("name") == "twitter:image:alt"):
|
||||
names.add("content")
|
||||
for name in names:
|
||||
text = node.attrs.get(name)
|
||||
if text and translatable(text):
|
||||
context = "shared" if "<!-- website:header:start -->" in source[:node.start] and "<!-- website:header:end -->" not in source[:node.start] else "page:" + path
|
||||
if name == "content" and node.attrs.get("name") != "description":
|
||||
context = "shared"
|
||||
units.append((context, text, {}, node, name))
|
||||
return units
|
||||
|
||||
|
||||
def runtime_units(root):
|
||||
"""Extract the constrained t("literal", namedValues) authoring form directly."""
|
||||
yield "fallback", FALLBACK_NOTICE, "scripts/website_i18n.py", ""
|
||||
site = root / "website/public"
|
||||
for path in sorted((site / "assets").glob("*.js")):
|
||||
if path.name == "i18n.js": # Owns t itself, rather than reader-facing calls.
|
||||
continue
|
||||
source = path.read_text(encoding="utf-8")
|
||||
calls = list(re.finditer(r'\bt\(\s*("(?:[^"\\]|\\.)*")', source))
|
||||
if len(calls) != len(re.findall(r"\bt\(", source)):
|
||||
raise ValueError(f'{path.name}: use t("one literal message", namedValues)')
|
||||
script = path.relative_to(site).as_posix()
|
||||
for call in calls:
|
||||
yield script, json.loads(call[1]), "website/public/" + script, source.count("\n", 0, call.start()) + 1
|
||||
|
||||
|
||||
def read_runtime(root, page=None):
|
||||
"""Scope automatically extracted interaction messages to a page's scripts."""
|
||||
units = list(runtime_units(root))
|
||||
if page is None:
|
||||
return list(dict.fromkeys(text for _, text, _, _ in units))
|
||||
site = root / "website/public"
|
||||
scripts = set()
|
||||
for node in Document(page.read_text(encoding="utf-8")).nodes:
|
||||
if node.tag == "script" and node.attrs.get("src"):
|
||||
reference = urlsplit(node.attrs["src"])
|
||||
if not reference.scheme and not reference.netloc:
|
||||
scripts.add((page.parent / unquote(reference.path)).resolve().relative_to(site.resolve()).as_posix())
|
||||
return list(dict.fromkeys(text for script, text, _, _ in units if script in scripts))
|
||||
|
||||
|
||||
def source_catalog(root):
|
||||
"""Build a deterministic POT with source references and protected-placeholder notes."""
|
||||
site = root / "website/public"
|
||||
catalog = polib.POFile(wrapwidth=88)
|
||||
catalog.metadata = {"Project-Id-Version": "REA website", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit"}
|
||||
entries = {}
|
||||
|
||||
def add(context, text, slots, reference, line):
|
||||
key = context, text
|
||||
if key not in entries:
|
||||
entry = polib.POEntry(msgctxt=context, msgid=text, msgstr="", flags=["python-brace-format"] if PLACEHOLDER.search(text) else [])
|
||||
entry.comment = "\n".join(f"{{{key}}}: {re.sub(r'\s+', ' ', value)}" for key, value in slots.items())
|
||||
entries[key] = entry
|
||||
catalog.append(entry)
|
||||
occurrence = (reference, str(line))
|
||||
if occurrence not in entries[key].occurrences:
|
||||
entries[key].occurrences.append(occurrence)
|
||||
|
||||
for path in reading_pages(site):
|
||||
relative = path.relative_to(site).as_posix()
|
||||
source = path.read_text(encoding="utf-8")
|
||||
for context, text, slots, _, _, line in prose_units(source, relative):
|
||||
add(context, text, slots, "website/public/" + relative, line)
|
||||
for context, text, slots, node, _ in attribute_units(source, relative):
|
||||
add(context, text, slots, "website/public/" + relative, node.line)
|
||||
for _, text, reference, line in runtime_units(root):
|
||||
add("runtime", text, {}, reference, line)
|
||||
for path in sorted((site / "assets/figures").glob("*.svg")):
|
||||
source = path.read_text(encoding="utf-8")
|
||||
for node in Document(source).nodes:
|
||||
if node.tag in {"text", "title", "desc"} and not any(isinstance(child, Node) for child in node.children):
|
||||
text = unescape(source[node.open_end:node.end_start]).strip()
|
||||
if translatable(text):
|
||||
add("figure:" + path.name, text, {}, "website/public/assets/figures/" + path.name, node.line)
|
||||
return catalog
|
||||
|
||||
|
||||
def extract_catalogs(root, check=False):
|
||||
"""Refresh standard catalogs, preserving translations/comments and archiving removed text."""
|
||||
inventory = languages(root)
|
||||
template = source_catalog(root)
|
||||
directory = root / "website/locales"
|
||||
target = directory / "messages.pot"
|
||||
changed = []
|
||||
if not target.exists() or target.read_text(encoding="utf-8") != str(template):
|
||||
changed.append(target)
|
||||
if not check:
|
||||
template.save(target)
|
||||
expected = {(entry.msgctxt, entry.msgid) for entry in template}
|
||||
for language in inventory[1:]:
|
||||
path = directory / language["code"] / "messages.po"
|
||||
catalog = polib.pofile(path, encoding="utf-8") if path.exists() else polib.POFile(wrapwidth=88)
|
||||
active = {(entry.msgctxt, entry.msgid) for entry in catalog if not entry.obsolete}
|
||||
if active != expected:
|
||||
changed.append(path)
|
||||
if not check:
|
||||
catalog.merge(template)
|
||||
catalog.metadata.update(template.metadata)
|
||||
catalog.metadata["Language"] = language["code"]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
catalog.save(path)
|
||||
if check and changed:
|
||||
raise ValueError("Translation sources changed: " + ", ".join(str(path.relative_to(root)) for path in changed[:8]) + ". Run python3 scripts/website.py translations extract, then update the affected msgstr entries.")
|
||||
print(f"{'Checked' if check else 'Extracted'} {len(template)} source messages for {len(inventory) - 1} translation catalogs.")
|
||||
return template
|
||||
|
||||
|
||||
def fields(text):
|
||||
"""Validate named format fields without accepting conversion or format instructions."""
|
||||
result = []
|
||||
for _, name, specification, conversion in Formatter().parse(text):
|
||||
if name is not None:
|
||||
if not re.fullmatch(r"[a-zA-Z][a-zA-Z0-9_]*", name) or specification or conversion:
|
||||
raise ValueError(f"Unsupported translation placeholder: {name}")
|
||||
result.append(name)
|
||||
return Counter(result)
|
||||
|
||||
|
||||
def checked_catalog(root, language, template, strict=False):
|
||||
"""Reject malformed catalogs and report missing or fuzzy messages independently."""
|
||||
path = root / "website/locales" / language["code"] / "messages.po"
|
||||
catalog = polib.pofile(path, encoding="utf-8", check_for_duplicates=True)
|
||||
if catalog.metadata.get("Language") != language["code"]:
|
||||
raise ValueError(f"{path}: wrong Language header")
|
||||
messages, missing = {}, []
|
||||
source = {(entry.msgctxt, entry.msgid): entry for entry in template}
|
||||
for entry in catalog:
|
||||
if entry.obsolete:
|
||||
continue
|
||||
key = entry.msgctxt, entry.msgid
|
||||
if key not in source:
|
||||
raise ValueError(f"{path}: obsolete source still active: {entry.msgid[:80]}")
|
||||
if entry.msgid_plural or entry.msgstr_plural:
|
||||
raise ValueError(f"{path}: use the declared named-count messages, not undeclared plural forms")
|
||||
if not entry.msgstr.strip() or "fuzzy" in entry.flags:
|
||||
missing.append(key)
|
||||
continue
|
||||
if fields(entry.msgid) != fields(entry.msgstr):
|
||||
raise ValueError(f"{path}: translation changed placeholders: {entry.msgid[:100]}")
|
||||
slots = dict(re.findall(r"^\{(ph\d+)\}: (.*)$", source[key].comment, re.MULTILINE))
|
||||
if slots:
|
||||
fragment = escape(entry.msgstr, quote=False)
|
||||
for name, value in slots.items():
|
||||
fragment = fragment.replace("{" + name + "}", value)
|
||||
try:
|
||||
Document("<div>" + fragment + "</div>")
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{path}: {entry.msgctxt}: invalid inline markup in {entry.msgid[:100]!r}: {error}") from error
|
||||
messages[key] = entry.msgstr.strip()
|
||||
missing.extend(key for key in source if key not in messages and key not in missing)
|
||||
if strict and missing:
|
||||
raise ValueError(f"{path}: {len(missing)} untranslated/fuzzy messages; first: {missing[0][1][:100]}")
|
||||
return messages, missing
|
||||
|
||||
|
||||
def replace_attribute(raw, name, value):
|
||||
"""Change one parsed start-tag attribute without modifying adjacent markup."""
|
||||
pattern = re.compile(r"(\s" + re.escape(name) + r"\s*=\s*)([\"'])(.*?)\2", re.DOTALL | re.IGNORECASE)
|
||||
updated, count = pattern.subn(lambda match: match[1] + '"' + escape(value, quote=True) + '"', raw)
|
||||
if count != 1:
|
||||
raise ValueError(f"Expected one quoted {name} attribute: {raw[:100]}")
|
||||
return updated
|
||||
|
||||
|
||||
def apply_edits(source, edits):
|
||||
"""Apply non-overlapping spans from the end, preserving untouched evidence byte-for-byte."""
|
||||
end = len(source)
|
||||
for start, stop, replacement in sorted(edits, reverse=True):
|
||||
if stop > end:
|
||||
raise ValueError("Overlapping translation spans")
|
||||
source = source[:start] + replacement + source[stop:]
|
||||
end = start
|
||||
return source
|
||||
|
||||
|
||||
def render_prose(source, relative, messages):
|
||||
"""Translate complete units and restore validated original inline markup."""
|
||||
edits = []
|
||||
for context, text, slots, start, end, _ in prose_units(source, relative):
|
||||
translated = messages[(context, text)]
|
||||
if fields(text) != fields(translated):
|
||||
raise ValueError(f"{relative}: changed placeholders")
|
||||
rendered = escape(translated, quote=False)
|
||||
for key, value in slots.items():
|
||||
rendered = rendered.replace("{" + key + "}", value)
|
||||
# Check translator-reordered markup before accepting it into the document.
|
||||
try:
|
||||
Document("<div>" + rendered + "</div>")
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{relative}: invalid translated markup in {text[:100]!r}: {error}") from error
|
||||
raw = source[start:end]
|
||||
leading = re.match(r"\s*", raw)[0]
|
||||
trailing = re.search(r"\s*$", raw)[0]
|
||||
edits.append((start, end, leading + rendered + trailing))
|
||||
return apply_edits(source, edits)
|
||||
|
||||
|
||||
def relative_reference(reference, source_path, output_path, site, available, code, figures, language_link=False):
|
||||
"""Resolve links once, keeping assets shared and reading links in the selected edition."""
|
||||
parsed = urlsplit(reference)
|
||||
if parsed.scheme or parsed.netloc or not parsed.path:
|
||||
return reference
|
||||
target = (source_path.parent / unquote(parsed.path)).resolve()
|
||||
if not target.is_relative_to(site.resolve()):
|
||||
raise ValueError(f"Reference leaves the website: {reference}")
|
||||
relative = target.relative_to(site.resolve())
|
||||
lookup = relative / "index.html" if target.is_dir() else relative
|
||||
if not language_link and lookup.as_posix() in available and code != "en":
|
||||
relative = Path(code) / relative
|
||||
if relative.as_posix() in figures and code != "en":
|
||||
relative = Path("assets/localized") / code / relative.relative_to("assets")
|
||||
destination = os.path.relpath(site / relative, output_path.parent).replace(os.sep, "/")
|
||||
if parsed.path.endswith("/") and not destination.endswith("/"):
|
||||
destination += "/"
|
||||
return urlunsplit(("", "", quote(destination, safe="/"), parsed.query, parsed.fragment))
|
||||
|
||||
|
||||
def replace_language_markup(source, root, relative, code, available):
|
||||
"""Derive selectors and alternates from the same page-level completeness decision."""
|
||||
selector, alternates = language_markup(root, relative, code, available)
|
||||
document = Document(source)
|
||||
picker = next(node for node in document.nodes if node.tag == "details" and node.attrs.get("class") == "language-picker")
|
||||
source = apply_edits(source, [(picker.start, picker.end, selector)])
|
||||
pattern = r"(<!-- website:languages:start -->).*?(<!-- website:languages:end -->)"
|
||||
return re.sub(pattern, lambda match: match[1] + "\n" + alternates + "\n" + match[2], source, flags=re.DOTALL)
|
||||
|
||||
|
||||
def render_page(root, source_path, language, messages, available, figures, page_languages):
|
||||
"""Create one complete translation, preserving routes, code, IDs and reference data."""
|
||||
from website import SITE_ORIGIN, PageMetadata, page_url
|
||||
site = root / "website/public"
|
||||
relative = source_path.relative_to(site).as_posix()
|
||||
code = language["code"]
|
||||
output_path = site / localized_path(relative, code)
|
||||
source = source_path.read_text(encoding="utf-8")
|
||||
source = render_prose(source, relative, messages)
|
||||
metadata = PageMetadata()
|
||||
# Translate attributes separately; units inside inline markup keep their source tags.
|
||||
replacements = {}
|
||||
for context, text, _, node, attribute in attribute_units(source, relative):
|
||||
raw = replacements.get(node.start, source[node.start:node.open_end])
|
||||
replacements[node.start] = replace_attribute(raw, attribute, messages[(context, text)])
|
||||
source = apply_edits(source, [(node.start, node.open_end, replacements[node.start]) for node in Document(source).nodes if node.start in replacements])
|
||||
metadata.feed(source)
|
||||
canonical = page_url(site, output_path)
|
||||
replacements = []
|
||||
for node in Document(source).nodes:
|
||||
raw = source[node.start:node.open_end]
|
||||
if node.tag == "html":
|
||||
raw = replace_attribute(raw, "lang", code)
|
||||
raw = raw[:-1] + f' dir="{language["dir"]}">'
|
||||
if node.tag == "meta":
|
||||
key = node.attrs.get("property") or node.attrs.get("name")
|
||||
if key in {"og:title", "twitter:title"}:
|
||||
raw = replace_attribute(raw, "content", metadata.titles[0])
|
||||
elif key in {"og:description", "twitter:description"}:
|
||||
raw = replace_attribute(raw, "content", metadata.descriptions[0])
|
||||
elif key == "og:url":
|
||||
raw = replace_attribute(raw, "content", canonical)
|
||||
for attribute in ("href", "src"):
|
||||
if attribute not in node.attrs:
|
||||
continue
|
||||
reference = node.attrs[attribute]
|
||||
if node.tag == "link" and node.attrs.get("rel") == "canonical":
|
||||
value = canonical
|
||||
else:
|
||||
value = relative_reference(reference, source_path, output_path, site, available, code, figures, "data-language" in node.attrs)
|
||||
raw = replace_attribute(raw, attribute, value)
|
||||
if raw != source[node.start:node.open_end]:
|
||||
replacements.append((node.start, node.open_end, raw))
|
||||
source = apply_edits(source, replacements)
|
||||
source = replace_language_markup(source, root, relative, code, page_languages)
|
||||
runtime = {text: messages[("runtime", text)] for text in read_runtime(root, source_path)}
|
||||
payload = json.dumps(runtime, ensure_ascii=False).replace("<", "\\u003c").replace("&", "\\u0026")
|
||||
source = source.replace("</head>", f'<script id="website-messages" type="application/json">{payload}</script>\n </head>')
|
||||
return source
|
||||
|
||||
|
||||
def required_messages(root, path, template):
|
||||
"""Bind a page to its prose, attributes, runtime labels and referenced figures."""
|
||||
site = root / "website/public"
|
||||
relative = path.relative_to(site).as_posix()
|
||||
source = path.read_text(encoding="utf-8")
|
||||
required = {(unit[0], unit[1]) for unit in prose_units(source, relative)}
|
||||
required.update((unit[0], unit[1]) for unit in attribute_units(source, relative))
|
||||
required.update(("runtime", text) for text in read_runtime(root, path))
|
||||
for node in Document(source).nodes:
|
||||
reference = node.attrs.get("src")
|
||||
if reference and not urlsplit(reference).scheme:
|
||||
target = (path.parent / unquote(urlsplit(reference).path)).resolve()
|
||||
if target.parent == (site / "assets/figures").resolve():
|
||||
required.update((entry.msgctxt, entry.msgid) for entry in template if entry.msgctxt == "figure:" + target.name)
|
||||
return required
|
||||
|
||||
|
||||
def render_fallback(root, path, language, page_languages, messages):
|
||||
"""Expose a clear English fallback instead of a stale or partially translated article."""
|
||||
from website import page_url
|
||||
site = root / "website/public"
|
||||
relative = path.relative_to(site).as_posix()
|
||||
code = language["code"]
|
||||
output_path = site / localized_path(relative, code)
|
||||
source = path.read_text(encoding="utf-8")
|
||||
edits = []
|
||||
canonical = page_url(site, output_path)
|
||||
for node in Document(source).nodes:
|
||||
raw = source[node.start:node.open_end]
|
||||
for attribute in ("href", "src"):
|
||||
if attribute in node.attrs:
|
||||
value = canonical if node.tag == "link" and node.attrs.get("rel") == "canonical" else relative_reference(node.attrs[attribute], path, output_path, site, set(), code, set(), "data-language" in node.attrs)
|
||||
raw = replace_attribute(raw, attribute, value)
|
||||
if node.tag == "meta" and node.attrs.get("property") == "og:url":
|
||||
raw = replace_attribute(raw, "content", canonical)
|
||||
if node.tag == "html":
|
||||
raw = raw[:-1] + ' data-translation-fallback="true">'
|
||||
if raw != source[node.start:node.open_end]:
|
||||
edits.append((node.start, node.open_end, raw))
|
||||
source = apply_edits(source, edits)
|
||||
source = source.replace("</head>", '<meta name="robots" content="noindex, follow" />\n </head>')
|
||||
source = replace_language_markup(source, root, relative, code, page_languages)
|
||||
notice = messages.get(("runtime", FALLBACK_NOTICE), FALLBACK_NOTICE)
|
||||
notice_lang = code if ("runtime", FALLBACK_NOTICE) in messages else "en"
|
||||
main = next(node for node in Document(source).nodes if node.tag == "main")
|
||||
source = source[:main.open_end] + f'<aside class="container translation-notice" lang="{notice_lang}" dir="{language["dir"]}">{escape(notice)}</aside>' + source[main.open_end:]
|
||||
return source
|
||||
|
||||
|
||||
def build_translations(root, destination, finalize=None):
|
||||
"""Build one clean publish directory from English sources and versioned PO catalogs."""
|
||||
site = root / "website/public"
|
||||
if destination.resolve() != (root / "website/_site").resolve():
|
||||
raise ValueError("Generated output must be website/_site")
|
||||
template = extract_catalogs(root, check=True)
|
||||
inventory = languages(root)
|
||||
pages = reading_pages(site)
|
||||
catalogs = {language["code"]: checked_catalog(root, language, template)[0] for language in inventory[1:]}
|
||||
required = {path.relative_to(site).as_posix(): required_messages(root, path, template) for path in pages}
|
||||
page_languages = {relative: {"en"} | {code for code, messages in catalogs.items() if keys <= messages.keys()} for relative, keys in required.items()}
|
||||
# Finish rendering before replacing the previous preview/publish artifact.
|
||||
with TemporaryDirectory(prefix=".i18n-", dir=root / "website") as directory:
|
||||
staging = Path(directory) / "site"
|
||||
shutil.copytree(site, staging, ignore=shutil.ignore_patterns("__pycache__"))
|
||||
for path in pages:
|
||||
relative = path.relative_to(site).as_posix()
|
||||
output = staging / relative
|
||||
output.write_text(replace_language_markup(path.read_text(encoding="utf-8"), root, relative, "en", page_languages[relative]), encoding="utf-8")
|
||||
for language in inventory[1:]:
|
||||
messages = catalogs[language["code"]]
|
||||
available = {relative for relative, codes in page_languages.items() if language["code"] in codes}
|
||||
figures = set()
|
||||
for path in sorted((site / "assets/figures").glob("*.svg")):
|
||||
source = path.read_text(encoding="utf-8")
|
||||
edits = []
|
||||
for node in Document(source).nodes:
|
||||
if node.tag in {"text", "title", "desc"} and not any(isinstance(child, Node) for child in node.children):
|
||||
text = unescape(source[node.open_end:node.end_start]).strip()
|
||||
if translatable(text) and ("figure:" + path.name, text) in messages:
|
||||
translated = messages[("figure:" + path.name, text)]
|
||||
edits.append((node.open_end, node.end_start, escape(translated, quote=False)))
|
||||
translated = apply_edits(source, edits)
|
||||
output = staging / "assets/localized" / language["code"] / path.relative_to(site / "assets")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(translated, encoding="utf-8")
|
||||
figures.add(path.relative_to(site).as_posix())
|
||||
for path in pages:
|
||||
relative = path.relative_to(site).as_posix()
|
||||
output = staging / localized_path(relative, language["code"])
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
rendered = render_page(root, path, language, messages, available, figures, page_languages[relative]) if relative in available else render_fallback(root, path, language, page_languages[relative], messages)
|
||||
output.write_text(rendered, encoding="utf-8")
|
||||
fallback_count = len(pages) - len(available)
|
||||
if fallback_count:
|
||||
print(f"{language['code']}: {fallback_count} English fallback pages, marked noindex; run translations status.")
|
||||
print(f"Prepared {len(pages) * len(inventory)} reading pages in {len(inventory)} languages.")
|
||||
if finalize is not None:
|
||||
finalize(staging)
|
||||
if destination.exists():
|
||||
shutil.rmtree(destination)
|
||||
staging.rename(destination)
|
||||
Reference in New Issue
Block a user