',
+ page.read_text(),
+ )
+ report = quietly(WEBSITE.translation_report, root)
+ self.assertIn(f' after updating the translation, record source-sha256="{WEBSITE.source_digest(changed)}"', report)
+
+ english_path.write_text(changed.replace("
Restart your client after setup.
", text, 1))
+ quietly(WEBSITE.sync_pages, root)
+ self.assertNotIn('class="translation-notice"', page.read_text())
+
+ page.write_text(page.read_text().replace("rea-agents@latest", "rea-agents@1", 1))
+ self.assertTrue(any("install-cli must match" in error for error in translation_errors(root)[0]))
+
+ def test_translations_keep_structure_and_cjk_spacing(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = site_copy(directory)
+ page = quietly(WEBSITE.new_translation, root, "zh-hant", "get-started")
+ original = page.read_text()
+ cases = (
+ ("
Restart your client when setup finishes.
", "
完成後\n 重新啟動。
", "renders as a gap"),
+ ('id="install-node"', 'id="node"', "element IDs differ"),
+ ('data-copy="install-cli"', 'data-copy="node-check"', "copy buttons differ"),
+ (' \n', "", "draft translation needs"),
+ ('status="draft"', 'status="final"', "status must be draft or reviewed"),
+ )
+ for old, new, message in cases:
+ with self.subTest(message=message):
+ self.assertIn(old, original)
+ page.write_text(original.replace(old, new, 1))
+ self.assertTrue(any(message in error for error in translation_errors(root)[0]))
+ page.write_text(original.replace("
Restart your client when setup finishes.
", "
重新啟動 REA 用戶端。
", 1))
+ self.assertEqual(translation_errors(root)[0], [])
+ page.write_text(original.replace("node -v", "node -V", 1))
+ errors, warnings = translation_errors(root)
+ self.assertEqual(errors, [])
+ self.assertTrue(any("inline code differs" in message for _, message in warnings))
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/scripts/verify-website.py b/scripts/verify-website.py
index 889d77394..cb59896f7 100644
--- a/scripts/verify-website.py
+++ b/scripts/verify-website.py
@@ -1,6 +1,8 @@
"""Check website references, search metadata and the sole production publisher."""
+from collections import Counter
from html.parser import HTMLParser
+import os
from pathlib import Path
import re
import struct
@@ -10,7 +12,21 @@ from urllib.parse import unquote, urlsplit
import xml.etree.ElementTree as ET
from zipfile import BadZipFile, ZipFile
-from website import NOTES_FIXTURES, SITE_ORIGIN, authored_markdown, page_url, sync_pages
+from website import (
+ LANGUAGES,
+ NOTES_FIXTURES,
+ SITE_ORIGIN,
+ SiteState,
+ authored_markdown,
+ code_block,
+ copy_targets,
+ page_language,
+ page_url,
+ source_digest,
+ sync_pages,
+ synced_code_ids,
+ translation_record,
+)
LEGACY_ORIGIN = "morluto.github.io"
SITE_TEXT_URL = re.compile(
@@ -167,8 +183,9 @@ def check_search_metadata(site, pages):
title = page.titles[0].strip() if len(page.titles) == 1 else ""
if not title:
errors.append(f"{label}: needs exactly one nonempty head title")
- if page.lang != "en" or page.h1_count != 1:
- errors.append(f"{label}: needs lang=en and exactly one h1")
+ tag = LANGUAGES[page_language(site, path)[0]].tag
+ if page.lang != tag or page.h1_count != 1:
+ errors.append(f"{label}: needs lang={tag} and exactly one h1")
if label in NOTES_FIXTURES and not page.noindex():
errors.append(
f"{label}: example application must have a head noindex directive"
@@ -280,15 +297,21 @@ def check_search_metadata(site, pages):
return len(indexable), errors
-def check_site(site):
- """Check assets, HTML fragment destinations and SVG syntax."""
+def parse_pages(site):
+ """Parse every HTML page once for reference, metadata and translation checks."""
pages = {}
- errors = []
- references = 0
for path in sorted(site.rglob("*.html")):
parser = HtmlReferences()
parser.feed(path.read_text(encoding="utf-8"))
pages[path] = parser
+ return pages
+
+
+def check_site(site):
+ """Check assets, HTML fragment destinations and SVG syntax."""
+ pages = parse_pages(site)
+ errors = []
+ references = 0
if site / "index.html" not in pages:
errors.append("website/public/index.html is missing")
@@ -346,6 +369,190 @@ def check_site(site):
return len(pages), references, indexable, errors
+# Han, kana and CJK punctuation; a space between two of them renders as a gap.
+CJK_GAP = re.compile(
+ r"([、-〿-ヿ㐀-䶿一-鿿豈-!-])"
+ r"\s+([、-〿-ヿ㐀-䶿一-鿿豈-!-])"
+)
+CODE_TAGS = {"code", "kbd", "pre", "script", "style"}
+
+
+class TranslationText(HTMLParser):
+ """Collect prose, inline code and message attributes for comparing translations."""
+
+ def __init__(self, synced):
+ super().__init__()
+ self.synced = set(synced)
+ self.stack = []
+ self.prose = []
+ self.inline_code = Counter()
+ self.messages = Counter()
+
+ def handle_starttag(self, tag, attrs):
+ attributes = dict(attrs)
+ self.messages.update(name for name in attributes if name.startswith("data-msg-"))
+ if tag in CODE_TAGS:
+ self.stack.append([tag, attributes.get("id"), ""])
+
+ def handle_startendtag(self, tag, attrs):
+ attributes = dict(attrs)
+ self.messages.update(name for name in attributes if name.startswith("data-msg-"))
+
+ def handle_endtag(self, tag):
+ if self.stack and self.stack[-1][0] == tag:
+ name, identifier, text = self.stack.pop()
+ if name in ("code", "kbd") and identifier not in self.synced:
+ self.inline_code[" ".join(text.split())] += 1
+ if self.stack:
+ self.stack[-1][2] += text
+
+ def handle_data(self, data):
+ if self.stack:
+ self.stack[-1][2] += data
+ else:
+ self.prose.append(data)
+
+
+def translation_warning(path, root, message):
+ """Return a warning tied to a repository file."""
+ return (path.relative_to(root).as_posix(), message)
+
+
+def check_translations(root, pages):
+ """Check translation records, copied code, structure parity and CJK spacing.
+
+ A changed English source is a warning: its translation keeps working, shows a
+ notice after sync and is updated later. Errors cover what sync or the
+ translator controls directly.
+ """
+ site = root / "website/public"
+ errors = []
+ warnings = []
+ try:
+ state = SiteState(root)
+ except ValueError as error:
+ return [str(error)], []
+ for path, page in pages.items():
+ language, english = page_language(site, path)
+ if language == "en":
+ continue
+ label = path.relative_to(site).as_posix()
+ source = path.read_text(encoding="utf-8")
+ record = translation_record(source)
+ english_path = site / english
+ if not english_path.is_file():
+ errors.append(f"{label}: translates {english}, which does not exist")
+ continue
+ english_source = english_path.read_text(encoding="utf-8")
+ if record is None:
+ errors.append(f"{label}: needs one record")
+ continue
+ status = record.get("status")
+ if status not in ("draft", "reviewed"):
+ errors.append(f"{label}: translation status must be draft or reviewed")
+ if status == "draft" and not page.noindex():
+ errors.append(f"{label}: a draft translation needs ")
+ if status == "reviewed":
+ if page.noindex():
+ errors.append(f"{label}: remove noindex from a reviewed translation")
+ if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", record.get("reviewed", "")):
+ errors.append(f"{label}: a reviewed translation needs reviewed=\"YYYY-MM-DD\"")
+ fresh = record.get("source-sha256") == source_digest(english_source)
+ if not fresh:
+ warnings.append(translation_warning(
+ path, root,
+ f"The English page {english} has changed since this translation. Update the translation, "
+ "then set source-sha256 to the new value from python3 scripts/website.py translations.",
+ ))
+
+ synced = synced_code_ids(english_source)
+ for identifier in synced:
+ expected = code_block(identifier).search(english_source).group(2)
+ found = code_block(identifier).findall(source)
+ if len(found) != 1 or found[0][1] != expected:
+ errors.append(f"{label}: code block {identifier} must match the English page; run sync")
+
+ english_page = pages[english_path]
+ differences = []
+ if page.ids != english_page.ids:
+ differences.append(
+ f"element IDs differ (missing {sorted(english_page.ids - page.ids)}, extra {sorted(page.ids - english_page.ids)})"
+ )
+ if copy_targets(source) != copy_targets(english_source):
+ differences.append("copy buttons differ from the English page")
+ translated = TranslationText(synced)
+ translated.feed(source)
+ original = TranslationText(synced)
+ original.feed(english_source)
+ if translated.messages != original.messages:
+ differences.append("data-msg-* message attributes differ from the English page")
+ for difference in differences:
+ if fresh:
+ errors.append(f"{label}: {difference}")
+ else:
+ warnings.append(translation_warning(path, root, difference))
+ if fresh and translated.inline_code != original.inline_code:
+ changed = sorted((translated.inline_code - original.inline_code) + (original.inline_code - translated.inline_code))
+ warnings.append(translation_warning(
+ path, root, f"inline code differs from the English page: {', '.join(repr(text) for text in changed[:5])}",
+ ))
+
+ if language in ("ja", "zh-hans", "zh-hant"):
+ for text in translated.prose:
+ for match in CJK_GAP.finditer(text):
+ errors.append(
+ f"{label}: whitespace between {match.group(1)!r} and {match.group(2)!r} renders as a gap; "
+ "keep CJK text on one line without spaces"
+ )
+
+ for reference in page.references:
+ url = urlsplit(reference)
+ if url.scheme or url.netloc or not url.path or reference.startswith("/"):
+ continue
+ target = (path.parent / unquote(url.path)).resolve()
+ if target.is_dir():
+ target /= "index.html"
+ if not target.is_relative_to(site.resolve()):
+ continue
+ target_language, target_english = page_language(site.resolve(), target)
+ if (
+ target_language == "en"
+ and target_english != english
+ and state.translations.get((language, target_english)) == "reviewed"
+ ):
+ warnings.append(translation_warning(
+ path, root, f"links to the English {target_english}; link to its reviewed translation instead",
+ ))
+
+ english_strings = state.locales["en"]["strings"]
+ for code in LANGUAGES:
+ if code == "en" or not any(key[0] == code for key in state.translations):
+ continue
+ locale = state.locales[code]
+ same = sorted(
+ key for key, value in locale["strings"].items()
+ if value == english_strings[key] and key not in locale.get("sameAsEnglish", [])
+ )
+ if same:
+ warnings.append((f"website/locales/{code}.json", f"UI strings still in English: {', '.join(same)}"))
+ return errors, warnings
+
+
+def report_warnings(warnings):
+ """Print warnings, as GitHub annotations and a job summary when running in Actions."""
+ for file, message in warnings:
+ print(f"Warning: {file}: {message}")
+ if os.environ.get("GITHUB_ACTIONS") != "true" or not warnings:
+ return
+ for file, message in warnings:
+ print(f"::warning file={file}::{message}")
+ summary = os.environ.get("GITHUB_STEP_SUMMARY")
+ if summary:
+ with open(summary, "a", encoding="utf-8") as output:
+ output.write("### Website warnings\n\n")
+ output.writelines(f"- `{file}`: {message}\n" for file, message in warnings)
+
+
def check_publisher(root):
"""Detect a second Pages action that could replace the website."""
expected = root / ".github/workflows/website-pages.yml"
@@ -404,10 +611,16 @@ def main():
errors.extend(check_example_archive(root / "website/public"))
errors.extend(check_publisher(root))
errors.extend(check_repository_links(root))
+ translation_errors, warnings = check_translations(root, parse_pages(root / "website/public"))
+ errors.extend(translation_errors)
try:
- sync_pages(root, check=True)
+ for path in sync_pages(root, check=True):
+ warnings.append(translation_warning(
+ path, root, "the out-of-date notice needs refreshing; run python3 scripts/website.py sync",
+ ))
except (OSError, ValueError, KeyError) as error:
errors.append(str(error))
+ report_warnings(warnings)
if errors:
print("Website checks failed:\n" + "\n".join(errors), file=sys.stderr)
return 1
diff --git a/scripts/website.py b/scripts/website.py
index fd636897a..6a0500516 100644
--- a/scripts/website.py
+++ b/scripts/website.py
@@ -1,26 +1,53 @@
"""Maintain shared HTML and run the static website's contributor commands."""
import argparse
+from collections import namedtuple
from functools import partial
+from hashlib import sha256
from html import escape
from html.parser import HTMLParser
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+import json
from pathlib import Path, PurePosixPath
+import posixpath
+import re
from string import Template
import subprocess
import sys
from textwrap import indent
-from urllib.parse import quote, urlsplit
+from urllib.parse import quote, urlsplit, urlunsplit
SITE_ORIGIN = "https://rea.tools"
NOTES_FIXTURES = {"examples/notes-electron/index.html", "examples/notes-web/index.html"}
+# Each label is a key in website/locales/.json.
NAVIGATION = (
- ("Showcases", "showcase/"),
- ("Blog", "blog/"),
- ("Get started", "get-started/"),
- ("Guides", "guides/"),
- ("FAQ", "faq/"),
+ ("navShowcases", "showcase/"),
+ ("navBlog", "blog/"),
+ ("navGetStarted", "get-started/"),
+ ("navGuides", "guides/"),
+ ("navFaq", "faq/"),
+)
+Language = namedtuple("Language", "tag name og_locale")
+# English stays at the root; every other language lives under its directory.
+# The order is the language menu's order.
+LANGUAGES = {
+ "en": Language("en", "English", "en_US"),
+ "zh-hans": Language("zh-Hans", "简体中文", "zh_CN"),
+ "zh-hant": Language("zh-Hant", "繁體中文", "zh_TW"),
+ "ja": Language("ja", "日本語", "ja_JP"),
+ "ko": Language("ko", "한국어", "ko_KR"),
+ "de": Language("de", "Deutsch", "de_DE"),
+ "fr": Language("fr", "Français", "fr_FR"),
+}
+TRANSLATION_STATUSES = ("draft", "reviewed")
+TRANSLATION_RECORD = re.compile(r"")
+SHARED_REGION = re.compile(r".*?", re.S)
+NOTICE_START = " \n"
+NOTICE_END = " "
+GLOBE_PATH = (
+ "M12 3a9 9 0 1 0 0 18a9 9 0 1 0 0-18Z M3.6 9h16.8 M3.6 15h16.8 "
+ "M12 3c2.4 2.5 3.6 5.5 3.6 9s-1.2 6.5-3.6 9c-2.4-2.5-3.6-5.5-3.6-9s1.2-6.5 3.6-9Z"
)
@@ -75,10 +102,14 @@ class PageMetadata(HTMLParser):
self.titles[-1] += data
+def file_route(relative):
+ """Map a site-relative HTML file to its public route."""
+ return relative.removesuffix("index.html") if PurePosixPath(relative).name == "index.html" else relative
+
+
def page_route(site, path):
"""Map an HTML file to its public route without a separate page registry."""
- route = path.relative_to(site).as_posix()
- return route.removesuffix("index.html") if path.name == "index.html" else route
+ return file_route(path.relative_to(site).as_posix())
def page_url(site, path):
@@ -86,30 +117,227 @@ def page_url(site, path):
return f"{SITE_ORIGIN}/{quote(page_route(site, path), safe='/')}"
-def shared_blocks(root, path, source):
+def page_language(site, path):
+ """Return a page's language directory and the site-relative file of its English source."""
+ relative = path.relative_to(site).as_posix()
+ first, separator, rest = relative.partition("/")
+ if separator and first in LANGUAGES and first != "en":
+ return first, rest
+ return "en", relative
+
+
+def load_locales(root):
+ """Read the UI strings for every language and require the same keys as English."""
+ directory = root / "website/locales"
+ locales = {}
+ for code in LANGUAGES:
+ path = directory / f"{code}.json"
+ try:
+ locale = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError) as error:
+ raise ValueError(f"website/locales/{code}.json: cannot read UI strings: {error}") from error
+ locales[code] = locale
+ expected = set(locales["en"]["strings"])
+ for code, locale in locales.items():
+ label = f"website/locales/{code}.json"
+ strings = locale.get("strings")
+ if not isinstance(strings, dict) or set(strings) != expected:
+ missing = sorted(expected - set(strings or {}))
+ extra = sorted(set(strings or {}) - expected)
+ raise ValueError(f"{label}: strings must match en.json (missing {missing}, extra {extra})")
+ if any(not isinstance(value, str) or not value.strip() for value in strings.values()):
+ raise ValueError(f"{label}: every UI string must be nonempty text")
+ stars = locale.get("githubStars")
+ if not isinstance(stars, dict) or "other" not in stars or not set(stars) <= {"zero", "one", "two", "few", "many", "other"}:
+ raise ValueError(f"{label}: githubStars needs an 'other' label and CLDR plural categories only")
+ if any("{count}" not in value for value in stars.values()):
+ raise ValueError(f"{label}: every githubStars label needs a {{count}} placeholder")
+ if "{link}" not in strings["staleNotice"]:
+ raise ValueError(f"{label}: staleNotice needs a {{link}} placeholder")
+ same = locale.get("sameAsEnglish", [])
+ if not isinstance(same, list) or not set(same) <= expected:
+ raise ValueError(f"{label}: sameAsEnglish must list UI string keys")
+ return locales
+
+
+def translation_record(source):
+ """Return the attributes of a translation's provenance comment, or None."""
+ records = TRANSLATION_RECORD.findall(source)
+ if len(records) != 1:
+ return None
+ return dict(re.findall(r'([\w-]+)="([^"]*)"', records[0]))
+
+
+def copy_targets(source):
+ """Return the IDs referenced by copy buttons, in page order."""
+ return list(dict.fromkeys(re.findall(r'\bdata-copy="([^"]+)"', source)))
+
+
+def code_block(identifier):
+ """Match a code element by ID, capturing its opening tag, contents and end tag."""
+ return re.compile(rf'(]*\bid="{re.escape(identifier)}"[^>]*>)(.*?)()', re.S)
+
+
+def synced_code_ids(source):
+ """Return the copy targets that are code blocks; translations take them from English."""
+ return [identifier for identifier in copy_targets(source) if code_block(identifier).search(source)]
+
+
+def source_digest(source):
+ """Hash an English page's translatable content.
+
+ Shared regions and copied code blocks are left out, so template edits and
+ command-only changes do not mark translations out of date.
+ """
+ content = SHARED_REGION.sub("", source)
+ for identifier in synced_code_ids(source):
+ content = code_block(identifier).sub(r"\1\3", content)
+ return sha256(" ".join(content.split()).encode("utf-8")).hexdigest()
+
+
+def sync_code_blocks(english, source, label):
+ """Copy every synced code block's contents from the English source, matched by ID."""
+ for identifier in synced_code_ids(english):
+ contents = code_block(identifier).search(english).group(2)
+ pattern = code_block(identifier)
+ if len(pattern.findall(source)) != 1:
+ raise ValueError(f"{label}: needs one block copied from the English page")
+ source = pattern.sub(lambda match: match.group(1) + contents + match.group(3), source)
+ return source
+
+
+class SiteState:
+ """Read the locales and every translation's status once for a sync or check."""
+
+ def __init__(self, root):
+ self.root = root
+ self.site = root / "website/public"
+ self.locales = load_locales(root)
+ self.translations = {}
+ for path in sorted(self.site.rglob("*.html")):
+ language, english = page_language(self.site, path)
+ if language != "en":
+ record = translation_record(path.read_text(encoding="utf-8")) or {}
+ self.translations[(language, english)] = record.get("status")
+
+ def linkable(self, language, english, from_draft=False):
+ """Report whether other pages may link to this language's version of a page."""
+ if language == "en":
+ return (self.site / english).is_file()
+ status = self.translations.get((language, english))
+ return status == "reviewed" or (from_draft and status == "draft")
+
+ def alternates(self, english):
+ """Return the languages whose versions of a page join its hreflang cluster."""
+ return [code for code in LANGUAGES if code == "en" or self.translations.get((code, english)) == "reviewed"]
+
+
+def language_url(language, english):
+ """Return the canonical URL of one language's version of a page."""
+ prefix = "" if language == "en" else f"{language}/"
+ return f"{SITE_ORIGIN}/{quote(prefix + file_route(english), safe='/')}"
+
+
+def shared_blocks(root, path, source, state=None):
"""Render shared regions while keeping the page's body and metadata authored."""
- site = root / "website/public"
+ state = state or SiteState(root)
+ site = state.site
metadata = PageMetadata()
metadata.feed(source)
for name, values in (("title", metadata.titles), ("description", metadata.descriptions)):
if len(values) != 1 or not values[0].strip():
raise ValueError(f"{path.relative_to(site)}: needs one nonempty head {name}")
+ language, english = page_language(site, path)
+ strings = state.locales[language]["strings"]
depth = len(path.relative_to(site).parent.parts)
base = "../" * depth or "./"
- route = page_route(site, path)
+ route = file_route(english)
+ draft = language != "en" and state.translations.get((language, english)) == "draft"
+
+ def href(target_language, target):
+ prefix = "" if target_language == "en" else f"{target_language}/"
+ return base + file_route(prefix + target)
+
+ def localized(target):
+ """Prefer this language's version of a target page, else the English page."""
+ if language != "en" and state.linkable(language, target, from_draft=draft):
+ return language, href(language, target)
+ return "en", href("en", target)
+
links = []
- for label, destination in NAVIGATION:
+ for key, destination in NAVIGATION:
+ target_language, url = localized(destination + "index.html")
attributes = ' class="nav-start"' if destination == "get-started/" else ""
if route.startswith(destination):
attributes += ' aria-current="page"'
- links.append(f' {label}')
+ badge = ""
+ if target_language != language:
+ attributes += ' hreflang="en"'
+ badge = 'EN'
+ links.append(f' {escape(strings[key])}{badge}')
+
+ entries = []
+ for code, details in LANGUAGES.items():
+ if code == language or state.linkable(code, english):
+ url = href(code, english)
+ elif state.linkable(code, "index.html"):
+ url = href(code, "index.html")
+ else:
+ continue
+ current = ' aria-current="true"' if code == language else ""
+ entries.append(
+ f'