]*\bid="{re.escape(identifier)}"[^>]*>)(.*?)()', re.S)
def synced_code_ids(source):
"""Return the copy targets that are code blocks; translations take them from English."""
return [identifier for identifier in copy_targets(source) if code_block(identifier).search(source)]
def source_digest(source):
"""Hash an English page's translatable content.
Shared regions and copied code blocks are left out, so template edits and
command-only changes do not mark translations out of date.
"""
content = SHARED_REGION.sub("", source)
for identifier in synced_code_ids(source):
content = code_block(identifier).sub(r"\1\3", content)
return sha256(" ".join(content.split()).encode("utf-8")).hexdigest()
def sync_code_blocks(english, source, label):
"""Copy every synced code block's contents from the English source, matched by ID."""
for identifier in synced_code_ids(english):
contents = code_block(identifier).search(english).group(2)
pattern = code_block(identifier)
if len(pattern.findall(source)) != 1:
raise ValueError(f"{label}: needs one block copied from the English page")
source = pattern.sub(lambda match: match.group(1) + contents + match.group(3), source)
return source
class SiteState:
"""Read the locales and every translation's status once for a sync or check."""
def __init__(self, root):
self.root = root
self.site = root / "website/public"
self.locales = load_locales(root)
self.translations = {}
for path in sorted(self.site.rglob("*.html")):
language, english = page_language(self.site, path)
if language != "en":
record = translation_record(path.read_text(encoding="utf-8")) or {}
self.translations[(language, english)] = record.get("status")
def linkable(self, language, english, from_draft=False):
"""Report whether other pages may link to this language's version of a page."""
if language == "en":
return (self.site / english).is_file()
status = self.translations.get((language, english))
return status == "reviewed" or (from_draft and status == "draft")
def alternates(self, english):
"""Return the languages whose versions of a page join its hreflang cluster."""
return [code for code in LANGUAGES if code == "en" or self.translations.get((code, english)) == "reviewed"]
def language_url(language, english):
"""Return the canonical URL of one language's version of a page."""
prefix = "" if language == "en" else f"{language}/"
return f"{SITE_ORIGIN}/{quote(prefix + file_route(english), safe='/')}"
def shared_blocks(root, path, source, state=None):
"""Render shared regions while keeping the page's body and metadata authored."""
state = state or SiteState(root)
site = state.site
metadata = PageMetadata()
metadata.feed(source)
for name, values in (("title", metadata.titles), ("description", metadata.descriptions)):
if len(values) != 1 or not values[0].strip():
raise ValueError(f"{path.relative_to(site)}: needs one nonempty head {name}")
language, english = page_language(site, path)
strings = state.locales[language]["strings"]
depth = len(path.relative_to(site).parent.parts)
base = "../" * depth or "./"
route = file_route(english)
draft = language != "en" and state.translations.get((language, english)) == "draft"
def href(target_language, target):
prefix = "" if target_language == "en" else f"{target_language}/"
return base + file_route(prefix + target)
def localized(target):
"""Prefer this language's version of a target page, else the English page."""
if language != "en" and state.linkable(language, target, from_draft=draft):
return language, href(language, target)
return "en", href("en", target)
links = []
for key, destination in NAVIGATION:
target_language, url = localized(destination + "index.html")
attributes = ' class="nav-start"' if destination == "get-started/" else ""
if route.startswith(destination):
attributes += ' aria-current="page"'
badge = ""
if target_language != language:
attributes += ' hreflang="en"'
badge = ''
links.append(f' {escape(strings[key])}{badge}')
entries = []
for code, details in LANGUAGES.items():
if code == language or state.linkable(code, english):
url = href(code, english)
elif state.linkable(code, "index.html"):
url = href(code, "index.html")
else:
continue
current = ' aria-current="true"' if code == language else ""
entries.append(
f' {details.name} '
)
language_menu = ""
if len(entries) > 1:
language_menu = "\n".join((
"",
' ",
))
members = [] if draft else state.alternates(english)
hreflang = ""
locale_alternates = ""
if len(members) > 1:
for code in [*members, "x-default"]:
tag = LANGUAGES[code].tag if code != "x-default" else code
url = language_url(code if code != "x-default" else "en", english)
hreflang += f'\n\n'
for code in members:
if code != language:
locale_alternates += (
f'\n\n'
)
labels = "".join(
f'\n data-label-{category}="{escape(label, quote=True)}"'
for category, label in state.locales[language]["githubStars"].items()
)
values = {
"base": base,
"brand": localized("index.html")[1],
"navigation": "\n" + "\n".join(links) + "\n ",
"language_menu": language_menu,
"canonical": escape(page_url(site, path), quote=True),
"hreflang": hreflang,
"og_locale": LANGUAGES[language].og_locale,
"og_locale_alternates": locale_alternates,
"origin": SITE_ORIGIN,
"title": escape(metadata.titles[0].strip(), quote=True),
"description": escape(metadata.descriptions[0].strip(), quote=True),
"github_star_labels": labels,
**{f"t_{key}": escape(value, quote=True) for key, value in strings.items()},
}
templates = root / "website/templates"
return {
name: indent(Template((templates / f"{name}.html.tmpl").read_text(encoding="utf-8")).substitute(values), " ")
for name in ("sharing", "header", "footer")
}
def translation_notice(state, path, english_source, record):
"""Render the stale-translation notice, or nothing while the source hash matches."""
if record is None or record.get("source-sha256") == source_digest(english_source):
return ""
language, english = page_language(state.site, path)
strings = state.locales[language]["strings"]
depth = len(path.relative_to(state.site).parent.parts)
link = (
f''
f'{escape(strings["staleNoticeLink"])}'
)
message = escape(strings["staleNotice"]).replace("{link}", link)
return f' \n {message}
\n'
def replace_region(source, start, end, block, label, name):
"""Replace the contents of one marked region."""
if source.count(start) != 1 or source.count(end) != 1:
raise ValueError(f"{label}: needs one {name} region; use new-page or translate for a new page")
before, _, remaining = source.partition(start)
_, found, after = remaining.partition(end)
if not found:
raise ValueError(f"{label}: {name} region is out of order")
return before + start + block + end + after
def region_contents(source, start, end):
"""Return the current contents of a marked region."""
return source.partition(start)[2].partition(end)[0]
def sync_pages(root, check=False):
"""Update shared regions, or reject drift without changing any authored file.
Returns the pages that changed. With check=True, structural drift raises, so
the returned pages are translations whose only drift is the stale notice;
an out-of-date translation is a warning rather than a failure.
"""
state = SiteState(root)
site = state.site
changes = []
notices = []
for path in sorted(site.rglob("*.html")):
if path.relative_to(site).as_posix() in NOTES_FIXTURES:
continue
label = path.relative_to(site).as_posix()
original = path.read_text(encoding="utf-8")
source = original
language, english = page_language(site, path)
english_source = None
if language != "en":
english_path = site / english
if not english_path.is_file():
raise ValueError(f"{label}: translates {english}, which does not exist")
english_source = english_path.read_text(encoding="utf-8")
source = sync_code_blocks(english_source, source, label)
for name, block in shared_blocks(root, path, source, state).items():
source = replace_region(
source, f" \n", f" ", block, label, name
)
if language != "en":
notice = translation_notice(state, path, english_source, translation_record(source))
structural = replace_region(
source, NOTICE_START, NOTICE_END, region_contents(original, NOTICE_START, NOTICE_END), label,
"translation-notice",
)
source = replace_region(source, NOTICE_START, NOTICE_END, notice, label, "translation-notice")
if structural == original and source != original:
notices.append((path, source))
continue
if source != original:
changes.append((path, source))
if check and changes:
names = ", ".join(str(path.relative_to(site)) for path, _ in changes)
raise ValueError(f"Shared HTML is stale: {names}. Run python3 scripts/website.py sync and commit the updated pages.")
if not check:
for path, source in changes + notices:
path.write_text(source, encoding="utf-8")
if check:
print(f"Checked shared HTML ({len(notices)} translation notices to refresh).")
else:
print(f"Synced shared HTML ({len(changes) + len(notices)} pages changed).")
return [path for path, _ in (notices if check else changes + notices)]
def new_page(root, route, title, description):
"""Create a draft reading page with correct depth, metadata and shared regions."""
parts = PurePosixPath(route)
if not route or parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
raise ValueError("Use a relative directory route, such as blog/my-article.")
if parts.parts[0] in LANGUAGES:
raise ValueError("Create the English page first, then use translate for another language.")
site = root / "website/public"
path = site.joinpath(*parts.parts, "index.html")
if not path.resolve().is_relative_to(site.resolve()):
raise ValueError("The new page must stay inside website/public.")
if path.exists():
raise ValueError(f"Page already exists: {path.relative_to(site)}")
base = "../" * len(parts.parts) or "./"
source = Template((root / "website/templates/page.html").read_text(encoding="utf-8")).substitute(
title=escape(title, quote=True), description=escape(description, quote=True), base=base,
)
# Render before writing, so invalid input cannot leave an incomplete page.
blocks = shared_blocks(root, path, source)
for name, block in blocks.items():
source = source.replace(f" \n", f" \n{block}")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source, encoding="utf-8")
print(f"Created {path.relative_to(root)}. Write the page, add its index link, then remove the Draft label and noindex directive when ready.")
return path
def relocate_references(source, english, language, state):
"""Rewrite relative references for the translation's directory.
Pages that already have a version in this language link to it; everything
else keeps pointing at the English page or asset.
"""
english_directory = posixpath.dirname(english)
translation_directory = posixpath.join(language, english_directory)
def rewrite(match):
value = match.group(2)
url = urlsplit(value)
if not value or url.scheme or url.netloc or value.startswith(("#", "/")):
return match.group(0)
target = posixpath.normpath(posixpath.join(english_directory, url.path))
directory = url.path.endswith("/") or target == "."
if directory:
page = "index.html" if target == "." else f"{target}/index.html"
if (language, page) in state.translations:
target = posixpath.normpath(posixpath.join(language, target))
relative = posixpath.relpath(target, translation_directory or ".")
if directory:
relative = "./" if relative == "." else relative + "/"
return f'{match.group(1)}="{urlunsplit(("", "", relative, url.query, url.fragment))}"'
return re.sub(r'\b(href|src)="([^"]*)"', rewrite, source)
def new_translation(root, language, route):
"""Create a draft translation of an English page, ready to be translated in place."""
if language not in LANGUAGES or language == "en":
raise ValueError(f"Choose one of: {', '.join(code for code in LANGUAGES if code != 'en')}.")
route = route.strip("/")
parts = PurePosixPath(route) if route not in ("", ".") else PurePosixPath()
if parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
raise ValueError("Use the English page's directory route, such as get-started, or / for the home page.")
if parts.parts and parts.parts[0] in LANGUAGES:
raise ValueError("Name the English route, without a language directory.")
english = PurePosixPath(*parts.parts, "index.html").as_posix()
state = SiteState(root)
site = state.site
source_path = site / english
if not source_path.is_file() or english in NOTES_FIXTURES:
raise ValueError(f"No English page to translate at {english}.")
path = site / language / english
if path.exists():
raise ValueError(f"Translation already exists: {path.relative_to(site)}")
source = source_path.read_text(encoding="utf-8")
if source.count('') != 1 or source.count(" \n") != 1:
raise ValueError(f"{english}: needs and one sharing region")
main = re.search(r"^ {4}]*>\n", source, re.M)
if main is None:
raise ValueError(f"{english}: needs a element on its own line")
source = source.replace('', f'')
source = relocate_references(source, english, language, state)
record = (
f' \n'
)
robots = "" if '\n'
source = source.replace(" \n", robots + record + " \n")
main = re.search(r"^ {4}]*>\n", source, re.M)
source = source[: main.end()] + NOTICE_START + NOTICE_END + "\n" + source[main.end():]
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source, encoding="utf-8")
sync_pages(root)
print(
f"Created {path.relative_to(root)} as a noindex draft. Translate its text in place following "
"website/translation.md, then run python3 scripts/website.py check."
)
return path
def translation_report(root, language=None):
"""Describe each translation's review status, freshness and remaining review items."""
state = SiteState(root)
site = state.site
lines = []
for (code, english), status in sorted(state.translations.items()):
if language and code != language:
continue
path = site / code / english
source = path.read_text(encoding="utf-8")
record = translation_record(source) or {}
english_path = site / english
current = source_digest(english_path.read_text(encoding="utf-8")) if english_path.is_file() else None
fresh = record.get("source-sha256") == current
reviewed = f" on {record['reviewed']}" if record.get("reviewed") else ""
lines.append(f"{code}/{file_route(english) or '(home)'}: {status or 'no record'}{reviewed}, "
f"{'up to date' if fresh else 'English source changed'}")
if not fresh and current:
lines.append(f" after updating the translation, record source-sha256=\"{current}\"")
if status != "reviewed":
for system, text in re.findall(r'<(?:em|span|strong|kbd)\b[^>]*\bdata-os-ui="([^"]+)"[^>]*>(.*?)', source, re.S):
lines.append(f" confirm on {system}: {' '.join(text.split())}")
english_strings = state.locales["en"]["strings"]
for code in LANGUAGES:
if code == "en" or (language and code != language):
continue
if not any(key[0] == code for key in state.translations):
continue
locale = state.locales[code]
same = sorted(
key for key, value in locale["strings"].items()
if value == english_strings[key] and key not in locale.get("sameAsEnglish", [])
)
if same:
lines.append(f"{code}: UI strings still in English: {', '.join(same)}")
print("\n".join(lines) or "No translated pages.")
return lines
def run_script(root, name, arguments=()):
"""Use the active Python environment for the existing preparation and checks."""
subprocess.run([sys.executable, str(root / "scripts" / name), *arguments], cwd=root, check=True)
def run_browser_script_tests(root):
"""Run the page-script regressions with the repository's Node.js."""
try:
subprocess.run(["node", str(root / "scripts/test-website-scripts.mjs")], cwd=root, check=True)
except FileNotFoundError as error:
raise OSError("The website script tests need Node.js; see website/README.md.") from error
def preview_handler(site, prefix):
"""Serve one static directory at root or a Pages prefix, preserving redirects."""
class Handler(SimpleHTTPRequestHandler):
def send_head(self):
if not urlsplit(self.path).path.startswith(prefix):
self.send_error(404)
return None
return super().send_head()
def translate_path(self, path):
return super().translate_path("/" + path[len(prefix):])
return partial(Handler, directory=str(site))
def main():
"""Expose one small command for synchronizing, checking and previewing the site."""
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
sync = commands.add_parser("sync", help="Update shared HTML after editing templates or page metadata")
sync.add_argument("--check", action="store_true", help="Report drift without rewriting pages")
commands.add_parser("check", help="Check shared HTML, prepare assets, verify links and run regression tests")
links = commands.add_parser("links", help="Audit external targets using gh and HTTP; separate from offline checks")
links.add_argument("--report", help="Write the full external-link JSON report")
create = commands.add_parser("new-page", help="Create a draft reading page")
create.add_argument("route")
create.add_argument("--title", required=True)
create.add_argument("--description", required=True)
translate = commands.add_parser("translate", help="Create a draft translation of an English page")
translate.add_argument("language", choices=[code for code in LANGUAGES if code != "en"])
translate.add_argument("route", help="The English page's directory route, or / for the home page")
report = commands.add_parser("translations", help="List translation status, freshness and review items")
report.add_argument("--lang", choices=[code for code in LANGUAGES if code != "en"])
serve = commands.add_parser("serve", help="Prepare and preview the static site")
serve.add_argument("--port", type=int, default=4173)
serve.add_argument("--base-path", choices=("/", "/rea/"), default="/")
args = parser.parse_args()
root = Path(__file__).resolve().parent.parent
try:
if args.command == "new-page":
new_page(root, args.route, args.title, args.description)
elif args.command == "translate":
new_translation(root, args.language, args.route)
elif args.command == "translations":
translation_report(root, args.lang)
elif args.command == "sync":
sync_pages(root, check=args.check)
elif args.command == "check":
sync_pages(root, check=True)
for name in ("prepare-website.py", "verify-website.py", "test-website.py"):
run_script(root, name)
run_browser_script_tests(root)
elif args.command == "links":
run_script(root, "audit-website-links.py", ["--report", args.report] if args.report else [])
else:
sync_pages(root)
run_script(root, "prepare-website.py")
prefix = args.base_path
handler = preview_handler(root / "website/public", prefix)
with ThreadingHTTPServer(("127.0.0.1", args.port), handler) as server:
print(f"Preview: http://127.0.0.1:{server.server_port}{prefix}", flush=True)
server.serve_forever()
except (OSError, ValueError, KeyError) as error:
print(str(error), file=sys.stderr)
return 1
except subprocess.CalledProcessError as error:
return error.returncode
except KeyboardInterrupt:
return 0
return 0
if __name__ == "__main__":
sys.exit(main())