From a1e9cf9df91b2ce1e4aca6174a6f4eb979f08375 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 23 Feb 2026 21:42:24 -0800 Subject: [PATCH] translations: replace gettext apt dependency with pure Python tools (#37372) --- selfdrive/ui/SConscript | 14 -- selfdrive/ui/translations/potools.py | 362 +++++++++++++++++++++++++++ selfdrive/ui/update_translations.py | 22 +- system/ui/lib/multilang.py | 154 ++++++++++-- tools/setup_dependencies.sh | 3 +- 5 files changed, 511 insertions(+), 44 deletions(-) create mode 100644 selfdrive/ui/translations/potools.py diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index a03a26bea8..20ba2e44d0 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -1,6 +1,4 @@ -import os import re -import json from pathlib import Path Import('env', 'arch', 'common') @@ -18,18 +16,6 @@ env.Command( action=f"python3 {generator}", ) -# compile gettext .po -> .mo translations -with open(File("translations/languages.json").abspath) as f: - languages = json.loads(f.read()) - -po_sources = [f"#selfdrive/ui/translations/app_{l}.po" for l in languages.values()] -po_sources = [src for src in po_sources if os.path.exists(File(src).abspath)] -mo_targets = [src.replace(".po", ".mo") for src in po_sources] -mo_build = [] -for src, tgt in zip(po_sources, mo_targets): - mo_build.append(env.Command(tgt, src, "msgfmt -o $TARGET $SOURCE")) -mo_alias = env.Alias('mo', mo_build) -env.AlwaysBuild(mo_alias) if GetOption('extras'): # build installers diff --git a/selfdrive/ui/translations/potools.py b/selfdrive/ui/translations/potools.py new file mode 100644 index 0000000000..7571cccdd6 --- /dev/null +++ b/selfdrive/ui/translations/potools.py @@ -0,0 +1,362 @@ +"""Pure Python tools for managing .po translation files. + +Replaces GNU gettext CLI tools (xgettext, msginit, msgmerge) with Python +implementations for extracting, creating, and updating .po files. +""" + +import ast +import os +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path + + +@dataclass +class POEntry: + msgid: str = "" + msgstr: str = "" + msgid_plural: str = "" + msgstr_plural: dict[int, str] = field(default_factory=dict) + comments: list[str] = field(default_factory=list) + source_refs: list[str] = field(default_factory=list) + flags: list[str] = field(default_factory=list) + + @property + def is_plural(self) -> bool: + return bool(self.msgid_plural) + + +# ──── PO file parsing ──── + +def _parse_quoted(s: str) -> str: + """Parse a PO-format quoted string, handling escape sequences.""" + s = s.strip() + if not (s.startswith('"') and s.endswith('"')): + raise ValueError(f"Expected quoted string: {s!r}") + s = s[1:-1] + result = [] + i = 0 + while i < len(s): + if s[i] == '\\' and i + 1 < len(s): + c = s[i + 1] + if c == 'n': + result.append('\n') + elif c == 't': + result.append('\t') + elif c == '"': + result.append('"') + elif c == '\\': + result.append('\\') + else: + result.append(s[i:i + 2]) + i += 2 + else: + result.append(s[i]) + i += 1 + return ''.join(result) + + +def parse_po(path: str | Path) -> tuple[POEntry | None, list[POEntry]]: + """Parse a .po/.pot file. Returns (header_entry, entries).""" + with open(path, encoding='utf-8') as f: + lines = f.readlines() + + entries: list[POEntry] = [] + header: POEntry | None = None + cur: POEntry | None = None + cur_field: str | None = None + plural_idx = 0 + + def finish(): + nonlocal cur, header + if cur is None: + return + if cur.msgid == "" and cur.msgstr: + header = cur + elif cur.msgid != "" or cur.is_plural: + entries.append(cur) + cur = None + + for raw in lines: + line = raw.rstrip('\n') + stripped = line.strip() + + if not stripped: + finish() + cur_field = None + continue + + # Skip obsolete entries + if stripped.startswith('#~'): + continue + + if stripped.startswith('#'): + if cur is None: + cur = POEntry() + if stripped.startswith('#:'): + cur.source_refs.append(stripped[2:].strip()) + elif stripped.startswith('#,'): + cur.flags.extend(f.strip() for f in stripped[2:].split(',') if f.strip()) + else: + cur.comments.append(line) + continue + + if stripped.startswith('msgid_plural '): + if cur is None: + cur = POEntry() + cur.msgid_plural = _parse_quoted(stripped[len('msgid_plural '):]) + cur_field = 'msgid_plural' + continue + + if stripped.startswith('msgid '): + if cur is None: + cur = POEntry() + cur.msgid = _parse_quoted(stripped[len('msgid '):]) + cur_field = 'msgid' + continue + + m = re.match(r'msgstr\[(\d+)]\s+(.*)', stripped) + if m: + plural_idx = int(m.group(1)) + cur.msgstr_plural[plural_idx] = _parse_quoted(m.group(2)) + cur_field = 'msgstr_plural' + continue + + if stripped.startswith('msgstr '): + cur.msgstr = _parse_quoted(stripped[len('msgstr '):]) + cur_field = 'msgstr' + continue + + if stripped.startswith('"'): + val = _parse_quoted(stripped) + if cur_field == 'msgid': + cur.msgid += val + elif cur_field == 'msgid_plural': + cur.msgid_plural += val + elif cur_field == 'msgstr': + cur.msgstr += val + elif cur_field == 'msgstr_plural': + cur.msgstr_plural[plural_idx] += val + + finish() + return header, entries + + +# ──── PO file writing ──── + +def _quote(s: str) -> str: + """Quote a string for .po file output.""" + s = s.replace('\\', '\\\\').replace('"', '\\"').replace('\t', '\\t') + if '\n' in s and s != '\n': + parts = s.split('\n') + lines = ['""'] + for i, part in enumerate(parts): + text = part + ('\\n' if i < len(parts) - 1 else '') + if text: + lines.append(f'"{text}"') + return '\n'.join(lines) + return f'"{s}"'.replace('\n', '\\n') + + +def write_po(path: str | Path, header: POEntry | None, entries: list[POEntry]) -> None: + """Write a .po/.pot file.""" + with open(path, 'w', encoding='utf-8') as f: + if header: + for c in header.comments: + f.write(c + '\n') + if header.flags: + f.write('#, ' + ', '.join(header.flags) + '\n') + f.write(f'msgid {_quote("")}\n') + f.write(f'msgstr {_quote(header.msgstr)}\n\n') + + for entry in entries: + for c in entry.comments: + f.write(c + '\n') + for ref in entry.source_refs: + f.write(f'#: {ref}\n') + if entry.flags: + f.write('#, ' + ', '.join(entry.flags) + '\n') + f.write(f'msgid {_quote(entry.msgid)}\n') + if entry.is_plural: + f.write(f'msgid_plural {_quote(entry.msgid_plural)}\n') + for idx in sorted(entry.msgstr_plural): + f.write(f'msgstr[{idx}] {_quote(entry.msgstr_plural[idx])}\n') + else: + f.write(f'msgstr {_quote(entry.msgstr)}\n') + f.write('\n') + + +# ──── String extraction (replaces xgettext) ──── + +def extract_strings(files: list[str], basedir: str) -> list[POEntry]: + """Extract tr/trn/tr_noop calls from Python source files.""" + seen: dict[str, POEntry] = {} + + for filepath in files: + full = os.path.join(basedir, filepath) + with open(full, encoding='utf-8') as f: + source = f.read() + try: + tree = ast.parse(source, filename=filepath) + except SyntaxError: + continue + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + + func = node.func + if isinstance(func, ast.Name): + name = func.id + elif isinstance(func, ast.Attribute): + name = func.attr + else: + continue + + if name not in ('tr', 'trn', 'tr_noop'): + continue + + ref = f'{filepath}:{node.lineno}' + is_flagged = name in ('tr', 'trn') + + if name in ('tr', 'tr_noop'): + if not node.args or not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str): + continue + msgid = node.args[0].value + if msgid in seen: + if ref not in seen[msgid].source_refs: + seen[msgid].source_refs.append(ref) + else: + flags = ['python-format'] if is_flagged else [] + seen[msgid] = POEntry(msgid=msgid, source_refs=[ref], flags=flags) + + elif name == 'trn': + if len(node.args) < 2: + continue + a1, a2 = node.args[0], node.args[1] + if not (isinstance(a1, ast.Constant) and isinstance(a1.value, str)): + continue + if not (isinstance(a2, ast.Constant) and isinstance(a2.value, str)): + continue + msgid, msgid_plural = a1.value, a2.value + if msgid in seen: + if ref not in seen[msgid].source_refs: + seen[msgid].source_refs.append(ref) + else: + flags = ['python-format'] if is_flagged else [] + seen[msgid] = POEntry( + msgid=msgid, msgid_plural=msgid_plural, + source_refs=[ref], flags=flags, + msgstr_plural={0: '', 1: ''}, + ) + + return list(seen.values()) + + +# ──── POT generation ──── + +def generate_pot(entries: list[POEntry], pot_path: str | Path) -> None: + """Generate a .pot template file from extracted entries.""" + now = datetime.now(UTC).strftime('%Y-%m-%d %H:%M%z') + header = POEntry( + comments=[ + '# SOME DESCRIPTIVE TITLE.', + "# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER", + '# This file is distributed under the same license as the PACKAGE package.', + '# FIRST AUTHOR , YEAR.', + '#', + ], + flags=['fuzzy'], + msgstr='Project-Id-Version: PACKAGE VERSION\n' + + 'Report-Msgid-Bugs-To: \n' + + f'POT-Creation-Date: {now}\n' + + 'PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n' + + 'Last-Translator: FULL NAME \n' + + 'Language-Team: LANGUAGE \n' + + 'Language: \n' + + 'MIME-Version: 1.0\n' + + 'Content-Type: text/plain; charset=UTF-8\n' + + 'Content-Transfer-Encoding: 8bit\n' + + 'Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n', + ) + write_po(pot_path, header, entries) + + +# ──── PO init (replaces msginit) ──── + +PLURAL_FORMS: dict[str, str] = { + 'en': 'nplurals=2; plural=(n != 1);', + 'de': 'nplurals=2; plural=(n != 1);', + 'fr': 'nplurals=2; plural=(n > 1);', + 'es': 'nplurals=2; plural=(n != 1);', + 'pt-BR': 'nplurals=2; plural=(n > 1);', + 'tr': 'nplurals=2; plural=(n != 1);', + 'uk': 'nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);', + 'th': 'nplurals=1; plural=0;', + 'zh-CHT': 'nplurals=1; plural=0;', + 'zh-CHS': 'nplurals=1; plural=0;', + 'ko': 'nplurals=1; plural=0;', + 'ja': 'nplurals=1; plural=0;', +} + + +def init_po(pot_path: str | Path, po_path: str | Path, language: str) -> None: + """Create a new .po file from a .pot template (replaces msginit).""" + _, entries = parse_po(pot_path) + plural_forms = PLURAL_FORMS.get(language, 'nplurals=2; plural=(n != 1);') + now = datetime.now(UTC).strftime('%Y-%m-%d %H:%M%z') + + header = POEntry( + comments=[ + f'# {language} translations for PACKAGE package.', + "# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER", + '# This file is distributed under the same license as the PACKAGE package.', + '# Automatically generated.', + '#', + ], + msgstr='Project-Id-Version: PACKAGE VERSION\n' + + 'Report-Msgid-Bugs-To: \n' + + f'POT-Creation-Date: {now}\n' + + f'PO-Revision-Date: {now}\n' + + 'Last-Translator: Automatically generated\n' + + 'Language-Team: none\n' + + f'Language: {language}\n' + + 'MIME-Version: 1.0\n' + + 'Content-Type: text/plain; charset=UTF-8\n' + + 'Content-Transfer-Encoding: 8bit\n' + + f'Plural-Forms: {plural_forms}\n', + ) + + nplurals = int(re.search(r'nplurals=(\d+)', plural_forms).group(1)) + for e in entries: + if e.is_plural: + e.msgstr_plural = dict.fromkeys(range(nplurals), '') + + write_po(po_path, header, entries) + + +# ──── PO merge (replaces msgmerge) ──── + +def merge_po(po_path: str | Path, pot_path: str | Path) -> None: + """Update a .po file with entries from a .pot template (replaces msgmerge --update).""" + po_header, po_entries = parse_po(po_path) + _, pot_entries = parse_po(pot_path) + + existing = {e.msgid: e for e in po_entries} + merged = [] + + for pot_e in pot_entries: + if pot_e.msgid in existing: + old = existing[pot_e.msgid] + old.source_refs = pot_e.source_refs + old.flags = pot_e.flags + old.comments = pot_e.comments + if pot_e.is_plural: + old.msgid_plural = pot_e.msgid_plural + merged.append(old) + else: + merged.append(pot_e) + + merged.sort(key=lambda e: e.msgid) + write_po(po_path, po_header, merged) diff --git a/selfdrive/ui/update_translations.py b/selfdrive/ui/update_translations.py index bded80b2e5..6ff3667d8a 100755 --- a/selfdrive/ui/update_translations.py +++ b/selfdrive/ui/update_translations.py @@ -3,6 +3,7 @@ from itertools import chain import os from openpilot.common.basedir import BASEDIR from openpilot.system.ui.lib.multilang import SYSTEM_UI_DIR, UI_DIR, TRANSLATIONS_DIR, multilang +from openpilot.selfdrive.ui.translations.potools import extract_strings, generate_pot, merge_po, init_po LANGUAGES_FILE = os.path.join(str(TRANSLATIONS_DIR), "languages.json") POT_FILE = os.path.join(str(TRANSLATIONS_DIR), "app.pot") @@ -18,24 +19,17 @@ def update_translations(): if filename.endswith(".py"): files.append(os.path.relpath(os.path.join(root, filename), BASEDIR)) - # Create main translation file - cmd = ("xgettext -L Python --keyword=tr --keyword=trn:1,2 --keyword=tr_noop --from-code=UTF-8 " + - "--flag=tr:1:python-brace-format --flag=trn:1:python-brace-format --flag=trn:2:python-brace-format " + - f"-D {BASEDIR} -o {POT_FILE} {' '.join(files)}") - - ret = os.system(cmd) - assert ret == 0 + # Extract translatable strings and generate .pot template + entries = extract_strings(files, BASEDIR) + generate_pot(entries, POT_FILE) # Generate/update translation files for each language for name in multilang.languages.values(): - if os.path.exists(os.path.join(TRANSLATIONS_DIR, f"app_{name}.po")): - cmd = f"msgmerge --update --no-fuzzy-matching --backup=none --sort-output {TRANSLATIONS_DIR}/app_{name}.po {POT_FILE}" - ret = os.system(cmd) - assert ret == 0 + po_file = os.path.join(TRANSLATIONS_DIR, f"app_{name}.po") + if os.path.exists(po_file): + merge_po(po_file, POT_FILE) else: - cmd = f"msginit -l {name} --no-translator --input {POT_FILE} --output-file {TRANSLATIONS_DIR}/app_{name}.po" - ret = os.system(cmd) - assert ret == 0 + init_po(POT_FILE, po_file, name) if __name__ == "__main__": diff --git a/system/ui/lib/multilang.py b/system/ui/lib/multilang.py index ad2a5f9399..343c06a1e8 100644 --- a/system/ui/lib/multilang.py +++ b/system/ui/lib/multilang.py @@ -1,7 +1,7 @@ from importlib.resources import files -import os import json -import gettext +import os +import re from openpilot.common.basedir import BASEDIR from openpilot.common.swaglog import cloudlog @@ -23,14 +23,137 @@ UNIFONT_LANGUAGES = [ "ja", ] +# Plural form selectors for supported languages +PLURAL_SELECTORS = { + 'en': lambda n: 0 if n == 1 else 1, + 'de': lambda n: 0 if n == 1 else 1, + 'fr': lambda n: 0 if n <= 1 else 1, + 'pt-BR': lambda n: 0 if n <= 1 else 1, + 'es': lambda n: 0 if n == 1 else 1, + 'tr': lambda n: 0 if n == 1 else 1, + 'uk': lambda n: 0 if n % 10 == 1 and n % 100 != 11 else (1 if 2 <= n % 10 <= 4 and not 12 <= n % 100 <= 14 else 2), + 'th': lambda n: 0, + 'zh-CHT': lambda n: 0, + 'zh-CHS': lambda n: 0, + 'ko': lambda n: 0, + 'ja': lambda n: 0, +} + + +def _parse_quoted(s: str) -> str: + """Parse a PO-format quoted string.""" + s = s.strip() + if not (s.startswith('"') and s.endswith('"')): + raise ValueError(f"Expected quoted string: {s!r}") + s = s[1:-1] + result: list[str] = [] + i = 0 + while i < len(s): + if s[i] == '\\' and i + 1 < len(s): + c = s[i + 1] + if c == 'n': + result.append('\n') + elif c == 't': + result.append('\t') + elif c == '"': + result.append('"') + elif c == '\\': + result.append('\\') + else: + result.append(s[i:i + 2]) + i += 2 + else: + result.append(s[i]) + i += 1 + return ''.join(result) + + +def load_translations(path) -> tuple[dict[str, str], dict[str, list[str]]]: + """Parse a .po file and return (translations, plurals) dicts. + + translations: msgid -> msgstr + plurals: msgid -> [msgstr[0], msgstr[1], ...] + """ + with open(str(path), encoding='utf-8') as f: + lines = f.readlines() + + translations: dict[str, str] = {} + plurals: dict[str, list[str]] = {} + + # Parser state + msgid = msgid_plural = msgstr = "" + msgstr_plurals: dict[int, str] = {} + field: str | None = None + plural_idx = 0 + + def finish(): + nonlocal msgid, msgid_plural, msgstr, msgstr_plurals, field + if msgid: # skip header (empty msgid) + if msgid_plural: + max_idx = max(msgstr_plurals.keys()) if msgstr_plurals else 0 + plurals[msgid] = [msgstr_plurals.get(i, '') for i in range(max_idx + 1)] + else: + translations[msgid] = msgstr + msgid = msgid_plural = msgstr = "" + msgstr_plurals = {} + field = None + + for raw in lines: + line = raw.strip() + + if not line: + finish() + continue + + if line.startswith('#'): + continue + + if line.startswith('msgid_plural '): + msgid_plural = _parse_quoted(line[len('msgid_plural '):]) + field = 'msgid_plural' + continue + + if line.startswith('msgid '): + msgid = _parse_quoted(line[len('msgid '):]) + field = 'msgid' + continue + + m = re.match(r'msgstr\[(\d+)]\s+(.*)', line) + if m: + plural_idx = int(m.group(1)) + msgstr_plurals[plural_idx] = _parse_quoted(m.group(2)) + field = 'msgstr_plural' + continue + + if line.startswith('msgstr '): + msgstr = _parse_quoted(line[len('msgstr '):]) + field = 'msgstr' + continue + + if line.startswith('"'): + val = _parse_quoted(line) + if field == 'msgid': + msgid += val + elif field == 'msgid_plural': + msgid_plural += val + elif field == 'msgstr': + msgstr += val + elif field == 'msgstr_plural': + msgstr_plurals[plural_idx] += val + + finish() + return translations, plurals + class Multilang: def __init__(self): self._params = Params() if Params is not None else None self._language: str = "en" - self.languages = {} - self.codes = {} - self._translation: gettext.NullTranslations | gettext.GNUTranslations = gettext.NullTranslations() + self.languages: dict[str, str] = {} + self.codes: dict[str, str] = {} + self._translations: dict[str, str] = {} + self._plurals: dict[str, list[str]] = {} + self._plural_selector = PLURAL_SELECTORS.get('en', lambda n: 0) self._load_languages() @property @@ -43,27 +166,30 @@ class Multilang: def setup(self): try: - with TRANSLATIONS_DIR.joinpath(f'app_{self._language}.mo').open('rb') as fh: - translation = gettext.GNUTranslations(fh) - translation.install() - self._translation = translation + po_path = TRANSLATIONS_DIR.joinpath(f'app_{self._language}.po') + self._translations, self._plurals = load_translations(po_path) + self._plural_selector = PLURAL_SELECTORS.get(self._language, lambda n: 0) cloudlog.debug(f"Loaded translations for language: {self._language}") except FileNotFoundError: cloudlog.error(f"No translation file found for language: {self._language}, using default.") - gettext.install('app') - self._translation = gettext.NullTranslations() + self._translations = {} + self._plurals = {} def change_language(self, language_code: str) -> None: - # Reinstall gettext with the selected language self._params.put("LanguageSetting", language_code) self._language = language_code self.setup() def tr(self, text: str) -> str: - return self._translation.gettext(text) + return self._translations.get(text, text) def trn(self, singular: str, plural: str, n: int) -> str: - return self._translation.ngettext(singular, plural, n) + if singular in self._plurals: + idx = self._plural_selector(n) + forms = self._plurals[singular] + if idx < len(forms) and forms[idx]: + return forms[idx] + return singular if n == 1 else plural def _load_languages(self): with LANGUAGES_FILE.open(encoding='utf-8') as f: diff --git a/tools/setup_dependencies.sh b/tools/setup_dependencies.sh index 79f56333cf..98f1d878f8 100755 --- a/tools/setup_dependencies.sh +++ b/tools/setup_dependencies.sh @@ -51,8 +51,7 @@ function install_ubuntu_deps() { $SUDO apt-get install -y --no-install-recommends \ python3-dev \ libncurses5-dev \ - libzstd-dev \ - gettext + libzstd-dev if [[ -d "/etc/udev/rules.d/" ]]; then # Setup jungle udev rules