translations: replace gettext apt dependency with pure Python tools (#37372)

selfdrive/ui/SConscript

@@ -1,6 +1,4 @@
-import os
 import re
-import json
 from pathlib import Path
 Import('env', 'arch', 'common')
 
@@ -18,18 +16,6 @@ env.Command(
   action=f"python3 {generator}",
 )
 
-# compile gettext .po -> .mo translations
-with open(File("translations/languages.json").abspath) as f:
-  languages = json.loads(f.read())
-
-po_sources = [f"#selfdrive/ui/translations/app_{l}.po" for l in languages.values()]
-po_sources = [src for src in po_sources if os.path.exists(File(src).abspath)]
-mo_targets = [src.replace(".po", ".mo") for src in po_sources]
-mo_build = []
-for src, tgt in zip(po_sources, mo_targets):
-  mo_build.append(env.Command(tgt, src, "msgfmt -o $TARGET $SOURCE"))
-mo_alias = env.Alias('mo', mo_build)
-env.AlwaysBuild(mo_alias)
 
 if GetOption('extras'):
   # build installers

selfdrive/ui/translations/potools.py

@@ -0,0 +1,362 @@
+"""Pure Python tools for managing .po translation files.
+
+Replaces GNU gettext CLI tools (xgettext, msginit, msgmerge) with Python
+implementations for extracting, creating, and updating .po files.
+"""
+
+import ast
+import os
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+
+@dataclass
+class POEntry:
+  msgid: str = ""
+  msgstr: str = ""
+  msgid_plural: str = ""
+  msgstr_plural: dict[int, str] = field(default_factory=dict)
+  comments: list[str] = field(default_factory=list)
+  source_refs: list[str] = field(default_factory=list)
+  flags: list[str] = field(default_factory=list)
+
+  @property
+  def is_plural(self) -> bool:
+    return bool(self.msgid_plural)
+
+
+# ──── PO file parsing ────
+
+def _parse_quoted(s: str) -> str:
+  """Parse a PO-format quoted string, handling escape sequences."""
+  s = s.strip()
+  if not (s.startswith('"') and s.endswith('"')):
+    raise ValueError(f"Expected quoted string: {s!r}")
+  s = s[1:-1]
+  result = []
+  i = 0
+  while i < len(s):
+    if s[i] == '\\' and i + 1 < len(s):
+      c = s[i + 1]
+      if c == 'n':
+        result.append('\n')
+      elif c == 't':
+        result.append('\t')
+      elif c == '"':
+        result.append('"')
+      elif c == '\\':
+        result.append('\\')
+      else:
+        result.append(s[i:i + 2])
+      i += 2
+    else:
+      result.append(s[i])
+      i += 1
+  return ''.join(result)
+
+
+def parse_po(path: str | Path) -> tuple[POEntry | None, list[POEntry]]:
+  """Parse a .po/.pot file. Returns (header_entry, entries)."""
+  with open(path, encoding='utf-8') as f:
+    lines = f.readlines()
+
+  entries: list[POEntry] = []
+  header: POEntry | None = None
+  cur: POEntry | None = None
+  cur_field: str | None = None
+  plural_idx = 0
+
+  def finish():
+    nonlocal cur, header
+    if cur is None:
+      return
+    if cur.msgid == "" and cur.msgstr:
+      header = cur
+    elif cur.msgid != "" or cur.is_plural:
+      entries.append(cur)
+    cur = None
+
+  for raw in lines:
+    line = raw.rstrip('\n')
+    stripped = line.strip()
+
+    if not stripped:
+      finish()
+      cur_field = None
+      continue
+
+    # Skip obsolete entries
+    if stripped.startswith('#~'):
+      continue
+
+    if stripped.startswith('#'):
+      if cur is None:
+        cur = POEntry()
+      if stripped.startswith('#:'):
+        cur.source_refs.append(stripped[2:].strip())
+      elif stripped.startswith('#,'):
+        cur.flags.extend(f.strip() for f in stripped[2:].split(',') if f.strip())
+      else:
+        cur.comments.append(line)
+      continue
+
+    if stripped.startswith('msgid_plural '):
+      if cur is None:
+        cur = POEntry()
+      cur.msgid_plural = _parse_quoted(stripped[len('msgid_plural '):])
+      cur_field = 'msgid_plural'
+      continue
+
+    if stripped.startswith('msgid '):
+      if cur is None:
+        cur = POEntry()
+      cur.msgid = _parse_quoted(stripped[len('msgid '):])
+      cur_field = 'msgid'
+      continue
+
+    m = re.match(r'msgstr\[(\d+)]\s+(.*)', stripped)
+    if m:
+      plural_idx = int(m.group(1))
+      cur.msgstr_plural[plural_idx] = _parse_quoted(m.group(2))
+      cur_field = 'msgstr_plural'
+      continue
+
+    if stripped.startswith('msgstr '):
+      cur.msgstr = _parse_quoted(stripped[len('msgstr '):])
+      cur_field = 'msgstr'
+      continue
+
+    if stripped.startswith('"'):
+      val = _parse_quoted(stripped)
+      if cur_field == 'msgid':
+        cur.msgid += val
+      elif cur_field == 'msgid_plural':
+        cur.msgid_plural += val
+      elif cur_field == 'msgstr':
+        cur.msgstr += val
+      elif cur_field == 'msgstr_plural':
+        cur.msgstr_plural[plural_idx] += val
+
+  finish()
+  return header, entries
+
+
+# ──── PO file writing ────
+
+def _quote(s: str) -> str:
+  """Quote a string for .po file output."""
+  s = s.replace('\\', '\\\\').replace('"', '\\"').replace('\t', '\\t')
+  if '\n' in s and s != '\n':
+    parts = s.split('\n')
+    lines = ['""']
+    for i, part in enumerate(parts):
+      text = part + ('\\n' if i < len(parts) - 1 else '')
+      if text:
+        lines.append(f'"{text}"')
+    return '\n'.join(lines)
+  return f'"{s}"'.replace('\n', '\\n')
+
+
+def write_po(path: str | Path, header: POEntry | None, entries: list[POEntry]) -> None:
+  """Write a .po/.pot file."""
+  with open(path, 'w', encoding='utf-8') as f:
+    if header:
+      for c in header.comments:
+        f.write(c + '\n')
+      if header.flags:
+        f.write('#, ' + ', '.join(header.flags) + '\n')
+      f.write(f'msgid {_quote("")}\n')
+      f.write(f'msgstr {_quote(header.msgstr)}\n\n')
+
+    for entry in entries:
+      for c in entry.comments:
+        f.write(c + '\n')
+      for ref in entry.source_refs:
+        f.write(f'#: {ref}\n')
+      if entry.flags:
+        f.write('#, ' + ', '.join(entry.flags) + '\n')
+      f.write(f'msgid {_quote(entry.msgid)}\n')
+      if entry.is_plural:
+        f.write(f'msgid_plural {_quote(entry.msgid_plural)}\n')
+        for idx in sorted(entry.msgstr_plural):
+          f.write(f'msgstr[{idx}] {_quote(entry.msgstr_plural[idx])}\n')
+      else:
+        f.write(f'msgstr {_quote(entry.msgstr)}\n')
+      f.write('\n')
+
+
+# ──── String extraction (replaces xgettext) ────
+
+def extract_strings(files: list[str], basedir: str) -> list[POEntry]:
+  """Extract tr/trn/tr_noop calls from Python source files."""
+  seen: dict[str, POEntry] = {}
+
+  for filepath in files:
+    full = os.path.join(basedir, filepath)
+    with open(full, encoding='utf-8') as f:
+      source = f.read()
+    try:
+      tree = ast.parse(source, filename=filepath)
+    except SyntaxError:
+      continue
+
+    for node in ast.walk(tree):
+      if not isinstance(node, ast.Call):
+        continue
+
+      func = node.func
+      if isinstance(func, ast.Name):
+        name = func.id
+      elif isinstance(func, ast.Attribute):
+        name = func.attr
+      else:
+        continue
+
+      if name not in ('tr', 'trn', 'tr_noop'):
+        continue
+
+      ref = f'{filepath}:{node.lineno}'
+      is_flagged = name in ('tr', 'trn')
+
+      if name in ('tr', 'tr_noop'):
+        if not node.args or not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str):
+          continue
+        msgid = node.args[0].value
+        if msgid in seen:
+          if ref not in seen[msgid].source_refs:
+            seen[msgid].source_refs.append(ref)
+        else:
+          flags = ['python-format'] if is_flagged else []
+          seen[msgid] = POEntry(msgid=msgid, source_refs=[ref], flags=flags)
+
+      elif name == 'trn':
+        if len(node.args) < 2:
+          continue
+        a1, a2 = node.args[0], node.args[1]
+        if not (isinstance(a1, ast.Constant) and isinstance(a1.value, str)):
+          continue
+        if not (isinstance(a2, ast.Constant) and isinstance(a2.value, str)):
+          continue
+        msgid, msgid_plural = a1.value, a2.value
+        if msgid in seen:
+          if ref not in seen[msgid].source_refs:
+            seen[msgid].source_refs.append(ref)
+        else:
+          flags = ['python-format'] if is_flagged else []
+          seen[msgid] = POEntry(
+            msgid=msgid, msgid_plural=msgid_plural,
+            source_refs=[ref], flags=flags,
+            msgstr_plural={0: '', 1: ''},
+          )
+
+  return list(seen.values())
+
+
+# ──── POT generation ────
+
+def generate_pot(entries: list[POEntry], pot_path: str | Path) -> None:
+  """Generate a .pot template file from extracted entries."""
+  now = datetime.now(UTC).strftime('%Y-%m-%d %H:%M%z')
+  header = POEntry(
+    comments=[
+      '# SOME DESCRIPTIVE TITLE.',
+      "# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER",
+      '# This file is distributed under the same license as the PACKAGE package.',
+      '# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.',
+      '#',
+    ],
+    flags=['fuzzy'],
+    msgstr='Project-Id-Version: PACKAGE VERSION\n' +
+      'Report-Msgid-Bugs-To: \n' +
+      f'POT-Creation-Date: {now}\n' +
+      'PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n' +
+      'Last-Translator: FULL NAME <EMAIL@ADDRESS>\n' +
+      'Language-Team: LANGUAGE <LL@li.org>\n' +
+      'Language: \n' +
+      'MIME-Version: 1.0\n' +
+      'Content-Type: text/plain; charset=UTF-8\n' +
+      'Content-Transfer-Encoding: 8bit\n' +
+      'Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n',
+  )
+  write_po(pot_path, header, entries)
+
+
+# ──── PO init (replaces msginit) ────
+
+PLURAL_FORMS: dict[str, str] = {
+  'en': 'nplurals=2; plural=(n != 1);',
+  'de': 'nplurals=2; plural=(n != 1);',
+  'fr': 'nplurals=2; plural=(n > 1);',
+  'es': 'nplurals=2; plural=(n != 1);',
+  'pt-BR': 'nplurals=2; plural=(n > 1);',
+  'tr': 'nplurals=2; plural=(n != 1);',
+  'uk': 'nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);',
+  'th': 'nplurals=1; plural=0;',
+  'zh-CHT': 'nplurals=1; plural=0;',
+  'zh-CHS': 'nplurals=1; plural=0;',
+  'ko': 'nplurals=1; plural=0;',
+  'ja': 'nplurals=1; plural=0;',
+}
+
+
+def init_po(pot_path: str | Path, po_path: str | Path, language: str) -> None:
+  """Create a new .po file from a .pot template (replaces msginit)."""
+  _, entries = parse_po(pot_path)
+  plural_forms = PLURAL_FORMS.get(language, 'nplurals=2; plural=(n != 1);')
+  now = datetime.now(UTC).strftime('%Y-%m-%d %H:%M%z')
+
+  header = POEntry(
+    comments=[
+      f'# {language} translations for PACKAGE package.',
+      "# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER",
+      '# This file is distributed under the same license as the PACKAGE package.',
+      '# Automatically generated.',
+      '#',
+    ],
+    msgstr='Project-Id-Version: PACKAGE VERSION\n' +
+      'Report-Msgid-Bugs-To: \n' +
+      f'POT-Creation-Date: {now}\n' +
+      f'PO-Revision-Date: {now}\n' +
+      'Last-Translator: Automatically generated\n' +
+      'Language-Team: none\n' +
+      f'Language: {language}\n' +
+      'MIME-Version: 1.0\n' +
+      'Content-Type: text/plain; charset=UTF-8\n' +
+      'Content-Transfer-Encoding: 8bit\n' +
+      f'Plural-Forms: {plural_forms}\n',
+  )
+
+  nplurals = int(re.search(r'nplurals=(\d+)', plural_forms).group(1))
+  for e in entries:
+    if e.is_plural:
+      e.msgstr_plural = dict.fromkeys(range(nplurals), '')
+
+  write_po(po_path, header, entries)
+
+
+# ──── PO merge (replaces msgmerge) ────
+
+def merge_po(po_path: str | Path, pot_path: str | Path) -> None:
+  """Update a .po file with entries from a .pot template (replaces msgmerge --update)."""
+  po_header, po_entries = parse_po(po_path)
+  _, pot_entries = parse_po(pot_path)
+
+  existing = {e.msgid: e for e in po_entries}
+  merged = []
+
+  for pot_e in pot_entries:
+    if pot_e.msgid in existing:
+      old = existing[pot_e.msgid]
+      old.source_refs = pot_e.source_refs
+      old.flags = pot_e.flags
+      old.comments = pot_e.comments
+      if pot_e.is_plural:
+        old.msgid_plural = pot_e.msgid_plural
+      merged.append(old)
+    else:
+      merged.append(pot_e)
+
+  merged.sort(key=lambda e: e.msgid)
+  write_po(po_path, po_header, merged)

selfdrive/ui/update_translations.py

@@ -3,6 +3,7 @@ from itertools import chain
 import os
 from openpilot.common.basedir import BASEDIR
 from openpilot.system.ui.lib.multilang import SYSTEM_UI_DIR, UI_DIR, TRANSLATIONS_DIR, multilang
+from openpilot.selfdrive.ui.translations.potools import extract_strings, generate_pot, merge_po, init_po
 
 LANGUAGES_FILE = os.path.join(str(TRANSLATIONS_DIR), "languages.json")
 POT_FILE = os.path.join(str(TRANSLATIONS_DIR), "app.pot")
@@ -18,24 +19,17 @@ def update_translations():
       if filename.endswith(".py"):
         files.append(os.path.relpath(os.path.join(root, filename), BASEDIR))
 
-  # Create main translation file
-  cmd = ("xgettext -L Python --keyword=tr --keyword=trn:1,2 --keyword=tr_noop --from-code=UTF-8 " +
-         "--flag=tr:1:python-brace-format --flag=trn:1:python-brace-format --flag=trn:2:python-brace-format " +
-         f"-D {BASEDIR} -o {POT_FILE} {' '.join(files)}")
-
-  ret = os.system(cmd)
-  assert ret == 0
+  # Extract translatable strings and generate .pot template
+  entries = extract_strings(files, BASEDIR)
+  generate_pot(entries, POT_FILE)
 
   # Generate/update translation files for each language
   for name in multilang.languages.values():
-    if os.path.exists(os.path.join(TRANSLATIONS_DIR, f"app_{name}.po")):
-      cmd = f"msgmerge --update --no-fuzzy-matching --backup=none --sort-output {TRANSLATIONS_DIR}/app_{name}.po {POT_FILE}"
-      ret = os.system(cmd)
-      assert ret == 0
+    po_file = os.path.join(TRANSLATIONS_DIR, f"app_{name}.po")
+    if os.path.exists(po_file):
+      merge_po(po_file, POT_FILE)
     else:
-      cmd = f"msginit -l {name} --no-translator --input {POT_FILE} --output-file {TRANSLATIONS_DIR}/app_{name}.po"
-      ret = os.system(cmd)
-      assert ret == 0
+      init_po(POT_FILE, po_file, name)
 
 
 if __name__ == "__main__":

system/ui/lib/multilang.py

@@ -1,7 +1,7 @@
 from importlib.resources import files
-import os
 import json
-import gettext
+import os
+import re
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.swaglog import cloudlog
 
@@ -23,14 +23,137 @@ UNIFONT_LANGUAGES = [
   "ja",
 ]
 
+# Plural form selectors for supported languages
+PLURAL_SELECTORS = {
+  'en': lambda n: 0 if n == 1 else 1,
+  'de': lambda n: 0 if n == 1 else 1,
+  'fr': lambda n: 0 if n <= 1 else 1,
+  'pt-BR': lambda n: 0 if n <= 1 else 1,
+  'es': lambda n: 0 if n == 1 else 1,
+  'tr': lambda n: 0 if n == 1 else 1,
+  'uk': lambda n: 0 if n % 10 == 1 and n % 100 != 11 else (1 if 2 <= n % 10 <= 4 and not 12 <= n % 100 <= 14 else 2),
+  'th': lambda n: 0,
+  'zh-CHT': lambda n: 0,
+  'zh-CHS': lambda n: 0,
+  'ko': lambda n: 0,
+  'ja': lambda n: 0,
+}
+
+
+def _parse_quoted(s: str) -> str:
+  """Parse a PO-format quoted string."""
+  s = s.strip()
+  if not (s.startswith('"') and s.endswith('"')):
+    raise ValueError(f"Expected quoted string: {s!r}")
+  s = s[1:-1]
+  result: list[str] = []
+  i = 0
+  while i < len(s):
+    if s[i] == '\\' and i + 1 < len(s):
+      c = s[i + 1]
+      if c == 'n':
+        result.append('\n')
+      elif c == 't':
+        result.append('\t')
+      elif c == '"':
+        result.append('"')
+      elif c == '\\':
+        result.append('\\')
+      else:
+        result.append(s[i:i + 2])
+      i += 2
+    else:
+      result.append(s[i])
+      i += 1
+  return ''.join(result)
+
+
+def load_translations(path) -> tuple[dict[str, str], dict[str, list[str]]]:
+  """Parse a .po file and return (translations, plurals) dicts.
+
+  translations: msgid -> msgstr
+  plurals: msgid -> [msgstr[0], msgstr[1], ...]
+  """
+  with open(str(path), encoding='utf-8') as f:
+    lines = f.readlines()
+
+  translations: dict[str, str] = {}
+  plurals: dict[str, list[str]] = {}
+
+  # Parser state
+  msgid = msgid_plural = msgstr = ""
+  msgstr_plurals: dict[int, str] = {}
+  field: str | None = None
+  plural_idx = 0
+
+  def finish():
+    nonlocal msgid, msgid_plural, msgstr, msgstr_plurals, field
+    if msgid:  # skip header (empty msgid)
+      if msgid_plural:
+        max_idx = max(msgstr_plurals.keys()) if msgstr_plurals else 0
+        plurals[msgid] = [msgstr_plurals.get(i, '') for i in range(max_idx + 1)]
+      else:
+        translations[msgid] = msgstr
+    msgid = msgid_plural = msgstr = ""
+    msgstr_plurals = {}
+    field = None
+
+  for raw in lines:
+    line = raw.strip()
+
+    if not line:
+      finish()
+      continue
+
+    if line.startswith('#'):
+      continue
+
+    if line.startswith('msgid_plural '):
+      msgid_plural = _parse_quoted(line[len('msgid_plural '):])
+      field = 'msgid_plural'
+      continue
+
+    if line.startswith('msgid '):
+      msgid = _parse_quoted(line[len('msgid '):])
+      field = 'msgid'
+      continue
+
+    m = re.match(r'msgstr\[(\d+)]\s+(.*)', line)
+    if m:
+      plural_idx = int(m.group(1))
+      msgstr_plurals[plural_idx] = _parse_quoted(m.group(2))
+      field = 'msgstr_plural'
+      continue
+
+    if line.startswith('msgstr '):
+      msgstr = _parse_quoted(line[len('msgstr '):])
+      field = 'msgstr'
+      continue
+
+    if line.startswith('"'):
+      val = _parse_quoted(line)
+      if field == 'msgid':
+        msgid += val
+      elif field == 'msgid_plural':
+        msgid_plural += val
+      elif field == 'msgstr':
+        msgstr += val
+      elif field == 'msgstr_plural':
+        msgstr_plurals[plural_idx] += val
+
+  finish()
+  return translations, plurals
+
 
 class Multilang:
   def __init__(self):
     self._params = Params() if Params is not None else None
     self._language: str = "en"
-    self.languages = {}
-    self.codes = {}
-    self._translation: gettext.NullTranslations | gettext.GNUTranslations = gettext.NullTranslations()
+    self.languages: dict[str, str] = {}
+    self.codes: dict[str, str] = {}
+    self._translations: dict[str, str] = {}
+    self._plurals: dict[str, list[str]] = {}
+    self._plural_selector = PLURAL_SELECTORS.get('en', lambda n: 0)
     self._load_languages()
 
   @property
@@ -43,27 +166,30 @@ class Multilang:
 
   def setup(self):
     try:
-      with TRANSLATIONS_DIR.joinpath(f'app_{self._language}.mo').open('rb') as fh:
-        translation = gettext.GNUTranslations(fh)
-      translation.install()
-      self._translation = translation
+      po_path = TRANSLATIONS_DIR.joinpath(f'app_{self._language}.po')
+      self._translations, self._plurals = load_translations(po_path)
+      self._plural_selector = PLURAL_SELECTORS.get(self._language, lambda n: 0)
       cloudlog.debug(f"Loaded translations for language: {self._language}")
     except FileNotFoundError:
       cloudlog.error(f"No translation file found for language: {self._language}, using default.")
-      gettext.install('app')
-      self._translation = gettext.NullTranslations()
+      self._translations = {}
+      self._plurals = {}
 
   def change_language(self, language_code: str) -> None:
-    # Reinstall gettext with the selected language
     self._params.put("LanguageSetting", language_code)
     self._language = language_code
     self.setup()
 
   def tr(self, text: str) -> str:
-    return self._translation.gettext(text)
+    return self._translations.get(text, text)
 
   def trn(self, singular: str, plural: str, n: int) -> str:
-    return self._translation.ngettext(singular, plural, n)
+    if singular in self._plurals:
+      idx = self._plural_selector(n)
+      forms = self._plurals[singular]
+      if idx < len(forms) and forms[idx]:
+        return forms[idx]
+    return singular if n == 1 else plural
 
   def _load_languages(self):
     with LANGUAGES_FILE.open(encoding='utf-8') as f:

tools/setup_dependencies.sh

@@ -51,8 +51,7 @@ function install_ubuntu_deps() {
   $SUDO apt-get install -y --no-install-recommends \
     python3-dev \
     libncurses5-dev \
-    libzstd-dev \
-    gettext
+    libzstd-dev
 
   if [[ -d "/etc/udev/rules.d/" ]]; then
     # Setup jungle udev rules