translations: auto-generate with codex (#37462)

This commit is contained in:
Adeeb Shihadeh
2026-03-23 08:59:37 -07:00
committed by GitHub
parent 6871203c45
commit 5766202763
21 changed files with 3465 additions and 6486 deletions
-22
View File
@@ -9,28 +9,6 @@ env:
PYTHONPATH: ${{ github.workspace }}
jobs:
update_translations:
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
steps:
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Update translations
run: python3 selfdrive/ui/update_translations.py --vanish
- name: Create Pull Request
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
with:
author: Vehicle Researcher <user@comma.ai>
commit-message: "Update translations"
title: "[bot] Update translations"
body: "Automatic PR from repo-maintenance -> update_translations"
branch: "update-translations"
base: "master"
delete-branch: true
labels: bot
package_updates:
name: package_updates
runs-on: ubuntu-latest
+81 -99
View File
@@ -1,124 +1,106 @@
import pytest
import json
import os
import re
import xml.etree.ElementTree as ET
import string
import requests
from openpilot.common.parameterized import parameterized_class
from openpilot.system.ui.lib.multilang import TRANSLATIONS_DIR, LANGUAGES_FILE
from pathlib import Path
with open(str(LANGUAGES_FILE)) as f:
translation_files = json.load(f)
import pytest
UNFINISHED_TRANSLATION_TAG = "<translation type=\"unfinished\"" # non-empty translations can be marked unfinished
LOCATION_TAG = "<location "
FORMAT_ARG = re.compile("%[0-9]+")
from openpilot.selfdrive.ui.translations.potools import parse_po
from openpilot.system.ui.lib.multilang import LANGUAGES_FILE, TRANSLATIONS_DIR
PERCENT_PLACEHOLDER_RE = re.compile(r"%(?:n|\d+)")
BAD_ENTITY_RE = re.compile(r'@(\w+);')
LINE_NUMBER_REF_RE = re.compile(r'^#:\s+.+:\d+(?:\s|$)')
FORMATTER = string.Formatter()
PO_DIR = Path(str(TRANSLATIONS_DIR))
with LANGUAGES_FILE.open(encoding='utf-8') as f:
TRANSLATION_LANGUAGES = json.load(f)
@pytest.mark.skip("TODO: update for raylib")
@parameterized_class(("name", "file"), translation_files.items())
class TestTranslations:
name: str
file: str
def extract_placeholders(text: str) -> list[str]:
placeholders = PERCENT_PLACEHOLDER_RE.findall(text)
@staticmethod
def _read_translation_file(path, file):
tr_file = os.path.join(path, f"{file}.ts")
with open(tr_file) as f:
return f.read()
try:
parsed = list(FORMATTER.parse(text))
except ValueError as e:
raise AssertionError(f"invalid brace formatting in {text!r}: {e}") from e
def test_missing_translation_files(self):
assert os.path.exists(os.path.join(str(TRANSLATIONS_DIR), f"{self.file}.ts")), \
f"{self.name} has no XML translation file, run selfdrive/ui/update_translations.py"
for _, field_name, format_spec, conversion in parsed:
if field_name is None:
continue
@pytest.mark.skip("Only test unfinished translations before going to release")
def test_unfinished_translations(self):
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
assert UNFINISHED_TRANSLATION_TAG not in cur_translations, \
f"{self.file} ({self.name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist"
token = "{"
token += field_name
if conversion:
token += f"!{conversion}"
if format_spec:
token += f":{format_spec}"
token += "}"
placeholders.append(token)
def test_vanished_translations(self):
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
assert "<translation type=\"vanished\">" not in cur_translations, \
f"{self.file} ({self.name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them"
return sorted(placeholders)
def test_finished_translations(self):
"""
Tests ran on each translation marked "finished"
Plural:
- that any numerus (plural) translations have all plural forms non-empty
- that the correct format specifier is used (%n)
Non-plural:
- that translation is not empty
- that translation format arguments are consistent
"""
tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts"))
for context in tr_xml.getroot():
for message in context.iterfind("message"):
translation = message.find("translation")
source_text = message.find("source").text
def load_po_text(po_path: Path) -> str:
return po_path.read_text(encoding='utf-8')
# Do not test unfinished translations
if translation.get("type") == "unfinished":
continue
if message.get("numerus") == "yes":
numerusform = [t.text for t in translation.findall("numerusform")]
@pytest.mark.parametrize("language_code", sorted(TRANSLATION_LANGUAGES.values()))
def test_translation_file_exists(language_code: str):
po_path = PO_DIR / f"app_{language_code}.po"
assert po_path.exists(), f"missing translation file: {po_path}"
for nf in numerusform:
assert nf is not None, f"Ensure all plural translation forms are completed: {source_text}"
assert "%n" in nf, "Ensure numerus argument (%n) exists in translation."
assert FORMAT_ARG.search(nf) is None, f"Plural translations must use %n, not %1, %2, etc.: {numerusform}"
else:
assert translation.text is not None, f"Ensure translation is completed: {source_text}"
@pytest.mark.parametrize("po_path", sorted(PO_DIR.glob("app_*.po")), ids=lambda p: p.name)
def test_translation_placeholders_are_preserved(po_path: Path):
_, entries = parse_po(po_path)
language = po_path.stem.removeprefix("app_")
source_args = FORMAT_ARG.findall(source_text)
translation_args = FORMAT_ARG.findall(translation.text)
assert sorted(source_args) == sorted(translation_args), \
f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`"
for entry in entries:
source_placeholders = extract_placeholders(entry.msgid)
def test_no_locations(self):
for line in self._read_translation_file(TRANSLATIONS_DIR, self.file).splitlines():
assert not line.strip().startswith(LOCATION_TAG), \
f"Line contains location tag: {line.strip()}, remove all line numbers."
def test_entities_error(self):
cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
matches = re.findall(r'@(\w+);', cur_translations)
assert len(matches) == 0, f"The string(s) {matches} were found with '@' instead of '&'"
def test_bad_language(self):
IGNORED_WORDS = {'pédale'}
match = re.search(r'([a-zA-Z]{2,3})', self.file)
assert match, f"{self.name} - could not parse language"
try:
response = requests.get(
f"https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/{match.group(1)}"
if entry.is_plural:
plural_placeholders = extract_placeholders(entry.msgid_plural)
message = (
f"{language}: source plural placeholders do not match singular for "
+ f"{entry.msgid!r}: {source_placeholders} vs {plural_placeholders}"
)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
if e.response is not None and e.response.status_code == 429:
pytest.skip("word list rate limited")
raise
assert plural_placeholders == source_placeholders, message
banned_words = {line.strip() for line in response.text.splitlines()}
for context in ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")).getroot():
for message in context.iterfind("message"):
translation = message.find("translation")
if translation.get("type") == "unfinished":
for idx, msgstr in sorted(entry.msgstr_plural.items()):
if not msgstr:
continue
translation_text = " ".join([t.text for t in translation.findall("numerusform")]) if message.get("numerus") == "yes" else translation.text
translated_placeholders = extract_placeholders(msgstr)
message = (
f"{language}: plural form {idx} changes placeholders for {entry.msgid!r}: "
+ f"expected {source_placeholders}, got {translated_placeholders}"
)
assert translated_placeholders == source_placeholders, message
else:
if not entry.msgstr:
continue
if not translation_text:
continue
translated_placeholders = extract_placeholders(entry.msgstr)
message = (
f"{language}: translation changes placeholders for {entry.msgid!r}: "
+ f"expected {source_placeholders}, got {translated_placeholders}"
)
assert translated_placeholders == source_placeholders, message
words = set(translation_text.translate(str.maketrans('', '', string.punctuation + '%n')).lower().split())
bad_words_found = words & (banned_words - IGNORED_WORDS)
assert not bad_words_found, f"Bad language found in {self.name}: '{translation_text}'. Bad word(s): {', '.join(bad_words_found)}"
@pytest.mark.parametrize("po_path", sorted(PO_DIR.glob("app_*.po")), ids=lambda p: p.name)
def test_translation_refs_do_not_include_line_numbers(po_path: Path):
for line in load_po_text(po_path).splitlines():
assert not LINE_NUMBER_REF_RE.match(line), (
f"{po_path.name}: line-number source reference found: {line}"
)
@pytest.mark.parametrize("po_path", sorted(PO_DIR.glob("app_*.po")), ids=lambda p: p.name)
def test_translation_entities_are_valid(po_path: Path):
matches = BAD_ENTITY_RE.findall(load_po_text(po_path))
assert not matches, (
f"{po_path.name}: found '@...;' entity typo(s): {', '.join(sorted(set(matches)))}"
)
-3
View File
@@ -1,3 +0,0 @@
# Multilanguage
[![languages](https://raw.githubusercontent.com/commaai/openpilot/badges/translation_badge.svg)](#)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-138
View File
@@ -1,138 +0,0 @@
#!/usr/bin/env python3
import argparse
import json
import os
import pathlib
import xml.etree.ElementTree as ET
from typing import cast
import requests
TRANSLATIONS_DIR = pathlib.Path(__file__).resolve().parent
TRANSLATIONS_LANGUAGES = TRANSLATIONS_DIR / "languages.json"
OPENAI_MODEL = "gpt-4"
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
OPENAI_PROMPT = "You are a professional translator from English to {language} (ISO 639 language code). " + \
"The following sentence or word is in the GUI of a software called openpilot, translate it accordingly."
def get_language_files(languages: list[str] | None = None) -> dict[str, pathlib.Path]:
files = {}
with open(TRANSLATIONS_LANGUAGES) as fp:
language_dict = json.load(fp)
for filename in language_dict.values():
path = TRANSLATIONS_DIR / f"{filename}.ts"
language = path.stem
if languages is None or language in languages:
files[language] = path
return files
def translate_phrase(text: str, language: str) -> str:
response = requests.post(
"https://api.openai.com/v1/chat/completions",
json={
"model": OPENAI_MODEL,
"messages": [
{
"role": "system",
"content": OPENAI_PROMPT.format(language=language),
},
{
"role": "user",
"content": text,
},
],
"temperature": 0.8,
"max_tokens": 1024,
"top_p": 1,
},
headers={
"Authorization": f"Bearer {OPENAI_API_KEY}",
"Content-Type": "application/json",
},
)
if 400 <= response.status_code < 600:
raise requests.HTTPError(f'Error {response.status_code}: {response.json()}', response=response)
data = response.json()
return cast(str, data["choices"][0]["message"]["content"])
def translate_file(path: pathlib.Path, language: str, all_: bool) -> None:
tree = ET.parse(path)
root = tree.getroot()
for context in root.findall("./context"):
name = context.find("name")
if name is None:
raise ValueError("name not found")
print(f"Context: {name.text}")
for message in context.findall("./message"):
source = message.find("source")
translation = message.find("translation")
if source is None or translation is None:
raise ValueError("source or translation not found")
if not all_ and translation.attrib.get("type") != "unfinished":
continue
llm_translation = translate_phrase(cast(str, source.text), language)
print(f"Source: {source.text}\n" +
f"Current translation: {translation.text}\n" +
f"LLM translation: {llm_translation}")
translation.text = llm_translation
with path.open("w", encoding="utf-8") as fp:
fp.write('<?xml version="1.0" encoding="utf-8"?>\n' +
'<!DOCTYPE TS>\n' +
ET.tostring(root, encoding="utf-8").decode())
def main():
arg_parser = argparse.ArgumentParser("Auto translate")
group = arg_parser.add_mutually_exclusive_group(required=True)
group.add_argument("-a", "--all-files", action="store_true", help="Translate all files")
group.add_argument("-f", "--file", nargs="+", help="Translate the selected files. (Example: -f fr de)")
arg_parser.add_argument("-t", "--all-translations", action="store_true", default=False, help="Translate all sections. (Default: only unfinished)")
args = arg_parser.parse_args()
if OPENAI_API_KEY is None:
print("OpenAI API key is missing. (Hint: use `export OPENAI_API_KEY=YOUR-KEY` before you run the script).\n" +
"If you don't have one go to: https://beta.openai.com/account/api-keys.")
exit(1)
files = get_language_files(None if args.all_files else args.file)
if args.file:
missing_files = set(args.file) - set(files)
if len(missing_files):
print(f"No language files found: {missing_files}")
exit(1)
print(f"Translation mode: {'all' if args.all_translations else 'only unfinished'}. Files: {list(files)}")
for lang, path in files.items():
print(f"Translate {lang} ({path})")
translate_file(path, lang, args.all_translations)
if __name__ == "__main__":
main()
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd)"
ROOT="$DIR/../../../"
cd $DIR
./update_translations.py
command -v codex >/dev/null || {
echo "Install codex CLI to continue:"
echo "-> https://developers.openai.com/codex/cli"
echo
exit 1
}
codex exec --cd "$ROOT" -c 'model_reasoning_effort="low"' --dangerously-bypass-approvals-and-sandbox "$(cat <<EOF
Update openpilot UI translations in selfdrive/ui/translations.
- Translate English UI text naturally.
- Preserve placeholders (%n, %1, {}, {:.1f}), HTML/tags, and plural forms.
- Edit .po files in place.
- Print a short summary of changes.
- All strings should be translated. Don't stop until it's 100%.
- Be mindful of the layout/style of the UI and length of the original English string.
EOF
)"
-108
View File
@@ -1,108 +0,0 @@
#!/usr/bin/env python3
import json
import os
import requests
import xml.etree.ElementTree as ET
from openpilot.common.basedir import BASEDIR
from openpilot.selfdrive.ui.update_translations import LANGUAGES_FILE, TRANSLATIONS_DIR
BADGE_HEIGHT = 20 + 8
SHIELDS_URL = "https://img.shields.io/badge"
def parse_po_file(file_path):
"""
Parse a .po file and count total and unfinished translations.
Returns: (total_translations, unfinished_translations)
"""
with open(file_path) as f:
content = f.read()
total_translations = 0
unfinished_translations = 0
# Split into entries (separated by blank lines)
entries = content.split('\n\n')
for entry in entries:
# Skip header entry (contains Project-Id-Version)
if 'Project-Id-Version' in entry:
continue
# Check if this entry has a msgid (translation entry)
# After skipping header, any entry with msgid " is a translation
# (both msgid "content" and msgid "" for multiline contain msgid ")
if 'msgid "' not in entry:
continue
total_translations += 1
# Check if msgstr is empty (unfinished translation)
if 'msgstr ""' in entry:
# Check if there are continuation lines with content after msgstr ""
lines = entry.split('\n')
msgstr_idx = None
for i, line in enumerate(lines):
if line.strip().startswith('msgstr ""'):
msgstr_idx = i
break
if msgstr_idx is not None:
# Check if any continuation lines have content
has_content = False
for line in lines[msgstr_idx + 1:]:
stripped = line.strip()
# Continuation line with content
if stripped.startswith('"') and len(stripped) > 2:
has_content = True
break
# End of entry
if stripped.startswith(('msgid', '#')) or not stripped:
break
if not has_content:
unfinished_translations += 1
return (total_translations, unfinished_translations)
if __name__ == "__main__":
with open(LANGUAGES_FILE) as f:
translation_files = json.load(f)
badge_svg = []
max_badge_width = 0 # keep track of max width to set parent element
for idx, (name, file) in enumerate(translation_files.items()):
po_file_path = os.path.join(str(TRANSLATIONS_DIR), f"app_{file}.po")
total_translations, unfinished_translations = parse_po_file(po_file_path)
percent_finished = int(100 - (unfinished_translations / total_translations * 100.)) if total_translations > 0 else 0
color = f"rgb{(94, 188, 0) if percent_finished == 100 else (248, 255, 50) if percent_finished > 90 else (204, 55, 27)}"
# Download badge
badge_label = f"LANGUAGE {name}"
badge_message = f"{percent_finished}% complete"
if unfinished_translations != 0:
badge_message += f" ({unfinished_translations} unfinished)"
r = requests.get(f"{SHIELDS_URL}/{badge_label}-{badge_message}-{color}", timeout=10)
assert r.status_code == 200, "Error downloading badge"
content_svg = r.content.decode("utf-8")
xml = ET.fromstring(content_svg)
assert "width" in xml.attrib
max_badge_width = max(max_badge_width, int(xml.attrib["width"]))
# Make tag ids in each badge unique to combine them into one svg
for tag in ("r", "s"):
content_svg = content_svg.replace(f'id="{tag}"', f'id="{tag}{idx}"')
content_svg = content_svg.replace(f'"url(#{tag})"', f'"url(#{tag}{idx})"')
badge_svg.extend([f'<g transform="translate(0, {idx * BADGE_HEIGHT})">', content_svg, "</g>"])
badge_svg.insert(0, '<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" ' +
f'height="{len(translation_files) * BADGE_HEIGHT}" width="{max_badge_width}">')
badge_svg.append("</svg>")
with open(os.path.join(BASEDIR, "translation_badge.svg"), "w") as badge_f:
badge_f.write("\n".join(badge_svg))
+25 -54
View File
@@ -8,7 +8,6 @@ import ast
import os
import re
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
@@ -165,18 +164,18 @@ def write_po(path: str | Path, header: POEntry | None, entries: list[POEntry]) -
if header:
for c in header.comments:
f.write(c + '\n')
if header.flags:
f.write('#, ' + ', '.join(header.flags) + '\n')
f.write(f'msgid {_quote("")}\n')
f.write(f'msgstr {_quote(header.msgstr)}\n\n')
for entry in entries:
for c in entry.comments:
f.write(c + '\n')
for ref in entry.source_refs:
# Keep file-level context for translators, but drop line numbers to
# avoid churning PO diffs on unrelated code edits.
source_files = sorted({ref.rsplit(':', 1)[0] for ref in entry.source_refs})
for ref in source_files:
f.write(f'#: {ref}\n')
if entry.flags:
f.write('#, ' + ', '.join(entry.flags) + '\n')
# Runtime loading ignores gettext flags; omit them to reduce noise.
f.write(f'msgid {_quote(entry.msgid)}\n')
if entry.is_plural:
f.write(f'msgid_plural {_quote(entry.msgid_plural)}\n')
@@ -256,31 +255,24 @@ def extract_strings(files: list[str], basedir: str) -> list[POEntry]:
# ──── POT generation ────
def _build_pot_header() -> POEntry:
return POEntry(
msgstr='Content-Type: text/plain; charset=UTF-8\n',
)
def _build_po_header(language: str) -> POEntry:
plural_forms = PLURAL_FORMS.get(language, 'nplurals=2; plural=(n != 1);')
return POEntry(
msgstr='Content-Type: text/plain; charset=UTF-8\n' +
f'Language: {language}\n' +
f'Plural-Forms: {plural_forms}\n',
)
def generate_pot(entries: list[POEntry], pot_path: str | Path) -> None:
"""Generate a .pot template file from extracted entries."""
now = datetime.now(UTC).strftime('%Y-%m-%d %H:%M%z')
header = POEntry(
comments=[
'# SOME DESCRIPTIVE TITLE.',
"# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER",
'# This file is distributed under the same license as the PACKAGE package.',
'# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.',
'#',
],
flags=['fuzzy'],
msgstr='Project-Id-Version: PACKAGE VERSION\n' +
'Report-Msgid-Bugs-To: \n' +
f'POT-Creation-Date: {now}\n' +
'PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n' +
'Last-Translator: FULL NAME <EMAIL@ADDRESS>\n' +
'Language-Team: LANGUAGE <LL@li.org>\n' +
'Language: \n' +
'MIME-Version: 1.0\n' +
'Content-Type: text/plain; charset=UTF-8\n' +
'Content-Transfer-Encoding: 8bit\n' +
'Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n',
)
write_po(pot_path, header, entries)
write_po(pot_path, _build_pot_header(), entries)
# ──── PO init (replaces msginit) ────
@@ -305,43 +297,22 @@ def init_po(pot_path: str | Path, po_path: str | Path, language: str) -> None:
"""Create a new .po file from a .pot template (replaces msginit)."""
_, entries = parse_po(pot_path)
plural_forms = PLURAL_FORMS.get(language, 'nplurals=2; plural=(n != 1);')
now = datetime.now(UTC).strftime('%Y-%m-%d %H:%M%z')
header = POEntry(
comments=[
f'# {language} translations for PACKAGE package.',
"# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER",
'# This file is distributed under the same license as the PACKAGE package.',
'# Automatically generated.',
'#',
],
msgstr='Project-Id-Version: PACKAGE VERSION\n' +
'Report-Msgid-Bugs-To: \n' +
f'POT-Creation-Date: {now}\n' +
f'PO-Revision-Date: {now}\n' +
'Last-Translator: Automatically generated\n' +
'Language-Team: none\n' +
f'Language: {language}\n' +
'MIME-Version: 1.0\n' +
'Content-Type: text/plain; charset=UTF-8\n' +
'Content-Transfer-Encoding: 8bit\n' +
f'Plural-Forms: {plural_forms}\n',
)
nplurals = int(re.search(r'nplurals=(\d+)', plural_forms).group(1))
for e in entries:
if e.is_plural:
e.msgstr_plural = dict.fromkeys(range(nplurals), '')
write_po(po_path, header, entries)
write_po(po_path, _build_po_header(language), entries)
# ──── PO merge (replaces msgmerge) ────
def merge_po(po_path: str | Path, pot_path: str | Path) -> None:
"""Update a .po file with entries from a .pot template (replaces msgmerge --update)."""
po_header, po_entries = parse_po(po_path)
_, po_entries = parse_po(po_path)
_, pot_entries = parse_po(pot_path)
language = Path(po_path).stem.removeprefix("app_")
existing = {e.msgid: e for e in po_entries}
merged = []
@@ -359,4 +330,4 @@ def merge_po(po_path: str | Path, pot_path: str | Path) -> None:
merged.append(pot_e)
merged.sort(key=lambda e: e.msgid)
write_po(po_path, po_header, merged)
write_po(po_path, _build_po_header(language), merged)