raylib: unifont for CJK languages (#36430)

* add rest of langs

* unifont

* all langs are supported

* add japanese translations

* fix strip!

* add language name chars

* use unifont in lang selection

* add korean

* test all langs

* doesn't work

* unifont font fallback for multilang

* add ar translations

* fix labels not updating until scrolling

* t chinese

* more chn

* we already default

* wrap

* update

* fix thai

* fix missing chinese langs and all are supported!

* clean up

* update

* ??? mypy r u ok ???

* fix default option font weight

system/ui/lib/application.py

@@ -51,6 +51,14 @@ class FontWeight(StrEnum):
   BOLD = "Inter-Bold.ttf"
   EXTRA_BOLD = "Inter-ExtraBold.ttf"
   BLACK = "Inter-Black.ttf"
+  UNIFONT = "unifont.otf"
+
+
+def font_fallback(font: rl.Font) -> rl.Font:
+  """Fall back to unifont for languages that require it."""
+  if multilang.requires_unifont():
+    return gui_app.font(FontWeight.UNIFONT)
+  return font
 
 
 @dataclass
@@ -335,7 +343,7 @@ class GuiApplication:
     except KeyboardInterrupt:
       pass
 
-  def font(self, font_weight: FontWeight = FontWeight.NORMAL):
+  def font(self, font_weight: FontWeight = FontWeight.NORMAL) -> rl.Font:
     return self._fonts[font_weight]
 
   @property
@@ -356,14 +364,16 @@ class GuiApplication:
     all_chars |= set("–‑✓×°§•")
 
     # Load only the characters used in translations
-    for language in multilang.codes:
+    for language, code in multilang.languages.items():
+      all_chars |= set(language)
       try:
-        with open(os.path.join(TRANSLATIONS_DIR, f"app_{language}.po")) as f:
+        with open(os.path.join(TRANSLATIONS_DIR, f"app_{code}.po")) as f:
           all_chars |= set(f.read())
       except FileNotFoundError:
-        cloudlog.warning(f"Translation file for language '{language}' not found when loading fonts.")
+        cloudlog.warning(f"Translation file for language '{code}' not found when loading fonts.")
 
     all_chars = "".join(all_chars)
+    cloudlog.debug(f"Loading fonts with {len(all_chars)} glyphs.")
 
     codepoint_count = rl.ffi.new("int *", 1)
     codepoints = rl.load_codepoints(all_chars, codepoint_count)
@@ -390,6 +400,7 @@ class GuiApplication:
       rl._orig_draw_text_ex = rl.draw_text_ex
 
     def _draw_text_ex_scaled(font, text, position, font_size, spacing, tint):
+      font = font_fallback(font)
       return rl._orig_draw_text_ex(font, text, position, font_size * FONT_SCALE, spacing, tint)
 
     rl.draw_text_ex = _draw_text_ex_scaled

system/ui/lib/multilang.py

@@ -3,25 +3,27 @@ import json
 import gettext
 from openpilot.common.params import Params
 from openpilot.common.basedir import BASEDIR
+from openpilot.common.swaglog import cloudlog
 
 SYSTEM_UI_DIR = os.path.join(BASEDIR, "system", "ui")
 UI_DIR = os.path.join(BASEDIR, "selfdrive", "ui")
 TRANSLATIONS_DIR = os.path.join(UI_DIR, "translations")
 LANGUAGES_FILE = os.path.join(TRANSLATIONS_DIR, "languages.json")
 
-SUPPORTED_LANGUAGES = [
-  "en",
-  "de",
-  "fr",
-  "pt-BR",
-  "es",
-  "tr",
+UNIFONT_LANGUAGES = [
+  "ar",
+  "th",
+  "zh-CHT",
+  "zh-CHS",
+  "ko",
+  "ja",
 ]
 
 
 class Multilang:
   def __init__(self):
     self._params = Params()
+    self._language: str = "en"
     self.languages = {}
     self.codes = {}
     self._translation: gettext.NullTranslations | gettext.GNUTranslations = gettext.NullTranslations()
@@ -29,28 +31,28 @@ class Multilang:
 
   @property
   def language(self) -> str:
-    lang = str(self._params.get("LanguageSetting")).strip("main_")
-    if lang not in SUPPORTED_LANGUAGES:
-      lang = "en"
-    return lang
+    return self._language
+
+  def requires_unifont(self) -> bool:
+    """Certain languages require unifont to render their glyphs."""
+    return self._language in UNIFONT_LANGUAGES
 
   def setup(self):
-    language = self.language
     try:
-      with open(os.path.join(TRANSLATIONS_DIR, f'app_{language}.mo'), 'rb') as fh:
+      with open(os.path.join(TRANSLATIONS_DIR, f'app_{self._language}.mo'), 'rb') as fh:
         translation = gettext.GNUTranslations(fh)
       translation.install()
       self._translation = translation
-      print(f"Loaded translations for language: {language}")
+      cloudlog.warning(f"Loaded translations for language: {self._language}")
     except FileNotFoundError:
-      print(f"No translation file found for language: {language}, using default.")
+      cloudlog.error(f"No translation file found for language: {self._language}, using default.")
       gettext.install('app')
       self._translation = gettext.NullTranslations()
-    return None
 
   def change_language(self, language_code: str) -> None:
     # Reinstall gettext with the selected language
     self._params.put("LanguageSetting", language_code)
+    self._language = language_code
     self.setup()
 
   def tr(self, text: str) -> str:
@@ -61,8 +63,12 @@ class Multilang:
 
   def _load_languages(self):
     with open(LANGUAGES_FILE, encoding='utf-8') as f:
-      self.languages = {k: v for k, v in json.load(f).items() if v in SUPPORTED_LANGUAGES}
-    self.codes = {v: k for k, v in self.languages.items() if v in SUPPORTED_LANGUAGES}
+      self.languages = json.load(f)
+    self.codes = {v: k for k, v in self.languages.items()}
+
+    lang = str(self._params.get("LanguageSetting")).removeprefix("main_")
+    if lang in self.codes:
+      self._language = lang
 
 
 multilang = Multilang()

system/ui/lib/text_measure.py

@@ -1,5 +1,5 @@
 import pyray as rl
-from openpilot.system.ui.lib.application import FONT_SCALE
+from openpilot.system.ui.lib.application import FONT_SCALE, font_fallback
 from openpilot.system.ui.lib.emoji import find_emoji
 
 _cache: dict[int, rl.Vector2] = {}
@@ -7,6 +7,7 @@ _cache: dict[int, rl.Vector2] = {}
 
 def measure_text_cached(font: rl.Font, text: str, font_size: int, spacing: int = 0) -> rl.Vector2:
   """Caches text measurements to avoid redundant calculations."""
+  font = font_fallback(font)
   key = hash((font.texture.id, text, font_size, spacing))
   if key in _cache:
     return _cache[key]

system/ui/lib/wrap_text.py

@@ -1,5 +1,6 @@
 import pyray as rl
 from openpilot.system.ui.lib.text_measure import measure_text_cached
+from openpilot.system.ui.lib.application import font_fallback
 
 
 def _break_long_word(font: rl.Font, word: str, font_size: int, max_width: int) -> list[str]:
@@ -40,6 +41,7 @@ _cache: dict[int, list[str]] = {}
 
 
 def wrap_text(font: rl.Font, text: str, font_size: int, max_width: int) -> list[str]:
+  font = font_fallback(font)
   key = hash((font.texture.id, text, font_size, max_width))
   if key in _cache:
     return _cache[key]