zipapp pack (#35253)

Used to ship python UI in agnos without an openpilot clone

* add a main method to target

* pack script

* validate inputs

* refactors

* copy into temp, dont keep this

* cleanup

* help messages

* rename to pack.py

* pack.py

* updates for device

* moar

* don't use cereal

* just log normally

* use importlib.resources

* revert

* Revert "don't use cereal"

This reverts commit 7208524d422d88a1b07e209359aeb25e8b3bf4e7.

* fix cereal?

* cleanup

* Revert "cleanup"

This reverts commit 921edfe5020f244dbdf4f26767af7c98ca837d1c.

* cython hotfix

* Reapply "cleanup"

This reverts commit 9b54552f784dea1b1eb4ffc03937571e4fc851ba.

* more cleanup

* any script?

* slightly clearer

* rm print

* nothing python should use SVGs

---------

Co-authored-by: Trey Moen <trey@moen.ai>

cereal/__init__.py

@@ -1,9 +1,11 @@
 import os
 import capnp
+from importlib.resources import as_file, files
 
-CEREAL_PATH = os.path.dirname(os.path.abspath(__file__))
 capnp.remove_import_hook()
 
-log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
-car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
-custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))
+with as_file(files("cereal")) as fspath:
+  CEREAL_PATH = fspath.as_posix()
+  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

release/pack.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import importlib
+import shutil
+import sys
+import tempfile
+import zipapp
+from argparse import ArgumentParser
+from pathlib import Path
+
+from openpilot.common.basedir import BASEDIR
+
+
+DIRS = ['cereal', 'openpilot']
+EXTS = ['.png', '.py', '.ttf', '.capnp']
+INTERPRETER = '/usr/bin/env python3'
+
+
+def copy(src, dest):
+  if any(src.endswith(ext) for ext in EXTS):
+    shutil.copy2(src, dest, follow_symlinks=True)
+
+
+if __name__ == '__main__':
+  parser = ArgumentParser(prog='pack.py', description="package script into a portable executable", epilog='comma.ai')
+  parser.add_argument('-e', '--entrypoint', help="function to call in module, default is 'main'", default='main')
+  parser.add_argument('-o', '--output', help='output file')
+  parser.add_argument('module', help="the module to target, e.g. 'openpilot.system.ui.spinner'")
+  args = parser.parse_args()
+
+  if not args.output:
+    args.output = args.module
+
+  try:
+    mod = importlib.import_module(args.module)
+  except ModuleNotFoundError:
+    print(f'{args.module} not found, typo?')
+    sys.exit(1)
+
+  if not hasattr(mod, args.entrypoint):
+    print(f'{args.module} does not have a {args.entrypoint}() function, typo?')
+    sys.exit(1)
+
+  with tempfile.TemporaryDirectory() as tmp:
+    for directory in DIRS:
+      shutil.copytree(BASEDIR + '/' + directory, tmp + '/' + directory, symlinks=False, dirs_exist_ok=True, copy_function=copy)
+    entry = f'{args.module}:{args.entrypoint}'
+    zipapp.create_archive(tmp, target=args.output, interpreter=INTERPRETER, main=entry)
+
+  print(f'created executable {Path(args.output).resolve()}')

system/ui/lib/application.py

@@ -3,7 +3,7 @@ import os
 import time
 import pyray as rl
 from enum import IntEnum
-from openpilot.common.basedir import BASEDIR
+from importlib.resources import as_file, files
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware import HARDWARE
 
@@ -18,9 +18,9 @@ STRICT_MODE = os.getenv("STRICT_MODE") == '1'
 
 DEFAULT_TEXT_SIZE = 60
 DEFAULT_TEXT_COLOR = rl.WHITE
-ASSETS_DIR = os.path.join(BASEDIR, "selfdrive/assets")
-FONT_DIR = os.path.join(BASEDIR, "selfdrive/assets/fonts")
 
+ASSETS_DIR = files("openpilot.selfdrive").joinpath("assets")
+FONT_DIR = ASSETS_DIR.joinpath("fonts")
 
 class FontWeight(IntEnum):
   BLACK = 0
@@ -74,7 +74,8 @@ class GuiApplication:
     if cache_key in self._textures:
       return self._textures[cache_key]
 
-    texture_obj = self._load_texture_from_image(os.path.join(ASSETS_DIR, asset_path), width, height, alpha_premultiply, keep_aspect_ratio)
+    with as_file(ASSETS_DIR.joinpath(asset_path)) as fspath:
+      texture_obj = self._load_texture_from_image(fspath.as_posix(), width, height, alpha_premultiply, keep_aspect_ratio)
     self._textures[cache_key] = texture_obj
     return texture_obj
 
@@ -163,9 +164,10 @@ class GuiApplication:
       )
 
     for index, font_file in enumerate(font_files):
-      font = rl.load_font_ex(os.path.join(FONT_DIR, font_file), 120, None, 0)
-      rl.set_texture_filter(font.texture, rl.TextureFilter.TEXTURE_FILTER_BILINEAR)
-      self._fonts[index] = font
+      with as_file(FONT_DIR.joinpath(font_file)) as fspath:
+        font = rl.load_font_ex(fspath.as_posix(), 120, None, 0)
+        rl.set_texture_filter(font.texture, rl.TextureFilter.TEXTURE_FILTER_BILINEAR)
+        self._fonts[index] = font
 
     rl.gui_set_font(self._fonts[FontWeight.NORMAL])
 

system/ui/lib/wifi_manager.py

@@ -13,7 +13,11 @@ from dbus_next.aio import MessageBus
 from dbus_next import BusType, Variant, Message
 from dbus_next.errors import DBusError
 from dbus_next.constants import MessageType
-from openpilot.common.params import Params
+try:
+  from openpilot.common.params import Params
+except ImportError:
+  # Params/Cythonized modules are not available in zipapp
+  Params = None
 from openpilot.common.swaglog import cloudlog
 
 T = TypeVar("T")
@@ -81,9 +85,10 @@ class WifiManager:
     self.scan_task: asyncio.Task | None = None
     # Set tethering ssid as "weedle" + first 4 characters of a dongle id
     self._tethering_ssid = "weedle"
-    dongle_id = Params().get("DongleId", encoding="utf-8")
-    if dongle_id:
-      self._tethering_ssid += "-" + dongle_id[:4]
+    if Params is not None:
+      dongle_id = Params().get("DongleId", encoding="utf-8")
+      if dongle_id:
+        self._tethering_ssid += "-" + dongle_id[:4]
     self.running: bool = True
     self._current_connection_ssid: str | None = None
 

system/ui/spinner.py

@@ -98,7 +98,11 @@ class Spinner(BaseWindow[SpinnerRenderer]):
     self.update(str(round(100 * cur / total)))
 
 
-if __name__ == "__main__":
+def main():
   with Spinner() as s:
     s.update("Spinner text")
     time.sleep(5)
+
+
+if __name__ == "__main__":
+  main()