Boot logo refresh on panel open

selfdrive/ui/layouts/settings/starpilot/appearance.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 import re
+import threading
 
+from openpilot.common.params import Params
 from openpilot.system.ui.lib.application import gui_app
 from openpilot.system.ui.lib.multilang import tr, tr_noop
 from openpilot.system.ui.widgets import DialogResult
@@ -18,7 +20,8 @@ from openpilot.selfdrive.ui.layouts.settings.starpilot.aethergrid import (
     CardHubManagerView,
 )
 from openpilot.selfdrive.ui.layouts.settings.starpilot.simple_download_manager import SimpleDownloadManager
-from openpilot.starpilot.common.starpilot_variables import THEME_SAVE_PATH
+from openpilot.starpilot.assets.theme_manager import ThemeManager
+from openpilot.starpilot.common.starpilot_variables import THEME_SAVE_PATH, get_starpilot_toggles
 
 PANEL_STYLE = DEFAULT_PANEL_STYLE
 
@@ -130,6 +133,7 @@ class AppearanceManagerView(CardHubManagerView):
 class StarPilotAppearanceLayout(_SettingsPage):
     def __init__(self):
         super().__init__()
+        self._theme_refresh_thread = None
         self._build_view()
 
     def _make_parent(self, key: str, label: str, subtitle: str = "") -> ParentToggle:
@@ -747,10 +751,15 @@ class StarPilotAppearanceLayout(_SettingsPage):
     # ── Boot logo manager ──
 
     def _show_boot_logo_manager(self):
+        if self._theme_refresh_thread is None or not self._theme_refresh_thread.is_alive():
+            self._theme_refresh_thread = threading.Thread(
+                target=lambda: ThemeManager(Params(), self._params_memory).update_themes(get_starpilot_toggles(), update_assets=False), daemon=True,
+            )
+            self._theme_refresh_thread.start()
         def on_close(res, val):
             pass
 
-        gui_app.push_widget(SimpleDownloadManager(
+        manager = SimpleDownloadManager(
             title=tr("Boot Logo"),
             asset_type="boot logo",
             directory=THEME_SAVE_PATH / "bootlogos",
@@ -760,4 +769,6 @@ class StarPilotAppearanceLayout(_SettingsPage):
             params=self._params,
             params_memory=self._params_memory,
             on_close=on_close,
-        ))
+        )
+        manager.refresh_thread = self._theme_refresh_thread
+        gui_app.push_widget(manager)

selfdrive/ui/layouts/settings/starpilot/simple_download_manager.py

@@ -154,6 +154,7 @@ class SimpleDownloadManager(Widget):
 
     self._active_mode = self.MODE_SELECT
     self._downloading = False
+    self.refresh_thread = None
     self._download_progress = ""
     self._finalizing_until = 0.0
     self._pressed_target: str | None = None
@@ -316,6 +317,10 @@ class SimpleDownloadManager(Widget):
 
   def _update_state(self):
     now = time.monotonic()
+    if self.refresh_thread is not None and not self.refresh_thread.is_alive():
+      self.params.invalidate(self.downloadable_list_param)
+      self._refresh_list()
+      self.refresh_thread = None
     if self._pressed_target is not None and not self.is_pressed:
       self._pressed_target = None
     if self._info_message_until > 0 and now >= self._info_message_until:

starpilot/assets/tests/test_theme_catalog_refresh.py

@@ -0,0 +1,176 @@
+"""Exercise source methods with fake HTTP/Params; no native imports or live requests."""
+import ast
+import io
+import json
+import os
+from pathlib import Path
+from types import SimpleNamespace
+import zipfile
+
+import pytest
+import requests
+
+
+def load_source(path, names, namespace):
+  nodes = [node for node in ast.parse(path.read_text()).body if
+           isinstance(node, (ast.ClassDef, ast.FunctionDef)) and node.name in names or
+           isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id in names for target in node.targets)]
+  exec(compile(ast.Module(body=nodes, type_ignores=[]), str(path), "exec"), namespace)
+
+
+class Params:
+  def __init__(self):
+    self.values = {}
+  def get_bool(self, key):
+    return bool(self.values.get(key))
+  def put(self, key, value):
+    self.values[key] = value
+  def remove(self, key):
+    self.values.pop(key, None)
+
+
+class Response:
+  def __init__(self, data=None, payload=b"", status=200):
+    self.data, self.payload, self.status_code = data, payload, status
+    self.headers = {"Content-Length": str(len(payload))}
+  def json(self):
+    return self.data
+  def raise_for_status(self):
+    if self.status_code >= 400:
+      raise requests.exceptions.HTTPError(str(self.status_code), response=self)
+  def __enter__(self):
+    return self
+  def __exit__(self, *args):
+    pass
+  def iter_content(self, chunk_size):
+    yield self.payload
+
+
+@pytest.fixture
+def theme(tmp_path):
+  root = Path(__file__).resolve().parents[3]
+  archive = io.BytesIO()
+  with zipfile.ZipFile(archive, "w") as output:
+    output.writestr("asset.bin", b"downloaded asset")
+  payloads = {"Distance-Icons/frog-animated.zip": archive.getvalue(),
+              "Themes/frog/colors.zip": archive.getvalue(), "Steering-Wheels/frog.png": b"new wheel"}
+  trees = {branch: [dict(path="theme/" + path, type="file", size=len(payload))
+                    for path, payload in payloads.items() if path.startswith(branch + "/")]
+           for branch in ("Distance-Icons", "Steering-Wheels", "Themes")}
+  trees["Themes"].append(dict(path="theme/Themes/bootlogo/Preview.png", type="file", size=20))
+  catalogs, transfers = [], []
+  def get(url, **kwargs):
+    if "/tree/theme/" in url:
+      branch = url.split("/tree/theme/", 1)[1].split("?", 1)[0]
+      catalogs.append(branch)
+      return Response(data=trees[branch])
+    path = url.split("/theme/", 1)[1] if "/theme/" in url else url.split("example.invalid/", 1)[1]
+    transfers.append(path)
+    return Response(payload=payloads.get(path, b""), status=200 if path in payloads else 404)
+  def head(url, **kwargs):
+    path = url.split("/theme/", 1)[1] if "/theme/" in url else url.split("example.invalid/", 1)[1]
+    return Response(payload=payloads.get(path, b""), status=200 if path in payloads else 404)
+  errors = []
+  namespace = dict(Path=Path, requests=requests, json=json, os=os, zipfile=zipfile,
+                   HF_BUCKET="fixture/resources", GITHUB_URL="https://example.invalid", THEME_SAVE_PATH=tmp_path,
+                   get_resource_urls=lambda _: ["https://huggingface.co/buckets/fixture/resources/resolve", "https://example.invalid"],
+                   handle_error=lambda *args: errors.append(args), handle_request_error=lambda *args: errors.append(args))
+  load_source(root / "starpilot/common/starpilot_utilities.py", {"delete_file", "extract_zip", "update_json_file"}, namespace)
+  load_source(root / "starpilot/common/starpilot_download_utilities.py",
+              {"download_file", "get_remote_file_size", "verify_download"}, namespace)
+  load_source(root / "starpilot/assets/theme_manager.py",
+              {"ThemeManager", "THEME_COMPONENT_PARAMS", "CANCEL_DOWNLOAD_PARAM", "DOWNLOAD_PROGRESS_PARAM"}, namespace)
+  for path in ("bootlogos", "steering_wheels", "theme_packs/frog/colors", "theme_packs/frog-animated/distance_icons"):
+    (tmp_path / path).mkdir(parents=True)
+  (tmp_path / "steering_wheels/frog.png").write_bytes(b"old wheel")
+  manager = namespace["ThemeManager"].__new__(namespace["ThemeManager"])
+  manager.params, manager.params_memory = Params(), Params()
+  manager.downloading_theme = False
+  manager.theme_sizes, manager.theme_sizes_path = {}, tmp_path / "theme_sizes.json"
+  manager.session = SimpleNamespace(get=get, head=head)
+  manager.sync_local_resources = lambda: None
+  return SimpleNamespace(manager=manager, catalogs=catalogs, transfers=transfers, payloads=payloads, errors=errors, path=tmp_path)
+
+
+@pytest.mark.parametrize("recorded_size", [None, 1])
+def test_menu_refresh_lists_choices_without_downloading_existing_assets(theme, recorded_size):
+  if recorded_size is not None:
+    theme.manager.theme_sizes = {"wheels": {"frog": recorded_size}, "themes": {"frog": {"colors": recorded_size}}}
+  theme.manager.update_themes(object(), update_assets=False)
+  assert theme.catalogs == ["Distance-Icons", "Steering-Wheels", "Themes"]
+  assert not theme.transfers
+  assert theme.manager.params.values["DownloadableBootLogos"] == "Preview"
+  assert theme.manager.params.values["ThemesDownloaded"]["themes"]["Frog"] == ["colors"]
+  assert (theme.path / "steering_wheels/frog.png").read_bytes() == b"old wheel"
+
+
+def test_default_maintenance_updates_assets_without_nested_downloads(theme):
+  depth, maximum = 0, 0
+  original = theme.manager.download_theme
+  def download(*args):
+    nonlocal depth, maximum
+    depth += 1
+    maximum = max(maximum, depth)
+    try:
+      return original(*args)
+    finally:
+      depth -= 1
+  theme.manager.download_theme = download
+  theme.manager.update_themes(object())
+  assert maximum == 1
+  assert theme.transfers.count("Steering-Wheels/frog.png") == 1
+  assert (theme.path / "theme_packs/frog/colors/asset.bin").read_bytes() == b"downloaded asset"
+  assert theme.manager.theme_sizes["themes"]["frog"]["colors"] == len(theme.payloads["Themes/frog/colors.zip"])
+
+
+def test_selected_wheel_accepts_png_fallback_without_updating_other_assets(theme):
+  theme.manager.params_memory.put("WheelToDownload", "frog")
+  theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object())
+  assert theme.transfers == ["Steering-Wheels/frog.gif", "Steering-Wheels/frog.png"]
+  assert len(theme.catalogs) == 3
+  assert (theme.path / "steering_wheels/frog.png").read_bytes() == b"new wheel"
+  assert theme.manager.theme_sizes["wheels"]["frog"] == len(b"new wheel")
+  assert theme.manager.params_memory.values["ThemeDownloadProgress"] == "Downloaded!"
+  assert "WheelToDownload" not in theme.manager.params_memory.values
+  assert not theme.manager.downloading_theme
+
+
+def test_failed_fallback_does_not_confirm_existing_png(theme):
+  theme.payloads.pop("Steering-Wheels/frog.png")
+  theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object())
+  assert theme.errors
+  assert "ThemeDownloadProgress" not in theme.manager.params_memory.values
+  assert not theme.manager.theme_sizes
+  assert not theme.catalogs
+  assert not theme.manager.downloading_theme
+
+
+def test_rejected_png_fallback_can_retry_original_gif_from_next_origin(theme):
+  theme.payloads["Steering-Wheels/frog.gif"] = b"valid gif"
+  original_get, original_head = theme.manager.session.get, theme.manager.session.head
+  def get(url, **kwargs):
+    if "huggingface.co" in url and url.endswith(".gif"):
+      theme.transfers.append("Steering-Wheels/frog.gif")
+      return Response(status=404)
+    return original_get(url, **kwargs)
+  def head(url, **kwargs):
+    if "huggingface.co" in url and url.endswith(".png"):
+      return Response(payload=b"wrong remote size")
+    return original_head(url, **kwargs)
+  theme.manager.session.get, theme.manager.session.head = get, head
+  theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object())
+  assert theme.transfers == ["Steering-Wheels/frog.gif", "Steering-Wheels/frog.png", "Steering-Wheels/frog.gif"]
+  assert (theme.path / "steering_wheels/frog.gif").read_bytes() == b"valid gif"
+  assert theme.manager.theme_sizes["wheels"]["frog"] == len(b"valid gif")
+  assert theme.manager.params_memory.values["ThemeDownloadProgress"] == "Downloaded!"
+
+
+def test_cancelled_download_does_not_publish_success_or_refresh_catalog(theme):
+  theme.manager.params_memory.put("CancelThemeDownload", True)
+  theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object())
+  assert theme.errors
+  assert "ThemeDownloadProgress" not in theme.manager.params_memory.values
+  assert not theme.manager.theme_sizes
+  assert not theme.catalogs
+  assert not list(theme.path.rglob("*.tmp"))
+  assert not theme.manager.downloading_theme

starpilot/assets/theme_manager.py

@@ -242,7 +242,6 @@ class ThemeManager:
       name_candidates = [theme_name]
 
     for extension in extensions:
-      theme_path = download_path.with_suffix(extension)
       theme_urls = []
       for resource_url in resource_urls:
         source_prefix = f"{resource_url}/theme" if "huggingface.co/buckets/" in resource_url else resource_url
@@ -257,11 +256,13 @@ class ThemeManager:
           theme_urls.append(f"{source_prefix}/Themes/{theme_name}/{theme_component}{extension}")
 
       for theme_url in theme_urls:
+        theme_path = download_path.with_suffix(extension)
         delete_file(theme_path)
 
         source = "Hugging Face" if "huggingface.co/buckets/" in theme_url else "GitHub"
         print(f"Downloading theme from {source}: {theme_name}")
-        download_file(CANCEL_DOWNLOAD_PARAM, theme_path, asset_param, self.params_memory, DOWNLOAD_PROGRESS_PARAM, self.session, theme_url)
+        downloaded_path = download_file(CANCEL_DOWNLOAD_PARAM, theme_path, asset_param, self.params_memory, DOWNLOAD_PROGRESS_PARAM, self.session, theme_url)
+        theme_path = downloaded_path or theme_path
 
         if self.params_memory.get_bool(CANCEL_DOWNLOAD_PARAM):
           delete_file(theme_path)
@@ -283,13 +284,13 @@ class ThemeManager:
 
           self.downloading_theme = False
 
-          self.update_themes(starpilot_toggles)
+          self.update_themes(starpilot_toggles, update_assets=False)
           return
 
     handle_error(download_path, asset_param, "Download failed...", "Download failed...", self.params_memory, DOWNLOAD_PROGRESS_PARAM)
     self.downloading_theme = False
 
-  def fetch_assets(self, repo_url, starpilot_toggles):
+  def fetch_assets(self, repo_url, starpilot_toggles, *, update_assets=True):
     is_github = "github" in repo_url
     is_huggingface = "huggingface.co/buckets/" in repo_url
 
@@ -297,7 +298,7 @@ class ThemeManager:
     try:
       def list_files(branch):
         if is_huggingface:
-          response = self.session.get(f"https://huggingface.co/api/buckets/{HF_BUCKET}/tree?recursive=true", timeout=10)
+          response = self.session.get(f"https://huggingface.co/api/buckets/{HF_BUCKET}/tree/theme/{branch}?recursive=true", timeout=10)
           response.raise_for_status()
           prefix = {
             "Themes": "theme/Themes/",
@@ -346,7 +347,7 @@ class ThemeManager:
             assets["wheels"].append(path)
             theme_name = Path(path).stem
             local_files = list((THEME_SAVE_PATH / "steering_wheels").glob(f"{theme_name}.*"))
-            if local_files and size > 0:
+            if update_assets and local_files and size > 0:
               local_size = self.theme_sizes.get("wheels", {}).get(theme_name)
               if local_size != size:
                 self.download_theme("steering_wheels", theme_name, THEME_COMPONENT_PARAMS["steering_wheels"], starpilot_toggles)
@@ -357,7 +358,7 @@ class ThemeManager:
             assets["themes"].setdefault(theme_name, set()).add(component_name)
 
             local_path = THEME_SAVE_PATH / "theme_packs" / theme_name / component_name
-            if local_path.exists() and size > 0:
+            if update_assets and local_path.exists() and size > 0:
               local_size = self.theme_sizes.get("themes", {}).get(theme_name, {}).get(component_name)
               if local_size != size:
                 self.download_theme(component_name, theme_name, THEME_COMPONENT_PARAMS[component_name], starpilot_toggles)
@@ -379,7 +380,7 @@ class ThemeManager:
           assets["boot_logos"].append(sub_path)
           logo_name = Path(sub_path).stem
           local_files = list((THEME_SAVE_PATH / "bootlogos").glob(f"{logo_name}.*"))
-          if local_files and expected_size > 0:
+          if update_assets and local_files and expected_size > 0:
             local_size = self.theme_sizes.get("boot_logos", {}).get(logo_name)
             if local_size != expected_size:
               print(f"boot logo {logo_name} is outdated, redownloading...")
@@ -391,7 +392,7 @@ class ThemeManager:
             assets["themes"].setdefault(theme_name, set()).add(key)
 
             local_path = THEME_SAVE_PATH / "theme_packs" / theme_name / key
-            if local_path.exists():
+            if update_assets and local_path.exists():
               local_size = self.theme_sizes.get("themes", {}).get(theme_name, {}).get(key)
               if local_size != expected_size:
                 print(f"{key} {theme_name} is outdated, redownloading...")
@@ -746,7 +747,7 @@ class ThemeManager:
 
     update_json_file(self.theme_sizes_path, self.theme_sizes)
 
-  def update_themes(self, starpilot_toggles, boot_run=False):
+  def update_themes(self, starpilot_toggles, boot_run=False, *, update_assets=True):
     if self.downloading_theme:
       return
 
@@ -760,7 +761,7 @@ class ThemeManager:
 
     assets = {}
     for repo_url in resource_urls:
-      assets = self.fetch_assets(repo_url, starpilot_toggles)
+      assets = self.fetch_assets(repo_url, starpilot_toggles, update_assets=update_assets)
       if assets:
         break
     if not assets:
@@ -799,7 +800,7 @@ class ThemeManager:
     print(f"Downloadable Distance Icons: {downloadable_distance_icons}")
     print(f"Downloadable Wheels: {downloadable_wheels}")
 
-    if boot_run:
+    if boot_run and update_assets:
       self.validate_themes(downloadable_boot_logos, downloadable_colors, downloadable_distance_icons, downloadable_icons, downloadable_signals, downloadable_sounds, downloadable_wheels, starpilot_toggles)
 
     self.update_theme_params(downloadable_boot_logos, downloadable_colors, downloadable_distance_icons, downloadable_icons, downloadable_signals, downloadable_sounds, downloadable_wheels)

starpilot/common/starpilot_download_utilities.py

@@ -54,6 +54,7 @@ def download_file(cancel_param, destination, download_param, params_memory, prog
                 params_memory.put(progress_param, "Verifying authenticity...")
 
         temp_file_path.replace(destination)
+        return destination
 
       except InterruptedError:
         temp_file_path.unlink(missing_ok=True)