diff --git a/selfdrive/ui/layouts/settings/starpilot/appearance.py b/selfdrive/ui/layouts/settings/starpilot/appearance.py index 528e9c4902..038c5a1152 100644 --- a/selfdrive/ui/layouts/settings/starpilot/appearance.py +++ b/selfdrive/ui/layouts/settings/starpilot/appearance.py @@ -1,6 +1,8 @@ from __future__ import annotations import re +import threading +from openpilot.common.params import Params from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.lib.multilang import tr, tr_noop from openpilot.system.ui.widgets import DialogResult @@ -18,7 +20,8 @@ from openpilot.selfdrive.ui.layouts.settings.starpilot.aethergrid import ( CardHubManagerView, ) from openpilot.selfdrive.ui.layouts.settings.starpilot.simple_download_manager import SimpleDownloadManager -from openpilot.starpilot.common.starpilot_variables import THEME_SAVE_PATH +from openpilot.starpilot.assets.theme_manager import ThemeManager +from openpilot.starpilot.common.starpilot_variables import THEME_SAVE_PATH, get_starpilot_toggles PANEL_STYLE = DEFAULT_PANEL_STYLE @@ -130,6 +133,7 @@ class AppearanceManagerView(CardHubManagerView): class StarPilotAppearanceLayout(_SettingsPage): def __init__(self): super().__init__() + self._theme_refresh_thread = None self._build_view() def _make_parent(self, key: str, label: str, subtitle: str = "") -> ParentToggle: @@ -747,10 +751,15 @@ class StarPilotAppearanceLayout(_SettingsPage): # ── Boot logo manager ── def _show_boot_logo_manager(self): + if self._theme_refresh_thread is None or not self._theme_refresh_thread.is_alive(): + self._theme_refresh_thread = threading.Thread( + target=lambda: ThemeManager(Params(), self._params_memory).update_themes(get_starpilot_toggles(), update_assets=False), daemon=True, + ) + self._theme_refresh_thread.start() def on_close(res, val): pass - gui_app.push_widget(SimpleDownloadManager( + manager = SimpleDownloadManager( title=tr("Boot Logo"), asset_type="boot logo", directory=THEME_SAVE_PATH / "bootlogos", @@ -760,4 +769,6 @@ class StarPilotAppearanceLayout(_SettingsPage): params=self._params, params_memory=self._params_memory, on_close=on_close, - )) + ) + manager.refresh_thread = self._theme_refresh_thread + gui_app.push_widget(manager) diff --git a/selfdrive/ui/layouts/settings/starpilot/simple_download_manager.py b/selfdrive/ui/layouts/settings/starpilot/simple_download_manager.py index c75957c814..6ce972a120 100644 --- a/selfdrive/ui/layouts/settings/starpilot/simple_download_manager.py +++ b/selfdrive/ui/layouts/settings/starpilot/simple_download_manager.py @@ -154,6 +154,7 @@ class SimpleDownloadManager(Widget): self._active_mode = self.MODE_SELECT self._downloading = False + self.refresh_thread = None self._download_progress = "" self._finalizing_until = 0.0 self._pressed_target: str | None = None @@ -316,6 +317,10 @@ class SimpleDownloadManager(Widget): def _update_state(self): now = time.monotonic() + if self.refresh_thread is not None and not self.refresh_thread.is_alive(): + self.params.invalidate(self.downloadable_list_param) + self._refresh_list() + self.refresh_thread = None if self._pressed_target is not None and not self.is_pressed: self._pressed_target = None if self._info_message_until > 0 and now >= self._info_message_until: diff --git a/starpilot/assets/tests/test_theme_catalog_refresh.py b/starpilot/assets/tests/test_theme_catalog_refresh.py new file mode 100644 index 0000000000..7570c83971 --- /dev/null +++ b/starpilot/assets/tests/test_theme_catalog_refresh.py @@ -0,0 +1,176 @@ +"""Exercise source methods with fake HTTP/Params; no native imports or live requests.""" +import ast +import io +import json +import os +from pathlib import Path +from types import SimpleNamespace +import zipfile + +import pytest +import requests + + +def load_source(path, names, namespace): + nodes = [node for node in ast.parse(path.read_text()).body if + isinstance(node, (ast.ClassDef, ast.FunctionDef)) and node.name in names or + isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id in names for target in node.targets)] + exec(compile(ast.Module(body=nodes, type_ignores=[]), str(path), "exec"), namespace) + + +class Params: + def __init__(self): + self.values = {} + def get_bool(self, key): + return bool(self.values.get(key)) + def put(self, key, value): + self.values[key] = value + def remove(self, key): + self.values.pop(key, None) + + +class Response: + def __init__(self, data=None, payload=b"", status=200): + self.data, self.payload, self.status_code = data, payload, status + self.headers = {"Content-Length": str(len(payload))} + def json(self): + return self.data + def raise_for_status(self): + if self.status_code >= 400: + raise requests.exceptions.HTTPError(str(self.status_code), response=self) + def __enter__(self): + return self + def __exit__(self, *args): + pass + def iter_content(self, chunk_size): + yield self.payload + + +@pytest.fixture +def theme(tmp_path): + root = Path(__file__).resolve().parents[3] + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as output: + output.writestr("asset.bin", b"downloaded asset") + payloads = {"Distance-Icons/frog-animated.zip": archive.getvalue(), + "Themes/frog/colors.zip": archive.getvalue(), "Steering-Wheels/frog.png": b"new wheel"} + trees = {branch: [dict(path="theme/" + path, type="file", size=len(payload)) + for path, payload in payloads.items() if path.startswith(branch + "/")] + for branch in ("Distance-Icons", "Steering-Wheels", "Themes")} + trees["Themes"].append(dict(path="theme/Themes/bootlogo/Preview.png", type="file", size=20)) + catalogs, transfers = [], [] + def get(url, **kwargs): + if "/tree/theme/" in url: + branch = url.split("/tree/theme/", 1)[1].split("?", 1)[0] + catalogs.append(branch) + return Response(data=trees[branch]) + path = url.split("/theme/", 1)[1] if "/theme/" in url else url.split("example.invalid/", 1)[1] + transfers.append(path) + return Response(payload=payloads.get(path, b""), status=200 if path in payloads else 404) + def head(url, **kwargs): + path = url.split("/theme/", 1)[1] if "/theme/" in url else url.split("example.invalid/", 1)[1] + return Response(payload=payloads.get(path, b""), status=200 if path in payloads else 404) + errors = [] + namespace = dict(Path=Path, requests=requests, json=json, os=os, zipfile=zipfile, + HF_BUCKET="fixture/resources", GITHUB_URL="https://example.invalid", THEME_SAVE_PATH=tmp_path, + get_resource_urls=lambda _: ["https://huggingface.co/buckets/fixture/resources/resolve", "https://example.invalid"], + handle_error=lambda *args: errors.append(args), handle_request_error=lambda *args: errors.append(args)) + load_source(root / "starpilot/common/starpilot_utilities.py", {"delete_file", "extract_zip", "update_json_file"}, namespace) + load_source(root / "starpilot/common/starpilot_download_utilities.py", + {"download_file", "get_remote_file_size", "verify_download"}, namespace) + load_source(root / "starpilot/assets/theme_manager.py", + {"ThemeManager", "THEME_COMPONENT_PARAMS", "CANCEL_DOWNLOAD_PARAM", "DOWNLOAD_PROGRESS_PARAM"}, namespace) + for path in ("bootlogos", "steering_wheels", "theme_packs/frog/colors", "theme_packs/frog-animated/distance_icons"): + (tmp_path / path).mkdir(parents=True) + (tmp_path / "steering_wheels/frog.png").write_bytes(b"old wheel") + manager = namespace["ThemeManager"].__new__(namespace["ThemeManager"]) + manager.params, manager.params_memory = Params(), Params() + manager.downloading_theme = False + manager.theme_sizes, manager.theme_sizes_path = {}, tmp_path / "theme_sizes.json" + manager.session = SimpleNamespace(get=get, head=head) + manager.sync_local_resources = lambda: None + return SimpleNamespace(manager=manager, catalogs=catalogs, transfers=transfers, payloads=payloads, errors=errors, path=tmp_path) + + +@pytest.mark.parametrize("recorded_size", [None, 1]) +def test_menu_refresh_lists_choices_without_downloading_existing_assets(theme, recorded_size): + if recorded_size is not None: + theme.manager.theme_sizes = {"wheels": {"frog": recorded_size}, "themes": {"frog": {"colors": recorded_size}}} + theme.manager.update_themes(object(), update_assets=False) + assert theme.catalogs == ["Distance-Icons", "Steering-Wheels", "Themes"] + assert not theme.transfers + assert theme.manager.params.values["DownloadableBootLogos"] == "Preview" + assert theme.manager.params.values["ThemesDownloaded"]["themes"]["Frog"] == ["colors"] + assert (theme.path / "steering_wheels/frog.png").read_bytes() == b"old wheel" + + +def test_default_maintenance_updates_assets_without_nested_downloads(theme): + depth, maximum = 0, 0 + original = theme.manager.download_theme + def download(*args): + nonlocal depth, maximum + depth += 1 + maximum = max(maximum, depth) + try: + return original(*args) + finally: + depth -= 1 + theme.manager.download_theme = download + theme.manager.update_themes(object()) + assert maximum == 1 + assert theme.transfers.count("Steering-Wheels/frog.png") == 1 + assert (theme.path / "theme_packs/frog/colors/asset.bin").read_bytes() == b"downloaded asset" + assert theme.manager.theme_sizes["themes"]["frog"]["colors"] == len(theme.payloads["Themes/frog/colors.zip"]) + + +def test_selected_wheel_accepts_png_fallback_without_updating_other_assets(theme): + theme.manager.params_memory.put("WheelToDownload", "frog") + theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object()) + assert theme.transfers == ["Steering-Wheels/frog.gif", "Steering-Wheels/frog.png"] + assert len(theme.catalogs) == 3 + assert (theme.path / "steering_wheels/frog.png").read_bytes() == b"new wheel" + assert theme.manager.theme_sizes["wheels"]["frog"] == len(b"new wheel") + assert theme.manager.params_memory.values["ThemeDownloadProgress"] == "Downloaded!" + assert "WheelToDownload" not in theme.manager.params_memory.values + assert not theme.manager.downloading_theme + + +def test_failed_fallback_does_not_confirm_existing_png(theme): + theme.payloads.pop("Steering-Wheels/frog.png") + theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object()) + assert theme.errors + assert "ThemeDownloadProgress" not in theme.manager.params_memory.values + assert not theme.manager.theme_sizes + assert not theme.catalogs + assert not theme.manager.downloading_theme + + +def test_rejected_png_fallback_can_retry_original_gif_from_next_origin(theme): + theme.payloads["Steering-Wheels/frog.gif"] = b"valid gif" + original_get, original_head = theme.manager.session.get, theme.manager.session.head + def get(url, **kwargs): + if "huggingface.co" in url and url.endswith(".gif"): + theme.transfers.append("Steering-Wheels/frog.gif") + return Response(status=404) + return original_get(url, **kwargs) + def head(url, **kwargs): + if "huggingface.co" in url and url.endswith(".png"): + return Response(payload=b"wrong remote size") + return original_head(url, **kwargs) + theme.manager.session.get, theme.manager.session.head = get, head + theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object()) + assert theme.transfers == ["Steering-Wheels/frog.gif", "Steering-Wheels/frog.png", "Steering-Wheels/frog.gif"] + assert (theme.path / "steering_wheels/frog.gif").read_bytes() == b"valid gif" + assert theme.manager.theme_sizes["wheels"]["frog"] == len(b"valid gif") + assert theme.manager.params_memory.values["ThemeDownloadProgress"] == "Downloaded!" + + +def test_cancelled_download_does_not_publish_success_or_refresh_catalog(theme): + theme.manager.params_memory.put("CancelThemeDownload", True) + theme.manager.download_theme("steering_wheels", "frog", "WheelToDownload", object()) + assert theme.errors + assert "ThemeDownloadProgress" not in theme.manager.params_memory.values + assert not theme.manager.theme_sizes + assert not theme.catalogs + assert not list(theme.path.rglob("*.tmp")) + assert not theme.manager.downloading_theme diff --git a/starpilot/assets/theme_manager.py b/starpilot/assets/theme_manager.py index f19cf262eb..4a3f24dfd4 100644 --- a/starpilot/assets/theme_manager.py +++ b/starpilot/assets/theme_manager.py @@ -242,7 +242,6 @@ class ThemeManager: name_candidates = [theme_name] for extension in extensions: - theme_path = download_path.with_suffix(extension) theme_urls = [] for resource_url in resource_urls: source_prefix = f"{resource_url}/theme" if "huggingface.co/buckets/" in resource_url else resource_url @@ -257,11 +256,13 @@ class ThemeManager: theme_urls.append(f"{source_prefix}/Themes/{theme_name}/{theme_component}{extension}") for theme_url in theme_urls: + theme_path = download_path.with_suffix(extension) delete_file(theme_path) source = "Hugging Face" if "huggingface.co/buckets/" in theme_url else "GitHub" print(f"Downloading theme from {source}: {theme_name}") - download_file(CANCEL_DOWNLOAD_PARAM, theme_path, asset_param, self.params_memory, DOWNLOAD_PROGRESS_PARAM, self.session, theme_url) + downloaded_path = download_file(CANCEL_DOWNLOAD_PARAM, theme_path, asset_param, self.params_memory, DOWNLOAD_PROGRESS_PARAM, self.session, theme_url) + theme_path = downloaded_path or theme_path if self.params_memory.get_bool(CANCEL_DOWNLOAD_PARAM): delete_file(theme_path) @@ -283,13 +284,13 @@ class ThemeManager: self.downloading_theme = False - self.update_themes(starpilot_toggles) + self.update_themes(starpilot_toggles, update_assets=False) return handle_error(download_path, asset_param, "Download failed...", "Download failed...", self.params_memory, DOWNLOAD_PROGRESS_PARAM) self.downloading_theme = False - def fetch_assets(self, repo_url, starpilot_toggles): + def fetch_assets(self, repo_url, starpilot_toggles, *, update_assets=True): is_github = "github" in repo_url is_huggingface = "huggingface.co/buckets/" in repo_url @@ -297,7 +298,7 @@ class ThemeManager: try: def list_files(branch): if is_huggingface: - response = self.session.get(f"https://huggingface.co/api/buckets/{HF_BUCKET}/tree?recursive=true", timeout=10) + response = self.session.get(f"https://huggingface.co/api/buckets/{HF_BUCKET}/tree/theme/{branch}?recursive=true", timeout=10) response.raise_for_status() prefix = { "Themes": "theme/Themes/", @@ -346,7 +347,7 @@ class ThemeManager: assets["wheels"].append(path) theme_name = Path(path).stem local_files = list((THEME_SAVE_PATH / "steering_wheels").glob(f"{theme_name}.*")) - if local_files and size > 0: + if update_assets and local_files and size > 0: local_size = self.theme_sizes.get("wheels", {}).get(theme_name) if local_size != size: self.download_theme("steering_wheels", theme_name, THEME_COMPONENT_PARAMS["steering_wheels"], starpilot_toggles) @@ -357,7 +358,7 @@ class ThemeManager: assets["themes"].setdefault(theme_name, set()).add(component_name) local_path = THEME_SAVE_PATH / "theme_packs" / theme_name / component_name - if local_path.exists() and size > 0: + if update_assets and local_path.exists() and size > 0: local_size = self.theme_sizes.get("themes", {}).get(theme_name, {}).get(component_name) if local_size != size: self.download_theme(component_name, theme_name, THEME_COMPONENT_PARAMS[component_name], starpilot_toggles) @@ -379,7 +380,7 @@ class ThemeManager: assets["boot_logos"].append(sub_path) logo_name = Path(sub_path).stem local_files = list((THEME_SAVE_PATH / "bootlogos").glob(f"{logo_name}.*")) - if local_files and expected_size > 0: + if update_assets and local_files and expected_size > 0: local_size = self.theme_sizes.get("boot_logos", {}).get(logo_name) if local_size != expected_size: print(f"boot logo {logo_name} is outdated, redownloading...") @@ -391,7 +392,7 @@ class ThemeManager: assets["themes"].setdefault(theme_name, set()).add(key) local_path = THEME_SAVE_PATH / "theme_packs" / theme_name / key - if local_path.exists(): + if update_assets and local_path.exists(): local_size = self.theme_sizes.get("themes", {}).get(theme_name, {}).get(key) if local_size != expected_size: print(f"{key} {theme_name} is outdated, redownloading...") @@ -746,7 +747,7 @@ class ThemeManager: update_json_file(self.theme_sizes_path, self.theme_sizes) - def update_themes(self, starpilot_toggles, boot_run=False): + def update_themes(self, starpilot_toggles, boot_run=False, *, update_assets=True): if self.downloading_theme: return @@ -760,7 +761,7 @@ class ThemeManager: assets = {} for repo_url in resource_urls: - assets = self.fetch_assets(repo_url, starpilot_toggles) + assets = self.fetch_assets(repo_url, starpilot_toggles, update_assets=update_assets) if assets: break if not assets: @@ -799,7 +800,7 @@ class ThemeManager: print(f"Downloadable Distance Icons: {downloadable_distance_icons}") print(f"Downloadable Wheels: {downloadable_wheels}") - if boot_run: + if boot_run and update_assets: self.validate_themes(downloadable_boot_logos, downloadable_colors, downloadable_distance_icons, downloadable_icons, downloadable_signals, downloadable_sounds, downloadable_wheels, starpilot_toggles) self.update_theme_params(downloadable_boot_logos, downloadable_colors, downloadable_distance_icons, downloadable_icons, downloadable_signals, downloadable_sounds, downloadable_wheels) diff --git a/starpilot/common/starpilot_download_utilities.py b/starpilot/common/starpilot_download_utilities.py index aeb0b6f1e9..d91bc5b18c 100644 --- a/starpilot/common/starpilot_download_utilities.py +++ b/starpilot/common/starpilot_download_utilities.py @@ -54,6 +54,7 @@ def download_file(cancel_param, destination, download_param, params_memory, prog params_memory.put(progress_param, "Verifying authenticity...") temp_file_path.replace(destination) + return destination except InterruptedError: temp_file_path.unlink(missing_ok=True)