diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp index 921fa600e..fb90ecd3a 100644 --- a/openpilot/cereal/custom.capnp +++ b/openpilot/cereal/custom.capnp @@ -131,6 +131,7 @@ struct ModelManagerSP @0xaedffd8f31e7b55d { downloaded @2; cached @3; failed @4; + verifying @5; } struct DownloadProgress { diff --git a/openpilot/common/params_keys.h b/openpilot/common/params_keys.h index 47eb57c54..77fcab2b3 100644 --- a/openpilot/common/params_keys.h +++ b/openpilot/common/params_keys.h @@ -217,11 +217,10 @@ inline static std::unordered_map keys = { // Model Manager params {"ModelManager_ActiveBundle", {PERSISTENT, JSON}}, - {"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, STRING}}, - {"ModelManager_PrevBundle", {PERSISTENT, JSON}}, - {"ModelManager_PrevBundle_USBGPU", {PERSISTENT, JSON}}, + {"ModelManager_ActiveBundleUSBGPU", {PERSISTENT, JSON}}, + {"ModelManager_ActiveJson", {CLEAR_ON_MANAGER_START, JSON}}, {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}}, - {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}}, + {"ModelManager_DownloadRef", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}}, {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}}, {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}}, {"ModelManager_LastSyncTime_USBGPU", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}}, @@ -269,6 +268,7 @@ inline static std::unordered_map keys = { // mapd {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}}, + {"Mapd_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}}, {"MapdVersion", {PERSISTENT, STRING}}, {"MapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0.0"}}, {"NextMapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, JSON}}, diff --git a/openpilot/selfdrive/ui/layouts/sidebar.py b/openpilot/selfdrive/ui/layouts/sidebar.py index f950edaa4..5429a3585 100644 --- a/openpilot/selfdrive/ui/layouts/sidebar.py +++ b/openpilot/selfdrive/ui/layouts/sidebar.py @@ -168,9 +168,16 @@ class Sidebar(Widget, SidebarSP): # Home/Flag button flag_pressed = mouse_down and rl.check_collision_point_rec(mouse_pos, HOME_BTN) button_img = self._flag_img if ui_state.started else self._home_img + button_pos = rl.Vector2(HOME_BTN.x, HOME_BTN.y) + icon_opacity = 1.0 + + if gui_app.sunnypilot_ui(): + button_img, button_pos, icon_opacity = SidebarSP._get_home_icon(self, button_img) tint = Colors.BUTTON_PRESSED if (ui_state.started and flag_pressed) else Colors.BUTTON_NORMAL - rl.draw_texture_ex(button_img, rl.Vector2(HOME_BTN.x, HOME_BTN.y), 0.0, 1.0, tint) + if icon_opacity < 1.0: + tint = rl.Color(tint[0], tint[1], tint[2], int(255 * icon_opacity)) + rl.draw_texture_ex(button_img, button_pos, 0.0, 1.0, tint) # Microphone button if self._recording_audio: diff --git a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py index e4a6bea6e..3aa115139 100644 --- a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py +++ b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/models.py @@ -10,9 +10,10 @@ import time import pyray as rl from openpilot.cereal import custom -from openpilot.sunnypilot.models.default_model import get_default_model +from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_selected_bundle, resolve_bundle_by_ref from openpilot.common.constants import CV from openpilot.selfdrive.ui.ui_state import device, ui_state +from openpilot.selfdrive.ui.sunnypilot.model_info import big_model_state, bundles_for_source, carrying_model, default_model_name, queued_name from openpilot.system.ui.lib.multilang import tr from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.widgets import DialogResult, Widget @@ -36,7 +37,10 @@ class ModelsLayout(Widget): super().__init__() self.model_manager = None self.model_dialog = None + self._selection_source = None self._downloading = False + self._verifying = False + self._last_note = None self.last_cache_calc_time = 0 self._initialize_items() @@ -48,17 +52,24 @@ class ModelsLayout(Widget): self._scroller = Scroller(self.items, line_separator=True, spacing=0) def _initialize_items(self): - self.current_model_item = ListItemSP( - title=tr("Current Model"), + self.small_model_item = ListItemSP( + title=tr("Small Model"), description="", action_item=ScrollingButtonAction(tr("SELECT")), - callback=self._handle_current_model_clicked + callback=lambda: self._open_source_dialog("qcom") + ) + + self.big_model_item = ListItemSP( + title=tr("Big Model"), + action_item=ScrollingButtonAction(tr("SELECT")), + callback=lambda: self._open_source_dialog("usbgpu") ) self.download_item = download_status_item(lambda: tr("Download") if self._downloading else tr("Model Status")) self.refresh_item = button_item(tr("Refresh Model List"), tr("REFRESH"), "", lambda: (ui_state.params.put("ModelManager_LastSyncTime", 0), + ui_state.params.put("ModelManager_LastSyncTime_USBGPU", 0), gui_app.push_widget(alert_dialog(tr("Fetching Latest Models"))))) self.clear_cache_item = ListItemSP( @@ -68,7 +79,9 @@ class ModelsLayout(Widget): callback=self._clear_cache ) - self.cancel_download_item = button_item(tr("Cancel Download"), tr("Cancel"), "", lambda: ui_state.params.remove("ModelManager_DownloadIndex")) + self.cancel_download_item = button_item(lambda: tr("Cancel Verification") if self._verifying else tr("Cancel Download"), + tr("Cancel"), "", + lambda: ui_state.params.remove("ModelManager_DownloadRef")) self.lane_turn_value_control = option_item_sp(tr("Adjust Lane Turn Speed"), "LaneTurnValue", 500, 2000, tr("Set the maximum speed for lane turn desires. Default is 19 mph."), @@ -93,7 +106,7 @@ class ModelsLayout(Widget): 1, None, True, "", style.BUTTON_ACTION_WIDTH, None, True, lambda v: f"{v / 100:.2f} m") - self.items = [self.current_model_item, self.cancel_download_item, self.download_item, self.refresh_item, self.clear_cache_item, + self.items = [self.small_model_item, self.big_model_item, self.cancel_download_item, self.download_item, self.refresh_item, self.clear_cache_item, self.lane_turn_desire_toggle, self.lane_turn_value_control, self.lagd_toggle, self.delay_control, self.camera_offset] def _update_lagd_description(self, lagd_toggle: bool): @@ -107,10 +120,6 @@ class ModelsLayout(Widget): desc += f"
{tr('Actuator Delay:')} {cp:.2f} s + {tr('Software Delay:')} {sw:.2f} s = {tr('Total Delay:')} {cp + sw:.2f} s" self.lagd_toggle.set_description(desc) - def _is_downloading(self): - return (self.model_manager and self.model_manager.selectedBundle and - self.model_manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.downloading) - @staticmethod def calculate_cache_size(): cache_size = 0.0 @@ -133,36 +142,90 @@ class ModelsLayout(Widget): gui_app.push_widget(dialog) def _handle_bundle_download_progress(self): - self.download_item.set_visible(False) self.cancel_download_item.set_visible(False) self._downloading = False - - if not self.model_manager or (not self.model_manager.selectedBundle and not self.model_manager.activeBundle): - return - - bundle = self.model_manager.selectedBundle if self._is_downloading() or ( - self.model_manager.selectedBundle and self.model_manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.failed - ) else self.model_manager.activeBundle - if not bundle: - return - - self.cancel_download_item.set_visible(bool(self.model_manager.selectedBundle) and ui_state.params.get("ModelManager_DownloadIndex") is not None) + self._verifying = False + self.download_item.set_visible(True) if (current_time := time.monotonic()) - self.last_cache_calc_time > 0.5: self.last_cache_calc_time = current_time self.clear_cache_item.action_item.set_value(f"{self.calculate_cache_size():.2f} MB") + bundle = self.model_manager.selectedBundle if self.model_manager else None + progresses = [model.artifact.downloadProgress for model in bundle.models if model.artifact.fileName] if bundle else [] + if not progresses or bundle.status not in (custom.ModelManagerSP.DownloadStatus.downloading, + custom.ModelManagerSP.DownloadStatus.failed): + self.download_item.action_item.update(name="", segments=self._slot_segments()) + return + + self.cancel_download_item.set_visible(ui_state.params.get("ModelManager_DownloadRef") is not None) if bundle.status == custom.ModelManagerSP.DownloadStatus.downloading: device._reset_interactive_timeout() - # every bundle is a single chunked artifact now - progresses = [model.artifact.downloadProgress for model in bundle.models if model.artifact.fileName] - if not progresses: - return - - self.download_item.set_visible(True) - self.download_item.action_item.update(**self._download_row_state(progresses, bundle.internalName)) + state = self._download_row_state(progresses, bundle.internalName) + if queued := queued_name(bundle.ref): + state["name"] += f" | {queued} {tr('queued')}" + self.download_item.action_item.update(**state) self._downloading = self.download_item.action_item.downloading + ds = custom.ModelManagerSP.DownloadStatus + self._verifying = any(getattr(p.status, 'raw', p.status) == ds.verifying for p in progresses) + + def _slot_segments(self): + """small and big slots side by side; green marks the slot whose pick is actually + driving (runner-matched, so a failed Default big greens neither slot), an empty + slot shows its default.""" + big_state = big_model_state() + carry_source, carry_internal, _ = carrying_model() + segments = [] + for source, label in (("qcom", tr("small")), ("usbgpu", tr("big"))): + if segments: + segments.append(("|", rl.GRAY, None, None)) + bundle = get_selected_bundle(ui_state.params, source) + name = bundle.internalName if bundle else default_model_name(source) + color = ON_COLOR if (source == carry_source and name == carry_internal) else rl.LIGHTGRAY + name = "● " + name + if source == "usbgpu": + if big_state == 'failed': + color = rl.RED + elif big_state == 'loading': + color = rl.GOLD + segments.append((label, rl.GRAY, None, None)) + segments.append((name, color, None, None)) + return segments + + @staticmethod + def _set_item_note(item, text): + # a description renders only while shown; hide before clearing or the + # empty description keeps its visible state + if text: + item.set_description(text) + item.show_description(True) + else: + item.show_description(False) + item.set_description("") + + def _status_note(self) -> str: + """The failover story for the Model Status row. One-way big -> small, and the + fallback is runner-matched: a Default big can only fall back to the Default + small (stock modeld), a custom big has no automatic fallback yet.""" + if not ui_state.usbgpu: + return "" + big_bundle = get_selected_bundle(ui_state.params, "usbgpu") + big_name = big_bundle.internalName if big_bundle else default_model_name("usbgpu") + big_is_default = big_bundle is None + fallback_name = default_model_name("qcom") + state = big_model_state() + if state == 'failed': + if big_is_default: + return tr("Big model unavailable, {} is driving until the next drive.").format(fallback_name) + return tr("Big model unavailable until the next drive.") + if state == 'loading': + if big_is_default: + return tr("{} drives until the big model is ready.").format(fallback_name) + return tr("Getting the big model ready.") + if big_is_default: + return tr("{} will drive. If it fails during a drive, {} takes over until the next drive.").format(big_name, fallback_name) + return tr("{} will drive when the eGPU is ready.").format(big_name) @staticmethod def _download_row_state(progresses, name: str) -> dict: @@ -175,6 +238,8 @@ class ModelsLayout(Widget): if ds.failed in statuses: # close.png is authored black and a tint cannot lift it, hence close2 return {"name": name, "status_text": tr("download failed"), "text_color": rl.RED, "icon": "icons/close2.png"} + if ds.verifying in statuses: + return {"name": name, "downloading": True, "progress": progress, "status_text": tr("verifying")} if ds.downloading in statuses: return {"name": name, "downloading": True, "progress": progress} if statuses <= {ds.downloaded, ds.cached}: @@ -184,50 +249,71 @@ class ModelsLayout(Widget): def _on_model_selected(self, result): if result != DialogResult.CONFIRM: + self.model_dialog = None return selected_ref = self.model_dialog.selection_ref - if selected_ref == "Default": - ui_state.params.remove("ModelManager_ActiveBundle") - elif selected_bundle := next((bundle for bundle in self.model_manager.availableBundles if bundle.ref == selected_ref), None): - ui_state.params.put("ModelManager_DownloadIndex", selected_bundle.index) self.model_dialog = None + if selected_ref == "Default": + if self._selection_source in ACTIVE_BUNDLE_KEYS: + ui_state.params.remove(ACTIVE_BUNDLE_KEYS[self._selection_source]) + return + if selected_bundle := self._resolve_selected_bundle(selected_ref): + ui_state.params.put("ModelManager_DownloadRef", selected_bundle.ref) + + def _resolve_selected_bundle(self, ref): + source_bundles = {source: bundles_for_source(source) for source in ("qcom", "usbgpu")} + resolved = resolve_bundle_by_ref(ref, source_bundles) + return resolved[0] if resolved else None @staticmethod def _bundle_to_node(bundle): return TreeNode(bundle.ref, {'display_name': bundle.displayName, 'short_name': bundle.internalName}) - def _get_folders(self, favorites): - bundles = self.model_manager.availableBundles + def _get_folders(self, favorites, bundles): folders = {} for bundle in bundles: folders.setdefault(next((ov_ride.value for ov_ride in bundle.overrides if ov_ride.key == "folder"), ""), []).append(bundle) - folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': f"{get_default_model()} (Default)", - 'short_name': "Default"})])] + folders_list = [] for folder, folder_bundles in sorted(folders.items(), key=lambda x: max((bundle.index for bundle in x[1]), default=-1), reverse=True): folder_bundles.sort(key=lambda bundle: bundle.index, reverse=True) name = folder + (f" - (Updated: {m.group(1)})" if folder_bundles and (m := re.search(r'\(([^)]*)\)[^(]*$', folder_bundles[0].displayName)) else "") folders_list.append(TreeFolder(name, [self._bundle_to_node(bundle) for bundle in folder_bundles])) if favorites and (fav_bundles := [bundle for bundle in bundles if bundle.ref in favorites]): - folders_list.insert(1, TreeFolder("Favorites", [self._bundle_to_node(bundle) for bundle in fav_bundles])) + folders_list.insert(0, TreeFolder("Favorites", [self._bundle_to_node(bundle) for bundle in fav_bundles])) return folders_list - def _handle_current_model_clicked(self): + def _open_source_dialog(self, source): + self._selection_source = source favs = ui_state.params.get("ModelManager_Favs") favorites = set(favs.split(';')) if favs else set() - folders_list = self._get_folders(favorites) - - active_ref = self.model_manager.activeBundle.ref if self.model_manager.activeBundle else "Default" - self.model_dialog = TreeOptionDialog(tr("Select a Model"), folders_list, active_ref, "ModelManager_Favs", - get_folders_fn=self._get_folders, on_exit=self._on_model_selected) + folders_list = self._source_folders(favorites, source) + if not folders_list: + gui_app.push_widget(alert_dialog(tr("No models are available for this hardware yet. Connect to the internet and refresh the model list."))) + return + self.model_dialog = TreeOptionDialog(tr("Select a Model"), folders_list, self._slot_active_ref(source), "ModelManager_Favs", + get_folders_fn=lambda favs: self._source_folders(favs, source), on_exit=self._on_model_selected) gui_app.push_widget(self.model_dialog) + def _source_folders(self, favorites, source): + bundles = bundles_for_source(source) + if not bundles: + return [] + folders_list = [TreeFolder("", [TreeNode("Default", {'display_name': default_model_name(source)})])] + folders_list.extend(self._get_folders(favorites, bundles)) + return folders_list + + @staticmethod + def _slot_active_ref(source: str) -> str: + bundle = get_selected_bundle(ui_state.params, source) + return bundle.ref if bundle else "Default" + def _update_state(self): advanced_controls: bool = ui_state.params.get_bool("ShowAdvancedControls") turn_desire: bool = ui_state.params.get_bool("LaneTurnDesire") live_delay: bool = ui_state.params.get_bool("LagdToggle") - camera_offset: bool = ui_state.params.get("ModelManager_ActiveBundle") is not None + camera_offset: bool = ui_state.active_bundle is not None self.lane_turn_desire_toggle.action_item.set_state(turn_desire) self.lane_turn_value_control.set_visible(turn_desire and advanced_controls) @@ -241,19 +327,27 @@ class ModelsLayout(Widget): self._update_lagd_description(live_delay) self.model_manager = ui_state.sm["modelManagerSP"] self._handle_bundle_download_progress() - default_label = f"{get_default_model()} (Default)" - active_name = self.model_manager.activeBundle.displayName if self.model_manager and self.model_manager.activeBundle.ref else default_label - self.current_model_item.action_item.set_value(active_name) - if not ui_state.is_offroad(): - self.current_model_item.action_item.set_enabled(False) - self.current_model_item.set_description(tr("Only available when vehicle is off, or always offroad mode is on")) - else: - self.current_model_item.action_item.set_enabled(True) - self.current_model_item.set_description("") + carry_source, _, carry_display = carrying_model() + for item, item_source in ((self.small_model_item, "qcom"), (self.big_model_item, "usbgpu")): + bundle = get_selected_bundle(ui_state.params, item_source) + name = bundle.displayName if bundle else default_model_name(item_source) + color = ON_COLOR if (item_source == carry_source and name == carry_display) else style.ITEM_TEXT_VALUE_COLOR + item.action_item.set_value(name, color) + + note = self._status_note() + if note != self._last_note: + self._last_note = note + self._set_item_note(self.download_item, note) + + offroad = ui_state.is_offroad() + self.small_model_item.action_item.set_enabled(offroad) + self.big_model_item.action_item.set_enabled(offroad) + self.small_model_item.set_description("" if offroad else tr("Only available when vehicle is off, or always offroad mode is on")) def _render(self, rect): self._scroller.render(rect) def show_event(self): self._scroller.show_event() + self._last_note = None # re-expand the failover note every time the page opens diff --git a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/osm.py b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/osm.py index 7b30e880f..8e1c4afe7 100644 --- a/openpilot/selfdrive/ui/sunnypilot/layouts/settings/osm.py +++ b/openpilot/selfdrive/ui/sunnypilot/layouts/settings/osm.py @@ -8,7 +8,6 @@ import datetime import os import platform import requests -import shutil import threading from pathlib import Path from time import monotonic @@ -75,22 +74,12 @@ class OSMLayout(Widget): def _update_map_size(self): threading.Thread(target=self.calculate_size, daemon=True).start() - def _do_delete_maps(self): - if MAP_PATH.exists(): - shutil.rmtree(MAP_PATH) - - for param in ("OsmDownloadedDate", "OsmLocal", "OsmLocationName", "OsmLocationTitle", "OsmStateName", "OsmStateTitle"): - ui_state.params.remove(param) - + def _on_confirm_delete_maps(self): + ui_state.params.put_bool("Mapd_ClearCache", True) self._delete_maps_btn.action_item.set_enabled(True) self._delete_maps_btn.action_item.set_text(tr("DELETE")) self._update_map_size() - def _on_confirm_delete_maps(self): - self._delete_maps_btn.action_item.set_enabled(False) - self._delete_maps_btn.action_item.set_text("DELETING...") - threading.Thread(target=self._do_delete_maps).start() - def _delete_maps(self): self._show_confirm(tr("This will delete ALL downloaded maps\n\nAre you sure you want to delete all maps?"), tr("Yes, delete all maps"), self._on_confirm_delete_maps) diff --git a/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py b/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py index 79bb15dbb..7c74c4846 100644 --- a/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py +++ b/openpilot/selfdrive/ui/sunnypilot/layouts/sidebar.py @@ -4,11 +4,14 @@ Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ +import math + import pyray as rl import time from dataclasses import dataclass from openpilot.selfdrive.ui.ui_state import ui_state from openpilot.sunnypilot.sunnylink.api import UNREGISTERED_SUNNYLINK_DONGLE_ID +from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.lib.multilang import tr_noop @@ -18,6 +21,9 @@ METRIC_MARGIN = 30 METRIC_START_Y = 300 HOME_BTN = rl.Rectangle(60, 860, 180, 180) +EGPU_ICON_WIDTH = 180 +EGPU_ICON_HEIGHT = 133 + # Color scheme class Colors: @@ -53,6 +59,10 @@ class MetricData: class SidebarSP: def __init__(self): self._sunnylink_status = MetricData(tr_noop("SUNNYLINK"), tr_noop("OFFLINE"), Colors.WARNING) + self._egpu_green_img = gui_app.texture("icons_mici/egpu_green.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) + self._egpu_default_img = gui_app.texture("icons_mici/egpu.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) + self._egpu_orange_img = gui_app.texture("icons_mici/egpu_orange.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) + self._egpu_gray_img = gui_app.texture("icons_mici/egpu_gray.png", EGPU_ICON_WIDTH, EGPU_ICON_HEIGHT) def _update_sunnylink_status(self): if not ui_state.params.get_bool("SunnylinkEnabled"): @@ -78,6 +88,29 @@ class SidebarSP: self._sunnylink_status.update(tr_noop("SUNNYLINK"), status, color) + def _get_home_icon(self, default_img: rl.Texture) -> tuple[rl.Texture, rl.Vector2, float]: + default_pos = rl.Vector2(HOME_BTN.x, HOME_BTN.y) + if not ui_state.sm["deviceState"].chestnutPresent: + return default_img, default_pos, 1.0 + + big_model_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad + big_model_failed = ui_state.started and ui_state.big_model_failed + loading = ui_state.usbgpu_loading or (big_model_selected and ui_state.started and ui_state.usbgpu_active is None) + + if loading: + icon = self._egpu_default_img + opacity = 0.35 + 0.65 * (0.5 - 0.5 * math.cos(rl.get_time() * 6.0)) + elif big_model_selected and big_model_failed: + icon, opacity = self._egpu_orange_img, 1.0 + elif big_model_selected: + icon, opacity = self._egpu_green_img, 1.0 + else: + icon, opacity = self._egpu_gray_img, 1.0 + + x = HOME_BTN.x + (HOME_BTN.width - icon.width) / 2 + y = HOME_BTN.y + (HOME_BTN.height - icon.height) / 2 + return icon, rl.Vector2(x, y), opacity + def _draw_metrics_w_sunnylink(self, rect: rl.Rectangle, _temp, _panda, _connect): metrics = [_temp, _panda, _connect, self._sunnylink_status] start_y = int(rect.y) + METRIC_START_Y diff --git a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py index e2f1b4fb6..b26137394 100644 --- a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py +++ b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/home.py @@ -37,7 +37,7 @@ class MiciHomeLayoutSP(MiciHomeLayout): return big_model_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad - big_model_failed = ui_state.started and (ui_state.usbgpu_active is False) + big_model_failed = ui_state.started and ui_state.big_model_failed loading = ui_state.usbgpu_loading or (big_model_selected and ui_state.started and ui_state.usbgpu_active is None) if loading: diff --git a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py index 6eff45655..183b47fa5 100644 --- a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py +++ b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/models.py @@ -7,16 +7,37 @@ See the LICENSE.md file in the root directory for more details. import pyray as rl from openpilot.cereal import custom -from openpilot.sunnypilot.models.default_model import get_default_model +from openpilot.selfdrive.ui.mici.widgets.dialog import BigDialog +from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_selected_bundle from openpilot.selfdrive.ui.mici.widgets.button import BigButton -from openpilot.selfdrive.ui.sunnypilot.layouts.settings.models import ModelsLayout from openpilot.selfdrive.ui.ui_state import ui_state, device +from openpilot.selfdrive.ui.sunnypilot.model_info import (active_source, big_model_state, bundles_for_source, carrying_model, + default_model_name, model_info, queued_name) from openpilot.system.ui.lib.application import FontWeight, gui_app from openpilot.system.ui.lib.multilang import tr from openpilot.system.ui.widgets import Widget from openpilot.system.ui.widgets.label import UnifiedLabel from openpilot.system.ui.widgets.scroller import NavScroller +def _model_info() -> tuple[str, str, str]: + """(active model, info header, info text) for the panel. Runner-matched: the + active line names what actually drives, and a notable big-model state takes + the info pair.""" + source, active_name, other_name = model_info() + state = big_model_state() + _, _, carry_display = carrying_model() + if carry_display is None: + big = get_selected_bundle(ui_state.params, "usbgpu") + carry_display = big.displayName if big else default_model_name("usbgpu") + active_text = (carry_display or active_name).lower() + if state == 'failed': + return active_text, tr("big model"), tr("unavailable") + if state == 'loading': + return active_text, tr("big model"), tr("getting ready") + header = tr("small model") if source == "usbgpu" else tr("big model") + return active_text, header, other_name.lower() + + class CurrentModelInfo(Widget): def __init__(self): super().__init__() @@ -26,12 +47,12 @@ class CurrentModelInfo(Widget): header_color = rl.Color(255, 255, 255, int(255 * 0.9)) subheader_color = rl.Color(255, 255, 255, int(255 * 0.9 * 0.65)) max_width = int(self._rect.width - 20) + active_text, info_header, info_text = _model_info() self.current_model_header = UnifiedLabel(tr("active model"), 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY) - default_text = f"{get_default_model()} (Default)".lower() - self.current_model_text = UnifiedLabel(default_text, 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN, scroll=True) + self.current_model_text = UnifiedLabel(active_text, 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN, scroll=True) - self.info_header = UnifiedLabel("cache size", 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY) - self.info_text = UnifiedLabel("0 mb", 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN) + self.info_header = UnifiedLabel(info_header, 48, max_width=max_width, text_color=header_color, font_weight=FontWeight.DISPLAY) + self.info_text = UnifiedLabel(info_text, 32, max_width=max_width, text_color=subheader_color, font_weight=FontWeight.ROMAN, scroll=True) def _render(self, _): self.current_model_header.set_position(self._rect.x + 20, self._rect.y - 10) @@ -55,12 +76,13 @@ class ModelsLayoutMici(NavScroller): self._download_progress = "." self._download_frame = 0 self._was_downloading = False + self._selection_source: str | None = None self.select_model_btn = BigButton(tr("select model")) self.select_model_btn.set_click_callback(self._show_folders) self.cancel_download_btn = BigButton(tr("cancel download")) - self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadIndex")) + self.cancel_download_btn.set_click_callback(lambda: ui_state.params.remove("ModelManager_DownloadRef")) self.main_items = [self.current_model_info, self.select_model_btn, self.cancel_download_btn] self._scroller.add_widgets(self.main_items) @@ -69,8 +91,7 @@ class ModelsLayoutMici(NavScroller): def model_manager(self): return ui_state.sm["modelManagerSP"] - def _get_grouped_bundles(self, favorites = None): - bundles = self.model_manager.availableBundles + def _get_grouped_bundles(self, bundles, favorites = None): folders = {} for bundle in bundles: folder = next((override.value for override in bundle.overrides if override.key == "folder"), "") @@ -90,47 +111,70 @@ class ModelsLayoutMici(NavScroller): def _show_folders(self): self.focused_widget = self.select_model_btn + hardware_btns = [] + active = active_source() + for source, label in (("qcom", tr("small models")), ("usbgpu", tr("big models"))): + bundle = get_selected_bundle(ui_state.params, source) + value = (bundle.internalName if bundle else default_model_name(source)).lower() + if source == active: + value += f" ({tr('active')})" + btn = BigButton(label.lower(), value=value) + btn.set_click_callback(lambda s=source: self._select_hardware(s)) + hardware_btns.append(btn) + self._push_selection_view(hardware_btns) + + def _select_hardware(self, source): + self._selection_source = source + favs = ui_state.params.get("ModelManager_Favs") favorites = set(favs.split(';')) if favs else set() - folders = self._get_grouped_bundles(favorites) + bundles = bundles_for_source(source) + if not bundles: + gui_app.push_widget(BigDialog(title=tr("No models available"), + description=tr("No models are available for this hardware yet. Connect to the internet and refresh the model list."))) + return + folders = self._get_grouped_bundles(bundles, favorites) + folder_buttons = [] - default_btn = BigButton(f"{get_default_model()} (Default)".lower()) - default_btn.set_click_callback(self._select_default) + default_btn = BigButton(default_model_name(source).lower()) + default_btn.set_click_callback(lambda s=source: self._select_default(s)) folder_buttons.append(default_btn) for folder in sorted(folders.keys(), key=lambda f: max((bundle.index for bundle in folders[f]), default=-1), reverse=True): - if folder.lower() in ["release models", "master models", "favorites"]: - btn = BigButton(folder.lower()) - btn.set_click_callback(lambda f=folder: self._select_folder(f)) - if folder.lower() == "favorites": - folder_buttons.insert(0, btn) - else: - folder_buttons.append(btn) + btn = BigButton(folder.lower()) + btn.set_click_callback(lambda f=folder: self._select_folder(f)) + if folder.lower() == "favorites": + folder_buttons.insert(0, btn) + else: + folder_buttons.append(btn) self._push_selection_view(folder_buttons) def _pop_to_main(self): gui_app.pop_widgets_to(self) + self._scroller.scroll_panel.set_offset(0.0) def _select_model(self, bundle): - ui_state.params.put("ModelManager_DownloadIndex", bundle.index) + ui_state.params.put("ModelManager_DownloadRef", bundle.ref) self._pop_to_main() - def _select_default(self): - ui_state.params.remove("ModelManager_ActiveBundle") + def _select_default(self, source): + ui_state.params.remove(ACTIVE_BUNDLE_KEYS[source]) self._pop_to_main() def _select_folder(self, folder_name): + source = self._selection_source + if source is None: # folders are only reachable after picking a hardware + return favs = ui_state.params.get("ModelManager_Favs") favorites = set(favs.split(';')) if favs else set() - folders = self._get_grouped_bundles(favorites) + folders = self._get_grouped_bundles(bundles_for_source(source), favorites) bundles = sorted(folders.get(folder_name, []), key=lambda b: b.index, reverse=True) btns = [] for bundle in bundles: - txt = bundle.displayName.lower() - btn = BigButton(txt) + btn = BigButton(bundle.displayName.lower()) btn.set_click_callback(lambda b=bundle: self._select_model(b)) btns.append(btn) self._push_selection_view(btns) @@ -162,11 +206,10 @@ class ModelsLayoutMici(NavScroller): self._was_downloading = is_downloading self.current_model_info.current_model_header.set_text(tr("active model")) - default_model_text = f"{get_default_model()} (Default)".lower() - model_text = manager.activeBundle.displayName.lower() if manager.activeBundle.ref else default_model_text - self.current_model_info.current_model_text.set_text(model_text) - self.current_model_info.info_header.set_text(tr("cache size")) - self.current_model_info.info_text.set_text(f"{ModelsLayout.calculate_cache_size():.2f} MB") + active_text, info_header, info_text = _model_info() + self.current_model_info.current_model_text.set_text(active_text) + self.current_model_info.info_header.set_text(info_header) + self.current_model_info.info_text.set_text(info_text) if manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.failed: self.current_model_info.info_header.set_text(tr("error") + self._download_progress) @@ -177,18 +220,29 @@ class ModelsLayoutMici(NavScroller): device.set_override_interactive_timeout(5) progress = 0.0 count = 0 + verifying = False for model in manager.selectedBundle.models: count += 1 p = model.artifact.downloadProgress - if p.status == custom.ModelManagerSP.DownloadStatus.downloading: + if p.status in (custom.ModelManagerSP.DownloadStatus.downloading, + custom.ModelManagerSP.DownloadStatus.verifying): progress += p.progress + verifying = verifying or p.status == custom.ModelManagerSP.DownloadStatus.verifying elif p.status in (custom.ModelManagerSP.DownloadStatus.downloaded, custom.ModelManagerSP.DownloadStatus.cached): progress += 100.0 - self.current_model_info.current_model_header.set_text(tr("downloading")) + self.current_model_info.current_model_header.set_text(tr("verifying") if verifying else tr("downloading")) + self.cancel_download_btn.set_text(tr("cancel verification") if verifying else tr("cancel download")) self.current_model_info.current_model_header._shimmer = True - self.current_model_info.current_model_text.set_text(f"{manager.selectedBundle.internalName.lower()}") + name_text = manager.selectedBundle.internalName.lower() + if queued := queued_name(manager.selectedBundle.ref): + name_text += f" | {queued.lower()} {tr('queued')}" + self.current_model_info.current_model_text.set_text(name_text) self.current_model_info.info_header.set_text(tr("progress") + self._download_progress) self.current_model_info.info_header._shimmer = True self.current_model_info.info_text.set_text(f"{progress/count:.2f}%") + + elif manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.downloaded: + self.current_model_info.info_header.set_text(tr("downloaded")) + self.current_model_info.info_text.set_text(tr("downloaded")) diff --git a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/settings.py b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/settings.py index 0581f330a..d57b76717 100644 --- a/openpilot/selfdrive/ui/sunnypilot/mici/layouts/settings.py +++ b/openpilot/selfdrive/ui/sunnypilot/mici/layouts/settings.py @@ -12,13 +12,23 @@ from openpilot.selfdrive.ui.mici.widgets.dialog import BigConfirmationDialog, Bi from openpilot.selfdrive.ui.sunnypilot.mici.layouts.sunnylink import SunnylinkLayoutMici from openpilot.selfdrive.ui.sunnypilot.mici.layouts.models import ModelsLayoutMici from openpilot.selfdrive.ui.ui_state import ui_state -from openpilot.system.ui.lib.application import gui_app +from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.lib.multilang import tr ICON_SIZE = 70 BIG_ICON_SIZE = 110 +class SunnylinkBigButton(SettingsBigButton): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._label.set_font_weight(FontWeight.AUDIOWIDE) + + def _get_label_font_size(self): + # Audiowide runs wider than Inter: "sunnylink" wraps to two lines at 64 + return 56 + + class SettingsLayoutSP(OP.SettingsLayout): def __init__(self): OP.SettingsLayout.__init__(self) @@ -33,7 +43,7 @@ class SettingsLayoutSP(OP.SettingsLayout): self.icon_offroad_slider = gui_app.texture("icons_mici/settings/device/lkas.png", BIG_ICON_SIZE, BIG_ICON_SIZE) sunnylink_panel = SunnylinkLayoutMici() - sunnylink_btn = SettingsBigButton(tr("sunnylink"), "", gui_app.texture("icons_mici/settings/developer/ssh.png", 55, 55)) + sunnylink_btn = SunnylinkBigButton(tr("sunnylink"), "", gui_app.texture("../../sunnypilot/selfdrive/assets/icons_mici/sunnylink.png", 76, 44)) sunnylink_btn.set_click_callback(lambda: gui_app.push_widget(sunnylink_panel)) models_panel = ModelsLayoutMici() @@ -56,8 +66,8 @@ class SettingsLayoutSP(OP.SettingsLayout): items = self._scroller._items.copy() - items.insert(1, sunnylink_btn) - items.insert(2, models_btn) + items.insert(1, models_btn) + items.insert(5, sunnylink_btn) # front slots (only one ever visible at a time): exit-always-offroad, then enable-onroad items.insert(0, self._enable_offroad_btn_onroad) diff --git a/openpilot/selfdrive/ui/sunnypilot/model_info.py b/openpilot/selfdrive/ui/sunnypilot/model_info.py new file mode 100644 index 000000000..a93a06f18 --- /dev/null +++ b/openpilot/selfdrive/ui/sunnypilot/model_info.py @@ -0,0 +1,88 @@ +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" +from openpilot.selfdrive.ui.ui_state import ui_state +from openpilot.sunnypilot.models.fetcher import get_cached_bundles +from openpilot.sunnypilot.models.helpers import get_active_source, get_selected_bundle, resolve_bundle_by_ref +from openpilot.sunnypilot.models.model_name import DEFAULT_BIG_MODEL, DEFAULT_MODEL + + +def active_source() -> str: + return get_active_source(usbgpu=ui_state.usbgpu, + usbgpu_active=ui_state.usbgpu_active, usbgpu_loading=ui_state.usbgpu_loading, + offroad=ui_state.is_offroad()) + + +def bundles_for_source(source: str): + if source == active_source(): + return ui_state.sm["modelManagerSP"].availableBundles + return get_cached_bundles(ui_state.params, source) + + +def default_model(source: str) -> str: + return DEFAULT_BIG_MODEL if source == 'usbgpu' else DEFAULT_MODEL + + +def default_model_name(source: str) -> str: + return f"{default_model(source)} (Default)" + + +def big_model_state() -> str | None: + """'failed' | 'loading' | None, mirroring the sidebar's detection (#1969).""" + if ui_state.started and ui_state.usbgpu and ui_state.big_model_failed: + return 'failed' + big_selected = ui_state.usbgpu_compiled or ui_state.model_runner_tinygrad + if ui_state.usbgpu_loading or (big_selected and ui_state.started and ui_state.usbgpu_active is None): + return 'loading' + return None + + +def carrying_model() -> tuple[str | None, str | None, str | None]: + """(source, internal name, display name) of what actually drives. Runner-matched: + when a Default big cannot carry, stock modeld runs the Default small, never the + small slot's pick; a custom big has no automatic fallback yet -> (None, None, None).""" + source = active_source() + if source == "usbgpu": + bundle = get_selected_bundle(ui_state.params, "usbgpu") + if bundle: + return "usbgpu", bundle.internalName, bundle.displayName + name = default_model_name("usbgpu") + return "usbgpu", name, name + if ui_state.usbgpu: + if get_selected_bundle(ui_state.params, "usbgpu") is None: + name = default_model_name("qcom") + return "qcom", name, name + return None, None, None + bundle = get_selected_bundle(ui_state.params, "qcom") + if bundle: + return "qcom", bundle.internalName, bundle.displayName + name = default_model_name("qcom") + return "qcom", name, name + + +def queued_name(current_ref) -> str | None: + ref = ui_state.params.get("ModelManager_DownloadRef") + if ref and ref != current_ref: + source_bundles = {source: bundles_for_source(source) for source in ("qcom", "usbgpu")} + if resolved := resolve_bundle_by_ref(ref, source_bundles): + return resolved[0].internalName + return None + + +def model_info() -> tuple[str, str, str]: + """returns (active source, active model name, other model name) + + Names come from the params slots, never modelManagerSP.activeBundle — the + manager republishes a tick after a chestnut change, so the stale bundle + would flash the wrong model.""" + source = active_source() + other = "qcom" if source == "usbgpu" else "usbgpu" + active_bundle = get_selected_bundle(ui_state.params, source) + other_bundle = get_selected_bundle(ui_state.params, other) + + active_name = active_bundle.displayName if active_bundle else default_model_name(source) + other_name = other_bundle.displayName if other_bundle else default_model_name(other) + return source, active_name, other_name diff --git a/openpilot/selfdrive/ui/sunnypilot/ui_state.py b/openpilot/selfdrive/ui/sunnypilot/ui_state.py index e4bdabb16..14eff01e9 100644 --- a/openpilot/selfdrive/ui/sunnypilot/ui_state.py +++ b/openpilot/selfdrive/ui/sunnypilot/ui_state.py @@ -10,6 +10,7 @@ from openpilot.cereal import messaging, log, custom from opendbc.car.structs import car from openpilot.common.params import Params from openpilot.selfdrive.ui.sunnypilot.layouts.settings.display import OnroadBrightness +from openpilot.sunnypilot.models.helpers import ACTIVE_BUNDLE_KEYS, get_active_source from openpilot.sunnypilot.sunnylink.sunnylink_state import SunnylinkState from openpilot.system.ui.lib.application import gui_app from openpilot.system.ui.sunnypilot.widgets.screen_saver import ScreenSaverSP @@ -151,7 +152,9 @@ class UIStateSP: self.has_icbm = self.CP_SP.intelligentCruiseButtonManagementAvailable and self.params.get_bool("IntelligentCruiseButtonManagement") self._enforce_constraints() - self.active_bundle = self.params.get("ModelManager_ActiveBundle") + source = get_active_source(usbgpu=self.usbgpu, usbgpu_active=self.usbgpu_active, + usbgpu_loading=self.usbgpu_loading, offroad=self.is_offroad()) + self.active_bundle = self.params.get(ACTIVE_BUNDLE_KEYS[source]) self.model_runner_tinygrad = self.active_bundle is not None and self.active_bundle.get("runner") == "tinygrad" self.blindspot = self.params.get_bool("BlindSpot") self.chevron_metrics = self.params.get("ChevronInfo") diff --git a/openpilot/selfdrive/ui/ui_state.py b/openpilot/selfdrive/ui/ui_state.py index 500fc5712..e58b66cb3 100644 --- a/openpilot/selfdrive/ui/ui_state.py +++ b/openpilot/selfdrive/ui/ui_state.py @@ -127,6 +127,15 @@ class UIState(UIStateSP): def add_on_body_changed_callbacks(self, callback: Callable[[], None]): self._on_body_changed_callbacks.append(callback) + @property + def big_model_failed(self) -> bool: + # Mirrors the onroad HUD's four-condition check so sidebar and home icons reflect the same failure states + return (self.usbgpu_active is False or + not self.sm['deviceState'].chestnutPresent or + (self.usbgpu_active is True and self.sm.recv_frame['modelV2'] > self.started_frame and + not self.sm.alive['modelV2']) or + (self.usbgpu_active is None and self.sm.recv_frame['modelV2'] > self.started_frame)) + @property def engaged(self) -> bool: return self.started and (self.sm["selfdriveState"].enabled or self.sm["selfdriveStateSP"].mads.enabled) diff --git a/openpilot/sunnypilot/mapd/mapd_manager.py b/openpilot/sunnypilot/mapd/mapd_manager.py index 2251289bb..899b0c2bd 100755 --- a/openpilot/sunnypilot/mapd/mapd_manager.py +++ b/openpilot/sunnypilot/mapd/mapd_manager.py @@ -55,6 +55,19 @@ def cleanup_old_osm_data(files_to_remove: list[str]) -> None: shutil.rmtree(file, ignore_errors=False) +def clear_downloaded_maps() -> None: + """Deletes downloaded OSM map data and resets params.""" + path = f"{Paths.mapd_root()}/offline" + if os.path.exists(path): + shutil.rmtree(path, ignore_errors=True) + + for param in ("OsmDownloadedDate", "OsmLocal", "OsmLocationName", "OsmLocationTitle", + "OsmStateName", "OsmStateTitle"): + params.remove(param) + + cloudlog.info("mapd: downloaded maps cleared") + + def request_refresh_osm_location_data(nations: list[str], states: list[str] | None = None) -> None: params.put("OsmDownloadedDate", str(datetime.now().timestamp()), block=True) params.put_bool("OsmDbUpdatesCheck", False, block=True) @@ -131,6 +144,10 @@ def main_thread(): show_alert = bool(get_files_for_cleanup() and params.get_bool("OsmLocal")) set_offroad_alert("Offroad_OSMUpdateRequired", show_alert, "This alert will be cleared when new maps are downloaded.") + if params.get("Mapd_ClearCache"): + clear_downloaded_maps() + params.remove("Mapd_ClearCache") + update_osm_db() live_map_sp.tick() rk.keep_time() diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 9f3d70953..d18001227 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -91,7 +91,7 @@ class ModelState(ModelStateBase): if env_pkl and os.path.exists(env_pkl): model_bundle = None else: - model_bundle = get_active_bundle() + model_bundle = get_active_bundle(usbgpu=usbgpu) self.generation = model_bundle.generation if model_bundle is not None else None overrides = {override.key: override.value for override in model_bundle.overrides} if model_bundle else {} diff --git a/openpilot/sunnypilot/modeld_v2/tests/helpers.py b/openpilot/sunnypilot/modeld_v2/tests/helpers.py index ee59e8278..6e66bf771 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/helpers.py +++ b/openpilot/sunnypilot/modeld_v2/tests/helpers.py @@ -190,8 +190,8 @@ def tmp_path(): def patch_modeld(monkeypatch): def _patch(bundle): - monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None: bundle) - monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None: bundle) + monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) + monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) return _patch diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py index 3396649a1..ccd8cbc7f 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py @@ -59,8 +59,8 @@ class TestFindDrivingPkl(OpenpilotTestCase): class TestModelStateCombinedInit(OpenpilotTestCase): def test_asserts_when_no_pkl(self, monkeypatch): bundle = DummyBundle(models=[], is_20hz=True) - monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None: bundle) - monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None: bundle) + monkeypatch.setattr(helpers, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) + monkeypatch.setattr(modeld_module, 'get_active_bundle', lambda params=None, *, usbgpu=None: bundle) with self.assertRaisesRegex(AssertionError, "No driving pkl found"): ModelState(cam_w=CAM_W, cam_h=CAM_H) diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index c9e86edd0..1bbfb02f7 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -141,41 +141,50 @@ class ModelFetcher: MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v21.json" MODEL_URL_USBGPU = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_usbgpu_v22.json" + MODEL_SOURCES = { + "qcom": (MODEL_URL, ""), + "usbgpu": (MODEL_URL_USBGPU, "_USBGPU"), + } + def __init__(self, params: Params): self.params = params self.model_parser = ModelParser() - self._is_usbgpu: bool | None = None - self.model_cache = ModelCache(params) - self.model_url = self.MODEL_URL + self.model_caches = { + source: ModelCache(params, suffix=suffix) + for source, (_, suffix) in self.MODEL_SOURCES.items() + } + self._refetched: set[str] = set() + self.params.put("ModelManager_ActiveJson", { + "qcom": self.MODEL_URL, + "usbgpu": self.MODEL_URL_USBGPU, + }, block=True) - def _update_model_source(self, chestnut_present: bool) -> None: - """Updates what json to use based on chestnut hardware presence via deviceState""" - is_usbgpu = chestnut_present - if is_usbgpu != self._is_usbgpu: - self._is_usbgpu = is_usbgpu - self.model_cache = ModelCache(self.params, suffix="_USBGPU" if is_usbgpu else "") - self.model_url = self.MODEL_URL_USBGPU if is_usbgpu else self.MODEL_URL - self.params.put("ModelManager_ActiveJson", self.model_url, block=True) + @staticmethod + def active_source(chestnut_present: bool) -> str: + return "usbgpu" if chestnut_present else "qcom" - def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle] | None: + def _fetch_and_cache_models(self, source: str) -> list[custom.ModelManagerSP.ModelBundle] | None: """Fetches fresh model data from remote and updates cache. Returns None on transport errors. Raises on 404 and other fatal HTTP errors. """ + model_url, _ = self.MODEL_SOURCES[source] try: - response = requests.get(self.model_url, timeout=10) + response = requests.get(model_url, timeout=10) # Explicitly handle 404 differently if response.status_code == 404: - cloudlog.error(f"Models URL returned 404 Not Found: {self.model_url}") - raise HTTPError(f"404 Not Found: {self.model_url}", response=response) + cloudlog.error(f"Models URL returned 404 Not Found: {model_url}") + raise HTTPError(f"404 Not Found: {model_url}", response=response) # Raise for any other 4xx/5xx response.raise_for_status() json_data = response.json() - self.model_cache.set(json_data) - cloudlog.debug("Successfully updated models cache") - return self.model_parser.parse_models(json_data) + parsed = self.model_parser.parse_models(json_data) + if parsed: + self.model_caches[source].set(json_data) + cloudlog.debug(f"Successfully updated models cache for {source}") + return parsed except ConnectionError as e: cloudlog.warning(f"DNS/connection error while fetching models: {e}") @@ -188,16 +197,40 @@ class ModelFetcher: return None - def get_available_bundles(self, chestnut_present: bool = False) -> list[custom.ModelManagerSP.ModelBundle]: - """Gets the list of available models, with smart cache handling""" - self._update_model_source(chestnut_present) - cached_data, is_expired = self.model_cache.get() + @staticmethod + def _cache_matches_source(source: str, cached_data: dict) -> bool: + bundles = cached_data.get("bundles", []) + if source == "usbgpu": + return any(bundle.get("is_big") is True for bundle in bundles) + return not any(bundle.get("is_big") is True for bundle in bundles) + + def get_bundles_for_source(self, source: str) -> list[custom.ModelManagerSP.ModelBundle]: + if source not in self.MODEL_SOURCES: + cloudlog.warning(f"Unknown model source: {source}") + return [] + + cached_data, is_expired = self.model_caches[source].get() if cached_data and not is_expired: - cloudlog.debug("Using valid cached models data") - return self.model_parser.parse_models(cached_data) + # a source is refetched over a mismatch at most once per process: if the fresh + # manifest still mismatches, the URL is authoritative and the cache is trusted + if self._cache_matches_source(source, cached_data) or source in self._refetched: + try: + parsed = self.model_parser.parse_models(cached_data) + except Exception: + cloudlog.warning(f"Failed to parse cached models for {source}; refetching", exc_info=True) + else: + if parsed: + cloudlog.debug(f"Using valid cached models data for source {source}") + return parsed + # a source-matching cache that yields no valid bundles is stale (e.g. an old + # manifest version) - do not trust it, refetch so the source is repopulated + cloudlog.warning(f"Cached models for {source} have no valid bundles; refetching") + else: + self._refetched.add(source) + cloudlog.warning(f"Cached models for {source} not valid; refetching once") - fetched_bundles = self._fetch_and_cache_models() + fetched_bundles = self._fetch_and_cache_models(source) if fetched_bundles is not None: return fetched_bundles @@ -205,14 +238,33 @@ class ModelFetcher: cloudlog.warning("Failed to fetch fresh data and no cache available") cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback") - return self.model_parser.parse_models(cached_data) + try: + return self.model_parser.parse_models(cached_data) + except Exception: + return [] + + +def get_cached_bundles(params: Params, source: str) -> list[custom.ModelManagerSP.ModelBundle]: + + if source not in ModelFetcher.MODEL_SOURCES: + cloudlog.warning(f"Unknown model source: {source}") + return [] + _, suffix = ModelFetcher.MODEL_SOURCES[source] + cached_data = params.get(f"ModelManager_ModelsCache{suffix}") + if not cached_data: + return [] + try: + return ModelParser.parse_models(cached_data) + except Exception as e: + cloudlog.warning(f"Failed to parse cached models for source {source}: {e}") + return [] if __name__ == "__main__": from openpilot.selfdrive.modeld.helpers import usbgpu_present params = Params() model_fetcher = ModelFetcher(params) - bundles = model_fetcher.get_available_bundles(chestnut_present=usbgpu_present()) + bundles = model_fetcher.get_bundles_for_source(ModelFetcher.active_source(usbgpu_present())) for bundle in bundles: for model in bundle.models: model_overrides = {override.key: override.value for override in bundle.overrides} diff --git a/openpilot/sunnypilot/models/helpers.py b/openpilot/sunnypilot/models/helpers.py index e33cc445d..707b86f72 100644 --- a/openpilot/sunnypilot/models/helpers.py +++ b/openpilot/sunnypilot/models/helpers.py @@ -16,6 +16,7 @@ from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRaider from openpilot.common.hardware.hw import Paths +from openpilot.selfdrive.modeld.helpers import usbgpu_present # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO REQUIRED_JSON_VERSION = 18 @@ -24,6 +25,12 @@ CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' ModelManager = custom.ModelManagerSP +ACTIVE_BUNDLE_KEYS = { + "qcom": "ModelManager_ActiveBundle", + "usbgpu": "ModelManager_ActiveBundleUSBGPU", +} +_LAST_VALIDATED_RAW: dict[str, dict | None] = {} + def _compute_hash(file_path: str) -> str | None: from openpilot.common.file_chunker import open_file_chunked @@ -97,55 +104,81 @@ def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, availa return True if active_bundle.minimumSelectorVersion != matching_bundle.minimumSelectorVersion: return True - if active_bundle.runner.raw != matching_bundle.runner.raw: + if active_bundle.runner != matching_bundle.runner: return True if set(_bundle_artifacts(active_bundle)) != set(_bundle_artifacts(matching_bundle)): return True - # missing files trigger re-download, not selection reset - return False + return not _bundle_is_valid_locally(active_bundle) -def _prev_bundle_key(is_usbgpu: bool) -> str: - return "ModelManager_PrevBundle_USBGPU" if is_usbgpu else "ModelManager_PrevBundle" - - -def validate_active_bundle(params: Params, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None, - is_usbgpu: bool = False) -> None: - raw_bundle = params.get("ModelManager_ActiveBundle") - if not raw_bundle: - prev = params.get(_prev_bundle_key(is_usbgpu)) - if prev and (prev_bundle := get_active_bundle(params, raw_bundle_dict=prev)) is not None: - if not _bundle_needs_reset(prev_bundle, available_bundles): - params.put("ModelManager_ActiveBundle", prev, block=True) - return - - active_bundle = get_active_bundle(params, raw_bundle_dict=raw_bundle) - if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles): - cloudlog.warning("Active model bundle invalid; resetting to default") - params.put(_prev_bundle_key(not is_usbgpu), raw_bundle, block=True) - - prev = params.get(_prev_bundle_key(is_usbgpu)) - if prev and (prev_bundle := get_active_bundle(params, raw_bundle_dict=prev)) is not None: - if not _bundle_needs_reset(prev_bundle, available_bundles): - params.put("ModelManager_ActiveBundle", prev, block=True) - return - - params.remove("ModelManager_ActiveBundle") - params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True) - - -def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> "custom.ModelManagerSP.ModelBundle | None": - params = params or Params() +def _parse_active_bundle(raw_bundle) -> "custom.ModelManagerSP.ModelBundle | None": try: - active_bundle_dict = raw_bundle_dict if raw_bundle_dict is not None else (params.get("ModelManager_ActiveBundle") or {}) - if isinstance(active_bundle_dict, dict) and active_bundle_dict and is_bundle_version_compatible(active_bundle_dict): - return custom.ModelManagerSP.ModelBundle(**active_bundle_dict) + if isinstance(raw_bundle, dict) and raw_bundle and is_bundle_version_compatible(raw_bundle): + return custom.ModelManagerSP.ModelBundle(**raw_bundle) except Exception: pass return None +def get_selected_bundle(params: Params | None = None, source: str = "qcom") -> "custom.ModelManagerSP.ModelBundle | None": + params = params or Params() + return _parse_active_bundle(params.get(ACTIVE_BUNDLE_KEYS[source])) + + +def get_active_source(usbgpu: bool | None = None, usbgpu_active: bool | None = None, + usbgpu_loading: bool | None = None, offroad: bool | None = None) -> str: + if usbgpu is None: + usbgpu = usbgpu_present() + state_valid = usbgpu_active is not None or usbgpu_loading is not None or offroad is not None + big_active = usbgpu and (not state_valid or usbgpu_active or usbgpu_loading or offroad) + return "usbgpu" if big_active else "qcom" + + +def get_active_bundle(params: Params | None = None, *, usbgpu: bool | None = None) -> "custom.ModelManagerSP.ModelBundle | None": + # no cross-slot fallback: an empty active slot means the hardware default, which + # only stock modeld can run - modeld_v2 requires a real bundle + params = params or Params() + return get_selected_bundle(params, get_active_source(usbgpu=usbgpu)) + + +def resolve_bundle_by_ref( + ref: str, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]], +) -> "tuple[custom.ModelManagerSP.ModelBundle, str] | None": + for source, bundles in source_bundles.items(): + for bundle in bundles: + if bundle.ref == ref: + return bundle, source + return None + + +def _validate_active_bundle(params: Params, source: str, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None: + global _LAST_VALIDATED_RAW + + key = ACTIVE_BUNDLE_KEYS[source] + raw_bundle = params.get(key) + if not raw_bundle: + return + + if _LAST_VALIDATED_RAW.get(key) == raw_bundle: + return + + active_bundle = _parse_active_bundle(raw_bundle) + if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles): + cloudlog.warning(f"Active model bundle invalid for {source}; resetting to default") + params.remove(key) + _LAST_VALIDATED_RAW[key] = None + else: + _LAST_VALIDATED_RAW[key] = raw_bundle + + +def validate_active_bundles(params: Params, source_bundles: dict[str, list[custom.ModelManagerSP.ModelBundle]]) -> None: + # an empty list means the fetch failed, not that the catalog dropped the bundle + for source, bundles in source_bundles.items(): + _validate_active_bundle(params, source, bundles or None) + get_active_model_runner(params, force_check=True) + + def get_active_model_runner(params: Params | None = None, force_check: bool = False) -> int: params = params or Params() cached_runner_type = params.get("ModelRunnerTypeCache") diff --git a/openpilot/sunnypilot/models/manager.py b/openpilot/sunnypilot/models/manager.py index e47cf7536..178d6c04e 100644 --- a/openpilot/sunnypilot/models/manager.py +++ b/openpilot/sunnypilot/models/manager.py @@ -17,12 +17,17 @@ from openpilot.common.hardware.hw import Paths from openpilot.cereal import messaging, custom from openpilot.sunnypilot.models.fetcher import ModelFetcher -from openpilot.sunnypilot.models.helpers import get_active_bundle, validate_active_bundle, verify_file +from openpilot.sunnypilot.models.helpers import (ACTIVE_BUNDLE_KEYS, get_active_bundle, get_selected_bundle, + resolve_bundle_by_ref, validate_active_bundles, verify_file) # (connect, read) seconds. read is per-request inactivity, not a total cap DOWNLOAD_TIMEOUT = (30, 30) +class DownloadCancelled(Exception): + pass + + class ModelManagerSP: """Manages model downloads and status reporting""" @@ -31,11 +36,24 @@ class ModelManagerSP: self.model_fetcher = ModelFetcher(self.params) self.pm = messaging.PubMaster(["modelManagerSP"]) self.sm = messaging.SubMaster(["deviceState"]) + self.chestnut_present = False self.available_models: list[custom.ModelManagerSP.ModelBundle] = [] + self.source_models: dict[str, list[custom.ModelManagerSP.ModelBundle]] = {} self.selected_bundle: custom.ModelManagerSP.ModelBundle = None - self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params) + self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params, usbgpu=self.chestnut_present) self._chunk_size = 128 * 1000 # 128 KB chunks self._download_start_times: dict[str, float] = {} # Track start time per model + self._download_ref: bytes | str | None = None + + def _download_interrupted(self) -> bool: + # only removal cancels: a different ref is a queued selection that + # _release_download_ref leaves in place for the next tick + return self.params.get("ModelManager_DownloadRef") is None + + def _release_download_ref(self) -> None: + if self.params.get("ModelManager_DownloadRef") == self._download_ref: + self.params.remove("ModelManager_DownloadRef") + self._download_ref = None def _sync_artifact_progress(self, source_artifact) -> None: """Mirror download progress to all artifacts sharing the same filename in the selected bundle.""" @@ -77,8 +95,8 @@ class ModelManagerSP: f.write(chunk) bytes_downloaded += len(chunk) - if self.params.get("ModelManager_DownloadIndex") is None: - raise Exception("Download cancelled") + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") if total_size > 0: progress = (bytes_downloaded / total_size) * 100 @@ -91,7 +109,7 @@ class ModelManagerSP: # Clean up start time after download completes del self._download_start_times[model.fileName] - async def _download_chunked(self, base_url: str, base_path: str, artifact) -> None: + async def _download_chunked(self, base_url: str, base_path: str, artifact, skip: frozenset[int] | set[int] = frozenset()) -> None: from openpilot.common.file_chunker import get_chunk_name, get_manifest_path num_chunks = len(artifact.chunks) @@ -103,8 +121,11 @@ class ModelManagerSP: # Shared connection saves a TCP+TLS handshake per chunk. # Keep sequential: the link saturates on one stream and Session is not thread-safe. + completed = len(skip) with requests.Session() as session: for i, _ in enumerate(artifact.chunks): + if i in skip: + continue chunk_url = get_chunk_name(base_url, i, num_chunks) chunk_path = get_chunk_name(base_path, i, num_chunks) chunk_downloaded = 0 @@ -115,15 +136,16 @@ class ModelManagerSP: for data in response.iter_content(chunk_size=self._chunk_size): f.write(data) chunk_downloaded += len(data) - if self.params.get("ModelManager_DownloadIndex") is None: - raise Exception("Download cancelled") + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") intra = chunk_downloaded / max(chunk_size, 1) - progress = min(99.0, ((i + intra) / num_chunks) * 100) + progress = min(99.0, ((completed + intra) / num_chunks) * 100) artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading artifact.downloadProgress.progress = progress artifact.downloadProgress.eta = self._calculate_eta(artifact.fileName, progress) self._sync_artifact_progress(artifact) self._report_status() + completed += 1 with open(manifest_path, 'w') as f: # noqa: ASYNC230 f.write(str(num_chunks)) @@ -134,6 +156,8 @@ class ModelManagerSP: async def _process_artifact(self, artifact, destination_path: str) -> None: if not artifact.downloadUri.uri: return None + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") url = artifact.downloadUri.uri expected_hash = artifact.downloadUri.sha256 @@ -141,21 +165,23 @@ class ModelManagerSP: full_path = os.path.join(destination_path, filename) try: + # progress counts only valid chunks so a resumed download continues the + # bar from where verification left it, instead of falling back to zero is_cached = False + valid_chunks: set[int] = set() if len(artifact.chunks) > 0: from openpilot.common.file_chunker import get_chunk_name num_chunks = len(artifact.chunks) - chunks_valid = True for i, chunk in enumerate(artifact.chunks): - chunk_path = get_chunk_name(full_path, i, num_chunks) - if not await verify_file(chunk_path, chunk.sha256): - chunks_valid = False - break - artifact.downloadProgress.progress = ((i + 1) / num_chunks) * 100 + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") + if await verify_file(get_chunk_name(full_path, i, num_chunks), chunk.sha256): + valid_chunks.add(i) + artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.verifying + artifact.downloadProgress.progress = (len(valid_chunks) / num_chunks) * 100 self._sync_artifact_progress(artifact) self._report_status() - if chunks_valid and num_chunks > 0: - is_cached = True + is_cached = len(valid_chunks) == num_chunks else: if await verify_file(full_path, expected_hash): is_cached = True @@ -169,7 +195,7 @@ class ModelManagerSP: return if len(artifact.chunks) > 0: - await self._download_chunked(url, full_path, artifact) + await self._download_chunked(url, full_path, artifact, skip=valid_chunks) from openpilot.common.file_chunker import get_chunk_name for i, chunk in enumerate(artifact.chunks): chunk_path = get_chunk_name(full_path, i, len(artifact.chunks)) @@ -186,6 +212,17 @@ class ModelManagerSP: self._sync_artifact_progress(artifact) self._report_status() + except DownloadCancelled: + # a cancel keeps whatever is on disk: complete chunks resume the next attempt + self._download_start_times.pop(artifact.fileName, None) + artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.failed + artifact.downloadProgress.eta = 0 + self._sync_artifact_progress(artifact) + if self.selected_bundle: + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + self._report_status() + raise + except Exception as e: cloudlog.error(f"Error downloading {filename}: {str(e)}") for f in [full_path] + [p for p in (os.path.join(destination_path, f) for f in os.listdir(destination_path)) if filename in p]: @@ -217,8 +254,7 @@ class ModelManagerSP: model_manager_state.availableBundles = self.available_models self.pm.send('modelManagerSP', msg) - async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None: - """Downloads all models in a bundle""" + async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str, source: str) -> None: self.selected_bundle = model_bundle self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading for model in self.selected_bundle.models: @@ -240,10 +276,11 @@ class ModelManagerSP: seen_artifacts.add(artifact.fileName) await self._process_artifact(artifact, destination_path) - self.active_bundle = self.selected_bundle - self.active_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded - self.params.put("ModelManager_ActiveBundle", self.active_bundle.to_dict(), block=True) - self.selected_bundle = None + if self._download_interrupted(): + raise DownloadCancelled("Download cancelled") + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded + self.params.put(ACTIVE_BUNDLE_KEYS[source], model_bundle.to_dict(), block=True) + self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present) except Exception: if self.selected_bundle is not None: @@ -253,38 +290,45 @@ class ModelManagerSP: finally: self._report_status() - def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None: + def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str, source: str) -> None: """Main entry point for downloading a model bundle""" - asyncio.run(self._download_bundle(model_bundle, destination_path)) + asyncio.run(self._download_bundle(model_bundle, destination_path, source)) - BOOT_SETTLE_TICKS = 10 # seconds at 1 Hz before validating active bundle + def _process_download_requests(self) -> None: + # loops so a ref queued during a download starts in the same tick, without + # the bar dropping to idle for a tick between the two transfers + last_ref = None + while (ref_to_download := self.params.get("ModelManager_DownloadRef")) is not None: + if ref_to_download == last_ref: # a repeating ref falls back to the next tick instead of spinning + return + last_ref = ref_to_download + resolved = resolve_bundle_by_ref(ref_to_download, self.source_models) + if not resolved: + return + model_to_download, source = resolved + self._download_ref = ref_to_download + try: + self.download(model_to_download, Paths.model_root(), source) + except Exception as e: + cloudlog.exception(e) + finally: + self._release_download_ref() + self.selected_bundle = None def main_thread(self) -> None: """Main thread for model management""" rk = Ratekeeper(1, print_delay_threshold=None) - boot_ticks = 0 while True: try: self.sm.update(0) - chestnut_present = self.sm['deviceState'].chestnutPresent - self.available_models = self.model_fetcher.get_available_bundles(chestnut_present) - if boot_ticks >= self.BOOT_SETTLE_TICKS: - validate_active_bundle(self.params, self.available_models, is_usbgpu=chestnut_present) - boot_ticks = min(boot_ticks + 1, self.BOOT_SETTLE_TICKS) - self.active_bundle = get_active_bundle(self.params) + self.chestnut_present = self.sm['deviceState'].chestnutPresent + self.source_models = {source: self.model_fetcher.get_bundles_for_source(source) for source in ModelFetcher.MODEL_SOURCES} + self.available_models = self.source_models[ModelFetcher.active_source(self.chestnut_present)] + validate_active_bundles(self.params, self.source_models) + self.active_bundle = get_active_bundle(self.params, usbgpu=self.chestnut_present) - if (index_to_download := self.params.get("ModelManager_DownloadIndex")) is not None: - if self.active_bundle and self.active_bundle.index == index_to_download: - self.params.remove("ModelManager_DownloadIndex") - elif model_to_download := next((model for model in self.available_models if model.index == index_to_download), None): - try: - self.download(model_to_download, Paths.model_root()) - except Exception as e: - cloudlog.exception(e) - finally: - self.params.remove("ModelManager_DownloadIndex") - self.selected_bundle = None + self._process_download_requests() if self.params.get("ModelManager_ClearCache"): self.clear_model_cache() @@ -302,12 +346,14 @@ class ModelManagerSP: Clears the model cache directory of all files except those in the active model bundle. """ - # Get list of files used by active model bundle + # Get list of files used by both slots' selected bundles (either may become + # the truly active bundle depending on hardware availability) active_files = [] - if self.active_bundle is not None: # When the default model is active - for model in self.active_bundle.models: - if hasattr(model, 'artifact') and model.artifact.fileName: - active_files.append(model.artifact.fileName) + for source in ACTIVE_BUNDLE_KEYS: + if selected_bundle := get_selected_bundle(self.params, source): + for model in selected_bundle.models: + if model.artifact.fileName: + active_files.append(model.artifact.fileName) # Remove all files except active ones (including their chunk files) model_dir = Paths.model_root() diff --git a/openpilot/sunnypilot/models/tests/test_manager_download.py b/openpilot/sunnypilot/models/tests/test_manager_download.py index 67fb9023a..4d3b7989f 100644 --- a/openpilot/sunnypilot/models/tests/test_manager_download.py +++ b/openpilot/sunnypilot/models/tests/test_manager_download.py @@ -11,6 +11,7 @@ import http.server import os import tempfile import threading +import time import unittest from typing import Any from unittest import mock @@ -23,6 +24,10 @@ from openpilot.common.test import OpenpilotTestCase from openpilot.common.file_chunker import get_chunk_name, get_manifest_path from openpilot.selfdrive.test.helpers import http_server_context from openpilot.sunnypilot.models import manager as manager_module +from openpilot.sunnypilot.models.fetcher import ModelFetcher, get_cached_bundles +from openpilot.sunnypilot.models import helpers +from openpilot.sunnypilot.models.helpers import (get_active_bundle, get_active_source, get_selected_bundle, + resolve_bundle_by_ref, validate_active_bundles) from openpilot.sunnypilot.models.manager import ModelManagerSP CHUNK_BODIES = [b'A' * 5000, b'B' * 5000, b'C' * 3000] @@ -98,11 +103,13 @@ class ManagerDownloadTestBase(OpenpilotTestCase): self.manager = ModelManagerSP.__new__(ModelManagerSP) self.manager.params = mock.MagicMock() self.manager.params.get.return_value = b'0' # not cancelled + self.manager._download_ref = b'0' self.manager.pm = mock.MagicMock() self.manager.pm.send.side_effect = self._record_progress self.manager.selected_bundle = None self.manager.active_bundle = None self.manager.available_models = [] + self.manager.chestnut_present = False self.manager._chunk_size = 1024 self.manager._download_start_times = {} @@ -249,6 +256,166 @@ class TestManagerDownload(ManagerDownloadTestBase): assert self.manager._download_start_times == {} self.run_with_server(body) + def test_download_ref_present_keeps_download_alive(self): + """A pending download request (DownloadRef set) must not be cancelled mid-transfer.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + self.manager.params.get.side_effect = lambda key: b"ref" if key == "ModelManager_DownloadRef" else None + self.manager._download_ref = b"ref" + asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact)) + assert os.path.isfile(get_manifest_path(base_path)) + self.run_with_server(body) + + def test_cancellation_via_download_ref(self): + """Removing DownloadRef mid-transfer cancels the download.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + checks = {"n": 0} + + def get(key): + if key == "ModelManager_DownloadRef": + checks["n"] += 1 + return b"ref" if checks["n"] <= 2 else None + return b"0" + + self.manager.params.get.side_effect = get + self.manager._download_ref = b"ref" + with self.assertRaises(Exception) as ctx: + asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact)) + assert 'cancelled' in str(ctx.exception).lower() + assert not os.path.isfile(get_manifest_path(base_path)) + self.run_with_server(body) + + def test_replaced_download_ref_queues_instead_of_cancelling(self): + """Selecting another model mid-transfer lets the running download finish.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + self.manager.params.get.side_effect = lambda key: b"other-ref" if key == "ModelManager_DownloadRef" else None + self.manager._download_ref = b"ref" + asyncio.run(self.manager._download_chunked(artifact.downloadUri.uri, base_path, artifact)) + assert os.path.isfile(get_manifest_path(base_path)) + self.run_with_server(body) + + def test_replaced_download_ref_is_kept(self): + """A selection made during a download must survive that download's cleanup.""" + self.manager.params.get.return_value = b"new-ref" + self.manager._download_ref = b"old-ref" + self.manager._release_download_ref() + self.manager.params.remove.assert_not_called() + + def test_own_download_ref_is_released(self): + self.manager.params.get.return_value = b"ref" + self.manager._download_ref = b"ref" + self.manager._release_download_ref() + self.manager.params.remove.assert_called_once_with("ModelManager_DownloadRef") + + def test_cached_bundle_cancel_skips_slot_write(self): + """A cancel must stop an already-on-disk bundle before it is applied to the slot.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + for i, data in enumerate(CHUNK_BODIES): + with open(get_chunk_name(base_path, i, len(CHUNK_BODIES)), 'wb') as f: + f.write(data) + self._bundle.ref = "test-ref" + params, store = self._make_params_with_store() + store["ModelManager_DownloadRef"] = None # removed -> cancelled + self.manager.params = params + self.manager._download_ref = b"ref" + with self.assertRaises(Exception) as ctx: + asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom")) + assert 'cancelled' in str(ctx.exception).lower() + assert "ModelManager_ActiveBundle" not in store + assert all(os.path.isfile(p) for p in self.chunk_paths(base_path)), "cancel must not delete cached chunks" + self.run_with_server(body) + + def test_resume_skips_valid_chunks(self): + """A chunk already on disk is kept and not re-downloaded; progress starts above its share.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + with open(get_chunk_name(base_path, 0, len(CHUNK_BODIES)), 'wb') as f: + f.write(CHUNK_BODIES[0]) + + asyncio.run(self.manager._process_artifact(artifact, self.dest)) + + chunk0_suffix = get_chunk_name('', 0, len(CHUNK_BODIES)) + assert not any(p.endswith(chunk0_suffix) for p in DownloadHandler.request_paths), "valid chunk was re-downloaded" + for i, expected in enumerate(CHUNK_BODIES): + with open(get_chunk_name(base_path, i, len(CHUNK_BODIES)), 'rb') as f: + assert f.read() == expected + assert os.path.isfile(get_manifest_path(base_path)) + assert min(self.reported) >= (1 / len(CHUNK_BODIES)) * 100 - 1, "progress must not restart below the resumed share" + self.run_with_server(body) + + def test_verify_reports_valid_fraction_then_cached(self): + """A fully cached bundle publishes climbing verify progress and ends cached.""" + def body(): + artifact = self.make_artifact(chunked=True) + base_path = os.path.join(self.dest, artifact.fileName) + for i, data in enumerate(CHUNK_BODIES): + with open(get_chunk_name(base_path, i, len(CHUNK_BODIES)), 'wb') as f: + f.write(data) + + asyncio.run(self.manager._process_artifact(artifact, self.dest)) + + assert DownloadHandler.request_paths == [], "cached bundle must not hit the network" + assert [round(p) for p in self.reported[:3]] == [33, 67, 100] + assert artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.cached + self.run_with_server(body) + + def _make_params_with_store(self): + params = mock.MagicMock() + store = {} + + def get(key, *args, **kwargs): + return store.get(key, b"0") # b"0" -> download not cancelled + + def put(key, value, *args, **kwargs): + store[key] = value + + params.get.side_effect = get + params.put.side_effect = put + return params, store + + def test_download_writes_qcom_slot(self): + """A download resolved to the qcom source writes the qcom active bundle slot only.""" + def body(): + artifact = self.make_artifact(chunked=True) + self._bundle.ref = "test-ref" + self._bundle.minimumSelectorVersion = 18 + params, store = self._make_params_with_store() + self.manager.params = params + asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "qcom")) + + assert "ModelManager_ActiveBundle" in store, "qcom download must write the qcom slot" + assert "ModelManager_ActiveBundleUSBGPU" not in store, "qcom download must not touch the usbgpu slot" + assert self.manager.selected_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded + assert self.manager.active_bundle is not None and self.manager.active_bundle.ref == "test-ref" + assert self.manager.active_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded + chunk_names = [get_chunk_name(artifact.fileName, i, len(artifact.chunks)) for i in range(len(artifact.chunks))] + missing = [c for c in chunk_names if not os.path.isfile(os.path.join(self.dest, c))] + assert missing == [], f"chunks missing from the cache: {missing}" + self.run_with_server(body) + + def test_download_writes_usbgpu_slot(self): + """A download resolved to the usbgpu source writes the usbgpu active bundle slot only.""" + def body(): + self.make_artifact(chunked=True) + self._bundle.ref = "big-ref" + self._bundle.minimumSelectorVersion = 18 + params, store = self._make_params_with_store() + self.manager.params = params + asyncio.run(self.manager._download_bundle(self._bundle, self.dest, "usbgpu")) + + assert "ModelManager_ActiveBundleUSBGPU" in store, "usbgpu download must write the usbgpu slot" + assert "ModelManager_ActiveBundle" not in store, "usbgpu download must not touch the qcom slot" + assert self.manager.selected_bundle.status == custom.ModelManagerSP.DownloadStatus.downloaded + self.run_with_server(body) + class TestManagerImports(OpenpilotTestCase): """Catches undeclared dependencies. aiohttp lived only in the AGNOS venv; 19.6 dropped @@ -267,6 +434,352 @@ class TestManagerImports(OpenpilotTestCase): assert connect > 0 and read > 0, "requests defaults to no timeout; downloads would hang forever" +class TestResolveBundleByRef(OpenpilotTestCase): + """A ref resolves to (bundle, source) across both hardware manifests. Refs are + unique per manifest and never overlap across sources, so a ref maps to exactly + one slot. Shared by the manager's download flow and the settings UI.""" + + @staticmethod + def _bundle(ref: str): + bundle = custom.ModelManagerSP.ModelBundle.new_message() + bundle.ref = ref + return bundle + + def test_qcom_ref_resolves_to_qcom_slot(self): + small = self._bundle("small") + assert resolve_bundle_by_ref("small", {"qcom": [small], "usbgpu": []}) == (small, "qcom") + + def test_usbgpu_ref_resolves_to_usbgpu_slot(self): + big = self._bundle("big") + assert resolve_bundle_by_ref("big", {"qcom": [], "usbgpu": [big]}) == (big, "usbgpu") + + def test_unknown_ref_returns_none(self): + source_bundles = {"qcom": [self._bundle("small")], "usbgpu": []} + assert resolve_bundle_by_ref("nope", source_bundles) is None + + +def manifest_bundle(short_name: str, ref: str, index: int = 0, is_big: bool = False) -> dict: + """Minimal manifest bundle dict, version-compatible (no chunks to avoid disk side effects). + Big (usbgpu) bundles carry `is_big: true` in the manifest JSON.""" + return { + "index": index, + "short_name": short_name, + "display_name": short_name.upper(), + "generation": 1, + "environment": "release", + "runner": "tinygrad", + "is_big": is_big, + "minimum_selector_version": "18", + "ref": ref, + "models": [{ + "type": "supercombo", + "artifact": { + "file_name": f"{short_name}.pkl", + "download_uri": {"url": f"https://example.com/{short_name}.pkl", "sha256": "s"}, + }, + }], + } + + +def fresh_sync_time() -> int: + return int(time.monotonic() * 1e9) + + +class TestModelFetcherSources(OpenpilotTestCase): + """Both manifests are always maintained: get_bundles_for_source exposes either + source by name, and active_source picks which one matches the attached hardware.""" + + def _make_params(self, qcom_manifest, usbgpu_manifest): + params = mock.MagicMock() + + def get(key): + if key == "ModelManager_ModelsCache": + return qcom_manifest + if key == "ModelManager_ModelsCache_USBGPU": + return usbgpu_manifest + if key in ("ModelManager_LastSyncTime", "ModelManager_LastSyncTime_USBGPU"): + return fresh_sync_time() + return None + + params.get.side_effect = get + return params + + def test_active_source_follows_chestnut_presence(self): + assert ModelFetcher.active_source(False) == "qcom" + assert ModelFetcher.active_source(True) == "usbgpu" + + def test_get_bundles_for_source_returns_each_source(self): + params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]}, + {"bundles": [manifest_bundle("big", "bbb", is_big=True)]}) + fetcher = ModelFetcher(params) + assert [bundle.ref for bundle in fetcher.get_bundles_for_source("qcom")] == ["aaa"] + assert [bundle.ref for bundle in fetcher.get_bundles_for_source("usbgpu")] == ["bbb"] + + def test_get_bundles_for_source_unknown(self): + assert ModelFetcher(mock.MagicMock()).get_bundles_for_source("bogus") == [] + + def test_get_cached_bundles_parses_source(self): + params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]}, + {"bundles": [manifest_bundle("big", "bbb", is_big=True)]}) + qcom_bundles = get_cached_bundles(params, "qcom") + usbgpu_bundles = get_cached_bundles(params, "usbgpu") + assert [b.ref for b in qcom_bundles] == ["aaa"] + assert [b.ref for b in usbgpu_bundles] == ["bbb"] + assert qcom_bundles[0].displayName == "SMALL" + + def test_get_cached_bundles_empty_when_missing(self): + params = mock.MagicMock() + params.get.return_value = None + assert get_cached_bundles(params, "qcom") == [] + assert get_cached_bundles(params, "usbgpu") == [] + + def test_get_cached_bundles_unknown_source(self): + assert get_cached_bundles(mock.MagicMock(), "bogus") == [] + + def test_active_json_has_both_urls(self): + params = mock.MagicMock() + ModelFetcher(params) + active_json_calls = [call for call in params.put.call_args_list if call.args[0] == "ModelManager_ActiveJson"] + assert active_json_calls, "expected ModelManager_ActiveJson to be written" + assert active_json_calls[-1].args[1] == { + "qcom": ModelFetcher.MODEL_URL, + "usbgpu": ModelFetcher.MODEL_URL_USBGPU, + } + + + +class TestSourceCacheIntegrity(OpenpilotTestCase): + """Each source's cached manifest must contain only that source's models; the + `is_big` flag in the JSON marks the big (usbgpu) models. A mismatched cache is + legacy data from before the per-source split (the active manifest was cached + under the unsuffixed key regardless of hardware) and is refetched. This + replaces the old one-time bundle migration.""" + + def _make_params(self, qcom_manifest, usbgpu_manifest): + params = mock.MagicMock() + + def get(key): + if key == "ModelManager_ModelsCache": + return qcom_manifest + if key == "ModelManager_ModelsCache_USBGPU": + return usbgpu_manifest + if key in ("ModelManager_LastSyncTime", "ModelManager_LastSyncTime_USBGPU"): + return fresh_sync_time() + return None + + params.get.side_effect = get + return params + + def _fetched(self, *bundles): + return ModelFetcher(mock.MagicMock()).model_parser.parse_models({"bundles": list(bundles)}) + + def test_qcom_cache_with_big_models_is_refetched(self): + """Legacy: the unsuffixed cache holds the big manifest. is_big confirms it is + the wrong set for qcom, so a fresh fetch replaces it.""" + params = self._make_params({"bundles": [manifest_bundle("big", "bbb", is_big=True)]}, + {"bundles": [manifest_bundle("big2", "ccc", is_big=True)]}) + fetcher = ModelFetcher(params) + fetched = self._fetched(manifest_bundle("small", "aaa")) + with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched): + bundles = fetcher.get_bundles_for_source("qcom") + assert [bundle.ref for bundle in bundles] == ["aaa"] + + def test_usbgpu_cache_without_big_models_is_refetched(self): + params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]}, + {"bundles": [manifest_bundle("big2", "ccc")]}) + fetcher = ModelFetcher(params) + fetched = self._fetched(manifest_bundle("big", "bbb", is_big=True)) + with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched): + bundles = fetcher.get_bundles_for_source("usbgpu") + assert [bundle.ref for bundle in bundles] == ["bbb"] + + def test_matching_caches_are_used_without_fetch(self): + params = self._make_params({"bundles": [manifest_bundle("small", "aaa")]}, + {"bundles": [manifest_bundle("big", "bbb", is_big=True)]}) + fetcher = ModelFetcher(params) + with mock.patch.object(fetcher, "_fetch_and_cache_models", side_effect=AssertionError("cache should be used")): + assert [bundle.ref for bundle in fetcher.get_bundles_for_source("qcom")] == ["aaa"] + assert [bundle.ref for bundle in fetcher.get_bundles_for_source("usbgpu")] == ["bbb"] + + def test_stale_version_cache_is_refetched(self): + """A source-matching cache whose bundles are all filtered by the selector version + check parses to zero valid bundles; it is stale (e.g. an old manifest) and must be + refetched instead of silently returning an empty list forever.""" + stale = manifest_bundle("small", "aaa") + stale["minimum_selector_version"] = "16" + params = self._make_params({"bundles": [stale]}, + {"bundles": [manifest_bundle("big", "bbb", is_big=True)]}) + fetcher = ModelFetcher(params) + fetched = self._fetched(manifest_bundle("small2", "ddd")) + with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch: + bundles = fetcher.get_bundles_for_source("qcom") + fetch.assert_called_once_with("qcom") + assert [bundle.ref for bundle in bundles] == ["ddd"] + + def test_mismatched_refetch_happens_once(self): + """If the fresh manifest still fails the source check, the URL is authoritative: + trust it instead of refetching at 1 Hz forever.""" + params = self._make_params({"bundles": [manifest_bundle("big", "bbb", is_big=True)]}, + {"bundles": [manifest_bundle("big2", "ccc", is_big=True)]}) + fetcher = ModelFetcher(params) + fetched = self._fetched(manifest_bundle("big", "bbb", is_big=True)) + with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch: + first = fetcher.get_bundles_for_source("qcom") + second = fetcher.get_bundles_for_source("qcom") + fetch.assert_called_once_with("qcom") + assert [bundle.ref for bundle in first] == ["bbb"] + assert [bundle.ref for bundle in second] == ["bbb"] + + def test_corrupt_cache_is_refetched(self): + """A cache that fails to parse (e.g. truncated/foreign JSON) must trigger a + refetch instead of raising every loop and never recovering.""" + corrupt = {"bundles": [{"short_name": "broken"}]} # missing required fields + params = self._make_params(corrupt, {"bundles": [manifest_bundle("big", "bbb", is_big=True)]}) + fetcher = ModelFetcher(params) + fetched = self._fetched(manifest_bundle("small", "aaa")) + with mock.patch.object(fetcher, "_fetch_and_cache_models", return_value=fetched) as fetch: + bundles = fetcher.get_bundles_for_source("qcom") + fetch.assert_called_once_with("qcom") + assert [bundle.ref for bundle in bundles] == ["aaa"] + + +class TestActiveBundleValidation(OpenpilotTestCase): + """Validation is per-slot: a failed fetch (empty bundle list) must not reset a slot, + and resetting one slot must not stomp the runner cache derived from the other.""" + + def setUp(self): + super().setUp() + helpers._LAST_VALIDATED_RAW.clear() + + @staticmethod + def _raw_bundle(ref: str, runner: int | None = None) -> dict: + bundle = custom.ModelManagerSP.ModelBundle.new_message() + bundle.ref = ref + bundle.minimumSelectorVersion = 18 + if runner is not None: + bundle.runner = runner + return bundle.to_dict() + + def _params(self, qcom=None, usbgpu=None): + params = mock.MagicMock() + + def get(key, *args, **kwargs): + return {"ModelManager_ActiveBundle": qcom, "ModelManager_ActiveBundleUSBGPU": usbgpu}.get(key) + + params.get.side_effect = get + return params + + def test_empty_catalog_does_not_reset_slot(self): + params = self._params(qcom=self._raw_bundle("small")) + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False): + validate_active_bundles(params, {"qcom": [], "usbgpu": []}) + params.remove.assert_not_called() + + def test_reset_recomputes_runner_from_surviving_slot(self): + tinygrad = int(custom.ModelManagerSP.Runner.tinygrad) + big_raw = self._raw_bundle("big", runner=tinygrad) + params = self._params(qcom=self._raw_bundle("gone"), usbgpu=big_raw) + catalog = {"qcom": [custom.ModelManagerSP.ModelBundle(**self._raw_bundle("other"))], + "usbgpu": [custom.ModelManagerSP.ModelBundle(**big_raw)]} + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True): + validate_active_bundles(params, catalog) + params.remove.assert_called_once_with("ModelManager_ActiveBundle") + runner_puts = [call for call in params.put.call_args_list if call.args[0] == "ModelRunnerTypeCache"] + assert [call.args[1] for call in runner_puts] == [tinygrad] + + +class TestActiveBundleSelection(OpenpilotTestCase): + """The effective active bundle is the active source's slot: usbgpu when a GPU is + present, qcom otherwise. An empty active slot means the hardware default (stock + runner), never the other slot's pick - modeld_v2 requires a real bundle.""" + + @staticmethod + def _raw_bundle(ref: str) -> dict: + bundle = custom.ModelManagerSP.ModelBundle.new_message() + bundle.ref = ref + bundle.minimumSelectorVersion = 18 + return bundle.to_dict() + + def _params(self, qcom=None, usbgpu=None): + params = mock.MagicMock() + + def get(key, *args, **kwargs): + if key == "ModelManager_ActiveBundle": + return qcom + if key == "ModelManager_ActiveBundleUSBGPU": + return usbgpu + return None + + params.get.side_effect = get + return params + + def test_selected_bundle_is_per_slot(self): + params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big")) + assert get_selected_bundle(params, "qcom").ref == "small" + assert get_selected_bundle(params, "usbgpu").ref == "big" + + def test_no_gpu_uses_qcom_slot(self): + params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big")) + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False): + assert get_active_bundle(params).ref == "small" + + def test_gpu_uses_usbgpu_slot(self): + params = self._params(qcom=self._raw_bundle("small"), usbgpu=self._raw_bundle("big")) + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True): + assert get_active_bundle(params).ref == "big" + + def test_gpu_without_big_selection_is_hardware_default(self): + params = self._params(qcom=self._raw_bundle("small"), usbgpu=None) + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True): + assert get_active_bundle(params) is None + + +class TestEffectiveSource(OpenpilotTestCase): + """One gate decides the active source. With no flags it is runtime truth (GPU + attached); display callers (mici) pass the ui_state flags, which additionally + require the big model to be loading, active, or the device offroad. The active + bundle is simply the selected bundle of that source.""" + + @staticmethod + def _raw_bundle(ref: str) -> dict: + bundle = custom.ModelManagerSP.ModelBundle.new_message() + bundle.ref = ref + bundle.minimumSelectorVersion = 18 + return bundle.to_dict() + + def test_runtime_no_gpu(self): + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False): + assert get_active_source() == "qcom" + + def test_runtime_gpu_present(self): + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=True): + assert get_active_source() == "usbgpu" + + def test_display_offroad_gpu_present_shows_big(self): + assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=False, offroad=True) == "usbgpu" + + def test_display_onroad_gpu_loading_shows_big(self): + assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=True, offroad=False) == "usbgpu" + + def test_display_onroad_gpu_active_shows_big(self): + assert get_active_source(usbgpu=True, usbgpu_active=True, usbgpu_loading=False, offroad=False) == "usbgpu" + + def test_display_onroad_gpu_idle_shows_small(self): + assert get_active_source(usbgpu=True, usbgpu_active=False, usbgpu_loading=False, offroad=False) == "qcom" + + def test_display_active_none_is_idle(self): + assert get_active_source(usbgpu=True, usbgpu_active=None, usbgpu_loading=False, offroad=False) == "qcom" + + def test_active_bundle_follows_source(self): + params = mock.MagicMock() + params.get.side_effect = lambda key: {"ModelManager_ActiveBundle": self._raw_bundle("small"), + "ModelManager_ActiveBundleUSBGPU": self._raw_bundle("big")}.get(key) + with mock.patch("openpilot.sunnypilot.models.helpers.usbgpu_present", return_value=False): + assert get_active_bundle(params).ref == "small" + assert get_selected_bundle(params, get_active_source(usbgpu=True, usbgpu_active=False, + usbgpu_loading=False, offroad=True)).ref == "big" + + @unittest.skipUnless(os.environ.get('RUN_INTEGRATION_TESTS'), 'requires external network') class TestLiveModelManifest(OpenpilotTestCase): """Every artifact and chunk URL in the published manifest must resolve.""" diff --git a/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py b/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py index fd389f93c..d6d82dfb3 100644 --- a/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py +++ b/openpilot/sunnypilot/models/tests/test_tinygrad_ref.py @@ -1,13 +1,11 @@ import requests -from openpilot.common.params import Params from openpilot.sunnypilot.models.tinygrad_ref import get_tinygrad_ref from openpilot.sunnypilot.models.fetcher import ModelFetcher from openpilot.common.test import OpenpilotTestCase def fetch_tinygrad_ref(): - fetcher = ModelFetcher(Params()) - response = requests.get(fetcher.model_url, timeout=10) + response = requests.get(ModelFetcher.MODEL_URL, timeout=10) response.raise_for_status() json_data = response.json() return json_data.get("tinygrad_ref") diff --git a/openpilot/sunnypilot/selfdrive/assets/icons_mici/sunnylink.png b/openpilot/sunnypilot/selfdrive/assets/icons_mici/sunnylink.png new file mode 100644 index 000000000..6639536f9 --- /dev/null +++ b/openpilot/sunnypilot/selfdrive/assets/icons_mici/sunnylink.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:447099e93e303b29e7b3eac237bb0f27f8c5e12786991139aee2432532a75f58 +size 12310 diff --git a/openpilot/sunnypilot/sunnylink/statsd.py b/openpilot/sunnypilot/sunnylink/statsd.py index 7e8faf632..a221fc084 100755 --- a/openpilot/sunnypilot/sunnylink/statsd.py +++ b/openpilot/sunnypilot/sunnylink/statsd.py @@ -65,6 +65,7 @@ def sp_stats(end_event): 'MadsSteeringMode', 'MadsUnifiedEngagementMode', 'ModelManager_ActiveBundle', + 'ModelManager_ActiveBundleUSBGPU', 'ModelManager_Favs', 'EnableSunnylinkUploader', 'SunnylinkEnabled', diff --git a/openpilot/sunnypilot/system/params_migration.py b/openpilot/sunnypilot/system/params_migration.py index 130fd6431..f0f0d7248 100644 --- a/openpilot/sunnypilot/system/params_migration.py +++ b/openpilot/sunnypilot/system/params_migration.py @@ -84,6 +84,21 @@ def _migrate_tesla_mads_screen_button(_params): cloudlog.exception(f"Error migrating TeslaMadsScreenButton: {e}") +def _migrate_model_bundle_slots(_params): + # Pre-split, a chestnut user's big-model selection lived in the single + # ActiveBundle. Seed both slots; validation drops whichever does not match + # its own manifest. + try: + if _params.get("ModelManager_ActiveBundleUSBGPU") is not None: + return + if (bundle := _params.get("ModelManager_ActiveBundle")) is None: + return + _params.put("ModelManager_ActiveBundleUSBGPU", bundle, block=True) + cloudlog.info("params_migration: seeded ModelManager_ActiveBundleUSBGPU from ModelManager_ActiveBundle") + except Exception as e: + cloudlog.exception(f"Error migrating model bundle slots: {e}") + + def run_migration(_params): # migrate OnroadScreenOffBrightness if _params.get("OnroadScreenOffBrightnessMigrated") != ONROAD_BRIGHTNESS_MIGRATION_VERSION: @@ -120,3 +135,6 @@ def run_migration(_params): # seed TeslaMadsScreenButton for existing Tesla installs _migrate_tesla_mads_screen_button(_params) + + # seed the usbgpu model slot from the pre-split single slot + _migrate_model_bundle_slots(_params) diff --git a/openpilot/sunnypilot/system/tests/__init__.py b/openpilot/sunnypilot/system/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openpilot/sunnypilot/system/tests/test_params_migration.py b/openpilot/sunnypilot/system/tests/test_params_migration.py new file mode 100644 index 000000000..328a7a65a --- /dev/null +++ b/openpilot/sunnypilot/system/tests/test_params_migration.py @@ -0,0 +1,36 @@ +""" +Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + +This file is part of sunnypilot and is licensed under the MIT License. +See the LICENSE.md file in the root directory for more details. +""" + +from openpilot.common.params import Params +from openpilot.common.test import OpenpilotTestCase +from openpilot.sunnypilot.system.params_migration import _migrate_model_bundle_slots + + +class TestModelBundleSlotMigration(OpenpilotTestCase): + """Pre-split, a chestnut user's big-model selection lived in the single ActiveBundle. + The migration seeds both slots; per-source validation later drops whichever does not + match its own manifest.""" + + def test_seeds_usbgpu_slot_from_active_bundle(self): + params = Params() + bundle = {"ref": "big", "minimumSelectorVersion": 18} + params.put("ModelManager_ActiveBundle", bundle, block=True) + _migrate_model_bundle_slots(params) + assert params.get("ModelManager_ActiveBundleUSBGPU") == bundle + assert params.get("ModelManager_ActiveBundle") == bundle + + def test_noop_when_usbgpu_slot_already_set(self): + params = Params() + params.put("ModelManager_ActiveBundle", {"ref": "small"}, block=True) + params.put("ModelManager_ActiveBundleUSBGPU", {"ref": "big"}, block=True) + _migrate_model_bundle_slots(params) + assert params.get("ModelManager_ActiveBundleUSBGPU") == {"ref": "big"} + + def test_noop_when_no_selection(self): + params = Params() + _migrate_model_bundle_slots(params) + assert params.get("ModelManager_ActiveBundleUSBGPU") is None diff --git a/openpilot/system/ui/sunnypilot/lib/utils.py b/openpilot/system/ui/sunnypilot/lib/utils.py index b9ed152af..6ae30d13a 100644 --- a/openpilot/system/ui/sunnypilot/lib/utils.py +++ b/openpilot/system/ui/sunnypilot/lib/utils.py @@ -8,12 +8,26 @@ from collections.abc import Callable import pyray as rl -from openpilot.system.ui.lib.application import FontWeight +from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.sunnypilot.lib.styles import style from openpilot.system.ui.sunnypilot.widgets.list_view import ButtonActionSP -from openpilot.system.ui.widgets.label import UnifiedLabel +from openpilot.system.ui.widgets.label import ScrollState, UnifiedLabel from openpilot.system.ui.widgets.list_view import BUTTON_WIDTH, BUTTON_HEIGHT, TEXT_PADDING, _resolve_value +SCROLL_SPEED = 1.2 # stock is 0.8, boosted 50% to compensate for larger font (50 vs 32) +SCROLL_REFERENCE_FPS = 60. + + +class UnifiedLabelSP(UnifiedLabel): + # stock scroll formula (0.8 / 60 * fps) is inverted — pre-correct so speed is constant px/sec + def _render(self, _): + if self._needs_scroll and self._scroll_state == ScrollState.SCROLLING: + fps = gui_app.target_fps + wrong_step = 0.8 / SCROLL_REFERENCE_FPS * fps + correct_step = SCROLL_SPEED * SCROLL_REFERENCE_FPS / fps + self._scroll_offset -= (correct_step - wrong_step) + super()._render(_) + class NoElideButtonAction(ButtonActionSP): def get_width_hint(self): @@ -21,14 +35,12 @@ class NoElideButtonAction(ButtonActionSP): class ScrollingButtonAction(ButtonActionSP): - """ButtonActionSP whose value scrolls instead of eliding when it doesn't fit.""" - def __init__(self, text: str | Callable[[], str], width: int = style.BUTTON_ACTION_WIDTH, enabled: bool | Callable[[], bool] = True): super().__init__(text=text, width=width, enabled=enabled) - self._value_label = UnifiedLabel("", font_size=style.ITEM_TEXT_FONT_SIZE, font_weight=FontWeight.NORMAL, - text_color=self._value_color, scroll=True, - alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) + self._value_label = UnifiedLabelSP("", font_size=style.ITEM_TEXT_FONT_SIZE, font_weight=FontWeight.NORMAL, + text_color=self._value_color, scroll=True, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) def set_value(self, value: str | Callable[[], str], color: rl.Color = style.ITEM_TEXT_VALUE_COLOR): if self.value != _resolve_value(value, ""): diff --git a/openpilot/system/ui/sunnypilot/widgets/download_status.py b/openpilot/system/ui/sunnypilot/widgets/download_status.py index b299c464f..135bd151a 100644 --- a/openpilot/system/ui/sunnypilot/widgets/download_status.py +++ b/openpilot/system/ui/sunnypilot/widgets/download_status.py @@ -16,6 +16,7 @@ from openpilot.system.ui.lib.text_measure import measure_text_cached from openpilot.system.ui.sunnypilot.lib.styles import style from openpilot.system.ui.sunnypilot.widgets.list_view import ListItemSP from openpilot.system.ui.widgets.label import UnifiedLabel +from openpilot.system.ui.sunnypilot.lib.utils import UnifiedLabelSP from openpilot.system.ui.widgets.list_view import ItemAction FONT_SIZE = style.ITEM_TEXT_FONT_SIZE @@ -24,6 +25,8 @@ ICON_PADDING = 12 BAR_WIDTH = 1100 BAR_HEIGHT = 20 +SEGMENT_GAP = 24 +SEGMENT_NAME_MAX_WIDTH = 380 BAR_GAP = 16 BAR_RADIUS = BAR_HEIGHT / 2 CAPSULE_POINTS = 24 @@ -45,6 +48,8 @@ class DownloadStatusAction(ItemAction): super().__init__(width=BAR_WIDTH) self.name = "" self.status_text = "" + self.segments: list[tuple[str, rl.Color, str | None, rl.Color | None]] | None = None + self._segment_labels: list[UnifiedLabelSP] = [] self.downloading = False self.text_color = rl.GRAY self.icon: str | None = None @@ -62,7 +67,8 @@ class DownloadStatusAction(ItemAction): alignment=rl.GuiTextAlignment.TEXT_ALIGN_RIGHT, alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE) - def update(self, name, downloading=False, progress=0.0, status_text="", text_color=rl.GRAY, icon=None, icon_color=None): + def update(self, name, downloading=False, progress=0.0, status_text="", text_color=rl.GRAY, icon=None, icon_color=None, segments=None): + self.segments = segments if downloading and not self.downloading: self._name_label.reset_shimmer() self._progress.x = progress @@ -85,11 +91,22 @@ class DownloadStatusAction(ItemAction): def get_width_hint(self) -> float: if self.downloading: return BAR_WIDTH + if self.segments: + return sum(total for _, _, total in self._measured_segments()) width = measure_text_cached(self._font, self._idle_text, FONT_SIZE).x if self.icon: width += ICON_SIZE + ICON_PADDING return width + def _measured_segments(self): + """[(segment, text width, total width incl. icon and gap)]""" + out = [] + for i, seg in enumerate(self.segments or []): + text_width = min(measure_text_cached(self._font, seg[0], FONT_SIZE).x, SEGMENT_NAME_MAX_WIDTH) + total = text_width + (ICON_PADDING + ICON_SIZE if seg[2] else 0) + (SEGMENT_GAP if i else 0) + out.append((seg, text_width, total)) + return out + def _render(self, rect: rl.Rectangle): if self.downloading: self._render_downloading(rect) @@ -134,6 +151,8 @@ class DownloadStatusAction(ItemAction): def _render_downloading(self, rect: rl.Rectangle): percent = f"{int(self._progress.x)}%" + if self.status_text: + percent = f"{self.status_text} {percent}" text_height = measure_text_cached(self._font, percent, FONT_SIZE).y top = rect.y + (rect.height - (text_height + BAR_GAP + BAR_HEIGHT)) / 2 @@ -148,6 +167,9 @@ class DownloadStatusAction(ItemAction): self._draw_fill(rail, max(0.0, min(rect.width, rect.width * (self._progress.x / 100.0)))) def _render_idle(self, rect: rl.Rectangle): + if self.segments: + self._render_segments(rect) + return text = self._idle_text text_size = measure_text_cached(self._font, text, FONT_SIZE) right = rect.x + rect.width @@ -161,6 +183,29 @@ class DownloadStatusAction(ItemAction): rl.draw_text_ex(self._font, text, rl.Vector2(right - text_size.x, rect.y + (rect.height - text_size.y) / 2), FONT_SIZE, 0, self.text_color) + def _render_segments(self, rect: rl.Rectangle): + measured = self._measured_segments() + while len(self._segment_labels) < len(measured): + self._segment_labels.append(UnifiedLabelSP("", font_size=FONT_SIZE, max_width=SEGMENT_NAME_MAX_WIDTH, + scroll=True, wrap_text=False)) + x = rect.x + rect.width - sum(total for _, _, total in measured) + for i, ((text, color, icon, icon_color), text_width, _) in enumerate(measured): + if i: + x += SEGMENT_GAP + label = self._segment_labels[i] + if label.text != text: + label.set_text(text) + label.set_text_color(color) + text_height = measure_text_cached(self._font, text, FONT_SIZE).y + label.set_position(x, rect.y + (rect.height - text_height) / 2) + label.render() + x += text_width + if icon: + texture = gui_app.texture(icon, ICON_SIZE, ICON_SIZE, keep_aspect_ratio=True) + rl.draw_texture_v(texture, rl.Vector2(x + ICON_PADDING, rect.y + (rect.height - texture.height) / 2), + icon_color or color) + x += ICON_PADDING + ICON_SIZE + def download_status_item(title): return ListItemSP(title=title, action_item=DownloadStatusAction(), title_color=style.ITEM_TEXT_COLOR)