Merge branch 'master' into egpu-alert

openpilot/cereal/custom.capnp

@@ -353,6 +353,7 @@ struct OnroadEventSP @0xda96579883444c35 {
     speedLimitPending @22;
     e2eChime @23;
     laneChangeRoadEdge @24;
+    bigModelReady @25;
   }
 }
 

openpilot/common/params_keys.h

@@ -246,7 +246,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
 
     // mapd
     {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
-    {"Mapd_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
     {"MapdVersion", {PERSISTENT, STRING}},
     {"MapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0.0"}},
     {"NextMapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, JSON}},

openpilot/selfdrive/selfdrived/selfdrived.py

@@ -198,6 +198,7 @@ class SelfdriveD(CruiseHelper):
     loading = self.params.get_bool("UsbGpuLoading")
     if self.big_model_loading and not loading:
       self.big_model_ready_t = time.monotonic()
+      self.events_sp.add(custom.OnroadEventSP.EventName.bigModelReady)
     self.big_model_loading = loading
     if self.big_model_loading:
       self.events.add(EventName.bigModelLoading)

openpilot/selfdrive/ui/sunnypilot/layouts/settings/osm.py

@@ -8,6 +8,7 @@ import datetime
 import os
 import platform
 import requests
+import shutil
 import threading
 from pathlib import Path
 from time import monotonic
@@ -74,12 +75,22 @@ class OSMLayout(Widget):
   def _update_map_size(self):
     threading.Thread(target=self.calculate_size, daemon=True).start()
 
-  def _on_confirm_delete_maps(self):
-    ui_state.params.put_bool("Mapd_ClearCache", True)
+  def _do_delete_maps(self):
+    if MAP_PATH.exists():
+      shutil.rmtree(MAP_PATH)
+
+    for param in ("OsmDownloadedDate", "OsmLocal", "OsmLocationName", "OsmLocationTitle", "OsmStateName", "OsmStateTitle"):
+      ui_state.params.remove(param)
+
     self._delete_maps_btn.action_item.set_enabled(True)
     self._delete_maps_btn.action_item.set_text(tr("DELETE"))
     self._update_map_size()
 
+  def _on_confirm_delete_maps(self):
+    self._delete_maps_btn.action_item.set_enabled(False)
+    self._delete_maps_btn.action_item.set_text("DELETING...")
+    threading.Thread(target=self._do_delete_maps).start()
+
   def _delete_maps(self):
     self._show_confirm(tr("This will delete ALL downloaded maps\n\nAre you sure you want to delete all maps?"),
                        tr("Yes, delete all maps"), self._on_confirm_delete_maps)

openpilot/sunnypilot/mapd/mapd_manager.py

@@ -55,19 +55,6 @@ def cleanup_old_osm_data(files_to_remove: list[str]) -> None:
       shutil.rmtree(file, ignore_errors=False)
 
 
-def clear_downloaded_maps() -> None:
-  """Deletes downloaded OSM map data and resets params."""
-  path = f"{Paths.mapd_root()}/offline"
-  if os.path.exists(path):
-    shutil.rmtree(path, ignore_errors=True)
-
-  for param in ("OsmDownloadedDate", "OsmLocal", "OsmLocationName", "OsmLocationTitle",
-                "OsmStateName", "OsmStateTitle"):
-    params.remove(param)
-
-  cloudlog.info("mapd: downloaded maps cleared")
-
-
 def request_refresh_osm_location_data(nations: list[str], states: list[str] | None = None) -> None:
   params.put("OsmDownloadedDate", str(datetime.now().timestamp()), block=True)
   params.put_bool("OsmDbUpdatesCheck", False, block=True)
@@ -144,10 +131,6 @@ def main_thread():
     show_alert = bool(get_files_for_cleanup() and params.get_bool("OsmLocal"))
     set_offroad_alert("Offroad_OSMUpdateRequired", show_alert, "This alert will be cleared when new maps are downloaded.")
 
-    if params.get("Mapd_ClearCache"):
-      clear_downloaded_maps()
-      params.remove("Mapd_ClearCache")
-
     update_osm_db()
     live_map_sp.tick()
     rk.keep_time()

openpilot/sunnypilot/models/helpers.py

@@ -41,7 +41,12 @@ def _compute_hash(file_path: str) -> str | None:
     return None
 
 
-def verify_file(file_path: str, expected_hash: str) -> bool:
+async def verify_file(file_path: str, expected_hash: str) -> bool:
+  file_hash = _compute_hash(file_path)
+  return file_hash == expected_hash.lower() if file_hash else False
+
+
+def _verify_file(file_path: str, expected_hash: str) -> bool:
   file_hash = _compute_hash(file_path)
   return file_hash == expected_hash.lower() if file_hash else False
 
@@ -76,7 +81,7 @@ def _bundle_artifacts(bundle: custom.ModelManagerSP.ModelBundle) -> list[tuple[s
 
 def _bundle_is_valid_locally(bundle: custom.ModelManagerSP.ModelBundle) -> bool:
   model_root = Paths.model_root()
-  return all(verify_file(os.path.join(model_root, file_name), expected_hash)
+  return all(_verify_file(os.path.join(model_root, file_name), expected_hash)
              for file_name, expected_hash in _bundle_artifacts(bundle))
 
 

openpilot/sunnypilot/models/manager.py

@@ -10,7 +10,6 @@ import os
 import time
 
 import requests
-from openpilot.common.file_chunker import get_chunk_name
 from openpilot.common.params import Params
 from openpilot.common.realtime import Ratekeeper
 from openpilot.common.swaglog import cloudlog
@@ -23,8 +22,6 @@ from openpilot.sunnypilot.models.helpers import (ACTIVE_BUNDLE_KEYS, get_active_
 
 # (connect, read) seconds. read is per-request inactivity, not a total cap
 DOWNLOAD_TIMEOUT = (30, 30)
-# how many download+verify rounds before giving up on a chunk that won't verify
-MAX_CHUNK_VERIFY_ATTEMPTS = 3
 
 
 class DownloadCancelled(Exception):
@@ -156,35 +153,6 @@ class ModelManagerSP:
       os.remove(base_path)
     del self._download_start_times[artifact.fileName]
 
-  async def _verify_chunks_parallel(self, artifact, full_path: str) -> set[int]:
-    num_chunks = len(artifact.chunks)
-    start = time.monotonic()
-    futures = []
-    chunk_index: dict[asyncio.Future, int] = {}
-    for i, chunk in enumerate(artifact.chunks):
-      fut = asyncio.ensure_future(asyncio.to_thread(verify_file, get_chunk_name(full_path, i, num_chunks), chunk.sha256))
-      futures.append(fut)
-      chunk_index[fut] = i
-
-    valid_chunks: set[int] = set()
-    pending = set(futures)
-    while pending:
-      if self._download_interrupted():
-        for t in pending:
-          t.cancel()
-        raise DownloadCancelled("Download cancelled")
-      done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
-      for fut in done:
-        if fut.result():
-          valid_chunks.add(chunk_index[fut])
-      artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.verifying
-      artifact.downloadProgress.progress = (len(valid_chunks) / num_chunks) * 100
-      self._sync_artifact_progress(artifact)
-      self._report_status()
-    elapsed = time.monotonic() - start
-    cloudlog.info(f"Verified {len(valid_chunks)}/{num_chunks} chunks of {artifact.fileName} in {elapsed:.2f}s")
-    return valid_chunks
-
   async def _process_artifact(self, artifact, destination_path: str) -> None:
     if not artifact.downloadUri.uri:
       return None
@@ -202,10 +170,20 @@ class ModelManagerSP:
       is_cached = False
       valid_chunks: set[int] = set()
       if len(artifact.chunks) > 0:
-        valid_chunks = await self._verify_chunks_parallel(artifact, full_path)
-        is_cached = len(valid_chunks) == len(artifact.chunks)
+        from openpilot.common.file_chunker import get_chunk_name
+        num_chunks = len(artifact.chunks)
+        for i, chunk in enumerate(artifact.chunks):
+          if self._download_interrupted():
+            raise DownloadCancelled("Download cancelled")
+          if await verify_file(get_chunk_name(full_path, i, num_chunks), chunk.sha256):
+            valid_chunks.add(i)
+          artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.verifying
+          artifact.downloadProgress.progress = (len(valid_chunks) / num_chunks) * 100
+          self._sync_artifact_progress(artifact)
+          self._report_status()
+        is_cached = len(valid_chunks) == num_chunks
       else:
-        if verify_file(full_path, expected_hash):
+        if await verify_file(full_path, expected_hash):
           is_cached = True
 
       if is_cached:
@@ -217,18 +195,15 @@ class ModelManagerSP:
         return
 
       if len(artifact.chunks) > 0:
-        attempts = 0
-        while len(valid_chunks) < len(artifact.chunks):
-          attempts += 1
-          cloudlog.warning(f"Re-downloading {len(artifact.chunks) - len(valid_chunks)} invalid chunk(s) of {filename} (attempt {attempts})")
-          await self._download_chunked(url, full_path, artifact, skip=valid_chunks)
-          valid_chunks = await self._verify_chunks_parallel(artifact, full_path)
-          if len(valid_chunks) < len(artifact.chunks) and attempts >= MAX_CHUNK_VERIFY_ATTEMPTS:
-            missing = next(i for i in range(len(artifact.chunks)) if i not in valid_chunks)
-            raise ValueError(f"Hash validation failed for chunk {missing+1} of {filename} after {attempts} attempts")
+        await self._download_chunked(url, full_path, artifact, skip=valid_chunks)
+        from openpilot.common.file_chunker import get_chunk_name
+        for i, chunk in enumerate(artifact.chunks):
+          chunk_path = get_chunk_name(full_path, i, len(artifact.chunks))
+          if not await verify_file(chunk_path, chunk.sha256):
+            raise ValueError(f"Hash validation failed for chunk {i+1} of {filename}")
       else:
         await self._download_file(url, full_path, artifact)
-        if not verify_file(full_path, expected_hash):
+        if not await verify_file(full_path, expected_hash):
           raise ValueError(f"Hash validation failed for {filename}")
 
       artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloaded

openpilot/sunnypilot/selfdrive/selfdrived/events.py

@@ -252,4 +252,12 @@ EVENTS_SP: dict[int, dict[str, Alert | AlertCallbackType]] = {
       AlertStatus.userPrompt, AlertSize.small,
       Priority.LOW, VisualAlert.none, AudibleAlert.prompt, 0.1),
   },
+
+  EventNameSP.bigModelReady: {
+    ET.PERMANENT: Alert(
+      "Big Model Ready",
+      "",
+      AlertStatus.normal, AlertSize.small,
+      Priority.LOW, VisualAlert.none, AudibleAlert.prompt, 2.),
+  },
 }