fix chunking

openpilot/sunnypilot/models/fetcher.py

@@ -64,8 +64,6 @@ class ModelParser:
 
     model.type = model_data.get("type")
     model.artifact = ModelParser._parse_artifact(model_data.get("artifact", {}))
-    if metadata := model_data.get("metadata"):
-      model.metadata = ModelParser._parse_artifact(metadata)
     return model
 
   @staticmethod
@@ -211,5 +209,3 @@ if __name__ == "__main__":
       # Print metadata details
       if model.artifact.chunks:
         print(f"Contains {len(model.artifact.chunks)} chunks.")
-      if hasattr(model, 'metadata') and model.metadata and model.metadata.fileName:
-        print(f"Metadata: {model.metadata.fileName}, Download URI: {model.metadata.downloadUri.uri}")

openpilot/sunnypilot/models/helpers.py

@@ -18,7 +18,7 @@ from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRai
 from openpilot.common.hardware.hw import Paths
 
 # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO
-REQUIRED_JSON_VERSION = 15
+REQUIRED_JSON_VERSION = 16
 
 CUSTOM_MODEL_PATH = Paths.model_root()
 METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl'
@@ -56,12 +56,20 @@ def is_bundle_version_compatible(bundle: dict) -> bool:
 
 def _bundle_artifacts(bundle: custom.ModelManagerSP.ModelBundle) -> list[tuple[str, str]]:
   artifacts = []
+  from openpilot.common.file_chunker import get_chunk_name
   for model in getattr(bundle, 'models', []) or []:
-    for artifact in (getattr(model, 'artifact', None), getattr(model, 'metadata', None)):
-      if artifact and getattr(artifact, 'fileName', None) and getattr(artifact, 'downloadUri', None):
-        sha256 = getattr(artifact.downloadUri, 'sha256', None)
-        if sha256:
-          artifacts.append((artifact.fileName, sha256))
+    for artifact in (getattr(model, 'artifact', None),):
+      if artifact and getattr(artifact, 'fileName', None):
+        if len(artifact.chunks) > 0:
+          for i, chunk in enumerate(artifact.chunks):
+            chunk_name = get_chunk_name(artifact.fileName, i, len(artifact.chunks))
+            if getattr(chunk, 'sha256', None):
+              artifacts.append((chunk_name, chunk.sha256))
+        else:
+          if getattr(artifact, 'downloadUri', None):
+            sha256 = getattr(artifact.downloadUri, 'sha256', None)
+            if sha256:
+              artifacts.append((artifact.fileName, sha256))
   return artifacts
 
 
@@ -156,8 +164,7 @@ def _get_model():
 
 
 def load_metadata():
-  model = _get_model()
-  metadata_path = f"{CUSTOM_MODEL_PATH}/{model.metadata.fileName}" if model else METADATA_PATH
+  metadata_path = METADATA_PATH
 
   with open(metadata_path, 'rb') as f:
     return pickle.load(f)

openpilot/sunnypilot/models/manager.py

@@ -38,11 +38,11 @@ class ModelManagerSP:
     if not self.selected_bundle:
       return
     for model in self.selected_bundle.models:
-      for artifact in (model.artifact, model.metadata):
-        if artifact is not source_artifact and artifact.fileName == source_artifact.fileName:
-          artifact.downloadProgress.status = source_artifact.downloadProgress.status
-          artifact.downloadProgress.progress = source_artifact.downloadProgress.progress
-          artifact.downloadProgress.eta = source_artifact.downloadProgress.eta
+      artifact = model.artifact
+      if artifact is not source_artifact and artifact.fileName == source_artifact.fileName:
+        artifact.downloadProgress.status = source_artifact.downloadProgress.status
+        artifact.downloadProgress.progress = source_artifact.downloadProgress.progress
+        artifact.downloadProgress.eta = source_artifact.downloadProgress.eta
 
   def _calculate_eta(self, filename: str, progress: float) -> int:
     """Calculate ETA based on elapsed time and current progress"""
@@ -136,7 +136,22 @@ class ModelManagerSP:
     full_path = os.path.join(destination_path, filename)
 
     try:
-      if await verify_file(full_path, expected_hash):
+      is_cached = False
+      if len(artifact.chunks) > 0:
+        from openpilot.common.file_chunker import get_chunk_name
+        chunks_valid = True
+        for i, chunk in enumerate(artifact.chunks):
+          chunk_path = get_chunk_name(full_path, i, len(artifact.chunks))
+          if not await verify_file(chunk_path, chunk.sha256):
+            chunks_valid = False
+            break
+        if chunks_valid and len(artifact.chunks) > 0:
+          is_cached = True
+      else:
+        if await verify_file(full_path, expected_hash):
+          is_cached = True
+
+      if is_cached:
         artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached
         artifact.downloadProgress.progress = 100
         artifact.downloadProgress.eta = 0
@@ -146,11 +161,15 @@ class ModelManagerSP:
 
       if len(artifact.chunks) > 0:
         await self._download_chunked(url, full_path, artifact)
+        from openpilot.common.file_chunker import get_chunk_name
+        for i, chunk in enumerate(artifact.chunks):
+          chunk_path = get_chunk_name(full_path, i, len(artifact.chunks))
+          if not await verify_file(chunk_path, chunk.sha256):
+            raise ValueError(f"Hash validation failed for chunk {i+1} of {filename}")
       else:
         await self._download_file(url, full_path, artifact)
-
-      if not await verify_file(full_path, expected_hash):
-        raise ValueError(f"Hash validation failed for {filename}")
+        if not await verify_file(full_path, expected_hash):
+          raise ValueError(f"Hash validation failed for {filename}")
 
       artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloaded
       artifact.downloadProgress.progress = 100
@@ -198,16 +217,16 @@ class ModelManagerSP:
     try:
       seen_artifacts: set[str] = set()
       for model in self.selected_bundle.models:
-        for artifact in (model.metadata, model.artifact):
-          if not artifact.fileName:
-            continue
-          if artifact.fileName in seen_artifacts:
-            artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached
-            artifact.downloadProgress.progress = 100
-            artifact.downloadProgress.eta = 0
-          else:
-            seen_artifacts.add(artifact.fileName)
-            await self._process_artifact(artifact, destination_path)
+        artifact = model.artifact
+        if not artifact.fileName:
+          continue
+        if artifact.fileName in seen_artifacts:
+          artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached
+          artifact.downloadProgress.progress = 100
+          artifact.downloadProgress.eta = 0
+        else:
+          seen_artifacts.add(artifact.fileName)
+          await self._process_artifact(artifact, destination_path)
 
       self.active_bundle = self.selected_bundle
       self.active_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded
@@ -268,8 +287,6 @@ class ModelManagerSP:
       for model in self.active_bundle.models:
         if hasattr(model, 'artifact') and model.artifact.fileName:
           active_files.append(model.artifact.fileName)
-        if hasattr(model, 'metadata') and model.metadata.fileName:
-          active_files.append(model.metadata.fileName)
 
     # Remove all files except active ones (including their chunk files)
     model_dir = Paths.model_root()