diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index 655c4163a3..eda1117a2a 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -64,8 +64,6 @@ class ModelParser: model.type = model_data.get("type") model.artifact = ModelParser._parse_artifact(model_data.get("artifact", {})) - if metadata := model_data.get("metadata"): - model.metadata = ModelParser._parse_artifact(metadata) return model @staticmethod @@ -211,5 +209,3 @@ if __name__ == "__main__": # Print metadata details if model.artifact.chunks: print(f"Contains {len(model.artifact.chunks)} chunks.") - if hasattr(model, 'metadata') and model.metadata and model.metadata.fileName: - print(f"Metadata: {model.metadata.fileName}, Download URI: {model.metadata.downloadUri.uri}") diff --git a/openpilot/sunnypilot/models/helpers.py b/openpilot/sunnypilot/models/helpers.py index ad1333d4ed..101d8d196e 100644 --- a/openpilot/sunnypilot/models/helpers.py +++ b/openpilot/sunnypilot/models/helpers.py @@ -18,7 +18,7 @@ from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRai from openpilot.common.hardware.hw import Paths # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO -REQUIRED_JSON_VERSION = 15 +REQUIRED_JSON_VERSION = 16 CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' @@ -56,12 +56,20 @@ def is_bundle_version_compatible(bundle: dict) -> bool: def _bundle_artifacts(bundle: custom.ModelManagerSP.ModelBundle) -> list[tuple[str, str]]: artifacts = [] + from openpilot.common.file_chunker import get_chunk_name for model in getattr(bundle, 'models', []) or []: - for artifact in (getattr(model, 'artifact', None), getattr(model, 'metadata', None)): - if artifact and getattr(artifact, 'fileName', None) and getattr(artifact, 'downloadUri', None): - sha256 = getattr(artifact.downloadUri, 'sha256', None) - if sha256: - artifacts.append((artifact.fileName, sha256)) + for artifact in (getattr(model, 'artifact', None),): + if artifact and getattr(artifact, 'fileName', None): + if len(artifact.chunks) > 0: + for i, chunk in enumerate(artifact.chunks): + chunk_name = get_chunk_name(artifact.fileName, i, len(artifact.chunks)) + if getattr(chunk, 'sha256', None): + artifacts.append((chunk_name, chunk.sha256)) + else: + if getattr(artifact, 'downloadUri', None): + sha256 = getattr(artifact.downloadUri, 'sha256', None) + if sha256: + artifacts.append((artifact.fileName, sha256)) return artifacts @@ -156,8 +164,7 @@ def _get_model(): def load_metadata(): - model = _get_model() - metadata_path = f"{CUSTOM_MODEL_PATH}/{model.metadata.fileName}" if model else METADATA_PATH + metadata_path = METADATA_PATH with open(metadata_path, 'rb') as f: return pickle.load(f) diff --git a/openpilot/sunnypilot/models/manager.py b/openpilot/sunnypilot/models/manager.py index 6b04dcd04c..d742e968a9 100644 --- a/openpilot/sunnypilot/models/manager.py +++ b/openpilot/sunnypilot/models/manager.py @@ -38,11 +38,11 @@ class ModelManagerSP: if not self.selected_bundle: return for model in self.selected_bundle.models: - for artifact in (model.artifact, model.metadata): - if artifact is not source_artifact and artifact.fileName == source_artifact.fileName: - artifact.downloadProgress.status = source_artifact.downloadProgress.status - artifact.downloadProgress.progress = source_artifact.downloadProgress.progress - artifact.downloadProgress.eta = source_artifact.downloadProgress.eta + artifact = model.artifact + if artifact is not source_artifact and artifact.fileName == source_artifact.fileName: + artifact.downloadProgress.status = source_artifact.downloadProgress.status + artifact.downloadProgress.progress = source_artifact.downloadProgress.progress + artifact.downloadProgress.eta = source_artifact.downloadProgress.eta def _calculate_eta(self, filename: str, progress: float) -> int: """Calculate ETA based on elapsed time and current progress""" @@ -136,7 +136,22 @@ class ModelManagerSP: full_path = os.path.join(destination_path, filename) try: - if await verify_file(full_path, expected_hash): + is_cached = False + if len(artifact.chunks) > 0: + from openpilot.common.file_chunker import get_chunk_name + chunks_valid = True + for i, chunk in enumerate(artifact.chunks): + chunk_path = get_chunk_name(full_path, i, len(artifact.chunks)) + if not await verify_file(chunk_path, chunk.sha256): + chunks_valid = False + break + if chunks_valid and len(artifact.chunks) > 0: + is_cached = True + else: + if await verify_file(full_path, expected_hash): + is_cached = True + + if is_cached: artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached artifact.downloadProgress.progress = 100 artifact.downloadProgress.eta = 0 @@ -146,11 +161,15 @@ class ModelManagerSP: if len(artifact.chunks) > 0: await self._download_chunked(url, full_path, artifact) + from openpilot.common.file_chunker import get_chunk_name + for i, chunk in enumerate(artifact.chunks): + chunk_path = get_chunk_name(full_path, i, len(artifact.chunks)) + if not await verify_file(chunk_path, chunk.sha256): + raise ValueError(f"Hash validation failed for chunk {i+1} of {filename}") else: await self._download_file(url, full_path, artifact) - - if not await verify_file(full_path, expected_hash): - raise ValueError(f"Hash validation failed for {filename}") + if not await verify_file(full_path, expected_hash): + raise ValueError(f"Hash validation failed for {filename}") artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloaded artifact.downloadProgress.progress = 100 @@ -198,16 +217,16 @@ class ModelManagerSP: try: seen_artifacts: set[str] = set() for model in self.selected_bundle.models: - for artifact in (model.metadata, model.artifact): - if not artifact.fileName: - continue - if artifact.fileName in seen_artifacts: - artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached - artifact.downloadProgress.progress = 100 - artifact.downloadProgress.eta = 0 - else: - seen_artifacts.add(artifact.fileName) - await self._process_artifact(artifact, destination_path) + artifact = model.artifact + if not artifact.fileName: + continue + if artifact.fileName in seen_artifacts: + artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached + artifact.downloadProgress.progress = 100 + artifact.downloadProgress.eta = 0 + else: + seen_artifacts.add(artifact.fileName) + await self._process_artifact(artifact, destination_path) self.active_bundle = self.selected_bundle self.active_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded @@ -268,8 +287,6 @@ class ModelManagerSP: for model in self.active_bundle.models: if hasattr(model, 'artifact') and model.artifact.fileName: active_files.append(model.artifact.fileName) - if hasattr(model, 'metadata') and model.metadata.fileName: - active_files.append(model.metadata.fileName) # Remove all files except active ones (including their chunk files) model_dir = Paths.model_root()