Remember unchanged model verification across restarts

openpilot/starpilot/models/manager.py

@@ -296,6 +296,7 @@ class ModelManager:
     self.download_all = False
     self.job_id = ""
     self.checked: dict[str, tuple[tuple, bool]] = {}
+    self.snapshot_checked: dict[str, tuple[tuple, bool]] = {}
     self.verify_queue: deque[tuple[str, dict, tuple, str]] = deque()
     self.verifying: dict[str, tuple] = {}
     self.verify_worker: threading.Thread | None = None
@@ -417,6 +418,17 @@ class ModelManager:
       self.checked[key] = (signature, verified_artifact(mid, entries, self.root, variant) is not None)
     return self.checked[key][1]
 
+  def _verification_receipt(self, mid: str, variant: str) -> Path:
+    return self.root / ".verified" / f"{mid}-{variant}.json"
+
+  def _remember_verification(self, mid: str, variant: str, signature: tuple) -> None:
+    if len(signature) != 8:
+      return
+    try:
+      atomic_json(self._verification_receipt(mid, variant), {"version": 1, "signature": list(signature)})
+    except OSError:
+      logging.exception("Could not save model verification for %s", mid)
+
   def _snapshot_installed(self, mid: str, row: dict, variant: str = "standard") -> tuple[bool, bool]:
     if mid == DEFAULT_SMALL and variant == "standard" and self._signature(mid, row, variant) is None:
       row = {**row, "shipped": True, "artifact_sha256": DEFAULT_SMALL_SHA256, "artifact_size": DEFAULT_SMALL_SIZE}
@@ -430,6 +442,16 @@ class ModelManager:
     checked = self.checked.get(key)
     if checked is not None and checked[0] == signature:
       return checked[1], False
+    if len(signature) == 8:
+      if self.snapshot_checked.get(key, (None,))[0] != signature:
+        try:
+          receipt = read_json(self._verification_receipt(mid, variant), 4096)
+          valid = receipt == {"version": 1, "signature": list(signature)}
+        except (OSError, ValueError, TypeError):
+          valid = False
+        self.snapshot_checked[key] = (signature, valid)
+      if self.snapshot_checked[key][1]:
+        return True, False
     if self.verify_worker is not None and not self.verify_worker.is_alive():
       self.verify_worker = None
       self.verifying.clear()
@@ -462,6 +484,8 @@ class ModelManager:
           self.verifying.pop(key)
           if self._signature(mid, artifact_entry(mid, {mid: row}, variant), variant) == signature:
             self.checked[key] = (signature, valid)
+            if valid:
+              self._remember_verification(mid, variant, signature)
 
   def selected_gpu_status(self) -> tuple[bool, bool]:
     """Verify the selected download asynchronously, without starting a model or download job."""

openpilot/starpilot/models/tests/test_manager.py

@@ -180,6 +180,68 @@ class ModelManagerTest(unittest.TestCase):
     self.assertFalse(self.model_status()['installed'])
     self.assertFalse(self.wait_for_check()['installed'])
 
+  def test_verification_survives_manager_restart_but_selection_still_hashes(self):
+    self.download()
+    self.assertTrue(self.wait_for_check()['installed'])
+    self.manager.close()
+    self.manager = ModelManager(root=self.root, parked=lambda: True, gpu_present=lambda: True)
+    self.addCleanup(self.manager.close)
+    with patch('openpilot.starpilot.models.manager.verified_artifact', return_value=None) as verify:
+      row = self.model_status()
+      self.assertTrue(row['selectable'])
+      self.assertFalse(row['checking'])
+      verify.assert_not_called()
+      with self.assertRaises(ModelError):
+        self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'})
+      verify.assert_called()
+
+  def test_persisted_verification_invalidates_file_and_catalog_changes(self):
+    self.download()
+    self.assertTrue(self.wait_for_check()['installed'])
+    self.manager.close()
+    path = self.root / 'gwm8223/gwm8223_driving_tinygrad.pkl'
+    original_stat = path.stat()
+    path.write_bytes(b'x' * len(self.data))
+    import os
+    os.utime(path, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns))
+    self.manager = ModelManager(root=self.root, parked=lambda: True, gpu_present=lambda: True)
+    self.addCleanup(self.manager.close)
+    self.assertFalse(self.wait_for_check()['selectable'])
+    self.assertEqual(resolve_runtime(True, root=self.root).small_id, DEFAULT_SMALL)
+    row = catalog(self.root)['gwm8223']
+    row = {**row, 'artifact_sha256': '0' * 64}
+    installed, _ = self.manager._snapshot_installed('gwm8223', row)
+    self.assertFalse(installed)
+
+  def test_persisted_verification_rejects_changed_catalog_for_unchanged_file(self):
+    self.download()
+    self.assertTrue(self.wait_for_check()['installed'])
+    self.manager.close()
+    self.manager = ModelManager(root=self.root, parked=lambda: True, gpu_present=lambda: True)
+    self.addCleanup(self.manager.close)
+    row = {**catalog(self.root)['gwm8223'], 'artifact_sha256': '0' * 64}
+    with patch('openpilot.starpilot.models.manager.verified_artifact', return_value=None) as verify:
+      installed, _ = self.manager._snapshot_installed('gwm8223', row)
+      self.assertFalse(installed)
+      worker = self.manager.verify_worker
+      if worker is not None:
+        worker.join(2)
+        self.assertFalse(worker.is_alive())
+      self.manager.close()
+      verify.assert_called()
+
+  def test_invalid_verification_receipt_rehashes(self):
+    self.download()
+    self.assertTrue(self.wait_for_check()['installed'])
+    self.manager.close()
+    (self.root / '.verified/gwm8223-standard.json').write_text('{broken')
+    self.manager = ModelManager(root=self.root, parked=lambda: True, gpu_present=lambda: True)
+    self.addCleanup(self.manager.close)
+    from openpilot.starpilot.models.manager import verified_artifact
+    with patch('openpilot.starpilot.models.manager.verified_artifact', wraps=verified_artifact) as verify:
+      self.assertTrue(self.wait_for_check()['installed'])
+      verify.assert_called()
+
   def test_snapshot_does_not_wait_for_artifact_hash(self):
     self.download()
     self.manager.checked.clear()

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "966ef1025076c746a00053f58afbf63197dc24709fe0b17de5ec05a7829df616", "schema_version": 1}
+{"protected_inventory_sha256": "cc56278d30518c45d2a759b43335be5e40755c5cd1184e4d45ed868d22ab7de3", "schema_version": 1}