OS Updater improvements (#19914)

* Clear hashes before swapping

* add retry logic

* better retry logic

* actually fail on retries exceeded
old-commit-hash: fe91b8a1b4c13a8274d2f0e50ac2f402b9f00efe

launch_chffrplus.sh

@@ -146,6 +146,12 @@ function tici_init {
 
     if [[ "$SYSTEM_HASH" == "$SYSTEM_HASH_EXPECTED" && "$BOOT_HASH" == "$BOOT_HASH_EXPECTED" ]]; then
       echo "Swapping active slot to $OTHER_SLOT_NUMBER"
+
+      # Clean hashes before swapping to prevent looping
+      dd if=/dev/zero of="/dev/disk/by-partlabel/system$OTHER_SLOT" bs=1 seek="$SYSTEM_SIZE" count=64
+      dd if=/dev/zero of="/dev/disk/by-partlabel/boot$OTHER_SLOT" bs=1 seek="$BOOT_SIZE" count=64
+      sync
+
       abctl --set_active "$OTHER_SLOT_NUMBER"
 
       sleep 1
@@ -154,6 +160,12 @@ function tici_init {
       echo "Hash mismatch, downloading agnos"
       if $DIR/selfdrive/hardware/tici/agnos.py $MANIFEST; then
         echo "Download done, swapping active slot to $OTHER_SLOT_NUMBER"
+
+        # Clean hashes before swapping to prevent looping
+        dd if=/dev/zero of="/dev/disk/by-partlabel/system$OTHER_SLOT" bs=1 seek="$SYSTEM_SIZE" count=64
+        dd if=/dev/zero of="/dev/disk/by-partlabel/boot$OTHER_SLOT" bs=1 seek="$BOOT_SIZE" count=64
+        sync
+
         abctl --set_active "$OTHER_SLOT_NUMBER"
       fi
 

selfdrive/hardware/tici/agnos.py

@@ -76,6 +76,48 @@ def unsparsify(f):
       raise Exception("Unhandled sparse chunk type")
 
 
+def flash_partition(cloudlog, spinner, target_slot, partition):
+  cloudlog.info(f"Downloading and writing {partition['name']}")
+
+  downloader = StreamingDecompressor(partition['url'])
+  with open(f"/dev/disk/by-partlabel/{partition['name']}{target_slot}", 'wb') as out:
+    partition_size = partition['size']
+    # Clear hash before flashing
+    out.seek(partition_size)
+    out.write(b"\x00" * 64)
+    out.seek(0)
+    os.sync()
+
+    # Flash partition
+    if partition['sparse']:
+      raw_hash = hashlib.sha256()
+      for chunk in unsparsify(downloader):
+        raw_hash.update(chunk)
+        out.write(chunk)
+
+        if spinner is not None:
+          spinner.update_progress(out.tell(), partition_size)
+
+      if raw_hash.hexdigest().lower() != partition['hash_raw'].lower():
+        raise Exception(f"Unsparse hash mismatch '{raw_hash.hexdigest().lower()}'")
+    else:
+      while not downloader.eof:
+        out.write(downloader.read(1024 * 1024))
+
+        if spinner is not None:
+          spinner.update_progress(out.tell(), partition_size)
+
+    if downloader.sha256.hexdigest().lower() != partition['hash'].lower():
+      raise Exception("Uncompressed hash mismatch")
+
+    if out.tell() != partition['size']:
+      raise Exception("Uncompressed size mismatch")
+
+    # Write hash after successfull flash
+    os.sync()
+    out.write(partition['hash_raw'].lower().encode())
+
+
 def flash_agnos_update(manifest_path, cloudlog, spinner=None):
   update = json.load(open(manifest_path))
 
@@ -89,45 +131,22 @@ def flash_agnos_update(manifest_path, cloudlog, spinner=None):
   os.system(f"abctl --set_unbootable {target_slot_number}")
 
   for partition in update:
-    cloudlog.info(f"Downloading and writing {partition['name']}")
+    success = False
 
-    downloader = StreamingDecompressor(partition['url'])
-    with open(f"/dev/disk/by-partlabel/{partition['name']}{target_slot}", 'wb') as out:
-      partition_size = partition['size']
-      # Clear hash before flashing
-      out.seek(partition_size)
-      out.write(b"\x00" * 64)
-      out.seek(0)
-      os.sync()
+    for retries in range(10):
+      try:
+        flash_partition(cloudlog, spinner, target_slot, partition)
+        success = True
+        break
 
-      # Flash partition
-      if partition['sparse']:
-        raw_hash = hashlib.sha256()
-        for chunk in unsparsify(downloader):
-          raw_hash.update(chunk)
-          out.write(chunk)
+      except requests.exceptions.RequestException:
+        spinner.update("Waiting for internet...")
+        cloudlog.info(f"Failed to download {partition['name']}, retrying ({retries})")
+        time.sleep(10)
 
-          if spinner is not None:
-            spinner.update_progress(out.tell(), partition_size)
-
-        if raw_hash.hexdigest().lower() != partition['hash_raw'].lower():
-          raise Exception(f"Unsparse hash mismatch '{raw_hash.hexdigest().lower()}'")
-      else:
-        while not downloader.eof:
-          out.write(downloader.read(1024 * 1024))
-
-          if spinner is not None:
-            spinner.update_progress(out.tell(), partition_size)
-
-      if downloader.sha256.hexdigest().lower() != partition['hash'].lower():
-        raise Exception("Uncompressed hash mismatch")
-
-      if out.tell() != partition['size']:
-        raise Exception("Uncompressed size mismatch")
-
-      # Write hash after successfull flash
-      os.sync()
-      out.write(partition['hash_raw'].lower().encode())
+    if not success:
+      cloudlog.info(f"Failed to flash {partition['name']}, aborting")
+      raise Exception("Maximum retries exceeded")
 
   cloudlog.info(f"AGNOS ready on slot {target_slot}")