Add AGNOS download mirrors

system/hardware/tici/agnos.json

@@ -57,6 +57,9 @@
   {
     "name": "boot",
     "url": "https://files.firestar.link/x/ugiq4cqx08q7/boot9.img.xz",
+    "fallback_urls": [
+      "https://files-east.firestar.link/x/npthb0hzvtxx/boot9.img.xz"
+    ],
     "hash": "ab2eba0f96b2f48efa376330c3eb509158361adf3ad9c20f269ec92457aa841f",
     "hash_raw": "ab2eba0f96b2f48efa376330c3eb509158361adf3ad9c20f269ec92457aa841f",
     "size": 48343040,
@@ -68,6 +71,9 @@
   {
     "name": "system",
     "url": "https://files.firestar.link/x/07530yj1jd6a/system18.img.xz",
+    "fallback_urls": [
+      "https://files-east.firestar.link/x/5pmqh33uqrob/system18.img.xz"
+    ],
     "hash": "01c84930849f9be2bdbad5e9a8dda3a6fd2be95e81d1b3556574b18346934f49",
     "hash_raw": "01c84930849f9be2bdbad5e9a8dda3a6fd2be95e81d1b3556574b18346934f49",
     "size": 4718592000,

system/hardware/tici/agnos.py

@@ -16,6 +16,8 @@ SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
 CAIBX_URL = "https://commadist.azureedge.net/agnosupdate/"
 
 AGNOS_MANIFEST_FILE = "system/hardware/tici/agnos.json"
+PRIMARY_DOWNLOAD_ATTEMPTS = 3
+MAX_DOWNLOAD_ATTEMPTS = 10
 
 
 class StreamingDecompressor:
@@ -54,6 +56,11 @@ class StreamingDecompressor:
     return result
 
 
+def get_partition_download_url(partition: dict, attempt: int) -> str:
+  urls = [partition['url'], *partition.get('fallback_urls', [])]
+  return urls[min(attempt // PRIMARY_DOWNLOAD_ATTEMPTS, len(urls) - 1)]
+
+
 def unsparsify(f: StreamingDecompressor) -> Generator[bytes, None, None]:
   # https://source.android.com/devices/bootloader/images#sparse-format
   magic = struct.unpack("I", f.read(4))[0]
@@ -258,7 +265,8 @@ def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalo
 
 
 def swap(manifest_path: str, target_slot_number: int, cloudlog) -> None:
-  update = json.load(open(manifest_path))
+  with open(manifest_path) as manifest:
+    update = json.load(manifest)
   for partition in update:
     if not partition.get('full_check', False):
       clear_partition_hash(target_slot_number, partition)
@@ -273,7 +281,8 @@ def swap(manifest_path: str, target_slot_number: int, cloudlog) -> None:
 
 
 def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, standalone=False) -> None:
-  update = json.load(open(manifest_path))
+  with open(manifest_path) as manifest:
+    update = json.load(manifest)
 
   cloudlog.info(f"Target slot {target_slot_number}")
 
@@ -283,15 +292,20 @@ def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, st
   for partition in update:
     success = False
 
-    for retries in range(10):
+    for attempt in range(MAX_DOWNLOAD_ATTEMPTS):
+      download_partition = dict(partition)
+      download_partition['url'] = get_partition_download_url(partition, attempt)
+
       try:
-        flash_partition(target_slot_number, partition, cloudlog, standalone)
+        if download_partition['url'] != partition['url']:
+          cloudlog.info(f"Using fallback mirror for {partition['name']}: {download_partition['url']}")
+        flash_partition(target_slot_number, download_partition, cloudlog, standalone)
         success = True
         break
 
       except requests.exceptions.RequestException:
         cloudlog.exception("Failed")
-        cloudlog.info(f"Failed to download {partition['name']}, retrying ({retries})")
+        cloudlog.info(f"Failed to download {partition['name']}, retrying ({attempt + 1}/{MAX_DOWNLOAD_ATTEMPTS})")
         time.sleep(10)
 
     if not success:
@@ -302,7 +316,8 @@ def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, st
 
 
 def verify_agnos_update(manifest_path: str, target_slot_number: int) -> bool:
-  update = json.load(open(manifest_path))
+  with open(manifest_path) as manifest:
+    update = json.load(manifest)
   return all(verify_partition(target_slot_number, partition) for partition in update)
 
 

system/hardware/tici/tests/test_agnos_updater.py

@@ -1,11 +1,23 @@
 import json
 import os
+
+import pytest
 import requests
 
+from openpilot.system.hardware.tici import agnos
+
 TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)))
 MANIFEST = os.path.join(TEST_DIR, "../agnos.json")
 
 
+class FakeCloudlog:
+  def info(self, *_args):
+    pass
+
+  def exception(self, *_args):
+    pass
+
+
 class TestAgnosUpdater:
 
   def test_manifest(self):
@@ -13,8 +25,77 @@ class TestAgnosUpdater:
       m = json.load(f)
 
     for img in m:
-      r = requests.head(img['url'], timeout=10)
-      r.raise_for_status()
-      assert r.headers['Content-Type'] == "application/x-xz"
+      content_lengths = set()
+      for url in [img['url'], *img.get('fallback_urls', [])]:
+        r = requests.head(url, timeout=10)
+        r.raise_for_status()
+        assert r.headers['Content-Type'] == "application/x-xz"
+        content_lengths.add(r.headers.get('Content-Length'))
+
+      assert len(content_lengths) == 1
       if not img['sparse']:
         assert img['hash'] == img['hash_raw']
+
+  def test_download_url_uses_primary_for_first_three_attempts(self):
+    partition = {
+      "url": "https://primary/image.xz",
+      "fallback_urls": ["https://fallback/image.xz"],
+    }
+
+    assert [agnos.get_partition_download_url(partition, i) for i in range(5)] == [
+      "https://primary/image.xz",
+      "https://primary/image.xz",
+      "https://primary/image.xz",
+      "https://fallback/image.xz",
+      "https://fallback/image.xz",
+    ]
+
+  def test_network_failures_switch_to_fallback(self, monkeypatch, tmp_path):
+    partition = {
+      "name": "system",
+      "url": "https://primary/image.xz",
+      "fallback_urls": ["https://fallback/image.xz"],
+    }
+    manifest = tmp_path / "agnos.json"
+    manifest.write_text(json.dumps([partition]))
+    attempted_urls = []
+
+    def flash_partition(_slot, attempted_partition, _cloudlog, _standalone):
+      attempted_urls.append(attempted_partition['url'])
+      if len(attempted_urls) <= agnos.PRIMARY_DOWNLOAD_ATTEMPTS:
+        raise requests.exceptions.ReadTimeout()
+
+    monkeypatch.setattr(agnos, "flash_partition", flash_partition)
+    monkeypatch.setattr(agnos.os, "system", lambda *_args: 0)
+    monkeypatch.setattr(agnos.time, "sleep", lambda *_args: None)
+
+    agnos.flash_agnos_update(str(manifest), 1, FakeCloudlog())
+
+    assert attempted_urls == [
+      "https://primary/image.xz",
+      "https://primary/image.xz",
+      "https://primary/image.xz",
+      "https://fallback/image.xz",
+    ]
+
+  def test_integrity_failures_do_not_switch_mirrors(self, monkeypatch, tmp_path):
+    partition = {
+      "name": "system",
+      "url": "https://primary/image.xz",
+      "fallback_urls": ["https://fallback/image.xz"],
+    }
+    manifest = tmp_path / "agnos.json"
+    manifest.write_text(json.dumps([partition]))
+    attempted_urls = []
+
+    def flash_partition(_slot, attempted_partition, _cloudlog, _standalone):
+      attempted_urls.append(attempted_partition['url'])
+      raise ValueError("hash mismatch")
+
+    monkeypatch.setattr(agnos, "flash_partition", flash_partition)
+    monkeypatch.setattr(agnos.os, "system", lambda *_args: 0)
+
+    with pytest.raises(ValueError, match="hash mismatch"):
+      agnos.flash_agnos_update(str(manifest), 1, FakeCloudlog())
+
+    assert attempted_urls == ["https://primary/image.xz"]