Check offline Firefox against AGNOS

openpilot/tools/arabesque_firefox/README.md

@@ -521,6 +521,18 @@ Preparation requires root, ARM64 AGNOS `19.8.1`, `20.2-starpilot.1` or `20.2.1`,
 
 The `offline/` directory ships the original archives named by digest. The installer copies them into the private package cache and checks each size and digest before extraction. For a new pinned release, refresh the directory with `python3 openpilot/tools/arabesque_firefox/setup.py --stage-offline` and include the resulting archives in the device checkout. Packages absent from a checkout can still be fetched from their pinned HTTPS URLs during setup or an explicit update.
 
+### Check a new AGNOS image or Firefox bundle
+
+From the repo root on a Linux development machine with Python 3.12+, `dpkg-deb`, `glib-compile-schemas`, `debugfs`, and `simg2img`, run:
+
+```bash
+sudo /usr/bin/python3 -m openpilot.tools.arabesque_firefox.check_agnos
+```
+
+On x86-64, also install `qemu-user-static` for `qemu-aarch64-static`. Allow about 10 GiB of temporary space. Set `TMPDIR` via `sudo env TMPDIR=/path/to/scratch` if the default location is too small. The command downloads the pinned system image. Use `--image /path/to/system.img.xz` to reuse a local copy, or `--manifest openpilot/common/hardware/comma/agnos-c3.json` to check the C3 manifest if its system image differs.
+
+The check builds from the shipped archives with an empty cache and package downloads disabled. It verifies the decompressed system image against the manifest, then checks the browser and display libraries inside that image. Keep the existing pins if it passes. Use a parked comma to verify launch, rendering, input, and profile preservation before qualifying a new AGNOS version.
+
 ### Force the reference and SHM modes
 
 After preparation has completed, close the Firefox tool and run one diagnostic at a time from the checkout:

openpilot/tools/arabesque_firefox/check_agnos.py

@@ -0,0 +1,107 @@
+"""Check the shipped offline Firefox bundle against a pinned AGNOS system image."""
+
+import argparse
+import hashlib
+import json
+import lzma
+import os
+import platform
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from openpilot.tools.arabesque_firefox import setup
+from tools.laptop_device_build.extract_sysroot_from_agnos import download, ensure_debugfs_readable_image
+
+
+def check_loader(root, runtime, chroot, emulator):
+  multiarch = 'lib/aarch64-linux-gnu'
+  common = runtime / 'usr' / multiarch
+  display = runtime / 'display/usr'
+  system = Path('/usr') / multiarch
+  x_path = f'{display / multiarch}:{common}'
+  checks = [
+    (display / 'bin/Xvfb', x_path),
+    (display / 'bin/xauth', x_path),
+    (runtime / 'isolation/xkb/usr/bin/xkbcomp', x_path),
+    (runtime / 'firefox/firefox', str(common)),
+    # Gecko loads its sibling libraries before libxul. The standalone loader needs their directory.
+    (runtime / 'firefox/libxul.so', f'{runtime / "firefox"}:{common}'),
+    (common / 'libgtk-3.so.0', str(common)),
+    (common / 'libxcb-damage.so.0', str(common)),
+    (common / 'libxcb-xtest.so.0', str(common)),
+  ]
+  for name in ('libxcb.so.1', 'libxcb-shm.so.0'):
+    library = common / name
+    checks.append((library if os.path.lexists(root / library.relative_to('/')) else system / name, str(common)))
+
+  if emulator:
+    shutil.copyfile(emulator, root / 'arabesque-check-qemu')
+    (root / 'arabesque-check-qemu').chmod(0o755)
+  command = [chroot, f'--userspec=+{setup.BROWSER_UID}:+{setup.BROWSER_UID}', '--groups=', str(root)]
+  if emulator:
+    command.append('/arabesque-check-qemu')
+  command += ['/lib/ld-linux-aarch64.so.1', '--inhibit-cache']
+  for binary, libraries in checks:
+    result = subprocess.run([*command, '--library-path', libraries, '--list', str(binary)],
+                            env={'LC_ALL': 'C'}, capture_output=True, text=True, timeout=30)
+    if result.returncode or '=> not found' in result.stdout:
+      raise RuntimeError(f'AGNOS loader failed for {binary.name}:\n{result.stderr}{result.stdout}')
+    print(f'AGNOS loader OK for {binary.name}', flush=True)
+
+
+def main():
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument('--manifest', type=Path, default=Path('openpilot/common/hardware/comma/agnos.json'))
+  parser.add_argument('--image', type=Path, help='Use a local copy of the compressed system image (.img.xz).')
+  args = parser.parse_args()
+  if os.geteuid() != 0:
+    parser.error('Run with sudo on a Linux development machine.')
+  for tool in ('chroot', 'debugfs', 'simg2img', 'dpkg-deb', 'glib-compile-schemas'):
+    if shutil.which(tool) is None:
+      parser.error(f'{tool} is required.')
+  emulator = None
+  if platform.machine() != 'aarch64':
+    emulator = shutil.which('qemu-aarch64-static')
+    if emulator is None:
+      parser.error('qemu-aarch64-static is required on non-ARM64 hosts.')
+
+  system, = (entry for entry in json.loads(args.manifest.read_text()) if entry['name'] == 'system')
+  target = setup.approved()
+  with tempfile.TemporaryDirectory(prefix='arabesque-agnos-') as directory:
+    work = Path(directory)
+    setup.ROOT = work / 'bundle'
+    setup.ROOT.mkdir()
+    staging = setup.ROOT / 'runtime'
+    setup.build(target, staging, offline_only=True)
+
+    compressed = args.image
+    if compressed is None:
+      compressed = work / 'system.img.xz'
+      download(system['url'], compressed)
+    image = work / 'system.img'
+    with lzma.open(compressed, 'rb') as source, image.open('wb') as dest:
+      shutil.copyfileobj(source, dest)
+    with image.open('rb') as stream:
+      if hashlib.file_digest(stream, 'sha256').hexdigest() != system['hash'].lower():
+        raise ValueError(f'AGNOS system image checksum mismatch: {compressed}')
+
+    filesystem = ensure_debugfs_readable_image(image, work)
+    print('Extracting the verified AGNOS filesystem…', flush=True)
+    root = work / 'root'
+    root.mkdir()
+    # A real chroot keeps even absolute AGNOS symlinks and loader fallback paths away from the host.
+    result = subprocess.run(['debugfs', '-R', 'rdump / .', str(filesystem.resolve())],
+                            cwd=root, capture_output=True, text=True, check=True)
+    if result.stderr.splitlines()[1:]:
+      raise RuntimeError(result.stderr.strip())
+    runtime = Path('/arabesque-check-runtime')
+    staging.rename(root / runtime.relative_to('/'))
+    check_loader(root, runtime, shutil.which('chroot'), emulator)
+    version = (root / 'VERSION').read_text().strip()
+    print(f'Offline Firefox {target["version"]} revision {target["bundle_revision"]} passed on AGNOS {version}')
+
+
+if __name__ == '__main__':
+  main()

openpilot/tools/arabesque_firefox/setup.py

@@ -202,7 +202,7 @@ def copy_package(package, response, partial, label):
       report(f'{label}… {100 * (package["size"] - remaining) // package["size"]}%')
 
 
-def download(package, cache, *, cache_uid=0):
+def download(package, cache, *, cache_uid=0, offline_only=False):
   digest = package.get('sha512', package.get('sha256'))
   path = cache / (digest + '-' + package['name'])
   legacy = cache / package['name']
@@ -225,6 +225,8 @@ def download(package, cache, *, cache_uid=0):
       try:
         fd = os.open(bundled, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
       except FileNotFoundError:
+        if offline_only:
+          raise FileNotFoundError(f'Missing offline Firefox archive: {bundled.name}') from None
         with urllib.request.urlopen(package['url'], timeout=30) as response:
           if not response.url.startswith('https://'):
             raise ValueError('Refusing an insecure browser package redirect') from None
@@ -321,14 +323,14 @@ def quiescent():
     raise RuntimeError('Close all Firefox sessions before preparing the runtime')
 
 
-def build(manifest, staging):
+def build(manifest, staging, *, offline_only=False):
   cache = ROOT / 'packages'
   cache.mkdir(mode=0o700, exist_ok=True)
   safe(cache, private=True)
   staging.mkdir()
   for index, package in enumerate(manifest['packages'], 1):
     report(f'Preparing Firefox ({index}/{len(manifest["packages"])})…')
-    archive = download(package, cache)
+    archive = download(package, cache, offline_only=offline_only)
     safe(archive)
     report('Unpacking Firefox…' if package['name'].startswith('firefox-') else 'Preparing browser libraries…')
     # Cairo and Xvfb both need the pinned Pixman library.

openpilot/tools/arabesque_firefox/test_setup.py

@@ -79,6 +79,16 @@ def test_download_reports_progress_and_reuses_verified_package(tmp_path, monkeyp
   fetch.assert_called_once()
 
 
+def test_offline_only_missing_archive_fails_without_fetching(tmp_path, monkeypatch):
+  package = {'name': 'missing.deb', 'url': 'https://example.invalid/missing.deb', 'size': 4, 'sha256': '0' * 64}
+  fetch = Mock(side_effect=AssertionError('network during offline verification'))
+  monkeypatch.setattr(setup.urllib.request, 'urlopen', fetch)
+  with pytest.raises(FileNotFoundError, match='Missing offline Firefox archive: .*missing.deb'):
+    setup.download(package, tmp_path, offline_only=True)
+  fetch.assert_not_called()
+  assert not list(tmp_path.iterdir())
+
+
 def test_cancelled_download_never_publishes_partial_package(tmp_path, monkeypatch):
   data = b'partial'
   response = io.BytesIO(data)