From e381bf6a44540ab84d8c7dda44d9643d4e9c94f1 Mon Sep 17 00:00:00 2001 From: firestarsdog <229254897+firestarsdog@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:07:29 -0500 Subject: [PATCH] Check offline Firefox against AGNOS --- openpilot/tools/arabesque_firefox/README.md | 12 ++ .../tools/arabesque_firefox/check_agnos.py | 107 ++++++++++++++++++ openpilot/tools/arabesque_firefox/setup.py | 8 +- .../tools/arabesque_firefox/test_setup.py | 10 ++ 4 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 openpilot/tools/arabesque_firefox/check_agnos.py diff --git a/openpilot/tools/arabesque_firefox/README.md b/openpilot/tools/arabesque_firefox/README.md index 23d6853be7..b2be16c3f9 100644 --- a/openpilot/tools/arabesque_firefox/README.md +++ b/openpilot/tools/arabesque_firefox/README.md @@ -521,6 +521,18 @@ Preparation requires root, ARM64 AGNOS `19.8.1`, `20.2-starpilot.1` or `20.2.1`, The `offline/` directory ships the original archives named by digest. The installer copies them into the private package cache and checks each size and digest before extraction. For a new pinned release, refresh the directory with `python3 openpilot/tools/arabesque_firefox/setup.py --stage-offline` and include the resulting archives in the device checkout. Packages absent from a checkout can still be fetched from their pinned HTTPS URLs during setup or an explicit update. +### Check a new AGNOS image or Firefox bundle + +From the repo root on a Linux development machine with Python 3.12+, `dpkg-deb`, `glib-compile-schemas`, `debugfs`, and `simg2img`, run: + +```bash +sudo /usr/bin/python3 -m openpilot.tools.arabesque_firefox.check_agnos +``` + +On x86-64, also install `qemu-user-static` for `qemu-aarch64-static`. Allow about 10 GiB of temporary space. Set `TMPDIR` via `sudo env TMPDIR=/path/to/scratch` if the default location is too small. The command downloads the pinned system image. Use `--image /path/to/system.img.xz` to reuse a local copy, or `--manifest openpilot/common/hardware/comma/agnos-c3.json` to check the C3 manifest if its system image differs. + +The check builds from the shipped archives with an empty cache and package downloads disabled. It verifies the decompressed system image against the manifest, then checks the browser and display libraries inside that image. Keep the existing pins if it passes. Use a parked comma to verify launch, rendering, input, and profile preservation before qualifying a new AGNOS version. + ### Force the reference and SHM modes After preparation has completed, close the Firefox tool and run one diagnostic at a time from the checkout: diff --git a/openpilot/tools/arabesque_firefox/check_agnos.py b/openpilot/tools/arabesque_firefox/check_agnos.py new file mode 100644 index 0000000000..e3f32a50c9 --- /dev/null +++ b/openpilot/tools/arabesque_firefox/check_agnos.py @@ -0,0 +1,107 @@ +"""Check the shipped offline Firefox bundle against a pinned AGNOS system image.""" + +import argparse +import hashlib +import json +import lzma +import os +import platform +import shutil +import subprocess +import tempfile +from pathlib import Path + +from openpilot.tools.arabesque_firefox import setup +from tools.laptop_device_build.extract_sysroot_from_agnos import download, ensure_debugfs_readable_image + + +def check_loader(root, runtime, chroot, emulator): + multiarch = 'lib/aarch64-linux-gnu' + common = runtime / 'usr' / multiarch + display = runtime / 'display/usr' + system = Path('/usr') / multiarch + x_path = f'{display / multiarch}:{common}' + checks = [ + (display / 'bin/Xvfb', x_path), + (display / 'bin/xauth', x_path), + (runtime / 'isolation/xkb/usr/bin/xkbcomp', x_path), + (runtime / 'firefox/firefox', str(common)), + # Gecko loads its sibling libraries before libxul. The standalone loader needs their directory. + (runtime / 'firefox/libxul.so', f'{runtime / "firefox"}:{common}'), + (common / 'libgtk-3.so.0', str(common)), + (common / 'libxcb-damage.so.0', str(common)), + (common / 'libxcb-xtest.so.0', str(common)), + ] + for name in ('libxcb.so.1', 'libxcb-shm.so.0'): + library = common / name + checks.append((library if os.path.lexists(root / library.relative_to('/')) else system / name, str(common))) + + if emulator: + shutil.copyfile(emulator, root / 'arabesque-check-qemu') + (root / 'arabesque-check-qemu').chmod(0o755) + command = [chroot, f'--userspec=+{setup.BROWSER_UID}:+{setup.BROWSER_UID}', '--groups=', str(root)] + if emulator: + command.append('/arabesque-check-qemu') + command += ['/lib/ld-linux-aarch64.so.1', '--inhibit-cache'] + for binary, libraries in checks: + result = subprocess.run([*command, '--library-path', libraries, '--list', str(binary)], + env={'LC_ALL': 'C'}, capture_output=True, text=True, timeout=30) + if result.returncode or '=> not found' in result.stdout: + raise RuntimeError(f'AGNOS loader failed for {binary.name}:\n{result.stderr}{result.stdout}') + print(f'AGNOS loader OK for {binary.name}', flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--manifest', type=Path, default=Path('openpilot/common/hardware/comma/agnos.json')) + parser.add_argument('--image', type=Path, help='Use a local copy of the compressed system image (.img.xz).') + args = parser.parse_args() + if os.geteuid() != 0: + parser.error('Run with sudo on a Linux development machine.') + for tool in ('chroot', 'debugfs', 'simg2img', 'dpkg-deb', 'glib-compile-schemas'): + if shutil.which(tool) is None: + parser.error(f'{tool} is required.') + emulator = None + if platform.machine() != 'aarch64': + emulator = shutil.which('qemu-aarch64-static') + if emulator is None: + parser.error('qemu-aarch64-static is required on non-ARM64 hosts.') + + system, = (entry for entry in json.loads(args.manifest.read_text()) if entry['name'] == 'system') + target = setup.approved() + with tempfile.TemporaryDirectory(prefix='arabesque-agnos-') as directory: + work = Path(directory) + setup.ROOT = work / 'bundle' + setup.ROOT.mkdir() + staging = setup.ROOT / 'runtime' + setup.build(target, staging, offline_only=True) + + compressed = args.image + if compressed is None: + compressed = work / 'system.img.xz' + download(system['url'], compressed) + image = work / 'system.img' + with lzma.open(compressed, 'rb') as source, image.open('wb') as dest: + shutil.copyfileobj(source, dest) + with image.open('rb') as stream: + if hashlib.file_digest(stream, 'sha256').hexdigest() != system['hash'].lower(): + raise ValueError(f'AGNOS system image checksum mismatch: {compressed}') + + filesystem = ensure_debugfs_readable_image(image, work) + print('Extracting the verified AGNOS filesystem…', flush=True) + root = work / 'root' + root.mkdir() + # A real chroot keeps even absolute AGNOS symlinks and loader fallback paths away from the host. + result = subprocess.run(['debugfs', '-R', 'rdump / .', str(filesystem.resolve())], + cwd=root, capture_output=True, text=True, check=True) + if result.stderr.splitlines()[1:]: + raise RuntimeError(result.stderr.strip()) + runtime = Path('/arabesque-check-runtime') + staging.rename(root / runtime.relative_to('/')) + check_loader(root, runtime, shutil.which('chroot'), emulator) + version = (root / 'VERSION').read_text().strip() + print(f'Offline Firefox {target["version"]} revision {target["bundle_revision"]} passed on AGNOS {version}') + + +if __name__ == '__main__': + main() diff --git a/openpilot/tools/arabesque_firefox/setup.py b/openpilot/tools/arabesque_firefox/setup.py index 7248adc2af..23fb487b44 100755 --- a/openpilot/tools/arabesque_firefox/setup.py +++ b/openpilot/tools/arabesque_firefox/setup.py @@ -202,7 +202,7 @@ def copy_package(package, response, partial, label): report(f'{label}… {100 * (package["size"] - remaining) // package["size"]}%') -def download(package, cache, *, cache_uid=0): +def download(package, cache, *, cache_uid=0, offline_only=False): digest = package.get('sha512', package.get('sha256')) path = cache / (digest + '-' + package['name']) legacy = cache / package['name'] @@ -225,6 +225,8 @@ def download(package, cache, *, cache_uid=0): try: fd = os.open(bundled, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) except FileNotFoundError: + if offline_only: + raise FileNotFoundError(f'Missing offline Firefox archive: {bundled.name}') from None with urllib.request.urlopen(package['url'], timeout=30) as response: if not response.url.startswith('https://'): raise ValueError('Refusing an insecure browser package redirect') from None @@ -321,14 +323,14 @@ def quiescent(): raise RuntimeError('Close all Firefox sessions before preparing the runtime') -def build(manifest, staging): +def build(manifest, staging, *, offline_only=False): cache = ROOT / 'packages' cache.mkdir(mode=0o700, exist_ok=True) safe(cache, private=True) staging.mkdir() for index, package in enumerate(manifest['packages'], 1): report(f'Preparing Firefox ({index}/{len(manifest["packages"])})…') - archive = download(package, cache) + archive = download(package, cache, offline_only=offline_only) safe(archive) report('Unpacking Firefox…' if package['name'].startswith('firefox-') else 'Preparing browser libraries…') # Cairo and Xvfb both need the pinned Pixman library. diff --git a/openpilot/tools/arabesque_firefox/test_setup.py b/openpilot/tools/arabesque_firefox/test_setup.py index 0c7c282239..0571d3ecc7 100644 --- a/openpilot/tools/arabesque_firefox/test_setup.py +++ b/openpilot/tools/arabesque_firefox/test_setup.py @@ -79,6 +79,16 @@ def test_download_reports_progress_and_reuses_verified_package(tmp_path, monkeyp fetch.assert_called_once() +def test_offline_only_missing_archive_fails_without_fetching(tmp_path, monkeypatch): + package = {'name': 'missing.deb', 'url': 'https://example.invalid/missing.deb', 'size': 4, 'sha256': '0' * 64} + fetch = Mock(side_effect=AssertionError('network during offline verification')) + monkeypatch.setattr(setup.urllib.request, 'urlopen', fetch) + with pytest.raises(FileNotFoundError, match='Missing offline Firefox archive: .*missing.deb'): + setup.download(package, tmp_path, offline_only=True) + fetch.assert_not_called() + assert not list(tmp_path.iterdir()) + + def test_cancelled_download_never_publishes_partial_package(tmp_path, monkeypatch): data = b'partial' response = io.BytesIO(data)