diff --git a/.gitattributes b/.gitattributes index e3ae4c0b6c..e6b101e732 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,18 +1,20 @@ * text=auto eol=lf -# to move existing files into LFS: -# git add --renormalize . -*.onnx -text -openpilot/selfdrive/modeld/models/big_*_tinygrad.pkl -text -*.svg -text -*.png -text -*.gif -text -*.ttf -text -*.otf -text -*.wav -text +# Binary resources are ordinary Git blobs. +*.onnx -text +openpilot/selfdrive/modeld/models/big_*_tinygrad.pkl -text +*.svg -text +*.png -text +*.gif -text +*.ttf -text +*.otf -text +*.woff -text +*.woff2 -text +*.wav -text +openpilot/starpilot/galaxy/web/vendor/vue/vue.esm-browser.js -text -openpilot/selfdrive/car/tests/test_models_segs.txt -text -openpilot/common/hardware/comma/updater -text +openpilot/selfdrive/car/tests/test_models_segs.txt -text +openpilot/common/hardware/comma/updater -text # Dependency snapshots keep their own storage attributes. msgq_repo/** !filter !diff !merge -text @@ -21,3 +23,7 @@ panda/** !filter !diff !merge -text rednose_repo/** !filter !diff !merge -text teleoprtc_repo/** !filter !diff !merge -text tinygrad_repo/** !filter !diff !merge -text + +# Compiled model parts must retain exact binary bytes. +*.pkl.chunk* -text +*.pkl -text diff --git a/.gitignore b/.gitignore index d267608124..3f07e2f118 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,11 @@ venv/ a.out .hypothesis .cache/ +.host_runtime/ +.comma_sysroot/ +.venv-linux-arm64/ bin/ +!openpilot/starpilot/galaxy/bin/ # created at launch for comma hardware PYTHONPATH (PC uses editable installs via pyproject.toml) /msgq @@ -50,7 +54,6 @@ st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z] *.stats *.pkl *.pkl* -!openpilot/selfdrive/modeld/models/big_*_tinygrad.pkl config.json compile_commands.json compare_runtime*.html @@ -59,6 +62,8 @@ openpilot/selfdrive/modeld/models/tg_input_devices.json # build artifacts docs/_site/ openpilot/selfdrive/pandad/pandad +openpilot/selfdrive/pandad/tests/test_aol_protocol +openpilot/selfdrive/pandad/tests/test_aol_wire openpilot/cereal/services.h openpilot/cereal/gen openpilot/cereal/messaging/bridge @@ -98,3 +103,30 @@ build/ .context/ PLAN.md TASK.md + +# Disk-backed model cache reconstructed from published numbered parts. +openpilot/selfdrive/modeld/models/*.unchunked + +# Publish only the two small compiled model packages, never raw pickles or caches. +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9]of[0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9]of[0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9]of[0-9][0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9][0-9]of[0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9][0-9]of[0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9][0-9]of[0-9][0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9][0-9][0-9]of[0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9][0-9][0-9]of[0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunk[0-9][0-9][0-9][0-9]of[0-9][0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunkmanifest +!openpilot/selfdrive/modeld/models/driving_tinygrad.pkl.chunksha256 +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9]of[0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9]of[0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9]of[0-9][0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9][0-9]of[0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9][0-9]of[0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9][0-9]of[0-9][0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9][0-9][0-9]of[0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9][0-9][0-9]of[0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunk[0-9][0-9][0-9][0-9]of[0-9][0-9][0-9][0-9] +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunkmanifest +!openpilot/selfdrive/modeld/models/dmonitoring_model_tinygrad.pkl.chunksha256 diff --git a/SConstruct b/SConstruct index c01c17d9fa..7612bd1c4f 100644 --- a/SConstruct +++ b/SConstruct @@ -5,6 +5,8 @@ import sysconfig import platform import shlex import importlib +import re +import stat import numpy as np import SCons.Errors @@ -59,6 +61,10 @@ pkg_names = ['acados', 'capnproto', 'ffmpeg', 'json11', 'ncurses', 'zeromq', 'zs pkgs = [importlib.import_module(name) for name in pkg_names] acados = pkgs[pkg_names.index('acados')] ffmpeg = pkgs[pkg_names.index('ffmpeg')] +from tools.laptop_device_build.runtime_paths import runtime_library_path + +build_sysroot = os.environ.get("COMMA_BUILD_SYSROOT") if arch == "comma_arm64" else None +ffmpeg_runtime_dir = runtime_library_path(ffmpeg.LIB_DIR, sys.prefix, "/usr/local/venv" if build_sysroot else None) # Shared package ships .so/.dylib; older device venvs still have static .a only. # Keep static link deps (x264/z/va/drm) when the installed package is static so # COMMA_HARDWARE CI works without upgrading the device venv yet. @@ -159,7 +165,7 @@ env = Environment( "#rednose_repo/rednose/helpers", [x.LIB_DIR for x in pkgs], ], - RPATH=[ffmpeg.LIB_DIR] if ffmpeg_shared else [], + RPATH=[ffmpeg_runtime_dir] if ffmpeg_shared else [], CYTHONCFILESUFFIX=".cpp", COMPILATIONDB_USE_ABSPATH=True, REDNOSE_ROOT="#rednose_repo", @@ -184,6 +190,15 @@ if arch == "comma_arm64": arch_flags = ["-D__COMMA_HARDWARE__", "-mcpu=cortex-a57"] env.Append(CCFLAGS=arch_flags) env.Append(CXXFLAGS=arch_flags) + if build_sysroot: + if not all(os.path.isdir(os.path.join(build_sysroot, part)) for part in ("usr/include", "usr/lib/aarch64-linux-gnu", "lib/aarch64-linux-gnu")): + raise SCons.Errors.UserError(f"Incomplete COMMA_BUILD_SYSROOT: {build_sysroot}") + # Clang's --sysroot controls the standard headers and linker startup/libs; + # explicit SCons paths make device libraries win over container libraries. + env.Prepend(CPPPATH=[os.path.join(build_sysroot, part) for part in ("usr/local/include", "usr/include")]) + env.Prepend(LIBPATH=[os.path.join(build_sysroot, part) for part in ("usr/local/lib", "usr/lib/aarch64-linux-gnu", "lib/aarch64-linux-gnu")]) + env.Append(CCFLAGS=[f"--sysroot={build_sysroot}"]) + env.Append(LINKFLAGS=[f"--sysroot={build_sysroot}"]) elif arch == "Darwin": env.Append(LIBPATH=[ "/System/Library/Frameworks/OpenGL.framework/Libraries", @@ -226,6 +241,8 @@ if arch == "Darwin": envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"] else: envCython["LINKFLAGS"] = ["-pthread", "-shared"] + if build_sysroot: + envCython.Append(LINKFLAGS=[f"--sysroot={build_sysroot}"]) np_version = SCons.Script.Value(np.__version__) Export('envCython', 'np_version') @@ -233,19 +250,38 @@ Export('envCython', 'np_version') Export('env', 'arch', 'acados', 'ffmpeg_libs') # Setup cache dir -cache_dir = '/data/scons_cache' if arch == "comma_arm64" else '/tmp/scons_cache' +cache_dir = os.environ.get('SCONS_CACHE', '/data/scons_cache' if arch == "comma_arm64" else '/tmp/scons_cache') cache_size_limit = 4e9 if "CI" in os.environ else 2e9 CacheDir(cache_dir) Clean(["."], cache_dir) def prune_cache_dir(target=None, source=None, env=None): - cache_files = sorted((os.path.join(root, f) for root, _, files in os.walk(cache_dir) for f in files), key=os.path.getmtime) - cache_size = sum(os.path.getsize(f) for f in cache_files) - for f in cache_files: + cache_files = [] + cache_root = os.path.normpath(cache_dir) + for root, _, files in os.walk(cache_root): + shard = os.path.basename(root) + if os.path.dirname(root) != cache_root or re.fullmatch(r'[0-9A-Fa-f]{2}', shard) is None: + continue + for name in files: + if re.fullmatch(r'(?:[0-9a-f]{32}|[0-9a-f]{64})', name) is None or name[:2].lower() != shard.lower(): + continue + path = os.path.join(root, name) + try: + info = os.stat(path, follow_symlinks=False) + except FileNotFoundError: + continue + if stat.S_ISREG(info.st_mode): + cache_files.append((info.st_mtime, info.st_size, path)) + cache_files.sort() + cache_size = sum(size for _, size, _ in cache_files) + for _, size, path in cache_files: if cache_size < cache_size_limit: break - cache_size -= os.path.getsize(f) - os.unlink(f) + try: + os.unlink(path) + except FileNotFoundError: + pass + cache_size -= size # ********** start building stuff ********** @@ -281,6 +317,7 @@ SConscript([ if arch == "comma_arm64": SConscript(['openpilot/system/camerad/SConscript']) + SConscript(['openpilot/starpilot/system/android_auto/SConscript']) # Build selfdrive SConscript([ diff --git a/build b/build new file mode 100755 index 0000000000..d9101e60de --- /dev/null +++ b/build @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Project build shortcuts. AGNOS builds natively; laptops use the ARM64 container. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ "${1:-}" == -h || "${1:-}" == --help ]]; then + cat <<'EOF' +Usage: ./build [jobs] [scons options] + ./build [--params] [--cereal] [--panda] [jobs] [scons options] + ./build --mapd + ./build --check-startup + +On AGNOS/aarch64, builds run natively. On laptops, builds use a Linux/ARM64 container. +--params builds libparams_c.so; --cereal builds cereal outputs; --panda +builds signed panda_h7 and body_h7 firmware only. Numeric jobs become -jN. +RELEASE and CERT are environment variables, not trailing SCons arguments: + RELEASE=1 CERT=/path/to/certificate ./build --panda 4 +CERT must name a readable certificate file. Omit RELEASE for DEBUG firmware. +--mapd explicitly packages the pinned offline Linux/ARM64 Go provider; it +does not fetch map regions or participate in the default build. +Run scripts/laptop_device_build.sh doctor to check local prerequisites. +Full laptop builds require a source-matched QCOM artifact package in +.cache/qcom-artifact-package; device builds still compile models natively. +--check-startup checks retained settings admission read-only using the local +native developer runtime. Supply a matching Params namespace and recovery +storage. A relocated copy can fail its path-bound profile marker. +EOF + exit 0 +fi +if [[ "${1:-}" == --check-startup ]]; then + if [[ $# -ne 3 ]]; then + echo "Usage: ./build --check-startup " >&2 + exit 2 + fi + exec "${ROOT_DIR}/dev" python "${ROOT_DIR}/tools/check_startup.py" "$2" "$3" +fi +for arg in "$@"; do + case "${arg}" in + RELEASE=*|CERT=*) + echo "ERROR: RELEASE and CERT must be environment variables, not SCons arguments." >&2 + echo "Use: RELEASE=1 CERT=/path/to/certificate ./build --panda 4" >&2 + exit 2 + ;; + esac +done + +original_args=("$@") +build_helper="${ROOT_DIR}/scripts/laptop_device_build.sh" +if [[ -f /AGNOS && "$(uname -s)" == Linux && "$(uname -m)" == aarch64 ]]; then + build_helper="${ROOT_DIR}/scripts/native_device_build.sh" +fi +if [[ $# -eq 1 && "$1" == --mapd ]]; then + exec "${ROOT_DIR}/scripts/build_mapd_provider.sh" +fi +if [[ $# -gt 0 ]]; then + for arg in "$@"; do + if [[ "$arg" == --mapd ]]; then + echo "ERROR: --mapd is a standalone package operation; use ./build --mapd" >&2 + exit 2 + fi + done +fi +shortcut_targets=() +passthrough_args=() +shortcut_includes_panda=0 +panda_generated_files=(panda/board/obj/gitversion.h panda/board/obj/version) + +while [[ $# -gt 0 ]]; do + case "$1" in + --params|-params|-param) + shortcut_targets+=(openpilot/common/libparams_c.so) + ;; + --panda|-panda) + shortcut_includes_panda=1 + shortcut_targets+=(panda/board/obj/panda_h7.bin.signed panda/board/obj/body_h7.bin.signed) + ;; + --cereal|-cereal) + shortcut_targets+=(openpilot/cereal/libcereal.a openpilot/cereal/libsocketmaster.a + openpilot/cereal/messaging/bridge openpilot/cereal/services.h) + ;; + *) + if [[ "$1" =~ ^[0-9]+$ ]]; then + passthrough_args+=("-j$1") + else + passthrough_args+=("$1") + fi + ;; + esac + shift +done + +if [[ "${#shortcut_targets[@]}" -gt 0 ]]; then + scons_args=(--no-scrub "${shortcut_targets[@]}") + if [[ ${passthrough_args[0]+present} ]]; then + scons_args+=("${passthrough_args[@]}") + fi + if [[ "$shortcut_includes_panda" -eq 1 ]]; then + exec "${build_helper}" scons "${scons_args[@]}" + fi + + # Panda/SConscript updates these even for unrelated targets. Restore their + # prior bytes for the non-firmware shortcuts, including on build failure. + tmpdir="$(mktemp -d)" + restore_panda_metadata() { + local path safe + for path in "${panda_generated_files[@]}"; do + safe="${path//\//__}" + if [[ -e "${tmpdir}/${safe}.missing" ]]; then + rm -f "${ROOT_DIR}/${path}" + else + cp -p "${tmpdir}/${safe}" "${ROOT_DIR}/${path}" + fi + done + rm -rf "${tmpdir}" + } + trap restore_panda_metadata EXIT + for path in "${panda_generated_files[@]}"; do + safe="${path//\//__}" + if [[ -e "${ROOT_DIR}/${path}" ]]; then + cp -p "${ROOT_DIR}/${path}" "${tmpdir}/${safe}" + else + touch "${tmpdir}/${safe}.missing" + fi + done + "${build_helper}" scons "${scons_args[@]}" + exit $? +fi + +if [[ "${#original_args[@]}" -gt 0 ]]; then + exec "${build_helper}" build "${original_args[@]}" +fi +exec "${build_helper}" build diff --git a/c3 b/c3 new file mode 100755 index 0000000000..a6b2d574f7 --- /dev/null +++ b/c3 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" c3 "$@" diff --git a/c4 b/c4 new file mode 100755 index 0000000000..22437605b8 --- /dev/null +++ b/c4 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" c4 "$@" diff --git a/dev b/dev new file mode 100755 index 0000000000..30da28883f --- /dev/null +++ b/dev @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" "$@" diff --git a/docs/how-to/laptop-device-build.md b/docs/how-to/laptop-device-build.md new file mode 100644 index 0000000000..1637420275 --- /dev/null +++ b/docs/how-to/laptop-device-build.md @@ -0,0 +1,72 @@ +# Laptop and device builds + +`./build` uses a Linux/ARM64 container and a matching AGNOS sysroot on a laptop. On an actual AGNOS/aarch64 device it runs SCons natively in that checkout, using the device Python environment and vendored source imports. It does not copy host-built model files into a device build. The container image pins Cap'n Proto 1.0.1 to the generated cereal contract in this tree. + +## Prepare the build environment + +Install Docker or Podman with Linux/ARM64 support. Run `scripts/laptop_device_build.sh doctor` to see which local prerequisites are missing. The first setup/build can download a container image, Python dependencies, and an AGNOS system image; run it only when you intend that network activity. + +To prepare without a physical device, run: + +```sh +scripts/laptop_device_build.sh setup +``` + +To use an explicitly supplied device sysroot instead, run `scripts/laptop_device_build.sh setup-sysroot [user] [ssh-port]`, then `scripts/laptop_device_build.sh build-image`. The build does not connect to a device or flash firmware. The compatibility alias `scripts/starpilot_build_flow.sh laptop-setup` invokes the same explicit setup. + +If the matching AGNOS ext4 system image is already local, use `scripts/laptop_device_build.sh setup-sysroot-local ` instead. This extracts a separate ignored `.comma_sysroot`; it does not change the image. Verify its `VERSION` against the intended device OS. A sysroot copied from a different AGNOS build does not establish ABI compatibility. + +## Build + +```sh +./build # complete device-target build +./build 8 # complete build with eight SCons jobs +./build --params # current params shared library only +./build --cereal # cereal libraries, bridge, and service header +./build --panda # signed panda_h7 and body_h7 firmware targets only +./build --panda 8 # same H7 targets with eight jobs +./build --mapd # explicit offline Mapd provider package, separate from SCons +``` + +`--mapd` cannot be combined with SCons targets or jobs. It builds the pinned +Linux/ARM64 static Mapd binary and manifest in ignored +`openpilot/starpilot/maps/provider/`. It does not download map regions, change +the selected offline snapshot, start Mapd, or run during plain `./build`. + +The Mapd shortcut requires Go 1.25.1 and an already populated module cache. +On Linux/ARM64 it uses a local Go executable. On other hosts it uses the +`openpilot-local-mapd-builder:go1.25.1` Linux/ARM64 image, built **explicitly** +from `tools/laptop_device_build/Dockerfile.mapd`: + +```sh +docker build --platform linux/arm64 -f tools/laptop_device_build/Dockerfile.mapd \ + -t openpilot-local-mapd-builder:go1.25.1 . +mkdir -p .cache/mapd-go-modules +docker run --rm --platform linux/arm64 \ + --mount "type=bind,src=$PWD/mapd_repo,dst=/work,readonly" \ + --mount "type=bind,src=$PWD/.cache/mapd-go-modules,dst=/gomodcache" \ + -e GOMODCACHE=/gomodcache -w /work \ + openpilot-local-mapd-builder:go1.25.1 go mod download +./build --mapd +``` + +The image build and module-cache population above are explicit network setup +steps. `./build --mapd` itself uses `--network none`, `--pull never`, +`GOPROXY=off`, and a read-only module cache. An existing offline cache can be +selected with `COMMA_MAPD_MODULE_CACHE`; an existing reviewed ARM64 Go image +can be selected with `COMMA_MAPD_BUILD_IMAGE`. The generated package is checked +for the exact source digest and ARM64/static format by `package_shadow.py`. +It is a build artifact; stage it separately with the source manifest for a +device, rather than treating the source-only checkout as containing it. + +The historical wrapper's other Panda variants are not targets in the current `panda/SConscript`; `--panda` does not claim to build them. Shortcut targets use `--no-scrub` and leave `prebuilt` alone. Other SCons arguments pass through the container helper. A full-build invocation with options or explicit targets cannot mark `prebuilt`: clean, question, and dry-run commands do not compile the complete default graph. `scripts/starpilot_build_flow.sh laptop-device [jobs]` is the compatibility alias for a complete `./build`. + +The complete build selects the current `comma_arm64` SCons profile in its container, including Ion-backed VisionIPC and QCOM model compilation. It checks the driving model, driver-monitoring model, camera warps, and core runtime libraries before setting `prebuilt`. QCOM model compilation may require device-class hardware even with an ARM64 container; if it cannot complete, the build fails and `prebuilt` remains unset. A local `DEV=METAL` model artifact is not a device build substitute. + +`scripts/laptop_device_build.sh verify-artifacts` inspects the outputs without loading pickle files: it requires ARM64 ELF objects, exact current camera warp dimensions, and QCOM captured backends. It does not run inference. The matching sysroot is passed to Clang at both compile and link time. For a cached compatible base image, `COMMA_BASE_IMAGE=` overrides the Dockerfile's default base only while building the image; record that override when reporting build provenance. + +For a single explicit target, use `scripts/laptop_device_build.sh scons --no-scrub `. Sysroot, cache, and build-venv path overrides must resolve to direct children of this checkout. Relative overrides resolve from the checkout root before Docker bind mounts. The `doctor` command checks availability only; it does not prove the image or sysroot can compile every target. + +`scripts/laptop_device_build.sh manager` remains an **explicit** container runtime mode for developers with a complete build. Neither `./build` nor the checks above launch it. It can start ordinary manager processes; review that mode separately before using it with connected hardware. + +The legacy alias also supports `verify`, `mac`, and `device`. `mac` checks Python compilation only and does not produce device binaries. `device` requires AGNOS on ARM64 hardware and is never part of laptop validation. diff --git a/onroad b/onroad new file mode 100755 index 0000000000..f0f0116038 --- /dev/null +++ b/onroad @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" onroad "$@" diff --git a/openpilot/common/file_chunker.py b/openpilot/common/file_chunker.py new file mode 100644 index 0000000000..45465bd6e1 --- /dev/null +++ b/openpilot/common/file_chunker.py @@ -0,0 +1,149 @@ +"""Dom-compatible numbered model parts, with verified atomic disk materialization. + +The count-only .chunkmanifest format remains readable. New packages include an +integrity sidecar; model compilers and tinygrad's disk-backed loader stay unchanged. +""" +import hashlib +import json +import os +import re +import tempfile +from pathlib import Path + +CHUNK_SIZE = 45 * 1024 * 1024 + + +def get_chunk_name(name, idx, num_chunks): + return f'{name}.chunk{idx + 1:02d}of{num_chunks:02d}' + + +def get_manifest_path(name): + return f'{name}.chunkmanifest' + + +def get_existing_chunks(path): + path = Path(path) + manifest = Path(get_manifest_path(path)) + if not manifest.exists() and path.is_file(): + return [path] + text = manifest.read_text().strip() + if not text.isdecimal() or not 1 <= int(text) <= 1024: + raise ValueError('invalid model part count') + count = int(text) + parts = [path.with_name(get_chunk_name(path.name, i, count)) for i in range(count)] + for i, part in enumerate(parts): + size = part.stat().st_size + if not 0 < size <= CHUNK_SIZE or (i < count - 1 and size != CHUNK_SIZE): + raise ValueError('invalid model part size') + return [manifest, *parts] + + +def file_chunked_exists(path): + try: + get_existing_chunks(path) + return True + except (OSError, ValueError): + return False + + +def _digest_file(path): + digest = hashlib.sha256() + with Path(path).open('rb') as inp: + while block := inp.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def materialize_file_chunked(path, expected_sha256=None): + """Return a regular file for tinygrad OOB reads; refuse invalid packages. + + Cache publication uses os.replace so concurrent readers never see partial data. + Legacy count-only packages can supply an externally pinned expected_sha256. + """ + path = Path(path) + entries = get_existing_chunks(path) + if entries == [path]: + if expected_sha256 is not None and _digest_file(path) != expected_sha256: + raise ValueError('model digest mismatch') + return path + sidecar = Path(f'{path}.chunksha256') + metadata = json.loads(sidecar.read_text()) if sidecar.exists() else None + if metadata is not None: + if set(metadata) != {'sha256', 'bytes', 'parts'} or len(metadata['parts']) != len(entries) - 1: + raise ValueError('invalid model integrity manifest') + if expected_sha256 is not None and metadata['sha256'] != expected_sha256: + raise ValueError('model digest mismatch') + expected_sha256 = metadata['sha256'] + cache = Path(f'{path}.unchunked') + digest, total = hashlib.sha256(), 0 + for i, part in enumerate(entries[1:]): + part_hash, part_size = hashlib.sha256(), 0 + with part.open('rb') as inp: + while block := inp.read(1024 * 1024): + digest.update(block) + part_hash.update(block) + part_size += len(block) + total += part_size + if metadata is not None and metadata['parts'][i] != {'bytes': part_size, 'sha256': part_hash.hexdigest()}: + raise ValueError('model part digest mismatch') + if expected_sha256 is not None and digest.hexdigest() != expected_sha256: + raise ValueError('model digest mismatch') + if metadata is not None and total != metadata['bytes']: + raise ValueError('model size mismatch') + if cache.is_file() and cache.stat().st_size == total and _digest_file(cache) == digest.hexdigest(): + return cache + fd, temporary = tempfile.mkstemp(prefix=f'.{path.name}.', dir=path.parent) + copied_digest, copied_size = hashlib.sha256(), 0 + try: + with os.fdopen(fd, 'wb') as out: + for part in entries[1:]: + with part.open('rb') as inp: + while block := inp.read(1024 * 1024): + out.write(block) + copied_digest.update(block) + copied_size += len(block) + if copied_size != total or copied_digest.hexdigest() != digest.hexdigest(): + raise ValueError('model parts changed during materialization') + os.replace(temporary, cache) + return cache + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def read_file_chunked(path, expected_sha256=None): + return materialize_file_chunked(path, expected_sha256).read_bytes() + + +def open_file_chunked(path, expected_sha256=None): + return materialize_file_chunked(path, expected_sha256).open('rb') + + +def package_file(path, destination): + """Package a compiled artifact without deleting the compiler's original output.""" + path, destination = Path(path), Path(destination) + source_stat = path.stat() + size = source_stat.st_size + if size <= 0: + raise ValueError('empty model artifact') + count = (size + CHUNK_SIZE - 1) // CHUNK_SIZE + if count > 1024: + raise ValueError('model exceeds part count limit') + destination.parent.mkdir(parents=True, exist_ok=True) + digest, parts = hashlib.sha256(), [] + with path.open('rb') as inp: + for i in range(count): + block = inp.read(CHUNK_SIZE) + digest.update(block) + destination.with_name(get_chunk_name(destination.name, i, count)).write_bytes(block) + parts.append({'bytes': len(block), 'sha256': hashlib.sha256(block).hexdigest()}) + final_stat = path.stat() + if (source_stat.st_size, source_stat.st_mtime_ns, source_stat.st_ino) != (final_stat.st_size, final_stat.st_mtime_ns, final_stat.st_ino) or sum(part['bytes'] for part in parts) != size: + raise ValueError('compiled model changed while packaging') + Path(f'{destination}.chunksha256').write_text(json.dumps({'bytes': size, 'sha256': digest.hexdigest(), 'parts': parts}, indent=2) + '\n') + Path(get_manifest_path(destination)).write_text(str(count)) + active = {get_chunk_name(destination.name, i, count) for i in range(count)} + for old in destination.parent.glob(destination.name + '.chunk*of*'): + if old.name not in active and re.fullmatch(re.escape(destination.name) + r'\.chunk[0-9]{2,}of[0-9]{2,}', old.name): + old.unlink() + return digest.hexdigest() diff --git a/openpilot/common/vendor_manifest.py b/openpilot/common/vendor_manifest.py index 7e473cca2b..e0ee013439 100644 --- a/openpilot/common/vendor_manifest.py +++ b/openpilot/common/vendor_manifest.py @@ -9,7 +9,7 @@ from pathlib import Path, PurePosixPath from urllib.parse import urlsplit MANIFEST = "upstream-sync.json" -DEPENDENCIES = frozenset({"msgq_repo", "opendbc_repo", "panda", "rednose_repo", "teleoprtc_repo", "tinygrad_repo"}) +DEPENDENCIES = frozenset({"msgq_repo", "opendbc_repo", "panda", "rednose_repo", "teleoprtc_repo", "tinygrad_repo", "mapd_repo"}) SHA = re.compile(r"[0-9a-f]{40}\Z") diff --git a/openpilot/starpilot/tests/test_schema_cache_normalize.py b/openpilot/starpilot/tests/test_schema_cache_normalize.py new file mode 100644 index 0000000000..3aeccd3133 --- /dev/null +++ b/openpilot/starpilot/tests/test_schema_cache_normalize.py @@ -0,0 +1,208 @@ +import hashlib +from contextlib import redirect_stderr +import io +import json +from pathlib import Path +import runpy +import struct +import tempfile +import unittest +from unittest.mock import patch + +from openpilot.common.params import Params +from openpilot.starpilot import schema_cache as cache +from openpilot.starpilot import schema_cache_normalize as normalize +from openpilot.starpilot.state_migration import load_snapshot +from tools.diagnostics.normalize_event_caches import main as cli_main + + +class TestCacheNormalization(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.params = Params(str(self.root / 'params')) + self.backup = self.root / 'backup' + cache.prewarm_cache_contracts() + + @staticmethod + def repack(header, payload): + encoded = json.dumps(header, sort_keys=True, separators=(',', ':')).encode() + return cache.MAGIC + struct.pack('>I', len(encoded)) + encoded + payload + + def populate(self, version=1): + values = { + 'CalibrationParams': {'rpyCalib': [.1, -.2, .3], 'validBlocks': 12}, + 'LiveParametersV2': {'steerRatio': 15.25, 'stiffnessFactor': .875, 'roll': -.015625}, + 'LiveTorqueParameters': {'latAccelFactorFiltered': 2.5, 'frictionCoefficientFiltered': .125, 'totalBucketPoints': 64}, + 'LiveDelay': {'lateralDelay': .25}, + } + result = {} + for index, key in enumerate(normalize.EVENT_KEYS): + contract = cache.CONTRACTS[key] + message = contract.root.new_message(logMonoTime=1234567890 + index, valid=bool(index % 2)) + message.init(contract.service) + setattr(message, contract.service, values[key]) + payload = message.to_bytes() + raw = self.repack(cache._header(contract, key, payload, version=version), payload) + self.params.put(key, raw, block=True) + result[key] = raw + return result + + def run_normalization(self): + return normalize.normalize_current_event_caches(self.params, self.backup, producers_stopped=True) + + def test_current_v1_all_services_values_backup_and_carparams_preserved(self): + originals = self.populate() + car_params = {} + for key, contract in cache.CONTRACTS.items(): + if contract.service is None: + cache.put_cache(self.params, key, contract.root.new_message(carFingerprint='EXACT_ORIGINAL_CAR'), block=True) + car_params[key] = self.params.get(key) + staged = self.root / 'staged_normalizer.py' + staged.write_bytes(Path(normalize.__file__).read_bytes()) + api = runpy.run_path(str(staged)) + report = api['normalize_current_event_caches'](self.params, self.backup, producers_stopped=True) + self.assertEqual(report['normalizer']['path'], str(staged)) + self.assertEqual(report['status'], 'normalized') + self.assertEqual(report['written'], list(normalize.EVENT_KEYS)) + self.assertEqual(load_snapshot(report['backup']), originals) + self.assertEqual(json.loads((Path(report['backup']) / 'normalization.json').read_bytes()), report) + for key, old in originals.items(): + updated = self.params.get(key) + self.assertNotEqual(old, updated) + self.assertEqual(normalize._header(updated)['version'], 2) + old_payload, new_payload = cache.inspect_cache(key, old).payload, cache.inspect_cache(key, updated).payload + with cache.CONTRACTS[key].root.from_bytes(old_payload) as before, cache.CONTRACTS[key].root.from_bytes(new_payload) as after: + self.assertEqual(before.to_dict(), after.to_dict()) + for key, raw in car_params.items(): + self.assertEqual(self.params.get(key), raw) + + def test_v2_and_missing_values_unchanged_without_backup_or_writes(self): + originals = self.populate(version=2) + self.params.remove('LiveDelay') + originals.pop('LiveDelay') + before = {key: Path(self.params.get_param_path(key)).stat().st_ino for key in originals} + with patch.object(normalize, '_atomic_write', side_effect=AssertionError('unnecessary write')): + report = self.run_normalization() + self.assertEqual(report['status'], 'unchanged') + self.assertEqual(report['written'], []) + self.assertEqual(report['caches']['LiveDelay']['action'], 'missing') + self.assertFalse(self.backup.exists()) + for key, raw in originals.items(): + self.assertEqual(self.params.get(key), raw) + self.assertEqual(Path(self.params.get_param_path(key)).stat().st_ino, before[key]) + + def test_any_incompatible_schema_or_payload_rejects_entire_batch_before_backup(self): + for damage in ('schema', 'payload'): + with self.subTest(damage=damage): + originals = self.populate() + key = normalize.EVENT_KEYS[-1] + old = originals[key] + header = normalize._header(old) + payload = cache.inspect_cache(key, old).payload + if damage == 'schema': + header['schema_sha256'] = '0' * 64 + else: + payload = b'not capnp' + header.update(payload_bytes=len(payload), payload_sha256=hashlib.sha256(payload).hexdigest()) + originals[key] = self.repack(header, payload) + self.params.put(key, originals[key], block=True) + with patch.object(normalize, '_atomic_write', side_effect=AssertionError('write before validation')): + with self.assertRaises(normalize.NormalizationError): + self.run_normalization() + self.assertFalse(self.backup.exists()) + self.assertEqual({key: self.params.get(key) for key in originals}, originals) + + def test_old_v1_writer_cannot_claim_success(self): + originals = self.populate() + def old_writer(params, key, message, block=False): + params.put(key, originals[key], block=block) + with patch.object(cache, 'put_cache', side_effect=old_writer), self.assertRaises(normalize.NormalizationError): + self.run_normalization() + self.assertFalse(self.backup.exists()) + self.assertEqual({key: self.params.get(key) for key in originals}, originals) + + def test_typed_traversal_refuses_lazy_invalid_service_pointer(self): + originals = self.populate() + key = normalize.EVENT_KEYS[-1] + original = cache.inspect_cache(key, originals[key]) + payload = bytearray(original.payload) + data_words = cache.CONTRACTS[key].root.schema.node.struct.dataWordCount + struct.pack_into('` namespace; run consumers under the +same prefix. Other namespaces receive a diagnostic and no Params seeding. Playback +remains available, but isolation does not establish compatibility of logged schemas +or qualify a replay for vehicle validation. + ## Setup Before starting a replay, you need to authenticate with your comma account using `auth.py`. This will allow you to access your routes from the server. @@ -140,3 +147,25 @@ positional arguments: optional arguments: -h, --help show this help message and exit ``` + +## Isolated StarPilot desktop replay + +`./onroad [jobs] [--c3|--c4|--all|--replay-only] [--prefix replay-NAME] +` starts the current replay binary and selected native +UI in one private host session. Without a UI choice, logged `initData.deviceType` +chooses compact for mici/C4 and large otherwise. `--replay-only` starts no UI +and does not load `initData` for display selection. The wrapper preserves replay +arguments after its own options, and `--help` needs no host build. It uses a +separate `replay-*` message namespace and disposable Params directory, copies +only validated logged display booleans, and stops its owned children on normal +exit or SIGINT/SIGTERM/SIGHUP. It creates and removes only its own native +message-queue directory; an existing explicit replay prefix is rejected. +Route playback may need the usual route access +and can download data requested by the route argument. + +This port has no fake nav, CEM, CSC, alert, or Galaxy demo publishers. Their old +flags fail clearly. The current replay binary exposes its ncurses playback +controls, not the old state-file control bar. `--auto` alone is rejected because +the current replay binary still requires a route. A native UI preview also +requires the complete reviewed bitmap font bundle described in the StarPilot UI +README; this is an open zero-configuration host dependency. diff --git a/openpilot/tools/replay/alert_demo.py b/openpilot/tools/replay/alert_demo.py new file mode 100644 index 0000000000..4cbb5a422e --- /dev/null +++ b/openpilot/tools/replay/alert_demo.py @@ -0,0 +1,59 @@ +"""Synthetic visual alert for a private desktop replay session only.""" + +import argparse +import os +import signal +import time + +from openpilot.tools.replay.onroad import PREFIX_RE, _ipc_root + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Preview a synthetic critical visual alert during private replay") + parser.add_argument("--delay", type=float, default=20.0) + parser.add_argument("--hold", type=float, default=5.0) + args = parser.parse_args(argv) + prefix = os.environ.get("OPENPILOT_PREFIX", "") + ipc = _ipc_root(prefix) + if (os.environ.get("SP_HOST_RUNTIME") != "1" or PREFIX_RE.fullmatch(prefix) is None or + ipc.is_symlink() or not ipc.is_dir() or ipc.stat().st_uid != os.getuid()): + parser.error("alert preview requires an isolated host replay session") + if not (0 <= args.delay <= 120 and 0 < args.hold <= 30): + parser.error("invalid alert preview timing") + + from openpilot.cereal import log, messaging + from openpilot.common.realtime import Ratekeeper + + running = True + def stop(_signum, _frame) -> None: + nonlocal running + running = False + + previous = {sig: signal.signal(sig, stop) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + publisher = messaging.PubMaster(["selfdriveState"]) + rate = Ratekeeper(100, print_delay_threshold=None) + def send(active: bool) -> None: + message = messaging.new_message("selfdriveState") + message.valid = True + state = message.selfdriveState + state.alertSize = log.SelfdriveState.AlertSize.full if active else log.SelfdriveState.AlertSize.none + state.alertStatus = log.SelfdriveState.AlertStatus.critical if active else log.SelfdriveState.AlertStatus.normal + state.alertText1 = "TAKE CONTROL IMMEDIATELY" if active else "" + state.alertText2 = "Developer visual alert preview" if active else "" + state.alertType = "developerVisualPreview" if active else "" + publisher.send("selfdriveState", message) + + start = time.monotonic() + while running: + now = time.monotonic() - start + send(args.delay <= now < args.delay + args.hold) + rate.keep_time() + return 0 + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/openpilot/tools/replay/replay.cc b/openpilot/tools/replay/replay.cc index 26e10e7c7b..bd1350b000 100644 --- a/openpilot/tools/replay/replay.cc +++ b/openpilot/tools/replay/replay.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include #include "openpilot/cereal/services.h" @@ -211,8 +212,15 @@ void Replay::startStream(const std::shared_ptr segment) { builder.setRoot(event.getCarParams()); auto words = capnp::messageToFlatArray(builder); auto bytes = words.asBytes(); - Params().put("CarParams", (const char *)bytes.begin(), bytes.size()); - Params().put("CarParamsPersistent", (const char *)bytes.begin(), bytes.size()); + const char *prefix = std::getenv("OPENPILOT_PREFIX"); + const std::string replay_prefix = prefix != nullptr ? prefix : ""; + if (replay_prefix.rfind("replay-", 0) == 0 && replay_prefix.size() > 7 && + replay_prefix.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-") == std::string::npos) { + Params().put("CarParams", (const char *)bytes.begin(), bytes.size()); + } else { + rWarning("refusing live CarParams seeding outside a dedicated replay- namespace; use --prefix replay-"); + } + rWarning("refusing persistent CarParams seeding from unqualified logs; source schema provenance and explicit conversion are required"); } else { rWarning("failed to read CarParams from current segment"); } diff --git a/openpilot/tools/replay/tests/test_onroad_host.py b/openpilot/tools/replay/tests/test_onroad_host.py new file mode 100644 index 0000000000..16d5a010d6 --- /dev/null +++ b/openpilot/tools/replay/tests/test_onroad_host.py @@ -0,0 +1,314 @@ +"""Host replay parsing, private Params seeding and child ownership.""" + +from pathlib import Path +from types import SimpleNamespace +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +from openpilot.common.params import Params +from openpilot.tools.replay.onroad import _block_alert_service, _ipc_root, owned_ipc_namespace, parse_onroad_args, run, seed_replay_params, supervise +from openpilot.tools.replay.onroad_config import first_log_identifier, parse_replay_args, seed_preview, select_ui_target + + +class TestOnroadHost(unittest.TestCase): + def test_supervisor_preserves_fast_clean_exit_and_maps_child_signal(self): + class FakeProcess: + def __init__(self, statuses): + self.statuses = iter(statuses) + self.returncode = None + + def poll(self): + self.returncode = next(self.statuses, self.returncode) + return self.returncode + + def terminate(self): + self.returncode = 0 + + clean = FakeProcess([0]) + self.assertEqual(supervise(["replay"], [], {}, spawn=lambda *_args, **_kwargs: clean), 0) + replay = FakeProcess([None, None]) + aborted_ui = FakeProcess([-6]) + queue = iter((replay, aborted_ui)) + self.assertEqual(supervise(["replay"], [["ui"]], {}, spawn=lambda *_args, **_kwargs: next(queue)), 134) + + def test_native_pubmaster_uses_owned_ipc_parent_and_cleanup(self): + prefix = f"replay-test-{os.getpid()}-{time.time_ns()}" + root = _ipc_root(prefix) + self.assertFalse(root.exists()) + with owned_ipc_namespace(prefix): + self.assertTrue(root.is_dir()) + env = dict(os.environ, OPENPILOT_PREFIX=prefix) + env.pop("CEREAL_FAKE", None) + result = subprocess.run([sys.executable, "-c", "from openpilot.cereal.messaging import PubMaster; PubMaster(['accelerometer'])"], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue((root / "accelerometer").exists()) + self.assertFalse(root.exists()) + + def test_existing_ipc_namespace_is_never_removed(self): + prefix = f"replay-test-{os.getpid()}-{time.time_ns()}" + root = _ipc_root(prefix) + root.mkdir() + try: + sentinel = root / "external" + sentinel.write_text("keep") + with self.assertRaisesRegex(RuntimeError, "already has an IPC namespace"): + with owned_ipc_namespace(prefix): + pass + self.assertEqual(sentinel.read_text(), "keep") + finally: + sentinel.unlink() + root.rmdir() + + def test_help_requires_no_runtime_or_route(self): + result = subprocess.run([sys.executable, "-m", "openpilot.tools.replay.onroad", "--help"], + capture_output=True, text=True, check=False) + self.assertEqual(result.returncode, 0) + self.assertIn("--replay-only", result.stdout) + + def test_route_ui_and_option_preservation(self): + plan = parse_onroad_args(["--c4", "--prefix", "replay-fixture", "--start", "30", "--data_dir=/private/routes", + "route/abc", "--no-loop"]) + self.assertEqual(plan.targets, ("c4",)) + self.assertEqual(plan.prefix, "replay-fixture") + self.assertEqual(plan.replay_args, ("--start", "30", "--data_dir=/private/routes", "route/abc", "--no-loop")) + self.assertEqual(select_ui_target(SimpleNamespace(deviceType="mici")), "c4") + self.assertEqual(select_ui_target(None), "c3") + self.assertEqual(parse_onroad_args(["--all", "--demo"]).targets, ("c3", "c4")) + self.assertEqual(parse_onroad_args(["--replay-only", "--demo"]).targets, ()) + + def test_unsupported_or_unsafe_options_fail_before_spawn(self): + for args in (["--nav", "--demo"], ["--replay-only", "--alert", "--demo"], + ["--ui=none", "-alert", "--demo"], + ["--prefix", "d", "--demo"], ["--demo", "--", "--prefix", "replay-other"], + ["--headless", "--demo"], ["--auto"], ["--ui=bogus", "--demo"]): + with self.subTest(args=args), self.assertRaises(ValueError): + parse_onroad_args(args) + + def test_alert_preview_preserves_route_and_combines_native_blocklist(self): + plan = parse_onroad_args(["--c4", "-alert", "-b", "carControl,selfdriveState", "--start", "30", "--demo"]) + self.assertTrue(plan.alert) + self.assertEqual(plan.targets, ("c4",)) + self.assertEqual(_block_alert_service(plan.replay_args), + ["-b", "carControl,selfdriveState", "--start", "30", "--demo"]) + self.assertEqual(_block_alert_service(("--block=carState", "--demo")), + ["-b", "carState,selfdriveState", "--demo"]) + self.assertEqual(_block_alert_service(("-b", "carState", "--block=carControl,selfdriveState", "--demo")), + ["-b", "carState,carControl,selfdriveState", "--demo"]) + self.assertFalse(parse_onroad_args(["--c3", "--demo"]).alert) + self.assertFalse(parse_onroad_args(["--demo"]).visual_preview) + + def test_cem_csc_aliases_are_ui_only_and_combine_with_alert(self): + plan = parse_onroad_args(["--c4", "--mici-widget-demo", "--csc-demo", "-alert", "--demo"]) + self.assertEqual(plan.visual_preview, frozenset(("cem", "csc"))) + self.assertTrue(plan.alert) + self.assertEqual(plan.replay_args, ("--demo",)) + self.assertEqual(parse_onroad_args(["--c3", "--widget-demo", "--demo"]).visual_preview, + frozenset(("cem",))) + self.assertEqual(parse_onroad_args(["--c3", "--csc", "--demo"]).visual_preview, + frozenset(("csc",))) + for args in (["--replay-only", "--cem", "--demo"], ["--ui=none", "--csc", "--demo"]): + with self.subTest(args=args), self.assertRaisesRegex(ValueError, "require a native UI"): + parse_onroad_args(args) + + def test_alert_publisher_failure_reaps_replay_and_ui(self): + class FakeProcess: + def __init__(self, statuses): + self.statuses = iter(statuses) + self.returncode = None + self.terminated = False + + def poll(self): + self.returncode = next(self.statuses, self.returncode) + return self.returncode + + def terminate(self): + self.terminated = True + self.returncode = 0 + + replay = FakeProcess([None, None]) + ui = FakeProcess([None]) + alert = FakeProcess([2]) + queue = iter((replay, ui, alert)) + self.assertEqual(supervise(["replay"], [["ui"]], {}, demo_commands=[["alert"]], + spawn=lambda *_args, **_kwargs: next(queue)), 2) + self.assertTrue(replay.terminated) + self.assertTrue(ui.terminated) + replay = FakeProcess([None, None]) + ui = FakeProcess([None]) + alert = FakeProcess([0]) + queue = iter((replay, ui, alert)) + self.assertEqual(supervise(["replay"], [["ui"]], {}, demo_commands=[["alert"]], + spawn=lambda *_args, **_kwargs: next(queue)), 1) + + def test_alert_run_passes_one_owned_publisher_and_replay_exclusion(self): + with tempfile.TemporaryDirectory() as directory: + private = Path(directory) / "params" + env = {"SP_HOST_RUNTIME": "1", "SP_HOST_PARAMS_ROOT": str(private), "PARAMS_ROOT": str(private), + "SP_HOST_PREFIX": "starpilot-dev-host", "OPENPILOT_PREFIX": "starpilot-dev-host"} + with patch.dict(os.environ, env), patch.object(Path, "is_file", return_value=True), \ + patch("openpilot.tools.replay.onroad.os.access", return_value=True), \ + patch("openpilot.tools.replay.onroad_config.route_init_data", return_value=None), \ + patch("openpilot.starpilot.ui.host_launch.launch_environment", return_value={}), \ + patch("openpilot.tools.replay.onroad.supervise", return_value=0) as owned: + self.assertEqual(run(parse_onroad_args(["--c4", "--cem", "--csc", "-alert", "--prefix", "replay-alertfixture", + "-b", "carControl", "--demo"])), 0) + replay, ui, child_env, kwargs = owned.call_args.args[0], owned.call_args.args[1], owned.call_args.args[2], owned.call_args.kwargs + self.assertEqual(replay[1:3], ["-b", "carControl,selfdriveState"]) + self.assertEqual(len(ui), 1) + self.assertEqual(child_env["SP_ONROAD_VISUAL_PREVIEW"], "cem,csc") + self.assertEqual(kwargs["demo_commands"], [[sys.executable, "-m", "openpilot.tools.replay.alert_demo"]]) + + def test_actual_private_alert_pubsub_shows_then_clears(self): + prefix = f"replay-alert-{os.getpid()}-{time.time_ns()}" + env = dict(os.environ, OPENPILOT_PREFIX=prefix, USE_MSGQ_PREFIX="true", SP_HOST_RUNTIME="1") + env.pop("CEREAL_FAKE", None) + receiver = """import time +from openpilot.cereal import messaging +from openpilot.starpilot.ui.runtime_snapshot import current_message, _alert +sm = messaging.SubMaster(['selfdriveState'], ignore_alive=['selfdriveState']) +seen = [] +deadline = time.monotonic() + 5 +while time.monotonic() < deadline: + sm.update(100) + if not sm.updated['selfdriveState']: + continue + msg = current_message(sm, 'selfdriveState', time.monotonic_ns()) + if msg is None: + continue + visual = _alert(msg) + if visual.size.value == 'full': + assert visual.critical and visual.text1 == 'TAKE CONTROL IMMEDIATELY' + seen.append('full') + elif visual.size.value == 'none' and 'full' in seen: + seen.append('clear') + break +assert seen[-2:] == ['full', 'clear'], seen +""" + with owned_ipc_namespace(prefix): + subscriber = subprocess.Popen([sys.executable, "-c", receiver], env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + publisher = subprocess.Popen([sys.executable, "-m", "openpilot.tools.replay.alert_demo", + "--delay", ".2", "--hold", ".5"], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + stdout, stderr = subscriber.communicate(timeout=7) + self.assertEqual(subscriber.returncode, 0, stdout + stderr) + self.assertIsNone(publisher.poll(), "Preview publisher must remain owned after clearing") + finally: + if subscriber.poll() is None: + subscriber.kill() + subscriber.wait(timeout=2) + if publisher.poll() is None: + publisher.terminate() + publisher.communicate(timeout=3) + + def test_alert_module_refuses_unowned_namespace_before_publishing(self): + env = dict(os.environ, SP_HOST_RUNTIME="1", OPENPILOT_PREFIX="replay-missing-namespace") + result = subprocess.run([sys.executable, "-m", "openpilot.tools.replay.alert_demo", "--delay", "0"], + env=env, capture_output=True, text=True, check=False, timeout=5) + self.assertEqual(result.returncode, 2) + self.assertIn("isolated host replay session", result.stderr) + + def test_local_first_log_choice_does_not_fetch(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "private.rlog.zst" + path.write_bytes(b"fixture") + self.assertEqual(first_log_identifier(parse_replay_args([str(path)])), str(path)) + self.assertIsNotNone(first_log_identifier(parse_replay_args(["--demo"]))) + + def test_data_dir_uses_native_timestamp_segment_layout_without_remote_fallback(self): + route = "0123456789abcdef/2024-01-02--03-04-05/0" + with tempfile.TemporaryDirectory() as directory: + segment = Path(directory) / "2024-01-02--03-04-05--0" + segment.mkdir() + local = segment / "rlog.zst" + local.write_bytes(b"fixture") + args = parse_replay_args(["--data_dir", directory, route]) + self.assertEqual(first_log_identifier(args), str(local)) + local.unlink() + self.assertIsNone(first_log_identifier(args)) + + def test_seed_only_registered_display_booleans_in_disposable_params(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, OPENPILOT_PREFIX="replay-test-seed"): + params = Params(directory) + entries = [SimpleNamespace(key="IsMetric", value=b"1"), SimpleNamespace(key="HideSpeed", value=b"0"), + SimpleNamespace(key="HideMaxSpeed", value=b"not-a-bool"), + SimpleNamespace(key="AccessToken", value=b"secret")] + init = SimpleNamespace(params=SimpleNamespace(entries=entries)) + self.assertEqual(seed_preview(init, params), 2) + self.assertIs(params.get("IsMetric"), True) + self.assertIs(params.get("HideSpeed"), False) + self.assertIsNone(params.get("HideMaxSpeed")) + self.assertIsNone(params.get("AccessToken")) + self.assertIs(params.get("OpenpilotEnabledToggle"), True) + + def test_seed_uses_replay_namespace_not_runner_namespace(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, OPENPILOT_PREFIX="starpilot-dev-original"): + init = SimpleNamespace(params=SimpleNamespace(entries=[SimpleNamespace(key="IsMetric", value=b"1")])) + self.assertEqual(seed_replay_params(init, directory, "replay-target"), 1) + self.assertEqual(os.environ["OPENPILOT_PREFIX"], "starpilot-dev-original") + self.assertEqual((Path(directory) / "replay-target" / "IsMetric").read_bytes(), b"1") + self.assertFalse((Path(directory) / "starpilot-dev-original").exists()) + + def test_replay_only_does_not_fetch_route_metadata(self): + for ui_args in (["--replay-only"], ["--ui=none"]): + with self.subTest(ui_args=ui_args), tempfile.TemporaryDirectory() as directory: + private = Path(directory) / "params" + env = {"SP_HOST_RUNTIME": "1", "SP_HOST_PARAMS_ROOT": str(private), "PARAMS_ROOT": str(private), + "SP_HOST_PREFIX": "starpilot-dev-host", "OPENPILOT_PREFIX": "starpilot-dev-host"} + with patch.dict(os.environ, env), \ + patch.object(Path, "is_file", return_value=True), \ + patch("openpilot.tools.replay.onroad.os.access", return_value=True), \ + patch("openpilot.tools.replay.onroad_config.route_init_data", side_effect=AssertionError("fetched")), \ + patch("openpilot.tools.replay.onroad.supervise", return_value=0) as owned: + self.assertEqual(run(parse_onroad_args([*ui_args, "--prefix", "replay-onlyfixture", "--demo"])), 0) + self.assertEqual(owned.call_count, 1) + self.assertFalse((Path(directory) / "replay-session-replay-onlyfixture").exists()) + + def test_replay_exit_reaps_other_child(self): + with tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "terminated" + waiting = "; ".join(("import signal,time,sys", + "signal.signal(signal.SIGTERM, lambda *_: (open(sys.argv[1], 'w').write('yes'), sys.exit(0)))", + "time.sleep(10)")) + exiting = "import time; time.sleep(.2)" + result = supervise([sys.executable, "-c", exiting], [[sys.executable, "-c", waiting, str(marker)]], dict(os.environ)) + self.assertEqual(result, 0) + self.assertEqual(marker.read_text(), "yes") + + def test_termination_reaps_replay_and_ui(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + ready = root / "ready" + replay_stopped = root / "replay-stopped" + ui_stopped = root / "ui-stopped" + worker = "; ".join(("import signal,time,sys", "from pathlib import Path", + "Path(sys.argv[1]).write_text('ready')", + "signal.signal(signal.SIGTERM, lambda *_: (Path(sys.argv[2]).write_text('stopped'), sys.exit(0)))", + "time.sleep(20)")) + call = "raise SystemExit(supervise([sys.executable,'-c',worker,paths[0],paths[1]]," + \ + "[[sys.executable,'-c',worker,paths[2],paths[3]]],dict(os.environ)))" + supervisor = "; ".join(("import os,sys", "from openpilot.tools.replay.onroad import supervise", + "worker=sys.argv[1]", "paths=sys.argv[2:]", call)) + proc = subprocess.Popen([sys.executable, "-c", supervisor, worker, str(ready), str(replay_stopped), + str(root / "ui-ready"), str(ui_stopped)], env=dict(os.environ)) + try: + deadline = time.monotonic() + 5 + while not (ready.exists() and (root / "ui-ready").exists()) and time.monotonic() < deadline: + time.sleep(.02) + self.assertTrue(ready.exists() and (root / "ui-ready").exists()) + proc.send_signal(signal.SIGTERM) + self.assertEqual(proc.wait(timeout=5), 143) + self.assertEqual(replay_stopped.read_text(), "stopped") + self.assertEqual(ui_stopped.read_text(), "stopped") + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=2) diff --git a/pyproject.toml b/pyproject.toml index 14416fedd2..54fe397fb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,18 @@ dependencies = [ ] [project.optional-dependencies] +dev = [ + "pytest>=8,<9", + "pytest-xdist>=3,<4", + "pytest-mock>=3,<4", +] +safety = [ + "cffi", + "gcovr==8.6", + "tree-sitter==0.26.0", + "tree-sitter-c==0.24.2", + "cppcheck @ git+https://github.com/commaai/dependencies.git@b7253ddb101ed6add04fc2a2ce0688d5e21a55ee#subdirectory=cppcheck", +] testing = [ "coverage", # line coverage "ty", # type checking @@ -104,9 +116,10 @@ allow-direct-references = true [tool.codespell] quiet-level = 3 # if you've got a short variable name that's getting flagged, add it here -ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn,ws,uint,grey,deque,stdio,amin,BA,LITE,atEnd,UIs,errorString,arange,FocusIn,od,tim,relA,hist,copyable,jupyter,thead,TGE,abl,lite,ser" +ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn,ws,uint,grey,deque,stdio,amin,BA,LITE,atEnd,UIs,errorString,arange,FocusIn,od,tim,relA,hist,copyable,jupyter,thead,TGE,abl,lite,ser,assertIn,HomeState,cancelled,CAS,acknowledgement" builtin = "clear,rare,informal,code,names,en-GB_to_en-US" -skip = "*.po, uv.lock, *.onnx, *.pem, */c_generated_code/*, docs/assets/*, openpilot/tools/plotjuggler/layouts/*, openpilot/selfdrive/assets/offroad/mici_fcc.html" +skip = "*.po, uv.lock, *.onnx, *.pem, */c_generated_code/*, docs/assets/*, openpilot/tools/plotjuggler/layouts/*, openpilot/selfdrive/assets/offroad/mici_fcc.html, openpilot/starpilot/galaxy/web/vendor/vue/vue.esm-browser.js, openpilot/starpilot/ui/assets/fonts/notices/Unifont-COPYING.txt" +ignore-regex = "scale\\.numer" # https://docs.astral.sh/ruff/configuration/#using-pyprojecttoml [tool.ruff] @@ -143,6 +156,9 @@ lint.flake8-implicit-str-concat.allow-multiline = false [tool.ruff.format] quote-style = "preserve" +[tool.ty.environment] +extra-paths = ["opendbc_repo", "msgq_repo", "rednose_repo", "tinygrad_repo", "teleoprtc_repo"] + [tool.ty.rules] unresolved-import = "ignore" # Cython-compiled modules (.pyx) unresolved-attribute = "ignore" # many from capnp and Cython modules diff --git a/scripts/host_tool_runner.sh b/scripts/host_tool_runner.sh new file mode 100755 index 0000000000..1f119dc487 --- /dev/null +++ b/scripts/host_tool_runner.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -x "${ROOT_DIR}/.venv/bin/python3" ]]; then + exec "${ROOT_DIR}/.venv/bin/python3" "${ROOT_DIR}/tools/host_runtime.py" "$@" +fi +exec python3 "${ROOT_DIR}/tools/host_runtime.py" "$@" diff --git a/scripts/laptop_device_build.sh b/scripts/laptop_device_build.sh new file mode 100755 index 0000000000..7df6e07cc4 --- /dev/null +++ b/scripts/laptop_device_build.sh @@ -0,0 +1,855 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +cd "${ROOT_DIR}" + +HOST_ROOT_DIR="${COMMA_HOST_ROOT_DIR:-${ROOT_DIR}}" +DOCKER_RUN_USER="${COMMA_DOCKER_RUN_USER:-$(id -u):$(id -g)}" +HOST_VENV_DIR="${COMMA_HOST_VENV_DIR:-${HOST_ROOT_DIR}/.venv-linux-arm64}" + +# Make Docker Desktop binaries discoverable (docker + credential helpers) even +# when the caller shell PATH is minimal. +if [[ -d /Applications/Docker.app/Contents/Resources/bin ]]; then + export PATH="/Applications/Docker.app/Contents/Resources/bin:${PATH}" +fi + +IMAGE_NAME="${COMMA_BUILD_IMAGE:-openpilot-local-arm64-builder:latest}" +SYSROOT_DIR_DEFAULT="${ROOT_DIR}/.comma_sysroot" +SYSROOT_DIR="${COMMA_SYSROOT_DIR:-${SYSROOT_DIR_DEFAULT}}" +HOST_SYSROOT_DIR="${COMMA_HOST_SYSROOT_DIR:-${SYSROOT_DIR}}" +HOST_CACHE_DIR="${COMMA_HOST_CACHE_DIR:-${HOST_ROOT_DIR}/.cache}" +HOST_DOCKERFILE_PATH="${COMMA_HOST_DOCKERFILE_PATH:-${HOST_ROOT_DIR}/tools/laptop_device_build/Dockerfile}" + +# Docker treats a relative bind source as a named volume. Resolve accepted +# overrides against this checkout before any mount is assembled. +for path_name in HOST_VENV_DIR SYSROOT_DIR HOST_SYSROOT_DIR HOST_CACHE_DIR; do + path_value="${!path_name}" + if [[ "${path_value}" != /* ]]; then + printf -v "${path_name}" '%s/%s' "${ROOT_DIR}" "${path_value}" + fi +done + +# A Docker bind mount must never point at a different checkout than the tree +# whose outputs this script scrubs and marks prebuilt. Keep all writable build +# locations under this checkout, including env-configured overrides. +[[ "$(cd "${HOST_ROOT_DIR}" && pwd -P)" == "$(cd "${ROOT_DIR}" && pwd -P)" ]] || { + echo "ERROR: COMMA_HOST_ROOT_DIR must resolve to this checkout" >&2; exit 1; +} +for writable in "${SYSROOT_DIR}" "${HOST_SYSROOT_DIR}" "${HOST_VENV_DIR}" "${HOST_CACHE_DIR}"; do + parent="$(cd "$(dirname "${writable}")" && pwd -P)" || exit 1 + [[ "${parent}" == "${ROOT_DIR}" ]] || { + echo "ERROR: writable build paths must be direct children of this checkout: ${writable}" >&2; exit 1; + } + if [[ -e "${writable}" ]]; then + [[ "$(cd "${writable}" && pwd -P)" == "${ROOT_DIR}"/* ]] || { + echo "ERROR: writable build path resolves outside this checkout: ${writable}" >&2; exit 1; + } + fi +done +[[ "${HOST_SYSROOT_DIR}" == "${SYSROOT_DIR}" ]] || { + echo "ERROR: COMMA_HOST_SYSROOT_DIR must match COMMA_SYSROOT_DIR" >&2; exit 1; +} + +usage() { + cat <<'EOF' +Usage: + scripts/laptop_device_build.sh doctor + scripts/laptop_device_build.sh verify-artifacts + scripts/laptop_device_build.sh setup [device-host] [device-user] [ssh-port] + scripts/laptop_device_build.sh setup-sysroot [device-user] [ssh-port] + scripts/laptop_device_build.sh setup-sysroot-agnos [manifest-path] + scripts/laptop_device_build.sh setup-sysroot-local + scripts/laptop_device_build.sh build-image + scripts/laptop_device_build.sh build [jobs] [scons args...] + scripts/laptop_device_build.sh scons [--no-scrub] [scons args...] + scripts/laptop_device_build.sh manager [jobs] [--no-build] [-- manager args...] + scripts/laptop_device_build.sh shell + +Modes: + doctor Check host/container prerequisites for laptop-side device builds. + verify-artifacts Check complete device outputs without running a build. + setup One-time setup for laptop builds (venv + image + sysroot). + setup-sysroot Sync required runtime/linker libs from a comma device over SSH. + setup-sysroot-agnos Download AGNOS system image and extract sysroot locally. + setup-sysroot-local Extract sysroot from an existing local AGNOS system image. + build-image Build the Linux/aarch64 container image used for device-target builds. + build Run the complete device-target build; set prebuilt only after artifact checks. + scons Run raw scons in the larch64 container (no prebuilt touch). + manager Explicitly run openpilot/system/manager/manager.py in the container. + shell Open an interactive shell in the build container with all mounts. + +Environment overrides: + COMMA_BUILD_IMAGE Container image tag (default: openpilot-local-arm64-builder:latest) + COMMA_SYSROOT_DIR Sysroot location (default: ./.comma_sysroot) + COMMA_AUTOSTART_DOCKER Auto-launch Docker Desktop on macOS when needed (default: 1) + RELEASE, CERT For release firmware, set RELEASE to a nonempty value and CERT to + a readable certificate file; it is mounted read-only. +EOF +} + +err() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || err "Missing required command: $1" +} + +detect_engine() { + local docker_cmd="" + if command -v docker >/dev/null 2>&1; then + docker_cmd="$(command -v docker)" + elif [[ -x /Applications/Docker.app/Contents/Resources/bin/docker ]]; then + docker_cmd="/Applications/Docker.app/Contents/Resources/bin/docker" + fi + + if [[ -n "${docker_cmd}" ]]; then + # Needed for docker credential helpers (e.g. docker-credential-desktop). + export PATH="$(dirname "${docker_cmd}"):${PATH}" + if ! "${docker_cmd}" info >/dev/null 2>&1; then + if [[ "$(uname -s)" == "Darwin" ]] && [[ "${COMMA_AUTOSTART_DOCKER:-1}" =~ ^(1|true|yes|on)$ ]]; then + if [[ -d /Applications/Docker.app ]] && command -v open >/dev/null 2>&1; then + echo "Docker daemon is not running. Launching Docker Desktop..." >&2 + open -g -a Docker >/dev/null 2>&1 || open -a Docker >/dev/null 2>&1 || true + for ((i=0; i<300; i++)); do + if "${docker_cmd}" info >/dev/null 2>&1; then + break + fi + sleep 1 + done + fi + fi + if ! "${docker_cmd}" info >/dev/null 2>&1; then + err "Docker CLI found, but Docker daemon is not running. Start Docker Desktop (open -a Docker) and retry." + fi + fi + echo "${docker_cmd}" + return + fi + if command -v podman >/dev/null 2>&1; then + local podman_cmd + podman_cmd="$(command -v podman)" + if ! "${podman_cmd}" info >/dev/null 2>&1; then + err "Podman CLI found, but Podman service is not running." + fi + echo "${podman_cmd}" + return + fi + err "No container runtime found (install docker or podman)." +} + +build_container_image_cmd() { + local engine="$1" + shift + local build_args=("$@") + if [[ -n "${COMMA_BASE_IMAGE:-}" ]]; then + build_args=(--build-arg "BASE_IMAGE=${COMMA_BASE_IMAGE}" "${build_args[@]}") + fi + + if [[ "${engine##*/}" == "docker" ]] && "${engine}" buildx version >/dev/null 2>&1; then + "${engine}" buildx build --load --platform linux/arm64 "${build_args[@]}" + return + fi + + "${engine}" build --platform linux/arm64 "${build_args[@]}" +} + +ensure_image_exists() { + local engine="$1" + if ! "${engine}" image inspect "${IMAGE_NAME}" >/dev/null 2>&1; then + echo "Container image ${IMAGE_NAME} not found. Building it now..." + build_container_image_cmd "${engine}" -f "${HOST_DOCKERFILE_PATH}" -t "${IMAGE_NAME}" "${HOST_ROOT_DIR}" + fi + assert_image_arch "${engine}" + ensure_image_capnp_version "${engine}" +} + +assert_image_arch() { + local engine="$1" + local arch + arch="$("${engine}" image inspect "${IMAGE_NAME}" --format '{{.Architecture}}' 2>/dev/null || true)" + if [[ "${arch}" != "arm64" ]]; then + err "Container image ${IMAGE_NAME} is '${arch:-unknown}', expected 'arm64'. Rebuild with: ${engine} build --pull --platform linux/arm64 -f tools/laptop_device_build/Dockerfile -t ${IMAGE_NAME} ." + fi +} + +expected_capnp_version() { + if [[ -n "${COMMA_EXPECTED_CAPNP_VERSION:-}" ]]; then + echo "${COMMA_EXPECTED_CAPNP_VERSION}" + return + fi + + local dockerfile_version="" + dockerfile_version="$(sed -n 's/^ARG CAPNP_VERSION=\(.*\)$/\1/p' "${ROOT_DIR}/tools/laptop_device_build/Dockerfile" | head -n 1)" + if [[ -n "${dockerfile_version}" ]]; then + echo "${dockerfile_version}" + return + fi + + local header_path="${ROOT_DIR}/openpilot/cereal/gen/cpp/custom.capnp.h" + if [[ ! -f "${header_path}" ]]; then + err "Unable to determine expected Cap'n Proto version. Set COMMA_EXPECTED_CAPNP_VERSION or restore tools/laptop_device_build/Dockerfile ARG CAPNP_VERSION." + fi + + local raw_version="" + raw_version="$(sed -n 's/^#elif CAPNP_VERSION != \([0-9][0-9]*\)$/\1/p' "${header_path}" | head -n 1)" + [[ -n "${raw_version}" ]] || err "Unable to determine expected Cap'n Proto version from ${header_path}." + + local major=$(( raw_version / 1000000 )) + local minor=$(( (raw_version / 1000) % 1000 )) + local micro=$(( raw_version % 1000 )) + echo "${major}.${minor}.${micro}" +} + +image_capnp_version() { + local engine="$1" + local version_output="" + + version_output="$("${engine}" run --rm --platform linux/arm64 "${IMAGE_NAME}" capnp --version 2>&1 || true)" + printf '%s\n' "${version_output}" \ + | sed -nE 's/.*version[[:space:]]+([0-9]+\.[0-9]+\.[0-9]+).*/\1/p' \ + | head -n 1 +} + +ensure_image_capnp_version() { + local engine="$1" + local expected actual + expected="$(expected_capnp_version)" + actual="$(image_capnp_version "${engine}")" + + if [[ "${actual}" == "${expected}" ]]; then + return + fi + + echo "Container image ${IMAGE_NAME} has Cap'n Proto ${actual:-unknown}, expected ${expected}. Rebuilding it now..." + build_container_image_cmd "${engine}" -f "${HOST_DOCKERFILE_PATH}" -t "${IMAGE_NAME}" "${HOST_ROOT_DIR}" + + actual="$(image_capnp_version "${engine}")" + if [[ "${actual}" != "${expected}" ]]; then + err "Container image ${IMAGE_NAME} still has Cap'n Proto ${actual:-unknown} after rebuild; expected ${expected}." + fi +} + +assert_runtime_machine() { + local engine="$1" + local machine + machine="$("${engine}" run --rm --platform linux/arm64 "${IMAGE_NAME}" bash -lc "readelf -h \$(command -v python3) | awk -F: '/Machine/ {gsub(/^ +/, \"\", \$2); print \$2}'" 2>/dev/null || true)" + if [[ "${machine}" != "AArch64" ]]; then + err "Container Python runtime is '${machine:-unknown}', expected 'AArch64'. This image can still build device artifacts, but cannot run manager with larch64 Python extensions. Rebuild from an arm64-capable base image before using 'manager' mode." + fi +} + +default_jobs() { + if command -v nproc >/dev/null 2>&1; then + nproc + elif command -v sysctl >/dev/null 2>&1; then + sysctl -n hw.ncpu + else + echo 8 + fi +} + +build_container_image() { + local engine + engine="$(detect_engine)" + build_container_image_cmd "${engine}" \ + -f "${HOST_DOCKERFILE_PATH}" \ + -t "${IMAGE_NAME}" "${HOST_ROOT_DIR}" + assert_image_arch "${engine}" +} + +sanitize_sysroot_headers() { + # Device /usr/local/include can include older capnp/kj headers that conflict + # with generated cereal headers in this tree. Keep ffmpeg headers, drop capnp. + local removed=0 + local path="" + for path in \ + "${SYSROOT_DIR}/usr/local/include/capnp" \ + "${SYSROOT_DIR}/usr/local/include/kj"; do + if [[ -e "${path}" ]]; then + rm -rf "${path}" + removed=1 + fi + done + if [[ "${removed}" -eq 1 ]]; then + echo "Sanitized sysroot headers: removed capnp/kj from usr/local/include" + fi +} + +ensure_sysroot_layout() { + [[ -d "${SYSROOT_DIR}/usr/local/lib" ]] || err "Missing ${SYSROOT_DIR}/usr/local/lib. Run setup-sysroot first." + [[ -d "${SYSROOT_DIR}/usr/local/include" ]] || err "Missing ${SYSROOT_DIR}/usr/local/include. Run setup-sysroot first." + [[ -d "${SYSROOT_DIR}/lib/aarch64-linux-gnu" ]] || err "Missing ${SYSROOT_DIR}/lib/aarch64-linux-gnu. Run setup-sysroot first." + [[ -d "${SYSROOT_DIR}/usr/lib/aarch64-linux-gnu" ]] || err "Missing ${SYSROOT_DIR}/usr/lib/aarch64-linux-gnu. Run setup-sysroot first." + [[ -d "${SYSROOT_DIR}/usr/include" ]] || err "Missing ${SYSROOT_DIR}/usr/include. Run setup-sysroot first." + [[ -d "${SYSROOT_DIR}/system/vendor/lib64" ]] || err "Missing ${SYSROOT_DIR}/system/vendor/lib64. Run setup-sysroot first." + sanitize_sysroot_headers + repair_sysroot_linker +} + +is_aarch64_elf() { + local path="${1:-}" + [[ -f "${path}" ]] || return 1 + local desc + desc="$(file -b "${path}" 2>/dev/null || true)" + [[ "${desc}" == *"ELF 64-bit LSB"* && "${desc}" == *"ARM aarch64"* ]] +} + +repair_sysroot_linker() { + local root_ld="${SYSROOT_DIR}/lib/ld-linux-aarch64.so.1" + local lib_ld="${SYSROOT_DIR}/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1" + local usr_ld="${SYSROOT_DIR}/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1" + + # Some sysroot extraction paths can create this as an empty directory. + if [[ -d "${lib_ld}" ]]; then + rm -rf "${lib_ld}" + fi + + # Ensure lib-path loader exists (symlink to usr/lib loader is fine). + if [[ ! -e "${lib_ld}" && -f "${usr_ld}" ]]; then + mkdir -p "$(dirname "${lib_ld}")" + ln -s "../../usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1" "${lib_ld}" + fi + # Debian's libc.so linker script names the loader at /lib directly, while + # AGNOS stores it under the multiarch directory. + if [[ ! -e "${root_ld}" && -f "${lib_ld}" ]]; then + ln -s "aarch64-linux-gnu/ld-linux-aarch64.so.1" "${root_ld}" + fi +} + +resolve_ldso_source() { + local lib_ld="${SYSROOT_DIR}/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1" + local usr_ld="${SYSROOT_DIR}/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1" + + repair_sysroot_linker + + if is_aarch64_elf "${lib_ld}"; then + echo "${lib_ld}" + return 0 + fi + if is_aarch64_elf "${usr_ld}"; then + echo "${usr_ld}" + return 0 + fi + + err "Missing valid aarch64 dynamic loader in sysroot. Expected one of: ${lib_ld} or ${usr_ld}. Re-run setup-sysroot." +} + +scrub_mixed_arch_artifacts() { + # Desktop launchers can leave macOS objects with the same paths as larch64 outputs. + # Remove known cross-target extension intermediates so device builds always relink. + rm -f \ + "${ROOT_DIR}/.sconsign.dblite" \ + "${ROOT_DIR}/openpilot/common/libcommon.a" \ + "${ROOT_DIR}/openpilot/common/params.o" \ + "${ROOT_DIR}/openpilot/common/params_c.os" \ + "${ROOT_DIR}/openpilot/cereal/libcereal.a" \ + "${ROOT_DIR}/openpilot/cereal/libsocketmaster.a" \ + "${ROOT_DIR}/openpilot/cereal/gen/cpp/"*.capnp.c++ \ + "${ROOT_DIR}/openpilot/cereal/gen/cpp/"*.capnp.h \ + "${ROOT_DIR}/openpilot/cereal/gen/cpp/"*.capnp.o \ + "${ROOT_DIR}/openpilot/cereal/messaging/"*.o \ + "${ROOT_DIR}/openpilot/cereal/messaging/bridge" \ + "${ROOT_DIR}/msgq_repo/msgq/"*.os \ + "${ROOT_DIR}/msgq_repo/msgq/"*.o \ + "${ROOT_DIR}/msgq_repo/libmsgq.a" \ + "${ROOT_DIR}/msgq_repo/libvisionipc.a" \ + "${ROOT_DIR}/msgq_repo/msgq/ipc_pyx.so" \ + "${ROOT_DIR}/msgq_repo/msgq/visionipc/"*.os \ + "${ROOT_DIR}/msgq_repo/msgq/visionipc/"*.o \ + "${ROOT_DIR}/msgq_repo/msgq/visionipc/visionipc_pyx.so" \ + "${ROOT_DIR}/openpilot/common/libparams_c.so" \ + "${ROOT_DIR}/openpilot/selfdrive/pandad/can_list_to_can_capnp.o" \ + "${ROOT_DIR}/openpilot/selfdrive/pandad/libcan_list_to_can_capnp.a" \ + "${ROOT_DIR}/openpilot/selfdrive/pandad/pandad" +} + +sync_sysroot_from_device() { + local host="${1:-}" + local user="${2:-comma}" + local port="${3:-22}" + [[ -n "${host}" ]] || err "setup-sysroot requires ." + + require_cmd rsync + mkdir -p "${SYSROOT_DIR}" + local ssh_cmd="ssh -p ${port} -o StrictHostKeyChecking=accept-new" + + rsync -a --delete -e "${ssh_cmd}" "${user}@${host}:/usr/local/lib/" "${SYSROOT_DIR}/usr/local/lib/" + rsync -a --delete -e "${ssh_cmd}" "${user}@${host}:/usr/local/include/" "${SYSROOT_DIR}/usr/local/include/" + rsync -a --delete -e "${ssh_cmd}" "${user}@${host}:/lib/aarch64-linux-gnu/" "${SYSROOT_DIR}/lib/aarch64-linux-gnu/" + rsync -a --delete -e "${ssh_cmd}" "${user}@${host}:/usr/lib/aarch64-linux-gnu/" "${SYSROOT_DIR}/usr/lib/aarch64-linux-gnu/" + rsync -a --delete -e "${ssh_cmd}" "${user}@${host}:/usr/include/" "${SYSROOT_DIR}/usr/include/" + + local vendor_src="" + for candidate in /system/vendor/lib64 /vendor/lib64; do + if ssh -p "${port}" -o StrictHostKeyChecking=accept-new "${user}@${host}" "test -d '${candidate}'"; then + vendor_src="${candidate}" + break + fi + done + + if [[ -n "${vendor_src}" ]]; then + rsync -a --delete -e "${ssh_cmd}" "${user}@${host}:${vendor_src}/" "${SYSROOT_DIR}/system/vendor/lib64/" + else + local vendor_dst="${SYSROOT_DIR}/system/vendor/lib64" + echo "Warning: no vendor lib64 dir found on device; falling back to usr/lib/aarch64-linux-gnu" + if [[ -L "${vendor_dst}" || -f "${vendor_dst}" ]]; then + rm -f "${vendor_dst}" + elif [[ -d "${vendor_dst}" ]]; then + rm -rf "${vendor_dst}" + fi + mkdir -p "${SYSROOT_DIR}/system/vendor" + ln -s ../../usr/lib/aarch64-linux-gnu "${vendor_dst}" + fi + + sanitize_sysroot_headers + + echo "Sysroot synced to: ${SYSROOT_DIR}" +} + +setup_sysroot_from_agnos() { + local manifest="${1:-openpilot/system/hardware/tici/agnos.json}" + local engine + engine="$(detect_engine)" + ensure_image_exists "${engine}" + mkdir -p "${SYSROOT_DIR}" "${ROOT_DIR}/.cache/agnos" "${HOST_CACHE_DIR}/agnos" + + "${engine}" run --rm --platform linux/arm64 \ + --user "${DOCKER_RUN_USER}" \ + -v "${HOST_ROOT_DIR}:/work" \ + -v "${HOST_SYSROOT_DIR}:/opt/tici-sysroot" \ + -v "${HOST_CACHE_DIR}:/work/.cache" \ + -w /work \ + "${IMAGE_NAME}" \ + /usr/bin/python3 tools/laptop_device_build/extract_sysroot_from_agnos.py \ + --manifest "${manifest}" \ + --output-dir /opt/tici-sysroot \ + --cache-dir /work/.cache/agnos +} + +setup_sysroot_from_local_image() { + local image_path="${1:-}" + [[ -f "${image_path}" ]] || err "setup-sysroot-local requires an existing system image path." + image_path="$(cd "$(dirname "${image_path}")" && pwd)/$(basename "${image_path}")" + local engine + engine="$(detect_engine)" + ensure_image_exists "${engine}" + mkdir -p "${SYSROOT_DIR}" "${HOST_CACHE_DIR}/agnos" + "${engine}" run --rm --platform linux/arm64 \ + --user "${DOCKER_RUN_USER}" \ + -v "${HOST_ROOT_DIR}:/work" \ + -v "${HOST_SYSROOT_DIR}:/opt/tici-sysroot" \ + -v "${HOST_CACHE_DIR}:/work/.cache" \ + -v "${image_path}:/opt/source-system.img:ro" \ + -w /work \ + "${IMAGE_NAME}" \ + /usr/bin/python3 tools/laptop_device_build/extract_sysroot_from_agnos.py \ + --image-path /opt/source-system.img \ + --output-dir /opt/tici-sysroot \ + --cache-dir /work/.cache/agnos +} + +run_larch64_scons() { + local jobs="$1" + shift || true + local cert_path="" + if [[ ${RELEASE+x} ]]; then + [[ -n "${RELEASE}" ]] || err "RELEASE must be nonempty; unset it for a DEBUG build." + [[ -n "${CERT:-}" ]] || err "RELEASE requires CERT to name a certificate file." + require_cmd realpath + cert_path="$(realpath "${CERT}")" || err "CERT cannot be resolved: ${CERT}" + [[ -f "${cert_path}" && -r "${cert_path}" ]] || err "CERT must be a readable regular file: ${CERT}" + [[ "${cert_path}" != *:* ]] || err "CERT path cannot contain a colon: ${CERT}" + fi + local scrub_mode="${COMMA_SCONS_SCRUB_MODE:-full}" + if [[ "${1:-}" == "--no-scrub" ]]; then + scrub_mode="none" + shift || true + fi + local firmware_only=1 + local firmware_targets=0 + local arg + for arg in "$@"; do + case "${arg}" in + panda/board/obj/panda_h7.bin.signed|panda/board/obj/body_h7.bin.signed) + firmware_targets=$((firmware_targets + 1)) ;; + -j*|--jobs=*|--verbose|--cache-disable) ;; + *) firmware_only=0 ;; + esac + done + [[ "${firmware_targets}" -gt 0 ]] || firmware_only=0 + local engine + local started_at + engine="$(detect_engine)" + ensure_image_exists "${engine}" + if [[ "${firmware_only}" -eq 0 ]]; then + ensure_sysroot_layout + fi + if [[ "${scrub_mode}" != "none" ]]; then + scrub_mixed_arch_artifacts + fi + started_at="$(date +%s)" + + echo "==> Starting larch64 scons build" + echo " jobs: ${jobs}" + echo " note: warp artifact precompile can take several minutes on first run" + + mkdir -p "${ROOT_DIR}/.cache/scons" "${HOST_CACHE_DIR}/scons" "${HOST_VENV_DIR}" + # SConstruct selects the device build only when /AGNOS exists on arm64. + # These are build-time file probes, not a simulated runtime device. + : > "${HOST_CACHE_DIR}/build-agnos-marker" + if [[ "${firmware_only}" -eq 0 ]]; then + : > "${HOST_CACHE_DIR}/build-ion-marker" + fi + + local extra_args="" + local jobs_prefix="-j${jobs}" + if [[ "$#" -gt 0 ]]; then + for arg in "$@"; do + case "${arg}" in + -j|--jobs|-j*|--jobs=*) + jobs_prefix="" + ;; + esac + extra_args+=" $(printf '%q' "${arg}")" + done + fi + + local cmd +cmd="$(cat < larch64 scons completed in $(( $(date +%s) - started_at ))s" +} + +setup_host_venv() { + if [[ ! -f "${ROOT_DIR}/.venv/bin/activate" ]]; then + if command -v uv >/dev/null 2>&1; then + (cd "${ROOT_DIR}" && uv sync --frozen --all-extras) + else + err "Missing host .venv and uv. Install uv, then run uv sync --frozen --all-extras." + fi + fi +} + +setup_all() { + local host="${1:-}" + local user="${2:-comma}" + local port="${3:-22}" + + setup_host_venv + build_container_image + + if [[ -n "${host}" ]]; then + sync_sysroot_from_device "${host}" "${user}" "${port}" + else + setup_sysroot_from_agnos "openpilot/system/hardware/tici/agnos.json" + fi +} + +run_larch64_build() { + local jobs="${1:-$(default_jobs)}" + shift || true + rm -f "${ROOT_DIR}/prebuilt" + echo "==> Complete device-target project build" + run_larch64_scons "${jobs}" "$@" + # Pass-through options and explicit targets are useful, but they may be a + # dry run, clean, question, or partial build. Only the plain default graph + # may claim a complete current build. + if [[ "$#" -ne 0 ]]; then + echo "Build options/targets were supplied; prebuilt remains unset." + return 0 + fi + # A successful SCons invocation alone is insufficient if stale/prebuilt model + # files were omitted. These are the current device-runtime outputs, including + # QCOM-compiled driving/DM models and both camera warp families. + python3 -m tools.laptop_device_build.package_model_chunks --source "${ROOT_DIR}/openpilot/selfdrive/modeld/models" --destination "${ROOT_DIR}/openpilot/selfdrive/modeld/models" + verify_device_artifacts + touch "${ROOT_DIR}/prebuilt" +} + +verify_device_artifacts() { + python3 "${ROOT_DIR}/tools/laptop_device_build/validate_artifacts.py" "${ROOT_DIR}" || + err "Device artifacts are incomplete or wrong-backend; prebuilt was not set." +} + +manager_artifacts_ready() { + [[ -f "${ROOT_DIR}/prebuilt" ]] && + python3 "${ROOT_DIR}/tools/laptop_device_build/validate_artifacts.py" "${ROOT_DIR}" >/dev/null 2>&1 +} + +run_manager() { + local jobs="$(default_jobs)" + local auto_build=1 + local manager_args=() + + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift || true + fi + + while [[ $# -gt 0 ]]; do + case "${1}" in + --no-build) + auto_build=0 + shift + ;; + --) + shift || true + manager_args=("$@") + break + ;; + *) + manager_args+=("$1") + shift + ;; + esac + done + + if [[ "${auto_build}" -eq 1 ]]; then + echo "Checking current source with the complete device-target build before manager..." + run_larch64_build "${jobs}" + elif ! manager_artifacts_ready; then + err "--no-build requires a prior verified device build." + fi + + local engine + engine="$(detect_engine)" + ensure_image_exists "${engine}" + assert_runtime_machine "${engine}" + ensure_sysroot_layout + mkdir -p "${ROOT_DIR}/.cache/scons" "${HOST_CACHE_DIR}/scons" "${HOST_VENV_DIR}" + + local manager_args_q="" + if [[ "${#manager_args[@]}" -gt 0 ]]; then + for arg in "${manager_args[@]}"; do + manager_args_q+=" $(printf '%q' "${arg}")" + done + fi + + local cmd +cmd="$(cat </dev/null 2>&1; then + if docker info >/dev/null 2>&1; then + echo "OK: docker daemon available" + else + echo "FAIL: docker CLI found but daemon unavailable" + failed=1 + fi + elif command -v podman >/dev/null 2>&1; then + if podman info >/dev/null 2>&1; then + echo "OK: podman service available" + else + echo "FAIL: podman CLI found but service unavailable" + failed=1 + fi + else + echo "FAIL: install docker or podman" + failed=1 + fi + + if command -v rsync >/dev/null 2>&1; then + echo "OK: rsync found" + else + echo "WARN: rsync missing (needed only for setup-sysroot from a physical device)" + fi + + if [[ -d "${SYSROOT_DIR}" ]]; then + if [[ -d "${SYSROOT_DIR}/usr/local/lib" && -d "${SYSROOT_DIR}/usr/local/include" && -d "${SYSROOT_DIR}/lib/aarch64-linux-gnu" && -d "${SYSROOT_DIR}/usr/lib/aarch64-linux-gnu" && -d "${SYSROOT_DIR}/usr/include" && -d "${SYSROOT_DIR}/system/vendor/lib64" ]]; then + echo "OK: sysroot present at ${SYSROOT_DIR}" + else + echo "WARN: sysroot dir exists but is incomplete (${SYSROOT_DIR})" + failed=1 + fi + else + echo "WARN: sysroot missing (${SYSROOT_DIR})" + failed=1 + fi + + if [[ -f "${ROOT_DIR}/.venv/bin/activate" ]]; then + echo "OK: host .venv present" + else + echo "WARN: host .venv missing (run scripts/laptop_device_build.sh setup)" + failed=1 + fi + + if [[ "${failed}" -ne 0 ]]; then + exit 1 + fi +} + +main() { + local mode="${1:-}" + if [[ "${mode}" == build || "${mode}" == scons ]]; then + local arg + for arg in "$@"; do + case "${arg}" in + RELEASE=*|CERT=*) + echo "ERROR: RELEASE and CERT must be environment variables, not SCons arguments." >&2 + echo "Use: RELEASE=1 CERT=/path/to/certificate ./build --panda 4" >&2 + exit 2 + ;; + esac + done + fi + case "${mode}" in + doctor) + doctor + ;; + package-models) + shift + python3 -m tools.laptop_device_build.package_model_chunks --source "${ROOT_DIR}/openpilot/selfdrive/modeld/models" --destination "${1:?destination directory required}" + ;; + verify-artifacts) + verify_device_artifacts + ;; + setup) + shift || true + setup_all "${1:-}" "${2:-comma}" "${3:-22}" + ;; + setup-sysroot) + shift + sync_sysroot_from_device "${1:-}" "${2:-comma}" "${3:-22}" + ;; + setup-sysroot-agnos) + shift || true + setup_sysroot_from_agnos "${1:-openpilot/system/hardware/tici/agnos.json}" + ;; + setup-sysroot-local) + shift || true + setup_sysroot_from_local_image "${1:-}" + ;; + build-image) + build_container_image + ;; + build) + shift || true + local jobs_arg="${1:-$(default_jobs)}" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + shift || true + else + jobs_arg="$(default_jobs)" + fi + run_larch64_build "${jobs_arg}" "$@" + ;; + scons) + shift || true + run_larch64_scons "$(default_jobs)" "$@" + ;; + manager) + shift || true + run_manager "$@" + ;; + shell) + run_shell + ;; + ""|-h|--help|help) + usage + ;; + *) + usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/launch_onroad_desktop.sh b/scripts/launch_onroad_desktop.sh new file mode 100755 index 0000000000..d066fa0010 --- /dev/null +++ b/scripts/launch_onroad_desktop.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" =~ ^[0-9]+$ ]]; then shift; fi +exec "${VIRTUAL_ENV:?Host Python environment required}/bin/python" -m openpilot.tools.replay.onroad "$@" diff --git a/scripts/launch_ui_c3_desktop.sh b/scripts/launch_ui_c3_desktop.sh new file mode 100755 index 0000000000..6d8578b056 --- /dev/null +++ b/scripts/launch_ui_c3_desktop.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" =~ ^[0-9]+$ ]]; then shift; fi +exec "${VIRTUAL_ENV:?Host Python environment required}/bin/python" -m openpilot.starpilot.ui.host_launch large "$@" diff --git a/scripts/launch_ui_c4_desktop.sh b/scripts/launch_ui_c4_desktop.sh new file mode 100755 index 0000000000..8b12261f54 --- /dev/null +++ b/scripts/launch_ui_c4_desktop.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" =~ ^[0-9]+$ ]]; then shift; fi +exec "${VIRTUAL_ENV:?Host Python environment required}/bin/python" -m openpilot.starpilot.ui.host_launch compact "$@" diff --git a/scripts/lint/check_added_large_files.py b/scripts/lint/check_added_large_files.py index 571f85296a..fd9a2c83cf 100755 --- a/scripts/lint/check_added_large_files.py +++ b/scripts/lint/check_added_large_files.py @@ -29,8 +29,8 @@ def check_added_large_files(filenames: list[str], max_kb: int) -> int: print(f'{filename}: placeholder pointer payload.') failed = True continue - if os.stat(filename).st_size > 95 * 1024 * 1024: - print(f'{filename}: ordinary Git blob exceeds 95 MiB.') + if os.stat(filename).st_size > 100 * 1024 * 1024: + print(f'{filename}: ordinary Git blob exceeds 100 MiB.') failed = True continue if filename in ignored: diff --git a/scripts/lint/check_dependencies.py b/scripts/lint/check_dependencies.py index f3707035b3..378e43d7cf 100755 --- a/scripts/lint/check_dependencies.py +++ b/scripts/lint/check_dependencies.py @@ -5,15 +5,49 @@ import importlib.metadata from pathlib import Path +DIRECT_LIMITS = {"baseline": 37, "dev": 3, "safety": 5} +TOOLING_PACKAGES = frozenset({ + "colorlog", "cppcheck", "execnet", "gcovr", "iniconfig", "jinja2", "lxml", "markupsafe", + "pluggy", "pygments", "pytest", "pytest-mock", "pytest-xdist", "tree-sitter", "tree-sitter-c", +}) +BASELINE_INSTALLED_LIMIT = 65 +TOOLING_INSTALLED_LIMIT = 15 + + +def direct_dependency_counts(project: dict) -> dict[str, int]: + extras = project["optional-dependencies"] + return { + "baseline": len(project["dependencies"]) + sum(len(deps) for name, deps in extras.items() if name not in ("dev", "safety")), + "dev": len(extras.get("dev", ())), + "safety": len(extras.get("safety", ())), + } + + +def direct_dependencies_within_limits(project: dict) -> bool: + counts = direct_dependency_counts(project) + return all(counts[name] <= limit for name, limit in DIRECT_LIMITS.items()) + + +def installed_dependency_counts(packages: set[str], tooling_names: frozenset[str] = TOOLING_PACKAGES) -> tuple[int, int]: + tooling = len(packages & tooling_names) + return len(packages) - tooling, tooling + + +def installed_dependencies_within_limits(packages: set[str], tooling_names: frozenset[str] = TOOLING_PACKAGES) -> bool: + baseline, tooling = installed_dependency_counts(packages, tooling_names) + return baseline <= BASELINE_INSTALLED_LIMIT and tooling <= TOOLING_INSTALLED_LIMIT + + def main() -> int: if sys.prefix == sys.base_prefix: print("Dependency checks require a virtual environment. Run tools/op.sh setup first.") return 1 project = tomllib.loads((Path(__file__).resolve().parents[2] / "pyproject.toml").read_text())["project"] - direct = len(project["dependencies"]) + sum(len(deps) for deps in project["optional-dependencies"].values()) + direct = direct_dependency_counts(project) # Count each installed package once, including transitive dependencies and all extras. packages = {dist.metadata["Name"].lower().replace("_", "-") for dist in importlib.metadata.distributions()} + installed_baseline, installed_tooling = installed_dependency_counts(packages) # Logical file sizes avoid filesystem block-size differences. Don't follow links to the interpreter or source tree. size = sum(path.stat().st_size for path in Path(sys.prefix).rglob("*") if not path.is_symlink() and path.is_file()) @@ -22,10 +56,12 @@ def main() -> int: These values are *not* intended to be increased, and we expect to strictly drive these down over time. """ - failed = False + failed = not direct_dependencies_within_limits(project) or not installed_dependencies_within_limits(packages) + for name, limit in DIRECT_LIMITS.items(): + print(f"Direct dependencies ({name}): {direct[name]} (limit: {limit})") + print(f"Installed dependencies (baseline): {installed_baseline} (limit: {BASELINE_INSTALLED_LIMIT})") + print(f"Installed dependencies (CI tooling): {installed_tooling} (limit: {TOOLING_INSTALLED_LIMIT})") for name, value, limit in ( - ("Direct dependencies (all extras)", direct, 37), - ("Total dependencies", len(packages), 65), ("Venv size (MiB)", size / 1024**2, 550), ): print(f"{name}: {value:g} (limit: {limit})") diff --git a/scripts/lint/check_shebang_format.sh b/scripts/lint/check_shebang_format.sh index 6a3657a10b..8757bb8a08 100755 --- a/scripts/lint/check_shebang_format.sh +++ b/scripts/lint/check_shebang_format.sh @@ -2,12 +2,12 @@ FAIL=0 -if grep '^#!.*python' "$@" | grep -v '#!/usr/bin/env python3$'; then +if grep -I '^#!.*python' "$@" | grep -v '#!/usr/bin/env python3$'; then echo -e "Invalid shebang! Must use '#!/usr/bin/env python3'\n" FAIL=1 fi -if grep '^#!.*bash' "$@" | grep -v '#!/usr/bin/env bash$'; then +if grep -I '^#!.*bash' "$@" | grep -v '#!/usr/bin/env bash$'; then echo -e "Invalid shebang! Must use '#!/usr/bin/env bash'" FAIL=1 fi diff --git a/scripts/lint/lint.sh b/scripts/lint/lint.sh index b05b2475cb..79781c6850 100755 --- a/scripts/lint/lint.sh +++ b/scripts/lint/lint.sh @@ -116,9 +116,12 @@ while IFS= read -r -d '' f; do fi done < <(git ls-files -z openpilot) -# Include tooling, launchers, and the extensionless Git hook. +# Keep inherited pinned dependency scripts outside the first-party shell check. SHELL_FILES=() while IFS= read -r -d '' f; do + case $f in + msgq_repo/*|opendbc_repo/*|panda/*|rednose_repo/*|teleoprtc_repo/*|tinygrad_repo/*) continue ;; + esac if [[ -f $f ]]; then SHELL_FILES+=("$f") fi diff --git a/scripts/native_device_build.sh b/scripts/native_device_build.sh new file mode 100755 index 0000000000..7b4674adf1 --- /dev/null +++ b/scripts/native_device_build.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Build in a staged checkout on actual AGNOS/aarch64 hardware. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +[[ -f /AGNOS && "$(uname -s)" == Linux && "$(uname -m)" == aarch64 ]] || { + echo "ERROR: native build requires AGNOS/aarch64 hardware" >&2 + exit 1 +} +cd "${ROOT_DIR}" + +if [[ -x /usr/local/venv/bin/scons ]]; then + SCONS=/usr/local/venv/bin/scons +elif [[ -x "${ROOT_DIR}/.venv/bin/scons" ]]; then + SCONS="${ROOT_DIR}/.venv/bin/scons" +else + echo "ERROR: SCons is missing from the device Python environment" >&2 + exit 1 +fi + +# Bind vendored build imports to this checkout, not an editable active install. +export PYTHONPATH="${ROOT_DIR}:${ROOT_DIR}/msgq_repo:${ROOT_DIR}/opendbc_repo:${ROOT_DIR}/rednose_repo:${ROOT_DIR}/teleoprtc_repo:${ROOT_DIR}/tinygrad_repo" +export PATH="$(dirname "${SCONS}"):${PATH}" + +mode="${1:-}" +shift || true +for arg in "$@"; do + case "${arg}" in + RELEASE=*|CERT=*) + echo "ERROR: RELEASE and CERT must be environment variables, not SCons arguments." >&2 + echo "Use: RELEASE=1 CERT=/path/to/certificate ./build --panda 4" >&2 + exit 2 + ;; + esac +done +case "${mode}" in + scons) + if [[ "${1:-}" == --no-scrub ]]; then + shift + fi + exec "${SCONS}" "$@" + ;; + build) + jobs=4 + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift + fi + [[ "${jobs}" -gt 0 ]] || { echo "ERROR: jobs must be positive" >&2; exit 2; } + rm -f prebuilt + "${SCONS}" "-j${jobs}" "$@" + if [[ "$#" -ne 0 ]]; then + echo "Build options/targets were supplied; prebuilt remains unset." + exit 0 + fi + python3 -m tools.laptop_device_build.package_model_chunks --source "${ROOT_DIR}/openpilot/selfdrive/modeld/models" --destination "${ROOT_DIR}/openpilot/selfdrive/modeld/models" + python3 "${ROOT_DIR}/tools/laptop_device_build/validate_artifacts.py" "${ROOT_DIR}" + touch prebuilt + ;; + *) + echo "ERROR: native build accepts only build or scons" >&2 + exit 2 + ;; +esac diff --git a/scripts/starpilot_build_flow.sh b/scripts/starpilot_build_flow.sh new file mode 100755 index 0000000000..bfb67ef92f --- /dev/null +++ b/scripts/starpilot_build_flow.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Compatibility entry point for the historical build-flow commands. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" + +usage() { + cat <<'EOF' +Usage: scripts/starpilot_build_flow.sh {verify|mac|device|laptop-setup|laptop-device} [jobs] + +verify Validate the local build entry points and current target files. +mac Run local Python compilation checks (no device binaries). +device Build on AGNOS/aarch64 hardware and set prebuilt on success. +laptop-setup Explicitly prepare image and sysroot; may access network/device. +laptop-device Run the complete device-target container build. +EOF +} + +case "${1:-}" in + verify) + bash -n build scripts/laptop_device_build.sh "$0" + for path in SConstruct openpilot/common/SConscript openpilot/cereal/SConscript \ + panda/SConscript openpilot/selfdrive/modeld/SConscript \ + tools/laptop_device_build/Dockerfile; do + [[ -f "${path}" ]] || { echo "Missing ${path}" >&2; exit 1; } + done + echo "Build entry points and current target files found." + ;; + mac) + [[ -x .venv/bin/python ]] || { echo "Missing .venv/bin/python" >&2; exit 1; } + .venv/bin/python -m py_compile openpilot/system/timed.py openpilot/common/spinner.py openpilot/common/text_window.py + .venv/bin/python -m compileall -q openpilot/starpilot openpilot/selfdrive openpilot/system + .venv/bin/python -c 'import openpilot.system.loggerd.xattr_cache' + echo "Local Python compilation passed; this does not produce device binaries." + ;; + device) + [[ -f /AGNOS && "$(uname -m)" == aarch64 ]] || { echo "device mode requires AGNOS/aarch64 hardware" >&2; exit 1; } + [[ -x .venv/bin/scons ]] || { echo "Missing .venv/bin/scons" >&2; exit 1; } + jobs="${2:-4}" + [[ "${jobs}" =~ ^[0-9]+$ ]] || { echo "jobs must be numeric" >&2; exit 1; } + rm -f prebuilt + .venv/bin/scons "-j${jobs}" + scripts/laptop_device_build.sh verify-artifacts + touch prebuilt + ;; + laptop-setup) + shift + exec scripts/laptop_device_build.sh setup "$@" + ;; + laptop-device) + shift + exec scripts/laptop_device_build.sh build "$@" + ;; + *) usage; exit 1 ;; +esac diff --git a/tool b/tool new file mode 100755 index 0000000000..30da28883f --- /dev/null +++ b/tool @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" "$@" diff --git a/tools/README.md b/tools/README.md index ae36282828..bb53e0771f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,5 +1,8 @@ # openpilot tools +For StarPilot's `./build`, `./dev`, `./onroad`, `./c3`, and `./c4` workflows, +see [StarPilot developer commands](STARPILOT_DEVELOPMENT.md). + ## System Requirements openpilot is developed and tested on **Ubuntu 24.04**, which is the primary development target aside from the [supported embedded hardware](https://github.com/commaai/openpilot#running-on-a-dedicated-device-in-a-car). diff --git a/tools/STARPILOT_DEVELOPMENT.md b/tools/STARPILOT_DEVELOPMENT.md new file mode 100644 index 0000000000..bafecd474e --- /dev/null +++ b/tools/STARPILOT_DEVELOPMENT.md @@ -0,0 +1,106 @@ +# StarPilot developer commands + +The root commands are a supported developer interface. `./build` produces +device artifacts. `./dev` builds and runs desktop tools in a separate source +cache. The UI remains the custom large and compact Raylib UI. + +| Command | Purpose | +| --- | --- | +| `./build [jobs] [SCons arguments]` | Complete Linux ARM64 device build | +| `./build --panda [jobs]` | Build the currently supported signed H7 firmware targets | +| `./build --params [jobs]` | Build the device Params library | +| `./build --cereal [jobs]` | Build the current schema, messaging libraries and bridge | +| `./c3 [jobs]`, `./c4 [jobs]` | Large and compact custom desktop UI | +| `./onroad [jobs] [--c3\|--c4\|--all\|--replay-only] ` | Route replay with selected desktop UI(s) | +| `./dev replay [jobs] ` | Native replay and its terminal controls | +| `./dev cabana [jobs] ` | Native Cabana in an independent cache | +| `./dev plotjuggler [jobs] ` | Current PlotJuggler helper; `juggle` is an alias | +| `./dev galaxy [jobs] [--port N]` | Authenticated local Galaxy server | +| `./dev python ` | Python with host native libraries and current source | +| `./dev pytest ` | pytest, including its xdist and mock plugins | +| `./dev shell` | Shell with host imports and private settings | +| `./dev sync [shared\|cabana]` | Refresh source and dependencies without launching a tool | + +`./tool` and `./tools/host` are aliases for `./dev`. The `c3`, `c4`, and +`onroad` commands can also be written as `./dev c3`, etc. Run `./dev help`, +`./c3 --help`, `./c4 --help`, or `./onroad --help` without installing or +building anything. Scripts resolve paths relative to the checkout, so invoking +an absolute launcher path from another directory also works. + +## Setup and build separation + +Desktop tools require macOS or Linux, Git, uv, and a working native C/C++ +compiler. Run `tools/setup_dependencies.sh` in this checkout for prerequisites. The first +desktop command installs the locked Python 3.12 environment and selected +developer dependencies in its own cache. It does not reuse a device venv. +Native bindings are built with the current SCons graph before launching. +Python, pytest and shell also build the generated longitudinal MPC and location +filter libraries. There is no automatic manager launch, firmware flash, or +vehicle connection in these host commands. + +Device build setup and sysroot instructions are in +[laptop-device-build.md](../docs/how-to/laptop-device-build.md). +`./build --panda` builds the targets present in this branch: `panda_h7` and +`body_h7`. Other historical firmware variants are not implemented by this +branch's Panda graph; their migration is a separate vehicle/platform obligation. +Compilation alone does not qualify a vehicle port or a road-test candidate. + +## Incremental source and state + +Host runtime files live under `.host_runtime/-/shared/`. +Cabana uses the adjacent `cabana/` bucket, allowing it to run alongside +PlotJuggler or replay. Commands in one bucket wait for the current session to +exit before changing its source or native libraries. The lock is held until +the launched command exits and is released by the OS after a crash. + +Each launch copies current tracked source and unignored new files, including +uncommitted edits. Deleted source is removed from the cache; ignored native +build outputs are not copied from the main checkout. Unchanged host build +outputs stay available for incremental builds. The cache has its own Git index +and refs, with read-only access to source objects; it never shares the source +index or runs checkout/reset cleanup against the developer's tree. Cache pushes +are disabled. Make source edits in the main checkout, since the next sync +replaces cached source edits. + +The host venv and Params live in the bucket. Messaging uses a stable, unique +checkout-and-bucket prefix. PC runtime support files, including Galaxy's local +credentials, use the corresponding `~/.commastarpilot-dev-/` directory; +ordinary `~/.comma` settings are not reused. Keeping the user's normal HOME +also preserves route authentication. Download caches are separate from Params. +Onroad sessions use disposable Params and a dedicated `replay-...` namespace. + +To force a clean desktop rebuild, stop the relevant command and remove only +that bucket under `.host_runtime`. This does not delete the main checkout's +device artifacts. Deleting a bucket removes its private Params; the separately +prefixed home directory holds the local Galaxy credentials described above. + +## UI assets and replay compatibility + +The preserved bitmap UI requires its exact reviewed font bundle. Set +`STARPILOT_UI_FONT_DIR` to that complete local directory. Inter and Unifont +bitmaps and their notices are bundled; Como Heavy is still externally +provisioned. An incomplete or changed bundle produces an explicit error rather +than silently substituting fonts and changing UI metrics. + +`./onroad` forwards current replay options such as `--start`, `--playback`, +`--data_dir`, `--no-loop`, `--no-vipc`, and `--demo`. An explicit UI choice avoids +automatic selection from logged device metadata. `--prefix replay-NAME` selects +a private session name. Playback uses the current native terminal controls. +Route access may require authentication or download the route the developer +requests. This is log playback; it does not run a driving manager or send CAN. + +`./onroad -alert` previews a synthetic critical visual alert in the private +replay session; it never publishes car control. `./onroad --cem` and `--csc` +show labeled synthetic CEM and curve visuals in that private UI session. They +change presentation only; they do not create driving acknowledgments or qualify +the current control features. The legacy navigation and combined Galaxy replay +demo switches still need adapters for this branch's current services. They fail +explicitly while absent. `./dev galaxy` starts the implemented local Galaxy +server independently; set its local password in the desktop UI first. Galaxy +continues to use its existing authenticated loopback interface. + +Developer compatibility remains a release requirement. Launcher/parser tests, +native desktop builds, UI/replay smoke checks, complete device builds, and the +existing safety/vehicle suites provide different evidence; none substitutes +for the others. Remaining demo and firmware variants must not be counted as +full compatibility just because the root command exists. diff --git a/tools/check_startup.py b/tools/check_startup.py new file mode 100644 index 0000000000..b546718c22 --- /dev/null +++ b/tools/check_startup.py @@ -0,0 +1,56 @@ +"""Read-only local manager admission check for an existing Params namespace.""" + +import argparse +from pathlib import Path +from tempfile import TemporaryDirectory + +from openpilot.common.params import Params +from openpilot.starpilot.state_migration import MigrationRequired, prepare_manager_start + + +class ExistingParams: + def __init__(self, namespace: Path, keys): + self.namespace = namespace + self.keys = keys + + def get_param_path(self): + return str(self.namespace) + + def all_keys(self): + return self.keys + + +def check_startup(params_root: Path, storage: Path) -> None: + if not params_root.is_dir(): + raise ValueError("Existing Params namespace required") + standard = params_root / "d" + if standard.is_symlink() and standard.is_dir(): + namespace = standard + elif params_root.is_symlink() or (not any(path.is_symlink() for path in params_root.iterdir()) and + any(path.is_file() and path.name != ".lock" for path in params_root.iterdir())): + namespace = params_root + else: + raise ValueError("Supply an existing Params namespace or a root with d; empty or named-link roots are ambiguous") + if not storage.is_dir(): + raise ValueError("Existing recovery storage directory required") + with TemporaryDirectory(prefix="startup-preflight-") as temporary: + keys = Params(temporary).all_keys() + prepare_manager_start(ExistingParams(namespace, keys), storage, dry_run=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("params_root", type=Path) + parser.add_argument("storage", type=Path) + args = parser.parse_args() + try: + check_startup(args.params_root, args.storage) + except (MigrationRequired, ValueError, OSError) as error: + print(f"Startup admission rejected: {error}") + return 1 + print("Startup admission accepted (read-only point-in-time check)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/tests/test_check_startup.py b/tools/ci/tests/test_check_startup.py new file mode 100644 index 0000000000..452a9fe2fb --- /dev/null +++ b/tools/ci/tests/test_check_startup.py @@ -0,0 +1,62 @@ +import tempfile +import json +import struct +import unittest +from pathlib import Path + +from openpilot.common.params import Params +from openpilot.starpilot import schema_cache as cache +from openpilot.starpilot.state_migration import MigrationRequired, prepare_manager_start +from openpilot.cereal import messaging +from tools.check_startup import check_startup + + +class TestCheckStartup(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.params = Params(str(self.root / "params")) + self.storage = self.root / "recovery" + self.storage.mkdir(mode=0o700) + + def tree(self): + return {str(path.relative_to(self.root)): path.read_bytes() if path.is_file() else None + for path in self.root.rglob("*")} + + def test_valid_initialized_cache_accepted_without_writes(self): + prepare_manager_start(self.params, self.storage) + cache.put_cache(self.params, "LiveDelay", messaging.new_message("lateralDelay"), block=True) + before = self.tree() + check_startup(Path(self.params.get_param_path()), self.storage) + self.assertEqual(self.tree(), before) + + def test_incompatible_retained_cache_rejected_without_writes(self): + prepare_manager_start(self.params, self.storage) + payload = messaging.new_message("lateralDelay").to_bytes() + header = cache._header(cache.CONTRACTS["LiveDelay"], "LiveDelay", payload, version=1) + header["schema_sha256"] = "0" * 64 + encoded = json.dumps(header, separators=(",", ":")).encode() + raw = cache.MAGIC + struct.pack(">I", len(encoded)) + encoded + payload + self.assertEqual(cache.inspect_cache("LiveDelay", raw).status, "incompatible") + (Path(self.params.get_param_path()) / "LiveDelay").write_bytes(raw) + before = self.tree() + with self.assertRaisesRegex(MigrationRequired, "incompatible retained cache"): + check_startup(Path(self.params.get_param_path()), self.storage) + self.assertEqual(self.tree(), before) + + def test_missing_input_rejected_without_creating_namespace(self): + missing = self.root / "missing" + with self.assertRaisesRegex(ValueError, "Existing Params namespace required"): + check_startup(missing, self.storage) + self.assertFalse(missing.exists()) + + def test_root_with_named_link_or_empty_directory_is_rejected(self): + before = self.tree() + with self.assertRaisesRegex(ValueError, "named-link roots are ambiguous"): + check_startup(self.root / "params", self.storage) + empty = self.root / "empty" + empty.mkdir() + with self.assertRaisesRegex(ValueError, "empty or named-link roots are ambiguous"): + check_startup(empty, self.storage) + self.assertEqual({key: value for key, value in self.tree().items() if key != "empty"}, before) diff --git a/tools/ci/tests/test_dev_runtime.py b/tools/ci/tests/test_dev_runtime.py new file mode 100644 index 0000000000..5695cd885f --- /dev/null +++ b/tools/ci/tests/test_dev_runtime.py @@ -0,0 +1,95 @@ +import hashlib +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from tools.host_runtime import HostRuntime, parse + + +class TestDeveloperRuntime(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) / 'source with spaces' + self.root.mkdir() + self.git('init', '-q') + self.git('config', 'user.email', 'test@example.invalid') + self.git('config', 'user.name', 'Test') + (self.root / '.gitignore').write_text('.host_runtime/\n*.so\n') + (self.root / 'source.py').write_text('original\n') + self.git('add', '.') + self.git('commit', '-qm', 'fixture') + self.runtime = HostRuntime(self.root, 'shared', system='Darwin', machine='arm64') + + def git(self, *args): + return subprocess.check_output(['git', *args], cwd=self.root, text=True) + + def test_sync_includes_local_edits_and_untracked_but_not_device_artifacts(self): + (self.root / 'source.py').write_text('local edit\n') + (self.root / 'new.py').write_text('new\n') + (self.root / 'native.so').write_bytes(b'device binary') + index_before = hashlib.sha256((self.root / '.git/index').read_bytes()).hexdigest() + with self.runtime.locked(): + self.runtime.sync() + self.assertEqual((self.runtime.work / 'source.py').read_text(), 'local edit\n') + self.assertTrue((self.runtime.work / 'new.py').is_file()) + self.assertFalse((self.runtime.work / 'native.so').exists()) + self.assertFalse((self.runtime.work / '.git').is_symlink()) + self.assertEqual(hashlib.sha256((self.root / '.git/index').read_bytes()).hexdigest(), index_before) + subprocess.check_call(['git', 'add', 'source.py'], cwd=self.runtime.work) + self.assertEqual(hashlib.sha256((self.root / '.git/index').read_bytes()).hexdigest(), index_before) + + def test_resync_removes_deleted_source_and_keeps_host_builds_incremental(self): + self.runtime.sync() + native = self.runtime.work / 'native.so' + native.write_bytes(b'host binary') + source = self.runtime.work / 'source.py' + before = source.stat().st_mtime_ns + self.runtime.sync() + self.assertEqual(source.stat().st_mtime_ns, before) + (self.root / 'source.py').unlink() + self.runtime.sync() + self.assertFalse(source.exists()) + self.assertEqual(native.read_bytes(), b'host binary') + + def test_cache_environment_ignores_external_imports_and_settings(self): + with patch.dict(os.environ, PYTHONPATH='/elsewhere', PARAMS_ROOT='/real/params', OPENPILOT_PREFIX='real', CC='cross-compiler'): + env = self.runtime.environment() + self.assertNotIn('/elsewhere', env['PYTHONPATH']) + self.assertEqual(env['CC'], '/usr/bin/clang') + self.assertTrue(Path(env['PARAMS_ROOT']).is_relative_to(self.runtime.cache)) + self.assertNotEqual(env['OPENPILOT_PREFIX'], 'real') + second = HostRuntime(self.root, 'shared', system='Darwin', machine='arm64') + cabana = HostRuntime(self.root, 'cabana', system='Darwin', machine='arm64') + self.assertEqual(second.prefix, self.runtime.prefix) + self.assertNotEqual(cabana.prefix, self.runtime.prefix) + + def test_cache_symlink_cannot_write_into_source(self): + self.runtime.cache.mkdir(parents=True) + self.runtime.work.symlink_to(self.root, target_is_directory=True) + with self.assertRaises(RuntimeError): + self.runtime.sync() + self.assertEqual((self.root / 'source.py').read_text(), 'original\n') + + def test_cache_cannot_borrow_source_git_index(self): + self.runtime.work.mkdir(parents=True) + (self.runtime.work / '.git').symlink_to(self.root / '.git', target_is_directory=True) + with self.assertRaisesRegex(RuntimeError, 'shared Git'): + self.runtime.sync_git() + + def test_command_parsing_preserves_tool_arguments(self): + self.assertEqual(parse(['c3', '8', '--demo']), ('c3', 8, ['--demo'])) + self.assertEqual(parse(['juggle', '4', '--demo']), ('plotjuggler', 4, ['--demo'])) + self.assertEqual(parse(['python', '-c', 'print(1)'])[2], ['-c', 'print(1)']) + self.assertEqual(parse(['pytest', '-n', '2'])[2], ['-n', '2']) + self.assertEqual(parse(['--help'])[0], 'help') + for args in (['unknown'], ['c4', '0'], ['sync', '../oops'], ['sync', 'shared', 'extra']): + with self.subTest(args=args), self.assertRaises(ValueError): + parse(args) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ci/tests/test_dev_runtime_review.py b/tools/ci/tests/test_dev_runtime_review.py new file mode 100644 index 0000000000..fb9e16c3dd --- /dev/null +++ b/tools/ci/tests/test_dev_runtime_review.py @@ -0,0 +1,134 @@ +"""Source-cache transitions that must not require a native build or device.""" + +import os +import fcntl +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +from tools.host_runtime import HostRuntime + + +def git(root: Path, *args: str) -> None: + subprocess.run(['git', *args], cwd=root, check=True, capture_output=True) + + +class TestHostRuntimeSourceSync(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) / 'source' + self.root.mkdir() + git(self.root, 'init', '-q') + git(self.root, 'config', 'user.name', 'Host Runtime Test') + git(self.root, 'config', 'user.email', 'host-runtime@example.invalid') + (self.root / 'README').write_text('fixture\n') + git(self.root, 'add', 'README') + git(self.root, 'commit', '-qm', 'fixture') + self.runtime = HostRuntime(self.root, 'shared', system='Darwin', machine='arm64') + + def test_same_size_and_mtime_edit_reaches_cache(self) -> None: + source = self.root / 'README' + self.runtime.sync() + before = source.stat() + source.write_text('changed\n') + os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns)) + + self.runtime.sync() + + self.assertEqual((self.runtime.work / 'README').read_text(), 'changed\n') + + def test_tracked_child_replaced_by_file_reaches_cache(self) -> None: + directory = self.root / 'path' + directory.mkdir() + (directory / 'child').write_text('old\n') + git(self.root, 'add', 'path/child') + git(self.root, 'commit', '-qm', 'child') + self.runtime.sync() + + (directory / 'child').unlink() + directory.rmdir() + directory.write_text('new\n') + self.runtime.sync() + + cached = self.runtime.work / 'path' + self.assertTrue(cached.is_file()) + self.assertEqual(cached.read_text(), 'new\n') + + def test_tracked_file_replaced_by_child_reaches_cache(self) -> None: + source = self.root / 'path' + source.write_text('old\n') + git(self.root, 'add', 'path') + git(self.root, 'commit', '-qm', 'file') + self.runtime.sync() + + source.unlink() + source.mkdir() + (source / 'child').write_text('new\n') + self.runtime.sync() + + cached = self.runtime.work / 'path' + self.assertTrue(cached.is_dir()) + self.assertEqual((cached / 'child').read_text(), 'new\n') + + def test_wrapper_sigkill_keeps_lock_until_child_exits(self) -> None: + runtime = self.runtime + runtime.work.mkdir(parents=True) + (runtime.venv / 'bin').mkdir(parents=True) + (runtime.venv / 'bin/python').symlink_to(sys.executable) + child_pid_file = self.root / 'child.pid' + source_root = Path(__file__).resolve().parents[3] + wrapper_code = '''import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from tools.host_runtime import HostRuntime +runtime = HostRuntime(sys.argv[2], 'shared', system='Darwin', machine='arm64') +child = "import os,sys,time; from pathlib import Path; Path(sys.argv[1]).write_text(str(os.getpid())); time.sleep(30)" +with runtime.locked(): + runtime.launch('python', 1, ['-c', child, sys.argv[3]]) +''' + wrapper = subprocess.Popen([sys.executable, '-c', wrapper_code, str(source_root), str(self.root), + str(child_pid_file)], start_new_session=True) + ipc = Path('/tmp') / f'msgq_{runtime.prefix}' + lock_path = runtime.cache / 'lock' + try: + deadline = time.monotonic() + 5 + while not child_pid_file.exists() and wrapper.poll() is None and time.monotonic() < deadline: + time.sleep(.02) + self.assertTrue(child_pid_file.exists(), 'child did not start') + wrapper.kill() + wrapper.wait(timeout=3) + with lock_path.open('a+') as contender: + with self.assertRaises(BlockingIOError): + fcntl.flock(contender, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + try: + os.killpg(wrapper.pid, signal.SIGTERM) + except ProcessLookupError: + pass + if wrapper.poll() is None: + wrapper.kill() + wrapper.wait(timeout=3) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + with lock_path.open('a+') as contender: + try: + fcntl.flock(contender, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + time.sleep(.02) + else: + fcntl.flock(contender, fcntl.LOCK_UN) + break + else: + self.fail('owned child did not release the lock after stopping') + if ipc.exists(): + shutil.rmtree(ipc) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ci/tests/test_qcom_artifact_contract.py b/tools/ci/tests/test_qcom_artifact_contract.py new file mode 100644 index 0000000000..61de8eedbc --- /dev/null +++ b/tools/ci/tests/test_qcom_artifact_contract.py @@ -0,0 +1,307 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from tools.laptop_device_build import qcom_artifact_contract as contract + + +ROOT = Path(__file__).resolve().parents[3] + + +class TestQcomArtifactContract(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.package = Path(temporary.name) + self.artifacts = self.package / "artifacts" + self.artifacts.mkdir() + self.commands = contract.commands(ROOT) + for name in self.commands: + (self.artifacts / name).write_bytes(f"artifact:{name}".encode()) + with patch("tools.laptop_device_build.validate_artifacts.require_qcom_pickle"): + manifest = contract.record(ROOT, self.artifacts) + (self.package / "manifest.json").write_text(json.dumps(manifest)) + + def verify(self, name): + with patch("tools.laptop_device_build.validate_artifacts.require_qcom_pickle"): + return contract.verify(self.package, ROOT, name, self.commands[name]) + + def compatible_package(self): + manifest = json.loads((self.package / "manifest.json").read_text()) + current = contract.source_inventory(ROOT) + old_hash = "0" * 64 + manifest["source"]["sources"][contract.COMPATIBILITY_SOURCE] = old_hash + manifest["source_sha256"] = contract._inventory_signature(manifest["source"]) + (self.package / "manifest.json").write_text(json.dumps(manifest)) + evidence_dir = self.package / "compatibility" + evidence_dir.mkdir() + (evidence_dir / "qualification.txt").write_text("reviewed paired output evidence\n") + observations = {name: [{"input": label, "shape": contract.COMPATIBILITY_OUTPUTS[name][0], + "dtype": contract.COMPATIBILITY_OUTPUTS[name][1], + "build_sha256": str(index + 4) * 64, + "runtime_sha256": str(index + 4) * 64} + for index, label in enumerate(("A", "B", "A"))] + for name in self.commands} + evidence = {"version": 1, "method": "paired-qcom-output-sha256", + "build_manifest_sha256": contract._sha(self.package / "manifest.json"), + "compatible_source_sha256": contract._inventory_signature(current), + "changed_source": {"path": contract.COMPATIBILITY_SOURCE, "build_sha256": old_hash, + "runtime_sha256": current["sources"][contract.COMPATIBILITY_SOURCE]}, + "artifact_sha256": {name: entry["sha256"] for name, entry in manifest["artifacts"].items()}, + "paired_outputs": observations, + "evidence_files": {"qualification.txt": contract._sha(evidence_dir / "qualification.txt")}} + (self.package / "compatibility.json").write_text(json.dumps(evidence)) + return evidence, current + + def test_exact_source_and_artifacts_select_expected_target(self): + name = "driving_tinygrad.pkl" + self.assertEqual(self.verify(name)[0], self.artifacts / name) + self.assertEqual(set(json.loads((self.package / "manifest.json").read_text())["artifacts"]), set(self.commands)) + + def test_source_and_command_changes_reject(self): + name = "driving_tinygrad.pkl" + with patch.object(contract, "_sources", return_value={"changed.py": "0" * 64}), \ + self.assertRaisesRegex(ValueError, "sources changed"): + self.verify(name) + with self.assertRaisesRegex(ValueError, "command recipe changed"): + contract.verify(self.package, ROOT, name, self.commands[name] + " --different") + + def test_one_artifact_change_or_missing_member_rejects_whole_package(self): + (self.artifacts / "dm_warp_1344x760_tinygrad.pkl").write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "artifact mismatch"): + self.verify("driving_tinygrad.pkl") + manifest = json.loads((self.package / "manifest.json").read_text()) + del manifest["artifacts"]["dm_warp_1344x760_tinygrad.pkl"] + (self.package / "manifest.json").write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "source or artifact set"): + self.verify("driving_tinygrad.pkl") + + def test_wrong_backend_pickle_rejects_even_with_matching_hash(self): + with self.assertRaises(ValueError): + contract.verify(self.package, ROOT, "driving_tinygrad.pkl", self.commands["driving_tinygrad.pkl"]) + + def test_malformed_manifest_artifacts_reject_cleanly(self): + manifest = json.loads((self.package / "manifest.json").read_text()) + for malformed in (None, [], "not-a-map"): + manifest["artifacts"] = malformed + (self.package / "manifest.json").write_text(json.dumps(manifest)) + with self.subTest(malformed=malformed), self.assertRaisesRegex(ValueError, "artifact set"): + self.verify("driving_tinygrad.pkl") + + def test_changed_bytes_during_copy_leave_existing_target(self): + name = "driving_tinygrad.pkl" + target = self.package / name + target.write_bytes(b"previous") + with patch("tools.laptop_device_build.validate_artifacts.require_qcom_pickle"), \ + patch.object(contract.shutil, "copyfileobj", side_effect=lambda source, output: output.write(b"changed")), \ + self.assertRaisesRegex(ValueError, "changed during copy"): + contract.import_artifact(self.package, ROOT, target, self.commands[name]) + self.assertEqual(target.read_bytes(), b"previous") + self.assertFalse(list(self.package.glob(".qcom-import-*"))) + + def test_exact_compatible_source_and_all_six_artifacts_verify(self): + self.compatible_package() + for name in self.commands: + with self.subTest(name=name): + self.assertEqual(self.verify(name)[0], self.artifacts / name) + + def test_compatible_source_rejects_unrelated_source_or_recipe_change(self): + _, current = self.compatible_package() + changed = copy.deepcopy(current) + changed["sources"]["unrelated.py"] = "1" * 64 + with patch.object(contract, "source_inventory", return_value=changed), \ + self.assertRaisesRegex(ValueError, "outside reviewed"): + self.verify("driving_tinygrad.pkl") + changed = copy.deepcopy(current) + changed["commands"]["driving_tinygrad.pkl"].append("--new-option") + with patch.object(contract, "source_inventory", return_value=changed), \ + self.assertRaisesRegex(ValueError, "outside reviewed"): + self.verify("driving_tinygrad.pkl") + + def test_compatible_source_rejects_artifact_or_paired_output_change(self): + evidence, _ = self.compatible_package() + del evidence["paired_outputs"]["dmonitoring_model_tinygrad.pkl"] + (self.package / "compatibility.json").write_text(json.dumps(evidence)) + with self.assertRaisesRegex(ValueError, "lacks a stock target"): + self.verify("driving_tinygrad.pkl") + evidence["paired_outputs"]["dmonitoring_model_tinygrad.pkl"] = [ + {"input": label, "shape": contract.COMPATIBILITY_OUTPUTS["dmonitoring_model_tinygrad.pkl"][0], + "dtype": "float32", "build_sha256": str(index + 4) * 64, + "runtime_sha256": str(index + 4) * 64} + for index, label in enumerate(("A", "B", "A"))] + (self.package / "compatibility.json").write_text(json.dumps(evidence)) + (self.artifacts / "dmonitoring_model_tinygrad.pkl").write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "artifact mismatch"): + self.verify("driving_tinygrad.pkl") + + def test_compatible_source_rejects_changed_or_unsafe_evidence(self): + evidence, _ = self.compatible_package() + evidence_file = self.package / "compatibility" / "qualification.txt" + evidence_file.write_text("changed evidence\n") + with self.assertRaisesRegex(ValueError, "evidence file mismatch"): + self.verify("driving_tinygrad.pkl") + evidence_file.write_text("reviewed paired output evidence\n") + evidence["evidence_files"] = {"../qualification.txt": contract._sha(evidence_file)} + (self.package / "compatibility.json").write_text(json.dumps(evidence)) + with self.assertRaisesRegex(ValueError, "evidence filename"): + self.verify("driving_tinygrad.pkl") + evidence["evidence_files"] = {"qualification.txt": contract._sha(evidence_file)} + (self.package / "compatibility.json").write_text(json.dumps(evidence)) + outside = self.package / "outside.txt" + outside.write_text("reviewed paired output evidence\n") + evidence_file.unlink() + evidence_file.symlink_to(outside) + with self.assertRaisesRegex(ValueError, "evidence file mismatch"): + self.verify("driving_tinygrad.pkl") + evidence_file.unlink() + evidence_file.write_text("reviewed paired output evidence\n") + evidence_dir = self.package / "compatibility" + evidence_dir.rename(self.package / "actual_evidence") + evidence_dir.symlink_to(self.package / "actual_evidence", target_is_directory=True) + with self.assertRaisesRegex(ValueError, "evidence directory"): + self.verify("driving_tinygrad.pkl") + + def test_compatible_source_rejects_unequal_outputs_and_stale_source_binding(self): + evidence, _ = self.compatible_package() + original = copy.deepcopy(evidence) + evidence["paired_outputs"]["driving_tinygrad.pkl"][1]["runtime_sha256"] = "a" * 64 + (self.package / "compatibility.json").write_text(json.dumps(evidence)) + with self.assertRaisesRegex(ValueError, "Invalid QCOM paired outputs"): + self.verify("driving_tinygrad.pkl") + original["compatible_source_sha256"] = "b" * 64 + (self.package / "compatibility.json").write_text(json.dumps(original)) + with self.assertRaisesRegex(ValueError, "not bound"): + self.verify("driving_tinygrad.pkl") + + def test_compatible_source_rejects_bool_versions_and_duplicate_json_fields(self): + evidence, _ = self.compatible_package() + evidence["version"] = True + (self.package / "compatibility.json").write_text(json.dumps(evidence)) + with self.assertRaisesRegex(ValueError, "not bound"): + self.verify("driving_tinygrad.pkl") + (self.package / "compatibility.json").write_text('{"version":1,"version":1}') + with self.assertRaisesRegex(ValueError, "Duplicate QCOM manifest field"): + self.verify("driving_tinygrad.pkl") + manifest = json.loads((self.package / "manifest.json").read_text()) + manifest["version"] = True + (self.package / "manifest.json").write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "source or artifact set"): + self.verify("driving_tinygrad.pkl") + + +class TestQcomSupplementalRuntimeCompatibility(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.package = Path(temporary.name) + self.current = {"sources": {name: hashes[1] for name, hashes in contract.RUNTIME_EXTENSION_CHANGES.items()}, + "commands": {"driving_tinygrad.pkl": ["DEV=QCOM", "IMAGE=1", "compile"]}} + self.current["sources"].update(contract.RUNTIME_EXTENSION_ADDITIONS) + self.current["sources"].update({contract.COMPATIBILITY_SOURCE: "a" * 64, "model.onnx": "b" * 64}) + parent = copy.deepcopy(self.current) + for name, hashes in contract.RUNTIME_EXTENSION_CHANGES.items(): + parent["sources"][name] = hashes[0] + for name in contract.RUNTIME_EXTENSION_ADDITIONS: + del parent["sources"][name] + saved = copy.deepcopy(parent) + saved["sources"][contract.COMPATIBILITY_SOURCE] = "c" * 64 + self.manifest = {"source": saved, "source_sha256": contract._inventory_signature(saved), + "artifacts": {name: {"sha256": "d" * 64} for name in contract.COMPATIBILITY_OUTPUTS}} + self.manifest_path = self.package / "manifest.json" + self.manifest_path.write_text(json.dumps(self.manifest)) + evidence_dir = self.package / "compatibility" + evidence_dir.mkdir() + (evidence_dir / "outputs.txt").write_text("A/B/A output hashes\n") + self.evidence = {"version": 1, "method": "paired-qcom-output-sha256", + "build_manifest_sha256": contract._sha(self.manifest_path), + "compatible_source_sha256": contract._inventory_signature(parent), + "changed_source": {"path": contract.COMPATIBILITY_SOURCE, "build_sha256": "c" * 64, "runtime_sha256": "a" * 64}, + "artifact_sha256": {name: "d" * 64 for name in contract.COMPATIBILITY_OUTPUTS}, + "paired_outputs": {name: [{"input": label, "shape": shape, "dtype": dtype, + "build_sha256": "e" * 64, "runtime_sha256": "e" * 64} + for label in ("A", "B", "A")] + for name, (shape, dtype) in contract.COMPATIBILITY_OUTPUTS.items()}, + "evidence_files": {"outputs.txt": contract._sha(evidence_dir / "outputs.txt")}} + self.parent_path = self.package / "compatibility.json" + self.parent_path.write_text(json.dumps(self.evidence)) + self.bind_extension() + + def bind_extension(self): + extension = {"version": 1, "profile": contract.RUNTIME_EXTENSION_PROFILE, + "parent_evidence_sha256": contract._sha(self.parent_path), + "compatible_source_sha256": contract._inventory_signature(self.current)} + (self.package / "runtime-compatibility.json").write_text(json.dumps(extension)) + + def verify(self): + contract._verify_runtime_compatibility(self.package, self.manifest_path, self.manifest, + self.current, dict.fromkeys(contract.COMPATIBILITY_OUTPUTS)) + + def test_reviewed_extension_preserves_original_manifest_and_parent_attestation(self): + before = self.manifest_path.read_bytes(), self.parent_path.read_bytes() + self.verify() + self.assertEqual(before, (self.manifest_path.read_bytes(), self.parent_path.read_bytes())) + + def test_each_reviewed_source_and_added_helper_requires_exact_bytes(self): + original = copy.deepcopy(self.current) + for name in (*contract.RUNTIME_EXTENSION_CHANGES, *contract.RUNTIME_EXTENSION_ADDITIONS): + with self.subTest(name=name): + self.current = copy.deepcopy(original) + self.current["sources"][name] = "f" * 64 + self.bind_extension() + with self.assertRaisesRegex(ValueError, "Unreviewed QCOM runtime"): + self.verify() + + def test_extra_missing_or_model_changed_source_is_not_normalized_away(self): + original = copy.deepcopy(self.current) + for kind in ("extra", "missing", "model"): + with self.subTest(kind=kind): + self.current = copy.deepcopy(original) + if kind == "extra": self.current["sources"]["unexpected.py"] = "f" * 64 + elif kind == "missing": del self.current["sources"]["model.onnx"] + else: self.current["sources"]["model.onnx"] = "f" * 64 + self.bind_extension() + with self.assertRaisesRegex(ValueError, "outside reviewed"): + self.verify() + + def test_compile_command_change_remains_rejected(self): + self.current["commands"]["driving_tinygrad.pkl"].append("FLOAT16=0") + self.bind_extension() + with self.assertRaisesRegex(ValueError, "outside reviewed"): + self.verify() + + def test_parent_evidence_mutation_and_missing_supplement_reject(self): + original = self.parent_path.read_bytes() + self.parent_path.write_bytes(original + b"\n") + with self.assertRaisesRegex(ValueError, "parent evidence"): + self.verify() + self.parent_path.write_bytes(original) + (self.package / "runtime-compatibility.json").unlink() + with self.assertRaisesRegex(ValueError, "outside reviewed"): + self.verify() + + def test_artifact_substitution_and_unequal_paired_output_reject_after_rebinding(self): + original = copy.deepcopy(self.evidence) + for kind in ("artifact", "output"): + with self.subTest(kind=kind): + self.evidence = copy.deepcopy(original) + if kind == "artifact": self.evidence["artifact_sha256"]["driving_tinygrad.pkl"] = "f" * 64 + else: self.evidence["paired_outputs"]["driving_tinygrad.pkl"][0]["runtime_sha256"] = "f" * 64 + self.parent_path.write_text(json.dumps(self.evidence)) + self.bind_extension() + with self.assertRaises(ValueError): + self.verify() + + def test_supplement_symlink_unknown_profile_and_extra_fields_reject(self): + path = self.package / "runtime-compatibility.json" + good = json.loads(path.read_text()) + for mutation in ({"profile": "unreviewed"}, {"extra": True}, {"version": True}): + path.write_text(json.dumps(good | mutation)) + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + self.verify() + path.unlink() + path.symlink_to(self.parent_path) + with self.assertRaisesRegex(ValueError, "Invalid QCOM runtime extension"): + self.verify() diff --git a/tools/ci/tests/test_runtime_library_paths.py b/tools/ci/tests/test_runtime_library_paths.py new file mode 100644 index 0000000000..c2c9f1537c --- /dev/null +++ b/tools/ci/tests/test_runtime_library_paths.py @@ -0,0 +1,50 @@ +import struct +import unittest + +from tools.laptop_device_build.runtime_paths import runtime_library_path +from tools.laptop_device_build.validate_artifacts import require_aarch64_elf + + +def linked_elf(runpath: str) -> bytes: + strings = b"\0" + runpath.encode() + b"\0" + data = bytearray(256 + len(strings)) + data[:6] = b"\x7fELF\x02\x01" + struct.pack_into(" max_gap_s: + rejected[f"{service}:gap"] += 1 + elif mode != prior[2]: + rejected[f"{service}:mode_changed"] += 1 + else: + intervals[signal].add(elapsed) + delta_rates[signal].add((value - prior[1]) / elapsed) + previous[service] = (timestamp, value, mode) + + return {"limits": {"maxContinuousGapSeconds": max_gap_s, "maxContextAgeSeconds": context_age_s}, + "messageCounts": dict(counts), "transitions": dict(transitions), "pedalContextSamples": dict(pedal_context), + "rejectedPairsOrSamples": dict(rejected), + "signals": {name: {"valuesMps2": samples[name].summary(), + "continuousFrameDeltaSeconds": intervals[name].summary(), + "continuousDeltaRateMps3": delta_rates[name].summary()} + for name in SIGNALS.values()}} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("rlog", type=Path, help="Existing local rlog or qlog segment") + parser.add_argument("--max-gap-s", type=float, default=0.25) + parser.add_argument("--context-age-s", type=float, default=0.5) + args = parser.parse_args() + path = args.rlog.resolve(strict=True) + if not path.is_file() or path.suffix not in (".zst", ".bz2", ".rlog", ".qlog"): + parser.error("Expected a local log file") + from openpilot.tools.lib.logreader import LogReader + report = analyze(LogReader(str(path)), max_gap_s=args.max_gap_s, context_age_s=args.context_age_s) + report["input"] = {"path": str(path), "sha256": _sha(path), "sizeBytes": path.stat().st_size, + "kind": "qlog_decimated" if "qlog" in path.name else "rlog_segment"} + root = Path(__file__).resolve().parents[2] + report["sourceHashes"] = {name: _sha(root / name) for name in ( + "tools/diagnostics/longitudinal.py", "openpilot/cereal/log.capnp", "opendbc_repo/opendbc/car/car.capnp")} + report["provenance"] = {"plannerATarget": "longitudinalPlan.aTarget planner target, m/s²", + "commandAccel": "carControl.actuators.accel command, m/s²", + "aEgoEstimate": "carState.aEgo speed-derived vehicle estimate, m/s²", + "deltaRate": "sample-to-sample signal derivative, not physical jerk or publisher rate"} + print(json.dumps(report, sort_keys=True, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/tools/diagnostics/normalize_event_caches.py b/tools/diagnostics/normalize_event_caches.py new file mode 100644 index 0000000000..15ba169e39 --- /dev/null +++ b/tools/diagnostics/normalize_event_caches.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Normalize valid current-schema v1 Event caches before an offline source update.""" + +import argparse +import json +import os +from pathlib import Path +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--params-path', required=True, help='Existing Params root containing the active namespace') + parser.add_argument('--backup-dir', required=True, help='Private backup directory outside Params') + parser.add_argument('--producers-stopped', action='store_true', required=True, + help='Assert the caller has stopped every cache producer before this operation') + args = parser.parse_args() + + namespace = Path(args.params_path) / os.environ.get('OPENPILOT_PREFIX', 'd') + if not Path(args.params_path).is_dir() or not namespace.is_dir(): + parser.error('--params-path must already contain the existing Params namespace') + + from openpilot.common.params import Params + from openpilot.starpilot.schema_cache_normalize import normalize_current_event_caches + + try: + params = Params(args.params_path) + report = normalize_current_event_caches(params, args.backup_dir, producers_stopped=args.producers_stopped) + except (OSError, RuntimeError, ValueError) as error: + print(json.dumps({'status': 'failed', 'error': f'{type(error).__name__}: {error}'}), file=sys.stderr) + return 1 + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/diagnostics/performance.py b/tools/diagnostics/performance.py new file mode 100644 index 0000000000..f56022e14f --- /dev/null +++ b/tools/diagnostics/performance.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 +"""Bounded, read-only local runtime sampling and offline capture comparison.""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import UTC, datetime +import hashlib +import importlib.util +import json +from itertools import islice +import math +import os +from pathlib import Path +import platform +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[2] +SERVICES = ('uiDebug', 'managerState', 'deviceState', 'modelV2', 'controlsState', 'selfdriveState') +MAX_PROCESSES = 64 +MAX_INSTANCES = 256 +FIELDS = { + 'uiDebug': {'reportedFrameIntervalMs': ('frameTimeMillis', 1), 'reportedPrePresentWallMs': ('cpuTimeMillis', 1)}, + 'modelV2': {'reportedModelExecutionMs': ('modelExecutionTime', 1000), 'reportedFrameDropPercent': ('frameDropPerc', 1)}, + 'deviceState': {name: (field, 1) for name, field in ( + ('reportedMemoryUsagePercent', 'memoryUsagePercent'), ('reportedGpuUsagePercent', 'gpuUsagePercent'), + ('reportedMaxTemperatureC', 'maxTempC'), ('reportedPowerDrawW', 'powerDrawW'))}, +} +CATEGORIES = {'deviceState': ('started', 'thermalStatus', 'deviceType', 'chestnutPresent'), + 'selfdriveState': ('enabled', 'active'), 'modelV2': ('big',)} +METRICS = { + 'reportedFrameIntervalMs': 'uiDebug.frameTimeMillis; Raylib reported frame interval, not measured present duration', + 'reportedPrePresentWallMs': 'uiDebug.cpuTimeMillis; publisher monotonic wall time for drawing/update before present, not thread CPU', + 'reportedModelExecutionMs': 'modelV2.modelExecutionTime * 1000; publisher execution interval, not sensor-to-actuator latency', + 'observedReceiveRateHz': 'fresh conflated messages observed / actual capture seconds; capped by collector polling, not publisher rate', + 'receiveAgeMs': 'time since this collector last received a message, sampled each poll after first receipt', + 'publicationAgeAtReceiveMs': 'collector monotonic time minus Event.logMonoTime at receipt; includes publication/transport/sampling delay', + 'cpuPercentOneCore': '100 * delta process(user+system ticks) / clock ticks per second / elapsed sample seconds; can exceed 100', + 'rssMiB': 'Linux /proc/PID/stat resident pages * page size / 1048576; excludes swapped/nonresident memory', +} + + +class Distribution: + def __init__(self): + self.values: list[float] = [] + self.missing: Counter[str] = Counter() + + def add(self, value, reason='missing_field'): + if value is None: + self.missing[reason] += 1 + elif not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value): + self.missing['nonfinite_or_nonnumeric'] += 1 + else: + self.values.append(float(value)) + + def summary(self): + values = sorted(self.values) + result = {'status': 'observed' if values else 'no_samples', 'sampleCount': len(values), 'missingCounts': dict(self.missing)} + if not values: + return {**result, **dict.fromkeys(('min', 'mean', 'p50', 'p90', 'p95', 'p99', 'max'))} + + def percentile(q): + index = (len(values) - 1) * q + lo, hi = math.floor(index), math.ceil(index) + return values[lo] + (values[hi] - values[lo]) * (index - lo) + + return {**result, 'min': values[0], 'mean': math.fsum(values) / len(values), 'max': values[-1], + **{f'p{q}': percentile(q / 100) for q in (50, 90, 95, 99)}} + + +def field(body, name): + try: + return body.get(name) if isinstance(body, dict) else getattr(body, name) + except (AttributeError, RuntimeError): + return None + + +class MessageSamples: + def __init__(self): + self.count = 0 + self.invalid = 0 + self.last_received = None + self.last_published = None + self.timestamps = Counter() + self.receive_age = Distribution() + self.publication_age = Distribution() + self.intervals = Distribution() + self.metrics: dict[str, Distribution] = {} + self.states: dict[str, Counter] = {} + + def observe(self, now, event): + self.count += 1 + self.invalid += not event['valid'] + if self.last_received is not None: + self.intervals.add((now - self.last_received) * 1000) + self.last_received = now + published = event['logMonoTime'] + if not isinstance(published, int) or published <= 0: + self.publication_age.add(None, 'missing_or_zero_timestamp') + elif published > now * 1e9: + self.publication_age.add(None, 'future_timestamp_clock_mismatch') + else: + self.publication_age.add((now - published / 1e9) * 1000) + if isinstance(published, int) and published > 0: + if self.last_published is not None: + if published == self.last_published: + self.timestamps['repeated'] += 1 + elif published < self.last_published: + self.timestamps['regressed'] += 1 + self.last_published = published + + def summary(self, duration, source_status): + return {'status': 'observed' if self.count else ('no_messages' if source_status == 'available' else source_status), + 'observedMessages': self.count, 'invalidMessages': self.invalid, + 'observedReceiveRateHz': self.count / duration if duration > 0 and source_status == 'available' else None, + 'timestampDiscontinuities': dict(self.timestamps), 'receiveAgeMs': self.receive_age.summary(), + 'publicationAgeAtReceiveMs': self.publication_age.summary(), 'observedReceiveIntervalMs': self.intervals.summary(), + 'reportedStateCounts': {name: dict(value) for name, value in self.states.items()}, + 'metrics': {name: value.summary() for name, value in self.metrics.items()}} + + +class LocalMessages: + def __init__(self): + self.status = 'available' + self.error = None + self.master = None + self.expected_rates = {} + self.module_path = None + try: + from openpilot.cereal import messaging + from openpilot.cereal.services import SERVICE_LIST + self.module_path = messaging.__file__ + self.master = messaging.SubMaster(list(SERVICES)) + self.expected_rates = {name: SERVICE_LIST[name].frequency for name in SERVICES} + except Exception as error: + self.status, self.error = 'unavailable', type(error).__name__ + + def poll(self): + if self.master is None: + return {} + try: + self.master.update(0) + return {name: {'body': self.master[name], 'valid': self.master.valid[name], 'logMonoTime': self.master.logMonoTime[name]} + for name in SERVICES if self.master.updated[name]} + except Exception as error: + self.status, self.error, self.master = 'error', type(error).__name__, None + return {} + + +class ProcReader: + def __init__(self, root=Path('/proc'), ticks=None, page_size=None): + self.root = root + self.available = (root / 'self/stat').exists() + self.ticks = ticks or (os.sysconf('SC_CLK_TCK') if self.available else None) + self.page_size = page_size or (os.sysconf('SC_PAGE_SIZE') if self.available else None) + + def read(self, pid): + if not self.available: + return None, 'proc_unavailable' + try: + raw = (self.root / str(pid) / 'stat').read_text() + begin, end = raw.index('('), raw.rindex(')') + parts = raw[end + 2:].split() + if int(raw[:begin].strip()) != pid: + raise ValueError('PID mismatch') + result = {'comm': raw[begin + 1:end], 'cpuTicks': int(parts[11]) + int(parts[12]), + 'startTicks': int(parts[19]), 'rssPages': int(parts[21])} + if min(result['cpuTicks'], result['startTicks'], result['rssPages']) < 0: + raise ValueError('negative stat') + return result, None + except FileNotFoundError: + return None, 'process_gone' + except PermissionError: + return None, 'permission_denied' + except (OSError, ValueError, IndexError): + return None, 'unreadable_or_malformed_stat' + + +class ProcessSamples: + def __init__(self, reader): + self.reader = reader + self.instances = {} + self.previous = {} + self.read_errors = Counter() + self.pid_reuses = 0 + self.skipped_instances = 0 + self.skipped_targets = 0 + self.target_samples = 0 + + def poll(self, now, targets): + self.skipped_targets += max(0, len(targets) - MAX_PROCESSES) + self.target_samples += min(MAX_PROCESSES, len(targets)) + for pid, name in sorted(targets.items())[:MAX_PROCESSES]: + record, error = self.reader.read(pid) + if error: + self.read_errors[error] += 1 + previous = self.previous.pop(pid, None) + if previous is not None: + for metric in ('cpuPercentOneCore', 'rssMiB'): + self.instances[previous[0]][metric].add(None, error) + continue + identity = (pid, record['startTicks']) + if identity not in self.instances: + if len(self.instances) >= MAX_INSTANCES: + self.skipped_instances += 1 + continue + self.instances[identity] = {'pid': pid, 'startTicks': record['startTicks'], 'requestedName': name, 'comm': record['comm'], + 'firstObservedMonotonic': now, 'lastObservedMonotonic': now, + 'cpuPercentOneCore': Distribution(), 'rssMiB': Distribution()} + item = self.instances[identity] + item['lastObservedMonotonic'] = now + item['rssMiB'].add(record['rssPages'] * self.reader.page_size / 1048576) + before = self.previous.get(pid) + reason = None + if before is None: + reason = 'baseline_required' + elif before[0] != identity: + reason = 'pid_reused_or_restarted' + self.pid_reuses += 1 + elif now <= before[1]: + reason = 'nonpositive_sample_interval' + elif record['cpuTicks'] < before[2]: + reason = 'cpu_counter_regressed' + if reason: + item['cpuPercentOneCore'].add(None, reason) + else: + assert before is not None + item['cpuPercentOneCore'].add(100 * (record['cpuTicks'] - before[2]) / self.reader.ticks / (now - before[1])) + self.previous[pid] = (identity, now, record['cpuTicks']) + self.previous = {pid: value for pid, value in self.previous.items() if pid in targets} + + def summary(self): + names = Counter(item['requestedName'] for item in self.instances.values()) + return {'status': 'available' if self.reader.available else 'unavailable', 'readErrors': dict(self.read_errors), + 'samplingStatus': 'observed' if self.instances else ('no_targets' if not self.target_samples else 'no_readable_processes'), + 'requestedTargetSamples': self.target_samples, + 'clockTicksPerSecond': self.reader.ticks, 'pageSizeBytes': self.reader.page_size, + 'pidReuseOrRestartEvents': self.pid_reuses, 'skippedInstanceSamples': self.skipped_instances, + 'observedInstanceCountsByName': dict(names), + 'skippedTargetSamples': self.skipped_targets, + 'instances': [{name: value.summary() if isinstance(value, Distribution) else value for name, value in item.items()} + for item in self.instances.values()]} + + +def capture(duration=30, rate=20, *, messages=None, proc=None, pids=(), clock=time.monotonic, sleep=time.sleep): + if not math.isfinite(duration) or not 0.1 <= duration <= 120 or not math.isfinite(rate) or not 1 <= rate <= 50: + raise ValueError('duration must be 0.1..120 seconds and sample rate 1..50 Hz') + if len(pids) > MAX_PROCESSES or any(type(pid) is not int or pid <= 0 for pid in pids): + raise ValueError('at most 64 positive PIDs are supported') + messages = messages if messages is not None else LocalMessages() + processes = ProcessSamples(proc if proc is not None else ProcReader()) + samples = {name: MessageSamples() for name in SERVICES} + for service, metrics in FIELDS.items(): + samples[service].metrics = {name: Distribution() for name in metrics} + for name in ('reportedCpuUsageMeanPercent', 'reportedCpuUsageMaxPercent'): + samples['deviceState'].metrics[name] = Distribution() + explicit = {pid: f'explicit_pid_{pid}' for pid in pids} + targets = {} + manager_counts = Counter() + polls, missed_slots = 0, 0 + poll_intervals = Distribution() + started = clock() + previous_poll = None + scheduled = started + while polls < math.ceil(duration * rate) and clock() < started + duration: + if clock() < scheduled: + sleep(min(scheduled, started + duration) - clock()) + if clock() >= started + duration: + break + events = messages.poll() + now = clock() + if previous_poll is not None: + poll_intervals.add((now - previous_poll) * 1000) + previous_poll = now + for service, event in events.items(): + sample = samples[service] + sample.observe(now, event) + body = event['body'] + for name in CATEGORIES.get(service, ()): + value = field(body, name) + sample.states.setdefault(name, Counter())['missing' if value is None else str(value)[:128]] += 1 + for name, (source, scale) in FIELDS.get(service, {}).items(): + value = field(body, source) + if isinstance(value, (int, float)) and value < 0: + sample.metrics[name].add(None, 'negative_reported_value') + else: + sample.metrics[name].add(value * scale if isinstance(value, (int, float)) else value) + if service == 'deviceState': + values = field(body, 'cpuUsagePercent') + values = list(islice(values, 256)) if values is not None else [] + valid = [value for value in values if isinstance(value, (int, float)) and math.isfinite(value) and value >= 0] + reason = 'missing_or_empty_cpu_list' if not values else 'invalid_cpu_list' + for name, value in [('reportedCpuUsageMeanPercent', sum(valid) / len(valid) if valid else None), + ('reportedCpuUsageMaxPercent', max(valid) if valid else None)]: + sample.metrics[name].add(value if len(valid) == len(values) else None, reason) + if service == 'managerState': + targets = {} + entries = field(body, 'processes') + if entries is None: + manager_counts['missing_process_list'] += 1 + else: + for index, process in enumerate(entries): + if index >= MAX_PROCESSES: + manager_counts['truncated_process_lists'] += 1 + break + running, expected = field(process, 'running'), field(process, 'shouldBeRunning') + manager_counts['process_observations'] += 1 + manager_counts['reported_running'] += running is True + manager_counts['expected_but_stopped'] += expected is True and running is False + pid = field(process, 'pid') + if running is True and isinstance(pid, int) and pid > 0: + targets[pid] = str(field(process, 'name'))[:128] + for sample in samples.values(): + if sample.last_received is not None: + sample.receive_age.add((now - sample.last_received) * 1000) + processes.poll(now, {**targets, **explicit}) + polls += 1 + next_slot = max(polls + missed_slots, math.floor((clock() - started) * rate) + 1) + missed_slots = next_slot - polls + scheduled = started + next_slot / rate + if clock() < started + duration: + sleep(started + duration - clock()) + elapsed = clock() - started + return {'schemaVersion': 1, 'kind': 'runtime-performance-capture', + 'capture': {'requestedDurationSeconds': duration, 'actualDurationSeconds': elapsed, 'requestedSampleRateHz': rate, + 'pollCount': polls, 'missedScheduledPolls': missed_slots, 'pollIntervalMs': poll_intervals.summary()}, + 'messageSource': {'status': messages.status, 'errorType': messages.error, 'transport': 'local conflated SubMaster', + 'modulePath': getattr(messages, 'module_path', None), + 'configuredPublisherRatesHz': messages.expected_rates}, + 'services': {name: value.summary(elapsed, messages.status) for name, value in samples.items()}, + 'managerObservations': dict(manager_counts), 'processes': processes.summary(), 'metricDefinitions': METRICS, + 'bounds': {'maxSeconds': 120, 'maxPollHz': 50, 'maxProcessesPerPoll': MAX_PROCESSES, 'maxProcessInstances': MAX_INSTANCES}} + + +def metadata(root=None): + try: + spec = importlib.util.find_spec('openpilot') + imported_root = Path(spec.origin).resolve().parent.parent if spec is not None and spec.origin else None + except (ImportError, ValueError): + imported_root = None + binding = 'explicit' if root is not None else 'imported_openpilot' + if root is None: + root = imported_root + if root is None and (ROOT / 'openpilot/cereal/log.capnp').is_file(): + root, binding = ROOT, 'tool_repository_fallback' + if root is not None: + root = Path(root).resolve() + if not (root / 'openpilot/cereal/log.capnp').is_file(): + raise ValueError('source root must contain openpilot/cereal/log.capnp') + + def read(path): + try: + return path.read_text().strip()[:256] + except OSError: + return None + + def git(*args): + if root is None: + return None + try: + return subprocess.run(['git', '-C', str(root), *args], capture_output=True, text=True, timeout=3, check=True).stdout.strip() + except (OSError, subprocess.SubprocessError): + return None + + paths = ('openpilot/cereal/log.capnp', 'openpilot/cereal/services.py', + 'openpilot/selfdrive/ui/ui.py', 'openpilot/system/ui/lib/application.py', 'openpilot/selfdrive/modeld/modeld.py') + return {'recordedAtUtc': datetime.now(UTC).isoformat(), 'system': platform.system(), 'release': platform.release(), + 'machine': platform.machine(), 'python': platform.python_version(), 'logicalCpuCount': os.cpu_count(), + 'agnosVersion': read(Path('/VERSION')), 'bootId': read(Path('/proc/sys/kernel/random/boot_id')), + 'sourceRoot': str(root) if root is not None else None, 'sourceBinding': binding if root is not None else 'unavailable', + 'importedOpenpilotRoot': str(imported_root) if imported_root is not None else None, + 'declaredRootMatchesImported': root == imported_root if root is not None and imported_root is not None else None, + 'tool': {'path': str(Path(__file__).resolve()), 'sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}, + 'gitHead': git('rev-parse', 'HEAD'), + 'gitDirty': bool(status) if (status := git('status', '--porcelain')) is not None else None, + 'sourceSha256': {name: hashlib.sha256((root / name).read_bytes()).hexdigest() + for name in paths if root is not None and (root / name).is_file()}, + 'collectorEnvironment': {name: os.environ.get(name) for name in ('UI_FRAME_TIMING', 'SIMULATION', 'SCALE', 'ZMQ', 'MSGQ_PREFIX')}} + + +def compare(before, after): + for report in (before, after): + if report.get('schemaVersion') != 1 or report.get('kind') != 'runtime-performance-capture': + raise ValueError('unsupported capture schema') + differences = [] + for field_name in ('system', 'machine', 'agnosVersion', 'logicalCpuCount'): + left, right = before.get('environment', {}).get(field_name), after.get('environment', {}).get(field_name) + if left != right or left is None: + differences.append({'field': field_name, 'before': left, 'after': right}) + for name in ('requestedDurationSeconds', 'requestedSampleRateHz'): + if before['capture'][name] != after['capture'][name]: + differences.append({'field': name, 'before': before['capture'][name], 'after': after['capture'][name]}) + + def flatten(report): + result = {} + for service, value in report['services'].items(): + result[f'{service}.observedReceiveRateHz'] = value['observedReceiveRateHz'] + for metric, data in {**value['metrics'], **{name: value[name] for name in + ('receiveAgeMs', 'publicationAgeAtReceiveMs', 'observedReceiveIntervalMs')}}.items(): + for statistic in ('sampleCount', 'p50', 'p95', 'max'): + result[f'{service}.{metric}.{statistic}'] = data[statistic] + groups = {} + for item in report['processes']['instances']: + group = groups.setdefault(item['requestedName'], []) + group.append(item) + for name, items in groups.items(): + result[f'process.{name}.observedInstances'] = len(items) + # Do not average per-incarnation percentiles into a false pooled percentile. + for metric in ('cpuPercentOneCore', 'rssMiB'): + available = [item[metric]['max'] for item in items if item[metric]['sampleCount']] + result[f'process.{name}.{metric}.max'] = max(available) if available else None + return result + + left, right = flatten(before), flatten(after) + rows = [] + for key in sorted(left.keys() | right.keys()): + a, b = left.get(key), right.get(key) + rows.append({'metric': key, 'before': a, 'after': b, 'status': 'comparable' if a is not None and b is not None else 'missing_data', + 'delta': b - a if a is not None and b is not None else None, + 'changePercent': (b - a) / abs(a) * 100 if a is not None and b is not None and a != 0 else None}) + return {'schemaVersion': 1, 'kind': 'runtime-performance-comparison', 'environmentOrSamplingDifferences': differences, + 'interpretation': 'Descriptive differences only; workload, thermal state, conflation and instrumentation affect results.', + 'captureContext': {name: {'gitHead': report.get('environment', {}).get('gitHead'), + 'messageSource': report['messageSource'], 'processStatus': report['processes']['status'], + 'reportedStateCounts': {service: value['reportedStateCounts'] for service, value in report['services'].items()}} + for name, report in (('before', before), ('after', after))}, + 'summary': [row for row in rows if row['metric'] in ( + 'uiDebug.reportedFrameIntervalMs.p95', 'uiDebug.reportedPrePresentWallMs.p95', + 'modelV2.reportedModelExecutionMs.p95', 'modelV2.reportedFrameDropPercent.max', + 'deviceState.reportedMaxTemperatureC.max', 'deviceState.reportedMemoryUsagePercent.max') + or row['metric'].startswith('process.')], + 'metrics': rows} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + collect = commands.add_parser('capture', help='Read local messages and Linux /proc; write one final JSON file') + collect.add_argument('--duration', type=float, default=30) + collect.add_argument('--rate', type=float, default=20) + collect.add_argument('--pid', type=int, action='append', default=[], help='Also sample this local PID (repeatable, maximum 64)') + collect.add_argument('--source-root', type=Path, help='Source checkout to identify; default is the imported openpilot checkout') + collect.add_argument('--output', type=Path, required=True) + diff = commands.add_parser('compare', help='Compare two saved captures without connecting to anything') + diff.add_argument('before', type=Path) + diff.add_argument('after', type=Path) + diff.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + try: + if args.output.exists(): + raise FileExistsError(f'output already exists: {args.output}') + if args.command == 'capture': + environment = metadata(args.source_root) + report = capture(args.duration, args.rate, pids=args.pid) + report['environment'] = environment + else: + report = compare(json.loads(args.before.read_text()), json.loads(args.after.read_text())) + report['inputs'] = {name: {'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} + for name, path in (('before', args.before), ('after', args.after))} + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open('x') as output: + json.dump(report, output, indent=2, allow_nan=False) + output.write('\n') + except (ValueError, OSError) as error: + parser.exit(1, f'{type(error).__name__}: {error}\n') + print(f'{report["kind"]}: {args.output}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/diagnostics/tests/test_longitudinal.py b/tools/diagnostics/tests/test_longitudinal.py new file mode 100644 index 0000000000..4c0df0bdd0 --- /dev/null +++ b/tools/diagnostics/tests/test_longitudinal.py @@ -0,0 +1,83 @@ +import math +from types import SimpleNamespace as NS +import unittest + +from tools.diagnostics.longitudinal import analyze + + +class Event: + def __init__(self, service, seconds, **fields): + self.service = service + self.logMonoTime = int(seconds * 1e9) + self.valid = True + setattr(self, service, NS(**fields)) + + def which(self): + return self.service + + +def cluster(t, *, target=1.0, command=0.5, ego=0.2, source="cruise", experimental=False, + active=True, gas=False, brake=False): + return [Event("carState", t, aEgo=ego, gasPressed=gas, brakePressed=brake), + Event("selfdriveState", t, experimentalMode=experimental), + Event("longitudinalPlan", t, aTarget=target, longitudinalPlanSource=source), + Event("carControl", t, longActive=active, actuators=NS(accel=command))] + + +class TestLongitudinalTrace(unittest.TestCase): + def test_continuous_same_mode_rates_are_signal_derivatives(self): + messages = cluster(1) + cluster(1.1, target=1.2, command=0.6, ego=0.3) + cluster(1.2, target=1.4, command=0.7, ego=0.4) + report = analyze(messages) + command = report["signals"]["commandAccel"] + self.assertEqual(command["continuousFrameDeltaSeconds"]["sampleCount"], 2) + self.assertAlmostEqual(command["continuousDeltaRateMps3"]["mean"], 1) + self.assertEqual(report["pedalContextSamples"]["gas=False,brake=False"], 7) + + def test_gap_and_mode_transitions_exclude_derivatives(self): + messages = cluster(1) + cluster(1.1) + cluster(2, active=False, source="lead0", experimental=True, gas=True) + report = analyze(messages) + self.assertEqual(report["transitions"]["source"], 1) + self.assertEqual(report["transitions"]["experimental"], 1) + self.assertEqual(report["transitions"]["longActive"], 1) + self.assertEqual(report["signals"]["commandAccel"]["continuousDeltaRateMps3"]["sampleCount"], 1) + gap = analyze(cluster(1) + cluster(1.1) + cluster(2)) + self.assertGreater(gap["rejectedPairsOrSamples"]["carControl:gap"], 0) + + def test_nonfinite_missing_and_timestamp_regression_reset_baseline(self): + messages = cluster(1) + cluster(1.1, command=math.nan) + cluster(1.05) + cluster(1.2) + report = analyze(messages) + self.assertEqual(report["signals"]["commandAccel"]["valuesMps2"]["missingCounts"]["missing_or_nonfinite"], 1) + self.assertGreater(report["rejectedPairsOrSamples"]["carControl:timestamp_regression"], 0) + self.assertEqual(report["signals"]["commandAccel"]["continuousDeltaRateMps3"]["sampleCount"], 0) + + def test_stale_context_does_not_create_pair(self): + messages = cluster(1) + [Event("carControl", 2, longActive=True, actuators=NS(accel=1.0))] + report = analyze(messages) + self.assertEqual(report["rejectedPairsOrSamples"]["carControl:missing_or_stale_mode"], 1) + self.assertEqual(report["signals"]["commandAccel"]["continuousFrameDeltaSeconds"]["sampleCount"], 0) + + def test_invalid_envelope_is_excluded(self): + messages = cluster(1) + cluster(1.1) + messages[-1].valid = False + messages.append(Event("longitudinalPlan", 1.15, aTarget=1.0, longitudinalPlanSource="cruise")) + report = analyze(messages) + self.assertEqual(report["rejectedPairsOrSamples"]["carControl:invalid_envelope"], 1) + self.assertEqual(report["rejectedPairsOrSamples"]["longitudinalPlan:missing_or_stale_mode"], 2) + self.assertEqual(report["signals"]["commandAccel"]["valuesMps2"]["sampleCount"], 1) + + def test_away_and_back_or_invalid_recovery_breaks_other_signal_pair(self): + baseline = cluster(1) + cluster(1.1) + toggled = baseline + [Event("selfdriveState", 1.15, experimentalMode=True), + Event("selfdriveState", 1.16, experimentalMode=False), + Event("longitudinalPlan", 1.2, aTarget=1.4, longitudinalPlanSource="cruise")] + report = analyze(toggled) + self.assertEqual(report["transitions"]["experimental"], 2) + self.assertEqual(report["signals"]["plannerATarget"]["continuousDeltaRateMps3"]["sampleCount"], 0) + + invalid = Event("carControl", 1.15, longActive=True, actuators=NS(accel=0.5)) + invalid.valid = None + recovered = baseline + [invalid, Event("carControl", 1.16, longActive=True, actuators=NS(accel=0.5)), + Event("longitudinalPlan", 1.2, aTarget=1.4, longitudinalPlanSource="cruise")] + report = analyze(recovered) + self.assertEqual(report["rejectedPairsOrSamples"]["carControl:invalid_envelope"], 1) + self.assertEqual(report["signals"]["plannerATarget"]["continuousDeltaRateMps3"]["sampleCount"], 0) diff --git a/tools/diagnostics/tests/test_performance.py b/tools/diagnostics/tests/test_performance.py new file mode 100644 index 0000000000..21a18addd7 --- /dev/null +++ b/tools/diagnostics/tests/test_performance.py @@ -0,0 +1,248 @@ +import copy +import json +from pathlib import Path +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from tools.diagnostics import performance as perf + + +class FakeClock: + def __init__(self): + self.now = 10.0 + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class FakeMessages: + status = 'available' + error = None + expected_rates = {'uiDebug': 0} + + def __init__(self, frames=()): + self.frames = iter(frames) + + def poll(self): + return next(self.frames, {}) + + +def event(body, timestamp=10_000_000_000, valid=True): + return {'body': body, 'logMonoTime': timestamp, 'valid': valid} + + +def proc_stat(pid=42, cpu=20, start=100, rss=512, comm='ui (worker)'): + parts = ['0'] * 22 + parts[0], parts[11], parts[12], parts[19], parts[21] = 'S', str(cpu), '0', str(start), str(rss) + return f'{pid} ({comm}) ' + ' '.join(parts) + + +class PerformanceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + (self.root / 'self').mkdir() + (self.root / 'self/stat').write_text(proc_stat()) + (self.root / '42').mkdir() + self.reader = perf.ProcReader(self.root, ticks=100, page_size=4096) + + def write_stat(self, **values): + (self.root / '42/stat').write_text(proc_stat(**values)) + + def capture(self, frames=(), **kwargs): + clock = FakeClock() + return perf.capture(1, 10, messages=FakeMessages(frames), proc=self.reader, clock=clock, sleep=clock.sleep, **kwargs) + + def test_distribution_interpolation_and_missing_are_distinct_from_zero(self): + values = perf.Distribution() + self.assertIsNone(values.summary()['p95']) + for value in (0, 10, 20, 30): + values.add(value) + values.add(None) + values.add(float('nan')) + result = values.summary() + self.assertEqual(result['sampleCount'], 4) + self.assertEqual(result['min'], 0) + self.assertEqual(result['p50'], 15) + self.assertAlmostEqual(result['p95'], 28.5) + self.assertEqual(result['missingCounts'], {'missing_field': 1, 'nonfinite_or_nonnumeric': 1}) + json.dumps(result, allow_nan=False) + + def test_messages_are_sampled_once_and_receive_age_grows(self): + report = self.capture([{'uiDebug': event({'frameTimeMillis': 50, 'cpuTimeMillis': 7}, valid=False)}]) + ui = report['services']['uiDebug'] + self.assertEqual(report['capture']['pollCount'], 10) + self.assertAlmostEqual(report['capture']['actualDurationSeconds'], 1) + self.assertEqual(ui['observedMessages'], 1) + self.assertEqual(ui['invalidMessages'], 1) + self.assertEqual(ui['observedReceiveRateHz'], 1) + self.assertEqual(ui['metrics']['reportedPrePresentWallMs']['sampleCount'], 1) + self.assertAlmostEqual(ui['receiveAgeMs']['max'], 900) + self.assertEqual(report['services']['modelV2']['status'], 'no_messages') + self.assertIsNone(report['services']['modelV2']['metrics']['reportedModelExecutionMs']['max']) + + def test_missing_message_backend_and_proc_are_unavailable(self): + source = FakeMessages() + source.status, source.error = 'unavailable', 'ImportError' + clock = FakeClock() + report = perf.capture(0.1, 10, messages=source, proc=perf.ProcReader(self.root / 'absent'), + pids=[42], clock=clock, sleep=clock.sleep) + self.assertEqual(report['services']['uiDebug']['status'], 'unavailable') + self.assertIsNone(report['services']['uiDebug']['observedReceiveRateHz']) + self.assertEqual(report['processes']['readErrors'], {'proc_unavailable': 1}) + self.assertEqual(report['processes']['status'], 'unavailable') + + def test_reported_model_units_and_device_sentinels(self): + report = self.capture([{'modelV2': event({'modelExecutionTime': 0.025, 'frameDropPerc': 2}), + 'deviceState': event({'gpuUsagePercent': -1, 'memoryUsagePercent': 0, 'cpuUsagePercent': [10, 30]})}]) + self.assertEqual(report['services']['modelV2']['metrics']['reportedModelExecutionMs']['p50'], 25) + device = report['services']['deviceState']['metrics'] + self.assertEqual(device['reportedGpuUsagePercent']['missingCounts'], {'negative_reported_value': 1}) + self.assertEqual(device['reportedMemoryUsagePercent']['min'], 0) + self.assertEqual(device['reportedCpuUsageMeanPercent']['mean'], 20) + self.assertIsNone(device['reportedPowerDrawW']['max']) + + def test_timestamp_discontinuities_are_not_negative_latencies(self): + sample = perf.MessageSamples() + for now, stamp in [(10, 0), (11, 12_000_000_000), (12, 12_000_000_000), (13, 11_000_000_000)]: + sample.observe(now, event({}, stamp)) + result = sample.summary(4, 'available') + self.assertEqual(result['publicationAgeAtReceiveMs']['missingCounts'], + {'missing_or_zero_timestamp': 1, 'future_timestamp_clock_mismatch': 1}) + self.assertEqual(result['timestampDiscontinuities'], {'repeated': 1, 'regressed': 1}) + self.assertEqual(result['publicationAgeAtReceiveMs']['min'], 0) + + def test_proc_cpu_rss_and_parentheses_in_comm(self): + samples = perf.ProcessSamples(self.reader) + self.write_stat(cpu=20) + samples.poll(10, {42: 'ui'}) + self.write_stat(cpu=220) + samples.poll(11, {42: 'ui'}) + item = samples.summary()['instances'][0] + self.assertEqual(item['comm'], 'ui (worker)') + self.assertEqual(item['cpuPercentOneCore']['max'], 200) + self.assertEqual(item['cpuPercentOneCore']['missingCounts'], {'baseline_required': 1}) + self.assertEqual(item['rssMiB']['mean'], 2) + + def test_restart_with_new_pid_is_a_separate_instance(self): + samples = perf.ProcessSamples(self.reader) + self.write_stat(cpu=20) + samples.poll(10, {42: 'ui'}) + (self.root / '43').mkdir() + (self.root / '43/stat').write_text(proc_stat(pid=43, cpu=200, start=300)) + samples.poll(11, {43: 'ui'}) + result = samples.summary() + self.assertEqual(result['observedInstanceCountsByName'], {'ui': 2}) + self.assertEqual(result['instances'][1]['cpuPercentOneCore']['sampleCount'], 0) + self.assertEqual(result['pidReuseOrRestartEvents'], 0) + + def test_pid_reuse_does_not_attribute_previous_cpu_to_new_process(self): + samples = perf.ProcessSamples(self.reader) + self.write_stat(cpu=20, start=100) + samples.poll(10, {42: 'ui'}) + self.write_stat(cpu=900, start=200) + samples.poll(11, {42: 'ui'}) + self.write_stat(cpu=950, start=200) + samples.poll(12, {42: 'ui'}) + result = samples.summary() + self.assertEqual(result['pidReuseOrRestartEvents'], 1) + self.assertEqual(len(result['instances']), 2) + self.assertEqual(result['instances'][1]['cpuPercentOneCore']['max'], 50) + self.assertEqual(result['instances'][1]['cpuPercentOneCore']['missingCounts'], {'pid_reused_or_restarted': 1}) + + def test_process_disappears_and_reappearance_requires_new_baseline(self): + samples = perf.ProcessSamples(self.reader) + self.write_stat(cpu=20) + samples.poll(10, {42: 'ui'}) + (self.root / '42/stat').unlink() + samples.poll(11, {42: 'ui'}) + self.write_stat(cpu=500) + samples.poll(12, {42: 'ui'}) + result = samples.summary() + self.assertEqual(result['readErrors'], {'process_gone': 1}) + self.assertEqual(result['instances'][0]['cpuPercentOneCore']['sampleCount'], 0) + self.assertEqual(result['instances'][0]['cpuPercentOneCore']['missingCounts'], {'baseline_required': 2, 'process_gone': 1}) + + def test_proc_permissions_malformed_and_counter_regression(self): + self.write_stat() + with patch.object(Path, 'read_text', side_effect=PermissionError): + self.assertEqual(self.reader.read(42)[1], 'permission_denied') + (self.root / '42/stat').write_text('not a stat') + self.assertEqual(self.reader.read(42)[1], 'unreadable_or_malformed_stat') + samples = perf.ProcessSamples(self.reader) + self.write_stat(cpu=100) + samples.poll(10, {42: 'ui'}) + self.write_stat(cpu=50) + samples.poll(11, {42: 'ui'}) + self.assertEqual(samples.summary()['instances'][0]['cpuPercentOneCore']['missingCounts'], + {'baseline_required': 1, 'cpu_counter_regressed': 1}) + + def test_manager_stopped_and_no_targets_are_not_proc_failures(self): + report = self.capture([{'managerState': event({'processes': [ + {'name': 'ui', 'pid': 0, 'running': False, 'shouldBeRunning': True}]})}]) + self.assertEqual(report['managerObservations']['expected_but_stopped'], 1) + self.assertEqual(report['processes']['samplingStatus'], 'no_targets') + self.assertEqual(report['processes']['readErrors'], {}) + + def test_capture_and_process_instance_bounds(self): + for duration, rate in [(121, 20), (30, 51), (float('nan'), 20)]: + with self.assertRaises(ValueError): + perf.capture(duration, rate) + samples = perf.ProcessSamples(self.reader) + with patch.object(perf, 'MAX_INSTANCES', 2): + for start in range(3): + self.write_stat(start=start) + samples.poll(10 + start, {42: 'ui'}) + self.assertEqual(len(samples.instances), 2) + self.assertEqual(samples.skipped_instances, 1) + + def test_slow_polling_skips_schedule_slots_instead_of_bursting(self): + clock = FakeClock() + source = FakeMessages() + with patch.object(source, 'poll', side_effect=lambda: (clock.sleep(0.25) or {})): + report = perf.capture(1, 10, messages=source, proc=self.reader, clock=clock, sleep=clock.sleep) + self.assertLessEqual(report['capture']['pollCount'], 4) + self.assertGreater(report['capture']['missedScheduledPolls'], 0) + self.assertGreaterEqual(report['capture']['actualDurationSeconds'], 1) + + def test_comparison_missing_zero_and_different_sampling(self): + before = self.capture([{'uiDebug': event({'frameTimeMillis': 0, 'cpuTimeMillis': 0})}]) + after = copy.deepcopy(before) + after['capture']['requestedSampleRateHz'] = 30 + after['services']['uiDebug']['metrics']['reportedFrameIntervalMs']['p95'] = 5 + result = perf.compare(before, after) + rows = {row['metric']: row for row in result['metrics']} + self.assertEqual(rows['uiDebug.reportedFrameIntervalMs.p95']['delta'], 5) + self.assertIsNone(rows['uiDebug.reportedFrameIntervalMs.p95']['changePercent']) + self.assertEqual(rows['modelV2.reportedModelExecutionMs.p95']['status'], 'missing_data') + self.assertTrue(any(item['field'] == 'requestedSampleRateHz' for item in result['environmentOrSamplingDifferences'])) + json.dumps(result, allow_nan=False) + + def test_standalone_metadata_uses_imported_checkout_not_tool_directory(self): + (self.root / 'openpilot/cereal').mkdir(parents=True) + (self.root / 'openpilot/cereal/log.capnp').write_text('schema fixture') + with patch.object(perf.importlib.util, 'find_spec', return_value=SimpleNamespace(origin=str(self.root / 'openpilot/__init__.py'))): + report = perf.metadata() + self.assertEqual(report['sourceRoot'], str(self.root.resolve())) + self.assertEqual(report['sourceBinding'], 'imported_openpilot') + self.assertTrue(report['declaredRootMatchesImported']) + self.assertEqual(report['tool']['path'], str(Path(perf.__file__).resolve())) + self.assertNotIn('tools/diagnostics/performance.py', report['sourceSha256']) + + def test_explicit_metadata_root_reports_import_origin_mismatch(self): + (self.root / 'openpilot/cereal').mkdir(parents=True) + (self.root / 'openpilot/cereal/log.capnp').write_text('schema fixture') + with patch.object(perf.importlib.util, 'find_spec', return_value=SimpleNamespace(origin='/other/openpilot/__init__.py')): + report = perf.metadata(self.root) + self.assertEqual(report['sourceBinding'], 'explicit') + self.assertFalse(report['declaredRootMatchesImported']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/host b/tools/host new file mode 100755 index 0000000000..6d389de52e --- /dev/null +++ b/tools/host @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" "$@" diff --git a/tools/host_runtime.py b/tools/host_runtime.py new file mode 100755 index 0000000000..5c37467706 --- /dev/null +++ b/tools/host_runtime.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""StarPilot desktop commands with an independent build tree, index and Params.""" + +from __future__ import annotations + +from contextlib import contextmanager +import fcntl +import hashlib +import json +import os +from pathlib import Path +import platform +import shutil +import signal +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +HELP = """Usage: ./dev [args...] + ./tool [args...] ./tools/host [args...] + + c3 / c4 Build and launch the large / compact StarPilot Raylib UI. + onroad Build replay and launch route UI(s); see ./onroad --help. + replay Build and run upstream replay. + cabana Build and run Cabana in its own concurrent cache. + plotjuggler Run PlotJuggler (alias: juggle). + galaxy Run authenticated Galaxy on loopback (optional --port). + python Run Python with current source and native extensions. + pytest Run pytest with current source and native extensions. + shell Open a shell in the isolated host worktree. + sync [bucket] Refresh all caches, or shared/cabana, without launching tools. + help Show this help without installing or building anything. + +Build/launch commands accept an optional leading jobs count, e.g. ./c3 8. +The host cache is .host_runtime/-//. +Native build artifacts, editable installs and Params stay in that cache. +Shared commands wait while another shared command is running; Cabana is separate. +./build remains the Linux ARM64 device build, including ./build --panda. +""" +COMMANDS = {'c3', 'c4', 'onroad', 'replay', 'cabana', 'plotjuggler', 'galaxy', 'python', 'pytest', 'shell', 'sync'} +BUCKET_ALIASES = dict.fromkeys(('shared', 'default', 'ui', 'c3', 'c4', 'onroad', 'replay', 'shell', 'plotjuggler', 'juggle'), 'shared') +BUCKET_ALIASES['cabana'] = 'cabana' +VENDORS = ('msgq_repo', 'opendbc_repo', 'rednose_repo', 'teleoprtc_repo', 'tinygrad_repo') +# Never inherit a device/cross compiler or another project's editable imports. +REMOVE_ENV = ('PYTHONPATH', 'PYTHONHOME', 'VIRTUAL_ENV', 'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'LDFLAGS', + 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'PKG_CONFIG_PATH', 'CPATH', 'LIBRARY_PATH', + 'GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR') +EXCLUDE_DIRS = {'.git', '.venv', '.venv-linux-arm64', '.host_runtime', '.comma_sysroot', '.cache', '__pycache__'} + + +def run(arguments, *, cwd, env=None, capture=False): + return subprocess.run([str(arg) for arg in arguments], cwd=cwd, env=env, check=True, + stdout=subprocess.PIPE if capture else None, text=capture) + + +def git(root, *arguments): + env = {key: value for key, value in os.environ.items() if key not in REMOVE_ENV} + return run(['git', *arguments], cwd=root, env=env, capture=True).stdout + + +def parse(arguments): + command = arguments[0] if arguments else 'help' + if command in ('help', '-h', '--help'): + return 'help', 0, [] + if command == 'juggle': + command = 'plotjuggler' + if command not in COMMANDS: + raise ValueError(f'Unknown developer command: {command}. Run ./dev help.') + args = list(arguments[1:]) + jobs = max(1, os.cpu_count() or 8) + if command not in ('python', 'pytest', 'shell', 'sync') and args and args[0].isdigit(): + jobs = int(args.pop(0)) + if jobs < 1: + raise ValueError('The jobs count must be positive.') + if command == 'sync' and (len(args) > 1 or (args and args[0] not in BUCKET_ALIASES)): + raise ValueError('Usage: ./dev sync [shared|cabana]') + return command, jobs, args + + +def inside(path, root): + """Reject cache symlinks escaping the disposable tree before any mutation.""" + if not path.resolve().is_relative_to(root.resolve()): + raise RuntimeError(f'Cache path escapes its root: {path}') + return path + + +def same_contents(source, destination): + if not destination.is_file() or source.stat().st_size != destination.stat().st_size: + return False + with source.open('rb') as left, destination.open('rb') as right: + while block := left.read(1024 * 1024): + if block != right.read(len(block)): + return False + return True + + +class HostRuntime: + def __init__(self, root, bucket, *, system=None, machine=None): + self.root = Path(root).resolve() + self.system = system or platform.system() + machine = machine or platform.machine() + if self.system not in ('Darwin', 'Linux'): + raise RuntimeError('Desktop tools support macOS and Linux.') + self.cache = self.root / '.host_runtime' / f'{self.system.lower()}-{machine}' / bucket + if (self.root / '.host_runtime').is_symlink(): + raise RuntimeError('.host_runtime must be a local directory, not a symlink.') + inside(self.cache, self.root / '.host_runtime') + self.work = self.cache / 'worktree' + self.venv = self.cache / 'venv' + self.prefix = 'starpilot-dev-' + hashlib.sha256(str(self.cache).encode()).hexdigest()[:12] + self.lock_fd = None + + @contextmanager + def locked(self): + self.cache.mkdir(parents=True, exist_ok=True) + lock_path = inside(self.cache / 'lock', self.cache) + with lock_path.open('a+') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print(f'Waiting for the {self.cache.name} host session to exit…', flush=True) + fcntl.flock(lock, fcntl.LOCK_EX) + lock.seek(0) + lock.truncate() + lock.write(str(os.getpid())) + lock.flush() + self.lock_fd = lock.fileno() + try: + yield + finally: + self.lock_fd = None + fcntl.flock(lock, fcntl.LOCK_UN) + + def sync(self): + inside(self.work, self.cache).mkdir(parents=True, exist_ok=True) + # Include uncommitted development, exclude ignored native/device outputs. + listed = git(self.root, 'ls-files', '--cached', '--others', '--exclude-standard', '-z').split('\0') + paths = {name for name in listed if name and not (set(Path(name).parts) & EXCLUDE_DIRS) + and (self.root / name).is_file()} + manifest = inside(self.cache / 'source-files.json', self.cache) + previous = set(json.loads(manifest.read_text())) if manifest.exists() else set() + for name in sorted(previous - paths): + path = inside(self.work / name, self.work) + if path.is_file() or path.is_symlink(): + path.unlink() + for name in sorted(paths): + source = self.root / name + destination = inside(self.work / name, self.work) + destination.parent.mkdir(parents=True, exist_ok=True) + if source.is_symlink(): + # Keep repository-relative links; never point the host cache at device builds. + inside(source, self.root) + target = os.readlink(source) + if os.path.isabs(target): + raise RuntimeError(f'Absolute source symlink is not portable: {name}') + if destination.is_symlink() and os.readlink(destination) == target: + continue + if destination.exists() or destination.is_symlink(): + destination.unlink() + destination.symlink_to(target) + elif destination.is_symlink() or not same_contents(source, destination): + old_mtime = destination.stat().st_mtime_ns if destination.exists() else None + if destination.is_symlink(): + destination.unlink() + elif destination.is_dir(): + shutil.rmtree(destination) + shutil.copy2(source, destination) + # SCons' MD5-timestamp decider must notice edits even if an editor kept mtime. + if destination.stat().st_mtime_ns == old_mtime: + os.utime(destination, None) + elif source.stat().st_mode != destination.stat().st_mode: + destination.chmod(source.stat().st_mode) + self.sync_git() + manifest.write_text(json.dumps(sorted(paths)) + '\n') + + def sync_git(self): + # Same read-only object sharing as `git clone --shared`, independent index/refs. + metadata = self.work / '.git' + if metadata.is_symlink() or (metadata.exists() and not metadata.is_dir()): + raise RuntimeError('Host worktree has shared Git metadata; remove this host cache and retry.') + if not metadata.exists(): + git(self.work, 'init', '-q') + common = Path(git(self.root, 'rev-parse', '--git-common-dir').strip()) + if not common.is_absolute(): + common = self.root / common + (metadata / 'objects/info/alternates').write_text(str(common.resolve() / 'objects') + '\n') + head = git(self.root, 'rev-parse', 'HEAD').strip() + branch = git(self.root, 'rev-parse', '--abbrev-ref', 'HEAD').strip() + branch = 'host-detached' if branch == 'HEAD' else branch + git(self.work, 'symbolic-ref', 'HEAD', f'refs/heads/{branch}') + git(self.work, 'update-ref', 'HEAD', head) + git(self.work, 'read-tree', head) + config = git(self.root, 'config', '--list') + origin = next((line.split('=', 1)[1] for line in config.splitlines() if line.startswith('remote.origin.url=')), '') + if origin: + git(self.work, 'config', 'remote.origin.url', origin) + # A host helper must never accidentally push through this diagnostic clone. + git(self.work, 'config', 'remote.origin.pushurl', 'disabled://host-runtime') + git(self.work, 'config', 'core.hooksPath', '/dev/null') + + def environment(self): + env = {key: value for key, value in os.environ.items() if key not in REMOVE_ENV} + paths = [self.work, *(self.work / name for name in VENDORS)] + env.update(PYTHONPATH=os.pathsep.join(map(str, paths)), PYTHONNOUSERSITE='1', + PATH=f'{self.venv}/bin' + os.pathsep + env.get('PATH', ''), VIRTUAL_ENV=str(self.venv), + BASEDIR=str(self.work), PWD=str(self.work), PARAMS_ROOT=str(inside(self.cache / 'params', self.cache)), + OPENPILOT_PREFIX=self.prefix, SP_HOST_PREFIX=self.prefix, + SP_HOST_PARAMS_ROOT=str(self.cache / 'params'), SP_HOST_RUNTIME='1', + SP_SCONS_CACHE_DIR=str(self.cache / 'scons-cache'), + SCONS_CACHE=str(self.cache / 'scons-cache'), + NOBOARD='1', SIMULATION='1', SKIP_FW_QUERY='1', STARPILOT_UI_DEV='1', + COMMA_CACHE=str(self.cache.parent / 'downloads')) + if self.system == 'Darwin': + env.update(CC='/usr/bin/clang', CXX='/usr/bin/clang++', ZMQ='1') + return env + + def prepare(self): + uv = shutil.which('uv') + if uv is None: + candidate = Path.home() / '.local/bin/uv' + if candidate.is_file() and os.access(candidate, os.X_OK): + uv = str(candidate) + if uv is None: + raise RuntimeError('uv is required. Run tools/setup_dependencies.sh, then retry ./dev.') + inside(self.venv, self.cache) + env = self.environment() + env.update(UV_PROJECT_ENVIRONMENT=str(self.venv), UV_PYTHON='3.12') + run([uv, 'sync', '--frozen', '--extra', 'tools', '--extra', 'testing', '--extra', 'dev'], cwd=self.work, env=env) + link = self.work / '.venv' + if link.is_symlink(): + if link.resolve() != self.venv.resolve(): + raise RuntimeError('Host worktree .venv points outside its environment.') + elif link.exists(): + raise RuntimeError('Host worktree .venv is not the managed environment; remove this host cache and retry.') + else: + link.symlink_to(self.venv, target_is_directory=True) + + def build(self, command, jobs): + params = 'openpilot/common/libparams_c' + ('.dylib' if self.system == 'Darwin' else '.so') + targets = [params, 'msgq_repo/msgq/ipc_pyx.so', 'msgq_repo/msgq/visionipc/visionipc_pyx.so'] + if command in ('python', 'pytest', 'shell'): + targets += ['rednose_repo/rednose/helpers/ekf_sym_pyx.so', 'openpilot/selfdrive/locationd', + 'openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so'] + if command in ('onroad', 'replay', 'cabana'): + targets += ['openpilot/tools/' + ('cabana/cabana' if command == 'cabana' else 'replay/replay')] + run([self.venv / 'bin/python', '-m', 'SCons', f'-j{jobs}', *targets], cwd=self.work, env=self.environment()) + + def launch(self, command, jobs, arguments): + env = self.environment() + python = self.venv / 'bin/python' + if command in ('c3', 'c4', 'onroad'): + script = 'launch_onroad_desktop.sh' if command == 'onroad' else f'launch_ui_{command}_desktop.sh' + argv = [self.work / 'scripts' / script, str(jobs), *arguments] + elif command in ('replay', 'cabana'): + argv = [self.work / 'openpilot/tools' / command / command, *arguments] + elif command == 'plotjuggler': + argv = [python, '-m', 'openpilot.tools.plotjuggler.juggle', *arguments] + elif command == 'galaxy': + argv = [python, '-m', 'openpilot.starpilot.galaxy.server', *arguments] + elif command == 'pytest': + argv = [python, '-m', 'pytest', *arguments] + env.setdefault('SCALE', '1') # UI imports must not probe a monitor in test workers. + if self.system == 'Darwin': + env['PYTEST_ADDOPTS'] = env.get('PYTEST_ADDOPTS', '') + ' -n0' + elif command == 'shell': + argv = [env.get('SHELL', '/bin/bash'), *arguments] + else: + argv = [python, *arguments] + ipc = Path('/tmp' if self.system == 'Darwin' else '/dev/shm') / f'msgq_{self.prefix}' + if ipc.is_symlink() or (ipc.exists() and (not ipc.is_dir() or ipc.stat().st_uid != os.getuid())): + raise RuntimeError(f'Private messaging path is not owned by this user: {ipc}') + ipc.mkdir(mode=0o700, exist_ok=True) + # Keep the bucket locked if this wrapper is killed while its child is alive. + inherited = () if self.lock_fd is None else (self.lock_fd,) + with subprocess.Popen([str(arg) for arg in argv], cwd=self.work, env=env, pass_fds=inherited) as child: + def forward(signum, _frame): + if child.poll() is None: + child.send_signal(signum) + previous = {sig: signal.signal(sig, forward) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + result = child.wait() + return result if result >= 0 else 128 - result + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +def main(arguments=None): + command, jobs, args = parse(sys.argv[1:] if arguments is None else arguments) + if command == 'help': + print(HELP) + return 0 + if command in ('c3', 'c4', 'onroad') and args in (['--help'], ['-h']): + helper = ROOT / ('openpilot/tools/replay/onroad.py' if command == 'onroad' else 'openpilot/starpilot/ui/host_launch.py') + profile = [] if command == 'onroad' else ['large' if command == 'c3' else 'compact'] + return subprocess.call([sys.executable, str(helper), *profile, '--help'], cwd=ROOT) + if command == 'sync': + buckets = [BUCKET_ALIASES[args[0]]] if args else ['shared', 'cabana'] + else: + buckets = ['cabana' if command == 'cabana' else 'shared'] + for bucket in buckets: + runtime = HostRuntime(ROOT, bucket) + with runtime.locked(): + runtime.sync() + runtime.prepare() + if command != 'sync': + runtime.build(command, jobs) + return runtime.launch(command, jobs, args) + return 0 + + +if __name__ == '__main__': + try: + raise SystemExit(main()) + except (ValueError, RuntimeError, OSError) as error: + print(f'Host tools: {error}', file=sys.stderr) + raise SystemExit(1) from None + except subprocess.CalledProcessError as error: + raise SystemExit(error.returncode if error.returncode > 0 else 128 - error.returncode) from None + except KeyboardInterrupt: + raise SystemExit(130) from None diff --git a/tools/laptop_device_build/Dockerfile b/tools/laptop_device_build/Dockerfile new file mode 100644 index 0000000000..ede273db44 --- /dev/null +++ b/tools/laptop_device_build/Dockerfile @@ -0,0 +1,49 @@ +ARG BASE_IMAGE=ghcr.io/commaai/openpilot-base-aarch64@sha256:c4f206250ec2cbee493589b836e517a3de91281d8d9e4b01f370b5b600a0d143 +FROM ${BASE_IMAGE} + +ARG CAPNP_VERSION=1.0.1 +ARG CAPNP_SHA256=5bdb16f6b389a9e29b04214b9bae1759e8b7fe2b45049d7e3f1f286ba050a200 + +ENV PYTHONUNBUFFERED=1 +ENV UV_LINK_MODE=copy +WORKDIR /work + +# Keep this image minimal; dependency resolution is done by uv inside the workspace. +# rsync/ssh-client are needed for optional sysroot sync from a comma device. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + rsync \ + openssh-client \ + e2fsprogs \ + android-sdk-libsparse-utils \ + qemu-user-static \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + gcc-arm-none-eabi \ + binutils-arm-none-eabi \ + cmake \ + python3-pip \ + xz-utils \ + git \ + && rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + tmpdir="$(mktemp -d)"; \ + curl -fsSL "https://github.com/capnproto/capnproto/archive/refs/tags/v${CAPNP_VERSION}.tar.gz" -o "${tmpdir}/capnproto.tar.gz"; \ + printf '%s %s\n' "${CAPNP_SHA256}" "${tmpdir}/capnproto.tar.gz" | sha256sum -c -; \ + tar -xzf "${tmpdir}/capnproto.tar.gz" -C "${tmpdir}"; \ + cmake -S "${tmpdir}/capnproto-${CAPNP_VERSION}/c++" -B "${tmpdir}/build" \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local; \ + cmake --build "${tmpdir}/build" -j"$(nproc)"; \ + cmake --install "${tmpdir}/build"; \ + ldconfig; \ + capnp --version; \ + rm -rf "${tmpdir}" + +RUN /usr/bin/python3 -m pip install --no-cache-dir --break-system-packages uv==0.12.5 + +# Fix for "dubious ownership" git errors when mounting folders on Linux hosts. +RUN git config --global --add safe.directory /work diff --git a/tools/laptop_device_build/VERIFIED_QCOM_ARTIFACTS.md b/tools/laptop_device_build/VERIFIED_QCOM_ARTIFACTS.md new file mode 100644 index 0000000000..c37dcbb790 --- /dev/null +++ b/tools/laptop_device_build/VERIFIED_QCOM_ARTIFACTS.md @@ -0,0 +1,26 @@ +# Verified stock QCOM artifacts for laptop builds + +The ARM64 laptop container cannot compile QCOM models without `/dev/kgsl-3d0`. Full `./build` imports six stock model/warp artifacts only when `.cache/qcom-artifact-package` contains `manifest.json` and `artifacts/` for every stock target. The package is local and Git ignored. Device builds keep the native compiler commands. + +Generate the manifest on the source device or a trusted matching source tree, using its already compiled stock artifact directory: + +```sh +python3 tools/laptop_device_build/qcom_artifact_contract.py record "$SOURCE_ROOT" "$SOURCE_ROOT/openpilot/selfdrive/modeld/models" > manifest.json +``` + +Copy the six manifest-listed artifacts into `.cache/qcom-artifact-package/artifacts/` and the manifest into `.cache/qcom-artifact-package/manifest.json`. The import checks both ONNX files, selected tinygrad compiler sources, camera/model geometry, native QCOM command recipes, every artifact's size and SHA-256, and the QCOM pickle backend before copying. A changed input or incomplete package fails the build. The manifest records provenance; it is not a cryptographic attestation of who produced the artifacts. A relocated device source can generate the same normalized recipe signature when its actual inputs match. + +## Reusing artifacts after a runtime change + +A reviewed change limited to `tinygrad_repo/tinygrad/runtime/ops_qcom.py` can reuse the original artifacts after paired QCOM runs compare the old and new runtime. Preserve `manifest.json`: running `record` again would incorrectly attribute the existing binaries to new build inputs. Put the compatibility record in `compatibility.json` and its supporting files in `compatibility/` instead. Other source or compiler-command changes still require a new build package. + +The version-1 compatibility record contains: + +- `method`: `paired-qcom-output-sha256`. +- `build_manifest_sha256` and `compatible_source_sha256`: the original manifest's byte hash and the new checkout's `source-signature`. +- `changed_source`: `path`, `build_sha256`, and `runtime_sha256` for the changed runtime file. +- `artifact_sha256`: every original manifest target and its unchanged artifact hash. +- `paired_outputs`: every stock target's three observations in A/B/A input order. Each observation records `input`, `shape`, `dtype`, `build_sha256`, and `runtime_sha256`; each old/new output pair must match. +- `evidence_files`: one to eight filenames mapped to SHA-256 hashes. Files must be regular, nonempty, at most 128 KiB each, and directly inside `compatibility/`. + +Keep the input construction, recurrent-state handling, runtime identities and run results in the evidence files so another developer can review the comparison. This contract validates the recorded bindings and sampled equivalence; it does not independently attest who ran the hardware or establish equivalence for every input. The regular import checks still validate all six artifacts, even when only one is requested. diff --git a/tools/laptop_device_build/extract_sysroot_from_agnos.py b/tools/laptop_device_build/extract_sysroot_from_agnos.py new file mode 100755 index 0000000000..5aba10a5a3 --- /dev/null +++ b/tools/laptop_device_build/extract_sysroot_from_agnos.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 + +import argparse +import json +import lzma +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import urllib.parse +from pathlib import Path + + +REQUIRED_DIRS = [ + ("usr/local/lib", "/usr/local/lib"), + ("usr/local/include", "/usr/local/include"), + ("usr/local/venv/lib/python3.12/site-packages/raylib/install", "/usr/local/venv/lib/python3.12/site-packages/raylib/install"), + ("lib/aarch64-linux-gnu", "/lib/aarch64-linux-gnu"), + ("usr/lib/aarch64-linux-gnu", "/usr/lib/aarch64-linux-gnu"), + ("usr/include", "/usr/include"), +] +VENDOR_CANDIDATES = ["/system/vendor/lib64", "/vendor/lib64"] +OPTIONAL_INCLUDE_DIRS: list[str] = [] +DEBUGFS_NOT_FOUND_MARKERS = ( + "file not found by ext2_lookup", + "while trying to resolve filename", + "couldn't allocate", + "no such file", + "not found", +) +ANDROID_SPARSE_MAGIC = 0xED26FF3A + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Extract TICI sysroot dirs from AGNOS system image") + p.add_argument("--manifest", default="openpilot/system/hardware/tici/agnos.json", help="Path to AGNOS manifest JSON") + p.add_argument("--image-path", default=None, help="Use an existing local system image instead of downloading") + p.add_argument("--output-dir", required=True, help="Destination sysroot directory") + p.add_argument("--cache-dir", default=".cache/agnos", help="Cache directory for downloaded images") + p.add_argument("--url", default=None, help="Override AGNOS system image URL") + p.add_argument("--force-download", action="store_true", help="Redownload system image even if cached") + return p.parse_args() + + +def get_system_url(manifest_path: Path) -> str: + data = json.loads(manifest_path.read_text()) + for entry in data: + if entry.get("name") != "system": + continue + if isinstance(entry.get("url"), str): + return entry["url"] + alt = entry.get("alt") if isinstance(entry, dict) else None + if isinstance(alt, dict) and isinstance(alt.get("url"), str): + return alt["url"] + raise RuntimeError(f"No system entry in manifest: {manifest_path}") + + +def download(url: str, dst: Path) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + tmp = dst.with_suffix(dst.suffix + ".part") + print(f"Downloading {url} -> {dst}", flush=True) + with urllib.request.urlopen(url) as src, open(tmp, "wb") as out: + shutil.copyfileobj(src, out, length=1024 * 1024) + tmp.replace(dst) + + +def download_and_prepare_image(url: str, cache_dir: Path, force_download: bool) -> Path: + cache_dir.mkdir(parents=True, exist_ok=True) + compressed = cache_dir / "agnos_system.img.xz" + raw_image = cache_dir / "agnos_system.img" + ext4_image = cache_dir / "agnos_system.ext4.img" + + if force_download: + compressed.unlink(missing_ok=True) + raw_image.unlink(missing_ok=True) + ext4_image.unlink(missing_ok=True) + + # Prefer an already-cached raw image to avoid unnecessary multi-GB downloads. + if raw_image.exists(): + return raw_image + + parsed_url = urllib.parse.urlparse(url) + if parsed_url.path.endswith(".xz"): + if not compressed.exists(): + download(url, compressed) + if not raw_image.exists(): + print(f"Decompressing {compressed} -> {raw_image}", flush=True) + with lzma.open(compressed, "rb") as f_in, open(raw_image, "wb") as f_out: + shutil.copyfileobj(f_in, f_out, length=1024 * 1024) + else: + if not raw_image.exists(): + download(url, raw_image) + + return raw_image + + +def is_android_sparse(image_path: Path) -> bool: + with open(image_path, "rb") as f: + header = f.read(4) + if len(header) != 4: + return False + magic = int.from_bytes(header, "little") + return magic == ANDROID_SPARSE_MAGIC + + +def ensure_debugfs_readable_image(image_path: Path, cache_dir: Path) -> Path: + if not is_android_sparse(image_path): + return image_path + + if shutil.which("simg2img") is None: + raise RuntimeError( + f"{image_path} is an Android sparse image, but simg2img is not installed." + ) + + converted = cache_dir / "agnos_system.ext4.img" + needs_convert = not converted.exists() + if converted.exists(): + needs_convert = os.path.getmtime(converted) < os.path.getmtime(image_path) + + if needs_convert: + print(f"Converting sparse image {image_path} -> {converted}", flush=True) + proc = subprocess.run(["simg2img", str(image_path), str(converted)], capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError(f"simg2img failed:\n{proc.stdout}\n{proc.stderr}") + return converted + + +def _has_content(path: Path) -> bool: + if not path.exists(): + return False + if path.is_file(): + return path.stat().st_size > 0 + return any(path.iterdir()) + + +def _pick_extracted_root(tmp_path: Path, src_path: str) -> Path | None: + rel = Path(src_path.lstrip("/")) + candidates = [ + tmp_path / rel, + tmp_path / rel.name, + tmp_path, + ] + for c in candidates: + if _has_content(c): + return c + return None + + +def run_debugfs(image_path: Path, src_path: str, dst_path: Path) -> None: + attempts: list[str] = [] + for candidate_src in (src_path, src_path.lstrip("/")): + if not candidate_src: + continue + with tempfile.TemporaryDirectory(prefix="sysroot_extract_") as tmp: + tmp_path = Path(tmp) + cmd = ["debugfs", "-R", f"rdump {candidate_src} {tmp_path}", str(image_path)] + proc = subprocess.run(cmd, capture_output=True, text=True) + out = f"{proc.stdout}\n{proc.stderr}".lower() + attempts.append(f"{' '.join(cmd)}\n{proc.stdout}\n{proc.stderr}") + + if proc.returncode != 0: + continue + if any(marker in out for marker in DEBUGFS_NOT_FOUND_MARKERS): + continue + + source_to_copy = _pick_extracted_root(tmp_path, candidate_src) + if source_to_copy is None: + continue + + if dst_path.exists(): + shutil.rmtree(dst_path) + dst_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(source_to_copy, dst_path, symlinks=True, dirs_exist_ok=True) + return + + joined_attempts = "\n---\n".join(attempts) + raise RuntimeError(f"debugfs could not extract '{src_path}'. Attempts:\n{joined_attempts}") + + +def is_non_empty_dir(path: Path) -> bool: + return path.is_dir() and any(path.iterdir()) + + +def populate_optional_host_includes(output_dir: Path) -> None: + dst_root = output_dir / "usr/include" + if not dst_root.exists(): + return + + include_roots = [ + Path("/usr/include/aarch64-linux-gnu"), + Path("/usr/include"), + Path("/usr/include/x86_64-linux-gnu"), + ] + + for include_dir in OPTIONAL_INCLUDE_DIRS: + dst = dst_root / include_dir + if dst.exists(): + continue + for root in include_roots: + src = root / include_dir + if not src.is_dir(): + continue + shutil.copytree(src, dst, symlinks=True) + break + + openssl_dst = dst_root / "openssl" + for src_dir in ( + Path("/usr/include/aarch64-linux-gnu/openssl"), + Path("/usr/include/x86_64-linux-gnu/openssl"), + ): + if not src_dir.is_dir(): + continue + openssl_dst.mkdir(parents=True, exist_ok=True) + shutil.copytree(src_dir, openssl_dst, symlinks=True, dirs_exist_ok=True) + break + + +def main() -> int: + args = parse_args() + if shutil.which("debugfs") is None: + raise RuntimeError("debugfs is required (install e2fsprogs)") + + manifest_path = Path(args.manifest).resolve() + output_dir = Path(args.output_dir).resolve() + cache_dir = Path(args.cache_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + if args.image_path: + image_path = Path(args.image_path).resolve() + if not image_path.is_file(): + raise RuntimeError(f"Local AGNOS system image not found: {image_path}") + else: + url = args.url or get_system_url(manifest_path) + image_path = download_and_prepare_image(url, cache_dir, args.force_download) + image_path = ensure_debugfs_readable_image(image_path, cache_dir) + + for rel_dst, src_path in REQUIRED_DIRS: + dst = output_dir / rel_dst + print(f"Extracting {src_path} -> {dst}", flush=True) + run_debugfs(image_path, src_path, dst) + + vendor_ok = False + vendor_dst = output_dir / "system/vendor/lib64" + for src_path in VENDOR_CANDIDATES: + try: + print(f"Extracting {src_path} -> {vendor_dst}", flush=True) + run_debugfs(image_path, src_path, vendor_dst) + vendor_ok = True + break + except RuntimeError: + continue + + if not vendor_ok: + print("WARN: vendor libs not found in AGNOS image at /system/vendor/lib64 or /vendor/lib64; falling back to usr/lib/aarch64-linux-gnu", flush=True) + if vendor_dst.is_symlink() or vendor_dst.is_file(): + vendor_dst.unlink() + elif vendor_dst.exists(): + shutil.rmtree(vendor_dst) + vendor_dst.parent.mkdir(parents=True, exist_ok=True) + os.symlink("../../usr/lib/aarch64-linux-gnu", vendor_dst, target_is_directory=True) + + populate_optional_host_includes(output_dir) + + missing = [] + for rel in ("usr/local/lib", "usr/local/include", "lib/aarch64-linux-gnu", "usr/lib/aarch64-linux-gnu", "usr/include", "system/vendor/lib64"): + if not is_non_empty_dir(output_dir / rel): + missing.append(rel) + if missing: + raise RuntimeError(f"Extracted sysroot is incomplete, missing content in: {', '.join(missing)}") + + print(f"Sysroot ready: {output_dir}", flush=True) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except KeyboardInterrupt: + raise + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/tools/laptop_device_build/package_model_chunks.py b/tools/laptop_device_build/package_model_chunks.py new file mode 100644 index 0000000000..26f8724e45 --- /dev/null +++ b/tools/laptop_device_build/package_model_chunks.py @@ -0,0 +1,23 @@ +"""Package the compiled small-model outputs using established Dom model parts.""" +import argparse +from pathlib import Path + + +# Resolve this checkout's stdlib-only owner, even for an absolute CLI invocation +# with no PYTHONPATH or installed workspace package. +import importlib.util +_spec = importlib.util.spec_from_file_location("model_file_chunker", Path(__file__).resolve().parents[2] / "openpilot/common/file_chunker.py") +_chunk_owner = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_chunk_owner) +package_file = _chunk_owner.package_file + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--source', type=Path, required=True) + parser.add_argument('--destination', type=Path, required=True) + args = parser.parse_args() + for name in ('driving_tinygrad.pkl', 'dmonitoring_model_tinygrad.pkl'): + package_file(args.source / name, args.destination / name) + +if __name__ == '__main__': + main() diff --git a/tools/laptop_device_build/qcom_artifact_contract.py b/tools/laptop_device_build/qcom_artifact_contract.py new file mode 100644 index 0000000000..c5c1fce573 --- /dev/null +++ b/tools/laptop_device_build/qcom_artifact_contract.py @@ -0,0 +1,312 @@ +"""Exact source and artifact contract for laptop-only stock QCOM imports.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import sys +import tempfile + +VERSION = 1 +FLAGS = "DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" +MODEL_DIR = Path("openpilot/selfdrive/modeld/models") +COMPATIBILITY_SOURCE = "tinygrad_repo/tinygrad/runtime/ops_qcom.py" +MAX_COMPATIBILITY_FILES = 8 +MAX_COMPATIBILITY_FILE_BYTES = 131072 +COMPATIBILITY_OUTPUTS = { + "dm_warp_1344x760_tinygrad.pkl": ([1, 1382400], "uint8"), + "dm_warp_1928x1208_tinygrad.pkl": ([1, 1382400], "uint8"), + "driving_warp_1344x760_tinygrad.pkl": ([2, 6, 128, 256], "uint8"), + "driving_warp_1928x1208_tinygrad.pkl": ([2, 6, 128, 256], "uint8"), + "dmonitoring_model_tinygrad.pkl": ([1, 553], "float32"), + "driving_tinygrad.pkl": ([1, 2576], "float32"), +} +STATIC_SOURCES = ( + "openpilot/common/transformations/camera.py", + "openpilot/common/transformations/model.py", + "openpilot/system/camerad/cameras/nv12_info.py", + "openpilot/selfdrive/modeld/models/driving_supercombo.onnx", + "openpilot/selfdrive/modeld/models/dmonitoring_model.onnx", +) +TINYGRAD_SOURCE_ROOTS = ("tinygrad", "extra", "examples/openpilot") +SOURCE_SUFFIXES = frozenset((".py", ".c", ".cc", ".cpp", ".h", ".hpp", ".cl", ".metal", ".json")) + + +def _sha(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while data := source.read(1024 * 1024): + digest.update(data) + return digest.hexdigest() + + +def commands(root: Path) -> dict[str, str]: + sys.path.insert(0, str(root)) + from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye + from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE, DM_INPUT_SIZE + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + compiler = root / "tinygrad_repo/examples/openpilot" + commands = {} + for source, target in (("dmonitoring_model.onnx", "dmonitoring_model_tinygrad.pkl"), + ("driving_supercombo.onnx", "driving_tinygrad.pkl")): + commands[target] = " ".join((f'{FLAGS} taskset -c 0-7 python3 "{compiler}/compile_onnx.py"', + f'"{root / MODEL_DIR / source}" "{root / MODEL_DIR / target}"', + "--out-of-band --benchmark-runs 1")) + for camera in (_ar_ox_fisheye, _os_fisheye): + width, height = camera.width, camera.height + stride, y_height, uv_height, frame_size = get_nv12_info(width, height) + target = f"driving_warp_{width}x{height}_tinygrad.pkl" + commands[target] = " ".join((f'{FLAGS} taskset -c 0-7 python3 "{compiler}/compile_warp.py"', + f'--frame {width},{height},{stride},{y_height},{uv_height},{stride * (y_height + uv_height)}', + f'--warp-to {MEDMODEL_INPUT_SIZE[0]}x{MEDMODEL_INPUT_SIZE[1]} --layout yuv420 --frames 2', + f'--output {root / MODEL_DIR / target}')) + target = f"dm_warp_{width}x{height}_tinygrad.pkl" + commands[target] = " ".join((f'{FLAGS} python3 "{compiler}/compile_warp.py"', + f'--frame {width},{height},{stride},{y_height},{uv_height},{frame_size}', + f'--warp-to {DM_INPUT_SIZE[0]}x{DM_INPUT_SIZE[1]}', + f'--layout luma --border-fill 16 --transform-device NPY --output {root / MODEL_DIR / target}')) + return commands + + +def _sources(root: Path) -> dict[str, str]: + paths = [root / source for source in STATIC_SOURCES] + for directory in TINYGRAD_SOURCE_ROOTS: + paths.extend(path for path in (root / "tinygrad_repo" / directory).rglob("*") + if path.is_file() and path.suffix in SOURCE_SUFFIXES and "__pycache__" not in path.parts) + return {path.relative_to(root).as_posix(): _sha(path) for path in sorted(paths)} + + +def _tokens(command: str, root: Path) -> list[str]: + return [token.replace(str(root), "") for token in shlex.split(command)] + + +def source_inventory(root: Path) -> dict: + root = root.resolve() + return {"sources": _sources(root), "commands": {name: _tokens(command, root) for name, command in commands(root).items()}} + + +def source_signature(root: Path) -> str: + return _inventory_signature(source_inventory(root)) + + +def _inventory_signature(inventory: dict) -> str: + return hashlib.sha256(json.dumps(inventory, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def record(root: Path, artifacts: Path) -> dict: + expected = commands(root.resolve()) + from tools.laptop_device_build.validate_artifacts import require_qcom_pickle + entries = {} + for name in expected: + path = artifacts / name + if not path.is_file() or path.is_symlink(): + raise ValueError(f"Missing regular QCOM artifact: {name}") + require_qcom_pickle(path) + entries[name] = {"sha256": _sha(path), "size": path.stat().st_size} + return {"version": VERSION, "source_sha256": source_signature(root), + "source": source_inventory(root), "artifacts": entries} + + +def _unique_pairs(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError(f"Duplicate QCOM manifest field: {key}") + value[key] = item + return value + + +def _valid_sha(value) -> bool: + return type(value) is str and len(value) == 64 and all(char in "0123456789abcdef" for char in value) + + +# Exact reviewed import/diagnostic changes; compiled provenance stays in manifest.json. +RUNTIME_EXTENSION_PROFILE = "deferred-benchmark-and-startup-trace-v1" +RUNTIME_EXTENSION_CHANGES = { + 'tinygrad_repo/examples/openpilot/helpers.py': ( + 'f4bae8ab81931781dc7134d6450c9fa2f24410b81cbc6ecfcebd6c708eb6d912', + '7849829700c575339de7d95ffa869507ad2d65143af4c23a4411ad4d95ba3ae2', + ), + 'tinygrad_repo/tinygrad/runtime/support/am/amdev.py': ( + '3ceddc6b7aabe0f03019c6a6d60b99da11d18ea29065f804dbe15a6ba8d1dd98', + '90222e5c3817b7358f2155971f2e1c7c83480ace83832c4e35a27cef320e82e7', + ), + 'tinygrad_repo/tinygrad/runtime/support/am/ip.py': ( + '92e13084d4603c908721584411f2587f73792ca318b0978640caefbbb54366c4', + '7739c7284002379addcc192e870ef1d7ea7e24a5faec7cab154428833001f923', + ), + 'tinygrad_repo/tinygrad/runtime/support/usb.py': ( + '310ff4f4743a17c80de975196fd4dc21835cf6f672d0a82485d6edd635ad9275', + '546b2b8b2ea773be48e3ba16904b53aae8c2774af21aa519d64ec113b58397c4', + ), +} +RUNTIME_EXTENSION_ADDITIONS = { + 'tinygrad_repo/tinygrad/runtime/support/am/startup_trace.py': + '321e1fd0513f8f8ee98cb7c1dc637b609350eeb77ca4d84c3821e88e1f2da8ad', +} + +def _runtime_parent_source(package: Path, current_source: dict) -> dict: + """Bind supplemental runtime changes to the existing paired-output attestation.""" + path = package / "runtime-compatibility.json" + if not path.exists(): + return current_source + if path.is_symlink() or not path.is_file() or not 0 < path.stat().st_size <= 4096: + raise ValueError("Invalid QCOM runtime extension") + extension = json.loads(path.read_text(), object_pairs_hook=_unique_pairs) + parent = package / "compatibility.json" + if (type(extension) is not dict or set(extension) != { + "version", "profile", "parent_evidence_sha256", "compatible_source_sha256"} or + type(extension["version"]) is not int or extension["version"] != 1 or + extension["profile"] != RUNTIME_EXTENSION_PROFILE or + not parent.is_file() or parent.is_symlink() or + extension["parent_evidence_sha256"] != _sha(parent) or + extension["compatible_source_sha256"] != _inventory_signature(current_source)): + raise ValueError("QCOM runtime extension is not bound to source and parent evidence") + sources = dict(current_source["sources"]) + for name, (before, after) in RUNTIME_EXTENSION_CHANGES.items(): + if sources.get(name) != after: + raise ValueError(f"Unreviewed QCOM runtime source: {name}") + sources[name] = before + for name, expected_sha in RUNTIME_EXTENSION_ADDITIONS.items(): + if sources.pop(name, None) != expected_sha: + raise ValueError(f"Unreviewed QCOM runtime addition: {name}") + return {"sources": sources, "commands": current_source["commands"]} + + +def _verify_runtime_compatibility(package: Path, manifest_path: Path, manifest: dict, + current_source: dict, expected: dict[str, str]) -> None: + """Accept one reviewed runtime-only source delta, never a new build provenance.""" + saved = manifest.get("source") + if type(saved) is not dict or type(saved.get("sources")) is not dict or type(saved.get("commands")) is not dict: + raise ValueError("Invalid QCOM build source inventory") + current_source = _runtime_parent_source(package, current_source) + before, after = saved["sources"], current_source["sources"] + if (set(before) != set(after) or saved["commands"] != current_source["commands"] or + {key for key in before if before[key] != after[key]} != {COMPATIBILITY_SOURCE}): + raise ValueError("QCOM source change is outside reviewed runtime compatibility") + if manifest.get("source_sha256") != _inventory_signature(saved): + raise ValueError("QCOM build source signature mismatch") + sidecar = package / "compatibility.json" + if not sidecar.is_file() or sidecar.is_symlink() or sidecar.stat().st_size > 65536: + raise ValueError("QCOM runtime compatibility evidence is missing or invalid") + evidence = json.loads(sidecar.read_text(), object_pairs_hook=_unique_pairs) + if type(evidence) is not dict or set(evidence) != { + "version", "method", "build_manifest_sha256", "compatible_source_sha256", + "changed_source", "artifact_sha256", "paired_outputs", "evidence_files"}: + raise ValueError("Invalid QCOM runtime compatibility evidence") + files = evidence["evidence_files"] + if (set(expected) != set(COMPATIBILITY_OUTPUTS) or type(files) is not dict or + not 1 <= len(files) <= MAX_COMPATIBILITY_FILES): + raise ValueError("Incomplete QCOM compatibility evidence files") + evidence_dir = package / "compatibility" + if not evidence_dir.is_dir() or evidence_dir.is_symlink(): + raise ValueError("Invalid QCOM compatibility evidence directory") + for name, expected_sha in files.items(): + if type(name) is not str or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}", name) is None: + raise ValueError("Invalid QCOM compatibility evidence filename") + path = evidence_dir / name + if (not _valid_sha(expected_sha) or not path.is_file() or path.is_symlink() or + not 0 < path.stat().st_size <= MAX_COMPATIBILITY_FILE_BYTES or _sha(path) != expected_sha): + raise ValueError(f"QCOM compatibility evidence file mismatch: {name}") + if any(type(entry) is not dict for entry in manifest["artifacts"].values()): + raise ValueError("Invalid QCOM artifact entries") + if (type(evidence["version"]) is not int or evidence["version"] != 1 or + evidence["method"] != "paired-qcom-output-sha256" or + evidence["build_manifest_sha256"] != _sha(manifest_path) or + evidence["compatible_source_sha256"] != _inventory_signature(current_source) or + evidence["changed_source"] != {"path": COMPATIBILITY_SOURCE, + "build_sha256": before[COMPATIBILITY_SOURCE], + "runtime_sha256": after[COMPATIBILITY_SOURCE]} or + evidence["artifact_sha256"] != {name: entry.get("sha256") for name, entry in manifest["artifacts"].items()}): + raise ValueError("QCOM runtime compatibility evidence is not bound to this package and source") + observations = evidence["paired_outputs"] + if type(observations) is not dict or set(observations) != set(expected): + raise ValueError("QCOM runtime compatibility evidence lacks a stock target") + for target, rows in observations.items(): + if type(rows) is not list or len(rows) != 3: + raise ValueError(f"Invalid QCOM paired outputs: {target}") + for index, row in enumerate(rows): + if (type(row) is not dict or set(row) != {"input", "shape", "dtype", "build_sha256", "runtime_sha256"} or + row["input"] != ("A" if index != 1 else "B") or + row["shape"] != COMPATIBILITY_OUTPUTS[target][0] or + row["dtype"] != COMPATIBILITY_OUTPUTS[target][1] or + not _valid_sha(row["build_sha256"]) or row["build_sha256"] != row["runtime_sha256"]): + raise ValueError(f"Invalid QCOM paired outputs: {target}") + + +def verify(package: Path, root: Path, target_name: str, actual_command: str) -> tuple[Path, str]: + expected = commands(root.resolve()) + from tools.laptop_device_build.validate_artifacts import require_qcom_pickle + if target_name not in expected or _tokens(actual_command, root.resolve()) != _tokens(expected[target_name], root.resolve()): + raise ValueError(f"QCOM command recipe changed: {target_name}") + manifest_path = package / "manifest.json" + manifest = json.loads(manifest_path.read_text(), object_pairs_hook=_unique_pairs) + if type(manifest) is not dict: + raise ValueError("Invalid QCOM manifest") + current_source = source_inventory(root) + if manifest.get("source") != current_source and not (package / "compatibility.json").exists(): + saved = manifest.get("source", {}) + for section in ("sources", "commands"): + old = saved.get(section, {}) if type(saved) is dict else {} + if type(old) is not dict: + old = {} + new = current_source[section] + for path in sorted(set(old) | set(new)): + if old.get(path) != new.get(path): + raise ValueError(f"QCOM {section} changed: {path}") + if (type(manifest.get("version")) is not int or manifest["version"] != VERSION or + type(manifest.get("artifacts")) is not dict or + set(manifest["artifacts"]) != set(expected)): + raise ValueError("QCOM package source or artifact set does not match this checkout") + if manifest.get("source") == current_source: + if manifest.get("source_sha256") != _inventory_signature(current_source): + raise ValueError("QCOM package source signature mismatch") + else: + _verify_runtime_compatibility(package, manifest_path, manifest, current_source, expected) + for name, entry in manifest["artifacts"].items(): + path = package / "artifacts" / name + if (type(entry) is not dict or not path.is_file() or path.is_symlink() or + type(entry.get("size")) is not int or path.stat().st_size != entry["size"] or + _sha(path) != entry.get("sha256")): + raise ValueError(f"QCOM package artifact mismatch: {name}") + require_qcom_pickle(path) + return package / "artifacts" / target_name, manifest["artifacts"][target_name]["sha256"] + + +def import_artifact(package: Path, root: Path, target: Path, command: str) -> None: + source, expected_sha = verify(package, root, target.name, command) + with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".qcom-import-", delete=False) as output: + temporary = Path(output.name) + try: + with source.open("rb") as original: + shutil.copyfileobj(original, output) + output.flush() + os.fsync(output.fileno()) + if _sha(temporary) != expected_sha: + raise ValueError(f"QCOM artifact changed during copy: {target.name}") + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("source-signature", "record")) + parser.add_argument("root", type=Path) + parser.add_argument("artifacts", nargs="?", type=Path) + args = parser.parse_args() + if args.command == "source-signature": + print(source_signature(args.root)) + elif args.artifacts is None: + parser.error("record requires artifact directory") + else: + print(json.dumps(record(args.root, args.artifacts), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tools/laptop_device_build/runtime_paths.py b/tools/laptop_device_build/runtime_paths.py new file mode 100644 index 0000000000..297303c8ec --- /dev/null +++ b/tools/laptop_device_build/runtime_paths.py @@ -0,0 +1,12 @@ +"""Map virtual-environment library paths to their deployment location.""" + +from pathlib import PurePosixPath + + +def runtime_library_path(library_dir: str, build_venv: str, target_venv: str | None = None) -> str: + if target_venv is None: + return library_dir + library, build, target = map(PurePosixPath, (library_dir, build_venv, target_venv)) + if not all(path.is_absolute() and ".." not in path.parts for path in (library, build, target)): + raise ValueError("Library and virtual-environment paths must be absolute") + return str(target / library.relative_to(build)) diff --git a/tools/laptop_device_build/tests/test_model_artifact_git_policy.py b/tools/laptop_device_build/tests/test_model_artifact_git_policy.py new file mode 100644 index 0000000000..3c178d86b2 --- /dev/null +++ b/tools/laptop_device_build/tests/test_model_artifact_git_policy.py @@ -0,0 +1,31 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + + +class TestModelArtifactGitPolicy(unittest.TestCase): + def test_only_small_numbered_packages_are_publishable(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + subprocess.run(['git', 'init', '--quiet', str(root)], check=True) + for name in ('.gitignore', '.gitattributes'): + (root / name).write_bytes((ROOT / name).read_bytes()) + base = 'openpilot/selfdrive/modeld/models/' + publishable = [name + suffix for name in ('driving_tinygrad.pkl', 'dmonitoring_model_tinygrad.pkl') + for suffix in ('.chunk01of02', '.chunk01of100', '.chunk100of100', '.chunk1024of1024', '.chunkmanifest', '.chunksha256')] + ignored = ['driving_tinygrad.pkl', 'dmonitoring_model_tinygrad.pkl', 'driving_tinygrad.pkl.unchunked', + 'big_driving_tinygrad.pkl', 'big_driving_tinygrad.pkl.chunk01of02', + 'driving_tinygrad.pkl.chunkNOTESof02', 'other_driving_tinygrad.pkl.chunk01of02'] + for name, expected in [(n, False) for n in publishable] + [(n, True) for n in ignored]: + with self.subTest(name=name): + result = subprocess.run(['git', 'check-ignore', '--quiet', '--no-index', '--', base + name], cwd=root) + self.assertEqual(result.returncode, 0 if expected else 1) + result = subprocess.check_output(['git', 'check-attr', 'text', '--', base + 'driving_tinygrad.pkl.chunk01of02'], cwd=root, text=True) + self.assertTrue(result.endswith('text: unset\n')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/laptop_device_build/tests/test_ordinary_resources.py b/tools/laptop_device_build/tests/test_ordinary_resources.py new file mode 100644 index 0000000000..4adb245d38 --- /dev/null +++ b/tools/laptop_device_build/tests/test_ordinary_resources.py @@ -0,0 +1,53 @@ +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] + + +def owner(path): + spec = importlib.util.spec_from_file_location(path.stem, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestOrdinaryResources(unittest.TestCase): + def test_release_revision_pointer_rejected_without_checkout_dependence(self): + check = owner(ROOT / 'tools/resources/check.py') + responses = ['100644 blob abc 127\tasset.onnx\n', check.POINTER + b'\noid sha256:bad\n'] + with patch.object(check.subprocess, 'check_output', side_effect=responses), patch.object(sys, 'argv', ['check', '--revision', 'HEAD']): + with self.assertRaisesRegex(ValueError, 'placeholder pointer'): + check.main() + + def test_release_ordinary_blob_and_binary_attributes_accepted(self): + check = owner(ROOT / 'tools/resources/check.py') + check.validate_blob('asset.wav', 2048) + check.validate_blob('.gitattributes', 12, b'*.wav -text\n') + + def test_active_filter_and_oversized_blob_rejected(self): + check = owner(ROOT / 'tools/resources/check.py') + with self.assertRaisesRegex(ValueError, 'active checkout filter'): + check.validate_blob('.gitattributes', 22, b'*.wav filter=lfs -text\n') + check.validate_blob('exact-limit.pkl', 100 * 1024 * 1024) + with self.assertRaisesRegex(ValueError, 'exceeds 100 MiB'): + check.validate_blob('too-large.pkl', check.MAX_BLOB_BYTES + 1) + + def test_binary_lint_uses_byte_attributes_but_never_waives_pointer(self): + lint = owner(ROOT / 'scripts/lint/check_added_large_files.py') + with tempfile.TemporaryDirectory() as directory: + file = Path(directory) / 'asset.wav' + file.write_bytes(b'W' * 2048) + attr = f'{file}\0text\0unset\0' + with patch.object(lint.subprocess, 'run') as run: + run.return_value.stdout = attr + self.assertEqual(lint.check_added_large_files([str(file)], 1), 0) + file.write_bytes(b'version https://git-lfs.github.com/spec/v1\noid sha256:bad\n') + self.assertEqual(lint.check_added_large_files([str(file)], 1), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/laptop_device_build/validate_artifacts.py b/tools/laptop_device_build/validate_artifacts.py new file mode 100755 index 0000000000..0170bd54f4 --- /dev/null +++ b/tools/laptop_device_build/validate_artifacts.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Inspect device-build outputs without importing binaries or executing pickles.""" + +import argparse +from pathlib import Path + + +# Resolve this checkout's stdlib-only owner, even for an absolute CLI invocation +# with no PYTHONPATH or installed workspace package. +import importlib.util +_spec = importlib.util.spec_from_file_location("model_file_chunker", Path(__file__).resolve().parents[2] / "openpilot/common/file_chunker.py") +_chunk_owner = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_chunk_owner) +materialize_file_chunked = _chunk_owner.materialize_file_chunked +import pickletools +import struct + + +EM_AARCH64 = 183 +ELF_FILES = ( + "openpilot/common/libparams_c.so", + "openpilot/selfdrive/pandad/pandad", + "openpilot/system/loggerd/loggerd", + "openpilot/system/loggerd/encoderd", + "msgq_repo/msgq/ipc_pyx.so", + "msgq_repo/msgq/visionipc/visionipc_pyx.so", + "rednose_repo/rednose/helpers/ekf_sym_pyx.so", +) +ARCHIVES = ("openpilot/cereal/libcereal.a", "openpilot/cereal/libsocketmaster.a") +MODEL_DIR = Path("openpilot/selfdrive/modeld/models") +# Both current fisheye camera configs in transformations/camera.py. +WARP_SIZES = ("1344x760", "1928x1208") +MODELS = ("driving_tinygrad.pkl", "dmonitoring_model_tinygrad.pkl") +WARPS = tuple(f"{kind}_warp_{size}_tinygrad.pkl" for kind in ("driving", "dm") for size in WARP_SIZES) + + +def require_runtime_search_paths(data: bytes, name: str) -> None: + phoff = struct.unpack_from(" len(data) or size % 16: + raise ValueError(f"{name}: malformed dynamic table") + tags = [] + for pos in range(offset, offset + size, 16): + tag, value = struct.unpack_from(" len(data): + raise ValueError(f"{name}: truncated dynamic string table") + strings = data[start:start + length] + for value in paths: + end = strings.find(b"\0", value) + if value >= length or end < 0: + raise ValueError(f"{name}: invalid runtime search path") + for path in strings[value:end].decode().split(":"): + if path.startswith(("/work/", "/Users/", "/opt/tici-sysroot/")) or ".venv-linux-arm64" in path: + raise ValueError(f"{name}: build-host runtime search path: {path}") + + +def require_aarch64_elf(data: bytes, name: str, *, relocatable: bool = False) -> None: + if len(data) < 64 or data[:4] != b"\x7fELF" or data[4] != 2 or data[5] != 1: + raise ValueError(f"{name}: expected little-endian ELF64") + if struct.unpack_from(" len(data): + raise ValueError(f"{name}: invalid ELF section table") + sections = [struct.unpack_from(" len(data) for section in sections): + raise ValueError(f"{name}: object section extends beyond file") + if not any(section[1] == 1 and section[5] > 0 for section in sections): + raise ValueError(f"{name}: missing object code/data section") + else: + if not phoff or not phnum or phentsize < 56 or phoff + phentsize * phnum > len(data): + raise ValueError(f"{name}: invalid ELF program table") + if not any(struct.unpack_from(" 0 and + struct.unpack_from(" None: + if not data.startswith(b"!\n"): + raise ValueError(f"{name}: expected a regular ar archive") + offset, objects = 8, 0 + while offset < len(data): + header = data[offset:offset + 60] + if len(header) != 60 or header[58:] != b"`\n": + raise ValueError(f"{name}: malformed archive member") + try: + size = int(header[48:58].strip()) + except ValueError as exc: + raise ValueError(f"{name}: invalid archive member size") from exc + start, end = offset + 60, offset + 60 + size + if end > len(data): + raise ValueError(f"{name}: truncated archive member") + member = data[start:end] + if member.startswith(b"\x7fELF"): + require_aarch64_elf(member, name, relocatable=True) + objects += 1 + elif not header[:16].strip().startswith((b"/", b"__.SYMDEF")): + raise ValueError(f"{name}: non-ELF object member") + offset = end + (size % 2) + if not objects or offset != len(data): + raise ValueError(f"{name}: no complete AArch64 object members") + + +def pickle_opcodes(path: Path) -> bytes: + with path.open("rb") as file: + prefix = file.read(8) + if prefix.startswith(b"\x80"): + file.seek(0) + return file.read(100_000_001) + if len(prefix) != 8: + raise ValueError(f"{path}: truncated model") + size = struct.unpack(" None: + name = path.name + path = materialize_file_chunked(path) + data = pickle_opcodes(path) + if len(data) > 100_000_000: + raise ValueError(f"{path}: pickle opcodes exceed inspection bound") + try: + ops = list(pickletools.genops(data)) + tokens = {arg for _, arg, _ in ops if isinstance(arg, str)} + except (ValueError, EOFError) as exc: + raise ValueError(f"{path}: malformed pickle opcode stream") from exc + if not ops or ops[-1][0].name != "STOP" or not {"metadata", "run", "input_specs", "QCOM"}.issubset(tokens): + raise ValueError(f"{path}: missing captured model/warp graph") + if name in MODELS: + if not {"output_specs", "output_slices", "input_shapes"}.issubset(tokens): + raise ValueError(f"{path}: missing model input/output specification") + if path.stat().st_size <= 8 + len(data): + raise ValueError(f"{path}: missing out-of-band model buffers") + # QCOM captures also serialize CPU host-dispatch/renderer nodes. A CPU token + # therefore does not identify the device backend; require QCOM above and + # reject explicitly captured foreign GPU backends here. + if any(token in ("METAL", "AMD") or token.startswith(("METAL:", "AMD:", "USB+AMD")) for token in tokens): + raise ValueError(f"{path}: foreign GPU captured backend") + tinyjit_global = False + captured_global = False + for index, (op, _arg, _) in enumerate(ops): + if op.name != "STACK_GLOBAL": + continue + names = [value for _, value, _ in ops[max(0, index - 4):index] if isinstance(value, str)] + if names[-2:] == ["tinygrad.engine.jit", "_TinyJit"]: + tinyjit_global = True + if names and names[-1] == "CapturedJit": + captured_global = True + if not tinyjit_global or not captured_global: + raise ValueError(f"{path}: missing TinyJit capture structure") + + +def validate(root: Path) -> None: + for name in ELF_FILES: + path = root / name + require_aarch64_elf(path.read_bytes(), name) + for name in ARCHIVES: + path = root / name + require_aarch64_archive(path.read_bytes(), name) + + model_dir = root / MODEL_DIR + actual_warps = {p.name for p in model_dir.glob("driving_warp_*_tinygrad.pkl")} + actual_warps.update(p.name for p in model_dir.glob("dm_warp_*_tinygrad.pkl")) + if actual_warps != set(WARPS): + raise ValueError(f"camera warp set differs from current camera configs: {sorted(actual_warps)}") + for name in (*MODELS, *WARPS): + require_qcom_pickle(model_dir / name) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + args = parser.parse_args() + try: + validate(args.root) + except (OSError, ValueError) as exc: + parser.exit(1, f"Device artifact validation failed: {exc}\n") + print("Device artifacts: static AArch64 and QCOM capture structure checks passed; runtime inference remains unverified.") + + +if __name__ == "__main__": + main() diff --git a/tools/op.sh b/tools/op.sh index d56c5d416c..0f71ad111c 100755 --- a/tools/op.sh +++ b/tools/op.sh @@ -119,13 +119,8 @@ function op_check_git() { echo -e " ↳ [${GREEN}✔${NC}] git found." fi - echo "Checking checked-in model resources..." - if [[ $(file -b "$OPENPILOT_ROOT/openpilot/selfdrive/modeld/models/dmonitoring_model.onnx") == "data" ]]; then - echo -e " ↳ [${GREEN}✔${NC}] model resources found." - else - echo -e " ↳ [${RED}✗${NC}] model resources missing or invalid; restore the ordinary checkout" - return 1 - fi + echo "Checking ordinary checkout resources..." + python3 "$OPENPILOT_ROOT/tools/resources/check.py" || return 1 echo "Checking tracked dependency sources..." op_check_dependencies @@ -318,7 +313,7 @@ function op_lint() { function op_test() { op_before_cmd - op_run_command tools/test_runner.py "$@" + op_run_command python3 tools/ci/run_host_tests.py "$@" } function op_replay() { @@ -352,14 +347,25 @@ function op_check_agnos_update() { return 0 fi - local choice current_version target_version + local choice current_version target_version update_policy target_config current_version="$(< /VERSION)" - target_version="$(unset AGNOS_VERSION; source "$OPENPILOT_ROOT/launch_env.sh"; echo "$AGNOS_VERSION")" + target_config="$(unset AGNOS_VERSION AGNOS_UPDATE_POLICY; source "$OPENPILOT_ROOT/launch_env.sh"; printf '%s\n%s\n' "$AGNOS_VERSION" "${AGNOS_UPDATE_POLICY:-auto}")" + target_version="${target_config%%$'\n'*}" + update_policy="${target_config#*$'\n'}" + if [[ -z "$target_version" || ( "$update_policy" != "auto" && "$update_policy" != "retain" ) ]]; then + echo "Invalid AGNOS installation policy. No OS update was attempted." + return 1 + fi if [[ "$current_version" == "$target_version" ]]; then return 0 fi + if [[ "$update_policy" == "retain" ]]; then + echo "This StarPilot build retains AGNOS $target_version; installed $current_version. No OS update was attempted." + return 1 + fi + echo -e "${BOLD}AGNOS update available:${NC} $current_version → $target_version" if read -r -p "Install it now? [y/N] " choice && [[ "$choice" =~ ^[Yy]$ ]]; then op_run_command "$OPENPILOT_ROOT/openpilot/common/hardware/comma/agnos.py" --swap \ diff --git a/tools/release/README.md b/tools/release/README.md index 25ae32f704..08e4a922b4 100644 --- a/tools/release/README.md +++ b/tools/release/README.md @@ -1,3 +1,10 @@ +# Release builds + +Run `./build --mapd` in the source checkout before assembling a release. Both +release scripts validate and include the generated Mapd executable and manifest; +missing or stale packages stop assembly. Regenerate the package when Mapd source +or its upstream pin changes. The generated files stay out of the source branch. + # openpilot releases ``` diff --git a/tools/release/build_release.sh b/tools/release/build_release.sh index cdff327c7d..735e11238d 100755 --- a/tools/release/build_release.sh +++ b/tools/release/build_release.sh @@ -17,6 +17,8 @@ if [ -z "$RELEASE_BRANCH" ]; then exit 1 fi +python3 "$SOURCE_DIR/tools/release/stage_mapd_provider.py" --source "$SOURCE_DIR" + BUILD_BRANCH=release-mici-staging @@ -38,6 +40,7 @@ git read-tree --empty echo "[-] copying files T=$SECONDS" cd "$SOURCE_DIR" ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" -- +python3 "$SOURCE_DIR/tools/release/stage_mapd_provider.py" --source "$SOURCE_DIR" --destination "$BUILD_DIR" # in the directory cd "$BUILD_DIR" @@ -60,11 +63,7 @@ else scons panda/ fi -if [ -n "$INCLUDE_BIG_MODEL" ]; then - test -f openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl - test -f openpilot/selfdrive/modeld/models/big_driving_warp_1344x760_tinygrad.pkl - test -f openpilot/selfdrive/modeld/models/big_driving_warp_1928x1208_tinygrad.pkl -fi +# Big models are selected and verified through the external model catalog. # Cleanup find . -name '*.a' -delete diff --git a/tools/release/build_stripped.sh b/tools/release/build_stripped.sh index 4ae97f2579..ad01531ffe 100755 --- a/tools/release/build_stripped.sh +++ b/tools/release/build_stripped.sh @@ -6,6 +6,7 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" SOURCE_DIR="$(git -C "$DIR" rev-parse --show-toplevel)" python3 "$SOURCE_DIR/tools/vendor/check.py" +python3 "$SOURCE_DIR/tools/release/stage_mapd_provider.py" --source "$SOURCE_DIR" if [ -z "$TARGET_DIR" ]; then TARGET_DIR="$(mktemp -d)" fi @@ -31,6 +32,7 @@ fi echo "[-] copying files T=$SECONDS" cd "$SOURCE_DIR" ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$TARGET_DIR" -- +python3 "$SOURCE_DIR/tools/release/stage_mapd_provider.py" --source "$SOURCE_DIR" --destination "$TARGET_DIR" # in the directory cd "$TARGET_DIR" @@ -55,8 +57,9 @@ date: $DATETIME master commit: $GIT_HASH " -# Check the packaged dependency layout and ordinary checkout assets. +# Check packaged dependency layout and reject pointer payloads. python3 tools/vendor/check.py --revision HEAD +python3 tools/resources/check.py --revision HEAD source "$SOURCE_DIR/tools/release/check_file_sizes.sh" diff --git a/tools/release/stage_mapd_provider.py b/tools/release/stage_mapd_provider.py new file mode 100644 index 0000000000..a5524b02cd --- /dev/null +++ b/tools/release/stage_mapd_provider.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Verify a prebuilt Mapd provider, then copy it into a release tree.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import sys + +SOURCE_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(SOURCE_ROOT)) +from openpilot.starpilot.maps.artifact import elf_arm64_static_file, source_digest + + +PROVIDER = Path('openpilot/starpilot/maps/provider') +MANIFEST_LIMIT = 4096 +BINARY_LIMIT = 128 * 1024 * 1024 +HEX40 = re.compile(r'[0-9a-f]{40}\Z') +HEX64 = re.compile(r'[0-9a-f]{64}\Z') + + +def _regular(path: Path, *, max_bytes: int, executable: bool = False): + info = path.lstat() + if (not stat.S_ISREG(info.st_mode) or not 0 < info.st_size <= max_bytes or + executable and not info.st_mode & 0o111): + raise ValueError(f'invalid Mapd package file: {path}') + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + file = os.fdopen(descriptor, 'rb') + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + file.close() + raise ValueError(f'invalid Mapd package file: {path}') + return file + + +def _directories(root: Path, *, provider: bool = True) -> None: + relatives = (Path('.'), Path('openpilot'), Path('openpilot/starpilot'), Path('openpilot/starpilot/maps')) + for relative in relatives + ((PROVIDER,) if provider else ()): + path = root / relative + if not stat.S_ISDIR(path.lstat().st_mode): + raise ValueError(f'unsafe Mapd package directory: {path}') + + +def _unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate Mapd manifest field') + result[key] = value + return result + + +def validate_provider(source: Path) -> dict: + source = Path(source) + _directories(source) + binary = source / PROVIDER / 'mapd' + manifest_path = source / PROVIDER / 'manifest.json' + with _regular(manifest_path, max_bytes=MANIFEST_LIMIT) as file: + manifest = json.loads(file.read(MANIFEST_LIMIT + 1), object_pairs_hook=_unique) + fields = {'schemaVersion', 'goVersion', 'target', 'sourceRevision', 'upstreamRevision', + 'sourceDigest', 'binarySha256'} + if (type(manifest) is not dict or set(manifest) != fields or type(manifest['schemaVersion']) is not int or + manifest['schemaVersion'] != 1 or manifest['goVersion'] != 'go1.25.1' or + manifest['target'] != 'linux-arm64-static' or + any(type(manifest[name]) is not str or pattern.fullmatch(manifest[name]) is None + for name, pattern in (('sourceRevision', HEX40), ('upstreamRevision', HEX40), + ('sourceDigest', HEX64), ('binarySha256', HEX64)))): + raise ValueError('invalid Mapd package manifest') + pinned = json.loads((source / 'upstream-sync.json').read_text()) + upstream = next(item['commit'] for item in pinned['dependencies'] if item['path'] == 'mapd_repo') + if manifest['upstreamRevision'] != upstream or manifest['sourceDigest'] != source_digest(source / 'mapd_repo'): + raise ValueError('Mapd package source changed; run ./build --mapd') + stamps = {manifest[name].encode() for name in ('sourceRevision', 'upstreamRevision', 'sourceDigest')} + digest, seen, carry = hashlib.sha256(), set(), b'' + with _regular(binary, max_bytes=BINARY_LIMIT, executable=True) as file: + if not elf_arm64_static_file(file): + raise ValueError('Mapd package is not a static Linux ARM64 executable') + file.seek(0) + for chunk in iter(lambda: file.read(1024 * 1024), b''): + digest.update(chunk) + window = carry + chunk + seen.update(stamp for stamp in stamps if stamp in window) + carry = window[-128:] + if digest.hexdigest() != manifest['binarySha256'] or seen != stamps: + raise ValueError('Mapd package hash or embedded source stamps differ; run ./build --mapd') + return manifest + + +def stage_provider(source: Path, destination: Path) -> dict: + source, destination = Path(source), Path(destination) + manifest = validate_provider(source) + _directories(destination, provider=False) + if (source_digest(destination / 'mapd_repo') != manifest['sourceDigest'] or + (destination / 'upstream-sync.json').read_bytes() != (source / 'upstream-sync.json').read_bytes()): + raise ValueError('release Mapd source differs from the validated package') + provider = destination / PROVIDER + if provider.exists() or provider.is_symlink(): + _directories(destination) + else: + provider.mkdir(mode=0o755) + outputs = [] + try: + for name, mode in (('mapd', 0o755), ('manifest.json', 0o644)): + target = destination / PROVIDER / name + with _regular(source / PROVIDER / name, max_bytes=BINARY_LIMIT if name == 'mapd' else MANIFEST_LIMIT, + executable=name == 'mapd') as input_file: + descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode) + outputs.append(target) + with os.fdopen(descriptor, 'wb') as output: + for chunk in iter(lambda: input_file.read(1024 * 1024), b''): + output.write(chunk) + with (destination / PROVIDER / 'mapd').open('rb') as file: + if hashlib.file_digest(file, 'sha256').hexdigest() != manifest['binarySha256']: + raise ValueError('Mapd package changed while staging') + if json.loads((destination / PROVIDER / 'manifest.json').read_text(), object_pairs_hook=_unique) != manifest: + raise ValueError('Mapd manifest changed while staging') + except Exception: + for path in outputs: + path.unlink(missing_ok=True) + raise + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path, required=True) + parser.add_argument('--destination', type=Path) + args = parser.parse_args() + try: + manifest = stage_provider(args.source, args.destination) if args.destination else validate_provider(args.source) + except (OSError, ValueError, KeyError, TypeError, StopIteration) as error: + parser.exit(1, f'Mapd release package unavailable: {error}\nBuild it in the source checkout with ./build --mapd.\n') + print(f"Verified Mapd release package {manifest['binarySha256']}") + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/release/tests/test_stage_mapd_provider.py b/tools/release/tests/test_stage_mapd_provider.py new file mode 100644 index 0000000000..2234e80aa2 --- /dev/null +++ b/tools/release/tests/test_stage_mapd_provider.py @@ -0,0 +1,106 @@ +import hashlib +import json +from pathlib import Path +import stat +import struct +import subprocess +import tempfile +import unittest + +from openpilot.starpilot.maps.artifact import source_digest +from tools.release.stage_mapd_provider import PROVIDER, stage_provider, validate_provider + + +class TestStageMapdProvider(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.base = Path(temporary.name) + self.source = self.base / 'source' + self.source.mkdir() + self.destination = self.base / 'release' + self.destination.mkdir() + self.git('init', '--quiet') + (self.source / 'mapd_repo').mkdir() + (self.source / 'mapd_repo/main.go').write_text('package main\n') + (self.source / 'upstream-sync.json').write_text(json.dumps({'dependencies': [ + {'path': 'mapd_repo', 'commit': 'a' * 40}, + ]})) + self.git('add', 'mapd_repo/main.go', 'upstream-sync.json') + self.git('-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', + 'commit', '--quiet', '-m', 'fixture') + self.revision = self.git('rev-parse', 'HEAD').strip() + (self.source / PROVIDER).mkdir(parents=True) + (self.destination / 'openpilot/starpilot/maps').mkdir(parents=True) + (self.destination / 'mapd_repo').mkdir() + (self.destination / 'mapd_repo/main.go').write_text('package main\n') + (self.destination / 'upstream-sync.json').write_bytes((self.source / 'upstream-sync.json').read_bytes()) + self.make_package() + + def git(self, *args): + return subprocess.check_output(['git', '-C', str(self.source), *args], text=True) + + def make_package(self): + digest = source_digest(self.source / 'mapd_repo') + header = bytearray(64) + header[:6] = b'\x7fELF\x02\x01' + struct.pack_into(' MAX_BLOB_BYTES: + raise ValueError(f'ordinary Git blob exceeds 100 MiB: {path}') + if data is not None and data.startswith(POINTER): + raise ValueError(f'placeholder pointer payload: {path}') + if Path(path).name == '.gitattributes' and data is not None: + for line in data.decode().splitlines(): + if line.strip() and not line.lstrip().startswith('#') and 'filter=lfs' in line.split(): + raise ValueError(f'active checkout filter: {path}') + if path == '.lfsconfig': + raise ValueError('obsolete checkout configuration') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--revision') + args = parser.parse_args() + if args.revision: + entries = subprocess.check_output(['git', 'ls-tree', '-rl', args.revision], cwd=ROOT, text=True).splitlines() + for entry in entries: + fields, path = entry.split('\t', 1) + mode, kind, oid, size = fields.split() + if kind != 'blob': + continue + size = int(size) + data = subprocess.check_output(['git', 'cat-file', 'blob', oid], cwd=ROOT) if size <= 1024 or Path(path).name == '.gitattributes' else None + validate_blob(path, size, data) + else: + paths = subprocess.check_output(['git', 'ls-files', '-z'], cwd=ROOT).split(b'\0') + for raw in paths: + if not raw: + continue + path = raw.decode() + file = ROOT / path + if not file.is_file(): + continue # tracked symlinks/deletions are handled by source/release gates + with file.open('rb') as f: + data = f.read() if file.name == '.gitattributes' else f.read(1024) + validate_blob(path, file.stat().st_size, data) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/test_scons_cache.py b/tools/test_scons_cache.py new file mode 100644 index 0000000000..0976bfbce2 --- /dev/null +++ b/tools/test_scons_cache.py @@ -0,0 +1,111 @@ +import ast +import os +from pathlib import Path +import re +import stat +import tempfile +import unittest + + +SCONSTRUCT = Path(__file__).resolve().parents[1] / 'SConstruct' + + +class CacheOS: + path = os.path + + def __init__(self, *, vanish_on_walk=None, vanish_on_unlink=None, fail_stat=None, fail_unlink=None): + self.vanish_on_walk = vanish_on_walk + self.vanish_on_unlink = vanish_on_unlink + self.fail_stat = fail_stat + self.fail_unlink = fail_unlink + + def walk(self, root): + for entry in os.walk(root): + if self.vanish_on_walk is not None and self.vanish_on_walk.parent == Path(entry[0]): + self.vanish_on_walk.unlink(missing_ok=True) + yield entry + + def stat(self, path, *, follow_symlinks=True): + if self.fail_stat == Path(path): + raise PermissionError(path) + return os.stat(path, follow_symlinks=follow_symlinks) + + def unlink(self, path): + if self.fail_unlink == Path(path): + raise PermissionError(path) + if self.vanish_on_unlink == Path(path): + os.unlink(path) + raise FileNotFoundError(path) + os.unlink(path) + + +def prune(root, limit, os_impl=os): + tree = ast.parse(SCONSTRUCT.read_text()) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == 'prune_cache_dir') + scope = {'os': os_impl, 're': re, 'stat': stat, 'cache_dir': str(root), 'cache_size_limit': limit} + exec(compile(ast.fix_missing_locations(ast.Module(body=[function], type_ignores=[])), str(SCONSTRUCT), 'exec'), scope) + scope['prune_cache_dir']() + + +class TestSconsCachePrune(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.shard = self.root / '8B' + self.shard.mkdir() + + def artifact(self, suffix='a', length=32): + path = self.shard / ('8b' + suffix * (length - 2)) + path.write_bytes(b'finalized-artifact') + return path + + def test_only_finalized_artifacts_are_pruned(self): + final32 = self.artifact('a') + final64 = self.artifact('b', 64) + temp = self.shard / (final32.name + '.tmpwriter') + temp.write_bytes(b'active-writer') + metadata = self.root / 'config' + metadata.write_bytes(b'cache-metadata') + tag = self.root / 'CACHEDIR.TAG' + tag.write_bytes(b'tag') + outside = self.root / 'outside' + outside.write_bytes(b'outside') + link = self.shard / ('8b' + 'c' * 30) + link.symlink_to(outside) + prune(str(self.root) + '/', 1) + self.assertFalse(final32.exists()) + self.assertFalse(final64.exists()) + self.assertTrue(temp.exists()) + self.assertTrue(metadata.exists()) + self.assertTrue(tag.exists()) + self.assertTrue(link.is_symlink()) + self.assertEqual(outside.read_bytes(), b'outside') + + def test_disappearing_temp_and_finalized_stat_are_harmless(self): + final = self.artifact() + temp = self.shard / (final.name + '.tmpwriter') + temp.write_bytes(b'active-writer') + prune(self.root, 1, CacheOS(vanish_on_walk=temp)) + self.assertFalse(final.exists()) + final = self.artifact() + prune(self.root, 1, CacheOS(vanish_on_walk=final)) + self.assertFalse(final.exists()) + + def test_disappearing_unlink_is_harmless(self): + final = self.artifact() + prune(self.root, 1, CacheOS(vanish_on_unlink=final)) + self.assertFalse(final.exists()) + + def test_other_io_errors_remain_visible(self): + final = self.artifact() + with self.assertRaises(PermissionError): + prune(self.root, 1, CacheOS(fail_stat=final)) + self.assertTrue(final.exists()) + with self.assertRaises(PermissionError): + prune(self.root, 1, CacheOS(fail_unlink=final)) + self.assertTrue(final.exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/uv.lock b/uv.lock index 9a552a0efa..69f748e064 100644 --- a/uv.lock +++ b/uv.lock @@ -96,6 +96,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "colorlog" +version = "6.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/55/ba79756cb90c8d69d599d57785398ac87bba7b19c80e87f4e8a562197c93/colorlog-6.12.0.tar.gz", hash = "sha256:2a7924c1dadf18b22a0eb8b06d1c7b01d5341707ec1641eb6fcc4fde0c3e8e5f", size = 18151, upload-time = "2026-07-23T13:40:40.71Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/19/0b6647bf5e331521e55d2b63bfbdc210bd9cd605189273f03614a05f702d/colorlog-6.12.0-py3-none-any.whl", hash = "sha256:30d392604e9110045a2c2aeefc27d7a017abbab63f3a8aee594eac0801df784e", size = 12239, upload-time = "2026-07-23T13:40:39.562Z" }, +] + [[package]] name = "comma-deps-acados" version = "0.2.2.post103" @@ -276,6 +288,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/5a/234e8fadf85c3cc48cb31c247b9e8e0c7f06ece80f5b29f9b8c241f9da4c/coverage-7.16.0-py3-none-any.whl", hash = "sha256:245f7de6d023a5bba375dbec9f2e0869bfa26ac0cc639bbb7b4c814884000b73", size = 214977, upload-time = "2026-08-28T21:54:35.189Z" }, ] +[[package]] +name = "cppcheck" +version = "2.21.0" +source = { git = "https://github.com/commaai/dependencies.git?subdirectory=cppcheck&rev=b7253ddb101ed6add04fc2a2ce0688d5e21a55ee#b7253ddb101ed6add04fc2a2ce0688d5e21a55ee" } + [[package]] name = "cryptography" version = "50.0.1" @@ -344,6 +361,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/77/67b0b24e45073a699610e50f00c18474ff9b09ea29ecc95083bdf5e60acd/cython-3.3.0-py3-none-any.whl", hash = "sha256:9b24b5c8cd536946b62086fcafee6d5509d3f549f72d553d2336af87ffbe0da1", size = 1349151, upload-time = "2026-08-22T05:16:36.741Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "fonttools" version = "4.65.0" @@ -361,6 +387,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/35/f894ceb867118c0261d0f69a9bd516b045a3754238f76c88a49513ac7a83/fonttools-4.65.0-py3-none-any.whl", hash = "sha256:3060b8c1fc2329fa20265b7c138614143ea7c1624e26c5c180c76aeb74deae6f", size = 1196441, upload-time = "2026-09-10T15:35:52.347Z" }, ] +[[package]] +name = "gcovr" +version = "8.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorlog" }, + { name = "jinja2" }, + { name = "lxml" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/37/b4a87dff166dc0a5002e9d03fcb6ca8eeff048247b011b67f047e31122c9/gcovr-8.6.tar.gz", hash = "sha256:b2e7042abca9321cadbab8a06eb34d19f801b831557b28cdc30a029313de8b9e", size = 199997, upload-time = "2026-01-13T20:04:30.019Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/be/f722c843e7875c7cf92cf0e0c1604cddda55a70278c768c6327a78fdba79/gcovr-8.6-py3-none-any.whl", hash = "sha256:dbf9d87c38042752ad6f530aa8210427e22b526611bb7b7bfed0e81977d1f1ef", size = 254618, upload-time = "2026-01-13T20:04:28.15Z" }, +] + [[package]] name = "idna" version = "3.19" @@ -379,6 +420,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/db/55a262f3606bebcae07cc14095338471ad7c0bbcaa37707e6f0ee49725b7/importlib_resources-7.1.0-py3-none-any.whl", hash = "sha256:1bd7b48b4088eddb2cd16382150bb515af0bd2c70128194392725f82ad2c96a1", size = 37232, upload-time = "2026-04-12T16:36:08.219Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "inputs" version = "0.5" @@ -397,6 +447,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "kiwisolver" version = "1.5.1" @@ -465,6 +527,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/08/02aecf6dad627534a5835244ece14d7f187ef430354d9b8551199934d059/libusb1-3.4.0-py3-none-win_amd64.whl", hash = "sha256:b7dcc1f324a895af6aac708bc5513a17373f97349cc2ab8a277519c788bd18ca", size = 141212, upload-time = "2026-05-16T20:59:17.286Z" }, ] +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1f/a180b57d9eeabaab77f9d5aa30356898ea749c4795596a8f66d1eb6bef2e/lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc", size = 8602094, upload-time = "2026-09-02T14:47:26.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/070c92013a1c029a602b03560d68772313d918268667fa993da7961759c9/lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d", size = 4638308, upload-time = "2026-09-02T14:47:29.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1c/722e88883173097a1a375153e3c2447eba3060d0231522cf6596e99f4195/lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5", size = 4939696, upload-time = "2026-09-02T14:47:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/db/36/aa413bc214dc4f785ad2b2ddd8cc99aae7062d49ab155e91e6011af00daf/lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11", size = 5105247, upload-time = "2026-09-02T14:47:36.734Z" }, + { url = "https://files.pythonhosted.org/packages/a3/a0/a1f7f1313795bfec67b77f01ef3b1128d49f2d7f66a8413fa55d47f4e25f/lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a", size = 5011915, upload-time = "2026-09-02T14:47:39.846Z" }, + { url = "https://files.pythonhosted.org/packages/b9/78/840e7e3f1d0cc7a5cfac5d8505b97e25b6427fd774ac4bae672aaebfb4b5/lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32", size = 5638175, upload-time = "2026-09-02T14:47:43.644Z" }, + { url = "https://files.pythonhosted.org/packages/0a/20/e022dbc6b4753a9bc9fc5fb28a27163430c1731b9913997f6544c1b2518c/lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c", size = 5244675, upload-time = "2026-09-02T14:47:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/83/82cde81d2b5eb38d1539fdfdf318abdd014a7e604f4df01c9cd3deb18f2a/lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56", size = 5358205, upload-time = "2026-09-02T14:47:50.306Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a1/f3b057371c8cb29f2a9c9c44ea320592446e40b74a4b0af68c3d8e65bc73/lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f", size = 4704495, upload-time = "2026-09-02T14:47:53.251Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/230eb28be5d412152ffc3c679b51fe1aeede5a53f3a8eb6e9748f2f4754f/lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5", size = 5255117, upload-time = "2026-09-02T14:47:55.963Z" }, + { url = "https://files.pythonhosted.org/packages/a3/18/1969f56763af24ce42ea156007b0b2d73fddea552e283b2010416394f0f4/lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385", size = 5054424, upload-time = "2026-09-02T14:47:58.131Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/2a90acc1f6fabaa3a8db9340437822bd8d041b205d626a4b3e8621aaa390/lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d", size = 4785572, upload-time = "2026-09-02T14:48:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1e/b90e845b1dcd0f2f3f26b98283d857f25909223aacd265eee032c34ab8b1/lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9", size = 5656516, upload-time = "2026-09-02T14:48:03.419Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ab/0a1b802c57f3fba5c4efd77d5c6b78adaa8f7b681f0c90456b140fe8bf6c/lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e", size = 5245982, upload-time = "2026-09-02T14:48:06.109Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/2c016fbceb3778137459292538d9dfa7e3ad9070fe409c15254ddd90d2cc/lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5", size = 5267340, upload-time = "2026-09-02T14:48:08.374Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b1/736d18fd6f0835761923b7bac1f0c27d60c1200384e9093f05d8c5100525/lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c", size = 3602606, upload-time = "2026-09-02T14:48:10.384Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/6ed903e4e6278a020c8a6f0dbbe78030d041840a6b4a64ea441a1e414077/lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c", size = 4005999, upload-time = "2026-09-02T14:48:12.51Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1b/7bcebb7b6332cb3ae85e9c13b139adb6f23f75c71d84041c56a5005d9a29/lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa", size = 3666631, upload-time = "2026-09-02T14:48:14.567Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, +] + [[package]] name = "matplotlib" version = "3.11.2" @@ -606,6 +713,18 @@ dependencies = [ ] [package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-mock" }, + { name = "pytest-xdist" }, +] +safety = [ + { name = "cffi" }, + { name = "cppcheck" }, + { name = "gcovr" }, + { name = "tree-sitter" }, + { name = "tree-sitter-c" }, +] testing = [ { name = "codespell" }, { name = "coverage" }, @@ -635,6 +754,7 @@ standalone = [ [package.metadata] requires-dist = [ + { name = "cffi", marker = "extra == 'safety'" }, { name = "codespell", marker = "extra == 'testing'" }, { name = "comma-deps-acados" }, { name = "comma-deps-bootstrap-icons", marker = "extra == 'tools'" }, @@ -649,6 +769,8 @@ requires-dist = [ { name = "comma-deps-zeromq" }, { name = "comma-deps-zstd" }, { name = "coverage", marker = "extra == 'testing'" }, + { name = "cppcheck", marker = "extra == 'safety'", git = "https://github.com/commaai/dependencies.git?subdirectory=cppcheck&rev=b7253ddb101ed6add04fc2a2ce0688d5e21a55ee" }, + { name = "gcovr", marker = "extra == 'safety'", specifier = "==8.6" }, { name = "inputs" }, { name = "jeepney" }, { name = "matplotlib", marker = "extra == 'tools'" }, @@ -659,6 +781,9 @@ requires-dist = [ { name = "pillow" }, { name = "pycapnp", specifier = "==2.1.0" }, { name = "pyjwt", extras = ["crypto"] }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8,<9" }, + { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3,<4" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3,<4" }, { name = "pyzmq" }, { name = "rednose", marker = "extra == 'vendored'", editable = "rednose_repo" }, { name = "requests" }, @@ -669,11 +794,13 @@ requires-dist = [ { name = "teleoprtc", marker = "extra == 'vendored'", editable = "teleoprtc_repo" }, { name = "tinygrad", marker = "extra == 'vendored'", editable = "tinygrad_repo" }, { name = "tqdm" }, + { name = "tree-sitter", marker = "extra == 'safety'", specifier = "==0.26.0" }, + { name = "tree-sitter-c", marker = "extra == 'safety'", specifier = "==0.24.2" }, { name = "ty", marker = "extra == 'testing'" }, { name = "websocket-client" }, { name = "zstandard" }, ] -provides-extras = ["testing", "tools", "vendored"] +provides-extras = ["dev", "safety", "testing", "tools", "vendored"] [package.metadata.requires-dev] standalone = [{ name = "openpilot", extras = ["vendored"] }] @@ -727,6 +854,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "pycapnp" version = "2.1.0" @@ -775,6 +911,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + [[package]] name = "pyjwt" version = "2.14.0" @@ -798,6 +943,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1065,6 +1251,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, ] +[[package]] +name = "tree-sitter" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/03/5600b84aff2e6c4fe80cfebb4063fe2f50299521befe5f6092ab8c082f4a/tree_sitter-0.26.0.tar.gz", hash = "sha256:b40c219edccc4564530c96f8f1556f6202b37cda964d1cbd7bd2b7e68b40a245", size = 191423, upload-time = "2026-06-30T12:14:27.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/ca/565702c44815393e3a973552ad546db4e5ca081ca8698640b4e93d809f51/tree_sitter-0.26.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6cb2bd20efb2544c19ac54486ab7cb8ec7b36f913bbe1ce95df84acb96743d9c", size = 148934, upload-time = "2026-06-30T12:14:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/54/6f/8bb61957f16ec1b1d92410a006cdc84a952b6352a7313b2ad299f2d21484/tree_sitter-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:918d89529786873f0982a0f59c2a303cd065fbfd1b903d71a8e4e1584f67b42e", size = 140820, upload-time = "2026-06-30T12:14:02.087Z" }, + { url = "https://files.pythonhosted.org/packages/78/0a/8a6f08559182643a814a4ab559948ae817b2851890fd9b995a4fff6541ce/tree_sitter-0.26.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30a88be89ff1f2755297f81e8080d88b795dd98720c3f9fa2acf93873182cc95", size = 638844, upload-time = "2026-06-30T12:14:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2f/6e6781b31677231366cb3cf27bc8269157f6d4b03c9032865a4f5f2bbe7e/tree_sitter-0.26.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a6b333b0282d8bb0af741f9b018bd2523d4eecb2686bf6717066a625fecfaa4", size = 667487, upload-time = "2026-06-30T12:14:04.669Z" }, + { url = "https://files.pythonhosted.org/packages/02/0b/0483078c8567445557a7015b0e5b187f6d7d4fda73464df9c4bdea7f7f3c/tree_sitter-0.26.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f3c44339dd34fe8eb2b8d5aa7610660499a795f70376b130bbee7a437337280", size = 647975, upload-time = "2026-06-30T12:14:05.797Z" }, + { url = "https://files.pythonhosted.org/packages/27/68/da83ca72c984e96ab4eb3bee0db1a6ffb5de1c8c455f92bd9f420cde7f0e/tree_sitter-0.26.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:94550e13b6ae576969da40246f4c4abb206380b5375ad43f26dd9151d55438e3", size = 665018, upload-time = "2026-06-30T12:14:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/d1/36/4d67927fd47b89af4a00f65f55a7370e28778cd50e972c2430487e3ecc27/tree_sitter-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca89e361a276dbc934b28a43dd881199e25d34ff5493ee0ce45f3c52a6124a37", size = 129619, upload-time = "2026-06-30T12:14:08.373Z" }, + { url = "https://files.pythonhosted.org/packages/ed/72/cdefad523eb78710679c6da6a79e3d90f5afd32b1c6aa5a17bac7eef99f6/tree_sitter-0.26.0-cp312-cp312-win_arm64.whl", hash = "sha256:bc6cb01d5ee75c85424aa1f1c72a82d8f07fd52539a0f3c4a6ed3e8721079b84", size = 116545, upload-time = "2026-06-30T12:14:09.273Z" }, +] + +[[package]] +name = "tree-sitter-c" +version = "0.24.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/c9/3834f3d9278251aea7312274971bc4c45b17aec2490fd4b884d93bd7019a/tree_sitter_c-0.24.2.tar.gz", hash = "sha256:1628584df0299b5a340aa63f8e67b6c97c91517f52fa7e7a4c557e40adb330a9", size = 228397, upload-time = "2026-04-22T08:06:14.491Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/c1/26ed17730ec2c17bedc1b673349e5e0a466c578e3eb0327c3b73cf52bf97/tree_sitter_c-0.24.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:4d4579a8b54f0a442f903d88d3304cab77cd5c2031d4015baa4f2f8e15d6dcb7", size = 81016, upload-time = "2026-04-22T08:06:07.208Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1c/1140db75e7e375cda3c68792a33826c4fd40b5b98c3259d93c75f6c8368f/tree_sitter_c-0.24.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:97bc80a224d48215d4e6e6376bf30d114f4c317b8145ff1b02afe785d4ba7bdd", size = 86213, upload-time = "2026-04-22T08:06:08.136Z" }, + { url = "https://files.pythonhosted.org/packages/e9/8c/0dfb88d726f8821d1c4c36042f092be974a800afd734307a595b8604190c/tree_sitter_c-0.24.2-cp310-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5041ef67eb68ce6bc8bb0b1f8ef3a5585ce523dae0c7eec109ab0627dd75aede", size = 94264, upload-time = "2026-04-22T08:06:08.918Z" }, + { url = "https://files.pythonhosted.org/packages/87/78/47dc570e7aee6b0a1ecc2520b30639cc2b06003154c9ab0672d86bf720d5/tree_sitter_c-0.24.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c098bedcd5ac86ff93fa734d51d1dd86aed40fd5ed7d634c7af11380a0469969", size = 94560, upload-time = "2026-04-22T08:06:09.852Z" }, + { url = "https://files.pythonhosted.org/packages/29/37/75d59d3f74f4cfc00f04472917e933d8a9c9fdc6eff980ef9552e010e6aa/tree_sitter_c-0.24.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:82842c5a5f2acd93f4de10038c33ac179c8979defc39376f990348d6289e933b", size = 94023, upload-time = "2026-04-22T08:06:10.682Z" }, + { url = "https://files.pythonhosted.org/packages/64/57/8fc655d5a446a70a637e92b98bd2fdaab88bf5bb5b36076ac4add544808d/tree_sitter_c-0.24.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2b42e8e22202c251f8629306f9321233542e07a6e01611b5fe83489272143eb", size = 94160, upload-time = "2026-04-22T08:06:11.497Z" }, + { url = "https://files.pythonhosted.org/packages/c1/f7/72a1d6b42dd31fd37e03ff67e7dc5ee572301499e6b216002b8dd42a1714/tree_sitter_c-0.24.2-cp310-abi3-win_amd64.whl", hash = "sha256:abb549225091f7b25df2dd3a0143ece6e208f7055d8bcb4700b41ee79b9ef1e1", size = 84669, upload-time = "2026-04-22T08:06:12.347Z" }, + { url = "https://files.pythonhosted.org/packages/e2/9d/7475d9ae8ef679aa36c7dfe6c903ab78e573651c68b6ef9862d6a3f994db/tree_sitter_c-0.24.2-cp310-abi3-win_arm64.whl", hash = "sha256:4a2f4371cd816cc3153458f69062135ebb2ea5f275ddd90494e5c823d778204a", size = 82956, upload-time = "2026-04-22T08:06:13.364Z" }, +] + [[package]] name = "ty" version = "0.0.80"